From 28e43d7d1d416d30f213336be96b7e9b8c8f83ab Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Tue, 27 Jun 2017 10:30:06 -0700 Subject: [PATCH 01/10] add one failing test for iri/decoding, over the roundtripping url list --- hyperlink/test/test_url.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hyperlink/test/test_url.py b/hyperlink/test/test_url.py index 9ad2cc7f..76f0f657 100644 --- a/hyperlink/test/test_url.py +++ b/hyperlink/test/test_url.py @@ -234,6 +234,14 @@ def test_roundtrip(self): result = URL.from_text(test).to_text() self.assertEqual(test, result) + def test_roundtrip_iri(self): + for test in ROUNDTRIP_TESTS: + url = URL.from_text(test) + iri = url.to_iri() + assert iri == url.to_iri() + assert iri.to_text() == url.to_iri().to_text() + return + def test_equality(self): """ Two URLs decoded using L{URL.from_text} will be equal (C{==}) if they From 66ef4beecd3298991986d286fd9682be722688d6 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Tue, 27 Jun 2017 10:38:29 -0700 Subject: [PATCH 02/10] fix an iri roundtripping test by having .replace() pass through family and uses_netloc. previously ipv6 hosts were being passed through without family and the colons were causing failures. this makes .replace()'s interface match URL.__init__ (and its own docstring\!) --- hyperlink/_url.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index 096a1eb8..f81b0285 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -819,7 +819,8 @@ def absolute(self): return bool(self.scheme and self.host) def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, - fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET): + fragment=_UNSET, port=_UNSET, rooted=_UNSET, userinfo=_UNSET, + family=_UNSET, uses_netloc=_UNSET): """:class:`URL` objects are immutable, which means that attributes are designed to be set only once, at construction. Instead of modifying an existing URL, one simply creates a copy with the @@ -861,6 +862,8 @@ def replace(self, scheme=_UNSET, host=_UNSET, path=_UNSET, query=_UNSET, port=_optional(port, self.port), rooted=_optional(rooted, self.rooted), userinfo=_optional(userinfo, self.userinfo), + family=_optional(family, self.family), + uses_netloc=_optional(uses_netloc, self.uses_netloc) ) @classmethod From ddec1487e7c95b59e07c16542e69a7f3193cd550 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Thu, 29 Jun 2017 22:16:43 -0700 Subject: [PATCH 03/10] WIP all double-IRI roundtrip working (path seems to have been the only part truly tested in the roundtripping) --- hyperlink/_url.py | 79 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index f81b0285..a0b962c4 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -141,7 +141,6 @@ def __nonzero__(self): for a in string.hexdigits for b in string.hexdigits]) _ASCII_RE = re.compile('([\x00-\x7f]+)') - # RFC 3986 section 2.2, Reserved Characters # https://tools.ietf.org/html/rfc3986#section-2.2 _GEN_DELIMS = frozenset(u':/?#[]@') @@ -160,6 +159,16 @@ def __nonzero__(self): _QUERY_DELIMS = _ALL_DELIMS - _QUERY_SAFE +_PATH_HEX_MAP = dict(_HEX_CHAR_MAP) +for delim in _PATH_DELIMS: + _hexord = hex(ord(delim))[2:].encode('ascii') + try: + _PATH_HEX_MAP.pop(_hexord.lower()) + _PATH_HEX_MAP.pop(_hexord.upper()) + except KeyError: + print('not present:', delim) + + def _make_quote_map(safe_chars): ret = {} # v is included in the dict for py3 mostly, because bytestrings @@ -276,6 +285,7 @@ def _encode_userinfo_part(text, maximal=True): else t for t in text]) + # This port list painstakingly curated by hand searching through # https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml # and @@ -413,7 +423,11 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): return value -def _percent_decode(text): +def _decode_path_part(text): + return _percent_decode(text, _hex_map=_PATH_HEX_MAP) + + +def _percent_decode(text, _hex_map=_HEX_CHAR_MAP): """ Replace percent-encoded characters with their UTF-8 equivalents. @@ -427,12 +441,69 @@ def _percent_decode(text): quotedBytes = text.encode("ascii") except UnicodeEncodeError: return text - unquotedBytes = urlunquote(quotedBytes) + unquotedBytes = _unquote_to_bytes(quotedBytes, _hex_map=_hex_map) + # unquotedBytes = urlunquote(quotedBytes) try: return unquotedBytes.decode("utf-8") except UnicodeDecodeError: return text +''' +def _unquote(text, encoding='utf-8', errors='replace', _hex_map=_HEX_CHAR_MAP): + """Percent-decode a string, by replacing %xx escapes with their + single-character equivalent. The optional *encoding* and *errors* + parameters specify how to decode percent-encoded sequences into + Unicode characters, as accepted by the :meth:`bytes.decode()` method. By + default, percent-encoded sequences are decoded with UTF-8, and + invalid sequences are replaced by a placeholder character. + + # >>> unquote(u'abc%20def') + # u'abc def' + """ + if '%' not in text: + text.split + return text + if encoding is None: + encoding = 'utf-8' + if errors is None: + errors = 'replace' + bits = _ASCII_RE.split(text) + res = [bits[0]] + append = res.append + for i in range(1, len(bits), 2): + append(_unquote_to_bytes(bits[i], + _hex_map=_hex_map).decode(encoding, errors)) + append(bits[i + 1]) + return ''.join(res) +''' + + +def _unquote_to_bytes(text, _hex_map=_HEX_CHAR_MAP): + """unquote_to_bytes('abc%20def') -> b'abc def'.""" + # Note: strings are encoded as UTF-8. This is only an issue if it contains + # unescaped non-ASCII characters, which URIs should not. + if not text: + # Is it a string-like object? + text.split + return b'' + if isinstance(text, unicode): + text = text.encode('utf-8') + bits = text.split(b'%') + if len(bits) == 1: + return text + + res = [bits[0]] + append = res.append + + for item in bits[1:]: + try: + append(_hex_map[item[:2]]) + append(item[2:]) + except KeyError: + append(b'%') + append(item) + return b''.join(res) + def _resolve_dot_segments(path): """Normalize the URL path by resolving segments of '.' and '..'. For @@ -1112,7 +1183,7 @@ def to_iri(self): textHost = self.host return self.replace(userinfo=new_userinfo, host=textHost, - path=[_percent_decode(segment) + path=[_decode_path_part(segment) for segment in self.path], query=[tuple(_percent_decode(x) if x is not None else None From 9ef30bcf59037057a6b0beeadcae65a74ce9279f Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Fri, 30 Jun 2017 20:03:04 -0700 Subject: [PATCH 04/10] clean up and generalize path decoding machinery --- hyperlink/_url.py | 58 +++++++++++++---------------------------------- 1 file changed, 16 insertions(+), 42 deletions(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index a0b962c4..bd0dbccc 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -159,14 +159,16 @@ def __nonzero__(self): _QUERY_DELIMS = _ALL_DELIMS - _QUERY_SAFE -_PATH_HEX_MAP = dict(_HEX_CHAR_MAP) -for delim in _PATH_DELIMS: - _hexord = hex(ord(delim))[2:].encode('ascii') - try: - _PATH_HEX_MAP.pop(_hexord.lower()) - _PATH_HEX_MAP.pop(_hexord.upper()) - except KeyError: - print('not present:', delim) +def _make_decode_map(delims): + ret = dict(_HEX_CHAR_MAP) + for delim in delims: + _hexord = hex(ord(delim))[2:].zfill(2).encode('ascii') + try: + ret.pop(_hexord.lower()) + ret.pop(_hexord.upper()) + except KeyError: + print('not present:', delim) # TODO: pass + return ret def _make_quote_map(safe_chars): @@ -184,6 +186,7 @@ def _make_quote_map(safe_chars): _USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE) _PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE) +_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS) _SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE) _QUERY_PART_QUOTE_MAP = _make_quote_map(_QUERY_SAFE) _FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE) @@ -424,10 +427,10 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): def _decode_path_part(text): - return _percent_decode(text, _hex_map=_PATH_HEX_MAP) + return _percent_decode(text, _decode_map=_PATH_DECODE_MAP) -def _percent_decode(text, _hex_map=_HEX_CHAR_MAP): +def _percent_decode(text, _decode_map=_HEX_CHAR_MAP): """ Replace percent-encoded characters with their UTF-8 equivalents. @@ -441,44 +444,15 @@ def _percent_decode(text, _hex_map=_HEX_CHAR_MAP): quotedBytes = text.encode("ascii") except UnicodeEncodeError: return text - unquotedBytes = _unquote_to_bytes(quotedBytes, _hex_map=_hex_map) + unquotedBytes = _unquote_to_bytes(quotedBytes, _decode_map=_decode_map) # unquotedBytes = urlunquote(quotedBytes) try: return unquotedBytes.decode("utf-8") except UnicodeDecodeError: return text -''' -def _unquote(text, encoding='utf-8', errors='replace', _hex_map=_HEX_CHAR_MAP): - """Percent-decode a string, by replacing %xx escapes with their - single-character equivalent. The optional *encoding* and *errors* - parameters specify how to decode percent-encoded sequences into - Unicode characters, as accepted by the :meth:`bytes.decode()` method. By - default, percent-encoded sequences are decoded with UTF-8, and - invalid sequences are replaced by a placeholder character. - - # >>> unquote(u'abc%20def') - # u'abc def' - """ - if '%' not in text: - text.split - return text - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'replace' - bits = _ASCII_RE.split(text) - res = [bits[0]] - append = res.append - for i in range(1, len(bits), 2): - append(_unquote_to_bytes(bits[i], - _hex_map=_hex_map).decode(encoding, errors)) - append(bits[i + 1]) - return ''.join(res) -''' - -def _unquote_to_bytes(text, _hex_map=_HEX_CHAR_MAP): +def _unquote_to_bytes(text, _decode_map=_HEX_CHAR_MAP): """unquote_to_bytes('abc%20def') -> b'abc def'.""" # Note: strings are encoded as UTF-8. This is only an issue if it contains # unescaped non-ASCII characters, which URIs should not. @@ -497,7 +471,7 @@ def _unquote_to_bytes(text, _hex_map=_HEX_CHAR_MAP): for item in bits[1:]: try: - append(_hex_map[item[:2]]) + append(_decode_map[item[:2]]) append(item[2:]) except KeyError: append(b'%') From 027c8c6c28b7ec0530256267df7f406de384a241 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 18:37:57 -0700 Subject: [PATCH 05/10] add many failing tests for double-iri roundtripping (per #16) --- hyperlink/test/test_url.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/hyperlink/test/test_url.py b/hyperlink/test/test_url.py index 76f0f657..e3f0a77d 100644 --- a/hyperlink/test/test_url.py +++ b/hyperlink/test/test_url.py @@ -115,6 +115,27 @@ '20Linux%20precise-5.7.1.iso&tr=udp://tracker.openbittorrent.com:80&' 'tr=udp://tracker.publicbt.com:80&tr=udp://tracker.istole.it:6969&' 'tr=udp://tracker.ccc.de:80&tr=udp://open.demonii.com:1337'), + + # percent-encoded delimiters in percent-encodable fields + + 'https://%3A@example.com/', # colon in username + 'https://%40@example.com/', # at sign in username + 'https://%2f@example.com/', # slash in username + 'https://a:%3a@example.com/', # colon in password + 'https://a:%40@example.com/', # at sign in password + 'https://a:%2f@example.com/', # slash in password + 'https://a:%3f@example.com/', # question mark in password + 'https://example.com/%2F/', # slash in path + 'https://example.com/%3F/', # question mark in path + 'https://example.com/%23/', # hash in path + 'https://example.com/?%23=b', # hash in query param name + 'https://example.com/?%3D=b', # equals in query param name + 'https://example.com/?%26=b', # ampersand in query param name + 'https://example.com/?a=%23', # hash in query param value + 'https://example.com/?a=%26', # ampersand in query param value + 'https://example.com/?a=%3D', # equals in query param value + # double-encoded percent sign in all percent-encodable positions: + "http://(%2525):(%2525)@example.com/(%2525)/?(%2525)=(%2525)#(%2525)", ) @@ -231,15 +252,18 @@ def test_roundtrip(self): L{URL.to_text} should invert L{URL.from_text}. """ for test in ROUNDTRIP_TESTS: - result = URL.from_text(test).to_text() + result = URL.from_text(test).to_text(with_password=True) self.assertEqual(test, result) - def test_roundtrip_iri(self): + def test_roundtrip_double_iri(self): for test in ROUNDTRIP_TESTS: url = URL.from_text(test) iri = url.to_iri() - assert iri == url.to_iri() - assert iri.to_text() == url.to_iri().to_text() + double_iri = iri.to_iri() + assert iri == double_iri + iri_text = iri.to_text(with_password=True) + double_iri_text = double_iri.to_text(with_password=True) + assert iri_text == double_iri_text return def test_equality(self): From d64b1e701117049565090479668da067132b9576 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 18:40:22 -0700 Subject: [PATCH 06/10] add appropriate percent decoding function () for each of the percent-encodable fields, disabling decoding of restricted delimiter characters as well as the all-important percent sign itself. also brings in the decode_to_bytes function, reducing reliance on the standard library. --- hyperlink/_url.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index bd0dbccc..1b12d05d 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -159,8 +159,10 @@ def __nonzero__(self): _QUERY_DELIMS = _ALL_DELIMS - _QUERY_SAFE -def _make_decode_map(delims): +def _make_decode_map(delims, allow_percent=False): ret = dict(_HEX_CHAR_MAP) + if not allow_percent: + delims = list(delims) + [u'%'] for delim in delims: _hexord = hex(ord(delim))[2:].zfill(2).encode('ascii') try: @@ -185,11 +187,14 @@ def _make_quote_map(safe_chars): _USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE) +_USERINFO_DECODE_MAP = _make_decode_map(_USERINFO_DELIMS) _PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE) _PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS) _SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE) _QUERY_PART_QUOTE_MAP = _make_quote_map(_QUERY_SAFE) +_QUERY_DECODE_MAP = _make_decode_map(_QUERY_DELIMS) _FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE) +_FRAGMENT_DECODE_MAP = _make_decode_map(_FRAGMENT_DELIMS) _ROOT_PATHS = frozenset(((), (u'',))) @@ -426,10 +431,22 @@ def _textcheck(name, value, delims=frozenset(), nullable=False): return value +def _decode_userinfo_part(text): + return _percent_decode(text, _decode_map=_USERINFO_DECODE_MAP) + + def _decode_path_part(text): return _percent_decode(text, _decode_map=_PATH_DECODE_MAP) +def _decode_query_part(text): + return _percent_decode(text, _decode_map=_QUERY_DECODE_MAP) + + +def _decode_fragment_part(text): + return _percent_decode(text, _decode_map=_FRAGMENT_DECODE_MAP) + + def _percent_decode(text, _decode_map=_HEX_CHAR_MAP): """ Replace percent-encoded characters with their UTF-8 equivalents. @@ -1143,7 +1160,7 @@ def to_iri(self): URL: A new instance with its path segments, query parameters, and hostname decoded for display purposes. """ - new_userinfo = u':'.join([_percent_decode(p) for p in + new_userinfo = u':'.join([_decode_userinfo_part(p) for p in self.userinfo.split(':', 1)]) try: asciiHost = self.host.encode("ascii") @@ -1159,11 +1176,11 @@ def to_iri(self): host=textHost, path=[_decode_path_part(segment) for segment in self.path], - query=[tuple(_percent_decode(x) + query=[tuple(_decode_query_part(x) if x is not None else None for x in (k, v)) for k, v in self.query], - fragment=_percent_decode(self.fragment)) + fragment=_decode_fragment_part(self.fragment)) def to_text(self, with_password=False): """Render this URL to its textual representation. From 2075d28da17d8041edc5fb82560c4100cdf94967 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 19:01:14 -0700 Subject: [PATCH 07/10] a little bit of docs and cleanup around percent decoding --- hyperlink/_url.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index 1b12d05d..eba4e40e 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -17,8 +17,8 @@ import re import string - import socket +from unicodedata import normalize try: from socket import inet_pton except ImportError: @@ -52,13 +52,6 @@ def inet_pton(address_family, ip_string): raise socket.error('unknown address family') -try: - from urllib import unquote as urlunquote -except ImportError: - from urllib.parse import unquote_to_bytes as urlunquote - -from unicodedata import normalize - unicode = type(u'') try: unichr @@ -162,14 +155,13 @@ def __nonzero__(self): def _make_decode_map(delims, allow_percent=False): ret = dict(_HEX_CHAR_MAP) if not allow_percent: - delims = list(delims) + [u'%'] + delims = set(delims) | set([u'%']) for delim in delims: - _hexord = hex(ord(delim))[2:].zfill(2).encode('ascii') - try: - ret.pop(_hexord.lower()) - ret.pop(_hexord.upper()) - except KeyError: - print('not present:', delim) # TODO: pass + _hexord = hex(ord(delim))[2:].zfill(2).encode('ascii').upper() + _hexord_lower = _hexord.lower() + ret.pop(_hexord) + if _hexord != _hexord_lower: + ret.pop(_hexord_lower) return ret @@ -189,8 +181,8 @@ def _make_quote_map(safe_chars): _USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE) _USERINFO_DECODE_MAP = _make_decode_map(_USERINFO_DELIMS) _PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE) -_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS) _SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE) +_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS) _QUERY_PART_QUOTE_MAP = _make_quote_map(_QUERY_SAFE) _QUERY_DECODE_MAP = _make_decode_map(_QUERY_DELIMS) _FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE) @@ -448,21 +440,30 @@ def _decode_fragment_part(text): def _percent_decode(text, _decode_map=_HEX_CHAR_MAP): - """ - Replace percent-encoded characters with their UTF-8 equivalents. + """Convert percent-encoded text characters to their normal, + human-readable equivalents. + + All characters in the input text must be valid ASCII. All special + characters underlying the values in the percent-encoding must be + valid UTF-8. + + Only called by field-tailored variants, e.g., + :func:`_decode_path_part`, as every percent-encodable part of the + URL has characters which should not be percent decoded. Args: - text (unicode): The text with percent-encoded UTF-8 in it. + text (unicode): The ASCII text with percent-encoding present. Returns: - unicode: The encoded version of *text*. + unicode: The percent-decoded version of *text*, with UTF-8 + decoding applied. + """ try: quotedBytes = text.encode("ascii") except UnicodeEncodeError: return text unquotedBytes = _unquote_to_bytes(quotedBytes, _decode_map=_decode_map) - # unquotedBytes = urlunquote(quotedBytes) try: return unquotedBytes.decode("utf-8") except UnicodeDecodeError: From 31af2815acd061c9263dcddd41130d2ea0cd0835 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 19:01:45 -0700 Subject: [PATCH 08/10] add a potentially tricky decoding test --- hyperlink/test/test_url.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hyperlink/test/test_url.py b/hyperlink/test/test_url.py index e3f0a77d..5e6e5cda 100644 --- a/hyperlink/test/test_url.py +++ b/hyperlink/test/test_url.py @@ -136,6 +136,8 @@ 'https://example.com/?a=%3D', # equals in query param value # double-encoded percent sign in all percent-encodable positions: "http://(%2525):(%2525)@example.com/(%2525)/?(%2525)=(%2525)#(%2525)", + # colon in first part of schemeless relative url + 'first_seg_rel_path__colon%3Anotok/second_seg__colon%3Aok', ) @@ -261,6 +263,7 @@ def test_roundtrip_double_iri(self): iri = url.to_iri() double_iri = iri.to_iri() assert iri == double_iri + iri_text = iri.to_text(with_password=True) double_iri_text = double_iri.to_text(with_password=True) assert iri_text == double_iri_text @@ -277,7 +280,7 @@ def test_equality(self): self.assertNotEqual( urlpath, URL.from_text('ftp://www.anotherinvaliddomain.com/' - 'foo/bar/baz/?zot=21&zut') + 'foo/bar/baz/?zot=21&zut') ) def test_fragmentEquality(self): From 0e877ecb71dddcece5b8bf5a058613383dfea693 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 19:23:16 -0700 Subject: [PATCH 09/10] merge _percent_decode and _unquote_to_bytes since the latter is only called by the former --- hyperlink/_url.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/hyperlink/_url.py b/hyperlink/_url.py index eba4e40e..840cbfdc 100644 --- a/hyperlink/_url.py +++ b/hyperlink/_url.py @@ -451,36 +451,22 @@ def _percent_decode(text, _decode_map=_HEX_CHAR_MAP): :func:`_decode_path_part`, as every percent-encodable part of the URL has characters which should not be percent decoded. + >>> _percent_decode(u'abc%20def') + u'abc def' + Args: text (unicode): The ASCII text with percent-encoding present. Returns: unicode: The percent-decoded version of *text*, with UTF-8 decoding applied. - """ try: - quotedBytes = text.encode("ascii") + quoted_bytes = text.encode("ascii") except UnicodeEncodeError: return text - unquotedBytes = _unquote_to_bytes(quotedBytes, _decode_map=_decode_map) - try: - return unquotedBytes.decode("utf-8") - except UnicodeDecodeError: - return text - -def _unquote_to_bytes(text, _decode_map=_HEX_CHAR_MAP): - """unquote_to_bytes('abc%20def') -> b'abc def'.""" - # Note: strings are encoded as UTF-8. This is only an issue if it contains - # unescaped non-ASCII characters, which URIs should not. - if not text: - # Is it a string-like object? - text.split - return b'' - if isinstance(text, unicode): - text = text.encode('utf-8') - bits = text.split(b'%') + bits = quoted_bytes.split(b'%') if len(bits) == 1: return text @@ -494,7 +480,13 @@ def _unquote_to_bytes(text, _decode_map=_HEX_CHAR_MAP): except KeyError: append(b'%') append(item) - return b''.join(res) + + unquoted_bytes = b''.join(res) + + try: + return unquoted_bytes.decode("utf-8") + except UnicodeDecodeError: + return text def _resolve_dot_segments(path): From c5fa760f9cf315ea10efe59bb168a14b79930e8d Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 2 Jul 2017 23:07:57 -0700 Subject: [PATCH 10/10] add test confirming that replacing uses_netloc works --- hyperlink/test/test_url.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hyperlink/test/test_url.py b/hyperlink/test/test_url.py index 5e6e5cda..cfb01c5e 100644 --- a/hyperlink/test/test_url.py +++ b/hyperlink/test/test_url.py @@ -941,6 +941,13 @@ def test_netloc_slashes(self): url2 = url.replace(scheme='mailto') self.assertEquals(url2.to_text(), 'mailto:/path/to/heck') + url_text = 'unregisteredscheme:///a/b/c' + url = URL.from_text(url_text) + no_netloc_url = url.replace(uses_netloc=False) + self.assertEquals(no_netloc_url.to_text(), 'unregisteredscheme:/a/b/c') + netloc_url = url.replace(uses_netloc=True) + self.assertEquals(netloc_url.to_text(), url_text) + return def test_wrong_constructor(self):