+77
-0
lines changedFilter options
+77
-0
lines changed Original file line number Diff line number Diff line change
@@ -312,6 +312,9 @@ or on combining URL components into a URL string.
312 312
``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
313 313
decomposed before parsing, no error will be raised.
314 314 315 +
Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
316 +
``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
317 + 315 318
.. versionchanged:: 3.6
316 319
Out-of-range port numbers now raise :exc:`ValueError`, instead of
317 320
returning :const:`None`.
@@ -320,6 +323,10 @@ or on combining URL components into a URL string.
320 323
Characters that affect netloc parsing under NFKC normalization will
321 324
now raise :exc:`ValueError`.
322 325 326 +
.. versionchanged:: 3.8.10
327 +
ASCII newline and tab characters are stripped from the URL.
328 + 329 +
.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
323 330 324 331
.. function:: urlunsplit(parts)
325 332
@@ -668,6 +675,10 @@ task isn't already covered by the URL parsing functions above.
668 675 669 676
.. seealso::
670 677 678 +
`WHATWG`_ - URL Living standard
679 +
Working Group for the URL Standard that defines URLs, domains, IP addresses, the
680 +
application/x-www-form-urlencoded format, and their API.
681 + 671 682
:rfc:`3986` - Uniform Resource Identifiers
672 683
This is the current standard (STD66). Any changes to urllib.parse module
673 684
should conform to this. Certain deviations could be observed, which are
@@ -691,3 +702,5 @@ task isn't already covered by the URL parsing functions above.
691 702 692 703
:rfc:`1738` - Uniform Resource Locators (URL)
693 704
This specifies the formal syntax and semantics of absolute URLs.
705 + 706 +
.. _WHATWG: https://url.spec.whatwg.org/
Original file line number Diff line number Diff line change
@@ -612,6 +612,54 @@ def test_urlsplit_attributes(self):
612 612
with self.assertRaisesRegex(ValueError, "out of range"):
613 613
p.port
614 614 615 +
def test_urlsplit_remove_unsafe_bytes(self):
616 +
# Remove ASCII tabs and newlines from input, for http common case scenario.
617 +
url = "h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
618 +
p = urllib.parse.urlsplit(url)
619 +
self.assertEqual(p.scheme, "http")
620 +
self.assertEqual(p.netloc, "www.python.org")
621 +
self.assertEqual(p.path, "/javascript:alert('msg')/")
622 +
self.assertEqual(p.query, "query=something")
623 +
self.assertEqual(p.fragment, "fragment")
624 +
self.assertEqual(p.username, None)
625 +
self.assertEqual(p.password, None)
626 +
self.assertEqual(p.hostname, "www.python.org")
627 +
self.assertEqual(p.port, None)
628 +
self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment")
629 + 630 +
# Remove ASCII tabs and newlines from input as bytes, for http common case scenario.
631 +
url = b"h\nttp://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
632 +
p = urllib.parse.urlsplit(url)
633 +
self.assertEqual(p.scheme, b"http")
634 +
self.assertEqual(p.netloc, b"www.python.org")
635 +
self.assertEqual(p.path, b"/javascript:alert('msg')/")
636 +
self.assertEqual(p.query, b"query=something")
637 +
self.assertEqual(p.fragment, b"fragment")
638 +
self.assertEqual(p.username, None)
639 +
self.assertEqual(p.password, None)
640 +
self.assertEqual(p.hostname, b"www.python.org")
641 +
self.assertEqual(p.port, None)
642 +
self.assertEqual(p.geturl(), b"http://www.python.org/javascript:alert('msg')/?query=something#fragment")
643 + 644 +
# any scheme
645 +
url = "x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
646 +
p = urllib.parse.urlsplit(url)
647 +
self.assertEqual(p.geturl(), "x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
648 + 649 +
# Remove ASCII tabs and newlines from input as bytes, any scheme.
650 +
url = b"x-new-scheme\t://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
651 +
p = urllib.parse.urlsplit(url)
652 +
self.assertEqual(p.geturl(), b"x-new-scheme://www.python.org/javascript:alert('msg')/?query=something#fragment")
653 + 654 +
# Unsafe bytes is not returned from urlparse cache.
655 +
# scheme is stored after parsing, sending an scheme with unsafe bytes *will not* return an unsafe scheme
656 +
url = "https://www.python\n.org\t/java\nscript:\talert('msg\r\n')/?query\n=\tsomething#frag\nment"
657 +
scheme = "htt\nps"
658 +
for _ in range(2):
659 +
p = urllib.parse.urlsplit(url, scheme=scheme)
660 +
self.assertEqual(p.scheme, "https")
661 +
self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment")
662 + 615 663
def test_attributes_bad_port(self):
616 664
"""Check handling of invalid ports."""
617 665
for bytes in (False, True):
@@ -77,6 +77,9 @@
77 77
'0123456789'
78 78
'+-.')
79 79 80 +
# Unsafe bytes to be removed per WHATWG spec
81 +
_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
82 + 80 83
# XXX: Consider replacing with functools.lru_cache
81 84
MAX_CACHE_SIZE = 20
82 85
_parse_cache = {}
@@ -414,13 +417,20 @@ def _checknetloc(netloc):
414 417
raise ValueError("netloc '" + netloc + "' contains invalid " +
415 418
"characters under NFKC normalization")
416 419 420 +
def _remove_unsafe_bytes_from_url(url):
421 +
for b in _UNSAFE_URL_BYTES_TO_REMOVE:
422 +
url = url.replace(b, "")
423 +
return url
424 + 417 425
def urlsplit(url, scheme='', allow_fragments=True):
418 426
"""Parse a URL into 5 components:
419 427
<scheme>://<netloc>/<path>?<query>#<fragment>
420 428
Return a 5-tuple: (scheme, netloc, path, query, fragment).
421 429
Note that we don't break the components up in smaller bits
422 430
(e.g. netloc is a single string) and we don't expand % escapes."""
423 431
url, scheme, _coerce_result = _coerce_args(url, scheme)
432 +
url = _remove_unsafe_bytes_from_url(url)
433 +
scheme = _remove_unsafe_bytes_from_url(scheme)
424 434
allow_fragments = bool(allow_fragments)
425 435
key = url, scheme, allow_fragments, type(url), type(scheme)
426 436
cached = _parse_cache.get(key, None)
@@ -0,0 +1,6 @@
1 +
The presence of newline or tab characters in parts of a URL could allow
2 +
some forms of attacks.
3 + 4 +
Following the controlling specification for URLs defined by WHATWG
5 +
:func:`urllib.parse` now removes ASCII newlines and tabs from URLs,
6 +
preventing such attacks.
You can’t perform that action at this time.
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4