diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 055971ec21aee3..22448c14f7a188 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -230,6 +230,29 @@ The following exceptions are raised as appropriate: Previously, :exc:`BadStatusLine`\ ``('')`` was raised. +.. exception:: ProxyTunnelError + + A subclass of :exc:`OSError`. Raised when the proxy server answers the + CONNECT request sent for :meth:`HTTPConnection.set_tunnel` with a status + other than 200. + + .. attribute:: code + + The HTTP status code of the proxy's response, such as ``407``. + + .. attribute:: reason + + The reason phrase of the proxy's response. + + .. attribute:: headers + + The headers of the proxy's response, as an + :class:`http.client.HTTPMessage` instance. + + .. versionadded:: next + Previously, :exc:`OSError` was raised. + + The constants defined in this module are: .. data:: HTTP_PORT @@ -398,6 +421,9 @@ HTTPConnection Objects one will be automatically generated and transmitted if not provided in the headers argument. + .. versionchanged:: next + :exc:`ProxyTunnelError` is raised if the proxy rejects the CONNECT request. + .. method:: HTTPConnection.get_proxy_response_headers() diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 8234be98a2d659..c3f749774b869c 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -445,6 +445,10 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: next + Also handles a proxy that requires authentication for the CONNECT + request used to tunnel ``https`` URLs. + .. class:: AbstractDigestAuthHandler(password_mgr=None) @@ -482,6 +486,10 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: next + Also handles a proxy that requires authentication for the CONNECT + request used to tunnel ``https`` URLs. + .. class:: HTTPHandler() @@ -1102,6 +1110,14 @@ ProxyBasicAuthHandler Objects Retry the request with authentication information, if available. + For an ``https`` URL, the request goes through a tunnel that the proxy + sets up in reply to a CONNECT request. A 407 response can then come + either from the proxy, rejecting the CONNECT, or from the origin server, + through the tunnel. Both are handled the same way: the request is retried + with the credentials for the proxy, and those credentials are sent only in + the CONNECT request, never to the origin server. If the origin server + keeps answering 407, :exc:`~urllib.error.HTTPError` is raised. + .. _abstract-digest-auth-handler: @@ -1141,6 +1157,14 @@ ProxyDigestAuthHandler Objects Retry the request with authentication information, if available. + For an ``https`` URL, the request goes through a tunnel that the proxy + sets up in reply to a CONNECT request. A 407 response can then come + either from the proxy, rejecting the CONNECT, or from the origin server, + through the tunnel. Both are handled the same way: the request is retried + with the credentials for the proxy, and those credentials are sent only in + the CONNECT request, never to the origin server. If the origin server + keeps answering 407, :exc:`~urllib.error.HTTPError` is raised. + .. _http-handler-objects: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4a1fb6b6b87e13..da81d8513da99d 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -508,6 +508,16 @@ gzip (Contributed by Marin Misur in :gh:`91372`.) +http.client +----------- + +* Add the :exc:`~http.client.ProxyTunnelError` exception, a subclass of + :exc:`OSError` raised when a proxy rejects the CONNECT request sent for + :meth:`~http.client.HTTPConnection.set_tunnel`. It exposes the status + code, reason and headers of the proxy's response. + (Contributed by Senthil Kumaran in :gh:`51540`.) + + imaplib ------- @@ -956,6 +966,18 @@ unicodedata * The Unicode database has been updated to Unicode 18.0.0. +urllib.request +-------------- + +* :class:`~urllib.request.ProxyBasicAuthHandler` and + :class:`~urllib.request.ProxyDigestAuthHandler` now authenticate with a + proxy that answers the CONNECT request for an ``https`` URL with + ``407 Proxy Authentication Required``. When no credentials are available, + :exc:`~urllib.error.HTTPError` with code 407 is raised instead of + :exc:`~urllib.error.URLError`. + (Contributed by Senthil Kumaran in :gh:`51540`.) + + uuid ---- diff --git a/Lib/http/client.py b/Lib/http/client.py index fe67d0f7963f78..96f266f32346ab 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -87,8 +87,8 @@ "UnknownTransferEncoding", "UnimplementedFileMode", "IncompleteRead", "InvalidURL", "ImproperConnectionState", "CannotSendRequest", "CannotSendHeader", "ResponseNotReady", - "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error", - "responses"] + "BadStatusLine", "LineTooLong", "RemoteDisconnected", + "ProxyTunnelError", "error", "responses"] HTTP_PORT = 80 HTTPS_PORT = 443 @@ -1045,7 +1045,8 @@ def _tunnel(self): if code != http.HTTPStatus.OK: self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + raise ProxyTunnelError(code, message.strip(), + _parse_header_lines(self._raw_proxy_headers)) finally: response.close() @@ -1617,5 +1618,15 @@ def __init__(self, *pos, **kw): BadStatusLine.__init__(self, "") ConnectionResetError.__init__(self, *pos, **kw) +class ProxyTunnelError(OSError): + def __init__(self, code, reason, headers): + OSError.__init__(self, f"Tunnel connection failed: {code} {reason}") + self.code = code + self.reason = reason + self.headers = headers + + def __reduce__(self): + return self.__class__, (self.code, self.reason, self.headers) + # for backwards compatibility error = HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1e5ec0d4e43c27..57eedee25a89e0 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -4,6 +4,7 @@ import io import itertools import os +import pickle import array import re import socket @@ -2786,6 +2787,36 @@ def test_no_proxy_response_headers(self): headers = self.conn.get_proxy_response_headers() self.assertIsNone(headers) + def test_tunnel_error(self): + response_text = ( + 'HTTP/1.1 407 Proxy Authentication Required\r\n' + 'Proxy-Authenticate: Basic realm="proxy"\r\n\r\n' + ) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with self.assertRaises(client.ProxyTunnelError) as cm: + self.conn.request('HEAD', '/', '') + exc = cm.exception + self.assertIsInstance(exc, OSError) + self.assertEqual(exc.code, 407) + self.assertEqual(exc.reason, 'Proxy Authentication Required') + self.assertEqual(exc.headers['Proxy-Authenticate'], + 'Basic realm="proxy"') + self.assertEqual(str(exc), 'Tunnel connection failed: ' + '407 Proxy Authentication Required') + self.assertIsNone(self.conn.sock) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + copy = pickle.loads(pickle.dumps(exc, proto)) + self.assertIs(type(copy), client.ProxyTunnelError) + self.assertEqual(copy.code, 407) + self.assertEqual(copy.reason, exc.reason) + self.assertEqual(copy.headers['Proxy-Authenticate'], + 'Basic realm="proxy"') + self.assertEqual(str(copy), str(exc)) + def test_tunnel_leak(self): sock = None diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index f01b94c581e87a..a198a9de68b64f 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import hashlib import ftplib import socket import array @@ -617,6 +618,42 @@ def https_open(self, req): return self.do_open(self.httpconn, req) +class MockHTTPTunnelClass(MockHTTPClass): + # Stands in for a proxy that requires authentication: CONNECT fails + # with tunnel_code until the credentials arrive in the tunnel headers + # (tunnel_code=None: no authentication). The origin then answers + # through the tunnel with origin_code. + def __init__(self, tunnel_code=407, challenge='Basic realm="proxy"', + origin_code=200): + super().__init__() + self.tunnel_code = tunnel_code + self.challenge = challenge + self.origin_code = origin_code + self.tunnel_requests = [] + + def challenge_headers(self): + return http.client.parse_headers(io.BytesIO( + b'Proxy-Authenticate: %s\r\n\r\n' % self.challenge.encode())) + + def request(self, method, url, body=None, headers=None, *, + encode_chunked=False): + self.tunnel_requests.append(dict(self._tunnel_headers)) + if (self.tunnel_code is not None and + "Proxy-Authorization" not in self._tunnel_headers): + raise http.client.ProxyTunnelError( + self.tunnel_code, http.client.responses[self.tunnel_code], + self.challenge_headers()) + super().request(method, url, body, headers, + encode_chunked=encode_chunked) + + def getresponse(self): + response = super().getresponse() + response.code = self.origin_code + if self.origin_code != 200: + response.info = self.challenge_headers + return response + + class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): # useful for testing auth # sends supplied code response @@ -1645,6 +1682,142 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + def _open_through_tunnel(self, httpconn, password_mgr, + auth_handler=urllib.request.ProxyBasicAuthHandler, + url="https://www.example.com/path"): + https_handler = MockHTTPSHandler() + https_handler.httpconn = httpconn + opener = urllib.request.build_opener( + urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")), + auth_handler(password_mgr), + https_handler) + return opener.open(url) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_basic_auth_handler(self): + # gh-51540: a 407 from the proxy to CONNECT is passed to + # ProxyBasicAuthHandler, which retries with credentials + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass() + response = self._open_through_tunnel(httpconn, password_mgr) + self.assertEqual(response.code, 200) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertEqual(httpconn.tunnel_requests[1]["Proxy-Authorization"], + "Basic dXNlcjpwYXNz") + # The credentials go to the proxy only, not to the origin. + self.assertEqual(httpconn._tunnel_host, "www.example.com") + self.assertEqual(httpconn.selector, "/path") + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + @skip_libssl_fips_mode # Uses MD5 algorithm in challenge + def test_proxy_https_digest_auth_handler(self): + # gh-51540: a 407 from the proxy to CONNECT is passed to + # ProxyDigestAuthHandler, which retries with credentials + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass( + challenge='Digest realm="proxy", nonce="abcdef0123456789", ' + 'qop="auth"') + response = self._open_through_tunnel( + httpconn, password_mgr, urllib.request.ProxyDigestAuthHandler) + self.assertEqual(response.code, 200) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertStartsWith( + httpconn.tunnel_requests[1]["Proxy-Authorization"], + 'Digest username="user", realm="proxy", ') + self.assertEqual(httpconn.selector, "/path") + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + @skip_libssl_fips_mode # Uses MD5 algorithm in challenge + def test_proxy_https_digest_auth_connect(self): + # Through an https tunnel, the digest covers the CONNECT request + # (method CONNECT, request-target host:port), not the request + # later sent to the origin + def md5(text): + return hashlib.md5(text.encode("ascii")).hexdigest() + + nonce = "abcdef0123456789" + for url, target in [ + ("https://www.example.com/path", "www.example.com:443"), + ("https://www.example.com:8443/path", "www.example.com:8443"), + ]: + with self.subTest(url=url): + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass( + challenge='Digest realm="proxy", nonce="%s"' % nonce) + self._open_through_tunnel( + httpconn, password_mgr, + urllib.request.ProxyDigestAuthHandler, url) + + expected = md5("%s:%s:%s" % (md5("user:proxy:pass"), nonce, + md5("CONNECT:" + target))) + auth = httpconn.tunnel_requests[1]["Proxy-Authorization"] + self.assertIn('uri="%s"' % target, auth) + self.assertIn('response="%s"' % expected, auth) + self.assertEqual(httpconn.selector, "/path") + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_origin_407(self): + # A 407 sent by the origin through the tunnel is retried like one + # from the proxy, but the credentials only go on the CONNECT + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass(tunnel_code=None, origin_code=407) + with self.assertRaises(urllib.error.HTTPError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + with cm.exception as err: + self.assertEqual(err.code, 407) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertEqual(httpconn.tunnel_requests[1]["Proxy-Authorization"], + "Basic dXNlcjpwYXNz") + # Neither request to the origin carried the credentials. + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_basic_auth_handler_no_password(self): + password_mgr = urllib.request.HTTPPasswordMgr() + httpconn = MockHTTPTunnelClass() + with self.assertRaises(urllib.error.HTTPError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + with cm.exception as err: + self.assertEqual(err.code, 407) + self.assertEqual(err.reason, "Proxy Authentication Required") + self.assertEqual(err.headers["Proxy-Authenticate"], + 'Basic realm="proxy"') + self.assertEqual(err.read(), b"") + self.assertEqual(len(httpconn.tunnel_requests), 1) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_tunnel_error(self): + # Other failures to CONNECT are still raised as URLError + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass(tunnel_code=403) + with self.assertRaises(urllib.error.URLError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + self.assertNotIsInstance(cm.exception, urllib.error.HTTPError) + self.assertIsInstance(cm.exception.reason, + http.client.ProxyTunnelError) + self.assertEqual(cm.exception.reason.code, 403) + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") def test_winreg_proxy_bypass(self): proxy_override = "www.example.com;*.example.net; 192.168.0.1" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index da634a205c195e..a5166caed586d7 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -357,8 +357,10 @@ def get_full_url(self): return self.full_url def set_proxy(self, host, type): - if self.type == 'https' and not self._tunnel_host: - self._tunnel_host = self.host + if self.type == 'https': + # A retried request keeps the tunnel host set on the first pass. + if not self._tunnel_host: + self._tunnel_host = self.host else: self.type= type self.selector = self.full_url @@ -1228,10 +1230,19 @@ def get_authorization(self, req, chal): # HTTP proxy. RFC 7616, Section 3.4 allows both. An empty path is # sent as '/' by http.client, so use the same here. - uri = req.selector or '/' + if self.auth_header == 'Proxy-Authorization' and req._tunnel_host: + # For an https URL the proxy authenticates the CONNECT request, + # whose request-target is host:port, as sent by http.client. + method = 'CONNECT' + host, port = _splitport(req._tunnel_host) + uri = '%s:%s' % (host.encode('idna').decode('ascii'), + port or http.client.HTTPS_PORT) + else: + method = req.get_method() + uri = req.selector or '/' A1 = "%s:%s:%s" % (user, realm, pw) - A2 = "%s:%s" % (req.get_method(), uri) + A2 = "%s:%s" % (method, uri) # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` # or `auth-int` to the response back. we use `auth` to send the response back. if qop is None: @@ -1314,6 +1325,26 @@ def http_error_407(self, req, fp, code, msg, headers): self.reset_retry_count() return retry +class _ProxyTunnelResponse(io.BytesIO): + """The proxy's error response to CONNECT, as an empty-bodied response.""" + + def __init__(self, err, url): + super().__init__() + self.url = url + self.code = self.status = err.code + self.msg = self.reason = err.reason + self.headers = err.headers + + def info(self): + return self.headers + + def geturl(self): + return self.url + + def getcode(self): + return self.code + + class AbstractHTTPHandler(BaseHandler): def __init__(self, debuglevel=None): @@ -1407,6 +1438,12 @@ def do_open(self, http_class, req, **http_conn_args): try: h.request(req.get_method(), req.selector, req.data, headers, encode_chunked=req.has_header('Transfer-encoding')) + except http.client.ProxyTunnelError as err: + if err.code != 407: + raise URLError(err) + # Pass the proxy's challenge to CONNECT on to the + # http_error_407 handlers, like any other error response. + return _ProxyTunnelResponse(err, req.full_url) except OSError as err: # timeout error raise URLError(err) r = h.getresponse() diff --git a/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst b/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst new file mode 100644 index 00000000000000..9efd6dd0cca7db --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst @@ -0,0 +1,8 @@ +:class:`urllib.request.ProxyBasicAuthHandler` and +:class:`urllib.request.ProxyDigestAuthHandler` now handle a ``407 Proxy +Authentication Required`` response to the CONNECT request for an ``https`` +URL. :mod:`http.client` raises the new :exc:`~http.client.ProxyTunnelError`, +a subclass of :exc:`OSError`, when a proxy rejects a CONNECT request. +For such a proxy, :class:`~urllib.request.ProxyDigestAuthHandler` computes +the digest over the CONNECT request (``CONNECT host:port``) instead of the +request sent to the origin server.