diff --git a/HISTORY.rst b/HISTORY.rst index 8015410a2..3b340e98f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,16 @@ Release History dev --- +API Changes (Backward-Compatible) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Added a new flag to the ``H2Connection`` constructor: ``header_encoding``, + that controls what encoding is used (if any) to decode the headers from bytes + to unicode. This defaults to UTF-8 for backward compatibility. To disable the + decode and use bytes exclusively, set the field to False, None, or the empty + string. +- Bumped the minimum version of HPACK allowed from 2.0 to 2.1. + 2.2.3 (2016-04-13) ------------------ diff --git a/h2/connection.py b/h2/connection.py index 615e1bf9d..7562f1754 100644 --- a/h2/connection.py +++ b/h2/connection.py @@ -231,11 +231,22 @@ class H2Connection(object): constraints of those state machines are met as well. Attempts to create frames that cannot be sent will raise a ``ProtocolError``. + .. versionchanged:: 2.3.0 + Added the ``header_encoding`` keyword argument. + :param client_side: Whether this object is to be used on the client side of a connection, or on the server side. Affects the logic used by the state machine, the default settings values, the allowable stream IDs, and several other properties. Defaults to ``True``. :type client_side: ``bool`` + + :param header_encoding: Controls whether the headers emitted by this object + in events are transparently decoded to ``unicode`` strings, and what + encoding is used to do that decoding. For historical reason, this + defaults to ``'utf-8'``. To prevent the decoding of headers (that is, + to force them to be returned as bytestrings), this can be set to + ``False`` or the empty string. + :type header_encoding: ``str`` or ``False`` """ # The initial maximum outbound frame size. This can be changed by receiving # a settings frame. @@ -251,7 +262,7 @@ class H2Connection(object): # The largest acceptable window increment. MAX_WINDOW_INCREMENT = 2**31 - 1 - def __init__(self, client_side=True): + def __init__(self, client_side=True, header_encoding='utf-8'): self.state_machine = H2ConnectionStateMachine() self.streams = {} self.highest_inbound_stream_id = 0 @@ -291,6 +302,16 @@ def __init__(self, client_side=True): #: bytes. self.max_inbound_frame_size = self.local_settings.max_frame_size + #: Controls whether the headers emitted by this object in events are + #: transparently decoded to ``unicode`` strings, and what encoding is + #: used to do that decoding. For historical reason, this defaults to + #: ``'utf-8'``. To prevent the decoding of headers (that is, to force + #: them to be returned as bytestrings), this can be set to ``False`` or + #: the empty string. + #: + #: .. versionadded:: 2.3.0 + self.header_encoding = header_encoding + # Buffer for incoming data. self.incoming_buffer = FrameBuffer(server=not client_side) @@ -1022,9 +1043,11 @@ def _receive_headers_frame(self, frame): (max_open_streams, self.open_outbound_streams) ) - # Let's decode the headers. + # Let's decode the headers. We handle headers as bytes internally up + # until we hang them off the event, at which point we may optionally + # convert them to unicode. try: - headers = self.decoder.decode(frame.data) + headers = self.decoder.decode(frame.data, raw=True) except (HPACKError, IndexError, TypeError, UnicodeDecodeError) as e: # We should only need HPACKError here, but versions of HPACK # older than 2.1.0 throw all three others as well. For maximum @@ -1040,7 +1063,8 @@ def _receive_headers_frame(self, frame): ) frames, stream_events = stream.receive_headers( headers, - 'END_STREAM' in frame.flags + 'END_STREAM' in frame.flags, + self.header_encoding ) if 'PRIORITY' in frame.flags: diff --git a/h2/stream.py b/h2/stream.py index bdddbc551..117247137 100644 --- a/h2/stream.py +++ b/h2/stream.py @@ -749,7 +749,7 @@ def remotely_pushed(self): ) return [], events - def receive_headers(self, headers, end_stream): + def receive_headers(self, headers, end_stream, header_encoding): """ Receive a set of headers (or trailers). """ @@ -775,6 +775,12 @@ def receive_headers(self, headers, end_stream): if not end_stream: raise ProtocolError("Trailers must have END_STREAM set") + if header_encoding: + headers = [ + (n.decode(header_encoding), v.decode(header_encoding)) + for n, v in headers + ] + events[0].headers = headers return [], events @@ -882,7 +888,7 @@ def _initialize_content_length(self, headers): Content-Length header to be present. """ for n, v in headers: - if n == 'content-length': + if n == b'content-length': try: self._expected_content_length = int(v, 10) except ValueError: diff --git a/h2/utilities.py b/h2/utilities.py index 001f39d89..b5e3362fd 100644 --- a/h2/utilities.py +++ b/h2/utilities.py @@ -9,16 +9,16 @@ from .exceptions import ProtocolError, FlowControlError -UPPER_RE = re.compile("[A-Z]") +UPPER_RE = re.compile(b"[A-Z]") # A set of headers that are hop-by-hop or connection-specific and thus # forbidden in HTTP/2. This list comes from RFC 7540 ยง 8.1.2.2. CONNECTION_HEADERS = set([ - 'connection', - 'proxy-connection', - 'keep-alive', - 'transfer-encoding', - 'upgrade', + b'connection', + b'proxy-connection', + b'keep-alive', + b'transfer-encoding', + b'upgrade', ]) @@ -117,8 +117,8 @@ def _reject_te(headers): its value is anything other than "trailers". """ for header in headers: - if header[0] == 'te': - if header[1].lower().strip() != 'trailers': + if header[0] == b'te': + if header[1].lower().strip() != b'trailers': raise ProtocolError( "Invalid value for Transfer-Encoding header: %s" % header[1] @@ -151,7 +151,7 @@ def _reject_pseudo_header_fields(headers): seen_regular_header = False for header in headers: - if header[0].startswith(':'): + if header[0].startswith(b':'): if header[0] in seen_pseudo_header_fields: raise ProtocolError( "Received duplicate pseudo-header field %s" % header[0] diff --git a/setup.py b/setup.py index eb00cb7d4..5e08df308 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ ], install_requires=[ 'hyperframe>=3.1, <5, !=4.0.0', - 'hpack>=2.0, <3', + 'hpack>=2.1, <3', ], extras_require={ ':python_version == "2.7" or python_version == "3.3"': ['enum34>=1.0.4, <2'], diff --git a/test/test_basic_logic.py b/test/test_basic_logic.py index b8b2203f8..bf83908b0 100644 --- a/test/test_basic_logic.py +++ b/test/test_basic_logic.py @@ -35,8 +35,12 @@ class TestBasicClient(object): (':method', 'GET'), ] example_response_headers = [ - (':status', '200'), - ('server', 'fake-serv/0.1.0') + (u':status', u'200'), + (u'server', u'fake-serv/0.1.0') + ] + bytes_example_response_headers = [ + (b':status', b'200'), + (b'server', b'fake-serv/0.1.0') ] def test_begin_connection(self, frame_factory): @@ -127,6 +131,28 @@ def test_receiving_a_response(self, frame_factory): assert event.stream_id == 1 assert event.headers == self.example_response_headers + def test_receiving_a_response_bytes(self, frame_factory): + """ + When receiving a response, the ResponseReceived event fires with bytes + headers if the encoding is set appropriately. + """ + c = h2.connection.H2Connection(header_encoding=False) + c.initiate_connection() + c.send_headers(1, self.example_request_headers, end_stream=True) + + # Clear the data + f = frame_factory.build_headers_frame( + self.example_response_headers + ) + events = c.receive_data(f.serialize()) + + assert len(events) == 1 + event = events[0] + + assert isinstance(event, h2.events.ResponseReceived) + assert event.stream_id == 1 + assert event.headers == self.bytes_example_response_headers + def test_end_stream_without_data(self, frame_factory): """ Ending a stream without data emits a zero-length DATA frame with @@ -528,10 +554,16 @@ class TestBasicServer(object): Basic server-side tests. """ example_request_headers = [ - (':authority', 'example.com'), - (':path', '/'), - (':scheme', 'https'), - (':method', 'GET'), + (u':authority', u'example.com'), + (u':path', u'/'), + (u':scheme', u'https'), + (u':method', u'GET'), + ] + bytes_example_request_headers = [ + (b':authority', b'example.com'), + (b':path', b'/'), + (b':scheme', b'https'), + (b':method', b'GET'), ] example_response_headers = [ (':status', '200'), @@ -597,6 +629,27 @@ def test_headers_event(self, frame_factory): assert event.stream_id == 1 assert event.headers == self.example_request_headers + def test_headers_event_bytes(self, frame_factory): + """ + When a headers frame is received a RequestReceived event fires with + bytes headers if the encoding is set appropriately. + """ + c = h2.connection.H2Connection( + client_side=False, header_encoding=False + ) + c.receive_data(frame_factory.preamble()) + + f = frame_factory.build_headers_frame(self.example_request_headers) + data = f.serialize() + events = c.receive_data(data) + + assert len(events) == 1 + event = events[0] + + assert isinstance(event, h2.events.RequestReceived) + assert event.stream_id == 1 + assert event.headers == self.bytes_example_request_headers + def test_data_event(self, frame_factory): """ Test that data received on a stream fires a DataReceived event. diff --git a/test/test_invalid_headers.py b/test/test_invalid_headers.py index c57f70b0a..0b13c97a7 100644 --- a/test/test_invalid_headers.py +++ b/test/test_invalid_headers.py @@ -15,9 +15,9 @@ import h2.utilities from hypothesis import given -from hypothesis.strategies import text, lists, tuples +from hypothesis.strategies import binary, lists, tuples -HEADERS_STRATEGY = lists(tuples(text(), text())) +HEADERS_STRATEGY = lists(tuples(binary(), binary())) class TestInvalidFrameSequences(object):