Allow users to obtain raw headers.

HISTORY.rst

@@ -4,6 +4,16 @@ Release History
 dev
 ---
 
+API Changes (Backward-Compatible)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Added a new flag to the ``H2Connection`` constructor: ``header_encoding``,
+  that controls what encoding is used (if any) to decode the headers from bytes
+  to unicode. This defaults to UTF-8 for backward compatibility. To disable the
+  decode and use bytes exclusively, set the field to False, None, or the empty
+  string.
+- Bumped the minimum version of HPACK allowed from 2.0 to 2.1.
+
 2.2.3 (2016-04-13)
 ------------------
 

h2/connection.py

@@ -231,11 +231,22 @@ class H2Connection(object):
     constraints of those state machines are met as well. Attempts to create
     frames that cannot be sent will raise a ``ProtocolError``.
 
+    .. versionchanged:: 2.3.0
+       Added the ``header_encoding`` keyword argument.
+
     :param client_side: Whether this object is to be used on the client side of
         a connection, or on the server side. Affects the logic used by the
         state machine, the default settings values, the allowable stream IDs,
         and several other properties. Defaults to ``True``.
     :type client_side: ``bool``
+
+    :param header_encoding: Controls whether the headers emitted by this object
+        in events are transparently decoded to ``unicode`` strings, and what
+        encoding is used to do that decoding. For historical reason, this
+        defaults to ``'utf-8'``. To prevent the decoding of headers (that is,
+        to force them to be returned as bytestrings), this can be set to
+        ``False`` or the empty string.
+    :type header_encoding: ``str`` or ``False``
     """
     # The initial maximum outbound frame size. This can be changed by receiving
     # a settings frame.
@@ -251,7 +262,7 @@ class H2Connection(object):
     # The largest acceptable window increment.
     MAX_WINDOW_INCREMENT = 2**31 - 1
 
-    def __init__(self, client_side=True):
+    def __init__(self, client_side=True, header_encoding='utf-8'):
         self.state_machine = H2ConnectionStateMachine()
         self.streams = {}
         self.highest_inbound_stream_id = 0
@@ -291,6 +302,16 @@ def __init__(self, client_side=True):
         #: bytes.
         self.max_inbound_frame_size = self.local_settings.max_frame_size
 
+        #: Controls whether the headers emitted by this object in events are
+        #: transparently decoded to ``unicode`` strings, and what encoding is
+        #: used to do that decoding. For historical reason, this defaults to
+        #: ``'utf-8'``. To prevent the decoding of headers (that is, to force
+        #: them to be returned as bytestrings), this can be set to ``False`` or
+        #: the empty string.
+        #:
+        #: .. versionadded:: 2.3.0
+        self.header_encoding = header_encoding
+
         # Buffer for incoming data.
         self.incoming_buffer = FrameBuffer(server=not client_side)
 
@@ -1022,9 +1043,11 @@ def _receive_headers_frame(self, frame):
                     (max_open_streams, self.open_outbound_streams)
                 )
 
-        # Let's decode the headers.
+        # Let's decode the headers. We handle headers as bytes internally up
+        # until we hang them off the event, at which point we may optionally
+        # convert them to unicode.
         try:
-            headers = self.decoder.decode(frame.data)
+            headers = self.decoder.decode(frame.data, raw=True)
         except (HPACKError, IndexError, TypeError, UnicodeDecodeError) as e:
             # We should only need HPACKError here, but versions of HPACK
             # older than 2.1.0 throw all three others as well. For maximum
@@ -1040,7 +1063,8 @@ def _receive_headers_frame(self, frame):
         )
         frames, stream_events = stream.receive_headers(
             headers,
-            'END_STREAM' in frame.flags
+            'END_STREAM' in frame.flags,
+            self.header_encoding
         )
 
         if 'PRIORITY' in frame.flags:

h2/stream.py

@@ -749,7 +749,7 @@ def remotely_pushed(self):
         )
         return [], events
 
-    def receive_headers(self, headers, end_stream):
+    def receive_headers(self, headers, end_stream, header_encoding):
         """
         Receive a set of headers (or trailers).
         """
@@ -775,6 +775,12 @@ def receive_headers(self, headers, end_stream):
             if not end_stream:
                 raise ProtocolError("Trailers must have END_STREAM set")
 
+        if header_encoding:
+            headers = [
+                (n.decode(header_encoding), v.decode(header_encoding))
+                for n, v in headers
+            ]
+
         events[0].headers = headers
         return [], events
 
@@ -882,7 +888,7 @@ def _initialize_content_length(self, headers):
         Content-Length header to be present.
         """
         for n, v in headers:
-            if n == 'content-length':
+            if n == b'content-length':
                 try:
                     self._expected_content_length = int(v, 10)
                 except ValueError:

h2/utilities.py

@@ -9,16 +9,16 @@
 
 from .exceptions import ProtocolError, FlowControlError
 
-UPPER_RE = re.compile("[A-Z]")
+UPPER_RE = re.compile(b"[A-Z]")
 
 # A set of headers that are hop-by-hop or connection-specific and thus
 # forbidden in HTTP/2. This list comes from RFC 7540 § 8.1.2.2.
 CONNECTION_HEADERS = set([
-    'connection',
-    'proxy-connection',
-    'keep-alive',
-    'transfer-encoding',
-    'upgrade',
+    b'connection',
+    b'proxy-connection',
+    b'keep-alive',
+    b'transfer-encoding',
+    b'upgrade',
 ])
 
 
@@ -117,8 +117,8 @@ def _reject_te(headers):
     its value is anything other than "trailers".
     """
     for header in headers:
-        if header[0] == 'te':
-            if header[1].lower().strip() != 'trailers':
+        if header[0] == b'te':
+            if header[1].lower().strip() != b'trailers':
                 raise ProtocolError(
                     "Invalid value for Transfer-Encoding header: %s" %
                     header[1]
@@ -151,7 +151,7 @@ def _reject_pseudo_header_fields(headers):
     seen_regular_header = False
 
     for header in headers:
-        if header[0].startswith(':'):
+        if header[0].startswith(b':'):
             if header[0] in seen_pseudo_header_fields:
                 raise ProtocolError(
                     "Received duplicate pseudo-header field %s" % header[0]

setup.py

@@ -62,7 +62,7 @@
     ],
     install_requires=[
         'hyperframe>=3.1, <5, !=4.0.0',
-        'hpack>=2.0, <3',
+        'hpack>=2.1, <3',
     ],
     extras_require={
         ':python_version == "2.7" or python_version == "3.3"': ['enum34>=1.0.4, <2'],

test/test_basic_logic.py

@@ -35,8 +35,12 @@ class TestBasicClient(object):
         (':method', 'GET'),
     ]
     example_response_headers = [
-        (':status', '200'),
-        ('server', 'fake-serv/0.1.0')
+        (u':status', u'200'),
+        (u'server', u'fake-serv/0.1.0')
+    ]
+    bytes_example_response_headers = [
+        (b':status', b'200'),
+        (b'server', b'fake-serv/0.1.0')
     ]
 
     def test_begin_connection(self, frame_factory):
@@ -127,6 +131,28 @@ def test_receiving_a_response(self, frame_factory):
         assert event.stream_id == 1
         assert event.headers == self.example_response_headers
 
+    def test_receiving_a_response_bytes(self, frame_factory):
+        """
+        When receiving a response, the ResponseReceived event fires with bytes
+        headers if the encoding is set appropriately.
+        """
+        c = h2.connection.H2Connection(header_encoding=False)
+        c.initiate_connection()
+        c.send_headers(1, self.example_request_headers, end_stream=True)
+
+        # Clear the data
+        f = frame_factory.build_headers_frame(
+            self.example_response_headers
+        )
+        events = c.receive_data(f.serialize())
+
+        assert len(events) == 1
+        event = events[0]
+
+        assert isinstance(event, h2.events.ResponseReceived)
+        assert event.stream_id == 1
+        assert event.headers == self.bytes_example_response_headers
+
     def test_end_stream_without_data(self, frame_factory):
         """
         Ending a stream without data emits a zero-length DATA frame with
@@ -528,10 +554,16 @@ class TestBasicServer(object):
     Basic server-side tests.
     """
     example_request_headers = [
-        (':authority', 'example.com'),
-        (':path', '/'),
-        (':scheme', 'https'),
-        (':method', 'GET'),
+        (u':authority', u'example.com'),
+        (u':path', u'/'),
+        (u':scheme', u'https'),
+        (u':method', u'GET'),
+    ]
+    bytes_example_request_headers = [
+        (b':authority', b'example.com'),
+        (b':path', b'/'),
+        (b':scheme', b'https'),
+        (b':method', b'GET'),
     ]
     example_response_headers = [
         (':status', '200'),
@@ -597,6 +629,27 @@ def test_headers_event(self, frame_factory):
         assert event.stream_id == 1
         assert event.headers == self.example_request_headers
 
+    def test_headers_event_bytes(self, frame_factory):
+        """
+        When a headers frame is received a RequestReceived event fires with
+        bytes headers if the encoding is set appropriately.
+        """
+        c = h2.connection.H2Connection(
+            client_side=False, header_encoding=False
+        )
+        c.receive_data(frame_factory.preamble())
+
+        f = frame_factory.build_headers_frame(self.example_request_headers)
+        data = f.serialize()
+        events = c.receive_data(data)
+
+        assert len(events) == 1
+        event = events[0]
+
+        assert isinstance(event, h2.events.RequestReceived)
+        assert event.stream_id == 1
+        assert event.headers == self.bytes_example_request_headers
+
     def test_data_event(self, frame_factory):
         """
         Test that data received on a stream fires a DataReceived event.

test/test_invalid_headers.py

@@ -15,9 +15,9 @@
 import h2.utilities
 
 from hypothesis import given
-from hypothesis.strategies import text, lists, tuples
+from hypothesis.strategies import binary, lists, tuples
 
-HEADERS_STRATEGY = lists(tuples(text(), text()))
+HEADERS_STRATEGY = lists(tuples(binary(), binary()))
 
 
 class TestInvalidFrameSequences(object):