diff --git a/.travis.yml b/.travis.yml index 4ea96d6f..642c7d41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.4" - "3.5" + - "3.6" - "pypy-5.3.1" env: @@ -19,7 +20,7 @@ matrix: install: - ".travis/install.sh" -before_script: "flake8 --max-complexity 15 hyper test" +before_script: "flake8 hyper test" script: - ".travis/run.sh" diff --git a/.travis/run.sh b/.travis/run.sh index 43d9dd65..321835de 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -6,7 +6,7 @@ set -x if [[ "$TEST_RELEASE" == true ]]; then py.test test_release.py else - if [[ $TRAVIS_PYTHON_VERSION == pypy ]]; then + if [[ $TRAVIS_PYTHON_VERSION == pypy* ]]; then py.test test/ else coverage run -m py.test test/ diff --git a/HISTORY.rst b/HISTORY.rst index c45c08ce..dab8eed7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,10 @@ Release History dev --- +*Bugfixes* + +- Stream end flag when length of last chunk equal to MAX_CHUNK + v0.7.0 (2016-09-27) ------------------- diff --git a/README.rst b/README.rst index 294a062b..99dce29d 100644 --- a/README.rst +++ b/README.rst @@ -2,10 +2,23 @@ Hyper: HTTP/2 Client for Python =============================== -.. image:: https://raw.github.com/Lukasa/hyper/development/docs/source/images/hyper.png +**This project is no longer maintained!** + +Please use an alternative, such as `HTTPX`_ or others. + +.. _HTTPX: https://www.python-httpx.org/ + +We will not publish further updates for ``hyper``. + +Potential security issues will not be addressed. -.. image:: https://travis-ci.org/Lukasa/hyper.svg?branch=master - :target: https://travis-ci.org/Lukasa/hyper +---- + +**So long, and thanks for all the fish!** + +---- + +.. image:: https://raw.github.com/Lukasa/hyper/development/docs/source/images/hyper.png HTTP is changing under our feet. HTTP/1.1, our old friend, is being supplemented by the brand new HTTP/2 standard. HTTP/2 provides many benefits: @@ -15,8 +28,8 @@ improved speed, lower bandwidth usage, better connection management, and more. from hyper import HTTPConnection - conn = HTTPConnection('http2bin.org:443') - conn.request('GET', '/get') + conn = HTTPConnection('nghttp2.org:443') + conn.request('GET', '/httpbin/get') resp = conn.get_response() print(resp.read()) diff --git a/hyper/contrib.py b/hyper/contrib.py index ff4f8ff8..79aa7d12 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -28,9 +28,10 @@ class HTTP20Adapter(HTTPAdapter): HTTP/2. This implements some degree of connection pooling to maximise the HTTP/2 gain. """ - def __init__(self, *args, **kwargs): + def __init__(self, window_manager=None, *args, **kwargs): #: A mapping between HTTP netlocs and ``HTTP20Connection`` objects. self.connections = {} + self.window_manager = window_manager def get_connection(self, host, port, scheme, cert=None, verify=True, proxy=None, timeout=None): @@ -75,6 +76,7 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, host, port, secure=secure, + window_manager=self.window_manager, ssl_context=ssl_context, proxy_host=proxy_netloc, proxy_headers=proxy_headers, diff --git a/hyper/http11/response.py b/hyper/http11/response.py index 8f3eb985..7ff7a523 100644 --- a/hyper/http11/response.py +++ b/hyper/http11/response.py @@ -9,6 +9,7 @@ import logging import weakref import zlib +import brotli from ..common.decoder import DeflateDecoder from ..common.exceptions import ChunkedDecodeError, InvalidResponseError @@ -53,10 +54,13 @@ def __init__(self, code, reason, headers, sock, connection=None, self._expect_close = True # The expected length of the body. - try: - self._length = int(self.headers[b'content-length'][0]) - except KeyError: - self._length = None + if request_method != b'HEAD': + try: + self._length = int(self.headers[b'content-length'][0]) + except KeyError: + self._length = None + else: + self._length = 0 # Whether we expect a chunked response. self._chunked = ( @@ -85,6 +89,8 @@ def __init__(self, code, reason, headers, sock, connection=None, # http://stackoverflow.com/a/2695466/1401686 if b'gzip' in self.headers.get(b'content-encoding', []): self._decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS) + elif b'br' in self.headers.get(b'content-encoding', []): + self._decompressobj = brotli.Decompressor() elif b'deflate' in self.headers.get(b'content-encoding', []): self._decompressobj = DeflateDecoder() else: diff --git a/hyper/http20/response.py b/hyper/http20/response.py index 280ffbb2..bed10475 100644 --- a/hyper/http20/response.py +++ b/hyper/http20/response.py @@ -8,6 +8,7 @@ """ import logging import zlib +import brotli from ..common.decoder import DeflateDecoder from ..common.headers import HTTPHeaderMap @@ -31,6 +32,7 @@ def strip_headers(headers): decompressors = { b'gzip': lambda: zlib.decompressobj(16 + zlib.MAX_WBITS), + b'br': brotli.Decompressor, b'deflate': DeflateDecoder } @@ -79,8 +81,9 @@ def __init__(self, headers, stream): # Stack Overflow answer for more: # http://stackoverflow.com/a/2695466/1401686 for c in self.headers.get(b'content-encoding', []): - self._decompressobj = decompressors.get(c)() - break + if c in decompressors: + self._decompressobj = decompressors.get(c)() + break @property def trailers(self): diff --git a/hyper/http20/stream.py b/hyper/http20/stream.py index 598a1490..3c064783 100644 --- a/hyper/http20/stream.py +++ b/hyper/http20/stream.py @@ -122,8 +122,18 @@ def file_iterator(fobj): chunks = (data[i:i+MAX_CHUNK] for i in range(0, len(data), MAX_CHUNK)) - for chunk in chunks: - self._send_chunk(chunk, final) + # since we need to know when we have a last package we need to know + # if there is another package in advance + cur_chunk = None + try: + cur_chunk = next(chunks) + while True: + next_chunk = next(chunks) + self._send_chunk(cur_chunk, False) + cur_chunk = next_chunk + except StopIteration: + if cur_chunk is not None: # cur_chunk none when no chunks to send + self._send_chunk(cur_chunk, final) def _read(self, amt=None): """ @@ -323,19 +333,12 @@ def _send_chunk(self, data, final): while len(data) > self._out_flow_control_window: self._recv_cb() - # If the length of the data is less than MAX_CHUNK, we're probably - # at the end of the file. If this is the end of the data, mark it - # as END_STREAM. - end_stream = False - if len(data) < MAX_CHUNK and final: - end_stream = True - # Send the frame and decrement the flow control window. with self._conn as conn: conn.send_data( - stream_id=self.stream_id, data=data, end_stream=end_stream + stream_id=self.stream_id, data=data, end_stream=final ) self._send_outstanding_data() - if end_stream: + if final: self.local_closed = True diff --git a/hyper/ssl_compat.py b/hyper/ssl_compat.py index 71ebcd3a..97e6fb2e 100644 --- a/hyper/ssl_compat.py +++ b/hyper/ssl_compat.py @@ -188,7 +188,7 @@ def resolve_alias(alias): C='countryName', ST='stateOrProvinceName', L='localityName', - O='organizationName', + O='organizationName', # noqa: E741 OU='organizationalUnitName', CN='commonName', ).get(alias, alias) diff --git a/setup.cfg b/setup.cfg index 5e409001..53d397a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [wheel] universal = 1 + +[flake8] +max-complexity = 15 diff --git a/setup.py b/setup.py index a2578a6b..94cd8d21 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def run_tests(self): 'Programming Language :: Python :: Implementation :: CPython', ], install_requires=[ - 'h2>=2.4,<3.0,!=2.5.0', 'hyperframe>=3.2,<4.0', 'rfc3986>=1.1.0,<2.0' + 'h2>=2.4,<3.0,!=2.5.0', 'hyperframe>=3.2,<4.0', 'rfc3986>=1.1.0,<2.0', 'brotlipy>=0.7.0,<1.0' ], tests_require=['pytest', 'requests', 'mock'], cmdclass={'test': PyTest}, diff --git a/test/test_SSLContext.py b/test/test_SSLContext.py index 4add16f3..e6051af7 100644 --- a/test/test_SSLContext.py +++ b/test/test_SSLContext.py @@ -40,7 +40,6 @@ def test_custom_context(self): assert not hyper.tls._context.check_hostname assert hyper.tls._context.verify_mode == ssl.CERT_NONE - assert hyper.tls._context.options & ssl.OP_NO_COMPRESSION == 0 def test_HTTPConnection_with_custom_context(self): context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) @@ -70,7 +69,7 @@ def test_missing_certs(self): succeeded = True except hyper.common.exceptions.MissingCertFile: threw_expected_exception = True - except: + except Exception: pass assert not succeeded diff --git a/test/test_http11.py b/test/test_http11.py index 21dd7f70..9f3fd3d0 100644 --- a/test/test_http11.py +++ b/test/test_http11.py @@ -7,6 +7,7 @@ """ import os import zlib +import brotli from collections import namedtuple from io import BytesIO, StringIO @@ -637,13 +638,23 @@ def test_response_transparently_decrypts_gzip(self): headers = {b'content-encoding': [b'gzip'], b'connection': [b'close']} r = HTTP11Response(200, 'OK', headers, d, None) - c = zlib_compressobj(wbits=24) + c = zlib_compressobj(wbits=25) body = c.compress(b'this is test data') body += c.flush() d._buffer = BytesIO(body) assert r.read() == b'this is test data' + def test_response_transparently_decrypts_brotli(self): + d = DummySocket() + headers = {b'content-encoding': [b'br'], b'connection': [b'close']} + r = HTTP11Response(200, 'OK', headers, d, None) + + body = brotli.compress(b'this is test data') + d._buffer = BytesIO(body) + + assert r.read() == b'this is test data' + def test_response_transparently_decrypts_real_deflate(self): d = DummySocket() headers = { @@ -719,7 +730,7 @@ def test_response_transparently_decrypts_chunked_gzip(self): } r = HTTP11Response(200, 'OK', headers, d, None) - c = zlib_compressobj(wbits=24) + c = zlib_compressobj(wbits=25) body = c.compress(b'this is test data') body += c.flush() @@ -804,7 +815,7 @@ def test_bounded_read_expect_close_with_content_length(self): def test_compressed_bounded_read_expect_close(self): headers = {b'connection': [b'close'], b'content-encoding': [b'gzip']} - c = zlib_compressobj(wbits=24) + c = zlib_compressobj(wbits=25) body = c.compress(b'hello there sir') body += c.flush() @@ -941,6 +952,18 @@ def test_response_version(self): r = HTTP11Response(200, 'OK', headers, d) assert r.version is HTTPVersion.http11 + def test_response_body_length(self): + methods = [b'HEAD', b'GET'] + headers = {b'content-length': [b'15']} + d = DummySocket() + for method in methods: + d.queue = [] + r = HTTP11Response(200, 'OK', headers, d, request_method=method) + if method == b'HEAD': + assert r._length == 0 + else: + assert r._length == int(r.headers[b'content-length'][0]) + class DummySocket(object): def __init__(self): diff --git a/test/test_hyper.py b/test/test_hyper.py index f4a5994d..b826c63c 100644 --- a/test/test_hyper.py +++ b/test/test_hyper.py @@ -26,6 +26,7 @@ import pytest import socket import zlib +import brotli from io import BytesIO TEST_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -211,6 +212,61 @@ def data_callback(chunk, **kwargs): assert frames[1].data == b'hello there' assert frames[1].flags == set(['END_STREAM']) + def test_request_correctly_sent_max_chunk(self, frame_buffer): + """ + Test that request correctly sent when data length multiple + max chunk. We check last chunk has a end flag and correct number + of chunks. + """ + def data_callback(chunk, **kwargs): + frame_buffer.add_data(chunk) + + # one chunk + c = HTTP20Connection('www.google.com') + c._sock = DummySocket() + c._send_cb = data_callback + c.putrequest('GET', '/') + c.endheaders(message_body=b'1'*1024, final=True) + + frames = list(frame_buffer) + assert len(frames) == 2 + assert isinstance(frames[1], DataFrame) + assert frames[1].flags == set(['END_STREAM']) + + # two chunks + c = HTTP20Connection('www.google.com') + c._sock = DummySocket() + c._send_cb = data_callback + c.putrequest('GET', '/') + c.endheaders(message_body=b'1' * 2024, final=True) + + frames = list(frame_buffer) + assert len(frames) == 3 + assert isinstance(frames[1], DataFrame) + assert frames[2].flags == set(['END_STREAM']) + + # two chunks with last chunk < 1024 + c = HTTP20Connection('www.google.com') + c._sock = DummySocket() + c._send_cb = data_callback + c.putrequest('GET', '/') + c.endheaders(message_body=b'1' * 2000, final=True) + + frames = list(frame_buffer) + assert len(frames) == 3 + assert isinstance(frames[1], DataFrame) + assert frames[2].flags == set(['END_STREAM']) + + # no chunks + c = HTTP20Connection('www.google.com') + c._sock = DummySocket() + c._send_cb = data_callback + c.putrequest('GET', '/') + c.endheaders(message_body=b'', final=True) + + frames = list(frame_buffer) + assert len(frames) == 1 + def test_that_we_correctly_send_over_the_socket(self): sock = DummySocket() c = HTTP20Connection('www.google.com') @@ -1026,13 +1082,22 @@ def test_response_transparently_decrypts_gzip(self): headers = HTTPHeaderMap( [(':status', '200'), ('content-encoding', 'gzip')] ) - c = zlib_compressobj(wbits=24) + c = zlib_compressobj(wbits=25) body = c.compress(b'this is test data') body += c.flush() resp = HTTP20Response(headers, DummyStream(body)) assert resp.read() == b'this is test data' + def test_response_transparently_decrypts_brotli(self): + headers = HTTPHeaderMap( + [(':status', '200'), ('content-encoding', 'br')] + ) + body = brotli.compress(b'this is test data') + resp = HTTP20Response(headers, DummyStream(body)) + + assert resp.read() == b'this is test data' + def test_response_transparently_decrypts_real_deflate(self): headers = HTTPHeaderMap( [(':status', '200'), ('content-encoding', 'deflate')] @@ -1055,6 +1120,15 @@ def test_response_transparently_decrypts_wrong_deflate(self): assert resp.read() == b'this is test data' + def test_response_ignored_unsupported_compression(self): + headers = HTTPHeaderMap( + [(':status', '200'), ('content-encoding', 'invalid')] + ) + body = b'this is test data' + resp = HTTP20Response(headers, DummyStream(body)) + + assert resp.read() == b'this is test data' + def test_response_calls_stream_close(self): headers = HTTPHeaderMap([(':status', '200')]) stream = DummyStream('') @@ -1144,7 +1218,7 @@ def test_read_compressed_frames(self): headers = HTTPHeaderMap( [(':status', '200'), ('content-encoding', 'gzip')] ) - c = zlib_compressobj(wbits=24) + c = zlib_compressobj(wbits=25) body = c.compress(b'this is test data') body += c.flush() diff --git a/test_release.py b/test_release.py index 903994a9..38138657 100644 --- a/test_release.py +++ b/test_release.py @@ -10,17 +10,19 @@ capable of achieving basic tasks. """ -from concurrent.futures import as_completed, ThreadPoolExecutor import logging import random +from concurrent.futures import as_completed, ThreadPoolExecutor + import requests -import threading + from hyper import HTTP20Connection, HTTP11Connection, HTTPConnection from hyper.common.util import HTTPVersion from hyper.contrib import HTTP20Adapter logging.basicConfig(level=logging.INFO) + class TestHyperActuallyWorks(object): def test_abusing_nghttp2_org(self): """ @@ -93,32 +95,32 @@ def do_one_page(path): assert text_data max_workers = len(paths) - with ThreadPoolExecutor(max_workers=len(paths)) as ex: + with ThreadPoolExecutor(max_workers=max_workers) as ex: futures = [ex.submit(do_one_page, p) for p in paths] for f in as_completed(futures): f.result() - def test_hitting_http2bin_org(self): + def test_hitting_nghttp2_org(self): """ - This test function uses the requests adapter and requests to talk to http2bin. + This test function uses the requests adapter and requests to talk to nghttp2.org/httpbin. """ s = requests.Session() a = HTTP20Adapter() - s.mount('https://http2bin', a) - s.mount('https://www.http2bin', a) + s.mount('https://nghttp2', a) + s.mount('https://www.nghttp2', a) # Here are some nice URLs. urls = [ - 'https://www.http2bin.org/', - 'https://www.http2bin.org/ip', - 'https://www.http2bin.org/user-agent', - 'https://www.http2bin.org/headers', - 'https://www.http2bin.org/get', - 'https://http2bin.org/', - 'https://http2bin.org/ip', - 'https://http2bin.org/user-agent', - 'https://http2bin.org/headers', - 'https://http2bin.org/get', + 'https://www.nghttp2.org/httpbin/', + 'https://www.nghttp2.org/httpbin/ip', + 'https://www.nghttp2.org/httpbin/user-agent', + 'https://www.nghttp2.org/httpbin/headers', + 'https://www.nghttp2.org/httpbin/get', + 'https://nghttp2.org/httpbin/', + 'https://nghttp2.org/httpbin/ip', + 'https://nghttp2.org/httpbin/user-agent', + 'https://nghttp2.org/httpbin/headers', + 'https://nghttp2.org/httpbin/get', ] # Go get everything. @@ -132,7 +134,7 @@ def test_hitting_httpbin_org_http11(self): """ This test function uses hyper's HTTP/1.1 support to talk to httpbin """ - c = HTTP11Connection('httpbin.org') + c = HTTP11Connection('httpbin.org:443') # Here are some nice URLs. urls = [ @@ -166,3 +168,27 @@ def test_hitting_nghttp2_org_via_h2c_upgrade(self): assert response.status == 200 assert response.read() assert response.version == HTTPVersion.http20 + + def test_http11_response_body_length(self): + """ + This test function uses check the expected length of the HTTP/1.1-response-body. + """ + c = HTTP11Connection('httpbin.org:443') + + # Make some HTTP/1.1 requests. + methods = ['GET', 'HEAD'] + for method in methods: + c.request(method, '/') + resp = c.get_response() + + # Check the expected length of the body. + if method == 'HEAD': + assert resp._length == 0 + assert resp.read() == b'' + else: + try: + content_length = int(resp.headers[b'Content-Length'][0]) + except KeyError: + continue + assert resp._length == content_length + assert resp.read() diff --git a/test_requirements.txt b/test_requirements.txt index dcaff945..cae2fbc6 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,4 @@ -pytest>=2.7 +pytest>=3.0 pytest-xdist pytest-cov requests diff --git a/tox.ini b/tox.ini index 311f9c97..046619e9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, pypy, lint +envlist = py{27,34,35,36}, pypy, lint [testenv] deps= -r{toxinidir}/test_requirements.txt @@ -12,6 +12,6 @@ commands= commands= py.test {toxinidir}/test/ [testenv:lint] -basepython=python3.5 +basepython=python3 deps = flake8==2.5.4 -commands = flake8 --max-complexity 15 hyper test +commands = flake8 hyper test