From ead5da19930ad848d55a5369465961af9f5c0c1a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 7 Jun 2023 13:36:35 +0100 Subject: [PATCH 01/18] Avoid test dependency on freezegun and python-dateutil (#3056) Co-authored-by: Quentin Pradet --- dev-requirements.txt | 1 - pyproject.toml | 2 -- test/test_retry.py | 11 +++++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 10b2f2baf8..071bd1fb14 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,4 @@ coverage==7.0.4 -freezegun==1.2.2 tornado==6.2 PySocks==1.7.1 pytest==7.2.0 diff --git a/pyproject.toml b/pyproject.toml index c979818eea..e916ea35e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,8 +102,6 @@ filterwarnings = [ # https://github.com/pytest-dev/pytest/issues/10977 '''default:ast\.(Num|NameConstant|Str) is deprecated and will be removed in Python 3\.14; use ast\.Constant instead:DeprecationWarning:_pytest''', '''default:Attribute s is deprecated and will be removed in Python 3\.14; use value instead:DeprecationWarning:_pytest''', - # https://github.com/dateutil/dateutil/issues/1284 - '''default:datetime\.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version\.*:DeprecationWarning:dateutil''', ] [tool.isort] diff --git a/test/test_retry.py b/test/test_retry.py index 4163a06081..4fb2bba14c 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -1,9 +1,9 @@ from __future__ import annotations +import datetime from test import DUMMY_POOL from unittest import mock -import freezegun # type: ignore[import] import pytest from urllib3.exceptions import ( @@ -400,9 +400,12 @@ def test_respect_retry_after_header_sleep( ) -> None: retry = Retry(respect_retry_after_header=respect_retry_after_header) - with freezegun.freeze_time("2019-06-03 11:00:00", tz_offset=0), mock.patch( - "time.sleep" - ) as sleep_mock: + with mock.patch( + "time.time", + return_value=datetime.datetime( + 2019, 6, 3, 11, tzinfo=datetime.timezone.utc + ).timestamp(), + ), mock.patch("time.sleep") as sleep_mock: # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES response = HTTPResponse( status=503, headers={"Retry-After": retry_after_header} From 7bdf9757c00e7935343ed6d4526683c1ed560052 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Fri, 9 Jun 2023 06:40:52 +0300 Subject: [PATCH 02/18] Fix OpenGraph metadata in docs --- docs/conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 69f5c4dd09..ba67335880 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,9 +40,8 @@ # Open Graph metadata ogp_title = "urllib3 documentation" -ogp_site_url = "https://urllib3.readthedocs.io" ogp_type = "website" -ogp_image = "https://github.com/urllib3/urllib3/raw/main/docs/_static/banner_github.svg" +ogp_social_cards = {"image": "images/logo.png", "line_color": "#F09837"} ogp_description = "urllib3 is a user-friendly HTTP client library for Python." # Test code blocks only when explicitly specified @@ -73,6 +72,9 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = "friendly" +# The base URL with a proper language and version. +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "/") + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "furo" From 0b657b0e9e6e95db4f73373c497c2c654522a2e8 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 12 Jun 2023 03:02:22 -0700 Subject: [PATCH 03/18] Fix JSON example in user guide (#3067) --- docs/user-guide.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 133579fc11..5cf9f50d8a 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -319,28 +319,27 @@ dictionary in the ``fields`` argument provided to JSON ~~~~ -You can send a JSON request by specifying the data as ``json`` argument, -urllib3 automatically encodes data using ``json`` module with ``UTF-8`` -encoding. Also by default ``"Content-Type"`` in headers is set to -``"application/json"`` if not specified when calling -:meth:`~urllib3.PoolManager.request`: +To send JSON in the body of a request, provide the data in the ``json`` argument to +:meth:`~urllib3.PoolManager.request` and urllib3 will automatically encode the data +using the ``json`` module with ``UTF-8`` encoding. +In addition, when ``json`` is provided, the ``"Content-Type"`` in headers is set to +``"application/json"`` if not specified otherwise. .. code-block:: python import urllib3 - data = {"attribute": "value"} - resp = urllib3.request( "POST", "https://httpbin.org/post", - body=data, + json={"attribute": "value"}, headers={"Content-Type": "application/json"} ) print(resp.json()) - # {"attribute": "value"} - + # {'headers': {'Content-Type': 'application/json', ...}, + # 'data': '{"attribute":"value"}', 'json': {'attribute': 'value'}, ...} + Files & Binary Data ~~~~~~~~~~~~~~~~~~~ From 77c8837dfb082a191b0df716c6ba1487ffb82f99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 20:58:22 +0000 Subject: [PATCH 04/18] Bump github/codeql-action from 2.3.0 to 2.13.4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2.3.0 to 2.13.4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/b2c19fb9a2a485599ccf4ed5d65527d94bc57226...cdcdbb579706841c47f7063dda365e292e5cad7a) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d27556450e..1442827def 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,13 +23,13 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Run CodeQL init" - uses: "github/codeql-action/init@b2c19fb9a2a485599ccf4ed5d65527d94bc57226" + uses: "github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a" with: config-file: "./.github/codeql.yml" languages: "python" - name: "Run CodeQL autobuild" - uses: "github/codeql-action/autobuild@b2c19fb9a2a485599ccf4ed5d65527d94bc57226" + uses: "github/codeql-action/autobuild@cdcdbb579706841c47f7063dda365e292e5cad7a" - name: "Run CodeQL analyze" - uses: "github/codeql-action/analyze@b2c19fb9a2a485599ccf4ed5d65527d94bc57226" + uses: "github/codeql-action/analyze@cdcdbb579706841c47f7063dda365e292e5cad7a" From 530999da3cd9ec9d11f50e6a1f0ae9176cccf2b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 20:58:27 +0000 Subject: [PATCH 05/18] Bump slsa-framework/slsa-github-generator from 1.6.0 to 1.7.0 Bumps [slsa-framework/slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/slsa-framework/slsa-github-generator/releases) - [Changelog](https://github.com/slsa-framework/slsa-github-generator/blob/main/CHANGELOG.md) - [Commits](https://github.com/slsa-framework/slsa-github-generator/compare/v1.6.0...v1.7.0) --- updated-dependencies: - dependency-name: slsa-framework/slsa-github-generator dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4e196a2589..1c178fa4c5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,7 +53,7 @@ jobs: actions: read contents: write id-token: write # Needed to access the workflow's OIDC identity. - uses: "slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.6.0" + uses: "slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.7.0" with: base64-subjects: "${{ needs.build.outputs.hashes }}" upload-assets: true From 10ceef1c8944a55b51ad53651998c8aa9caedacb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:57:54 +0000 Subject: [PATCH 06/18] Bump ossf/scorecard-action from 2.1.2 to 2.2.0 Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.1.2 to 2.2.0. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/e38b1902ae4f44df626f11ba0734b14fb91f8f86...08b4669551908b1024bb425080c797723083c031) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/scorecards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 72a345a451..f9f1015d96 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false - name: "Run Scorecard" - uses: "ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86" + uses: "ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031" with: results_file: results.sarif results_format: sarif From 91dc6fb646bcb1624ab69f609810f9e6265e4be2 Mon Sep 17 00:00:00 2001 From: Noah Jenner <64815328+Eutropios@users.noreply.github.com> Date: Thu, 29 Jun 2023 20:02:48 -0700 Subject: [PATCH 07/18] Added BaseHTTPResponse to __all__ --- changelog/3078.feature.rst | 1 + src/urllib3/__init__.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/3078.feature.rst diff --git a/changelog/3078.feature.rst b/changelog/3078.feature.rst new file mode 100644 index 0000000000..aaa208295e --- /dev/null +++ b/changelog/3078.feature.rst @@ -0,0 +1 @@ +Added ``BaseHTTPResponse`` to ``__all__`` in ``__init__.py`` diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 7ddfad2006..32c1f0025f 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -81,6 +81,7 @@ "make_headers", "proxy_from_url", "request", + "BaseHTTPResponse", ) logging.getLogger(__name__).addHandler(NullHandler()) From 2ac40569acb464074bdc3f308124d781d6aa0860 Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Fri, 30 Jun 2023 12:54:14 +0330 Subject: [PATCH 08/18] Update mypy to 1.4.1 (#3083) --- mypy-requirements.txt | 2 +- src/urllib3/util/request.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index c886dfa300..a7f2d3070a 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.3.0 +mypy==1.4.1 idna>=2.0.0 cryptography>=1.3.4 tornado>=6.1 diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index def099d745..7d6866f3ad 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -223,7 +223,7 @@ def chunk_readable() -> typing.Iterable[bytes]: nonlocal body, blocksize encode = isinstance(body, io.TextIOBase) while True: - datablock = body.read(blocksize) # type: ignore[union-attr] + datablock = body.read(blocksize) if not datablock: break if encode: From d0ac08d58511c4121138bd46436076409d21acbf Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 12 Jul 2023 07:41:25 +0400 Subject: [PATCH 09/18] Bump gh-action-pypi-publish to v1.8.8 It uses urllib3 2.0.3. --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1c178fa4c5..3749b5ec6a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -82,4 +82,4 @@ jobs: gh release upload ${{ github.ref_name }} dist/* --repo ${{ github.repository }} - name: "Publish dists to PyPI" - uses: "pypa/gh-action-pypi-publish@a56da0b891b3dc519c7ee3284aff1fad93cc8598" + uses: "pypa/gh-action-pypi-publish@f8c70e705ffc13c3b4d1221169b84f12a75d6ca8" # v1.8.8 From 326c4238cbabe85007aa7b1f2e80b880fd01d903 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Sat, 15 Jul 2023 05:29:02 +0300 Subject: [PATCH 10/18] Rely on the standard library for checking hostnames in supported PyPy releases Co-authored-by: Quentin Pradet --- changelog/3087.feature.rst | 1 + src/urllib3/util/ssl_.py | 41 ++++++++++++++++++++------------------ test/test_util.py | 19 +++++++++++------- 3 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 changelog/3087.feature.rst diff --git a/changelog/3087.feature.rst b/changelog/3087.feature.rst new file mode 100644 index 0000000000..c78c9dbc2a --- /dev/null +++ b/changelog/3087.feature.rst @@ -0,0 +1 @@ +Relied on the standard library for checking hostnames in supported PyPy releases. diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index 3314b903ae..7762803267 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -26,39 +26,41 @@ def _is_bpo_43522_fixed( - implementation_name: str, version_info: _TYPE_VERSION_INFO + implementation_name: str, + version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, ) -> bool: - """Return True for CPython 3.8.9+, 3.9.3+ or 3.10+ where setting - SSLContext.hostname_checks_common_name to False works. - - PyPy 7.3.7 doesn't work as it doesn't ship with OpenSSL 1.1.1l+ - so we're waiting for a version of PyPy that works before - allowing this function to return 'True'. + """Return True for CPython 3.8.9+, 3.9.3+ or 3.10+ and PyPy 7.3.8+ where + setting SSLContext.hostname_checks_common_name to False works. Outside of CPython and PyPy we don't know which implementations work or not so we conservatively use our hostname matching as we know that works on all implementations. https://github.com/urllib3/urllib3/issues/2192#issuecomment-821832963 - https://foss.heptapod.net/pypy/pypy/-/issues/3539# + https://foss.heptapod.net/pypy/pypy/-/issues/3539 """ - if implementation_name != "cpython": + if implementation_name == "pypy": + # https://foss.heptapod.net/pypy/pypy/-/issues/3129 + return pypy_version_info >= (7, 3, 8) and version_info >= (3, 8) # type: ignore[operator] + elif implementation_name == "cpython": + major_minor = version_info[:2] + micro = version_info[2] + return ( + (major_minor == (3, 8) and micro >= 9) + or (major_minor == (3, 9) and micro >= 3) + or major_minor >= (3, 10) + ) + else: # Defensive: return False - major_minor = version_info[:2] - micro = version_info[2] - return ( - (major_minor == (3, 8) and micro >= 9) - or (major_minor == (3, 9) and micro >= 3) - or major_minor >= (3, 10) - ) - def _is_has_never_check_common_name_reliable( openssl_version: str, openssl_version_number: int, implementation_name: str, version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, ) -> bool: # As of May 2023, all released versions of LibreSSL fail to reject certificates with # only common names, see https://github.com/urllib3/urllib3/pull/3024 @@ -71,7 +73,7 @@ def _is_has_never_check_common_name_reliable( return is_openssl and ( is_openssl_issue_14579_fixed - or _is_bpo_43522_fixed(implementation_name, version_info) + or _is_bpo_43522_fixed(implementation_name, version_info, pypy_version_info) ) @@ -117,6 +119,7 @@ class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False): OPENSSL_VERSION_NUMBER, sys.implementation.name, sys.version_info, + sys.pypy_version_info if sys.implementation.name == "pypy" else None, # type: ignore[attr-defined] ): HAS_NEVER_CHECK_COMMON_NAME = False @@ -341,7 +344,7 @@ def create_urllib3_context( try: context.hostname_checks_common_name = False - except AttributeError: + except AttributeError: # Defensive: for CPython < 3.8.9 and 3.9.3; for PyPy < 7.3.8 pass # Enable logging of TLS session keys via defacto standard environment variable diff --git a/test/test_util.py b/test/test_util.py index 587016ca75..0c46aa1dd3 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1068,19 +1068,22 @@ def test_ssl_wrap_socket_sni_none_no_warn(self) -> None: warn.assert_not_called() @pytest.mark.parametrize( - "openssl_version, openssl_version_number, implementation_name, version_info, reliable", + "openssl_version, openssl_version_number, implementation_name, version_info, pypy_version_info, reliable", [ # OpenSSL and Python OK -> reliable - ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 3), True), + ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 3), None, True), # Python OK -> reliable - ("OpenSSL 1.1.1", 0x10101000, "cpython", (3, 9, 3), True), - ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 6, 9), False), + ("OpenSSL 1.1.1", 0x10101000, "cpython", (3, 9, 3), None, True), + # PyPy: depends on the version + ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 6, 9), (7, 3, 7), False), + ("OpenSSL 1.1.1", 0x10101000, "pypy", (3, 7, 13), (7, 3, 9), False), + ("OpenSSL 1.1.1", 0x101010CF, "pypy", (3, 8, 12), (7, 3, 8), True), # OpenSSL OK -> reliable - ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 2), True), + ("OpenSSL 1.1.1", 0x101010CF, "cpython", (3, 9, 2), None, True), # not OpenSSSL -> unreliable - ("LibreSSL 2.8.3", 0x101010CF, "cpython", (3, 10, 0), False), + ("LibreSSL 2.8.3", 0x101010CF, "cpython", (3, 10, 0), None, False), # old OpenSSL and old Python, unreliable - ("OpenSSL 1.1.0", 0x10101000, "cpython", (3, 9, 2), False), + ("OpenSSL 1.1.0", 0x10101000, "cpython", (3, 9, 2), None, False), ], ) def test_is_has_never_check_common_name_reliable( @@ -1089,6 +1092,7 @@ def test_is_has_never_check_common_name_reliable( openssl_version_number: int, implementation_name: str, version_info: _TYPE_VERSION_INFO, + pypy_version_info: _TYPE_VERSION_INFO | None, reliable: bool, ) -> None: assert ( @@ -1097,6 +1101,7 @@ def test_is_has_never_check_common_name_reliable( openssl_version_number, implementation_name, version_info, + pypy_version_info, ) == reliable ) From 9aa0d4f6cb0d4b34e20fafed4481ac7d785d4969 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jul 2023 11:46:11 +0000 Subject: [PATCH 11/18] Bump cryptography from 39.0.1 to 41.0.0 (#3057) * Bump cryptography from 39.0.1 to 41.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.1 to 41.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.1...41.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Fix CI for PyPy 3.7 --------- Signed-off-by: dependabot[bot] Co-authored-by: Illia Volochii --- dev-requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 071bd1fb14..7d70febfe0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,10 @@ PySocks==1.7.1 pytest==7.2.0 pytest-timeout==2.1.0 trustme==0.9.0 -cryptography==39.0.1 +# We have to install at most cryptography 39.0.2 for PyPy<7.3.10 +# versions of Python 3.7, 3.8, and 3.9. +cryptography==39.0.2;implementation_name=="pypy" and implementation_version<"7.3.10" +cryptography==41.0.0;implementation_name!="pypy" or implementation_version>="7.3.10" backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==21.9.0 pytest-memray==1.4.0;python_version>="3.8" and python_version<"3.12" and sys_platform!="win32" and implementation_name=="cpython" From 05b21ca5d29764aae60c72e4e3bfceead0f70f95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jul 2023 11:46:56 +0000 Subject: [PATCH 12/18] Bump cryptography from 41.0.0 to 41.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.0 to 41.0.2. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.0...41.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7d70febfe0..3a1aae249f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,7 +7,7 @@ trustme==0.9.0 # We have to install at most cryptography 39.0.2 for PyPy<7.3.10 # versions of Python 3.7, 3.8, and 3.9. cryptography==39.0.2;implementation_name=="pypy" and implementation_version<"7.3.10" -cryptography==41.0.0;implementation_name!="pypy" or implementation_version>="7.3.10" +cryptography==41.0.2;implementation_name!="pypy" or implementation_version>="7.3.10" backports.zoneinfo==0.2.1;python_version<"3.9" towncrier==21.9.0 pytest-memray==1.4.0;python_version>="3.8" and python_version<"3.12" and sys_platform!="win32" and implementation_name=="cpython" From 609c5464cc9d2673549b651b713e07424517fa85 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Mon, 17 Jul 2023 13:38:02 -0500 Subject: [PATCH 13/18] Add support for union operators to `HTTPHeaderDict` (#2943) --- changelog/2254.feature.rst | 3 +++ src/urllib3/_collections.py | 29 +++++++++++++++++++++++ test/test_collections.py | 29 +++++++++++++++++++++++ test/with_dummyserver/test_poolmanager.py | 23 ++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 changelog/2254.feature.rst diff --git a/changelog/2254.feature.rst b/changelog/2254.feature.rst new file mode 100644 index 0000000000..3f189ff5f1 --- /dev/null +++ b/changelog/2254.feature.rst @@ -0,0 +1,3 @@ +Added support for union operators to ``HTTPHeaderDict`` + +This allows a simple way to merge headers provided in a request with the ``PoolManager`` headers, rather than replace them. diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index 3e43afbc90..7f9dca7fa8 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -432,3 +432,32 @@ def _has_value_for_header(self, header_name: str, potential_value: str) -> bool: if header_name in self: return potential_value in self._container[header_name.lower()][1:] return False + + def __ior__(self, other: object) -> HTTPHeaderDict: + # Supports extending a header dict in-place using operator |= + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + self.extend(maybe_constructable) + return self + + def __or__(self, other: object) -> HTTPHeaderDict: + # Supports merging header dicts using operator | + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = self.copy() + result.extend(maybe_constructable) + return result + + def __ror__(self, other: object) -> HTTPHeaderDict: + # Supports merging header dicts using operator | when other is on left side + # combining items with add instead of __setitem__ + maybe_constructable = ensure_can_construct_http_header_dict(other) + if maybe_constructable is None: + return NotImplemented + result = type(self)(maybe_constructable) + result.extend(self) + return result diff --git a/test/test_collections.py b/test/test_collections.py index 56cdea865d..8d0c1ce26f 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -423,3 +423,32 @@ def test_dunder_contains(self, d: HTTPHeaderDict) -> None: d._container[marker] = ["some", "strings"] # type: ignore[index] assert marker not in d assert marker in d._container + + def test_union(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + result = d | to_merge + assert result == HTTPHeaderDict({"Cookie": "foo, bar, tim-tam"}) + assert to_merge == {"Cookie": "tim-tam"} + assert d == HTTPHeaderDict({"Cookie": "foo, bar"}) + + def test_union_rhs(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + result = to_merge | d + assert result == HTTPHeaderDict({"Cookie": "tim-tam, foo, bar"}) + assert to_merge == {"Cookie": "tim-tam"} + assert d == HTTPHeaderDict({"Cookie": "foo, bar"}) + + def test_inplace_union(self, d: HTTPHeaderDict) -> None: + to_merge = {"Cookie": "tim-tam"} + d |= to_merge + assert d == HTTPHeaderDict({"Cookie": "foo, bar, tim-tam"}) + + def test_union_with_unsupported_type(self, d: HTTPHeaderDict) -> None: + with pytest.raises(TypeError, match="unsupported operand type.*'int'"): + d | 42 + with pytest.raises(TypeError, match="unsupported operand type.*'float'"): + 3.14 | d + + def test_inplace_union_with_unsupported_type(self, d: HTTPHeaderDict) -> None: + with pytest.raises(TypeError, match="unsupported operand type.*'NoneType'"): + d |= None diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index c4f1947037..e4c550714c 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -1,6 +1,7 @@ from __future__ import annotations import gzip +import typing from test import LONG_TIMEOUT from unittest import mock @@ -379,6 +380,28 @@ def test_headers_http_header_dict(self) -> None: ["Extra", "extra"], ] + def test_merge_headers_with_pool_manager_headers(self) -> None: + headers = HTTPHeaderDict() + headers.add("Cookie", "choc-chip") + headers.add("Cookie", "oatmeal-raisin") + orig = headers.copy() + added_headers = {"Cookie": "tim-tam"} + + with PoolManager(headers=headers) as http: + r = http.request( + "GET", + f"{self.base_url}/multi_headers", + headers=typing.cast(HTTPHeaderDict, http.headers) | added_headers, + ) + returned_headers = r.json()["headers"] + assert returned_headers[-3:] == [ + ["Cookie", "choc-chip"], + ["Cookie", "oatmeal-raisin"], + ["Cookie", "tim-tam"], + ] + # make sure the pool headers weren't modified + assert http.headers == orig + def test_headers_http_multi_header_multipart(self) -> None: headers = HTTPHeaderDict() headers.add("Multi", "1") From a1c184b298823f13dd34c783d56ea538848282f1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 17 Jul 2023 12:00:47 -0700 Subject: [PATCH 14/18] Remove warnings filters fixed in pytest 7.4.0 (#3086) Co-authored-by: Quentin Pradet --- dev-requirements.txt | 2 +- pyproject.toml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3a1aae249f..c4a4d089d8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ coverage==7.0.4 tornado==6.2 PySocks==1.7.1 -pytest==7.2.0 +pytest==7.4.0 pytest-timeout==2.1.0 trustme==0.9.0 # We have to install at most cryptography 39.0.2 for PyPy<7.3.10 diff --git a/pyproject.toml b/pyproject.toml index e916ea35e1..1e5f448cd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,9 +99,6 @@ filterwarnings = [ '''default:ssl\.PROTOCOL_TLSv1_2 is deprecated:DeprecationWarning''', '''default:unclosed .*:ResourceWarning''', '''default:ssl NPN is deprecated, use ALPN instead:DeprecationWarning''', - # https://github.com/pytest-dev/pytest/issues/10977 - '''default:ast\.(Num|NameConstant|Str) is deprecated and will be removed in Python 3\.14; use ast\.Constant instead:DeprecationWarning:_pytest''', - '''default:Attribute s is deprecated and will be removed in Python 3\.14; use value instead:DeprecationWarning:_pytest''', ] [tool.isort] From c056eb3df6aae4d1dff0365baded46235d413520 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 20:36:15 +0000 Subject: [PATCH 15/18] Bump actions/setup-python from 4.6.0 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/57ded4d7d5e986d7296eab16560982c6dd7c923b...61a6322f88396a6271a6ee3565807d608ecaddd1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- .github/workflows/integration.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/publish.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 500f39f3ca..8006a55661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: "3.x" cache: "pip" @@ -82,7 +82,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python ${{ matrix.python-version }}" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: ${{ matrix.python-version }} @@ -112,7 +112,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: "3.x" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 376391bee8..2a05744160 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -18,7 +18,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: "3.x" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a991884894..feccaad4c7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: "3.x" cache: pip diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3749b5ec6a..e33ff66627 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: uses: "actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3" - name: "Setup Python" - uses: "actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b" + uses: "actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1" with: python-version: "3.x" From 0a375d19243efb08c9d8a0f3356701ca11ef9791 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 19 Jul 2023 02:55:27 +0530 Subject: [PATCH 16/18] Raise `http.client.connect` audit events in `HTTPConnection` (#2859) --- changelog/2757.bugfix.rst | 1 + src/urllib3/connection.py | 7 +++++++ test/with_dummyserver/test_connection.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 changelog/2757.bugfix.rst diff --git a/changelog/2757.bugfix.rst b/changelog/2757.bugfix.rst new file mode 100644 index 0000000000..40ea75f657 --- /dev/null +++ b/changelog/2757.bugfix.rst @@ -0,0 +1 @@ +Fixed ``urllib3.connection.HTTPConnection`` to raise the ``http.client.connect`` audit event to have the same behavior as the standard library HTTP client diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 50f5e616c2..10109fa917 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -5,6 +5,7 @@ import os import re import socket +import sys import typing import warnings from http.client import HTTPConnection as _HTTPConnection @@ -76,6 +77,8 @@ class BaseSSLError(BaseException): # type: ignore[no-redef] _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") +_HAS_SYS_AUDIT = hasattr(sys, "audit") + class HTTPConnection(_HTTPConnection): """ @@ -216,6 +219,10 @@ def _new_conn(self) -> socket.socket: self, f"Failed to establish a new connection: {e}" ) from e + # Audit hooks are only available in Python 3.8+ + if _HAS_SYS_AUDIT: + sys.audit("http.client.connect", self, self.host, self.port) + return sock def set_tunnel( diff --git a/test/with_dummyserver/test_connection.py b/test/with_dummyserver/test_connection.py index d06a7551b5..2442c8ad2d 100644 --- a/test/with_dummyserver/test_connection.py +++ b/test/with_dummyserver/test_connection.py @@ -1,7 +1,9 @@ from __future__ import annotations +import sys import typing from http.client import ResponseNotReady +from unittest import mock import pytest @@ -33,6 +35,19 @@ def test_returns_urllib3_HTTPResponse(pool: HTTPConnectionPool) -> None: assert isinstance(response, HTTPResponse) +@pytest.mark.skipif(not hasattr(sys, "audit"), reason="requires python 3.8+") +@mock.patch("urllib3.connection.sys.audit") +def test_audit_event(audit_mock: mock.Mock, pool: HTTPConnectionPool) -> None: + conn = pool._get_conn() + conn.request("GET", "/") + audit_mock.assert_any_call("http.client.connect", conn, conn.host, conn.port) + # Ensure the event is raised only once. + connect_events = [ + call for call in audit_mock.mock_calls if call.args[0] == "http.client.connect" + ] + assert len(connect_events) == 1 + + def test_does_not_release_conn(pool: HTTPConnectionPool) -> None: conn = pool._get_conn() From d40d146765a7f7a899427c8add54de67b4b6599a Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Wed, 19 Jul 2023 17:01:45 +0300 Subject: [PATCH 17/18] Add Illia to CODEOWNERS --- .github/CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 64afa535c7..03a1992df7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,8 @@ # Restrict all files related to deploying to # require lead maintainer approval. -.github/workflows/ @sethmlarson @pquentin @shazow -.github/CODEOWNERS @sethmlarson @pquentin @shazow -src/urllib3/_version.py @sethmlarson @pquentin @shazow -pyproject.toml @sethmlarson @pquentin @shazow -ci/ @sethmlarson @pquentin @shazow +.github/workflows/ @sethmlarson @pquentin @shazow @illia-v +.github/CODEOWNERS @sethmlarson @pquentin @shazow @illia-v +src/urllib3/_version.py @sethmlarson @pquentin @shazow @illia-v +pyproject.toml @sethmlarson @pquentin @shazow @illia-v +ci/ @sethmlarson @pquentin @shazow @illia-v From c9fa144545eedb5dc4a2cc3f255e95602a1d7db0 Mon Sep 17 00:00:00 2001 From: Noah Jenner <64815328+Eutropios@users.noreply.github.com> Date: Wed, 19 Jul 2023 07:46:02 -0700 Subject: [PATCH 18/18] Release version 2.0.4 (#3084) Co-authored-by: Illia Volochii --- CHANGES.rst | 9 +++++++++ changelog/2254.feature.rst | 3 --- changelog/2757.bugfix.rst | 1 - changelog/3078.feature.rst | 1 - changelog/3087.feature.rst | 1 - src/urllib3/_version.py | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 changelog/2254.feature.rst delete mode 100644 changelog/2757.bugfix.rst delete mode 100644 changelog/3078.feature.rst delete mode 100644 changelog/3087.feature.rst diff --git a/CHANGES.rst b/CHANGES.rst index 1668b38304..3b311c2dd7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +2.0.4 (2023-07-19) +================== + +- Added support for union operators to ``HTTPHeaderDict`` (`#2254 `__) +- Added ``BaseHTTPResponse`` to ``urllib3.__all__`` (`#3078 `__) +- Fixed ``urllib3.connection.HTTPConnection`` to raise the ``http.client.connect`` audit event to have the same behavior as the standard library HTTP client (`#2757 `__) +- Relied on the standard library for checking hostnames in supported PyPy releases (`#3087 `__) + + 2.0.3 (2023-06-07) ================== diff --git a/changelog/2254.feature.rst b/changelog/2254.feature.rst deleted file mode 100644 index 3f189ff5f1..0000000000 --- a/changelog/2254.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added support for union operators to ``HTTPHeaderDict`` - -This allows a simple way to merge headers provided in a request with the ``PoolManager`` headers, rather than replace them. diff --git a/changelog/2757.bugfix.rst b/changelog/2757.bugfix.rst deleted file mode 100644 index 40ea75f657..0000000000 --- a/changelog/2757.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``urllib3.connection.HTTPConnection`` to raise the ``http.client.connect`` audit event to have the same behavior as the standard library HTTP client diff --git a/changelog/3078.feature.rst b/changelog/3078.feature.rst deleted file mode 100644 index aaa208295e..0000000000 --- a/changelog/3078.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added ``BaseHTTPResponse`` to ``__all__`` in ``__init__.py`` diff --git a/changelog/3087.feature.rst b/changelog/3087.feature.rst deleted file mode 100644 index c78c9dbc2a..0000000000 --- a/changelog/3087.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Relied on the standard library for checking hostnames in supported PyPy releases. diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py index e85dd3afd9..782af43345 100644 --- a/src/urllib3/_version.py +++ b/src/urllib3/_version.py @@ -1,4 +1,4 @@ # This file is protected via CODEOWNERS from __future__ import annotations -__version__ = "2.0.3" +__version__ = "2.0.4"