From 3d49ca04a23b36a3752d8609a20a298762531246 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 20 May 2024 12:03:50 -0400 Subject: [PATCH 1/5] Use truststore by default --- docs/html/topics/https-certificates.md | 45 ++++++++------------------ news/11647.feature.rst | 4 +++ src/pip/_internal/cli/cmdoptions.py | 3 +- src/pip/_internal/cli/index_command.py | 30 +++++++---------- tests/functional/test_truststore.py | 23 +++---------- tests/lib/certs.py | 15 ++++++++- 6 files changed, 50 insertions(+), 70 deletions(-) create mode 100644 news/11647.feature.rst diff --git a/docs/html/topics/https-certificates.md b/docs/html/topics/https-certificates.md index 0cf88b4b644..e769e0dfa4d 100644 --- a/docs/html/topics/https-certificates.md +++ b/docs/html/topics/https-certificates.md @@ -8,8 +8,8 @@ By default, pip will perform SSL certificate verification for network connections it makes over HTTPS. These serve to prevent man-in-the-middle -attacks against package downloads. This does not use the system certificate -store but, instead, uses a bundled CA certificate store from {pypi}`certifi`. +attacks against package downloads. Pip by default uses a bundled CA certificate +store from {pypi}`certifi`. ## Using a specific certificate store @@ -20,43 +20,24 @@ variables. ## Using system certificate stores -```{versionadded} 22.2 -Experimental support, behind `--use-feature=truststore`. -As with any other CLI option, this can be enabled globally via config or environment variables. -``` - -It is possible to use the system trust store, instead of the bundled certifi -certificates for verifying HTTPS certificates. This approach will typically -support corporate proxy certificates without additional configuration. - -In order to use system trust stores, you need to use Python 3.10 or newer. - - ```{pip-cli} - $ python -m pip install SomePackage --use-feature=truststore - [...] - Successfully installed SomePackage - ``` - -### When to use +```{versionadded} 24.2 -You should try using system trust stores when there is a custom certificate -chain configured for your system that pip isn't aware of. Typically, this -situation will manifest with an `SSLCertVerificationError` with the message -"certificate verify failed: unable to get local issuer certificate": - -```{pip-cli} -$ pip install -U SomePackage -[...] - SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (\_ssl.c:997)'))) - skipping ``` -This error means that OpenSSL wasn't able to find a trust anchor to verify the -chain against. Using system trust stores instead of certifi will likely solve -this issue. +If Python 3.10 or later is being used then by default +system certificates are used in addition to certifi to verify HTTPS connections. +This functionality is provided through the {pypi}`truststore` package. If you encounter a TLS/SSL error when using the `truststore` feature you should open an issue on the [truststore GitHub issue tracker] instead of pip's issue tracker. The maintainers of truststore will help diagnose and fix the issue. +To opt-out of using system certificates you can pass the `--use-deprecated=legacy-certs` +flag to pip. + +```{warning} +If Python 3.9 or earlier is in use then only certifi is used to verify HTTPS connections. +``` + [truststore github issue tracker]: https://github.com/sethmlarson/truststore/issues diff --git a/news/11647.feature.rst b/news/11647.feature.rst new file mode 100644 index 00000000000..26d04d49165 --- /dev/null +++ b/news/11647.feature.rst @@ -0,0 +1,4 @@ +Changed pip to use system certificates and certifi to verify HTTPS connections. +This change only affects Python 3.10 or later, Python 3.9 and earlier only use certifi. + +To revert to previous behavior pass the flag ``--use-deprecated=legacy-certs``. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index a47f8a3f46a..0b7cff77bdd 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -996,6 +996,7 @@ def check_list_path_option(options: Values) -> None: # Features that are now always on. A warning is printed if they are used. ALWAYS_ENABLED_FEATURES = [ + "truststore", # always on since 24.2 "no-binary-enable-wheel-cache", # always on since 23.1 ] @@ -1008,7 +1009,6 @@ def check_list_path_option(options: Values) -> None: default=[], choices=[ "fast-deps", - "truststore", ] + ALWAYS_ENABLED_FEATURES, help="Enable new functionality, that may be backward incompatible.", @@ -1023,6 +1023,7 @@ def check_list_path_option(options: Values) -> None: default=[], choices=[ "legacy-resolver", + "legacy-certs", ], help=("Enable deprecated functionality, that will be removed in the future."), ) diff --git a/src/pip/_internal/cli/index_command.py b/src/pip/_internal/cli/index_command.py index 4ff7b2c3a5d..d826df1ac40 100644 --- a/src/pip/_internal/cli/index_command.py +++ b/src/pip/_internal/cli/index_command.py @@ -12,9 +12,10 @@ from optparse import Values from typing import TYPE_CHECKING, List, Optional +from pip._vendor import certifi + from pip._internal.cli.base_command import Command from pip._internal.cli.command_context import CommandContextMixIn -from pip._internal.exceptions import CommandError if TYPE_CHECKING: from ssl import SSLContext @@ -26,7 +27,8 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: if sys.version_info < (3, 10): - raise CommandError("The truststore feature is only available for Python 3.10+") + logger.warning("Disabling truststore because Python version isn't 3.10+") + return None try: import ssl @@ -36,10 +38,13 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: try: from pip._vendor import truststore - except ImportError as e: - raise CommandError(f"The truststore feature is unavailable: {e}") + except ImportError: + logger.warning("Disabling truststore because platform isn't supported") + return None - return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(certifi.where()) + return ctx class SessionCommandMixin(CommandContextMixIn): @@ -80,20 +85,14 @@ def _build_session( options: Values, retries: Optional[int] = None, timeout: Optional[int] = None, - fallback_to_certifi: bool = False, ) -> "PipSession": from pip._internal.network.session import PipSession cache_dir = options.cache_dir assert not cache_dir or os.path.isabs(cache_dir) - if "truststore" in options.features_enabled: - try: - ssl_context = _create_truststore_ssl_context() - except Exception: - if not fallback_to_certifi: - raise - ssl_context = None + if "legacy-certs" not in options.deprecated_features_enabled: + ssl_context = _create_truststore_ssl_context() else: ssl_context = None @@ -162,11 +161,6 @@ def handle_pip_version_check(self, options: Values) -> None: options, retries=0, timeout=min(5, options.timeout), - # This is set to ensure the function does not fail when truststore is - # specified in use-feature but cannot be loaded. This usually raises a - # CommandError and shows a nice user-facing error, but this function is not - # called in that try-except block. - fallback_to_certifi=True, ) with session: _pip_self_version_check(session, options) diff --git a/tests/functional/test_truststore.py b/tests/functional/test_truststore.py index cc90343b52d..c534ddb954d 100644 --- a/tests/functional/test_truststore.py +++ b/tests/functional/test_truststore.py @@ -1,4 +1,3 @@ -import sys from typing import Any, Callable import pytest @@ -9,25 +8,13 @@ @pytest.fixture() -def pip(script: PipTestEnvironment) -> PipRunner: +def pip_no_truststore(script: PipTestEnvironment) -> PipRunner: def pip(*args: str, **kwargs: Any) -> TestPipResult: - return script.pip(*args, "--use-feature=truststore", **kwargs) + return script.pip(*args, "--use-deprecated=legacy-certs", **kwargs) return pip -@pytest.mark.skipif(sys.version_info >= (3, 10), reason="3.10 can run truststore") -def test_truststore_error_on_old_python(pip: PipRunner) -> None: - result = pip( - "install", - "--no-index", - "does-not-matter", - expect_error=True, - ) - assert "The truststore feature is only available for Python 3.10+" in result.stderr - - -@pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore") @pytest.mark.network @pytest.mark.parametrize( "package", @@ -37,10 +24,10 @@ def test_truststore_error_on_old_python(pip: PipRunner) -> None: ], ids=["PyPI", "GitHub"], ) -def test_trustore_can_install( +def test_no_truststore_can_install( script: PipTestEnvironment, - pip: PipRunner, + pip_no_truststore: PipRunner, package: str, ) -> None: - result = pip("install", package) + result = pip_no_truststore("install", package) assert "Successfully installed" in result.stdout diff --git a/tests/lib/certs.py b/tests/lib/certs.py index 9e6542d2d57..6f899acfe48 100644 --- a/tests/lib/certs.py +++ b/tests/lib/certs.py @@ -5,7 +5,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]: @@ -25,10 +25,23 @@ def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]: .serial_number(x509.random_serial_number()) .not_valid_before(datetime.now(timezone.utc)) .not_valid_after(datetime.now(timezone.utc) + timedelta(days=10)) + .add_extension( + x509.BasicConstraints(ca=True, path_length=9), + critical=True, + ) .add_extension( x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False, ) + .add_extension( + x509.ExtendedKeyUsage( + [ + ExtendedKeyUsageOID.CLIENT_AUTH, + ExtendedKeyUsageOID.SERVER_AUTH, + ] + ), + critical=True, + ) .sign(key, hashes.SHA256(), default_backend()) ) return cert, key From 540b66aa7eb5440018355fbb4ffc61f237f65717 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Mon, 20 May 2024 15:17:51 -0400 Subject: [PATCH 2/5] Clarify the new default certificate behavior for pip --- docs/html/topics/https-certificates.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/html/topics/https-certificates.md b/docs/html/topics/https-certificates.md index e769e0dfa4d..bad532a2632 100644 --- a/docs/html/topics/https-certificates.md +++ b/docs/html/topics/https-certificates.md @@ -8,8 +8,7 @@ By default, pip will perform SSL certificate verification for network connections it makes over HTTPS. These serve to prevent man-in-the-middle -attacks against package downloads. Pip by default uses a bundled CA certificate -store from {pypi}`certifi`. +attacks against package downloads. ## Using a specific certificate store @@ -37,6 +36,10 @@ flag to pip. ```{warning} If Python 3.9 or earlier is in use then only certifi is used to verify HTTPS connections. + +The system certificate store won't be used in this case, so some situations like proxies +with their own certificates may not work. Upgrading to at least Python 3.10 or later is +the recommended method to resolve this issue. ``` [truststore github issue tracker]: From bec252f6b0458ae83b2878cbfd47839d958624f8 Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Wed, 10 Jul 2024 00:45:49 +0100 Subject: [PATCH 3/5] Use `debug` level for logging truststore disablement on <3.10 --- src/pip/_internal/cli/index_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/index_command.py b/src/pip/_internal/cli/index_command.py index d826df1ac40..9991326f36b 100644 --- a/src/pip/_internal/cli/index_command.py +++ b/src/pip/_internal/cli/index_command.py @@ -27,7 +27,7 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]: if sys.version_info < (3, 10): - logger.warning("Disabling truststore because Python version isn't 3.10+") + logger.debug("Disabling truststore because Python version isn't 3.10+") return None try: From 71140405397e2e5dd49d2bc5f626200ab58fe74f Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 10 Jul 2024 08:37:01 -0500 Subject: [PATCH 4/5] Update docs to mention previous behavior --- docs/html/topics/https-certificates.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/html/topics/https-certificates.md b/docs/html/topics/https-certificates.md index bad532a2632..05ef35e48f0 100644 --- a/docs/html/topics/https-certificates.md +++ b/docs/html/topics/https-certificates.md @@ -23,7 +23,12 @@ variables. ``` -If Python 3.10 or later is being used then by default +```{note} +Versions of pip prior to v24.2 did not use system certificates by default. +To use system certificates with pip v22.2 or later, you must opt-in using the `--use-feature=truststore` CLI flag. +``` + +On Python 3.10 or later, by default system certificates are used in addition to certifi to verify HTTPS connections. This functionality is provided through the {pypi}`truststore` package. @@ -35,7 +40,7 @@ To opt-out of using system certificates you can pass the `--use-deprecated=legac flag to pip. ```{warning} -If Python 3.9 or earlier is in use then only certifi is used to verify HTTPS connections. +On Python 3.9 or earlier, by default only certifi is used to verify HTTPS connections. The system certificate store won't be used in this case, so some situations like proxies with their own certificates may not work. Upgrading to at least Python 3.10 or later is From 69874c79aa477fe9f4d7c5ece3e7668069daf5ce Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 10 Jul 2024 12:13:51 -0500 Subject: [PATCH 5/5] Add wording suggestion from review Co-authored-by: Richard Si --- docs/html/topics/https-certificates.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/html/topics/https-certificates.md b/docs/html/topics/https-certificates.md index 05ef35e48f0..ff640575e6c 100644 --- a/docs/html/topics/https-certificates.md +++ b/docs/html/topics/https-certificates.md @@ -40,7 +40,8 @@ To opt-out of using system certificates you can pass the `--use-deprecated=legac flag to pip. ```{warning} -On Python 3.9 or earlier, by default only certifi is used to verify HTTPS connections. +On Python 3.9 or earlier, only certifi is used to verify HTTPS connections as +`truststore` requires Python 3.10 or higher to function. The system certificate store won't be used in this case, so some situations like proxies with their own certificates may not work. Upgrading to at least Python 3.10 or later is