Reconcile computation of isolated build environment paths (#11740)

Use the same code to determine isolated environment paths at
dependency install time and at environment setup time. We do not care
about the exact paths but the paths needs to be consistent at package
installation time and environment setup.

This should fix issues observed on platforms that customize the
installation schemes, such as Debian and Homebrew, where dependency
installation and isolated build environment setup resolved to
different paths.

Author: dnicolodi

news/11740.bugfix.rst

@@ -0,0 +1,3 @@
+Improve handling of isolated build environments on platforms that
+customize the Python's installation schemes, such as Debian and
+Homebrew.

src/pip/_internal/build_env.py

@@ -9,20 +9,15 @@
 import textwrap
 from collections import OrderedDict
 from types import TracebackType
-from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type
+from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple, Type, Union
 
 from pip._vendor.certifi import where
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
-from pip._internal.locations import (
-    get_isolated_environment_bin_path,
-    get_isolated_environment_lib_paths,
-    get_platlib,
-    get_purelib,
-)
+from pip._internal.locations import get_platlib, get_purelib, get_scheme
 from pip._internal.metadata import get_default_environment, get_environment
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
@@ -33,12 +28,17 @@
 logger = logging.getLogger(__name__)
 
 
+def _dedup(a: str, b: str) -> Union[Tuple[str], Tuple[str, str]]:
+    return (a, b) if a != b else (a,)
+
+
 class _Prefix:
     def __init__(self, path: str) -> None:
         self.path = path
         self.setup = False
-        self.bin_dir = get_isolated_environment_bin_path(path)
-        self.lib_dirs = get_isolated_environment_lib_paths(path)
+        scheme = get_scheme("", prefix=path)
+        self.bin_dir = scheme.scripts
+        self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)
 
 
 def get_runnable_pip() -> str:

src/pip/_internal/locations/__init__.py

@@ -4,7 +4,7 @@
 import pathlib
 import sys
 import sysconfig
-from typing import Any, Dict, Generator, List, Optional, Tuple
+from typing import Any, Dict, Generator, Optional, Tuple
 
 from pip._internal.models.scheme import SCHEME_KEYS, Scheme
 from pip._internal.utils.compat import WINDOWS
@@ -25,8 +25,6 @@
     "USER_CACHE_DIR",
     "get_bin_prefix",
     "get_bin_user",
-    "get_isolated_environment_bin_path",
-    "get_isolated_environment_lib_paths",
     "get_major_minor_version",
     "get_platlib",
     "get_purelib",
@@ -467,67 +465,3 @@ def get_platlib() -> str:
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
         _log_context()
     return old
-
-
-def _deduplicated(v1: str, v2: str) -> List[str]:
-    """Deduplicate values from a list."""
-    if v1 == v2:
-        return [v1]
-    return [v1, v2]
-
-
-def _looks_like_apple_library(path: str) -> bool:
-    """Apple patches sysconfig to *always* look under */Library/Python*."""
-    if sys.platform[:6] != "darwin":
-        return False
-    return path == f"/Library/Python/{get_major_minor_version()}/site-packages"
-
-
-def get_isolated_environment_lib_paths(prefix: str) -> List[str]:
-    """Return the lib locations under ``prefix``."""
-    new_pure, new_plat = _sysconfig.get_isolated_environment_lib_paths(prefix)
-    if _USE_SYSCONFIG:
-        return _deduplicated(new_pure, new_plat)
-
-    old_pure, old_plat = _distutils.get_isolated_environment_lib_paths(prefix)
-    old_lib_paths = _deduplicated(old_pure, old_plat)
-
-    # Apple's Python (shipped with Xcode and Command Line Tools) hard-code
-    # platlib and purelib to '/Library/Python/X.Y/site-packages'. This will
-    # cause serious build isolation bugs when Apple starts shipping 3.10 because
-    # pip will install build backends to the wrong location. This tells users
-    # who is at fault so Apple may notice it and fix the issue in time.
-    if all(_looks_like_apple_library(p) for p in old_lib_paths):
-        deprecated(
-            reason=(
-                "Python distributed by Apple's Command Line Tools incorrectly "
-                "patches sysconfig to always point to '/Library/Python'. This "
-                "will cause build isolation to operate incorrectly on Python "
-                "3.10 or later. Please help report this to Apple so they can "
-                "fix this. https://developer.apple.com/bug-reporting/"
-            ),
-            replacement=None,
-            gone_in=None,
-        )
-        return old_lib_paths
-
-    warned = [
-        _warn_if_mismatch(
-            pathlib.Path(old_pure),
-            pathlib.Path(new_pure),
-            key="prefixed-purelib",
-        ),
-        _warn_if_mismatch(
-            pathlib.Path(old_plat),
-            pathlib.Path(new_plat),
-            key="prefixed-platlib",
-        ),
-    ]
-    if any(warned):
-        _log_context(prefix=prefix)
-
-    return old_lib_paths
-
-
-def get_isolated_environment_bin_path(prefix: str) -> str:
-    return _sysconfig.get_isolated_environment_paths(prefix)["scripts"]

src/pip/_internal/locations/_distutils.py

@@ -21,7 +21,7 @@
 from distutils.command.install import SCHEME_KEYS
 from distutils.command.install import install as distutils_install_command
 from distutils.sysconfig import get_python_lib
-from typing import Dict, List, Optional, Tuple, Union, cast
+from typing import Dict, List, Optional, Union, cast
 
 from pip._internal.models.scheme import Scheme
 from pip._internal.utils.compat import WINDOWS
@@ -171,10 +171,3 @@ def get_purelib() -> str:
 
 def get_platlib() -> str:
     return get_python_lib(plat_specific=True)
-
-
-def get_isolated_environment_lib_paths(prefix: str) -> Tuple[str, str]:
-    return (
-        get_python_lib(plat_specific=False, prefix=prefix),
-        get_python_lib(plat_specific=True, prefix=prefix),
-    )

src/pip/_internal/locations/_sysconfig.py

@@ -211,15 +211,3 @@ def get_purelib() -> str:
 
 def get_platlib() -> str:
     return sysconfig.get_paths()["platlib"]
-
-
-def get_isolated_environment_paths(prefix: str) -> typing.Dict[str, str]:
-    variables = {"base": prefix, "platbase": prefix}
-    if "venv" in sysconfig.get_scheme_names():
-        return sysconfig.get_paths(vars=variables, scheme="venv")
-    return sysconfig.get_paths(vars=variables)
-
-
-def get_isolated_environment_lib_paths(prefix: str) -> typing.Tuple[str, str]:
-    paths = get_isolated_environment_paths(prefix)
-    return (paths["purelib"], paths["platlib"])