529 lines
17 KiB
Python
529 lines
17 KiB
Python
import functools
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import sysconfig
|
|
from typing import Any, Dict, Generator, List, Optional, Tuple
|
|
|
|
from pip._internal.models.scheme import SCHEME_KEYS, Scheme
|
|
from pip._internal.utils.compat import WINDOWS
|
|
from pip._internal.utils.deprecation import deprecated
|
|
from pip._internal.utils.virtualenv import running_under_virtualenv
|
|
|
|
from . import _sysconfig
|
|
from .base import (
|
|
USER_CACHE_DIR,
|
|
get_major_minor_version,
|
|
get_src_prefix,
|
|
is_osx_framework,
|
|
site_packages,
|
|
user_site,
|
|
)
|
|
|
|
__all__ = [
|
|
"USER_CACHE_DIR",
|
|
"get_bin_prefix",
|
|
"get_bin_user",
|
|
"get_major_minor_version",
|
|
"get_platlib",
|
|
"get_prefixed_libs",
|
|
"get_purelib",
|
|
"get_scheme",
|
|
"get_src_prefix",
|
|
"site_packages",
|
|
"user_site",
|
|
]
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
|
|
|
|
_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10)
|
|
|
|
|
|
def _should_use_sysconfig() -> bool:
|
|
"""This function determines the value of _USE_SYSCONFIG.
|
|
|
|
By default, pip uses sysconfig on Python 3.10+.
|
|
But Python distributors can override this decision by setting:
|
|
sysconfig._PIP_USE_SYSCONFIG = True / False
|
|
Rationale in https://github.com/pypa/pip/issues/10647
|
|
|
|
This is a function for testability, but should be constant during any one
|
|
run.
|
|
"""
|
|
return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT))
|
|
|
|
|
|
_USE_SYSCONFIG = _should_use_sysconfig()
|
|
|
|
if not _USE_SYSCONFIG:
|
|
# Import distutils lazily to avoid deprecation warnings,
|
|
# but import it soon enough that it is in memory and available during
|
|
# a pip reinstall.
|
|
from . import _distutils
|
|
|
|
# Be noisy about incompatibilities if this platforms "should" be using
|
|
# sysconfig, but is explicitly opting out and using distutils instead.
|
|
if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
|
|
_MISMATCH_LEVEL = logging.WARNING
|
|
else:
|
|
_MISMATCH_LEVEL = logging.DEBUG
|
|
|
|
|
|
def _looks_like_bpo_44860() -> bool:
|
|
"""The resolution to bpo-44860 will change this incorrect platlib.
|
|
|
|
See <https://bugs.python.org/issue44860>.
|
|
"""
|
|
from distutils.command.install import INSTALL_SCHEMES
|
|
|
|
try:
|
|
unix_user_platlib = INSTALL_SCHEMES["unix_user"]["platlib"]
|
|
except KeyError:
|
|
return False
|
|
return unix_user_platlib == "$usersite"
|
|
|
|
|
|
def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool:
|
|
platlib = scheme["platlib"]
|
|
if "/$platlibdir/" in platlib:
|
|
platlib = platlib.replace("/$platlibdir/", f"/{_PLATLIBDIR}/")
|
|
if "/lib64/" not in platlib:
|
|
return False
|
|
unpatched = platlib.replace("/lib64/", "/lib/")
|
|
return unpatched.replace("$platbase/", "$base/") == scheme["purelib"]
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _looks_like_red_hat_lib() -> bool:
|
|
"""Red Hat patches platlib in unix_prefix and unix_home, but not purelib.
|
|
|
|
This is the only way I can see to tell a Red Hat-patched Python.
|
|
"""
|
|
from distutils.command.install import INSTALL_SCHEMES
|
|
|
|
return all(
|
|
k in INSTALL_SCHEMES
|
|
and _looks_like_red_hat_patched_platlib_purelib(INSTALL_SCHEMES[k])
|
|
for k in ("unix_prefix", "unix_home")
|
|
)
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _looks_like_debian_scheme() -> bool:
|
|
"""Debian adds two additional schemes."""
|
|
from distutils.command.install import INSTALL_SCHEMES
|
|
|
|
return "deb_system" in INSTALL_SCHEMES and "unix_local" in INSTALL_SCHEMES
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _looks_like_red_hat_scheme() -> bool:
|
|
"""Red Hat patches ``sys.prefix`` and ``sys.exec_prefix``.
|
|
|
|
Red Hat's ``00251-change-user-install-location.patch`` changes the install
|
|
command's ``prefix`` and ``exec_prefix`` to append ``"/local"``. This is
|
|
(fortunately?) done quite unconditionally, so we create a default command
|
|
object without any configuration to detect this.
|
|
"""
|
|
from distutils.command.install import install
|
|
from distutils.dist import Distribution
|
|
|
|
cmd: Any = install(Distribution())
|
|
cmd.finalize_options()
|
|
return (
|
|
cmd.exec_prefix == f"{os.path.normpath(sys.exec_prefix)}/local"
|
|
and cmd.prefix == f"{os.path.normpath(sys.prefix)}/local"
|
|
)
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _looks_like_slackware_scheme() -> bool:
|
|
"""Slackware patches sysconfig but fails to patch distutils and site.
|
|
|
|
Slackware changes sysconfig's user scheme to use ``"lib64"`` for the lib
|
|
path, but does not do the same to the site module.
|
|
"""
|
|
if user_site is None: # User-site not available.
|
|
return False
|
|
try:
|
|
paths = sysconfig.get_paths(scheme="posix_user", expand=False)
|
|
except KeyError: # User-site not available.
|
|
return False
|
|
return "/lib64/" in paths["purelib"] and "/lib64/" not in user_site
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _looks_like_msys2_mingw_scheme() -> bool:
|
|
"""MSYS2 patches distutils and sysconfig to use a UNIX-like scheme.
|
|
|
|
However, MSYS2 incorrectly patches sysconfig ``nt`` scheme. The fix is
|
|
likely going to be included in their 3.10 release, so we ignore the warning.
|
|
See msys2/MINGW-packages#9319.
|
|
|
|
MSYS2 MINGW's patch uses lowercase ``"lib"`` instead of the usual uppercase,
|
|
and is missing the final ``"site-packages"``.
|
|
"""
|
|
paths = sysconfig.get_paths("nt", expand=False)
|
|
return all(
|
|
"Lib" not in p and "lib" in p and not p.endswith("site-packages")
|
|
for p in (paths[key] for key in ("platlib", "purelib"))
|
|
)
|
|
|
|
|
|
def _fix_abiflags(parts: Tuple[str]) -> Generator[str, None, None]:
|
|
ldversion = sysconfig.get_config_var("LDVERSION")
|
|
abiflags = getattr(sys, "abiflags", None)
|
|
|
|
# LDVERSION does not end with sys.abiflags. Just return the path unchanged.
|
|
if not ldversion or not abiflags or not ldversion.endswith(abiflags):
|
|
yield from parts
|
|
return
|
|
|
|
# Strip sys.abiflags from LDVERSION-based path components.
|
|
for part in parts:
|
|
if part.endswith(ldversion):
|
|
part = part[: (0 - len(abiflags))]
|
|
yield part
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _warn_mismatched(old: pathlib.Path, new: pathlib.Path, *, key: str) -> None:
|
|
issue_url = "https://github.com/pypa/pip/issues/10151"
|
|
message = (
|
|
"Value for %s does not match. Please report this to <%s>"
|
|
"\ndistutils: %s"
|
|
"\nsysconfig: %s"
|
|
)
|
|
logger.log(_MISMATCH_LEVEL, message, key, issue_url, old, new)
|
|
|
|
|
|
def _warn_if_mismatch(old: pathlib.Path, new: pathlib.Path, *, key: str) -> bool:
|
|
if old == new:
|
|
return False
|
|
_warn_mismatched(old, new, key=key)
|
|
return True
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _log_context(
|
|
*,
|
|
user: bool = False,
|
|
home: Optional[str] = None,
|
|
root: Optional[str] = None,
|
|
prefix: Optional[str] = None,
|
|
) -> None:
|
|
parts = [
|
|
"Additional context:",
|
|
"user = %r",
|
|
"home = %r",
|
|
"root = %r",
|
|
"prefix = %r",
|
|
]
|
|
|
|
logger.log(_MISMATCH_LEVEL, "\n".join(parts), user, home, root, prefix)
|
|
|
|
|
|
def get_scheme(
|
|
dist_name: str,
|
|
user: bool = False,
|
|
home: Optional[str] = None,
|
|
root: Optional[str] = None,
|
|
isolated: bool = False,
|
|
prefix: Optional[str] = None,
|
|
) -> Scheme:
|
|
new = _sysconfig.get_scheme(
|
|
dist_name,
|
|
user=user,
|
|
home=home,
|
|
root=root,
|
|
isolated=isolated,
|
|
prefix=prefix,
|
|
)
|
|
if _USE_SYSCONFIG:
|
|
return new
|
|
|
|
old = _distutils.get_scheme(
|
|
dist_name,
|
|
user=user,
|
|
home=home,
|
|
root=root,
|
|
isolated=isolated,
|
|
prefix=prefix,
|
|
)
|
|
|
|
warning_contexts = []
|
|
for k in SCHEME_KEYS:
|
|
old_v = pathlib.Path(getattr(old, k))
|
|
new_v = pathlib.Path(getattr(new, k))
|
|
|
|
if old_v == new_v:
|
|
continue
|
|
|
|
# distutils incorrectly put PyPy packages under ``site-packages/python``
|
|
# in the ``posix_home`` scheme, but PyPy devs said they expect the
|
|
# directory name to be ``pypy`` instead. So we treat this as a bug fix
|
|
# and not warn about it. See bpo-43307 and python/cpython#24628.
|
|
skip_pypy_special_case = (
|
|
sys.implementation.name == "pypy"
|
|
and home is not None
|
|
and k in ("platlib", "purelib")
|
|
and old_v.parent == new_v.parent
|
|
and old_v.name.startswith("python")
|
|
and new_v.name.startswith("pypy")
|
|
)
|
|
if skip_pypy_special_case:
|
|
continue
|
|
|
|
# sysconfig's ``osx_framework_user`` does not include ``pythonX.Y`` in
|
|
# the ``include`` value, but distutils's ``headers`` does. We'll let
|
|
# CPython decide whether this is a bug or feature. See bpo-43948.
|
|
skip_osx_framework_user_special_case = (
|
|
user
|
|
and is_osx_framework()
|
|
and k == "headers"
|
|
and old_v.parent.parent == new_v.parent
|
|
and old_v.parent.name.startswith("python")
|
|
)
|
|
if skip_osx_framework_user_special_case:
|
|
continue
|
|
|
|
# On Red Hat and derived Linux distributions, distutils is patched to
|
|
# use "lib64" instead of "lib" for platlib.
|
|
if k == "platlib" and _looks_like_red_hat_lib():
|
|
continue
|
|
|
|
# On Python 3.9+, sysconfig's posix_user scheme sets platlib against
|
|
# sys.platlibdir, but distutils's unix_user incorrectly coninutes
|
|
# using the same $usersite for both platlib and purelib. This creates a
|
|
# mismatch when sys.platlibdir is not "lib".
|
|
skip_bpo_44860 = (
|
|
user
|
|
and k == "platlib"
|
|
and not WINDOWS
|
|
and sys.version_info >= (3, 9)
|
|
and _PLATLIBDIR != "lib"
|
|
and _looks_like_bpo_44860()
|
|
)
|
|
if skip_bpo_44860:
|
|
continue
|
|
|
|
# Slackware incorrectly patches posix_user to use lib64 instead of lib,
|
|
# but not usersite to match the location.
|
|
skip_slackware_user_scheme = (
|
|
user
|
|
and k in ("platlib", "purelib")
|
|
and not WINDOWS
|
|
and _looks_like_slackware_scheme()
|
|
)
|
|
if skip_slackware_user_scheme:
|
|
continue
|
|
|
|
# Both Debian and Red Hat patch Python to place the system site under
|
|
# /usr/local instead of /usr. Debian also places lib in dist-packages
|
|
# instead of site-packages, but the /usr/local check should cover it.
|
|
skip_linux_system_special_case = (
|
|
not (user or home or prefix or running_under_virtualenv())
|
|
and old_v.parts[1:3] == ("usr", "local")
|
|
and len(new_v.parts) > 1
|
|
and new_v.parts[1] == "usr"
|
|
and (len(new_v.parts) < 3 or new_v.parts[2] != "local")
|
|
and (_looks_like_red_hat_scheme() or _looks_like_debian_scheme())
|
|
)
|
|
if skip_linux_system_special_case:
|
|
continue
|
|
|
|
# On Python 3.7 and earlier, sysconfig does not include sys.abiflags in
|
|
# the "pythonX.Y" part of the path, but distutils does.
|
|
skip_sysconfig_abiflag_bug = (
|
|
sys.version_info < (3, 8)
|
|
and not WINDOWS
|
|
and k in ("headers", "platlib", "purelib")
|
|
and tuple(_fix_abiflags(old_v.parts)) == new_v.parts
|
|
)
|
|
if skip_sysconfig_abiflag_bug:
|
|
continue
|
|
|
|
# MSYS2 MINGW's sysconfig patch does not include the "site-packages"
|
|
# part of the path. This is incorrect and will be fixed in MSYS.
|
|
skip_msys2_mingw_bug = (
|
|
WINDOWS and k in ("platlib", "purelib") and _looks_like_msys2_mingw_scheme()
|
|
)
|
|
if skip_msys2_mingw_bug:
|
|
continue
|
|
|
|
# CPython's POSIX install script invokes pip (via ensurepip) against the
|
|
# interpreter located in the source tree, not the install site. This
|
|
# triggers special logic in sysconfig that's not present in distutils.
|
|
# https://github.com/python/cpython/blob/8c21941ddaf/Lib/sysconfig.py#L178-L194
|
|
skip_cpython_build = (
|
|
sysconfig.is_python_build(check_home=True)
|
|
and not WINDOWS
|
|
and k in ("headers", "include", "platinclude")
|
|
)
|
|
if skip_cpython_build:
|
|
continue
|
|
|
|
warning_contexts.append((old_v, new_v, f"scheme.{k}"))
|
|
|
|
if not warning_contexts:
|
|
return old
|
|
|
|
# Check if this path mismatch is caused by distutils config files. Those
|
|
# files will no longer work once we switch to sysconfig, so this raises a
|
|
# deprecation message for them.
|
|
default_old = _distutils.distutils_scheme(
|
|
dist_name,
|
|
user,
|
|
home,
|
|
root,
|
|
isolated,
|
|
prefix,
|
|
ignore_config_files=True,
|
|
)
|
|
if any(default_old[k] != getattr(old, k) for k in SCHEME_KEYS):
|
|
deprecated(
|
|
reason=(
|
|
"Configuring installation scheme with distutils config files "
|
|
"is deprecated and will no longer work in the near future. If you "
|
|
"are using a Homebrew or Linuxbrew Python, please see discussion "
|
|
"at https://github.com/Homebrew/homebrew-core/issues/76621"
|
|
),
|
|
replacement=None,
|
|
gone_in=None,
|
|
)
|
|
return old
|
|
|
|
# Post warnings about this mismatch so user can report them back.
|
|
for old_v, new_v, key in warning_contexts:
|
|
_warn_mismatched(old_v, new_v, key=key)
|
|
_log_context(user=user, home=home, root=root, prefix=prefix)
|
|
|
|
return old
|
|
|
|
|
|
def get_bin_prefix() -> str:
|
|
new = _sysconfig.get_bin_prefix()
|
|
if _USE_SYSCONFIG:
|
|
return new
|
|
|
|
old = _distutils.get_bin_prefix()
|
|
if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
|
|
_log_context()
|
|
return old
|
|
|
|
|
|
def get_bin_user() -> str:
|
|
return _sysconfig.get_scheme("", user=True).scripts
|
|
|
|
|
|
def _looks_like_deb_system_dist_packages(value: str) -> bool:
|
|
"""Check if the value is Debian's APT-controlled dist-packages.
|
|
|
|
Debian's ``distutils.sysconfig.get_python_lib()`` implementation returns the
|
|
default package path controlled by APT, but does not patch ``sysconfig`` to
|
|
do the same. This is similar to the bug worked around in ``get_scheme()``,
|
|
but here the default is ``deb_system`` instead of ``unix_local``. Ultimately
|
|
we can't do anything about this Debian bug, and this detection allows us to
|
|
skip the warning when needed.
|
|
"""
|
|
if not _looks_like_debian_scheme():
|
|
return False
|
|
if value == "/usr/lib/python3/dist-packages":
|
|
return True
|
|
return False
|
|
|
|
|
|
def get_purelib() -> str:
|
|
"""Return the default pure-Python lib location."""
|
|
new = _sysconfig.get_purelib()
|
|
if _USE_SYSCONFIG:
|
|
return new
|
|
|
|
old = _distutils.get_purelib()
|
|
if _looks_like_deb_system_dist_packages(old):
|
|
return old
|
|
if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"):
|
|
_log_context()
|
|
return old
|
|
|
|
|
|
def get_platlib() -> str:
|
|
"""Return the default platform-shared lib location."""
|
|
new = _sysconfig.get_platlib()
|
|
if _USE_SYSCONFIG:
|
|
return new
|
|
|
|
from . import _distutils
|
|
|
|
old = _distutils.get_platlib()
|
|
if _looks_like_deb_system_dist_packages(old):
|
|
return old
|
|
if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
|
|
_log_context()
|
|
return old
|
|
|
|
|
|
def _deduplicated(v1: str, v2: str) -> List[str]:
|
|
"""Deduplicate values from a list."""
|
|
if v1 == v2:
|
|
return [v1]
|
|
return [v1, v2]
|
|
|
|
|
|
def _looks_like_apple_library(path: str) -> bool:
|
|
"""Apple patches sysconfig to *always* look under */Library/Python*."""
|
|
if sys.platform[:6] != "darwin":
|
|
return False
|
|
return path == f"/Library/Python/{get_major_minor_version()}/site-packages"
|
|
|
|
|
|
def get_prefixed_libs(prefix: str) -> List[str]:
|
|
"""Return the lib locations under ``prefix``."""
|
|
new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
|
|
if _USE_SYSCONFIG:
|
|
return _deduplicated(new_pure, new_plat)
|
|
|
|
old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
|
|
old_lib_paths = _deduplicated(old_pure, old_plat)
|
|
|
|
# Apple's Python (shipped with Xcode and Command Line Tools) hard-code
|
|
# platlib and purelib to '/Library/Python/X.Y/site-packages'. This will
|
|
# cause serious build isolation bugs when Apple starts shipping 3.10 because
|
|
# pip will install build backends to the wrong location. This tells users
|
|
# who is at fault so Apple may notice it and fix the issue in time.
|
|
if all(_looks_like_apple_library(p) for p in old_lib_paths):
|
|
deprecated(
|
|
reason=(
|
|
"Python distributed by Apple's Command Line Tools incorrectly "
|
|
"patches sysconfig to always point to '/Library/Python'. This "
|
|
"will cause build isolation to operate incorrectly on Python "
|
|
"3.10 or later. Please help report this to Apple so they can "
|
|
"fix this. https://developer.apple.com/bug-reporting/"
|
|
),
|
|
replacement=None,
|
|
gone_in=None,
|
|
)
|
|
return old_lib_paths
|
|
|
|
warned = [
|
|
_warn_if_mismatch(
|
|
pathlib.Path(old_pure),
|
|
pathlib.Path(new_pure),
|
|
key="prefixed-purelib",
|
|
),
|
|
_warn_if_mismatch(
|
|
pathlib.Path(old_plat),
|
|
pathlib.Path(new_plat),
|
|
key="prefixed-platlib",
|
|
),
|
|
]
|
|
if any(warned):
|
|
_log_context(prefix=prefix)
|
|
|
|
return old_lib_paths
|