forked from s434650/CatOrNot
343 lines
12 KiB
Python
343 lines
12 KiB
Python
|
"""
|
||
|
|
Requirements file parsing
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import absolute_import
|
||
|
|
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import shlex
|
||
|
|
import sys
|
||
|
|
import optparse
|
||
|
|
import warnings
|
||
|
|
|
||
|
|
from pip._vendor.six.moves.urllib import parse as urllib_parse
|
||
|
|
from pip._vendor.six.moves import filterfalse
|
||
|
|
|
||
|
|
import pip
|
||
|
|
from pip.download import get_file_content
|
||
|
|
from pip.req.req_install import InstallRequirement
|
||
|
|
from pip.exceptions import (RequirementsFileParseError)
|
||
|
|
from pip.utils.deprecation import RemovedInPip10Warning
|
||
|
|
from pip import cmdoptions
|
||
|
|
|
||
|
|
__all__ = ['parse_requirements']
|
||
|
|
|
||
|
|
SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
|
||
|
|
COMMENT_RE = re.compile(r'(^|\s)+#.*$')
|
||
|
|
|
||
|
|
SUPPORTED_OPTIONS = [
|
||
|
|
cmdoptions.constraints,
|
||
|
|
cmdoptions.editable,
|
||
|
|
cmdoptions.requirements,
|
||
|
|
cmdoptions.no_index,
|
||
|
|
cmdoptions.index_url,
|
||
|
|
cmdoptions.find_links,
|
||
|
|
cmdoptions.extra_index_url,
|
||
|
|
cmdoptions.allow_external,
|
||
|
|
cmdoptions.allow_all_external,
|
||
|
|
cmdoptions.no_allow_external,
|
||
|
|
cmdoptions.allow_unsafe,
|
||
|
|
cmdoptions.no_allow_unsafe,
|
||
|
|
cmdoptions.use_wheel,
|
||
|
|
cmdoptions.no_use_wheel,
|
||
|
|
cmdoptions.always_unzip,
|
||
|
|
cmdoptions.no_binary,
|
||
|
|
cmdoptions.only_binary,
|
||
|
|
cmdoptions.pre,
|
||
|
|
cmdoptions.process_dependency_links,
|
||
|
|
cmdoptions.trusted_host,
|
||
|
|
cmdoptions.require_hashes,
|
||
|
|
]
|
||
|
|
|
||
|
|
# options to be passed to requirements
|
||
|
|
SUPPORTED_OPTIONS_REQ = [
|
||
|
|
cmdoptions.install_options,
|
||
|
|
cmdoptions.global_options,
|
||
|
|
cmdoptions.hash,
|
||
|
|
]
|
||
|
|
|
||
|
|
# the 'dest' string values
|
||
|
|
SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]
|
||
|
|
|
||
|
|
|
||
|
|
def parse_requirements(filename, finder=None, comes_from=None, options=None,
|
||
|
|
session=None, constraint=False, wheel_cache=None):
|
||
|
|
"""Parse a requirements file and yield InstallRequirement instances.
|
||
|
|
|
||
|
|
:param filename: Path or url of requirements file.
|
||
|
|
:param finder: Instance of pip.index.PackageFinder.
|
||
|
|
:param comes_from: Origin description of requirements.
|
||
|
|
:param options: cli options.
|
||
|
|
:param session: Instance of pip.download.PipSession.
|
||
|
|
:param constraint: If true, parsing a constraint file rather than
|
||
|
|
requirements file.
|
||
|
|
:param wheel_cache: Instance of pip.wheel.WheelCache
|
||
|
|
"""
|
||
|
|
if session is None:
|
||
|
|
raise TypeError(
|
||
|
|
"parse_requirements() missing 1 required keyword argument: "
|
||
|
|
"'session'"
|
||
|
|
)
|
||
|
|
|
||
|
|
_, content = get_file_content(
|
||
|
|
filename, comes_from=comes_from, session=session
|
||
|
|
)
|
||
|
|
|
||
|
|
lines_enum = preprocess(content, options)
|
||
|
|
|
||
|
|
for line_number, line in lines_enum:
|
||
|
|
req_iter = process_line(line, filename, line_number, finder,
|
||
|
|
comes_from, options, session, wheel_cache,
|
||
|
|
constraint=constraint)
|
||
|
|
for req in req_iter:
|
||
|
|
yield req
|
||
|
|
|
||
|
|
|
||
|
|
def preprocess(content, options):
|
||
|
|
"""Split, filter, and join lines, and return a line iterator
|
||
|
|
|
||
|
|
:param content: the content of the requirements file
|
||
|
|
:param options: cli options
|
||
|
|
"""
|
||
|
|
lines_enum = enumerate(content.splitlines(), start=1)
|
||
|
|
lines_enum = join_lines(lines_enum)
|
||
|
|
lines_enum = ignore_comments(lines_enum)
|
||
|
|
lines_enum = skip_regex(lines_enum, options)
|
||
|
|
return lines_enum
|
||
|
|
|
||
|
|
|
||
|
|
def process_line(line, filename, line_number, finder=None, comes_from=None,
|
||
|
|
options=None, session=None, wheel_cache=None,
|
||
|
|
constraint=False):
|
||
|
|
"""Process a single requirements line; This can result in creating/yielding
|
||
|
|
requirements, or updating the finder.
|
||
|
|
|
||
|
|
For lines that contain requirements, the only options that have an effect
|
||
|
|
are from SUPPORTED_OPTIONS_REQ, and they are scoped to the
|
||
|
|
requirement. Other options from SUPPORTED_OPTIONS may be present, but are
|
||
|
|
ignored.
|
||
|
|
|
||
|
|
For lines that do not contain requirements, the only options that have an
|
||
|
|
effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may
|
||
|
|
be present, but are ignored. These lines may contain multiple options
|
||
|
|
(although our docs imply only one is supported), and all our parsed and
|
||
|
|
affect the finder.
|
||
|
|
|
||
|
|
:param constraint: If True, parsing a constraints file.
|
||
|
|
:param options: OptionParser options that we may update
|
||
|
|
"""
|
||
|
|
parser = build_parser()
|
||
|
|
defaults = parser.get_default_values()
|
||
|
|
defaults.index_url = None
|
||
|
|
if finder:
|
||
|
|
# `finder.format_control` will be updated during parsing
|
||
|
|
defaults.format_control = finder.format_control
|
||
|
|
args_str, options_str = break_args_options(line)
|
||
|
|
if sys.version_info < (2, 7, 3):
|
||
|
|
# Prior to 2.7.3, shlex cannot deal with unicode entries
|
||
|
|
options_str = options_str.encode('utf8')
|
||
|
|
opts, _ = parser.parse_args(shlex.split(options_str), defaults)
|
||
|
|
|
||
|
|
# preserve for the nested code path
|
||
|
|
line_comes_from = '%s %s (line %s)' % (
|
||
|
|
'-c' if constraint else '-r', filename, line_number)
|
||
|
|
|
||
|
|
# yield a line requirement
|
||
|
|
if args_str:
|
||
|
|
isolated = options.isolated_mode if options else False
|
||
|
|
if options:
|
||
|
|
cmdoptions.check_install_build_global(options, opts)
|
||
|
|
# get the options that apply to requirements
|
||
|
|
req_options = {}
|
||
|
|
for dest in SUPPORTED_OPTIONS_REQ_DEST:
|
||
|
|
if dest in opts.__dict__ and opts.__dict__[dest]:
|
||
|
|
req_options[dest] = opts.__dict__[dest]
|
||
|
|
yield InstallRequirement.from_line(
|
||
|
|
args_str, line_comes_from, constraint=constraint,
|
||
|
|
isolated=isolated, options=req_options, wheel_cache=wheel_cache
|
||
|
|
)
|
||
|
|
|
||
|
|
# yield an editable requirement
|
||
|
|
elif opts.editables:
|
||
|
|
isolated = options.isolated_mode if options else False
|
||
|
|
default_vcs = options.default_vcs if options else None
|
||
|
|
yield InstallRequirement.from_editable(
|
||
|
|
opts.editables[0], comes_from=line_comes_from,
|
||
|
|
constraint=constraint, default_vcs=default_vcs, isolated=isolated,
|
||
|
|
wheel_cache=wheel_cache
|
||
|
|
)
|
||
|
|
|
||
|
|
# parse a nested requirements file
|
||
|
|
elif opts.requirements or opts.constraints:
|
||
|
|
if opts.requirements:
|
||
|
|
req_path = opts.requirements[0]
|
||
|
|
nested_constraint = False
|
||
|
|
else:
|
||
|
|
req_path = opts.constraints[0]
|
||
|
|
nested_constraint = True
|
||
|
|
# original file is over http
|
||
|
|
if SCHEME_RE.search(filename):
|
||
|
|
# do a url join so relative paths work
|
||
|
|
req_path = urllib_parse.urljoin(filename, req_path)
|
||
|
|
# original file and nested file are paths
|
||
|
|
elif not SCHEME_RE.search(req_path):
|
||
|
|
# do a join so relative paths work
|
||
|
|
req_path = os.path.join(os.path.dirname(filename), req_path)
|
||
|
|
# TODO: Why not use `comes_from='-r {} (line {})'` here as well?
|
||
|
|
parser = parse_requirements(
|
||
|
|
req_path, finder, comes_from, options, session,
|
||
|
|
constraint=nested_constraint, wheel_cache=wheel_cache
|
||
|
|
)
|
||
|
|
for req in parser:
|
||
|
|
yield req
|
||
|
|
|
||
|
|
# percolate hash-checking option upward
|
||
|
|
elif opts.require_hashes:
|
||
|
|
options.require_hashes = opts.require_hashes
|
||
|
|
|
||
|
|
# set finder options
|
||
|
|
elif finder:
|
||
|
|
if opts.allow_external:
|
||
|
|
warnings.warn(
|
||
|
|
"--allow-external has been deprecated and will be removed in "
|
||
|
|
"the future. Due to changes in the repository protocol, it no "
|
||
|
|
"longer has any effect.",
|
||
|
|
RemovedInPip10Warning,
|
||
|
|
)
|
||
|
|
|
||
|
|
if opts.allow_all_external:
|
||
|
|
warnings.warn(
|
||
|
|
"--allow-all-external has been deprecated and will be removed "
|
||
|
|
"in the future. Due to changes in the repository protocol, it "
|
||
|
|
"no longer has any effect.",
|
||
|
|
RemovedInPip10Warning,
|
||
|
|
)
|
||
|
|
|
||
|
|
if opts.allow_unverified:
|
||
|
|
warnings.warn(
|
||
|
|
"--allow-unverified has been deprecated and will be removed "
|
||
|
|
"in the future. Due to changes in the repository protocol, it "
|
||
|
|
"no longer has any effect.",
|
||
|
|
RemovedInPip10Warning,
|
||
|
|
)
|
||
|
|
|
||
|
|
if opts.index_url:
|
||
|
|
finder.index_urls = [opts.index_url]
|
||
|
|
if opts.use_wheel is False:
|
||
|
|
finder.use_wheel = False
|
||
|
|
pip.index.fmt_ctl_no_use_wheel(finder.format_control)
|
||
|
|
if opts.no_index is True:
|
||
|
|
finder.index_urls = []
|
||
|
|
if opts.extra_index_urls:
|
||
|
|
finder.index_urls.extend(opts.extra_index_urls)
|
||
|
|
if opts.find_links:
|
||
|
|
# FIXME: it would be nice to keep track of the source
|
||
|
|
# of the find_links: support a find-links local path
|
||
|
|
# relative to a requirements file.
|
||
|
|
value = opts.find_links[0]
|
||
|
|
req_dir = os.path.dirname(os.path.abspath(filename))
|
||
|
|
relative_to_reqs_file = os.path.join(req_dir, value)
|
||
|
|
if os.path.exists(relative_to_reqs_file):
|
||
|
|
value = relative_to_reqs_file
|
||
|
|
finder.find_links.append(value)
|
||
|
|
if opts.pre:
|
||
|
|
finder.allow_all_prereleases = True
|
||
|
|
if opts.process_dependency_links:
|
||
|
|
finder.process_dependency_links = True
|
||
|
|
if opts.trusted_hosts:
|
||
|
|
finder.secure_origins.extend(
|
||
|
|
("*", host, "*") for host in opts.trusted_hosts)
|
||
|
|
|
||
|
|
|
||
|
|
def break_args_options(line):
|
||
|
|
"""Break up the line into an args and options string. We only want to shlex
|
||
|
|
(and then optparse) the options, not the args. args can contain markers
|
||
|
|
which are corrupted by shlex.
|
||
|
|
"""
|
||
|
|
tokens = line.split(' ')
|
||
|
|
args = []
|
||
|
|
options = tokens[:]
|
||
|
|
for token in tokens:
|
||
|
|
if token.startswith('-') or token.startswith('--'):
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
args.append(token)
|
||
|
|
options.pop(0)
|
||
|
|
return ' '.join(args), ' '.join(options)
|
||
|
|
|
||
|
|
|
||
|
|
def build_parser():
|
||
|
|
"""
|
||
|
|
Return a parser for parsing requirement lines
|
||
|
|
"""
|
||
|
|
parser = optparse.OptionParser(add_help_option=False)
|
||
|
|
|
||
|
|
option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ
|
||
|
|
for option_factory in option_factories:
|
||
|
|
option = option_factory()
|
||
|
|
parser.add_option(option)
|
||
|
|
|
||
|
|
# By default optparse sys.exits on parsing errors. We want to wrap
|
||
|
|
# that in our own exception.
|
||
|
|
def parser_exit(self, msg):
|
||
|
|
raise RequirementsFileParseError(msg)
|
||
|
|
parser.exit = parser_exit
|
||
|
|
|
||
|
|
return parser
|
||
|
|
|
||
|
|
|
||
|
|
def join_lines(lines_enum):
|
||
|
|
"""Joins a line ending in '\' with the previous line (except when following
|
||
|
|
comments). The joined line takes on the index of the first line.
|
||
|
|
"""
|
||
|
|
primary_line_number = None
|
||
|
|
new_line = []
|
||
|
|
for line_number, line in lines_enum:
|
||
|
|
if not line.endswith('\\') or COMMENT_RE.match(line):
|
||
|
|
if COMMENT_RE.match(line):
|
||
|
|
# this ensures comments are always matched later
|
||
|
|
line = ' ' + line
|
||
|
|
if new_line:
|
||
|
|
new_line.append(line)
|
||
|
|
yield primary_line_number, ''.join(new_line)
|
||
|
|
new_line = []
|
||
|
|
else:
|
||
|
|
yield line_number, line
|
||
|
|
else:
|
||
|
|
if not new_line:
|
||
|
|
primary_line_number = line_number
|
||
|
|
new_line.append(line.strip('\\'))
|
||
|
|
|
||
|
|
# last line contains \
|
||
|
|
if new_line:
|
||
|
|
yield primary_line_number, ''.join(new_line)
|
||
|
|
|
||
|
|
# TODO: handle space after '\'.
|
||
|
|
|
||
|
|
|
||
|
|
def ignore_comments(lines_enum):
|
||
|
|
"""
|
||
|
|
Strips comments and filter empty lines.
|
||
|
|
"""
|
||
|
|
for line_number, line in lines_enum:
|
||
|
|
line = COMMENT_RE.sub('', line)
|
||
|
|
line = line.strip()
|
||
|
|
if line:
|
||
|
|
yield line_number, line
|
||
|
|
|
||
|
|
|
||
|
|
def skip_regex(lines_enum, options):
|
||
|
|
"""
|
||
|
|
Skip lines that match '--skip-requirements-regex' pattern
|
||
|
|
|
||
|
|
Note: the regex pattern is only built once
|
||
|
|
"""
|
||
|
|
skip_regex = options.skip_requirements_regex if options else None
|
||
|
|
if skip_regex:
|
||
|
|
pattern = re.compile(skip_regex)
|
||
|
|
lines_enum = filterfalse(
|
||
|
|
lambda e: pattern.search(e[1]),
|
||
|
|
lines_enum)
|
||
|
|
return lines_enum
|