Support relative paths and regularize path handling and formatting

piptools/_compat/pip_compat.py

@@ -1,17 +1,25 @@
 import optparse
+import platform
+import re
 from typing import Callable, Iterable, Iterator, Optional, cast
 
 import pip
+from pip._internal.exceptions import InstallationError
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.models.link import Link
 from pip._internal.network.session import PipSession
 from pip._internal.req import InstallRequirement
 from pip._internal.req import parse_requirements as _parse_requirements
 from pip._internal.req.constructors import install_req_from_parsed_requirement
+from pip._internal.req.req_file import ParsedRequirement
 from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.pkg_resources import Requirement
 
+from ..utils import abs_ireq, copy_install_requirement, fragment_string, working_dir
+
 PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))
 
+file_url_schemes_re = re.compile(r"^((git|hg|svn|bzr)\+)?file:")
 
 __all__ = [
     "get_build_tracker",
@@ -29,11 +37,71 @@ def parse_requirements(
     options: Optional[optparse.Values] = None,
     constraint: bool = False,
     isolated: bool = False,
+    from_dir: Optional[str] = None,
 ) -> Iterator[InstallRequirement]:
     for parsed_req in _parse_requirements(
         filename, session, finder=finder, options=options, constraint=constraint
     ):
-        yield install_req_from_parsed_requirement(parsed_req, isolated=isolated)
+        # This context manager helps pip locate relative paths specified
+        # with non-URI (non file:) syntax, e.g. '-e ..'
+        with working_dir(from_dir):
+            try:
+                ireq = install_req_from_parsed_requirement(
+                    parsed_req, isolated=isolated
+                )
+            except InstallationError:
+                # This can happen when the url is a relpath with a fragment,
+                # so we try again with the fragment stripped
+                preq_without_fragment = ParsedRequirement(
+                    requirement=re.sub(r"#[^#]+$", "", parsed_req.requirement),
+                    is_editable=parsed_req.is_editable,
+                    comes_from=parsed_req.comes_from,
+                    constraint=parsed_req.constraint,
+                    options=parsed_req.options,
+                    line_source=parsed_req.line_source,
+                )
+                ireq = install_req_from_parsed_requirement(
+                    preq_without_fragment, isolated=isolated
+                )
+
+        # At this point the ireq has two problems:
+        # - Sometimes the fragment is lost (even without an InstallationError)
+        # - It's now absolute (ahead of schedule),
+        #   so abs_ireq will not know to apply the _was_relative attribute,
+        #   which is needed for the writer to use the relpath.
+
+        # To account for the first:
+        if not fragment_string(ireq):
+            fragment = Link(parsed_req.requirement)._parsed_url.fragment
+            if fragment:
+                link_with_fragment = Link(
+                    url=f"{ireq.link.url_without_fragment}#{fragment}",
+                    comes_from=ireq.link.comes_from,
+                    requires_python=ireq.link.requires_python,
+                    yanked_reason=ireq.link.yanked_reason,
+                    cache_link_parsing=ireq.link.cache_link_parsing,
+                )
+                ireq = copy_install_requirement(ireq, link=link_with_fragment)
+
+        a_ireq = abs_ireq(ireq, from_dir)
+
+        # To account for the second, we guess if the path was initially relative and
+        # set _was_relative ourselves:
+        bare_path = file_url_schemes_re.sub(
+            "", parsed_req.requirement.split(" @ ", 1)[-1]
+        )
+        is_win = platform.system() == "Windows"
+        if is_win:
+            bare_path = bare_path.lstrip("/")
+        if (
+            a_ireq.link is not None
+            and a_ireq.link.scheme.endswith("file")
+            and not bare_path.startswith("/")
+        ):
+            if not (is_win and re.match(r"[a-zA-Z]:", bare_path)):
+                a_ireq._was_relative = True
+
+        yield a_ireq
 
 
 if PIP_VERSION[:2] <= (22, 0):

piptools/resolver.py

@@ -705,8 +705,8 @@ def _get_install_requirements(
         for extras_candidate in extras_candidates:
             project_name = canonicalize_name(extras_candidate.project_name)
             ireq = result_ireqs[project_name]
-            ireq.extras |= extras_candidate.extras
-            ireq.req.extras |= extras_candidate.extras
+            ireq.extras = set(ireq.extras) | set(extras_candidate.extras)
+            ireq.req.extras = set(ireq.req.extras) | set(extras_candidate.extras)
 
         return set(result_ireqs.values())
 
@@ -778,4 +778,13 @@ def _get_install_requirement_from_candidate(
         if source_ireq is not None and ireq_key not in self.existing_constraints:
             pinned_ireq._source_ireqs = [source_ireq]
 
+        # Preserve _was_relative attribute of local path requirements
+        if pinned_ireq.link.is_file:
+            # Install requirement keys may not match
+            for c in self.constraints:
+                if pinned_ireq.link == c.link:
+                    if hasattr(c, "_was_relative"):
+                        pinned_ireq._was_relative = True
+                    break
+
         return pinned_ireq

piptools/scripts/compile.py

@@ -28,6 +28,7 @@
     drop_extras,
     is_pinned_requirement,
     key_from_ireq,
+    working_dir,
 )
 from ..writer import OutputWriter
 
@@ -159,6 +160,24 @@ def _get_default_option(option_name: str) -> Any:
         "Will be derived from input file otherwise."
     ),
 )
+@click.option(
+    "--write-relative-to-output",
+    is_flag=True,
+    default=False,
+    help=(
+        "Construct relative paths as relative to the output file's parent. "
+        "Will be written as relative to the current folder otherwise."
+    ),
+)
+@click.option(
+    "--read-relative-to-input",
+    is_flag=True,
+    default=False,
+    help=(
+        "Resolve relative paths as relative to the input file's parent. "
+        "Will be resolved as relative to the current folder otherwise."
+    ),
+)
 @click.option(
     "--allow-unsafe/--no-allow-unsafe",
     is_flag=True,
@@ -272,6 +291,8 @@ def cli(
     upgrade: bool,
     upgrade_packages: Tuple[str, ...],
     output_file: Union[LazyFile, IO[Any], None],
+    write_relative_to_output: bool,
+    read_relative_to_input: bool,
     allow_unsafe: bool,
     strip_extras: bool,
     generate_hashes: bool,
@@ -306,24 +327,27 @@ def cli(
                 ).format(DEFAULT_REQUIREMENTS_FILE)
             )
 
+    src_files = tuple(src if src == "-" else os.path.abspath(src) for src in src_files)
+
     if not output_file:
         # An output file must be provided for stdin
         if src_files == ("-",):
             raise click.BadParameter("--output-file is required if input is from stdin")
-        # Use default requirements output file if there is a setup.py the source file
-        elif os.path.basename(src_files[0]) in METADATA_FILENAMES:
-            file_name = os.path.join(
-                os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE
-            )
         # An output file must be provided if there are multiple source files
         elif len(src_files) > 1:
             raise click.BadParameter(
                 "--output-file is required if two or more input files are given."
             )
+        # Use default requirements output file if there is only a setup.py source file
+        elif os.path.basename(src_files[0]) in METADATA_FILENAMES:
+            file_name = os.path.join(
+                os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE
+            )
         # Otherwise derive the output file from the source file
         else:
-            base_name = src_files[0].rsplit(".", 1)[0]
-            file_name = base_name + ".txt"
+            file_name = os.path.splitext(src_files[0])[0] + ".txt"
+            if file_name == src_files[0]:
+                file_name += ".txt"
 
         output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True)
 
@@ -385,6 +409,11 @@ def cli(
             finder=tmp_repository.finder,
             session=tmp_repository.session,
             options=tmp_repository.options,
+            from_dir=(
+                os.path.dirname(os.path.abspath(output_file.name))
+                if write_relative_to_output
+                else None
+            ),
         )
 
         for ireq in filter(is_pinned_requirement, ireqs):
@@ -408,8 +437,7 @@ def cli(
         if src_file == "-":
             # pip requires filenames and not files. Since we want to support
             # piping from stdin, we need to briefly save the input from stdin
-            # to a temporary file and have pip read that.  also used for
-            # reading requirements from install_requires in setup.py.
+            # to a temporary file and have pip read that.
             tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
             tmpfile.write(sys.stdin.read())
             comes_from = "-r -"
@@ -435,20 +463,29 @@ def cli(
                 log.error(str(e))
                 log.error(f"Failed to parse {os.path.abspath(src_file)}")
                 sys.exit(2)
-            comes_from = f"{metadata.get_all('Name')[0]} ({src_file})"
-            constraints.extend(
-                [
-                    install_req_from_line(req, comes_from=comes_from)
-                    for req in metadata.get_all("Requires-Dist") or []
-                ]
-            )
+            with working_dir(os.path.dirname(os.path.abspath(output_file.name))):
+                comes_from = (
+                    f"{metadata.get_all('Name')[0]} ({os.path.relpath(src_file)})"
+                )
+            with working_dir(
+                os.path.dirname(src_file) if read_relative_to_input else None
+            ):
+                constraints.extend(
+                    [
+                        install_req_from_line(req, comes_from=comes_from)
+                        for req in metadata.get_all("Requires-Dist") or []
+                    ]
+                )
         else:
             constraints.extend(
                 parse_requirements(
                     src_file,
                     finder=repository.finder,
                     session=repository.session,
                     options=repository.options,
+                    from_dir=(
+                        os.path.dirname(src_file) if read_relative_to_input else None
+                    ),
                 )
             )
 
@@ -526,6 +563,7 @@ def cli(
         find_links=repository.finder.find_links,
         emit_find_links=emit_find_links,
         emit_options=emit_options,
+        write_relative_to_output=write_relative_to_output,
     )
     writer.write(
         results=results,

piptools/scripts/sync.py

@@ -70,6 +70,15 @@
     is_flag=True,
     help="Ignore package index (only looking at --find-links URLs instead)",
 )
+@click.option(
+    "--read-relative-to-input",
+    is_flag=True,
+    default=False,
+    help=(
+        "Resolve relative paths as relative to the input file's parent. "
+        "Will be resolved as relative to the current folder otherwise."
+    ),
+)
 @click.option(
     "--python-executable",
     help="Custom python executable path if targeting an environment other than current.",
@@ -96,6 +105,7 @@ def cli(
     extra_index_url: Tuple[str, ...],
     trusted_host: Tuple[str, ...],
     no_index: bool,
+    read_relative_to_input: bool,
     python_executable: Optional[str],
     verbose: int,
     quiet: int,
@@ -139,7 +149,17 @@ def cli(
     # Parse requirements file. Note, all options inside requirements file
     # will be collected by the finder.
     requirements = flat_map(
-        lambda src: parse_requirements(src, finder=finder, session=session), src_files
+        lambda src: parse_requirements(
+            src,
+            finder=finder,
+            session=session,
+            from_dir=(
+                os.path.dirname(os.path.abspath(src))
+                if read_relative_to_input
+                else os.getcwd()
+            ),
+        ),
+        src_files,
     )
 
     try: