Introduce compile_pip_requirements rule

This uses pip-tools to compile a requirements.in file to a requirements.txt file,
allowing transitive dependency versions to be pinned so that builds are reproducible.

Fixes bazelbuild#176

examples/pip_install/.bazelrc

@@ -0,0 +1 @@
+test --test_output=errors

examples/pip_install/BUILD

@@ -1,5 +1,6 @@
 load("@pip//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")
 
 # Toolchain setup, this is optional.
 # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
@@ -40,3 +41,6 @@ py_test(
     srcs = ["test.py"],
     deps = [":main"],
 )
+
+# Check that our compiled requirements are up-to-date
+compile_pip_requirements()

examples/pip_install/WORKSPACE

@@ -8,6 +8,17 @@ http_archive(
     sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
 )
 
+# Should not be here, workaround for
+# https://github.com/bazelbuild/rules_python/issues/372
+http_archive(
+    name = "bazel_skylib",
+    urls = [
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+    ],
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
+)
+
 load("@rules_python//python:pip.bzl", "pip_install")
 
 pip_install(

examples/pip_install/requirements.in

@@ -0,0 +1 @@
+boto3==1.14.51

examples/pip_install/requirements.txt

@@ -1 +1,39 @@
-boto3==1.14.51
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bazel run //:requirements.update
+#
+boto3==1.14.51 \
+    --hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \
+    --hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71 \
+    # via -r ./requirements.in
+botocore==1.17.63 \
+    --hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \
+    --hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75 \
+    # via boto3, s3transfer
+docutils==0.15.2 \
+    --hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
+    --hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
+    --hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99 \
+    # via botocore
+jmespath==0.10.0 \
+    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
+    # via boto3, botocore
+python-dateutil==2.8.1 \
+    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
+    # via botocore
+s3transfer==0.3.3 \
+    --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+    --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
+    # via boto3
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via python-dateutil
+urllib3==1.25.11 \
+    --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
+    --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e \
+    # via botocore

python/BUILD

@@ -23,6 +23,7 @@
 In an ideal renaming, we'd move the packaging rules to a different package so
 that @rules_python//python is only concerned with the core rules.
 """
+load("//python/pip_install:requirements.bzl", "compile_pip_requirements")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -130,3 +131,5 @@ exports_files([
     "pip.bzl",
     "whl.bzl",
 ])
+
+compile_pip_requirements(extra_args = ["--allow-unsafe"])

python/pip_install/BUILD

@@ -1,9 +1,12 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
+exports_files(["pip_compile.py"])
+
 filegroup(
     name = "distribution",
     srcs = glob(["*.bzl"]) + [
         "BUILD",
+        "pip_compile.py",
         "//python/pip_install/extract_wheels:distribution",
     ],
     visibility = ["//:__pkg__"],

python/pip_install/pip_compile.py

@@ -0,0 +1,89 @@
+"Set defaults for the pip-compile command to run it under Bazel"
+
+import os
+import sys
+from shutil import copyfile
+
+from piptools.scripts.compile import cli
+
+if len(sys.argv) < 4:
+    print(
+        "Expected at least two arguments: requirements_in requirements_out",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+requirements_in = sys.argv.pop(1)
+requirements_txt = sys.argv.pop(1)
+update_target_name = sys.argv.pop(1)
+
+# Before loading click, set the locale for its parser.
+# If it leaks through to the system setting, it may fail:
+# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
+# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for
+# mitigation steps.
+os.environ["LC_ALL"] = "C.UTF-8"
+os.environ["LANG"] = "C.UTF-8"
+
+UPDATE = True
+# Detect if we are running under `bazel test`
+if "TEST_TMPDIR" in os.environ:
+    UPDATE = False
+    # pip-compile wants the cache files to be writeable, but if we point
+    # to the real user cache, Bazel sandboxing makes the file read-only
+    # and we fail.
+    # In theory this makes the test more hermetic as well.
+    sys.argv.append("--cache-dir")
+    sys.argv.append(os.environ["TEST_TMPDIR"])
+    # Make a copy for pip-compile to read and mutate
+    requirements_out = os.path.join(
+        os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
+    )
+    copyfile(requirements_txt, requirements_out)
+
+elif "BUILD_WORKING_DIRECTORY" in os.environ:
+    os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
+else:
+    print(
+        "Expected to find BUILD_WORKING_DIRECTORY in environment",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+update_target_pkg = "/".join(requirements_in.split('/')[:-1])
+# $(rootpath) in the workspace root gives ./requirements.in
+if update_target_pkg == ".":
+    update_target_pkg = ""
+update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name)
+
+os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
+
+sys.argv.append("--generate-hashes")
+sys.argv.append("--output-file")
+sys.argv.append(requirements_txt if UPDATE else requirements_out)
+sys.argv.append(requirements_in)
+
+if UPDATE:
+    print("Updating " + requirements_txt)
+    cli()
+else:
+    # cli will exit(0) on success
+    try:
+        print("Checking " + requirements_txt)
+        cli()
+        print("cl() should exit", file=sys.stderr)
+        sys.exit(1)
+    except SystemExit:
+        golden = open(requirements_txt).readlines()
+        out = open(requirements_out).readlines()
+        if golden != out:
+            import difflib
+
+            print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
+            print(
+                "Lock file out of date. Run '"
+                + update_command
+                + "' to update.",
+                file=sys.stderr,
+            )
+            sys.exit(1)

python/pip_install/repositories.bzl

@@ -4,10 +4,20 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 _RULE_DEPS = [
+    (
+        "pypi__click",
+        "https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl",
+        "dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc",
+    ),
     (
         "pypi__pip",
-        "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl",
-        "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7",
+        "https://files.pythonhosted.org/packages/43/84/23ed6a1796480a6f1a2d38f2802901d078266bda38388954d01d3f2e821d/pip-20.1.1-py2.py3-none-any.whl",
+        "b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4",
+    ),
+    (
+        "pypi__pip_tools",
+        "https://files.pythonhosted.org/packages/a1/d5/c0f282060c483e6fd4635ed8f4f88aff02c5490d0fc588edec52b0cb9d7b/pip_tools-5.3.1-py2.py3-none-any.whl",
+        "73787e23269bf8a9230f376c351297b9037ed0d32ab0f9bef4a187d976acc054",
     ),
     (
         "pypi__pkginfo",

python/pip_install/requirements.bzl

@@ -0,0 +1,71 @@
+"Rules to verify and update pip-compile locked requirements.txt"
+
+load("//python:defs.bzl", "py_binary", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+
+def compile_pip_requirements(
+        name = "requirements",
+        extra_args = [],
+        visibility = ["//visibility:private"],
+        **kwargs):
+    """
+    Produce two targets for checking pip-compile:
+
+    - validate with `bazel test <name>_test`
+    - update with   `bazel run <name>.update`
+    
+    By default requirements in file is expected to be <name>.in and output
+    requirements txt lock file is expected to be <name>.txt. These may be customized
+    with `requirements_in` and `requirements_locked` params.
+
+    Args:
+        name: string
+        extra_args: passed to pip-compile
+        visibility: passed to both the _test and .update rules
+        **kwargs: other bazel attributes passed to the "_test" rule
+    """
+    requirements_in = kwargs.pop("requirements_in", name + ".in")
+    requirements_txt = kwargs.pop("requirements_locked", name + ".txt")
+
+    data = kwargs.pop("data", []) + [requirements_in, requirements_txt]
+
+    loc = "$(rootpath %s)"
+
+    # Use the Label constructor so this is expanded in the context of the file
+    # where it appears, which is to say, in @rules_python
+    pip_compile = Label("//python/pip_install:pip_compile.py")
+
+    args = [
+        loc % requirements_in,
+        loc % requirements_txt,
+        name + ".update",
+    ] + extra_args
+
+    py_binary(
+        name = name + ".update",
+        srcs = [pip_compile],
+        main = pip_compile,
+        args = args,
+        visibility = visibility,
+        deps = [
+            requirement("click"),
+            requirement("pip"),
+            requirement("pip_tools"),
+        ],
+        data = data,
+    )
+
+    py_test(
+        name = name + "_test",
+        srcs = [pip_compile],
+        main = pip_compile,
+        args = args,
+        visibility = visibility,
+        deps = [
+            requirement("click"),
+            requirement("pip"),
+            requirement("pip_tools"),
+        ],
+        data = data,
+        **kwargs
+    )

python/requirements.in

@@ -0,0 +1,6 @@
+pip==9.0.3
+setuptools==44.0.0
+wheel==0.30.0a0
+
+# For tests
+mock==2.0.0

python/requirements.txt

@@ -1,6 +1,32 @@
-pip==9.0.3
-setuptools==44.0.0
-wheel==0.30.0a0
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bazel run //python:requirements.update
+#
+mock==2.0.0 \
+    --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \
+    # via -r python/requirements.in
+pbr==5.5.1 \
+    --hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \
+    --hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00 \
+    # via mock
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via mock
+wheel==0.30.0a0 \
+    --hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \
+    --hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0 \
+    # via -r python/requirements.in
 
-# For tests
-mock==2.0.0
+# The following packages are considered to be unsafe in a requirements file:
+pip==9.0.3 \
+    --hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \
+    --hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5 \
+    # via -r python/requirements.in
+setuptools==44.0.0 \
+    --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \
+    --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 \
+    # via -r python/requirements.in