Introduce pip_requirements rule

This uses pip-tools to compile a requirements.in file to a requirements.txt file,
allowing transitive dependency versions to be pinned so that builds are reproducible.

Fixes bazelbuild#176

python/BUILD

@@ -130,3 +130,7 @@ exports_files([
     "pip.bzl",
     "whl.bzl",
 ])
+
+load("//python/pip_install:pip_requirements.bzl", "pip_requirements")
+
+pip_requirements(extra_args = ["--allow-unsafe"])

python/pip_install/BUILD

@@ -1,5 +1,7 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
+exports_files(["pip_compile.py"])
+
 filegroup(
     name = "distribution",
     srcs = glob(["*.bzl"]) + [

python/pip_install/pip_compile.py

@@ -0,0 +1,83 @@
+"Set defaults for the pip-compile command to run it under Bazel"
+
+import os
+import sys
+from shutil import copyfile
+
+from piptools.scripts.compile import cli
+
+if len(sys.argv) < 3:
+    print(
+        "Expected at least two arguments: requirements_in requirements_out",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+requirements_in = sys.argv.pop(1)
+requirements_txt = sys.argv.pop(1)
+
+UPDATE = True
+# Detect if we are running under `bazel test`
+if "TEST_TMPDIR" in os.environ:
+    UPDATE = False
+    # pip-compile wants the cache files to be writeable, but if we point
+    # to the real user cache, Bazel sandboxing makes the file read-only
+    # and we fail.
+    # In theory this makes the test more hermetic as well.
+    sys.argv.append("--cache-dir")
+    sys.argv.append(os.environ["TEST_TMPDIR"])
+    # Make a copy for pip-compile to read and mutate
+    requirements_out = os.path.join(
+        os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
+    )
+    copyfile(requirements_txt, requirements_out)
+
+elif "BUILD_WORKING_DIRECTORY" in os.environ:
+    os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
+else:
+    print(
+        "Expected to find BUILD_WORKING_DIRECTORY in environment",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+update_target_segments = requirements_in[:-3].split('/')
+update_command = (
+    "bazel run //"
+    + "/".join(update_target_segments[:-1])
+    + ":"
+    + update_target_segments[-1]
+    + ".update"
+)
+
+os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
+
+sys.argv.append("--generate-hashes")
+sys.argv.append("--output-file")
+sys.argv.append(requirements_txt if UPDATE else requirements_out)
+sys.argv.append(requirements_in)
+
+if UPDATE:
+    print("Updating " + requirements_txt)
+    cli()
+else:
+    # cli will exit(0) on success
+    try:
+        print("Checking " + requirements_txt)
+        cli()
+        print("cl() should exit", file=sys.stderr)
+        sys.exit(1)
+    except SystemExit:
+        golden = open(requirements_txt).readlines()
+        out = open(requirements_out).readlines()
+        if golden != out:
+            import difflib
+
+            print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
+            print(
+                "Lock file out of date. Run '"
+                + update_command
+                + "' to update.",
+                file=sys.stderr,
+            )
+            sys.exit(1)

python/pip_install/pip_requirements.bzl

@@ -0,0 +1,64 @@
+"Rules to verify and update pip-compile locked requirements.txt"
+
+load("//python:defs.bzl", "py_binary", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+
+def pip_requirements(
+        name = "requirements_test",
+        extra_args = [],
+        **kwargs):
+    """
+    Produce two targets for checking pip-compile:
+
+    - validate with `bazel test some_test`
+    - update with   `bazel run some.update`
+
+    Args:
+        name: string
+        extra_args: passed to pip-compile
+        **kwargs: other bazel attributes
+    """
+    if name and not name.endswith("_test"):
+        fail("test target must be named with _test suffix")
+    requirements_in = name[:-5] + ".in"
+    requirements_txt = name[:-5] + ".txt"
+
+    data = kwargs.pop("data", []) + [requirements_in, requirements_txt]
+
+    loc = "$(rootpath %s)"
+
+    py_binary(
+        name = name[:-5] + ".update",
+        srcs = ["//python/pip_install:pip_compile.py"],
+        main = "//python/pip_install:pip_compile.py",
+        args = [
+            loc % requirements_in,
+            loc % requirements_txt,
+        ] + extra_args,
+        visibility = ["//visibility:public"],
+        deps = [
+            requirement("pip"),
+            requirement("pip_tools"),
+        ],
+        data = data,
+    )
+
+    timeout = kwargs.pop("timeout", "short")
+
+    py_test(
+        name = name,
+        srcs = ["//python/pip_install:pip_compile.py"],
+        main = "//python/pip_install:pip_compile.py",
+        args = [
+            loc % requirements_in,
+            loc % requirements_txt,
+        ] + extra_args,
+        visibility = ["//visibility:public"],
+        deps = [
+            requirement("pip"),
+            requirement("pip_tools"),
+        ],
+        data = data,
+        timeout = timeout,
+        **kwargs
+    )

python/pip_install/repositories.bzl

@@ -6,8 +6,13 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 _RULE_DEPS = [
     (
         "pypi__pip",
-        "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl",
-        "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7",
+        "https://files.pythonhosted.org/packages/43/84/23ed6a1796480a6f1a2d38f2802901d078266bda38388954d01d3f2e821d/pip-20.1.1-py2.py3-none-any.whl",
+        "b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4",
+    ),
+    (
+        "pypi__pip_tools",
+        "https://files.pythonhosted.org/packages/a1/d5/c0f282060c483e6fd4635ed8f4f88aff02c5490d0fc588edec52b0cb9d7b/pip_tools-5.3.1-py2.py3-none-any.whl",
+        "73787e23269bf8a9230f376c351297b9037ed0d32ab0f9bef4a187d976acc054",
     ),
     (
         "pypi__pkginfo",

python/requirements.in

@@ -0,0 +1,6 @@
+pip==9.0.3
+setuptools==44.0.0
+wheel==0.30.0a0
+
+# For tests
+mock==2.0.0

python/requirements.txt

@@ -1,6 +1,32 @@
-pip==9.0.3
-setuptools==44.0.0
-wheel==0.30.0a0
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bazel run //python:requirements.update
+#
+mock==2.0.0 \
+    --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
+    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba \
+    # via -r python/requirements.in
+pbr==5.5.1 \
+    --hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \
+    --hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00 \
+    # via mock
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    # via mock
+wheel==0.30.0a0 \
+    --hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \
+    --hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0 \
+    # via -r python/requirements.in
 
-# For tests
-mock==2.0.0
+# The following packages are considered to be unsafe in a requirements file:
+pip==9.0.3 \
+    --hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \
+    --hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5 \
+    # via -r python/requirements.in
+setuptools==44.0.0 \
+    --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \
+    --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 \
+    # via -r python/requirements.in