Skip to content

Commit

Permalink
Introduce compile_pip_requirements rule
Browse files Browse the repository at this point in the history
This uses pip-tools to compile a requirements.in file to a requirements.txt file,
allowing transitive dependency versions to be pinned so that builds are reproducible.

Fixes bazelbuild#176
  • Loading branch information
Alex Eagle authored and alexeagle committed Apr 20, 2021
1 parent 04c6d7f commit f73e99c
Show file tree
Hide file tree
Showing 11 changed files with 281 additions and 6 deletions.
1 change: 1 addition & 0 deletions examples/pip_install/.bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test --test_output=errors
6 changes: 6 additions & 0 deletions examples/pip_install/BUILD
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
load("@pip//:requirements.bzl", "requirement")
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements")

# Toolchain setup, this is optional.
# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
Expand Down Expand Up @@ -40,3 +41,8 @@ py_test(
srcs = ["test.py"],
deps = [":main"],
)

# Check that our compiled requirements are up-to-date
compile_pip_requirements(
name = "requirements",
)
1 change: 1 addition & 0 deletions examples/pip_install/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
boto3==1.14.51
44 changes: 43 additions & 1 deletion examples/pip_install/requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,43 @@
boto3==1.14.51
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bazel run //:requirements.update
#
boto3==1.14.51 \
--hash=sha256:a6bdb808e948bd264af135af50efb76253e85732c451fa605b7a287faf022432 \
--hash=sha256:f9dbccbcec916051c6588adbccae86547308ac4cd154f1eb7cf6422f0e391a71
# via -r ./requirements.in
botocore==1.17.63 \
--hash=sha256:40f13f6c9c29c307a9dc5982739e537ddce55b29787b90c3447b507e3283bcd6 \
--hash=sha256:aa88eafc6295132f4bc606f1df32b3248e0fa611724c0a216aceda767948ac75
# via
# boto3
# s3transfer
docutils==0.15.2 \
--hash=sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0 \
--hash=sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827 \
--hash=sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99
# via botocore
jmespath==0.10.0 \
--hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
--hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
# via
# boto3
# botocore
python-dateutil==2.8.1 \
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
# via botocore
s3transfer==0.3.3 \
--hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
--hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db
# via boto3
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
# via python-dateutil
urllib3==1.25.11 \
--hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \
--hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e
# via botocore
7 changes: 7 additions & 0 deletions python/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ In an ideal renaming, we'd move the packaging rules to a different package so
that @rules_python//python is only concerned with the core rules.
"""

load("//python/pip_install:requirements.bzl", "compile_pip_requirements")

package(default_visibility = ["//visibility:public"])

licenses(["notice"]) # Apache 2.0
Expand Down Expand Up @@ -145,3 +147,8 @@ exports_files([
"pip.bzl",
"whl.bzl",
])

compile_pip_requirements(
name = "requirements",
extra_args = ["--allow-unsafe"],
)
3 changes: 3 additions & 0 deletions python/pip_install/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
exports_files(["pip_compile.py"])

filegroup(
name = "distribution",
srcs = glob(["*.bzl"]) + [
"BUILD",
"pip_compile.py",
"//python/pip_install/extract_wheels:distribution",
"//python/pip_install/parse_requirements_to_bzl:distribution",
],
Expand Down
89 changes: 89 additions & 0 deletions python/pip_install/pip_compile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
"Set defaults for the pip-compile command to run it under Bazel"

import os
import sys
from shutil import copyfile

from piptools.scripts.compile import cli

if len(sys.argv) < 4:
print(
"Expected at least two arguments: requirements_in requirements_out",
file=sys.stderr,
)
sys.exit(1)

requirements_in = sys.argv.pop(1)
requirements_txt = sys.argv.pop(1)
update_target_name = sys.argv.pop(1)

# Before loading click, set the locale for its parser.
# If it leaks through to the system setting, it may fail:
# RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
# as encoding for the environment. Consult https://click.palletsprojects.com/python3/ for
# mitigation steps.
os.environ["LC_ALL"] = "C.UTF-8"
os.environ["LANG"] = "C.UTF-8"

UPDATE = True
# Detect if we are running under `bazel test`
if "TEST_TMPDIR" in os.environ:
UPDATE = False
# pip-compile wants the cache files to be writeable, but if we point
# to the real user cache, Bazel sandboxing makes the file read-only
# and we fail.
# In theory this makes the test more hermetic as well.
sys.argv.append("--cache-dir")
sys.argv.append(os.environ["TEST_TMPDIR"])
# Make a copy for pip-compile to read and mutate
requirements_out = os.path.join(
os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
)
copyfile(requirements_txt, requirements_out)

elif "BUILD_WORKING_DIRECTORY" in os.environ:
os.chdir(os.environ['BUILD_WORKING_DIRECTORY'])
else:
print(
"Expected to find BUILD_WORKING_DIRECTORY in environment",
file=sys.stderr,
)
sys.exit(1)

update_target_pkg = "/".join(requirements_in.split('/')[:-1])
# $(rootpath) in the workspace root gives ./requirements.in
if update_target_pkg == ".":
update_target_pkg = ""
update_command = "bazel run //%s:%s" % (update_target_pkg, update_target_name)

os.environ["CUSTOM_COMPILE_COMMAND"] = update_command

sys.argv.append("--generate-hashes")
sys.argv.append("--output-file")
sys.argv.append(requirements_txt if UPDATE else requirements_out)
sys.argv.append(requirements_in)

if UPDATE:
print("Updating " + requirements_txt)
cli()
else:
# cli will exit(0) on success
try:
print("Checking " + requirements_txt)
cli()
print("cl() should exit", file=sys.stderr)
sys.exit(1)
except SystemExit:
golden = open(requirements_txt).readlines()
out = open(requirements_out).readlines()
if golden != out:
import difflib

print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
print(
"Lock file out of date. Run '"
+ update_command
+ "' to update.",
file=sys.stderr,
)
sys.exit(1)
10 changes: 10 additions & 0 deletions python/pip_install/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")

_RULE_DEPS = [
(
"pypi__click",
"https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl",
"dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc",
),
(
"pypi__pip",
"https://files.pythonhosted.org/packages/fe/ef/60d7ba03b5c442309ef42e7d69959f73aacccd0d86008362a681c4698e83/pip-21.0.1-py3-none-any.whl",
"37fd50e056e2aed635dec96594606f0286640489b0db0ce7607f7e51890372d5",
),
(
"pypi__pip_tools",
"https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl",
"10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985",
),
(
"pypi__pkginfo",
"https://files.pythonhosted.org/packages/4f/3c/535287349af1b117e082f8e77feca52fbe2fdf61ef1e6da6bcc2a72a3a79/pkginfo-1.6.1-py2.py3-none-any.whl",
Expand Down
76 changes: 76 additions & 0 deletions python/pip_install/requirements.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"Rules to verify and update pip-compile locked requirements.txt"

load("//python:defs.bzl", "py_binary", "py_test")
load("//python/pip_install:repositories.bzl", "requirement")

def compile_pip_requirements(
name,
extra_args = [],
visibility = ["//visibility:private"],
requirements_in = "requirements.in",
requirements_txt = "requirements.txt",
**kwargs):
"""
Produce two targets for checking pip-compile:
- validate with `bazel test <name>_test`
- update with `bazel run <name>.update`
Args:
name: base name for generated targets
extra_args: passed to pip-compile
visibility: passed to both the _test and .update rules
requirements_in: file expressing desired dependencies
requirements_txt: result of "compiling" the requirements.in file
**kwargs: other bazel attributes passed to the "_test" rule
"""
requirements_in = kwargs.pop("requirements_in", name + ".in")
requirements_txt = kwargs.pop("requirements_locked", name + ".txt")

data = kwargs.pop("data", []) + [requirements_in, requirements_txt]

loc = "$(rootpath %s)"

# Use the Label constructor so this is expanded in the context of the file
# where it appears, which is to say, in @rules_python
pip_compile = Label("//python/pip_install:pip_compile.py")

args = [
loc % requirements_in,
loc % requirements_txt,
name + ".update",
] + extra_args

deps = [
requirement("click"),
requirement("pip"),
requirement("pip_tools"),
requirement("setuptools"),
]

attrs = {
"args": args,
"data": data,
"deps": deps,
"main": pip_compile,
"srcs": [pip_compile],
"visibility": visibility,
}

# cheap way to detect the bazel version
_bazel_version_4_or_greater = "propeller_optimize" in dir(native)

# Bazel 4.0 added the "env" attribute to py_test/py_binary
if _bazel_version_4_or_greater:
attrs["env"] = kwargs.pop("env", {})

py_binary(
name = name + ".update",
**attrs
)

py_test(
name = name + "_test",
# kwargs could contain test-specific attributes like size or timeout
**dict(attrs, **kwargs)
)
10 changes: 10 additions & 0 deletions python/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
pip==9.0.3
setuptools==44.0.0
wheel==0.30.0a0

# For tests
mock==2.0.0
# This is a transitive dependency of mock, which only appears on some python versions
# Right now our repo doesn't pin our Python interpreter, so our locked requirements
# may differ between local dev and CI, for example.
funcsigs==1.0.2
40 changes: 35 additions & 5 deletions python/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,36 @@
pip==9.0.3
setuptools==44.0.0
wheel==0.30.0a0
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bazel run //python:requirements.update
#
funcsigs==1.0.2 \
--hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
--hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
# via -r python/requirements.in
mock==2.0.0 \
--hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \
--hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
# via -r python/requirements.in
pbr==5.5.1 \
--hash=sha256:5fad80b613c402d5b7df7bd84812548b2a61e9977387a80a5fc5c396492b13c9 \
--hash=sha256:b236cde0ac9a6aedd5e3c34517b423cd4fd97ef723849da6b0d2231142d89c00
# via mock
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
# via mock
wheel==0.30.0a0 \
--hash=sha256:98f3e09b4ad7f5649a7e3d00e0e005ec1824ddcd6ec16c5086c05b1d91ada6da \
--hash=sha256:cd19aa9325d3af1c641b0a23502b12696159171d2a2f4b84308df9a075c2a4a0
# via -r python/requirements.in

# For tests
mock==2.0.0
# The following packages are considered to be unsafe in a requirements file:
pip==9.0.3 \
--hash=sha256:7bf48f9a693be1d58f49f7af7e0ae9fe29fd671cde8a55e6edca3581c4ef5796 \
--hash=sha256:c3ede34530e0e0b2381e7363aded78e0c33291654937e7373032fda04e8803e5
# via -r python/requirements.in
setuptools==44.0.0 \
--hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \
--hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008
# via -r python/requirements.in

0 comments on commit f73e99c

Please sign in to comment.