WIP: TUF Initialization using python-tuf 1.0.0

This work refactors the [Draft PR](pypi#7488) by @ woodruffw, to build a new repository tool on top of the Python-TUF Metadata API, and use it instead of the Python-TUF repository tool that was deprecated in v1.0.0. Part of pypi#10672 Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>
kairoaraujo · May 30, 2022 · 2332f8c · 2332f8c
1 parent d1fbea6
commit 2332f8c
Show file tree

Hide file tree

Showing 37 changed files with 1,522 additions and 700 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,7 +20,7 @@ docker-compose.override.yaml
 
 node_modules/
 
-dev/tuf.*
+dev/tufkeys/
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz

diff --git a/Makefile b/Makefile
@@ -1,6 +1,5 @@
 DB := example
 IPYTHON := no
-LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
 WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
@@ -96,14 +95,17 @@ initdb: .state/docker-build-web
 	$(MAKE) reindex
 
 inittuf:
-	$(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root
-	$(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot
-	$(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets
-	$(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp
-	$(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins
-	$(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n
-	$(WAREHOUSE_CLI) tuf new-repo
-	$(WAREHOUSE_CLI) tuf build-targets
+	$(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root
+	$(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets1
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets2
+	$(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp
+	$(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins
+	$(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n
+	$(WAREHOUSE_CLI) tuf dev init-repo
+	$(WAREHOUSE_CLI) tuf dev init-delegations
+	$(WAREHOUSE_CLI) tuf dev add-all-packages
+	$(WAREHOUSE_CLI) tuf dev add-all-indexes
 
 reindex: .state/docker-build-web
 	docker-compose run --rm web python -m warehouse search reindex
@@ -113,6 +115,7 @@ shell: .state/docker-build-web
 
 clean:
 	rm -rf dev/*.sql
+	rm -rf dev/tufkeys
 
 purge: stop clean
 	rm -rf .state

diff --git a/dev/environment b/dev/environment
@@ -59,4 +59,4 @@ TUF_SNAPSHOT_SECRET="an insecure private key password"
 TUF_TARGETS_SECRET="an insecure private key password"
 TUF_TIMESTAMP_SECRET="an insecure private key password"
 TUF_BINS_SECRET="an insecure private key password"
-TUF_BIN_N_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"
diff --git a/dev/tufkeys/bin-n b/dev/tufkeys/bin-n
diff --git a/dev/tufkeys/bin-n.pub b/dev/tufkeys/bin-n.pub
diff --git a/dev/tufkeys/bins b/dev/tufkeys/bins
diff --git a/dev/tufkeys/bins.pub b/dev/tufkeys/bins.pub
diff --git a/dev/tufkeys/root b/dev/tufkeys/root
diff --git a/dev/tufkeys/root.pub b/dev/tufkeys/root.pub
diff --git a/dev/tufkeys/snapshot b/dev/tufkeys/snapshot
diff --git a/dev/tufkeys/snapshot.pub b/dev/tufkeys/snapshot.pub
diff --git a/dev/tufkeys/targets b/dev/tufkeys/targets
diff --git a/dev/tufkeys/targets.pub b/dev/tufkeys/targets.pub
diff --git a/dev/tufkeys/timestamp b/dev/tufkeys/timestamp
diff --git a/dev/tufkeys/timestamp.pub b/dev/tufkeys/timestamp.pub
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,6 +4,7 @@ volumes:
   simple:
   packages:
   sponsorlogos:
+  tuf_metadata:
   vault:
 
 services:
@@ -81,6 +82,7 @@ services:
       # The :z option fixes permission issues with SELinux by setting a
       # permissive security context.
       - ./dev:/opt/warehouse/src/dev:z
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
       - ./docs:/opt/warehouse/src/docs:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./tests:/opt/warehouse/src/tests:z
@@ -89,6 +91,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
       - ./bin:/opt/warehouse/src/bin:z
       - ./requirements:/opt/warehouse/src/requirements:z
     ports:
@@ -110,6 +113,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -122,6 +126,7 @@ services:
     volumes:
       - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
     env_file: dev/environment
     environment:
       C_FORCE_ROOT: "1"

diff --git a/requirements/main.in b/requirements/main.in
@@ -48,6 +48,7 @@ requests
 requests-aws4auth
 redis>=2.8.0,<5.0.0
 rfc3986
+securesystemslib
 sentry-sdk
 setuptools
 sqlalchemy[asyncio]>=0.9,<1.5.0  # https://github.com/pypa/warehouse/pull/9228
@@ -57,7 +58,7 @@ stdlib-list
 structlog
 transaction
 trove-classifiers
-tuf==0.15.0
+tuf==1.0.0
 typeguard
 webauthn>=1.0.0,<2.0.0
 whitenoise

diff --git a/requirements/main.txt b/requirements/main.txt
@@ -1085,6 +1085,12 @@ sentry-sdk==1.5.10 \
     --hash=sha256:0a9eb20a84f4c17c08c57488d59fdad18669db71ebecb28fb0721423a33535f9 \
     --hash=sha256:972c8fe9318a415b5cf35f687f568321472ef94b36806407c370ce9c88a67f2e
     # via -r requirements/main.in
+securesystemslib==0.22.0 \
+    --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \
+    --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662
+    # via
+    #   -r requirements/main.in
+    #   tuf
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
@@ -1184,6 +1190,10 @@ trove-classifiers==2022.5.19 \
     --hash=sha256:83e281589b70e7ee23da9af84fbe3bf819341206d7115beee5c8e61efe6223f5 \
     --hash=sha256:93a2f00ace24485ca7999b8ca0bb8080bbb62f2649c99fbb4addb0217fd94843
     # via -r requirements/main.in
+tuf==1.0.0 \
+    --hash=sha256:6d967f992d494678b684a6f5f03d948130e2d9956310c7af74fa2f0296e8ec4d \
+    --hash=sha256:ec74150954d56e206fa72f3b0234ea802649370e528dea5a83e507bd408fda82
+    # via -r requirements/main.in
 typeguard==2.13.3 \
     --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \
     --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -243,8 +243,9 @@ def app_config(database):
         "docs.backend": "warehouse.packaging.services.LocalDocsStorage",
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
+        "tuf.storage_backend": "warehouse.tuf.services.LocalStorageService",
         "tuf.key_backend": "warehouse.tuf.services.LocalKeyService",
-        "tuf.repo_backend": "warehouse.tuf.services.LocalRepositoryService",
+        "tuf.repository_backend": "warehouse.tuf.services.RepositoryService",
         "malware_check.backend": (
             "warehouse.malware.services.PrinterMalwareCheckService"
         ),

diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
@@ -407,7 +407,6 @@ def __init__(self):
         ),
     ]
     assert configurator_obj.add_static_view.calls == [
-        pretend.call("tuf", "warehouse:tuf/dist/metadata.staged/"),
         pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000),
     ]
     assert configurator_obj.add_cache_buster.calls == [

diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py
@@ -26,6 +26,7 @@ def __init__(self):
                 settings={
                     "docs.url": docs_route_url,
                     "files.url": "https://files.example.com/packages/{path}",
+                    "tuf.url": "https://files.example.com/metadata/{path}",
                 }
             )
             if warehouse:
@@ -387,6 +388,7 @@ def add_policy(name, filename):
             domain=warehouse,
         ),
         pretend.call("packaging.file", "https://files.example.com/packages/{path}"),
+        pretend.call("tuf.metadata", "https://files.example.com/metadata/{path}"),
         pretend.call("ses.hook", "/_/ses-hook/", domain=warehouse),
         pretend.call("rss.updates", "/rss/updates.xml", domain=warehouse),
         pretend.call("rss.packages", "/rss/packages.xml", domain=warehouse),
@@ -475,7 +477,6 @@ def add_policy(name, filename):
             view_kw={"has_translations": True},
         ),
     ]
-
     assert config.add_redirect.calls == [
         pretend.call("/sponsor/", "/sponsors/", domain=warehouse),
         pretend.call("/u/{username}/", "/user/{username}/", domain=warehouse),
@@ -490,6 +491,11 @@ def add_policy(name, filename):
             "https://files.example.com/packages/{path}",
             domain=warehouse,
         ),
+        pretend.call(
+            "/metadata/{path:.*}",
+            "https://files.example.com/metadata/{path}",
+            domain=warehouse,
+        ),
     ]
 
     assert config.add_pypi_action_route.calls == [

diff --git a/tests/unit/tuf/__init__.py b/tests/unit/tuf/__init__.py
@@ -0,0 +1,11 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py
@@ -0,0 +1,54 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from zope.interface.verify import verifyClass
+
+from warehouse.tuf.interfaces import IKeyService
+from warehouse.tuf.services import LocalKeyService
+
+
+class TestLocalLocalKeyService:
+    def test_verify_service(self):
+        assert verifyClass(IKeyService, LocalKeyService)
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"tuf.key.path": "/tuf/key/path/"})
+        )
+        service = LocalKeyService.create_service(None, request)
+        assert service._key_path == "/tuf/key/path/"
+
+    def test_basic_init(self, db_request):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+        assert service._key_path == "/opt/warehouse/src/dev/tufkeys"
+
+    def test_get_private_key(self, db_request, monkeypatch):
+        service = LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request)
+
+        expected_priv_key_dict = {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {"public": "720a9a588deefd5...4d08984e87bfc5a18f34618e438434c7"},
+            "keyid": "2de4eb9afe9fb73...2235d3418bd63f4214d3ba7d23b516f23e",
+            "keyid_hash_algorithms": ["sha256", "sha512"],
+        }
+        db_request.registry.settings["tuf.root.secret"] = "tuf.root.secret"
+        monkeypatch.setattr(
+            "warehouse.tuf.services.import_ed25519_privatekey_from_file",
+            lambda *a, **kw: expected_priv_key_dict,
+        )
+
+        root_keyid = service.get("root", "private")
+
+        assert root_keyid == [expected_priv_key_dict]