From 479eaf16bd94143f21ec17f53bf66c099982a197 Mon Sep 17 00:00:00 2001 From: Laurentiu Niculae Date: Fri, 7 Apr 2023 12:57:03 +0300 Subject: [PATCH 1/2] refactor(artifact): remove oci artifact support - add header to referrers call to indicated applied artifact type filters Signed-off-by: Laurentiu Niculae --- codecov.yml | 1 + errors/errors.go | 1 - examples/config-dynamodb.json | 1 - go.mod | 2 +- go.sum | 33 +- golangcilint.yaml | 2 +- pkg/api/controller_test.go | 140 +----- pkg/api/routes.go | 3 +- pkg/cli/client.go | 6 +- pkg/cli/client_utils_test.go | 79 +--- pkg/cli/cve_cmd_test.go | 8 +- pkg/cli/image_cmd_test.go | 30 +- pkg/common/common.go | 9 + pkg/extensions/lint/lint.go | 3 +- pkg/extensions/search/convert/convert_test.go | 2 +- pkg/extensions/search/cve/cve_test.go | 2 +- pkg/extensions/search/resolver_test.go | 2 +- pkg/extensions/search/search_test.go | 133 +++--- pkg/extensions/sync/signatures.go | 132 +----- pkg/extensions/sync/sync_internal_test.go | 117 +++-- pkg/extensions/sync/sync_test.go | 446 ++++-------------- pkg/extensions/sync/utils.go | 21 - pkg/log/log.go | 2 +- pkg/meta/bolt/buckets.go | 1 - pkg/meta/dynamo/parameters.go | 2 +- .../repodb/boltdb-wrapper/boltdb_wrapper.go | 48 -- .../boltdb-wrapper/boltdb_wrapper_test.go | 12 - .../dynamodb-wrapper/dynamo_internal_test.go | 3 - .../repodb/dynamodb-wrapper/dynamo_test.go | 108 +---- .../repodb/dynamodb-wrapper/dynamo_wrapper.go | 84 ---- pkg/meta/repodb/repodb.go | 10 - pkg/meta/repodb/repodb_test.go | 62 +-- .../repodb/repodbfactory/repodb_factory.go | 4 - .../repodbfactory/repodb_factory_test.go | 1 - pkg/meta/repodb/storage_parsing.go | 64 +-- pkg/meta/repodb/storage_parsing_test.go | 8 +- pkg/meta/update.go | 2 +- pkg/meta/update_test.go | 28 +- pkg/meta/version/version_test.go | 1 - pkg/storage/common.go | 176 +++---- pkg/storage/common_test.go | 40 +- pkg/storage/local/local.go | 50 +- pkg/storage/s3/s3.go | 15 +- pkg/storage/s3/s3_test.go | 38 -- pkg/test/common.go | 121 +---- pkg/test/common_test.go | 12 - pkg/test/mocks/repo_db_mock.go | 20 - pkg/test/oci-layout/oci_layout_test.go | 2 +- pkg/test/test_http_server.go | 51 ++ test/blackbox/annotations.bats | 2 +- test/blackbox/cloud-only.bats | 1 - test/blackbox/pushpull.bats | 6 +- test/blackbox/referrers.bats | 18 - test/blackbox/sync.bats | 8 +- 54 files changed, 587 insertions(+), 1586 deletions(-) create mode 100644 pkg/test/test_http_server.go diff --git a/codecov.yml b/codecov.yml index 1aa3df7f4..915464c2f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,3 +6,4 @@ ignore: - "./cmd/zb/*.go" - "./pkg/test/mocks/*.go" - "./swagger/*.go" + - "./pkg/test/test_http_server.go" diff --git a/errors/errors.go b/errors/errors.go index 378c3b73b..9c974f456 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -63,7 +63,6 @@ var ( ErrManifestConflict = errors.New("manifest: multiple manifests found") ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference") ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest") - ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest") ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest") ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name") ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names") diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json index 51a2f5983..d154dc652 100644 --- a/examples/config-dynamodb.json +++ b/examples/config-dynamodb.json @@ -20,7 +20,6 @@ "cacheTablename": "ZotBlobTable", "repoMetaTablename": "ZotRepoMetadataTable", "manifestDataTablename": "ZotManifestDataTable", - "artifactDataTablename": "ZotArtifactDataTable", "userDataTablename": "ZotUserDataTable", "versionTablename": "ZotVersion" } diff --git a/go.mod b/go.mod index a2afec645..919961b1a 100644 --- a/go.mod +++ b/go.mod @@ -462,7 +462,7 @@ require ( ) replace ( - github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef + github.com/opencontainers/image-spec => github.com/project-zot/image-spec v0.0.0-20230428141304-a263befc67ea // v0.3.1-0.20230104082527-d6f58551be3f is taken from github.com/moby/buildkit v0.11.0 // spdx logic write on v0.3.0 and incompatible with v0.3.1-0.20230104082527-d6f58551be3f // see https://github.com/aquasecurity/trivy/commit/2033e05b6b563aa4bc46d6234b3657a54e71ee46 diff --git a/go.sum b/go.sum index db1152a81..6c09fb7bf 100644 --- a/go.sum +++ b/go.sum @@ -244,7 +244,6 @@ github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzS github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -480,6 +479,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -512,6 +512,7 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chartmuseum/auth v0.5.0 h1:ENNmoxvjxcR/JR0HrghAEtGQe7hToMNj16+UoS5CK9Y= github.com/chartmuseum/auth v0.5.0/go.mod h1:BvoSXHyvbsq+/bbhNgVTDQsModM+HERBTNY5o9Vyrig= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= @@ -521,6 +522,7 @@ github.com/chrismellard/docker-credential-acr-env v0.0.0-20220119192733-fe33c00c github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/clbanning/mxj/v2 v2.5.6 h1:Jm4VaCI/+Ug5Q57IzEoZbwx4iQFA6wkXv72juUSeK+g= github.com/clbanning/mxj/v2 v2.5.6/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= @@ -543,6 +545,7 @@ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= @@ -587,6 +590,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 h1:vU+EP9ZuFUCYE0NYLwTSob+3LNEJATzNfP/DC7SWGWI= github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -662,7 +666,7 @@ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yi github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -1182,7 +1186,8 @@ github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlW github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1190,6 +1195,7 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -1250,6 +1256,7 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= @@ -1316,18 +1323,18 @@ github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230117141039-067a0 github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230117141039-067a0f5b0e25/go.mod h1:uLvbyisMXc6uWXBmKrPegq9igec91coS/kGM2sHL660= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221020182949-4df8887994e8 h1:l9vfzobI7tZtG164u1Jf6NqDErHZoqAw8rlvBYQJpVI= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221020182949-4df8887994e8/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0-rc.2 h1:ucBtEms2tamYYW/SvGpvq9yUN0NEVL6oyLEwDcTSrk8= github.com/opencontainers/runtime-spec v1.1.0-rc.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opencontainers/umoci v0.4.8-0.20210922062158-e60a0cc726e6 h1:yRmw/21YcR/78dRoFyWNJ6c3VxOXqe97PhNZI0Ib7bM= +github.com/opencontainers/umoci v0.4.8-0.20210922062158-e60a0cc726e6/go.mod h1:kO0Bh4G4BZUh2QSlqsCR/OCtdqLjmt3mvD6okZhMBlU= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/oras-project/artifacts-spec v1.0.0-rc.2 h1:9SMCNSxkJEHqWGDiMCuy6TXHgvjgwXGdXZZGXLKQvVE= @@ -1368,8 +1375,8 @@ github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8 github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= -github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef h1:PMJXsXS3x5tiwsRrFgMSnjKmeH5c9ppFXrS5NKdIPEo= -github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef/go.mod h1:smZzWUkoRRHivXkcHQN7n+KGHVfR+HKJrtCaI8cMifg= +github.com/project-zot/image-spec v0.0.0-20230428141304-a263befc67ea h1:emnKit1I3avTxHcrDfGWXb9U+dG866wv2y8WcgAj3ys= +github.com/project-zot/image-spec v0.0.0-20230428141304-a263befc67ea/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1432,6 +1439,7 @@ github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcL github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/secure-systems-lab/go-securesystemslib v0.5.0 h1:oTiNu0QnulMQgN/hLK124wJD/r2f9ZhIUuKIeBsCBT8= github.com/secure-systems-lab/go-securesystemslib v0.5.0/go.mod h1:uoCqUC0Ap7jrBSEanxT+SdACYJTVplRXWLkGMuDjXqk= @@ -1457,6 +1465,7 @@ github.com/sigstore/sigstore v1.6.3/go.mod h1:BpLOp7N2IECbatk4sXE2toY2krw615NmwA github.com/sigstore/timestamp-authority v1.0.0 h1:UisIGA9anE6xyWctJaiEmsk1M+txQ6kAPQDCyu+ieQw= github.com/sigstore/timestamp-authority v1.0.0/go.mod h1:DvyJt4w4by/hsutEi+th0MsK2tB3IwCPVr9ACrJAgIA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -1583,9 +1592,9 @@ github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.10.3/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/vbatts/go-mtree v0.5.0/go.mod h1:7JbaNHyBMng+RP8C3Q4E+4Ca8JnGQA2R/MB+jb4tSOk= github.com/vbatts/go-mtree v0.5.2 h1:d8SAbLJiR1cR3pe1J+FBaalRkCQw95gP12/P+a9PUcA= github.com/vbatts/go-mtree v0.5.2/go.mod h1:e0NDJ+bT3jG7ZINeB9HR5AxTvjskCsOR54+9KoaXyDc= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= @@ -1718,6 +1727,7 @@ golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1939,6 +1949,7 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201005172224-997123666555/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1957,6 +1968,7 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1987,7 +1999,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/golangcilint.yaml b/golangcilint.yaml index e06ce5738..e2990605a 100644 --- a/golangcilint.yaml +++ b/golangcilint.yaml @@ -52,8 +52,8 @@ linters-settings: gomoddirectives: replace-allow-list: - helm.sh/helm/v3 - - github.com/opencontainers/umoci - github.com/spdx/tools-golang + - github.com/opencontainers/image-spec issues: exclude-rules: - path: pkg/extensions/search/schema.resolvers.go diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 2b1e445fc..a32bcd1b8 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -51,7 +51,6 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" "zotregistry.io/zot/pkg/storage" - storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" ) @@ -157,7 +156,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -173,7 +171,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -188,7 +185,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -222,7 +218,6 @@ func TestCreateRepoDBDriver(t *testing.T) { "cachetablename": "BlobTable", "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "userdatatablename": "UserDatatable", } @@ -236,7 +231,6 @@ func TestCreateRepoDBDriver(t *testing.T) { "cachetablename": "", "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "userDataTablename": "ZotUserDataTable", "versiontablename": 1, } @@ -252,7 +246,6 @@ func TestCreateRepoDBDriver(t *testing.T) { "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", "indexdatatablename": "IndexDataTable", - "artifactdatatablename": "ArtifactDataTable", "userdatatablename": "ZotUserDataTable", "versiontablename": "1", } @@ -4312,138 +4305,15 @@ func TestArtifactReferences(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex) - }) - }) - - Convey("Validate Artifact Manifest Reference", func() { - resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - - var referrers ispec.Index - err = json.Unmarshal(resp.Body(), &referrers) - So(err, ShouldBeNil) - So(referrers.Manifests, ShouldBeEmpty) - - // now upload a reference - - // upload image config blob - resp, err = resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := test.Location(baseURL, resp) - cblob, cdigest := test.GetEmptyImageConfig() - - resp, err = resty.R(). - SetContentLength(true). - SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). - SetHeader("Content-Type", "application/octet-stream"). - SetQueryParam("digest", cdigest.String()). - SetBody(cblob). - Put(loc) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - - // create a artifact - manifest := ispec.Artifact{ - MediaType: ispec.MediaTypeArtifactManifest, - ArtifactType: artifactType, - Blobs: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: digest, - Size: int64(len(content)), - }, - }, - Subject: &ispec.Descriptor{ - MediaType: ispec.MediaTypeImageManifest, - Digest: digest, - Size: int64(len(content)), - }, - Annotations: map[string]string{ - "key": "val", - }, - } - Convey("Using invalid content", func() { - content := []byte("invalid data") - So(err, ShouldBeNil) - mdigest := godigest.FromBytes(content) - So(mdigest, ShouldNotBeNil) - resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest) - - // unknown repo will return status not found - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", "unknown", digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType) - resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}). - Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - }) - Convey("Using valid content", func() { - content, err = json.Marshal(manifest) - So(err, ShouldBeNil) - mdigest := godigest.FromBytes(content) - So(mdigest, ShouldNotBeNil) - resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - - resp, err = resty.R().SetQueryParams(map[string]string{"artifact": "invalid"}). - Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - - resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": "invalid"}). - Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - - resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}). - Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String())) + resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType + + ",otherArtType"}).Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, + digest.String())) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex) - - var index ispec.Index - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(index.Manifests, ShouldNotBeEmpty) - So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty) - - // filter by multiple artifactTypes - req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, - baseURL+fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()), nil) - So(err, ShouldBeNil) - values := url.Values{} - values.Add("artifactType", artifactType) - values.Add("artifactType", "foobar") - req.URL.RawQuery = values.Encode() - rsp, err := http.DefaultClient.Do(req) - So(err, ShouldBeNil) - defer rsp.Body.Close() - So(rsp.StatusCode, ShouldEqual, http.StatusOK) - So(rsp.Header.Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex) - body, err := io.ReadAll(rsp.Body) - So(err, ShouldBeNil) - err = json.Unmarshal(body, &index) - So(err, ShouldBeNil) - So(index.Manifests, ShouldNotBeEmpty) - So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty) - So(len(strings.Split(index.Annotations[storageConstants.ReferrerFilterAnnotation], ",")), ShouldEqual, 2) + So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType+",otherArtType") }) }) }) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 82fd50525..b2c3758b6 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -408,7 +408,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http. } if rh.c.RepoDB != nil { - err := meta.OnGetManifest(name, reference, digest, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) + err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log) if errors.Is(err, zerr.ErrOrphanSignature) { rh.c.Log.Error().Err(err).Msg("image is an orphan signature") @@ -514,6 +514,7 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http return } + response.Header().Set("OCI-Filters-Applied", strings.Join(artifactTypes, ",")) WriteData(response, http.StatusOK, ispec.MediaTypeImageIndex, out) } diff --git a/pkg/cli/client.go b/pkg/cli/client.go index e2790e69c..0cb1e0fcc 100644 --- a/pkg/cli/client.go +++ b/pkg/cli/client.go @@ -490,10 +490,8 @@ func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf se return false } - for _, reference := range referrers.Manifests { - if reference.ArtifactType == notreg.ArtifactTypeNotation { - return true - } + if len(referrers.Manifests) > 0 { + return true } return false diff --git a/pkg/cli/client_utils_test.go b/pkg/cli/client_utils_test.go index 3580d19a6..20bf2fb7b 100644 --- a/pkg/cli/client_utils_test.go +++ b/pkg/cli/client_utils_test.go @@ -5,7 +5,6 @@ package cli //nolint:testpackage import ( "context" - "errors" "fmt" "io" "net/http" @@ -20,48 +19,6 @@ import ( "zotregistry.io/zot/pkg/test" ) -type RouteHandler struct { - Route string - // HandlerFunc is the HTTP handler function that receives a writer for output and an HTTP request as input. - HandlerFunc http.HandlerFunc - // AllowedMethods specifies the HTTP methods allowed for the current route. - AllowedMethods []string -} - -// Routes is a map that associates HTTP paths to their corresponding HTTP handlers. -type HTTPRoutes []RouteHandler - -func StartTestHTTPServer(routes HTTPRoutes, port string) *http.Server { - baseURL := test.GetBaseURL(port) - mux := mux.NewRouter() - - mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte("{}")) - if err != nil { - return - } - }).Methods(http.MethodGet) - - for _, routeHandler := range routes { - mux.HandleFunc(routeHandler.Route, routeHandler.HandlerFunc).Methods(routeHandler.AllowedMethods...) - } - - server := &http.Server{ //nolint:gosec - Addr: fmt.Sprintf(":%s", port), - Handler: mux, - } - - go func() { - if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { - return - } - }() - - test.WaitTillServerReady(baseURL + "/test") - - return server -} - func getDefaultSearchConf(baseURL string) searchConfig { verifyTLS := false debug := false @@ -81,7 +38,7 @@ func getDefaultSearchConf(baseURL string) searchConfig { func TestDoHTTPRequest(t *testing.T) { Convey("doHTTPRequest nil result pointer", t, func() { port := test.GetFreePort() - server := StartTestHTTPServer(nil, port) + server := test.StartTestHTTPServer(nil, port) defer server.Close() url := fmt.Sprintf("http://127.0.0.1:%s/asd", port) @@ -93,7 +50,7 @@ func TestDoHTTPRequest(t *testing.T) { Convey("doHTTPRequest bad return json", t, func() { port := test.GetFreePort() - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/test", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -135,7 +92,7 @@ func TestDoHTTPRequest(t *testing.T) { searchConf := getDefaultSearchConf(baseURL) // 404 erorr will appear - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() URL := baseURL + "/v2/repo/manifests/tag" @@ -158,7 +115,7 @@ func TestDoHTTPRequest(t *testing.T) { searchConf := getDefaultSearchConf(baseURL) Convey("makeGETRequest manifest error, context is done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() ctx, cancel := context.WithCancel(context.Background()) @@ -172,7 +129,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("makeGETRequest manifest error, context is not done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() _, err := fetchManifestStruct(context.Background(), "repo", "tag", searchConf, @@ -182,7 +139,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("makeGETRequest config error, context is not done", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -203,7 +160,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("Platforms on config", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -258,7 +215,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("fetchImageIndexStruct no errors", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(writer http.ResponseWriter, req *http.Request) { @@ -327,7 +284,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("fetchImageIndexStruct makeGETRequest errors context done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() ctx, cancel := context.WithCancel(context.Background()) @@ -349,7 +306,7 @@ func TestDoHTTPRequest(t *testing.T) { }) Convey("fetchImageIndexStruct makeGETRequest errors context not done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() URL := baseURL + "/v2/repo/manifests/indexRef" @@ -384,7 +341,7 @@ func TestDoJobErrors(t *testing.T) { reqPool.wtgrp.Add(1) Convey("Do Job makeHEADRequest error context done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() URL := baseURL + "/v2/repo/manifests/manifestRef" @@ -404,7 +361,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Do Job makeHEADRequest error context not done", func() { - server := StartTestHTTPServer(HTTPRoutes{}, port) + server := test.StartTestHTTPServer(test.HTTPRoutes{}, port) defer server.Close() URL := baseURL + "/v2/repo/manifests/manifestRef" @@ -426,7 +383,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Do Job fetchManifestStruct errors context canceled", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -459,7 +416,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Do Job fetchManifestStruct errors context not canceled", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -493,7 +450,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Do Job fetchIndexStruct errors context canceled", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -526,7 +483,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Do Job fetchIndexStruct errors context not canceled", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -559,7 +516,7 @@ func TestDoJobErrors(t *testing.T) { So(result.StrValue, ShouldResemble, "") }) Convey("Do Job fetchIndexStruct not supported content type", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { @@ -589,7 +546,7 @@ func TestDoJobErrors(t *testing.T) { }) Convey("Media type is MediaTypeImageIndex image.string erorrs", func() { - server := StartTestHTTPServer(HTTPRoutes{ + server := test.StartTestHTTPServer(test.HTTPRoutes{ { Route: "/v2/{name}/manifests/{reference}", HandlerFunc: func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 3f30409c2..36594f6c7 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -713,7 +713,7 @@ func TestServerCVEResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") str = strings.TrimSpace(str) So(err, ShouldBeNil) - So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 82836dd7 false 548B") + So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 40d1f749 false 605B") }) Convey("Test images by CVE ID - GQL - invalid CVE ID", t, func() { @@ -829,7 +829,7 @@ func TestServerCVEResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") So(err, ShouldBeNil) So(strings.TrimSpace(str), ShouldEqual, - "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 82836dd7 false 548B") + "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 40d1f749 false 605B") }) Convey("Test CVE by name and CVE ID - GQL - invalid name and CVE ID", t, func() { @@ -907,7 +907,7 @@ func TestServerCVEResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") str = strings.TrimSpace(str) So(err, ShouldBeNil) - So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 82836dd7 false 548B") + So(str, ShouldEqual, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 40d1f749 false 605B") }) Convey("Test images by CVE ID - invalid CVE ID", t, func() { @@ -992,7 +992,7 @@ func TestServerCVEResponse(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") So(err, ShouldBeNil) So(strings.TrimSpace(str), ShouldEqual, - "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 82836dd7 false 548B") + "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 40d1f749 false 605B") }) Convey("Test CVE by name and CVE ID - invalid name and CVE ID", t, func() { diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 6e52b065f..f24be698c 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -333,7 +333,7 @@ func TestSignature(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 6742241d true 447B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 8e59ed3b true 504B") t.Log("Test getting all images using rest calls to get catalog and individual manifests") cmd = MockNewImageCommand(new(searchService)) @@ -346,7 +346,7 @@ func TestSignature(t *testing.T) { str = space.ReplaceAllString(buff.String(), " ") actual = strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 6742241d true 447B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 linux/amd64 8e59ed3b true 504B") err = os.Chdir(currentWorkingDir) So(err, ShouldBeNil) @@ -410,7 +410,7 @@ func TestSignature(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 6742241d true 447B") + So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 8e59ed3b true 504B") t.Log("Test getting all images using rest calls to get catalog and individual manifests") cmd = MockNewImageCommand(new(searchService)) @@ -423,7 +423,7 @@ func TestSignature(t *testing.T) { str = space.ReplaceAllString(buff.String(), " ") actual = strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 6742241d true 447B") + So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 8e59ed3b true 504B") err = os.Chdir(currentWorkingDir) So(err, ShouldBeNil) @@ -1333,13 +1333,13 @@ func runDisplayIndexTests(baseURL string) { actual := strings.TrimSpace(str) // Actual cli output should be something similar to (order of images may differ): // IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE - // repo multi-arch * 46b78b06 false 1.4kB - // linux/amd64 97b0d65c false 577B + // repo multi-arch * 59b25ae4 false 1.5kB + // linux/amd64 97b0d65c false 634B // windows/arm64/v6 dcfa3a9c false 444B So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo multi-arch * 46b78b06 false 1.4kB ") - So(actual, ShouldContainSubstring, "linux/amd64 97b0d65c false 577B ") - So(actual, ShouldContainSubstring, "windows/arm64/v6 dcfa3a9c false 444B") + So(actual, ShouldContainSubstring, "repo multi-arch * 59b25ae4 false 1.5kB ") + So(actual, ShouldContainSubstring, "linux/amd64 2ab1a275 false 634B ") + So(actual, ShouldContainSubstring, "windows/arm64/v6 55fdd23a false 501B") }) Convey("Test Image Index Verbose", func() { @@ -1361,18 +1361,18 @@ func runDisplayIndexTests(baseURL string) { actual := strings.TrimSpace(str) // Actual cli output should be something similar to (order of images may differ): // IMAGE NAME TAG OS/ARCH DIGEST CONFIG SIGNED LAYERS SIZE - // repo multi-arch * 46b78b06 false 1.4kB - // linux/amd64 97b0d65c 58cc9abe false 577B + // repo multi-arch * 59b25ae4 false 1.5kB + // linux/amd64 2ab1a275 58cc9abe false 634B // cbb5b121 4B // a00291e8 4B - // windows/arm64/v6 dcfa3a9c 5132a1cd false 444B + // windows/arm64/v6 55fdd23a 5132a1cd false 501B // 7d08ce29 4B So(actual, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST CONFIG SIGNED LAYERS SIZE") - So(actual, ShouldContainSubstring, "repo multi-arch * 46b78b06 false 1.4kB") - So(actual, ShouldContainSubstring, "linux/amd64 97b0d65c 58cc9abe false 577B") + So(actual, ShouldContainSubstring, "repo multi-arch * 59b25ae4 false 1.5kB") + So(actual, ShouldContainSubstring, "linux/amd64 2ab1a275 58cc9abe false 634B") So(actual, ShouldContainSubstring, "cbb5b121 4B") So(actual, ShouldContainSubstring, "a00291e8 4B") - So(actual, ShouldContainSubstring, "windows/arm64/v6 dcfa3a9c 5132a1cd false 444B") + So(actual, ShouldContainSubstring, "windows/arm64/v6 55fdd23a 5132a1cd false 501B") So(actual, ShouldContainSubstring, "7d08ce29 4B") }) } diff --git a/pkg/common/common.go b/pkg/common/common.go index d99f6f2e7..6ff4f4aa7 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -17,6 +17,7 @@ import ( "unicode/utf8" "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.io/zot/pkg/log" ) @@ -270,3 +271,11 @@ func DContains(slice []digest.Digest, item digest.Digest) bool { return false } + +func GetManifestArtifactType(manifestContent ispec.Manifest) string { + if manifestContent.ArtifactType != "" { + return manifestContent.ArtifactType + } + + return manifestContent.Config.MediaType +} diff --git a/pkg/extensions/lint/lint.go b/pkg/extensions/lint/lint.go index 348649133..2e7cf7c30 100644 --- a/pkg/extensions/lint/lint.go +++ b/pkg/extensions/lint/lint.go @@ -78,7 +78,8 @@ func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godi content, err = imgStore.GetBlobContent(repo, configDigest) if err != nil { - linter.log.Error().Err(err).Msg("linter: couldn't get config JSON " + configDigest.String()) + linter.log.Error().Err(err).Msg("linter: couldn't get config JSON " + + configDigest.String()) return false, err } diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index d2b7a2ce1..316ac8894 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -183,7 +183,7 @@ func TestConvertErrors(t *testing.T) { Referrers: map[string][]repodb.ReferrerInfo{}, }, repodb.ManifestMetadata{ - ManifestBlob: []byte("{}"), + ManifestBlob: []byte(`{}`), ConfigBlob: []byte("bad json"), }, nil, diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index bee06ab87..9ca7142f1 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -923,7 +923,7 @@ func TestCVEStruct(t *testing.T) { }, Layers: []ispec.Descriptor{ { - MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, + MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck Size: 0, Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), }, diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index fb631317e..23c8ee21a 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -1449,7 +1449,7 @@ func TestGetReferrers(t *testing.T) { Convey("GetReferrers return index of descriptors", func() { testLogger := log.NewLogger("debug", "") referrerDescriptor := ispec.Descriptor{ - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, ArtifactType: "com.artifact.test", Size: 403, Digest: godigest.FromString("test"), diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index cea9d648e..2c82a48d0 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -21,6 +21,7 @@ import ( dbTypes "github.com/aquasecurity/trivy-db/pkg/types" "github.com/gobwas/glob" regTypes "github.com/google/go-containerregistry/pkg/v1/types" + notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -31,7 +32,7 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/api/constants" - "zotregistry.io/zot/pkg/common" + zcommon "zotregistry.io/zot/pkg/common" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" @@ -97,7 +98,7 @@ type ReferrersResp struct { } type ReferrersResult struct { - Referrers []common.Referrer `json:"referrers"` + Referrers []zcommon.Referrer `json:"referrers"` } type GlobalSearchResultResp struct { GlobalSearchResult GlobalSearchResult `json:"data"` @@ -109,24 +110,24 @@ type GlobalSearchResult struct { } type GlobalSearch struct { - Images []common.ImageSummary `json:"images"` - Repos []common.RepoSummary `json:"repos"` - Layers []common.LayerSummary `json:"layers"` - Page repodb.PageInfo `json:"page"` + Images []zcommon.ImageSummary `json:"images"` + Repos []zcommon.RepoSummary `json:"repos"` + Layers []zcommon.LayerSummary `json:"layers"` + Page repodb.PageInfo `json:"page"` } type ExpandedRepoInfo struct { - RepoInfo common.RepoInfo `json:"expandedRepoInfo"` + RepoInfo zcommon.RepoInfo `json:"expandedRepoInfo"` } type PaginatedReposResult struct { - Results []common.RepoSummary `json:"results"` - Page repodb.PageInfo `json:"page"` + Results []zcommon.RepoSummary `json:"results"` + Page repodb.PageInfo `json:"page"` } type PaginatedImagesResult struct { - Results []common.ImageSummary `json:"results"` - Page repodb.PageInfo `json:"page"` + Results []zcommon.ImageSummary `json:"results"` + Page repodb.PageInfo `json:"page"` } //nolint:tagliatelle // graphQL schema @@ -140,7 +141,7 @@ type ErrorGQL struct { } type SingleImageSummary struct { - ImageSummary common.ImageSummary `json:"Image"` //nolint:tagliatelle + ImageSummary zcommon.ImageSummary `json:"Image"` //nolint:tagliatelle } type ImageSummaryResult struct { SingleImageSummary SingleImageSummary `json:"data"` @@ -189,7 +190,7 @@ func readFileAndSearchString(filePath string, stringToMatch string, timeout time } func verifyRepoSummaryFields(t *testing.T, - actualRepoSummary, expectedRepoSummary *common.RepoSummary, + actualRepoSummary, expectedRepoSummary *zcommon.RepoSummary, ) { t.Helper() @@ -218,7 +219,7 @@ func verifyRepoSummaryFields(t *testing.T, } func verifyImageSummaryFields(t *testing.T, - actualImageSummary, expectedImageSummary *common.ImageSummary, + actualImageSummary, expectedImageSummary *zcommon.ImageSummary, ) { t.Helper() @@ -972,30 +973,37 @@ func TestGetReferrersGQL(t *testing.T) { artifactContentBlobDigest := godigest.FromBytes(artifactContentBlob) artifactType := "com.artifact.test" - err = UploadBlob(baseURL, repo, artifactContentBlob, artifactContentType) - So(err, ShouldBeNil) - - artifact := &ispec.Artifact{ - Blobs: []ispec.Descriptor{ - { - MediaType: artifactContentType, - Digest: artifactContentBlobDigest, - Size: artifactContentBlobSize, + artifactImg := Image{ + Manifest: ispec.Manifest{ + Layers: []ispec.Descriptor{ + { + MediaType: artifactContentType, + Digest: artifactContentBlobDigest, + Size: artifactContentBlobSize, + }, + }, + Subject: subjectDescriptor, + ArtifactType: artifactType, + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeScratch, + Digest: ispec.ScratchDescriptor.Digest, + Data: ispec.ScratchDescriptor.Data, + }, + MediaType: ispec.MediaTypeImageManifest, + Annotations: map[string]string{ + "com.artifact.format": "test", }, }, - Subject: subjectDescriptor, - ArtifactType: artifactType, - MediaType: ispec.MediaTypeArtifactManifest, - Annotations: map[string]string{ - "com.artifact.format": "test", - }, + Config: ispec.Image{}, + Layers: [][]byte{artifactContentBlob}, } - artifactManifestBlob, err := json.Marshal(artifact) + artifactManifestBlob, err := json.Marshal(artifactImg.Manifest) So(err, ShouldBeNil) artifactManifestDigest := godigest.FromBytes(artifactManifestBlob) + artifactImg.Reference = artifactManifestDigest.String() - err = UploadArtifactManifest(artifact, nil, baseURL, repo) + err = UploadImage(artifactImg, baseURL, repo) So(err, ShouldBeNil) gqlQuery := ` @@ -1029,7 +1037,7 @@ func TestGetReferrersGQL(t *testing.T) { So(err, ShouldBeNil) So(referrersResp.Errors, ShouldBeNil) So(referrersResp.ReferrersResult.Referrers[0].ArtifactType, ShouldEqual, artifactType) - So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest) + So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeImageManifest) So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Key, ShouldEqual, "com.artifact.format") So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Value, ShouldEqual, "test") @@ -1505,6 +1513,7 @@ func TestExpandedRepoInfo(t *testing.T) { }) Convey("Test expanded repo info with tagged referrers", t, func() { + const test = "test" rootDir := t.TempDir() port := GetFreePort() baseURL := GetBaseURL(port) @@ -1524,7 +1533,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlrManager.StartAndWait(port) defer ctlrManager.StopServer() - image, err := GetRandomImage("test") + image, err := GetRandomImage(test) So(err, ShouldBeNil) manifestDigest, err := image.Digest() So(err, ShouldBeNil) @@ -1532,17 +1541,15 @@ func TestExpandedRepoInfo(t *testing.T) { err = UploadImage(image, baseURL, "repo") So(err, ShouldBeNil) - referrer, err := GetRandomArtifact(&ispec.Descriptor{ - Digest: manifestDigest, - MediaType: ispec.MediaTypeImageManifest, - }) + referrer, err := GetImageWithSubject(manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) tag := "test-ref-tag" - err = UploadArtifactManifest(&referrer.Manifest, &tag, baseURL, "repo") + referrer.Reference = tag + err = UploadImage(referrer, baseURL, "repo") So(err, ShouldBeNil) - // ------- Make the call to GQL and see that it doesn't crash and that the referrer isn't in the list of tags + // ------- Make the call to GQL and see that it doesn't crash responseStruct := &ExpandedRepoInfoResp{} query := ` { @@ -1564,10 +1571,23 @@ func TestExpandedRepoInfo(t *testing.T) { err = json.Unmarshal(resp.Body(), responseStruct) So(err, ShouldBeNil) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 1) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 2) repoInfo := responseStruct.ExpandedRepoInfo.RepoInfo - So(repoInfo.ImageSummaries[0].Tag, ShouldEqual, "test") + + foundTagTest := false + foundTagRefTag := false + + for _, imgSum := range repoInfo.ImageSummaries { + switch imgSum.Tag { + case test: + foundTagTest = true + case "test-ref-tag": + foundTagRefTag = true + } + } + + So(foundTagTest || foundTagRefTag, ShouldEqual, true) }) Convey("Test image tags order", t, func() { @@ -3241,8 +3261,8 @@ func TestGlobalSearch(t *testing.T) { repos, err := olu.GetRepositories() So(err, ShouldBeNil) - allExpectedRepoInfoMap := make(map[string]common.RepoInfo) - allExpectedImageSummaryMap := make(map[string]common.ImageSummary) + allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo) + allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary) for _, repo := range repos { repoInfo, err := olu.GetExpandedRepoInfo(repo) So(err, ShouldBeNil) @@ -3310,8 +3330,8 @@ func TestGlobalSearch(t *testing.T) { t.Logf("returned layers: %v", responseStruct.GlobalSearchResult.GlobalSearch.Layers) So(responseStruct.GlobalSearchResult.GlobalSearch.Layers, ShouldBeEmpty) - newestImageMap := make(map[string]common.ImageSummary) - actualRepoMap := make(map[string]common.RepoSummary) + newestImageMap := make(map[string]zcommon.ImageSummary) + actualRepoMap := make(map[string]zcommon.RepoSummary) for _, repo := range responseStruct.GlobalSearchResult.GlobalSearch.Repos { newestImageMap[repo.Name] = repo.NewestImage actualRepoMap[repo.Name] = repo @@ -3570,8 +3590,8 @@ func TestGlobalSearch(t *testing.T) { repos, err := olu.GetRepositories() So(err, ShouldBeNil) - allExpectedRepoInfoMap := make(map[string]common.RepoInfo) - allExpectedImageSummaryMap := make(map[string]common.ImageSummary) + allExpectedRepoInfoMap := make(map[string]zcommon.RepoInfo) + allExpectedImageSummaryMap := make(map[string]zcommon.ImageSummary) for _, repo := range repos { repoInfo, err := olu.GetExpandedRepoInfo(repo) So(err, ShouldBeNil) @@ -3637,8 +3657,8 @@ func TestGlobalSearch(t *testing.T) { t.Logf("returned layers: %v", responseStruct.GlobalSearchResult.GlobalSearch.Layers) So(responseStruct.GlobalSearchResult.GlobalSearch.Layers, ShouldBeEmpty) - newestImageMap := make(map[string]common.ImageSummary) - actualRepoMap := make(map[string]common.RepoSummary) + newestImageMap := make(map[string]zcommon.ImageSummary) + actualRepoMap := make(map[string]zcommon.RepoSummary) for _, repo := range responseStruct.GlobalSearchResult.GlobalSearch.Repos { newestImageMap[repo.Name] = repo.NewestImage actualRepoMap[repo.Name] = repo @@ -5626,10 +5646,17 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { signatureReference := "" - var sigManifestContent ispec.Artifact + var sigManifestContent ispec.Manifest for _, manifest := range indexContent.Manifests { - if manifest.MediaType == ispec.MediaTypeArtifactManifest { + manifestBlob, _, _, err := storage.GetImageManifest(repo, manifest.Digest.String()) + So(err, ShouldBeNil) + var manifestContent ispec.Manifest + + err = json.Unmarshal(manifestBlob, &manifestContent) + So(err, ShouldBeNil) + + if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { signatureReference = manifest.Digest.String() manifestBlob, _, _, err := storage.GetImageManifest(repo, signatureReference) So(err, ShouldBeNil) @@ -6233,11 +6260,11 @@ func TestImageSummary(t *testing.T) { So(imgSummary.Vulnerabilities.Count, ShouldEqual, 0) So(imgSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "") So(len(imgSummary.Referrers), ShouldEqual, 1) - So(imgSummary.Referrers[0], ShouldResemble, common.Referrer{ + So(imgSummary.Referrers[0], ShouldResemble, zcommon.Referrer{ MediaType: ispec.MediaTypeImageManifest, ArtifactType: "test.artifact.type", Digest: referrerManifestDigest.String(), - Annotations: []common.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}}, + Annotations: []zcommon.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}}, }) t.Log("starting Test retrieve duplicated image same layers based on image identifier") diff --git a/pkg/extensions/sync/signatures.go b/pkg/extensions/sync/signatures.go index 8a6774609..1b93840eb 100644 --- a/pkg/extensions/sync/signatures.go +++ b/pkg/extensions/sync/signatures.go @@ -355,23 +355,8 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil { return err } - } else if ref.MediaType == ispec.MediaTypeArtifactManifest { - // read manifest - var manifest ispec.Artifact - - err = json.Unmarshal(OCIRefBody, &manifest) - if err != nil { - sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). - Str("manifest", getRefManifestURL.String()).Msg("couldn't unmarshal oci reference manifest") - - return err - } - - for _, layer := range manifest.Blobs { - if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil { - return err - } - } + } else { + continue } digest, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(), @@ -404,78 +389,6 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string return nil } -func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference string, - ociArtifactBuf []byte, -) error { - var ociArtifact ispec.Artifact - - err := json.Unmarshal(ociArtifactBuf, &ociArtifact) - if err != nil { - sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", reference). - Msg("couldn't unmarshal OCI artifact") - - return err - } - - canSkipOCIArtifact, err := sig.canSkipOCIArtifact(localRepo, reference, ociArtifact) - if err != nil { - sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", reference). - Msg("couldn't check if OCI artifact can be skipped") - } - - if canSkipOCIArtifact { - return nil - } - - imageStore := sig.storeController.GetImageStore(localRepo) - - sig.log.Info().Msg("syncing OCI artifacts") - - for _, blob := range ociArtifact.Blobs { - if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil { - return err - } - } - - artifactManifestBuf, err := json.Marshal(ociArtifact) - if err != nil { - sig.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't marshal OCI artifact") - - return err - } - - // push manifest - digest, err := imageStore.PutImageManifest(localRepo, reference, - ispec.MediaTypeArtifactManifest, artifactManifestBuf) - if err != nil { - sig.log.Error().Str("errorType", common.TypeOf(err)). - Err(err).Msg("couldn't upload OCI artifact manifest") - - return err - } - - if sig.repoDB != nil { - sig.log.Debug().Str("repository", localRepo).Str("digest", digest.String()). - Msg("trying to OCI refs for repo digest") - - err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeArtifactManifest, - digest, artifactManifestBuf, sig.storeController.GetImageStore(localRepo), - sig.repoDB, sig.log) - if err != nil { - return fmt.Errorf("failed to set metadata for OCI Artifact '%s@%s': %w", localRepo, digest.String(), err) - } - - sig.log.Info().Str("repository", localRepo).Str("digest", digest.String()). - Msg("successfully added oci artifacts to RepoDB for repo digest") - } - - sig.log.Info().Str("repository", localRepo).Str("tag", reference). - Msg("successfully synced OCI artifact for repo tag") - - return nil -} - func (sig *signaturesCopier) canSkipORASRefs(localRepo, digestStr string, refs ReferenceList, ) (bool, error) { imageStore := sig.storeController.GetImageStore(localRepo) @@ -509,47 +422,6 @@ func (sig *signaturesCopier) canSkipORASRefs(localRepo, digestStr string, refs R return true, nil } -func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, artifact ispec.Artifact, -) (bool, error) { - imageStore := sig.storeController.GetImageStore(localRepo) - - var localArtifactManifest ispec.Artifact - - localArtifactBuf, _, _, err := imageStore.GetImageManifest(localRepo, reference) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrRepoNotFound) { - return false, nil - } - - sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). - Str("repository", localRepo).Str("reference", reference). - Msg("couldn't get local OCI artifact manifest") - - return false, err - } - - err = json.Unmarshal(localArtifactBuf, &localArtifactManifest) - if err != nil { - sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). - Str("repository", localRepo).Str("reference", reference). - Msg("couldn't unmarshal local OCI artifact manifest") - - return false, err - } - - if !artifactsEqual(localArtifactManifest, artifact) { - sig.log.Info().Str("repository", localRepo).Str("reference", reference). - Msg("upstream OCI artifact changed, syncing again") - - return false, nil - } - - sig.log.Info().Str("repository", localRepo).Str("reference", reference). - Msg("skipping OCI artifact, already synced") - - return true, nil -} - func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string, cosignManifest *ispec.Manifest, ) (bool, error) { imageStore := sig.storeController.GetImageStore(localRepo) diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index f6db13804..d8e53fd7a 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -361,7 +361,7 @@ func TestSyncInternal(t *testing.T) { So(regURL, ShouldNotBeNil) ref := ispec.Descriptor{ - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, Digest: "fakeDigest", ArtifactType: "application/vnd.cncf.notary.signature", } @@ -406,7 +406,7 @@ func TestSyncInternal(t *testing.T) { refs := ispec.Index{Manifests: []ispec.Descriptor{ { - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, Digest: "fakeDigest", ArtifactType: "application/vnd.cncf.notary.signature", }, @@ -446,7 +446,7 @@ func TestSyncInternal(t *testing.T) { err = json.Unmarshal(buf, &index) So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, Digest: godigest.FromString(""), ArtifactType: "application/vnd.cncf.notary.signature", }) @@ -548,6 +548,59 @@ func TestSyncInternal(t *testing.T) { So(canBeSkipped, ShouldBeFalse) }) + Convey("Test syncOCIRefs with bad mediaType", t, func() { + downPort, upPort := test.GetFreePort(), test.GetFreePort() + + downStream := test.StartTestHTTPServer( + []test.RouteHandler{ + { + Route: "", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + AllowedMethods: []string{}, + }, + }, + downPort, + ) + defer downStream.Close() + + upStream := test.StartTestHTTPServer( + []test.RouteHandler{ + { + Route: "/v2/{name}/manifests/{digest}", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("{}")) + if err != nil { + t.FailNow() + } + }, + AllowedMethods: []string{"GET"}, + }, + }, + upPort, + ) + defer upStream.Close() + + upStreamURL, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%s", upPort)) + So(err, ShouldBeNil) + + client := &http.Client{} + mockRepoDB := mocks.RepoDBMock{} + mockImageStore := mocks.MockedImageStore{} + sig := newSignaturesCopier(client, syncconf.Credentials{}, + *upStreamURL, mockRepoDB, storage.StoreController{DefaultStore: mockImageStore}, + log.NewLogger("debug", ""), + ) + + digest := godigest.FromString("1") + + err = sig.syncOCIRefs("repo", "repo", digest.String(), + ispec.Index{Manifests: []ispec.Descriptor{{MediaType: "bad media type", Digest: digest}}}) + So(err, ShouldBeNil) + }) + Convey("Test filterRepos()", t, func() { repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"} contents := []syncconf.Content{ @@ -1265,61 +1318,3 @@ func TestCompareArtifactRefs(t *testing.T) { } }) } - -func TestCompareArtifactManifests(t *testing.T) { - testCases := []struct { - refs1 ispec.Artifact - refs2 ispec.Artifact - expected bool - }{ - { - refs1: ispec.Artifact{ - MediaType: "mediatype", - ArtifactType: "signature", - Blobs: []ispec.Descriptor{ - { - Digest: "digest1", - }, - }, - }, - refs2: ispec.Artifact{ - MediaType: "mediatype", - ArtifactType: "signature", - Blobs: []ispec.Descriptor{ - { - Digest: "digest1", - }, - }, - }, - expected: true, - }, - { - refs1: ispec.Artifact{ - MediaType: "mediatype", - ArtifactType: "signature", - Blobs: []ispec.Descriptor{ - { - Digest: "digest1", - }, - }, - }, - refs2: ispec.Artifact{ - MediaType: "mediatype", - ArtifactType: "signature", - Blobs: []ispec.Descriptor{ - { - Digest: "digest2", - }, - }, - }, - expected: false, - }, - } - - Convey("Test artifactsEqual()", t, func() { - for _, test := range testCases { - actualResult := artifactsEqual(test.refs1, test.refs2) - So(actualResult, ShouldEqual, test.expected) - } - }) -} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 9e83fa9e3..93ab581eb 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -400,19 +400,24 @@ func TestORAS(t *testing.T) { digest = godigest.FromBytes(resp.Body()) - content := []byte("blob content") - adigest := pushBlob(srcBaseURL, repoName, content) + // layer + layer := []byte("blob content") + blobDigest := pushBlob(srcBaseURL, repoName, layer) - artifactManifest := ispec.Artifact{ + // config + _ = pushBlob(srcBaseURL, repoName, ispec.ScratchDescriptor.Data) + + artifactManifest := ispec.Manifest{ MediaType: artifactspec.MediaTypeArtifactManifest, ArtifactType: "application/vnd.oras.artifact", - Blobs: []ispec.Descriptor{ + Layers: []ispec.Descriptor{ { MediaType: "application/octet-stream", - Digest: adigest, - Size: int64(len(content)), + Digest: blobDigest, + Size: int64(len(layer)), }, }, + Config: ispec.ScratchDescriptor, Subject: &ispec.Descriptor{ MediaType: "application/vnd.oci.image.manifest.v1+json", Digest: digest, @@ -420,19 +425,15 @@ func TestORAS(t *testing.T) { }, } - content, err = json.Marshal(artifactManifest) - if err != nil { - panic(err) - } + artManifestBlob, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) - adigest = godigest.FromBytes(content) + artifactDigest := godigest.FromBytes(artManifestBlob) // put OCI reference artifact mediaType artifact _, err = resty.R().SetHeader("Content-Type", artifactspec.MediaTypeArtifactManifest). - SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String())) - if err != nil { - panic(err) - } + SetBody(artManifestBlob).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String())) + So(err, ShouldBeNil) err = os.Chmod(path.Join(destDir, testImage, "index.json"), 0o000) So(err, ShouldBeNil) @@ -447,7 +448,7 @@ func TestORAS(t *testing.T) { So(err, ShouldBeNil) // trigger getORASRefs err - err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", adigest.Encoded()), 0o000) + err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", artifactDigest.Encoded()), 0o000) So(err, ShouldBeNil) resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + digest.String()) @@ -455,7 +456,7 @@ func TestORAS(t *testing.T) { So(resp, ShouldNotBeEmpty) So(resp.StatusCode(), ShouldEqual, http.StatusOK) - err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", adigest.Encoded()), 0o755) + err = os.Chmod(path.Join(srcDir, testImage, "blobs/sha256", artifactDigest.Encoded()), 0o755) So(err, ShouldBeNil) resp, err = resty.R().Get(getORASReferrersURL) @@ -473,11 +474,12 @@ func TestORAS(t *testing.T) { err = os.RemoveAll(path.Join(destDir, repoName)) So(err, ShouldBeNil) - err = os.WriteFile(path.Join(srcDir, repoName, "blobs", "sha256", adigest.Encoded()), []byte("wrong content"), 0o600) + err = os.WriteFile(path.Join(srcDir, repoName, "blobs", "sha256", artifactDigest.Encoded()), + []byte("wrong content"), 0o600) So(err, ShouldBeNil) _, err = resty.R().SetHeader("Content-Type", artifactspec.MediaTypeArtifactManifest). - SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String())) + SetBody(artManifestBlob).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String())) if err != nil { panic(err) } @@ -665,20 +667,33 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) // add OCI Ref - OCIRefManifest := ispec.Artifact{ + _ = pushBlob(srcBaseURL, "remote-repo", ispec.ScratchDescriptor.Data) + + OCIRefManifest := ispec.Manifest{ Subject: &ispec.Descriptor{ MediaType: ispec.MediaTypeImageManifest, Digest: manifestDigest, }, - Blobs: []ispec.Descriptor{}, - MediaType: ispec.MediaTypeArtifactManifest, + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeScratch, + Digest: ispec.ScratchDescriptor.Digest, + Size: 2, + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeScratch, + Digest: ispec.ScratchDescriptor.Digest, + Size: 2, + }, + }, + MediaType: ispec.MediaTypeImageManifest, } OCIRefManifestBlob, err := json.Marshal(OCIRefManifest) So(err, ShouldBeNil) resp, err := resty.R(). - SetHeader("Content-type", ispec.MediaTypeArtifactManifest). + SetHeader("Content-type", ispec.MediaTypeImageManifest). SetBody(OCIRefManifestBlob). Put(srcBaseURL + "/v2/remote-repo/manifests/oci.ref") @@ -763,7 +778,7 @@ func TestOnDemand(t *testing.T) { dctlr.RepoDB = mocks.RepoDBMock{ SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { - if mediaType == ispec.MediaTypeArtifactManifest { + if mediaType == ispec.MediaTypeImageManifest { return sync.ErrTestError } @@ -3018,7 +3033,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { So(err, ShouldBeNil) // read manifest - var artifactManifest ispec.Artifact + var artifactManifest ispec.Manifest for _, ref := range referrers.Manifests { refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) body, err := os.ReadFile(refPath) @@ -3028,7 +3043,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { So(err, ShouldBeNil) // triggers perm denied on sig blobs - for _, blob := range artifactManifest.Blobs { + for _, blob := range artifactManifest.Layers { blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) err := os.Chmod(blobPath, 0o000) So(err, ShouldBeNil) @@ -3145,7 +3160,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { Convey("of type OCI artifact", func() { //nolint: dupl // read manifest - var artifactManifest ispec.Artifact + var artifactManifest ispec.Manifest for _, ref := range referrers.Manifests { refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) body, err := os.ReadFile(refPath) @@ -3155,7 +3170,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { So(err, ShouldBeNil) // triggers perm denied on artifact blobs - for _, blob := range artifactManifest.Blobs { + for _, blob := range artifactManifest.Layers { blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) err := os.Chmod(blobPath, 0o000) So(err, ShouldBeNil) @@ -3349,7 +3364,7 @@ func TestSignatures(t *testing.T) { err = os.RemoveAll(path.Join(destDir, repoName)) So(err, ShouldBeNil) - var artifactManifest ispec.Artifact + var artifactManifest ispec.Manifest for _, ref := range referrers.Manifests { refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded()) body, err := os.ReadFile(refPath) @@ -3359,7 +3374,7 @@ func TestSignatures(t *testing.T) { So(err, ShouldBeNil) // triggers perm denied on notary sig blobs on downstream - for _, blob := range artifactManifest.Blobs { + for _, blob := range artifactManifest.Layers { blobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) err := os.MkdirAll(blobPath, 0o755) So(err, ShouldBeNil) @@ -3392,7 +3407,7 @@ func TestSignatures(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusOK) // triggers perm denied on sig blobs - for _, blob := range artifactManifest.Blobs { + for _, blob := range artifactManifest.Layers { blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded()) err := os.Chmod(blobPath, 0o000) So(err, ShouldBeNil) @@ -5390,331 +5405,6 @@ func TestSyncImageIndex(t *testing.T) { }) } -func TestSyncOCIArtifactsWithTag(t *testing.T) { - Convey("Verify syncing tagged OCI artifacts", t, func() { - updateDuration, _ := time.ParseDuration("10s") - - sctlr, srcBaseURL, _, _, _ := makeUpstreamServer(t, false, false) - - scm := test.NewControllerManager(sctlr) - scm.StartAndWait(sctlr.Config.HTTP.Port) - defer scm.StopServer() - - regex := ".*" - var semver bool - tlsVerify := false - - repoName := "artifact" - - syncRegistryConfig := syncconf.RegistryConfig{ - Content: []syncconf.Content{ - { - Prefix: repoName, - Tags: &syncconf.Tags{ - Regex: ®ex, - Semver: &semver, - }, - }, - }, - URLs: []string{srcBaseURL}, - OnDemand: false, - PollInterval: updateDuration, - TLSVerify: &tlsVerify, - } - - defaultVal := true - syncConfig := &syncconf.Config{ - Enable: &defaultVal, - Registries: []syncconf.RegistryConfig{syncRegistryConfig}, - } - - // create artifact blob - buf := []byte("this is an artifact") - digest := pushBlob(srcBaseURL, repoName, buf) - - // create artifact config blob - cbuf := []byte("{}") - cdigest := pushBlob(srcBaseURL, repoName, cbuf) - - // push a referrer artifact - manifest := ispec.Manifest{ - MediaType: ispec.MediaTypeImageManifest, - Config: ispec.Descriptor{ - MediaType: "application/vnd.cncf.icecream", - Digest: cdigest, - Size: int64(len(cbuf)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/octet-stream", - Digest: digest, - Size: int64(len(buf)), - }, - }, - } - - artifactManifest := ispec.Artifact{ - MediaType: ispec.MediaTypeArtifactManifest, - ArtifactType: "application/vnd.cncf.icecream", - Blobs: []ispec.Descriptor{ - { - MediaType: "application/octet-stream", - Digest: digest, - Size: int64(len(buf)), - }, - }, - } - - manifest.SchemaVersion = 2 - - content, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - // put OCI artifact mediatype oci image - _, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). - SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0")) - So(err, ShouldBeNil) - - content, err = json.Marshal(artifactManifest) - So(err, ShouldBeNil) - - artifactDigest := godigest.FromBytes(content) - - // put OCI artifact mediatype artifact - _, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - SetBody(content).Put(srcBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0")) - So(err, ShouldBeNil) - - Convey("sync periodically", func() { - // start downstream server - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - // give it time to set up sync - t.Logf("waitsync(%s, %s)", dctlr.Config.Storage.RootDirectory, repoName) - waitSync(dctlr.Config.Storage.RootDirectory, repoName) - - resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0")) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - So(resp.Body(), ShouldNotBeEmpty) - So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) - - var syncedManifest ispec.Manifest - err = json.Unmarshal(resp.Body(), &syncedManifest) - So(err, ShouldBeNil) - - So(reflect.DeepEqual(syncedManifest, manifest), ShouldEqual, true) - - resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0")) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - So(resp.Body(), ShouldNotBeEmpty) - So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) - - var syncedArtifact ispec.Artifact - err = json.Unmarshal(resp.Body(), &syncedArtifact) - So(err, ShouldBeNil) - - So(reflect.DeepEqual(syncedArtifact, artifactManifest), ShouldEqual, true) - - // for coverage - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "skipping OCI artifact", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - - waitSyncFinish(dctlr.Config.Log.Output) - }) - - Convey("sync on demand", func() { - // start downstream server - syncConfig.Registries[0].OnDemand = true - syncConfig.Registries[0].PollInterval = 0 - - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "2.0")) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - So(resp.Body(), ShouldNotBeEmpty) - So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) - - var syncedArtifact ispec.Artifact - err = json.Unmarshal(resp.Body(), &syncedArtifact) - So(err, ShouldBeNil) - - So(reflect.DeepEqual(syncedArtifact, artifactManifest), ShouldEqual, true) - - resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, "1.0")) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) - So(resp.Body(), ShouldNotBeEmpty) - So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) - - var syncedManifest ispec.Manifest - err = json.Unmarshal(resp.Body(), &syncedManifest) - So(err, ShouldBeNil) - - So(reflect.DeepEqual(syncedManifest, manifest), ShouldEqual, true) - }) - - Convey("sync periodically error on mediatype", func() { - manifestPath := path.Join(sctlr.Config.Storage.RootDirectory, repoName, "blobs", "sha256", artifactDigest.Encoded()) - So(os.Chmod(manifestPath, 0o000), ShouldBeNil) - - // start downstream server - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - defer func() { - err := os.Chmod(manifestPath, 0o755) - So(err, ShouldBeNil) - }() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - - resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - - found, err = test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get upstream image", 5*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - }) - - Convey("sync on demand error on mediatype", func() { - // start downstream server - syncConfig.Registries[0].OnDemand = true - syncConfig.Registries[0].PollInterval = 0 - - dctlr, destBaseURL, _, _ := makeDownstreamServer(t, false, syncConfig) - - dcm := test.NewControllerManager(dctlr) - dcm.StartAndWait(dctlr.Config.HTTP.Port) - defer dcm.StopServer() - - manifestPath := path.Join(sctlr.Config.Storage.RootDirectory, repoName, "blobs", "sha256", artifactDigest.Encoded()) - So(os.Chmod(manifestPath, 0o000), ShouldBeNil) - defer func() { - err := os.Chmod(manifestPath, 0o755) - So(err, ShouldBeNil) - }() - - resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - }) - - Convey("sync on demand and periodically error on PutImageManifest", func() { - // start downstream server - syncConfig.Registries[0].OnDemand = true - - destDir := t.TempDir() - destConfig := config.New() - - destConfig.HTTP.Port = test.GetFreePort() - destBaseURL := test.GetBaseURL(destConfig.HTTP.Port) - - destConfig.Storage.RootDirectory = destDir - - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = syncConfig - - destConfig.Log.Output = path.Join(destDir, "sync.log") - - dctlr := api.NewController(destConfig) - dcm := test.NewControllerManager(dctlr) - - manifestPath := path.Join(destDir, repoName, "blobs", "sha256", artifactDigest.Encoded()) - So(os.MkdirAll(manifestPath, 0o755), ShouldBeNil) - So(os.Chmod(manifestPath, 0o000), ShouldBeNil) - - dcm.StartAndWait(destConfig.HTTP.Port) - - defer dcm.StopServer() - - defer func() { - err := os.Chmod(manifestPath, 0o755) - So(err, ShouldBeNil) - }() - - found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't upload OCI artifact manifest", 15*time.Second) - if err != nil { - panic(err) - } - - if !found { - data, err := os.ReadFile(dctlr.Config.Log.Output) - So(err, ShouldBeNil) - - t.Logf("downstream log: %s", string(data)) - } - - So(found, ShouldBeTrue) - - resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). - Get(destBaseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, artifactDigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - - waitSyncFinish(dctlr.Config.Log.Output) - }) - }) -} - func generateKeyPairs(tdir string) { // generate a keypair os.Setenv("COSIGN_PASSWORD", "") @@ -5817,6 +5507,35 @@ func pushRepo(url, repoName string) godigest.Digest { panic(err) } + // upload scratch image config + resp, err = resty.R(). + Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName)) + if err != nil { + panic(err) + } + + if resp.StatusCode() != http.StatusAccepted { + panic(fmt.Errorf("invalid status code: %d %w", resp.StatusCode(), errBadStatus)) + } + + loc = test.Location(url, resp) + cblob, cdigest := ispec.ScratchDescriptor.Data, ispec.ScratchDescriptor.Digest + + resp, err = resty.R(). + SetContentLength(true). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + if err != nil { + panic(err) + } + + if resp.StatusCode() != http.StatusCreated { + panic(fmt.Errorf("invalid status code: %d %w", resp.StatusCode(), errBadStatus)) + } + // upload image config blob resp, err = resty.R(). Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName)) @@ -5829,7 +5548,7 @@ func pushRepo(url, repoName string) godigest.Digest { } loc = test.Location(url, resp) - cblob, cdigest := test.GetRandomImageConfig() + cblob, cdigest = test.GetRandomImageConfig() resp, err = resty.R(). SetContentLength(true). @@ -5907,10 +5626,15 @@ func pushRepo(url, repoName string) godigest.Digest { }, } - artifactManifest := ispec.Artifact{ - MediaType: ispec.MediaTypeArtifactManifest, + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, ArtifactType: "application/vnd.cncf.icecream", - Blobs: []ispec.Descriptor{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeScratch, + Digest: ispec.ScratchDescriptor.Digest, + Size: 2, + }, + Layers: []ispec.Descriptor{ { MediaType: "application/octet-stream", Digest: adigest, @@ -5948,7 +5672,7 @@ func pushRepo(url, repoName string) godigest.Digest { adigest = godigest.FromBytes(content) // put OCI reference artifact mediaType artifact - _, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest). + _, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String())) if err != nil { panic(err) diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 2ad0beb03..00bd99c77 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -590,17 +590,6 @@ func manifestsEqual(manifest1, manifest2 ispec.Manifest) bool { return false } -func artifactsEqual(manifest1, manifest2 ispec.Artifact) bool { - if manifest1.ArtifactType == manifest2.ArtifactType && - manifest1.MediaType == manifest2.MediaType { - if descriptorsEqual(manifest1.Blobs, manifest2.Blobs) { - return true - } - } - - return false -} - func artifactDescriptorsEqual(desc1, desc2 []artifactspec.Descriptor) bool { if len(desc1) != len(desc2) { return false @@ -690,16 +679,6 @@ func syncImageWithRefs(ctx context.Context, localRepo, upstreamRepo, reference s upstreamImageDigest := godigest.FromBytes(manifestBuf) if !isSupportedMediaType(mediaType) { - if mediaType == ispec.MediaTypeArtifactManifest { - err = sig.syncOCIArtifact(localRepo, upstreamRepo, reference, manifestBuf) //nolint - if err != nil { - log.Error().Err(err).Str("image", upstreamImageRef.DockerReference().String()). - Msg("couldn't sync oci artifact with artifact mediaType") - - return skipped, err - } - } - return skipped, nil } diff --git a/pkg/log/log.go b/pkg/log/log.go index 937982c3d..72b534ac8 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -92,7 +92,7 @@ func GoroutineID() int { type goroutineHook struct{} -func (h goroutineHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { +func (h goroutineHook) Run(e *zerolog.Event, level zerolog.Level, _ string) { if level != zerolog.NoLevel { e.Int("goroutine", GoroutineID()) } diff --git a/pkg/meta/bolt/buckets.go b/pkg/meta/bolt/buckets.go index 80237d061..388daf965 100644 --- a/pkg/meta/bolt/buckets.go +++ b/pkg/meta/bolt/buckets.go @@ -4,7 +4,6 @@ package bolt const ( ManifestDataBucket = "ManifestData" IndexDataBucket = "IndexData" - ArtifactDataBucket = "ArtifactData" RepoMetadataBucket = "RepoMetadata" UserDataBucket = "UserData" VersionBucket = "Version" diff --git a/pkg/meta/dynamo/parameters.go b/pkg/meta/dynamo/parameters.go index cc98debca..cbe717291 100644 --- a/pkg/meta/dynamo/parameters.go +++ b/pkg/meta/dynamo/parameters.go @@ -10,7 +10,7 @@ import ( type DBDriverParameters struct { Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, - ArtifactDataTablename, VersionTablename, UserDataTablename string + VersionTablename, UserDataTablename string } func GetDynamoClient(params DBDriverParameters) (*dynamodb.Client, error) { diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 1963d1f69..ef4f4b2f7 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -49,11 +49,6 @@ func NewBoltDBWrapper(boltDB *bbolt.DB, log log.Logger) (*DBWrapper, error) { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(bolt.ArtifactDataBucket)) - if err != nil { - return err - } - _, err = transaction.CreateBucketIfNotExists([]byte(bolt.RepoMetadataBucket)) if err != nil { return err @@ -258,49 +253,6 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat return indexMetadata, err } -func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) - - imBlob, err := json.Marshal(artifactData) - if err != nil { - return fmt.Errorf("repodb: error while calculating blob for artifact with digest %s %w", artifactDigest, err) - } - - err = buck.Put([]byte(artifactDigest), imBlob) - if err != nil { - return fmt.Errorf("repodb: error while setting artifact blob for digest %s %w", artifactDigest, err) - } - - return nil - }) - - return err -} - -func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { - var artifactData repodb.ArtifactData - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) - - blob := buck.Get([]byte(artifactDigest)) - - if len(blob) == 0 { - return zerr.ErrArtifactDataNotFound - } - - err := json.Unmarshal(blob, &artifactData) - if err != nil { - return fmt.Errorf("repodb: error while unmashaling artifact data for digest %s %w", artifactDigest, err) - } - - return nil - }) - - return artifactData, err -} - func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index 4b03d1b40..601ea3038 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -91,18 +91,6 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("GetArtifactData", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - artifactBuck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) - - return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetArtifactData("artifactDigest") - So(err, ShouldNotBeNil) - }) - Convey("SetReferrer", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go index 8721d1e2e..079340afa 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go @@ -30,7 +30,6 @@ func TestWrapperErrors(t *testing.T) { repoMetaTablename := "RepoMetadataTable" + uuid.String() manifestDataTablename := "ManifestDataTable" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() - artifactDataTablename := "ArtifactDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() versionTablename := "Version" + uuid.String() @@ -57,7 +56,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, UserDataTablename: userDataTablename, Patches: version.GetDynamoDBPatches(), @@ -99,7 +97,6 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, VersionTablename: versionTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index 349713b40..bddcd9a64 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -42,7 +42,6 @@ func TestIterator(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() - artifactDataTablename := "ArtifactDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() log := log.NewLogger("debug", "") @@ -54,7 +53,6 @@ func TestIterator(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, UserDataTablename: userDataTablename, } @@ -145,7 +143,6 @@ func TestWrapperErrors(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() - artifactDataTablename := "ArtifactData" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() ctx := context.Background() @@ -159,7 +156,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: versionTablename, } @@ -417,20 +413,6 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("GetArtifactData", func() { - dynamoWrapper.ArtifactDataTablename = badTablename - _, err = dynamoWrapper.GetArtifactData("dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetArtifactData unmarhsal error", func() { - err = setBadArtifactData(dynamoWrapper.Client, artifactDataTablename, "dig") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetArtifactData("dig") - So(err, ShouldNotBeNil) - }) - Convey("SetRepoReference client error", func() { dynamoWrapper.RepoMetaTablename = badTablename digest := digest.FromString("str") @@ -488,7 +470,6 @@ func TestWrapperErrors(t *testing.T) { Convey("GetReferrersInfo getData fails", func() { dynamoWrapper.ManifestDataTablename = badTablename - dynamoWrapper.ArtifactDataTablename = badTablename err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig1", MediaType: ispec.MediaTypeImageManifest, @@ -497,7 +478,7 @@ func TestWrapperErrors(t *testing.T) { err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig2", - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) @@ -506,11 +487,6 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetReferrersInfo bad descriptor blob", func() { - err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{ - ManifestBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - err = dynamoWrapper.SetManifestData("dig3", repodb.ManifestData{ ManifestBlob: []byte("bad json"), }) @@ -518,7 +494,7 @@ func TestWrapperErrors(t *testing.T) { err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig2", - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) @@ -1095,7 +1071,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: "", ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: versionTablename, } @@ -1111,7 +1086,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: "", IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: versionTablename, } @@ -1127,7 +1101,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: "", - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: versionTablename, } @@ -1143,7 +1116,6 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: "", } @@ -1159,41 +1131,8 @@ func TestWrapperErrors(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: "", - UserDataTablename: userDataTablename, - VersionTablename: versionTablename, - } - client, err = dynamo.GetDynamoClient(params) - So(err, ShouldBeNil) - - _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) - So(err, ShouldNotBeNil) - - params = dynamo.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: userDataTablename, - ArtifactDataTablename: artifactDataTablename, - } - client, err = dynamo.GetDynamoClient(params) - So(err, ShouldBeNil) - - _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) - So(err, ShouldBeNil) - - params = dynamo.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, UserDataTablename: "", - ArtifactDataTablename: artifactDataTablename, + VersionTablename: versionTablename, } client, err = dynamo.GetDynamoClient(params) So(err, ShouldBeNil) @@ -1228,26 +1167,26 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s return err } -func setBadArtifactData(client *dynamodb.Client, artifactDataTablename, digest string) error { - mdAttributeValue, err := attributevalue.Marshal("string") +func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error { + repoAttributeValue, err := attributevalue.Marshal("string") if err != nil { return err } _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ - "#AD": "ArtifactData", + "#RM": "RepoMetadata", }, ExpressionAttributeValues: map[string]types.AttributeValue{ - ":ArtifactData": mdAttributeValue, + ":RepoMetadata": repoAttributeValue, }, Key: map[string]types.AttributeValue{ - "ArtifactDigest": &types.AttributeValueMemberS{ - Value: digest, + "RepoName": &types.AttributeValueMemberS{ + Value: repoName, }, }, - TableName: aws.String(artifactDataTablename), - UpdateExpression: aws.String("SET #AD = :ArtifactData"), + TableName: aws.String(repoMetadataTableName), + UpdateExpression: aws.String("SET #RM = :RepoMetadata"), }) return err @@ -1278,31 +1217,6 @@ func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) return err } -func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error { - repoAttributeValue, err := attributevalue.Marshal("string") - if err != nil { - return err - } - - _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ - Value: repoName, - }, - }, - TableName: aws.String(repoMetadataTableName), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), - }) - - return err -} - func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error { userAttributeValue, err := attributevalue.Marshal("string") if err != nil { diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index 82267ae0f..fb674b76d 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -32,7 +32,6 @@ type DBWrapper struct { RepoMetaTablename string IndexDataTablename string ManifestDataTablename string - ArtifactDataTablename string UserDataTablename string VersionTablename string Patches []func(client *dynamodb.Client, tableNames map[string]string) error @@ -45,7 +44,6 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter RepoMetaTablename: params.RepoMetaTablename, ManifestDataTablename: params.ManifestDataTablename, IndexDataTablename: params.IndexDataTablename, - ArtifactDataTablename: params.ArtifactDataTablename, VersionTablename: params.VersionTablename, UserDataTablename: params.UserDataTablename, Patches: version.GetDynamoDBPatches(), @@ -67,11 +65,6 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter return nil, err } - err = dynamoWrapper.createArtifactDataTable() - if err != nil { - return nil, err - } - err = dynamoWrapper.createIndexDataTable() if err != nil { return nil, err @@ -301,58 +294,6 @@ func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat return indexData, nil } -func (dwr DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error { - artifactAttributeValue, err := attributevalue.Marshal(artifactData) - if err != nil { - return err - } - - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#AD": "ArtifactData", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":ArtifactData": artifactAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "ArtifactDigest": &types.AttributeValueMemberS{ - Value: artifactDigest.String(), - }, - }, - TableName: aws.String(dwr.ArtifactDataTablename), - UpdateExpression: aws.String("SET #AD = :ArtifactData"), - }) - - return err -} - -func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.ArtifactDataTablename), - Key: map[string]types.AttributeValue{ - "ArtifactDigest": &types.AttributeValueMemberS{ - Value: artifactDigest.String(), - }, - }, - }) - if err != nil { - return repodb.ArtifactData{}, err - } - - if resp.Item == nil { - return repodb.ArtifactData{}, zerr.ErrRepoMetaNotFound - } - - var artifactData repodb.ArtifactData - - err = attributevalue.Unmarshal(resp.Item["ArtifactData"], &artifactData) - if err != nil { - return repodb.ArtifactData{}, err - } - - return artifactData, nil -} - func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), @@ -1664,31 +1605,6 @@ func (dwr *DBWrapper) createIndexDataTable() error { return dwr.waitTableToBeCreated(dwr.IndexDataTablename) } -func (dwr DBWrapper) createArtifactDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.ArtifactDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("ArtifactDigest"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("ArtifactDigest"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { - return err - } - - return dwr.waitTableToBeCreated(dwr.ManifestDataTablename) -} - func (dwr *DBWrapper) createVersionTable() error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ TableName: aws.String(dwr.VersionTablename), diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index d8d3be76a..aeccb32f4 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -77,12 +77,6 @@ type RepoDB interface { //nolint:interfacebloat // GetIndexData returns indexData for a given Index from the database GetIndexData(indexDigest godigest.Digest) (IndexData, error) - // SetArtifactData sets artifactData for a given artifact in the database - SetArtifactData(artifactDigest godigest.Digest, artifactData ArtifactData) error - - // GetArtifactData returns artifactData for a given artifact from the database - GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error) - // SetReferrer adds a referrer to the referrers list of a manifest inside a repo SetReferrer(repo string, referredDigest godigest.Digest, referrer ReferrerInfo) error @@ -150,10 +144,6 @@ type ManifestData struct { ConfigBlob []byte } -type ArtifactData struct { - ManifestBlob []byte -} - type ReferrerInfo struct { Digest string MediaType string diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index aac0e012b..f81530622 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -87,7 +87,6 @@ func TestDynamoDBWrapper(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() - artifactDataTablename := "ArtifactDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() Convey("DynamoDB Wrapper", t, func() { @@ -96,7 +95,6 @@ func TestDynamoDBWrapper(t *testing.T) { RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, UserDataTablename: userDataTablename, Region: "us-east-2", @@ -2174,27 +2172,6 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(err, ShouldNotBeNil) }) - Convey("Test artifact logic", func() { - artifact, err := test.GetRandomArtifact(nil) - So(err, ShouldBeNil) - - artifactDigest, err := artifact.Digest() - So(err, ShouldBeNil) - - artifactData, err := artifact.ArtifactData() - So(err, ShouldBeNil) - - err = repoDB.SetArtifactData(artifactDigest, artifactData) - So(err, ShouldBeNil) - - result, err := repoDB.GetArtifactData(artifactDigest) - So(err, ShouldBeNil) - So(result, ShouldResemble, artifactData) - - _, err = repoDB.GetArtifactData(godigest.FromString("inexistent")) - So(err, ShouldNotBeNil) - }) - Convey("Test Referrers", func() { image, err := test.GetRandomImage("tag") So(err, ShouldBeNil) @@ -2221,10 +2198,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { // ------- Add Artifact 1 - artifact1, err := test.GetRandomArtifact(&ispec.Descriptor{ - Digest: referredDigest, - MediaType: ispec.MediaTypeImageManifest, - }) + artifact1, err := test.GetImageWithSubject( + referredDigest, + ispec.MediaTypeImageManifest, + ) So(err, ShouldBeNil) artifactDigest1, err := artifact1.Digest() @@ -2238,10 +2215,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { // ------- Add Artifact 2 - artifact2, err := test.GetRandomArtifact(&ispec.Descriptor{ - Digest: referredDigest, - MediaType: ispec.MediaTypeImageManifest, - }) + artifact2, err := test.GetImageWithSubject( + referredDigest, + ispec.MediaTypeImageManifest, + ) So(err, ShouldBeNil) artifactDigest2, err := artifact2.Digest() @@ -2249,7 +2226,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: artifactDigest2.String(), - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) @@ -2263,7 +2240,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(referrers, ShouldContain, repodb.ReferrerInfo{ Digest: artifactDigest2.String(), - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) @@ -2333,12 +2310,6 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ - Digest: "inexistendArtifactManifestDigest", - MediaType: ispec.MediaTypeArtifactManifest, - }) - So(err, ShouldBeNil) - // ------- Set existent manifest and artifact manifest err = repoDB.SetManifestData("goodManifest", repodb.ManifestData{ ManifestBlob: []byte(`{"artifactType": "unwantedType"}`), @@ -2347,20 +2318,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(err, ShouldBeNil) err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ - Digest: "goodManifest", + Digest: "goodManifestUnwanted", MediaType: ispec.MediaTypeImageManifest, ArtifactType: "unwantedType", }) So(err, ShouldBeNil) - err = repoDB.SetArtifactData("goodArtifact", repodb.ArtifactData{ - ManifestBlob: []byte(`{"artifactType": "wantedType"}`), - }) - So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ - Digest: "goodArtifact", - MediaType: ispec.MediaTypeArtifactManifest, + Digest: "goodManifest", + MediaType: ispec.MediaTypeImageManifest, ArtifactType: "wantedType", }) So(err, ShouldBeNil) @@ -2369,7 +2335,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(err, ShouldBeNil) So(len(referrerInfo), ShouldEqual, 1) So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType") - So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact") + So(referrerInfo[0].Digest, ShouldResemble, "goodManifest") }) Convey("FilterRepos", func() { diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory.go b/pkg/meta/repodb/repodbfactory/repodb_factory.go index a67b14055..f91efbcb8 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory.go @@ -89,9 +89,6 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log) allParametersOk = allParametersOk && ok - artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log) - allParametersOk = allParametersOk && ok - versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) allParametersOk = allParametersOk && ok @@ -108,7 +105,6 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d RepoMetaTablename: repoMetaTablename, ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, - ArtifactDataTablename: artifactDataTablename, UserDataTablename: userDataTablename, VersionTablename: versionTablename, } diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go index 28f73a7d5..484ae3de5 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go @@ -22,7 +22,6 @@ func TestCreateDynamo(t *testing.T) { RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", - ArtifactDataTablename: "ArtifactDataTable", UserDataTablename: "UserDataTable", VersionTablename: "Version", Region: "us-east-2", diff --git a/pkg/meta/repodb/storage_parsing.go b/pkg/meta/repodb/storage_parsing.go index 80464034b..7f2ec57fb 100644 --- a/pkg/meta/repodb/storage_parsing.go +++ b/pkg/meta/repodb/storage_parsing.go @@ -9,6 +9,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" ) @@ -263,13 +264,6 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageSto return indexData } -func NewArtifactData(repo string, descriptorBlob []byte, imageStore storage.ImageStore, -) ArtifactData { - return ArtifactData{ - ManifestBlob: descriptorBlob, - } -} - // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, @@ -295,15 +289,6 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig if err != nil { log.Error().Err(err).Msg("repodb: error while putting index data") - return err - } - case ispec.MediaTypeArtifactManifest: - artifactData := NewArtifactData(repo, descriptorBlob, imageStore) - - err := repoDB.SetArtifactData(digest, artifactData) - if err != nil { - log.Error().Err(err).Msg("repodb: error while putting artifact data") - return err } } @@ -335,43 +320,22 @@ func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string, referrerSubject *ispec.Descriptor ) - switch mediaType { - case ispec.MediaTypeImageManifest: - var manifestContent ispec.Manifest + var manifestContent ispec.Manifest - err := json.Unmarshal(descriptorBlob, &manifestContent) - if err != nil { - return "", referrerInfo, false, - fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err) - } - - referrerSubject = manifestContent.Subject - - referrerInfo = ReferrerInfo{ - Digest: referrerDigest, - MediaType: mediaType, - ArtifactType: manifestContent.Config.MediaType, - Size: len(descriptorBlob), - Annotations: manifestContent.Annotations, - } - case ispec.MediaTypeArtifactManifest: - manifestContent := ispec.Artifact{} - - err := json.Unmarshal(descriptorBlob, &manifestContent) - if err != nil { - return "", referrerInfo, false, - fmt.Errorf("repodb: can't unmarhsal artifact manifest for digest %s: %w", referrerDigest, err) - } + err := json.Unmarshal(descriptorBlob, &manifestContent) + if err != nil { + return "", referrerInfo, false, + fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err) + } - referrerSubject = manifestContent.Subject + referrerSubject = manifestContent.Subject - referrerInfo = ReferrerInfo{ - Digest: referrerDigest, - MediaType: manifestContent.MediaType, - ArtifactType: manifestContent.ArtifactType, - Size: len(descriptorBlob), - Annotations: manifestContent.Annotations, - } + referrerInfo = ReferrerInfo{ + Digest: referrerDigest, + MediaType: mediaType, + ArtifactType: zcommon.GetManifestArtifactType(manifestContent), + Size: len(descriptorBlob), + Annotations: manifestContent.Annotations, } if referrerSubject == nil || referrerSubject.Digest.String() == "" { diff --git a/pkg/meta/repodb/storage_parsing_test.go b/pkg/meta/repodb/storage_parsing_test.go index 985d6d002..9852dfe58 100644 --- a/pkg/meta/repodb/storage_parsing_test.go +++ b/pkg/meta/repodb/storage_parsing_test.go @@ -237,7 +237,7 @@ func TestParseStorageErrors(t *testing.T) { }) Convey("CheckIsImageSignature -> is signature", func() { - manifestContent := ispec.Artifact{ + manifestContent := ispec.Manifest{ Subject: &ispec.Descriptor{ Digest: "123", }, @@ -292,7 +292,6 @@ func TestParseStorageDynamoWrapper(t *testing.T) { RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", - ArtifactDataTablename: "ArtifactDataTable", UserDataTablename: "UserDataTable", VersionTablename: "Version", } @@ -515,10 +514,7 @@ func RunParseStorageTests(rootDir string, repoDB repodb.RepoDB) { func TestGetReferredSubject(t *testing.T) { Convey("GetReferredSubject error", t, func() { - _, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeArtifactManifest) - So(err, ShouldNotBeNil) - - _, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest) + _, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) } diff --git a/pkg/meta/update.go b/pkg/meta/update.go index 1cb6178e6..d4ae82f44 100644 --- a/pkg/meta/update.go +++ b/pkg/meta/update.go @@ -144,7 +144,7 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, } // OnDeleteManifest is called when a manifest is downloaded. It increments the download couter on that manifest. -func OnGetManifest(name, reference string, digest godigest.Digest, body []byte, +func OnGetManifest(name, reference string, body []byte, storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger, ) error { // check if image is a signature diff --git a/pkg/meta/update_test.go b/pkg/meta/update_test.go index 9d087d102..60abd33b7 100644 --- a/pkg/meta/update_test.go +++ b/pkg/meta/update_test.go @@ -100,7 +100,7 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("zerr.ErrOrphanSignature", func() { - manifestContent := ispec.Artifact{ + manifestContent := ispec.Manifest{ Subject: &ispec.Descriptor{ Digest: "123", }, @@ -126,7 +126,7 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("CheckIsImageSignature errors", func() { - manifestContent := ispec.Artifact{ + manifestContent := ispec.Manifest{ Subject: &ispec.Descriptor{ Digest: "123", }, @@ -179,7 +179,7 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") Convey("CheckIsImageSignature errors", func() { - manifestContent := ispec.Artifact{ + manifestContent := ispec.Manifest{ Subject: &ispec.Descriptor{ Digest: "123", }, @@ -192,7 +192,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", zerr.ErrManifestNotFound } - err = meta.OnGetManifest("repo", "tag1", "digest", manifestBlob, + err = meta.OnGetManifest("repo", "tag1", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) @@ -200,7 +200,7 @@ func TestUpdateErrors(t *testing.T) { return []byte{}, "", "", ErrTestError } - err = meta.OnGetManifest("repo", "tag1", "media", manifestBlob, + err = meta.OnGetManifest("repo", "tag1", manifestBlob, storeController, repoDB, log) So(err, ShouldNotBeNil) }) @@ -245,22 +245,18 @@ func TestUpdateErrors(t *testing.T) { err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte("{}"), imageStore, repoDB, log) So(err, ShouldNotBeNil) + }) - repoDB = mocks.RepoDBMock{ - SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error { - return ErrTestError - }, - } - err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", - []byte("{}"), imageStore, repoDB, log) - So(err, ShouldNotBeNil) + Convey("SetMetadataFromInput SetIndexData errors", func() { + imageStore := mocks.MockedImageStore{} + log := log.NewLogger("debug", "") - repoDB = mocks.RepoDBMock{ - SetArtifactDataFn: func(digest godigest.Digest, artifactData repodb.ArtifactData) error { + repoDB := mocks.RepoDBMock{ + SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error { return ErrTestError }, } - err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeArtifactManifest, "digest", + err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", []byte("{}"), imageStore, repoDB, log) So(err, ShouldNotBeNil) }) diff --git a/pkg/meta/version/version_test.go b/pkg/meta/version/version_test.go index 2ffbed001..c0f0e1a0c 100644 --- a/pkg/meta/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -125,7 +125,6 @@ func TestVersioningDynamoDB(t *testing.T) { Region: region, RepoMetaTablename: "RepoMetadataTable", ManifestDataTablename: "ManifestDataTable", - ArtifactDataTablename: "ArtifactDataTable", IndexDataTablename: "IndexDataTable", UserDataTablename: "UserDataTable", VersionTablename: "Version", diff --git a/pkg/storage/common.go b/pkg/storage/common.go index 41d5671ed..890746291 100644 --- a/pkg/storage/common.go +++ b/pkg/storage/common.go @@ -16,10 +16,16 @@ import ( "github.com/sigstore/cosign/v2/pkg/oci/remote" zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/scheduler" storageConstants "zotregistry.io/zot/pkg/storage/constants" ) +const ( + CosignType = "cosign" + NotationType = "notation" +) + func SignatureMediaTypes() map[string]bool { return map[string]bool{ notreg.ArtifactTypeNotation: true, @@ -106,22 +112,6 @@ func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, bo return "", zerr.ErrBadManifest } - case ispec.MediaTypeArtifactManifest: - var artifact ispec.Artifact - if err := json.Unmarshal(body, &artifact); err != nil { - log.Error().Err(err).Msg("unable to unmarshal JSON") - - return "", zerr.ErrBadManifest - } - - if artifact.Subject != nil { - var m ispec.Descriptor - if err := json.Unmarshal(body, &m); err != nil { - log.Error().Err(err).Msg("unable to unmarshal JSON") - - return "", zerr.ErrBadManifest - } - } } return "", nil @@ -303,6 +293,27 @@ func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log return imageIndex, nil } +func GetImageManifest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +) (ispec.Manifest, error) { + var manifestContent ispec.Manifest + + manifestBlob, err := imgStore.GetBlobContent(repo, digest) + if err != nil { + return manifestContent, err + } + + manifestPath := path.Join(imgStore.RootDir(), repo, "blobs", + digest.Algorithm().String(), digest.Encoded()) + + if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil { + log.Error().Err(err).Str("path", manifestPath).Msg("invalid JSON") + + return manifestContent, err + } + + return manifestContent, nil +} + func RemoveManifestDescByReference(index *ispec.Index, reference string, detectCollisions bool, ) (ispec.Descriptor, error) { var removedManifest ispec.Descriptor @@ -456,31 +467,50 @@ func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godig return prunedManifests, nil } -func ApplyLinter(imgStore ImageStore, linter Lint, repo string, manifestDesc ispec.Descriptor) (bool, error) { +func ApplyLinter(imgStore ImageStore, linter Lint, repo string, descriptor ispec.Descriptor) (bool, error) { pass := true - if linter != nil { - tag := manifestDesc.Annotations[ispec.AnnotationRefName] - // apply linter only on images, not signatures - if manifestDesc.MediaType == ispec.MediaTypeImageManifest && - // check that image manifest is not cosign signature - !strings.HasPrefix(tag, "sha256-") && - !strings.HasSuffix(tag, remote.SignatureTagSuffix) { - // lint new index with new manifest before writing to disk - pass, err := linter.Lint(repo, manifestDesc.Digest, imgStore) - if err != nil { - return false, err - } + // we'll skip anything that's not a image manifest + if descriptor.MediaType != ispec.MediaTypeImageManifest { + return pass, nil + } - if !pass { - return false, zerr.ErrImageLintAnnotations - } + if linter != nil && !IsSignature(descriptor) { + // lint new index with new manifest before writing to disk + pass, err := linter.Lint(repo, descriptor.Digest, imgStore) + if err != nil { + return false, err + } + + if !pass { + return false, zerr.ErrImageLintAnnotations } } return pass, nil } +func IsSignature(descriptor ispec.Descriptor) bool { + tag := descriptor.Annotations[ispec.AnnotationRefName] + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + // is cosgin signature + if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) { + return true + } + + // is notation signature + if descriptor.ArtifactType == notreg.ArtifactTypeNotation { + return true + } + default: + return false + } + + return false +} + func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string, log zerolog.Logger, ) ([]oras.Descriptor, error) { @@ -609,67 +639,18 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art } // filter by artifact type - if len(artifactTypes) > 0 { - found := false - - for _, artifactType := range artifactTypes { - if artifactType != "" && mfst.Config.MediaType != artifactType { - continue - } + manifestArtifactType := zcommon.GetManifestArtifactType(mfst) - found = true - - break - } - - if !found { - continue - } - } - - result = append(result, ispec.Descriptor{ - MediaType: manifest.MediaType, - ArtifactType: mfst.Config.MediaType, - Size: manifest.Size, - Digest: manifest.Digest, - Annotations: mfst.Annotations, - }) - } else if manifest.MediaType == ispec.MediaTypeArtifactManifest { - var art ispec.Artifact - if err := json.Unmarshal(buf, &art); err != nil { - log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON") - - return nilIndex, err - } - - if art.Subject == nil || art.Subject.Digest != gdigest { + if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, manifestArtifactType) { continue } - // filter by artifact type - if len(artifactTypes) > 0 { - found := false - for _, artifactType := range artifactTypes { - if artifactType != "" && art.ArtifactType != artifactType { - continue - } - - found = true - - break - } - - if !found { - continue - } - } - result = append(result, ispec.Descriptor{ MediaType: manifest.MediaType, - ArtifactType: art.ArtifactType, + ArtifactType: manifestArtifactType, Size: manifest.Size, Digest: manifest.Digest, - Annotations: art.Annotations, + Annotations: mfst.Annotations, }) } } @@ -719,14 +700,13 @@ func GetOrasManifestByDigest(imgStore ImageStore, repo string, digest godigest.D func IsSupportedMediaType(mediaType string) bool { return mediaType == ispec.MediaTypeImageIndex || mediaType == ispec.MediaTypeImageManifest || - mediaType == ispec.MediaTypeArtifactManifest || mediaType == oras.MediaTypeArtifactManifest } func IsNonDistributable(mediaType string) bool { - return mediaType == ispec.MediaTypeImageLayerNonDistributable || - mediaType == ispec.MediaTypeImageLayerNonDistributableGzip || - mediaType == ispec.MediaTypeImageLayerNonDistributableZstd + return mediaType == ispec.MediaTypeImageLayerNonDistributable || //nolint:staticcheck + mediaType == ispec.MediaTypeImageLayerNonDistributableGzip || //nolint:staticcheck + mediaType == ispec.MediaTypeImageLayerNonDistributableZstd //nolint:staticcheck } // CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function @@ -742,30 +722,30 @@ func IsNonDistributable(mediaType string) bool { func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string, storeController StoreController, ) (bool, string, godigest.Digest, error) { - const cosign = "cosign" - - var manifestContent ispec.Artifact + var manifestContent ispec.Manifest err := json.Unmarshal(manifestBlob, &manifestContent) if err != nil { return false, "", "", err } + manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) + // check notation signature - if _, ok := SignatureMediaTypes()[manifestContent.ArtifactType]; ok && manifestContent.Subject != nil { + if _, ok := SignatureMediaTypes()[manifestArtifactType]; ok && manifestContent.Subject != nil { imgStore := storeController.GetImageStore(repoName) _, signedImageManifestDigest, _, err := imgStore.GetImageManifest(repoName, manifestContent.Subject.Digest.String()) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { - return true, "notation", signedImageManifestDigest, zerr.ErrOrphanSignature + return true, NotationType, signedImageManifestDigest, zerr.ErrOrphanSignature } return false, "", "", err } - return true, "notation", signedImageManifestDigest, nil + return true, NotationType, signedImageManifestDigest, nil } // check cosign @@ -785,17 +765,17 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin signedImageManifestDigest.String()) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { - return true, cosign, signedImageManifestDigest, zerr.ErrOrphanSignature + return true, CosignType, signedImageManifestDigest, zerr.ErrOrphanSignature } return false, "", "", err } if signedImageManifestDigest.String() == "" { - return true, cosign, signedImageManifestDigest, zerr.ErrOrphanSignature + return true, CosignType, signedImageManifestDigest, zerr.ErrOrphanSignature } - return true, cosign, signedImageManifestDigest, nil + return true, CosignType, signedImageManifestDigest, nil } return false, "", "", nil diff --git a/pkg/storage/common_test.go b/pkg/storage/common_test.go index d94a2ddf6..c36761fc6 100644 --- a/pkg/storage/common_test.go +++ b/pkg/storage/common_test.go @@ -118,7 +118,7 @@ func TestValidateManifest(t *testing.T) { }, Layers: []ispec.Descriptor{ { - MediaType: ispec.MediaTypeImageLayerNonDistributable, + MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck Digest: digest, Size: int64(len(content)), }, @@ -299,38 +299,11 @@ func TestGetReferrersErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("Trigger unmarshal error on artifact mediaType", func(c C) { - index = ispec.Index{ - Manifests: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeArtifactManifest, - Digest: digest, - }, - }, - } - - indexBuf, err = json.Marshal(index) - So(err, ShouldBeNil) - - imgStore = &mocks.MockedImageStore{ - GetIndexContentFn: func(repo string) ([]byte, error) { - return indexBuf, nil - }, - GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { - return []byte{}, nil - }, - } - - _, err = storage.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) - So(err, ShouldNotBeNil) - }) - Convey("Trigger nil subject", func(c C) { index = ispec.Index{ Manifests: []ispec.Descriptor{ { - MediaType: ispec.MediaTypeArtifactManifest, + MediaType: ispec.MediaTypeImageManifest, Digest: digest, }, }, @@ -398,3 +371,12 @@ func TestGetImageIndexErrors(t *testing.T) { So(err, ShouldNotBeNil) }) } + +func TestIsSignature(t *testing.T) { + Convey("Unknown media type", t, func(c C) { + isSingature := storage.IsSignature(ispec.Descriptor{ + MediaType: "unknown media type", + }) + So(isSingature, ShouldBeFalse) + }) +} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 2d91be639..b1bd7726a 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -29,7 +29,7 @@ import ( "github.com/sigstore/cosign/v2/pkg/oci/remote" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/common" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" zreg "zotregistry.io/zot/pkg/regexp" @@ -66,7 +66,7 @@ func (is *ImageStoreLocal) RootDir() string { } func (is *ImageStoreLocal) DirExists(d string) bool { - return common.DirExists(d) + return zcommon.DirExists(d) } // NewImageStore returns a new image store backed by a file storage. @@ -535,7 +535,18 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return "", err } - // apply linter only on images, not signatures + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + + err := json.Unmarshal(body, &manifest) + if err != nil { + return "", err + } + + desc.ArtifactType = zcommon.GetManifestArtifactType(manifest) + } + + // apply linter only on images, not signatures or indexes pass, err := storage.ApplyLinter(is, is.linter, repo, desc) if !pass { is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") @@ -1479,10 +1490,24 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { // gather cosign signatures if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) { cosignDescriptors = append(cosignDescriptors, desc) + + continue } } - case ispec.MediaTypeArtifactManifest: - notationDescriptors = append(notationDescriptors, desc) + + manifestContent, err := storage.GetImageManifest(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { + notationDescriptors = append(notationDescriptors, desc) + + continue + } } } @@ -1519,7 +1544,7 @@ func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *is ) error { for _, desc := range index.Manifests { // skip manifests referenced in image indexex - if common.Contains(referencedByImageIndex, desc.Digest.String()) { + if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { continue } @@ -1621,28 +1646,23 @@ func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *i for _, notationDesc := range notationDescriptors { foundSubject := false // check if we can find the manifest which the signature points to - var artManifest ispec.Artifact + var artManifest ispec.Manifest buf, err := imgStore.GetBlobContent(repo, notationDesc.Digest) if err != nil { imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()). - Msg("gc: failed to get oras artifact manifest") + Msg("gc: failed to get notation manifest") return err } if err := json.Unmarshal(buf, &artManifest); err != nil { imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()). - Msg("gc: failed to get oras artifact manifest") + Msg("gc: failed to get notation manifest") return err } - // skip oci artifacts which are not signatures - if artManifest.ArtifactType != notreg.ArtifactTypeNotation { - continue - } - for _, desc := range index.Manifests { if desc.Digest == artManifest.Subject.Digest { foundSubject = true @@ -1808,7 +1828,7 @@ func (is *ImageStoreLocal) GetNextDigestWithBlobPaths(lastDigests []godigest.Dig return nil //nolint:nilerr // ignore files which are not blobs } - if digest == "" && !common.DContains(lastDigests, blobDigest) { + if digest == "" && !zcommon.DContains(lastDigests, blobDigest) { digest = blobDigest } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 1d4a10d87..b18c7e0aa 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -24,7 +24,7 @@ import ( "github.com/rs/zerolog" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/common" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" zreg "zotregistry.io/zot/pkg/regexp" @@ -438,6 +438,17 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return "", err } + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + + err := json.Unmarshal(body, &manifest) + if err != nil { + return "", err + } + + desc.ArtifactType = zcommon.GetManifestArtifactType(manifest) + } + // apply linter only on images, not signatures pass, err := storage.ApplyLinter(is, is.linter, repo, desc) if !pass { @@ -1419,7 +1430,7 @@ func (is *ObjectStorage) GetNextDigestWithBlobPaths(lastDigests []godigest.Diges return nil //nolint:nilerr // ignore files which are not blobs } - if digest == "" && !common.DContains(lastDigests, blobDigest) { + if digest == "" && !zcommon.DContains(lastDigests, blobDigest) { digest = blobDigest } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index e15127032..d4c5c7149 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -584,44 +584,6 @@ func TestGetOrasAndOCIReferrers(t *testing.T) { So(index.Manifests[0].Digest, ShouldEqual, manDigest) }) - Convey("Get oci referrers - application/vnd.oci.artifact.manifest.v1+json", func(c C) { - artifactType := "application/vnd.example.icecream.v1" - - artifactManifest := ispec.Artifact{ - MediaType: ispec.MediaTypeArtifactManifest, - ArtifactType: artifactType, - Blobs: []ispec.Descriptor{ - { - MediaType: "application/octet-stream", - Size: int64(buflen), - Digest: digest, - }, - }, - Subject: &ispec.Descriptor{ - MediaType: ispec.MediaTypeImageManifest, - Size: int64(mbuflen), - Digest: mdigest, - }, - } - - manBuf, err := json.Marshal(artifactManifest) - So(err, ShouldBeNil) - - manBufLen := len(manBuf) - manDigest := godigest.FromBytes(manBuf) - - _, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeArtifactManifest, manBuf) - So(err, ShouldBeNil) - - index, err := imgStore.GetReferrers(repo, mdigest, []string{artifactType}) - So(err, ShouldBeNil) - So(index, ShouldNotBeEmpty) - So(index.Manifests[1].ArtifactType, ShouldEqual, artifactType) - So(index.Manifests[1].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest) - So(index.Manifests[1].Size, ShouldEqual, manBufLen) - So(index.Manifests[1].Digest, ShouldEqual, manDigest) - }) - Convey("Get oras referrers", func(c C) { artifactManifest := artifactspec.Manifest{} artifactManifest.ArtifactType = "signature-example" diff --git a/pkg/test/common.go b/pkg/test/common.go index 47efe0d3b..9c4bdc0f1 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -105,32 +105,6 @@ func (img Image) Digest() (godigest.Digest, error) { return godigest.FromBytes(blob), nil } -type Artifact struct { - Manifest ispec.Artifact - Blobs []ArtifactBlobs - Reference string -} - -func (a Artifact) Digest() (godigest.Digest, error) { - blob, err := json.Marshal(a.Manifest) - if err != nil { - return "", err - } - - return godigest.FromBytes(blob), nil -} - -func (a Artifact) ArtifactData() (repodb.ArtifactData, error) { - blob, err := json.Marshal(a.Manifest) - if err != nil { - return repodb.ArtifactData{}, err - } - - return repodb.ArtifactData{ - ManifestBlob: blob, - }, nil -} - type ArtifactBlobs struct { Blob []byte MediaType string @@ -588,6 +562,7 @@ func GetImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, e schemaVersion := 2 manifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, Versioned: specs.Versioned{ SchemaVersion: schemaVersion, }, @@ -642,6 +617,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif schemaVersion := 2 manifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, Versioned: specs.Versioned{ SchemaVersion: schemaVersion, }, @@ -657,7 +633,6 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif Size: int64(len(layers[0])), }, }, - MediaType: ispec.MediaTypeImageManifest, } return config, layers, manifest, nil @@ -702,6 +677,7 @@ func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispe schemaVersion := 2 manifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, Versioned: specs.Versioned{ SchemaVersion: schemaVersion, }, @@ -760,6 +736,7 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error) const schemaVersion = 2 manifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, Versioned: specs.Versioned{ SchemaVersion: schemaVersion, }, @@ -784,49 +761,6 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error) }, nil } -func GetRandomArtifact(subject *ispec.Descriptor) (Artifact, error) { - var randBlob [10]byte - - _, err := rand.Read(randBlob[:]) - if err != nil { - return Artifact{}, err - } - - artifactBlobs := []ArtifactBlobs{ - { - Blob: randBlob[:], - MediaType: "application/octet-stream", - }, - } - - blobsDescriptors := make([]ispec.Descriptor, 0, len(artifactBlobs)) - - for _, artifactBlob := range artifactBlobs { - blobsDescriptors = append(blobsDescriptors, ispec.Descriptor{ - Digest: godigest.FromBytes(artifactBlob.Blob), - MediaType: artifactBlob.MediaType, - Size: int64(len(artifactBlob.Blob)), - }) - } - - artifactManifest := ispec.Artifact{ - MediaType: ispec.MediaTypeArtifactManifest, - Blobs: blobsDescriptors, - Subject: subject, - } - - artifactManifestBlob, err := json.Marshal(artifactManifest) - if err != nil { - return Artifact{}, err - } - - return Artifact{ - Manifest: artifactManifest, - Blobs: artifactBlobs, - Reference: godigest.FromBytes(artifactManifestBlob).String(), - }, nil -} - func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) { manifestBlob, err := json.Marshal(manifest) if err != nil { @@ -906,6 +840,11 @@ func UploadImage(img Image, baseURL, repo string) error { cdigest := godigest.FromBytes(cblob) + if img.Manifest.Config.MediaType == ispec.MediaTypeScratch { + cblob = ispec.ScratchDescriptor.Data + cdigest = ispec.ScratchDescriptor.Digest + } + resp, err := resty.R(). Post(baseURL + "/v2/" + repo + "/blobs/uploads/") if err = Error(err); err != nil { @@ -940,7 +879,7 @@ func UploadImage(img Image, baseURL, repo string) error { } resp, err = resty.R(). - SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). + SetHeader("Content-type", ispec.MediaTypeImageManifest). SetBody(manifestBlob). Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference) @@ -966,27 +905,6 @@ func DeleteImage(repo, reference, baseURL string) (int, error) { return resp.StatusCode(), err } -// UploadArtifactManifest is used in tests where we don't need to upload the blobs of the artifact. -func UploadArtifactManifest(artifactManifest *ispec.Artifact, ref *string, baseURL, repo string) error { - // put manifest - artifactManifestBlob, err := json.Marshal(artifactManifest) - if err != nil { - return err - } - reference := godigest.FromBytes(artifactManifestBlob).String() - - if ref != nil { - reference = *ref - } - - _, err = resty.R(). - SetHeader("Content-type", ispec.MediaTypeArtifactManifest). - SetBody(artifactManifestBlob). - Put(baseURL + "/v2/" + repo + "/manifests/" + reference) - - return err -} - func UploadBlob(baseURL, repo string, blob []byte, artifactBlobMediaType string) error { resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/") if err != nil { @@ -1252,7 +1170,11 @@ func SignWithNotation(keyName string, reference string, tdir string) error { PlainHTTP: plainHTTP, } - sigRepo := notreg.NewRepository(remoteRepo) + repositoryOpts := notreg.RepositoryOptions{ + OCIImageManifest: true, + } + + sigRepo := notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts) sigOpts := notation.RemoteSignOptions{ SignOptions: notation.SignOptions{ @@ -1334,7 +1256,11 @@ func VerifyWithNotation(reference string, tdir string) error { PlainHTTP: plainHTTP, } - repo := notreg.NewRepository(remoteRepo) + repositoryOpts := notreg.RepositoryOptions{ + OCIImageManifest: true, + } + + repo := notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts) manifestDesc, err := repo.Resolve(ctx, ref.Reference) if err != nil { @@ -1357,7 +1283,7 @@ func VerifyWithNotation(reference string, tdir string) error { PlainHTTP: plainHTTP, } - repo = notreg.NewRepository(remoteRepo) + repo = notreg.NewRepositoryWithOptions(remoteRepo, repositoryOpts) configs := map[string]string{} @@ -1587,6 +1513,11 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e cdigest := godigest.FromBytes(cblob) + if img.Manifest.Config.MediaType == ispec.MediaTypeScratch { + cblob = ispec.ScratchDescriptor.Data + cdigest = ispec.ScratchDescriptor.Digest + } + resp, err := resty.R(). SetBasicAuth(user, password). Post(baseURL + "/v2/" + repo + "/blobs/uploads/") diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 122a60031..24149c76d 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -234,18 +234,6 @@ func TestControllerManager(t *testing.T) { }) } -func TestUploadArtifact(t *testing.T) { - Convey("Put request results in an error", t, func() { - port := test.GetFreePort() - baseURL := test.GetBaseURL(port) - - artifact := ispec.Artifact{} - - err := test.UploadArtifactManifest(&artifact, nil, baseURL, "test") - So(err, ShouldNotBeNil) - }) -} - func TestUploadBlob(t *testing.T) { Convey("Post request results in an error", t, func() { port := test.GetFreePort() diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index a83ebd376..a57611408 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -44,10 +44,6 @@ type RepoDBMock struct { GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error) - SetArtifactDataFn func(digest godigest.Digest, artifactData repodb.ArtifactData) error - - GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error) - SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error @@ -348,22 +344,6 @@ func (sdm RepoDBMock) PatchDB() error { return nil } -func (sdm RepoDBMock) SetArtifactData(digest godigest.Digest, artifactData repodb.ArtifactData) error { - if sdm.SetArtifactDataFn != nil { - return sdm.SetArtifactDataFn(digest, artifactData) - } - - return nil -} - -func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) { - if sdm.GetArtifactDataFn != nil { - return sdm.GetArtifactDataFn(artifactDigest) - } - - return repodb.ArtifactData{}, nil -} - func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { if sdm.SetReferrerFn != nil { return sdm.SetReferrerFn(repo, referredDigest, referrer) diff --git a/pkg/test/oci-layout/oci_layout_test.go b/pkg/test/oci-layout/oci_layout_test.go index c962adb25..d7755a544 100644 --- a/pkg/test/oci-layout/oci_layout_test.go +++ b/pkg/test/oci-layout/oci_layout_test.go @@ -376,7 +376,7 @@ func TestExtractImageDetails(t *testing.T) { olu := ocilayout.NewBaseOciLayoutUtils(storeController, testLogger) resDigest, resManifest, resIspecImage, resErr := olu.ExtractImageDetails("zot-test", "latest", testLogger) - So(string(resDigest), ShouldContainSubstring, "sha256:c52f15d2d4") + So(string(resDigest), ShouldContainSubstring, "sha256:8492645f16") So(resManifest.Config.Digest.String(), ShouldContainSubstring, configDigest.Encoded()) So(resIspecImage.Architecture, ShouldContainSubstring, "amd64") diff --git a/pkg/test/test_http_server.go b/pkg/test/test_http_server.go new file mode 100644 index 000000000..e8f24f9f3 --- /dev/null +++ b/pkg/test/test_http_server.go @@ -0,0 +1,51 @@ +package test + +import ( + "errors" + "fmt" + "net/http" + + "github.com/gorilla/mux" +) + +type RouteHandler struct { + Route string + // HandlerFunc is the HTTP handler function that receives a writer for output and an HTTP request as input. + HandlerFunc http.HandlerFunc + // AllowedMethods specifies the HTTP methods allowed for the current route. + AllowedMethods []string +} + +// Routes is a map that associates HTTP paths to their corresponding HTTP handlers. +type HTTPRoutes []RouteHandler + +func StartTestHTTPServer(routes HTTPRoutes, port string) *http.Server { + baseURL := GetBaseURL(port) + mux := mux.NewRouter() + + mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte("{}")) + if err != nil { + return + } + }).Methods(http.MethodGet) + + for _, routeHandler := range routes { + mux.HandleFunc(routeHandler.Route, routeHandler.HandlerFunc).Methods(routeHandler.AllowedMethods...) + } + + server := &http.Server{ //nolint:gosec + Addr: fmt.Sprintf(":%s", port), + Handler: mux, + } + + go func() { + if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return + } + }() + + WaitTillServerReady(baseURL + "/test") + + return server +} diff --git a/test/blackbox/annotations.bats b/test/blackbox/annotations.bats index 2526be0f3..5f91e14aa 100644 --- a/test/blackbox/annotations.bats +++ b/test/blackbox/annotations.bats @@ -145,7 +145,7 @@ function teardown_file() { } EOF - run notation sign --key "notation-sign-test" --plain-http localhost:8080/annotations:latest + run notation sign --signature-manifest image --key "notation-sign-test" --plain-http localhost:8080/annotations:latest [ "$status" -eq 0 ] run notation verify --plain-http localhost:8080/annotations:latest [ "$status" -eq 0 ] diff --git a/test/blackbox/cloud-only.bats b/test/blackbox/cloud-only.bats index acb8b1560..d4b03c9b3 100644 --- a/test/blackbox/cloud-only.bats +++ b/test/blackbox/cloud-only.bats @@ -37,7 +37,6 @@ function setup() { "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", - "artifactDataTablename": "ArtifactDataTable", "indexDataTablename": "IndexDataTable", "userDataTablename": "UserDataTable", "versionTablename": "Version" diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats index 810f28c5c..b1e534c19 100644 --- a/test/blackbox/pushpull.bats +++ b/test/blackbox/pushpull.bats @@ -120,11 +120,11 @@ function teardown_file() { @test "attach oras artifacts" { # attach signature echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > signature.json - run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --artifact-type 'signature/example' ./signature.json:application/json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json [ "$status" -eq 0 ] # attach sbom echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json - run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --artifact-type 'sbom/example' ./sbom.json:application/json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json [ "$status" -eq 0 ] } @@ -248,7 +248,7 @@ EOF run regctl artifact list localhost:8080/artifact-ref:demo --format raw-body [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.manifests | length') -eq 0 ] - run regctl artifact put --media-type "application/vnd.oci.artifact.manifest.v1+json" --annotation demo=true --annotation format=oci --artifact-type "application/vnd.example.icecream.v1" --subject localhost:8080/artifact-ref:demo << EOF + run regctl artifact put --annotation demo=true --annotation format=oci --artifact-type "application/vnd.example.icecream.v1" --subject localhost:8080/artifact-ref:demo << EOF test reference EOF [ "$status" -eq 0 ] diff --git a/test/blackbox/referrers.bats b/test/blackbox/referrers.bats index 8948b3d10..3ee25151d 100644 --- a/test/blackbox/referrers.bats +++ b/test/blackbox/referrers.bats @@ -49,12 +49,6 @@ EOF This artifact is represented as an ispec image manifest, this is the layer inside the manifest. EOF - ARTIFACT_MANIFEST_REFERRER=${ARTIFACT_BLOBS_DIR}/artifact-manifest-ref-blob - touch ${ARTIFACT_MANIFEST_REFERRER} - cat >${ARTIFACT_MANIFEST_REFERRER} < Date: Wed, 3 May 2023 11:09:45 +0300 Subject: [PATCH 2/2] feat(gc): simplify gc logic to increase coverage Signed-off-by: Laurentiu Niculae --- pkg/api/routes.go | 5 +++- pkg/extensions/sync/sync_test.go | 2 +- pkg/meta/repodb/storage_parsing.go | 2 +- pkg/storage/local/local.go | 42 ++++++++++++------------------ 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index b2c3758b6..65406f4dd 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -514,7 +514,10 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http return } - response.Header().Set("OCI-Filters-Applied", strings.Join(artifactTypes, ",")) + if len(artifactTypes) > 0 { + response.Header().Set("OCI-Filters-Applied", strings.Join(artifactTypes, ",")) + } + WriteData(response, http.StatusOK, ispec.MediaTypeImageIndex, out) } diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 93ab581eb..cdec5210a 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -869,7 +869,7 @@ func TestSyncWithNonDistributableBlob(t *testing.T) { nonDistributableLayer := make([]byte, 10) nonDistributableDigest := godigest.FromBytes(nonDistributableLayer) nonDistributableLayerDesc := ispec.Descriptor{ - MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, + MediaType: ispec.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck Digest: nonDistributableDigest, Size: int64(len(nonDistributableLayer)), URLs: []string{ diff --git a/pkg/meta/repodb/storage_parsing.go b/pkg/meta/repodb/storage_parsing.go index 7f2ec57fb..9a5d97084 100644 --- a/pkg/meta/repodb/storage_parsing.go +++ b/pkg/meta/repodb/storage_parsing.go @@ -325,7 +325,7 @@ func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string, err := json.Unmarshal(descriptorBlob, &manifestContent) if err != nil { return "", referrerInfo, false, - fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err) + fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) } referrerSubject = manifestContent.Subject diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index b1bd7726a..46b6f141a 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1451,6 +1451,12 @@ func ensureDir(dir string, log zerolog.Logger) error { return nil } +type extendedManifest struct { + ispec.Manifest + + Digest godigest.Digest +} + func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { oci, err := umoci.OpenLayout(dir) if err := test.Error(err); err != nil { @@ -1466,7 +1472,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { referencedByImageIndex := []string{} cosignDescriptors := []ispec.Descriptor{} - notationDescriptors := []ispec.Descriptor{} + notationManifests := []extendedManifest{} /* gather manifests references by multiarch images (to skip gc) gather cosign and notation signatures descriptors */ @@ -1504,7 +1510,10 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { } if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { - notationDescriptors = append(notationDescriptors, desc) + notationManifests = append(notationManifests, extendedManifest{ + Digest: desc.Digest, + Manifest: manifestContent, + }) continue } @@ -1525,7 +1534,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { is.log.Info().Msg("gc: notation signatures") - if err := gcNotationSignatures(is, oci, &index, repo, notationDescriptors); err != nil { + if err := gcNotationSignatures(is, oci, &index, repo, notationManifests); err != nil { return err } @@ -1641,41 +1650,24 @@ func gcCosignSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *isp } func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - notationDescriptors []ispec.Descriptor, + notationManifests []extendedManifest, ) error { - for _, notationDesc := range notationDescriptors { + for _, notationManifest := range notationManifests { foundSubject := false - // check if we can find the manifest which the signature points to - var artManifest ispec.Manifest - - buf, err := imgStore.GetBlobContent(repo, notationDesc.Digest) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()). - Msg("gc: failed to get notation manifest") - - return err - } - - if err := json.Unmarshal(buf, &artManifest); err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", notationDesc.Digest.String()). - Msg("gc: failed to get notation manifest") - - return err - } for _, desc := range index.Manifests { - if desc.Digest == artManifest.Subject.Digest { + if desc.Digest == notationManifest.Subject.Digest { foundSubject = true } } if !foundSubject { // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", notationDesc.Digest.String()). + imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()). Msg("gc: removing notation signature without subject") // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = storage.RemoveManifestDescByReference(index, notationDesc.Digest.String(), false) + _, _ = storage.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) err := oci.PutIndex(context.Background(), *index) if err != nil {