refactor(artifact): remove oci artifact support

errors/errors.go

@@ -63,7 +63,6 @@ var (
 	ErrManifestConflict         = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound     = errors.New("repodb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound     = errors.New("repodb: image data not found for given manifest digest")
-	ErrArtifactDataNotFound     = errors.New("repodb: artifact data not found for given digest")
 	ErrIndexDataNotFount        = errors.New("repodb: index data not found for given digest")
 	ErrRepoMetaNotFound         = errors.New("repodb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound          = errors.New("repodb: tag metadata not found for given repo and tag names")

examples/config-dynamodb.json

@@ -20,7 +20,6 @@
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
             "manifestDataTablename": "ZotManifestDataTable",
-            "artifactDataTablename": "ZotArtifactDataTable",
             "versionTablename": "ZotVersion"
         }
     },

go.mod

@@ -494,6 +494,7 @@ require (
 )
 
 replace (
+	github.com/opencontainers/image-spec => github.com/project-zot/image-spec v0.0.0-20230407150525-6fa4879e23d2
 	github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef
 	// v0.3.1-0.20230104082527-d6f58551be3f is taken from github.com/moby/buildkit v0.11.0
 	// spdx logic write on v0.3.0 and incompatible with v0.3.1-0.20230104082527-d6f58551be3f

go.sum

@@ -1826,11 +1826,6 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ
 github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.1.0-rc2.0.20221020182949-4df8887994e8 h1:l9vfzobI7tZtG164u1Jf6NqDErHZoqAw8rlvBYQJpVI=
-github.com/opencontainers/image-spec v1.1.0-rc2.0.20221020182949-4df8887994e8/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
@@ -1920,6 +1915,8 @@ github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
 github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef h1:PMJXsXS3x5tiwsRrFgMSnjKmeH5c9ppFXrS5NKdIPEo=
 github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef/go.mod h1:smZzWUkoRRHivXkcHQN7n+KGHVfR+HKJrtCaI8cMifg=
+github.com/project-zot/image-spec v0.0.0-20230407150525-6fa4879e23d2 h1:Ai5WR7z82o9nNk2XC5MHz0dZkV01zKHOoW339Bn0EY0=
+github.com/project-zot/image-spec v0.0.0-20230407150525-6fa4879e23d2/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=

pkg/api/controller_test.go

@@ -51,7 +51,6 @@ import (
 	"zotregistry.io/zot/pkg/log"
 	"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
 	"zotregistry.io/zot/pkg/storage"
-	storageConstants "zotregistry.io/zot/pkg/storage/constants"
 	"zotregistry.io/zot/pkg/storage/local"
 	"zotregistry.io/zot/pkg/test"
 )
@@ -3457,6 +3456,7 @@ func TestCrossRepoMount(t *testing.T) {
 		// wait for dedupe task to run
 		time.Sleep(15 * time.Second)
 
+		// TODO: What is happening here?
 		params["mount"] = string(manifestDigest)
 		postResponse, err = client.R().
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
@@ -4156,6 +4156,8 @@ func TestImageSignatures(t *testing.T) {
 	})
 }
 
+// TODO: Refactor
+// Here are notes to follow:
 func TestArtifactReferences(t *testing.T) {
 	Convey("Validate Artifact References", t, func() {
 		// start a new server
@@ -4300,138 +4302,6 @@ func TestArtifactReferences(t *testing.T) {
 				So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
 			})
 		})
-
-		Convey("Validate Artifact Manifest Reference", func() {
-			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-			var referrers ispec.Index
-			err = json.Unmarshal(resp.Body(), &referrers)
-			So(err, ShouldBeNil)
-			So(referrers.Manifests, ShouldBeEmpty)
-
-			// now upload a reference
-
-			// upload image config blob
-			resp, err = resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName))
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
-			loc := test.Location(baseURL, resp)
-			cblob, cdigest := test.GetEmptyImageConfig()
-
-			resp, err = resty.R().
-				SetContentLength(true).
-				SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
-				SetHeader("Content-Type", "application/octet-stream").
-				SetQueryParam("digest", cdigest.String()).
-				SetBody(cblob).
-				Put(loc)
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
-
-			// create a artifact
-			manifest := ispec.Artifact{
-				MediaType:    ispec.MediaTypeArtifactManifest,
-				ArtifactType: artifactType,
-				Blobs: []ispec.Descriptor{
-					{
-						MediaType: "application/vnd.oci.image.layer.v1.tar",
-						Digest:    digest,
-						Size:      int64(len(content)),
-					},
-				},
-				Subject: &ispec.Descriptor{
-					MediaType: ispec.MediaTypeImageManifest,
-					Digest:    digest,
-					Size:      int64(len(content)),
-				},
-				Annotations: map[string]string{
-					"key": "val",
-				},
-			}
-			Convey("Using invalid content", func() {
-				content := []byte("invalid data")
-				So(err, ShouldBeNil)
-				mdigest := godigest.FromBytes(content)
-				So(mdigest, ShouldNotBeNil)
-				resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
-					SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest)
-
-				// unknown repo will return status not found
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", "unknown", digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
-
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-			})
-			Convey("Using valid content", func() {
-				content, err = json.Marshal(manifest)
-				So(err, ShouldBeNil)
-				mdigest := godigest.FromBytes(content)
-				So(mdigest, ShouldNotBeNil)
-				resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
-					SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
-
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifact": "invalid"}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": "invalid"}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-				So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
-
-				var index ispec.Index
-				err = json.Unmarshal(resp.Body(), &index)
-				So(err, ShouldBeNil)
-				So(index.Manifests, ShouldNotBeEmpty)
-				So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
-
-				// filter by multiple artifactTypes
-				req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet,
-					baseURL+fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()), nil)
-				So(err, ShouldBeNil)
-				values := url.Values{}
-				values.Add("artifactType", artifactType)
-				values.Add("artifactType", "foobar")
-				req.URL.RawQuery = values.Encode()
-				rsp, err := http.DefaultClient.Do(req)
-				So(err, ShouldBeNil)
-				defer rsp.Body.Close()
-				So(rsp.StatusCode, ShouldEqual, http.StatusOK)
-				So(rsp.Header.Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
-				body, err := io.ReadAll(rsp.Body)
-				So(err, ShouldBeNil)
-				err = json.Unmarshal(body, &index)
-				So(err, ShouldBeNil)
-				So(index.Manifests, ShouldNotBeEmpty)
-				So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
-				So(len(strings.Split(index.Annotations[storageConstants.ReferrerFilterAnnotation], ",")), ShouldEqual, 2)
-			})
-		})
 	})
 }
 

pkg/cli/client.go

@@ -490,10 +490,8 @@ func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf se
 		return false
 	}
 
-	for _, reference := range referrers.Manifests {
-		if reference.ArtifactType == notreg.ArtifactTypeNotation {
-			return true
-		}
+	if len(referrers.Manifests) > 0 {
+		return true
 	}
 
 	return false

pkg/common/common.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 
+	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"zotregistry.io/zot/pkg/log"
 )
 
@@ -245,3 +246,11 @@ func DContains(slice []digest.Digest, item digest.Digest) bool {
 
 	return false
 }
+
+func GetManifestArtifactType(manifestContent ispec.Manifest) string {
+	if manifestContent.ArtifactType != "" {
+		return manifestContent.ArtifactType
+	}
+
+	return manifestContent.Config.MediaType
+}

pkg/extensions/search/common/common_test.go

@@ -21,6 +21,7 @@ import (
 	dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
 	"github.com/gobwas/glob"
 	regTypes "github.com/google/go-containerregistry/pkg/v1/types"
+	notreg "github.com/notaryproject/notation-go/registry"
 	godigest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -31,6 +32,7 @@ import (
 	"zotregistry.io/zot/pkg/api"
 	"zotregistry.io/zot/pkg/api/config"
 	"zotregistry.io/zot/pkg/api/constants"
+	zcommon "zotregistry.io/zot/pkg/common"
 	extconf "zotregistry.io/zot/pkg/extensions/config"
 	"zotregistry.io/zot/pkg/extensions/monitoring"
 	"zotregistry.io/zot/pkg/extensions/search/common"
@@ -995,30 +997,37 @@ func TestGetReferrersGQL(t *testing.T) {
 		artifactContentBlobDigest := godigest.FromBytes(artifactContentBlob)
 		artifactType := "com.artifact.test"
 
-		err = UploadBlob(baseURL, repo, artifactContentBlob, artifactContentType)
-		So(err, ShouldBeNil)
-
-		artifact := &ispec.Artifact{
-			Blobs: []ispec.Descriptor{
-				{
-					MediaType: artifactContentType,
-					Digest:    artifactContentBlobDigest,
-					Size:      artifactContentBlobSize,
+		artifactImg := Image{
+			Manifest: ispec.Manifest{
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: artifactContentType,
+						Digest:    artifactContentBlobDigest,
+						Size:      artifactContentBlobSize,
+					},
+				},
+				Subject:      subjectDescriptor,
+				ArtifactType: artifactType,
+				Config: ispec.Descriptor{
+					MediaType: ispec.MediaTypeScratch,
+					Digest:    ispec.ScratchDigestSHA256,
+					Data:      []byte(ispec.ScratchDigestData),
+				},
+				MediaType: ispec.MediaTypeImageManifest,
+				Annotations: map[string]string{
+					"com.artifact.format": "test",
 				},
 			},
-			Subject:      subjectDescriptor,
-			ArtifactType: artifactType,
-			MediaType:    ispec.MediaTypeArtifactManifest,
-			Annotations: map[string]string{
-				"com.artifact.format": "test",
-			},
+			Config: ispec.Image{},
+			Layers: [][]byte{artifactContentBlob},
 		}
 
-		artifactManifestBlob, err := json.Marshal(artifact)
+		artifactManifestBlob, err := json.Marshal(artifactImg.Manifest)
 		So(err, ShouldBeNil)
 		artifactManifestDigest := godigest.FromBytes(artifactManifestBlob)
+		artifactImg.Reference = artifactManifestDigest.String()
 
-		err = UploadArtifactManifest(artifact, nil, baseURL, repo)
+		err = UploadImage(artifactImg, baseURL, repo)
 		So(err, ShouldBeNil)
 
 		gqlQuery := `
@@ -1052,7 +1061,7 @@ func TestGetReferrersGQL(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(referrersResp.Errors, ShouldBeNil)
 		So(referrersResp.ReferrersResult.Referrers[0].ArtifactType, ShouldEqual, artifactType)
-		So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest)
+		So(referrersResp.ReferrersResult.Referrers[0].MediaType, ShouldEqual, ispec.MediaTypeImageManifest)
 
 		So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Key, ShouldEqual, "com.artifact.format")
 		So(referrersResp.ReferrersResult.Referrers[0].Annotations[0].Value, ShouldEqual, "test")
@@ -1555,17 +1564,15 @@ func TestExpandedRepoInfo(t *testing.T) {
 		err = UploadImage(image, baseURL, "repo")
 		So(err, ShouldBeNil)
 
-		referrer, err := GetRandomArtifact(&ispec.Descriptor{
-			Digest:    manifestDigest,
-			MediaType: ispec.MediaTypeImageManifest,
-		})
+		referrer, err := GetImageWithSubject(manifestDigest, ispec.MediaTypeImageManifest)
 		So(err, ShouldBeNil)
 
 		tag := "test-ref-tag"
-		err = UploadArtifactManifest(&referrer.Manifest, &tag, baseURL, "repo")
+		referrer.Reference = tag
+		err = UploadImage(referrer, baseURL, "repo")
 		So(err, ShouldBeNil)
 
-		// ------- Make the call to GQL and see that it doesn't crash and that the referrer isn't in the list of tags
+		// ------- Make the call to GQL and see that it doesn't crash
 		responseStruct := &ExpandedRepoInfoResp{}
 		query := `
 		{
@@ -1587,10 +1594,23 @@ func TestExpandedRepoInfo(t *testing.T) {
 
 		err = json.Unmarshal(resp.Body(), responseStruct)
 		So(err, ShouldBeNil)
-		So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 1)
+		So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 2)
 
 		repoInfo := responseStruct.ExpandedRepoInfo.RepoInfo
-		So(repoInfo.ImageSummaries[0].Tag, ShouldEqual, "test")
+
+		foundTagTest := false
+		foundTagRefTag := false
+
+		for _, imgSum := range repoInfo.ImageSummaries {
+			switch imgSum.Tag {
+			case "test":
+				foundTagTest = true
+			case "test-ref-tag":
+				foundTagRefTag = true
+			}
+		}
+
+		So(foundTagTest || foundTagRefTag, ShouldEqual, true)
 	})
 
 	Convey("Test image tags order", t, func() {
@@ -5779,10 +5799,17 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
 
 			signatureReference := ""
 
-			var sigManifestContent ispec.Artifact
+			var sigManifestContent ispec.Manifest
 
 			for _, manifest := range indexContent.Manifests {
-				if manifest.MediaType == ispec.MediaTypeArtifactManifest {
+				manifestBlob, _, _, err := storage.GetImageManifest(repo, manifest.Digest.String())
+				So(err, ShouldBeNil)
+				var manifestContent ispec.Manifest
+
+				err = json.Unmarshal(manifestBlob, &manifestContent)
+				So(err, ShouldBeNil)
+
+				if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation {
 					signatureReference = manifest.Digest.String()
 					manifestBlob, _, _, err := storage.GetImageManifest(repo, signatureReference)
 					So(err, ShouldBeNil)