Skip to content

Commit

Permalink
finished implementation
Browse files Browse the repository at this point in the history 
Signed-off-by: Xiaoxuan Wang <wangxiaoxuan119@gmail.com>
  • Loading branch information
@wangxiaoxuan273
wangxiaoxuan273 committed Dec 25, 2023
1 parent dc0a65e commit 5654c23
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 146 deletions.
75 changes: 31 additions & 44 deletions content/oci/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,25 +158,21 @@ func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error {
s.sync.Lock()
defer s.sync.Unlock()

return s.delete(ctx, target, s.AutoGC)
}

func (s *Store) delete(ctx context.Context, target ocispec.Descriptor, autoGC bool) error {
// delete one node
danglings, err := s.deleteNode(ctx, target)
// delete the node itself
danglings, err := s.delete(ctx, target)
if err != nil {
return err
}

// delete the dangling nodes caused by the delete
if autoGC {
if s.AutoGC {
return s.cleanGraphs(ctx, danglings)
}
return nil
}

// deleteNode deletes one node and returns the dangling nodes created by the delete.
func (s *Store) deleteNode(ctx context.Context, target ocispec.Descriptor) ([]ocispec.Descriptor, error) {
// delete deletes one node and returns the dangling nodes created by the delete.
func (s *Store) delete(ctx context.Context, target ocispec.Descriptor) ([]ocispec.Descriptor, error) {
resolvers := s.tagResolver.Map()
untagged := false
for reference, desc := range resolvers {
Expand Down Expand Up @@ -206,7 +202,7 @@ func (s *Store) cleanGraphs(ctx context.Context, danglings []ocispec.Descriptor)
danglings = danglings[1:]
// don't delete the node if it exists in index.json
if _, err := s.tagResolver.Resolve(ctx, string(node.Digest)); err != nil {
descs, err := s.deleteNode(ctx, node)
descs, err := s.delete(ctx, node)
if err != nil {
return err
}
Expand Down Expand Up @@ -414,31 +410,6 @@ func (s *Store) saveIndex() error {
return s.writeIndexFile()
}

// // GC deletes dangling blobs in the store.
// func (s *Store) GC(ctx context.Context) error {
// s.sync.Lock()
// defer s.sync.Unlock()

// danglings := s.graph.ScanForGarbage()
// for _, desc := range danglings {
// if s.isGarbage(ctx, desc) {
// err := s.delete(ctx, desc, true)
// if err != nil {
// return err
// }
// }
// }
// return nil
// }

// func (s *Store) isGarbage(ctx context.Context, desc ocispec.Descriptor) bool {
// if !descriptor.IsManifest(desc) {
// return true
// }
// _, err := s.tagResolver.Resolve(ctx, string(desc.Digest))
// return err != nil
// }

// writeIndexFile writes the `index.json` file.
func (s *Store) writeIndexFile() error {
indexJSON, err := json.Marshal(s.index)
Expand All @@ -448,17 +419,33 @@ func (s *Store) writeIndexFile() error {
return os.WriteFile(s.indexPath, indexJSON, 0666)
}

// draft
func (s *Store) scan(ctx context.Context) {
blobpath := filepath.Join(s.root, "blobs", "sha256")
filepath.Walk(blobpath, func(path string, info fs.FileInfo, err error) error {
// skip root directory
// GC removes garbage from Store. The garbage to be cleaned are:
// - garbage blobs in the storage whose metadata is not stored in Store.
// - unreferenced (dangling) blobs in Store which have no predecessors
func (s *Store) GC(ctx context.Context) error {
rootpath := filepath.Join(s.root, "blobs", "sha256")
return filepath.Walk(rootpath, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// skip the working directory
if !info.IsDir() {
blobDigest := digest.NewDigestFromEncoded("sha256", info.Name())
fmt.Println("digest: ", blobDigest)
_, err := s.tagResolver.Resolve(ctx, string(blobDigest))
if err != nil {
fmt.Println("isDangling: ", s.graph.IsDangling(blobDigest))
exists, desc := s.graph.ExistsInStore(blobDigest)
if !exists {
// remove the blob from storage if it does not exist in Store
err = os.Remove(path)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("%s: %w", blobDigest, errdef.ErrNotFound)
}
return err
}
} else if !s.graph.HasPredecessors(blobDigest) {
if _, err := s.tagResolver.Resolve(ctx, string(blobDigest)); err != nil {
// remove the blob and its metadata from the Store
s.delete(ctx, desc)
}
}
}
return nil
Expand Down
144 changes: 45 additions & 99 deletions content/oci/oci_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2238,99 +2238,7 @@ func TestStore_PredecessorsAndDelete(t *testing.T) {
}
}

// func TestStore_GC(t *testing.T) {
// tempDir := t.TempDir()
// s, err := New(tempDir)
// if err != nil {
// t.Fatal("New() error =", err)
// }
// ctx := context.Background()

// // generate test content
// var blobs [][]byte
// var descs []ocispec.Descriptor
// appendBlob := func(mediaType string, blob []byte) {
// blobs = append(blobs, blob)
// descs = append(descs, ocispec.Descriptor{
// MediaType: mediaType,
// Digest: digest.FromBytes(blob),
// Size: int64(len(blob)),
// })
// }
// generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) {
// manifest := ocispec.Manifest{
// Config: config,
// Layers: layers,
// }
// manifestJSON, err := json.Marshal(manifest)
// if err != nil {
// t.Fatal(err)
// }
// appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
// }
// generateIndex := func(manifests ...ocispec.Descriptor) {
// index := ocispec.Index{
// Manifests: manifests,
// }
// indexJSON, err := json.Marshal(index)
// if err != nil {
// t.Fatal(err)
// }
// appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
// }

// appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
// appendBlob(ocispec.MediaTypeImageLayer, []byte("blob1")) // Blob 1
// appendBlob(ocispec.MediaTypeImageLayer, []byte("dangling layer 1")) // Blob 2
// appendBlob(ocispec.MediaTypeImageLayer, []byte("blob2")) // Blob 3
// appendBlob(ocispec.MediaTypeImageLayer, []byte("dangling layer 2")) // Blob 4
// generateManifest(descs[0], descs[1]) // Blob 5
// generateManifest(descs[0], descs[3]) // Blob 6
// generateIndex(descs[5]) // Blob 7

// eg, egCtx := errgroup.WithContext(ctx)
// for i := range blobs {
// eg.Go(func(i int) func() error {
// return func() error {
// err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
// if err != nil {
// return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
// }
// return nil
// }
// }(i))
// }
// if err := eg.Wait(); err != nil {
// t.Fatal(err)
// }

// if err := s.GC(egCtx); err != nil {
// t.Fatal(err)
// }

// // verify that blob2 and blob4 are deleted
// existence := []bool{
// true, // Blob 0
// true, // Blob 1
// false, // Blob 2
// true, // Blob 3
// false, // Blob 4
// true, // Blob 5
// true, // Blob 6
// true, // Blob 7
// }
// for i, want := range existence {
// exists, err := s.Exists(egCtx, descs[i])
// if err != nil {
// t.Fatal(err)
// }
// if exists != want {
// t.Fatalf("expect existence of desc[%d] to be %v, got %v", i, want, exists)
// }
// }
// }

func TestStore_GCdraft(t *testing.T) {
func TestStore_GC(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
Expand Down Expand Up @@ -2361,13 +2269,19 @@ func TestStore_GCdraft(t *testing.T) {
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("blob1")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("dangling layer 1")) // Blob 2
generateManifest(descs[0], descs[1]) // Blob 5
appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("dangling layer")) // Blob 2, dangling layer
generateManifest(descs[0], descs[1]) // Blob 3
appendBlob(ocispec.MediaTypeImageLayer, []byte("garbage layer 1")) // Blob 4, garbage layer 1
generateManifest(descs[0], descs[4]) // Blob 5, garbage manifest 1
appendBlob(ocispec.MediaTypeImageConfig, []byte("garbage config")) // Blob 6, garbage config
appendBlob(ocispec.MediaTypeImageLayer, []byte("garbage layer 2")) // Blob 7, garbage layer 2
generateManifest(descs[6], descs[7]) // Blob 8, garbage manifest 2

// push blobs[0] - blobs[3] into s
eg, egCtx := errgroup.WithContext(ctx)
for i := range blobs {
for i := 0; i <= 3; i++ {
eg.Go(func(i int) func() error {
return func() error {
err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
Expand All @@ -2382,7 +2296,39 @@ func TestStore_GCdraft(t *testing.T) {
t.Fatal(err)
}

s.scan(egCtx)
// push blobs[4] - blobs[8] into s.storage, making them garbage as their metadata
// doesn't exist in s
for i := 4; i < len(blobs); i++ {
eg.Go(func(i int) func() error {
return func() error {
err := s.storage.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
if err != nil {
return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
}
return nil
}
}(i))
}
if err := eg.Wait(); err != nil {
t.Fatal(err)
}

// perform GC
if err = s.GC(egCtx); err != nil {
t.Fatal(err)
}

// verify existence
wantExistence := []bool{true, true, false, true, false, false, false, false, false}
for i, wantValue := range wantExistence {
exists, err := s.Exists(egCtx, descs[i])
if err != nil {
t.Fatal(err)
}
if exists != wantValue {
t.Fatalf("want existence %d to be %v, got %v", i, wantValue, exists)
}
}
}

func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool {
Expand Down
13 changes: 10 additions & 3 deletions internal/graph/memory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,15 @@ func (m *Memory) index(ctx context.Context, fetcher content.Fetcher, node ocispe
return successors, nil
}

func (m *Memory) IsDangling(dgst digest.Digest) bool {
_, exists := m.nodes[dgst]
// ExistsInStore returns if a blob denoted by its digest exists in the memory.
// The descriptor associated with the digest is also returned.
func (m *Memory) ExistsInStore(dgst digest.Digest) (bool, ocispec.Descriptor) {
desc, exists := m.nodes[dgst]
return exists, desc
}

// HasPredecessors returns if a blob denoted by its digest has any predecessors.
func (m *Memory) HasPredecessors(dgst digest.Digest) bool {
_, hasPredecessors := m.predecessors[dgst]
return exists && !hasPredecessors
return hasPredecessors
}

0 comments on commit 5654c23

Please sign in to comment.