From f05962dbcf917b5fbcbee9e58a9f7713b55c8d83 Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 12 Jul 2022 11:26:44 +0200 Subject: [PATCH 1/8] add Interface->fs.FS translation logic --- backends/fs/fs_test.go | 24 ++++- backends/memory/memory_test.go | 4 +- backends/s3/s3_test.go | 9 ++ fs.go | 161 +++++++++++++++++++++++++++++++++ tester/tester.go | 86 ++++++++++++++++++ 5 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 fs.go diff --git a/backends/fs/fs_test.go b/backends/fs/fs_test.go index 67ae80e..d76844b 100644 --- a/backends/fs/fs_test.go +++ b/backends/fs/fs_test.go @@ -10,10 +10,8 @@ import ( "github.com/PowerDNS/simpleblob/tester" ) -func TestBackend(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "simpleblob-test-") - assert.NoError(t, err) - t.Cleanup(func() { +func cleanup(t *testing.T, tmpDir string) func() { + return func() { // Don't want to use the recursive os.RemoveAll() for safety if tmpDir == "" { return @@ -28,9 +26,25 @@ func TestBackend(t *testing.T) { } err = os.Remove(tmpDir) assert.NoError(t, err) - }) + } +} + +func TestBackend(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "simpleblob-test-") + assert.NoError(t, err) + t.Cleanup(cleanup(t, tmpDir)) b, err := New(Options{RootPath: tmpDir}) assert.NoError(t, err) tester.DoBackendTests(t, b) } + +func TestFilesystem(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "simpleblob-test-") + assert.NoError(t, err) + t.Cleanup(cleanup(t, tmpDir)) + + b, err := New(Options{RootPath: tmpDir}) + assert.NoError(t, err) + tester.DoFSWrapperTests(t, b) +} diff --git a/backends/memory/memory_test.go b/backends/memory/memory_test.go index a6e6a7d..f3e663f 100644 --- a/backends/memory/memory_test.go +++ b/backends/memory/memory_test.go @@ -7,6 +7,6 @@ import ( ) func TestBackend(t *testing.T) { - b := New() - tester.DoBackendTests(t, b) + tester.DoBackendTests(t, New()) + tester.DoFSWrapperTests(t, New()) } diff --git a/backends/s3/s3_test.go b/backends/s3/s3_test.go index 7ac5c07..851ecb0 100644 --- a/backends/s3/s3_test.go +++ b/backends/s3/s3_test.go @@ -103,3 +103,12 @@ func TestBackend_marker(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "bar-1", string(data)) } + +func TestFilesystem(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + + b := getBackend(ctx, t) + tester.DoFSWrapperTests(t, b) + assert.Equal(t, "", b.lastMarker) +} diff --git a/fs.go b/fs.go new file mode 100644 index 0000000..01ee40e --- /dev/null +++ b/fs.go @@ -0,0 +1,161 @@ +package simpleblob + +import ( + "bytes" + "context" + "errors" + "io" + "io/fs" + "os" + "path" + "strings" + "time" +) + +// fsInterfaceWrapper wraps an Interface and implements fs.FS. +type fsInterfaceWrapper struct{ Interface } + +// fsBlobWrapper represents data upstream and implements both fs.File +// and fs.FileInfo for convenience. +type fsBlobWrapper struct { + b *Blob + parent *fsInterfaceWrapper + r *bytes.Reader +} + +// AsFS casts the provided interface to a fs.FS interface if supported, +// else it wraps it to replicate its functionalities. +func AsFS(st Interface) fs.FS { + if fsys, ok := st.(fs.FS); ok { + return fsys + } + return &fsInterfaceWrapper{st} +} + +// find gets a Blob in wrapped Interface and returns it wrapped as a fsBlobWrapper. +func (stw *fsInterfaceWrapper) find(name string) (*fsBlobWrapper, error) { + blobs, err := stw.Interface.List(context.TODO(), name) + if err != nil { + return nil, err + } + for _, b := range blobs { + if b.Name == name { + return &fsBlobWrapper{&b, stw, nil}, nil + } + } + return nil, os.ErrNotExist +} + +// Open retrieves a Blob, wrapped as a fs.File, from the underlying Interface. +func (stw *fsInterfaceWrapper) Open(name string) (fs.File, error) { + if ret, err := stw.find(name); err != nil { + return nil, err + } else if ret != nil { + return ret, nil + } + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: os.ErrNotExist, + } +} + +// ReadFile implements fs.ReadFileFS on top of an Interface wrapped as a fs.FS. +func (stw *fsInterfaceWrapper) ReadFile(name string) ([]byte, error) { + return stw.Load(context.Background(), name) +} + +// Stat implements fs.StatFS on top of an Interface wrapped as a fs.FS. +func (stw *fsInterfaceWrapper) Stat(name string) (fs.FileInfo, error) { + if ret, err := stw.find(name); err != nil { + return nil, err + } else if ret != nil { + return ret.Stat() + } + return nil, nil +} + +func checkDirPath(name, op string) *fs.PathError { + if name != "" && name != "." { + return &fs.PathError{Op: op, Path: name, Err: errors.New("no subdirectory available")} + } + return nil +} + +// ReadDir satisfies fs.ReadDirFS +func (stw *fsInterfaceWrapper) ReadDir(name string) ([]fs.DirEntry, error) { + if err := checkDirPath(name, "readdir"); err != nil { + return nil, err + } + ls, err := stw.List(context.Background(), "") + if err != nil { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} + } + ret := make([]fs.DirEntry, len(ls)) + for i, entry := range ls { + ret[i] = &fsBlobWrapper{&entry, stw, nil} + } + return ret, nil +} + +// Glob satisfies fs.GlobFS +func (stw *fsInterfaceWrapper) Glob(pattern string) ([]string, error) { + var prefix string + if pattern != "" && !strings.ContainsAny(pattern, `*?[\`) { + prefix = pattern + } + ls, err := stw.List(context.Background(), prefix) + if err != nil { + return nil, &fs.PathError{Op: "glob", Path: pattern, Err: err} + } + var ret []string + for _, b := range ls { + if ok, _ := path.Match(pattern, b.Name); ok { + ret = append(ret, b.Name) + } + } + return ret, nil +} + +// Sub satisfies fs.SubFS +func (stw *fsInterfaceWrapper) Sub(dir string) (fs.FS, error) { + if err := checkDirPath(dir, "sub"); err != nil { + return nil, err + } + return stw, nil +} + +// fs.FileInfo implementation + +func (*fsBlobWrapper) IsDir() bool { return false } +func (*fsBlobWrapper) ModTime() time.Time { return time.Time{} } +func (*fsBlobWrapper) Mode() fs.FileMode { return 666 } +func (bw *fsBlobWrapper) Name() string { return bw.b.Name } +func (bw *fsBlobWrapper) Sys() interface{} { return bw.parent } +func (bw *fsBlobWrapper) Size() int64 { return bw.b.Size } + +// fs.File implementation + +func (bw *fsBlobWrapper) Stat() (fs.FileInfo, error) { + return bw, nil +} +func (bw *fsBlobWrapper) Read(p []byte) (int, error) { + if bw.r == nil { + b, err := bw.parent.Interface.Load(context.TODO(), bw.b.Name) + if err != nil { + return 0, err + } + bw.r = bytes.NewReader(b) + } + n, err := bw.r.Read(p) + if err == io.EOF { + bw.r = nil + } + return n, err +} +func (bw *fsBlobWrapper) Close() error { return nil } + +// fs.DirEntry implementation + +func (bw *fsBlobWrapper) Type() fs.FileMode { return bw.Mode() } +func (bw *fsBlobWrapper) Info() (fs.FileInfo, error) { return bw, nil } diff --git a/tester/tester.go b/tester/tester.go index f1e8ed4..13fcbc0 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -2,11 +2,15 @@ package tester import ( "context" + "crypto/rand" + "io" + "io/fs" "os" "testing" "github.com/PowerDNS/simpleblob" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // DoBackendTests tests a backend for conformance @@ -72,3 +76,85 @@ func DoBackendTests(t *testing.T, b simpleblob.Interface) { _, err = b.Load(ctx, "does-not-exist") assert.ErrorIs(t, err, os.ErrNotExist) } + +// DoFSWrapperTests confronts Interface to its fs.FS implementations +func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { + // Wrap provided interface into a filesystem + // and use the backend to check operations on the filesystem. + // The backend is considered working from DoBackendTests. + fsys := simpleblob.AsFS(b) + + // Filesystem is empty + ls, err := fs.Glob(fsys, "*") + assert.NoError(t, err) + assert.Len(t, ls, 0) + + // Opening random thing fails + f, err := fsys.Open("something") + assert.Error(t, err) + assert.Nil(t, f) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Test single item + fooData := make([]byte, 64) + _, err = rand.Read(fooData) + require.NoError(t, err) + b.Store(ctx, "foo", fooData) // error is ignored, Store is tested above + + // Item isn't listed when prefix is bad + ls, err = fs.Glob(fsys, "bar") + assert.NoError(t, err) + assert.Len(t, ls, 0) + + // Item is listed + ls, err = fs.Glob(fsys, "*") + assert.NoError(t, err) + assert.Len(t, ls, 1) + assert.Contains(t, ls, "foo") + + // Item can be loaded by name + f, err = fsys.Open("foo") + assert.NoError(t, err) + assert.NotNil(t, f) + defer func() { + assert.NoError(t, f.Close()) + }() + + // Item has right content + p, err := io.ReadAll(f) + assert.Equal(t, p, fooData) + + // Check file info + info, err := f.Stat() + assert.NoError(t, err) + assert.EqualValues(t, info.Mode(), 666) + assert.Equal(t, info.Name(), "foo") + assert.EqualValues(t, info.Size(), 64) + assert.Equal(t, info.Sys(), fsys) + + // fs.ReadFileFS is satisfied + p2, err := fs.ReadFile(fsys, "meh") + assert.Error(t, err) + assert.Empty(t, p2) + p2, err = fs.ReadFile(fsys, "foo") + assert.NoError(t, err) + assert.Equal(t, p, p2) + + // fs.ReadDirFS is satisfied and allows only "." subdir + direntries, err := fs.ReadDir(fsys, "meh") + assert.Error(t, err) + assert.Nil(t, direntries) + direntries, err = fs.ReadDir(fsys, ".") + assert.NoError(t, err) + assert.Contains(t, direntries, f) + + // fs.SubFS is satisfied and allows only "." subdir + subfsys, err := fs.Sub(fsys, "anything") + assert.Error(t, err) + assert.Nil(t, subfsys) + subfsys, err = fs.Sub(fsys, ".") + assert.NoError(t, err) + assert.Equal(t, fsys, subfsys) +} From 36ae907ad62f73f19b0e4be99b92b12d08de5a0c Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Wed, 13 Jul 2022 14:49:10 +0200 Subject: [PATCH 2/8] comply with golangci-lint for tester --- tester/tester.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tester/tester.go b/tester/tester.go index 13fcbc0..4719ccf 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -101,7 +101,8 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { fooData := make([]byte, 64) _, err = rand.Read(fooData) require.NoError(t, err) - b.Store(ctx, "foo", fooData) // error is ignored, Store is tested above + err = b.Store(ctx, "foo", fooData) + require.NoError(t, err) // Item isn't listed when prefix is bad ls, err = fs.Glob(fsys, "bar") @@ -123,7 +124,8 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { }() // Item has right content - p, err := io.ReadAll(f) + var p []byte + p, err = io.ReadAll(f) assert.Equal(t, p, fooData) // Check file info From aad39815e13a16d5fdde3782591736810bef700d Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Wed, 13 Jul 2022 14:58:52 +0200 Subject: [PATCH 3/8] fix ineffassign, continuing prev commit --- tester/tester.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tester/tester.go b/tester/tester.go index 4719ccf..43b621d 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -126,6 +126,7 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { // Item has right content var p []byte p, err = io.ReadAll(f) + require.NoError(t, err) assert.Equal(t, p, fooData) // Check file info From 7f96f7bbfcdf661424eeacf166c3f1b5205933fc Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 19 Jul 2022 11:42:54 +0200 Subject: [PATCH 4/8] fix FileMode being in decimal to octal --- fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs.go b/fs.go index 01ee40e..2b3d6e3 100644 --- a/fs.go +++ b/fs.go @@ -129,7 +129,7 @@ func (stw *fsInterfaceWrapper) Sub(dir string) (fs.FS, error) { func (*fsBlobWrapper) IsDir() bool { return false } func (*fsBlobWrapper) ModTime() time.Time { return time.Time{} } -func (*fsBlobWrapper) Mode() fs.FileMode { return 666 } +func (*fsBlobWrapper) Mode() fs.FileMode { return 0666 } func (bw *fsBlobWrapper) Name() string { return bw.b.Name } func (bw *fsBlobWrapper) Sys() interface{} { return bw.parent } func (bw *fsBlobWrapper) Size() int64 { return bw.b.Size } From 58ec33141386e131e50bf6f404b1babe7f1ebeb0 Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 19 Jul 2022 12:17:38 +0200 Subject: [PATCH 5/8] reset underlying reader for `fs.File` implementation on Close only --- fs.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fs.go b/fs.go index 2b3d6e3..d358e4d 100644 --- a/fs.go +++ b/fs.go @@ -141,19 +141,20 @@ func (bw *fsBlobWrapper) Stat() (fs.FileInfo, error) { } func (bw *fsBlobWrapper) Read(p []byte) (int, error) { if bw.r == nil { - b, err := bw.parent.Interface.Load(context.TODO(), bw.b.Name) + b, err := bw.parent.Interface.Load(context.Background(), bw.b.Name) if err != nil { return 0, err } bw.r = bytes.NewReader(b) } - n, err := bw.r.Read(p) - if err == io.EOF { - bw.r = nil + return bw.r.Read(p) +} +func (bw *fsBlobWrapper) Close() error { + if bw.r != nil { + bw.r = nil } - return n, err + return nil } -func (bw *fsBlobWrapper) Close() error { return nil } // fs.DirEntry implementation From 30f8c037e84cc4135460159c64243cc69d90a093 Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 19 Jul 2022 13:13:29 +0200 Subject: [PATCH 6/8] remove optional fs interfaces implementation where they're not added value --- fs.go | 99 +++--------------------------------------------- tester/tester.go | 34 +---------------- 2 files changed, 6 insertions(+), 127 deletions(-) diff --git a/fs.go b/fs.go index d358e4d..37694b8 100644 --- a/fs.go +++ b/fs.go @@ -3,12 +3,7 @@ package simpleblob import ( "bytes" "context" - "errors" - "io" "io/fs" - "os" - "path" - "strings" "time" ) @@ -32,32 +27,13 @@ func AsFS(st Interface) fs.FS { return &fsInterfaceWrapper{st} } -// find gets a Blob in wrapped Interface and returns it wrapped as a fsBlobWrapper. -func (stw *fsInterfaceWrapper) find(name string) (*fsBlobWrapper, error) { - blobs, err := stw.Interface.List(context.TODO(), name) - if err != nil { - return nil, err - } - for _, b := range blobs { - if b.Name == name { - return &fsBlobWrapper{&b, stw, nil}, nil - } - } - return nil, os.ErrNotExist -} - // Open retrieves a Blob, wrapped as a fs.File, from the underlying Interface. func (stw *fsInterfaceWrapper) Open(name string) (fs.File, error) { - if ret, err := stw.find(name); err != nil { - return nil, err - } else if ret != nil { - return ret, nil - } - return nil, &fs.PathError{ - Op: "open", - Path: name, - Err: os.ErrNotExist, + b, err := stw.Load(context.Background(), name) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} } + return &fsBlobWrapper{&Blob{name, int64(len(b))}, stw, nil}, nil } // ReadFile implements fs.ReadFileFS on top of an Interface wrapped as a fs.FS. @@ -65,71 +41,11 @@ func (stw *fsInterfaceWrapper) ReadFile(name string) ([]byte, error) { return stw.Load(context.Background(), name) } -// Stat implements fs.StatFS on top of an Interface wrapped as a fs.FS. -func (stw *fsInterfaceWrapper) Stat(name string) (fs.FileInfo, error) { - if ret, err := stw.find(name); err != nil { - return nil, err - } else if ret != nil { - return ret.Stat() - } - return nil, nil -} - -func checkDirPath(name, op string) *fs.PathError { - if name != "" && name != "." { - return &fs.PathError{Op: op, Path: name, Err: errors.New("no subdirectory available")} - } - return nil -} - -// ReadDir satisfies fs.ReadDirFS -func (stw *fsInterfaceWrapper) ReadDir(name string) ([]fs.DirEntry, error) { - if err := checkDirPath(name, "readdir"); err != nil { - return nil, err - } - ls, err := stw.List(context.Background(), "") - if err != nil { - return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} - } - ret := make([]fs.DirEntry, len(ls)) - for i, entry := range ls { - ret[i] = &fsBlobWrapper{&entry, stw, nil} - } - return ret, nil -} - -// Glob satisfies fs.GlobFS -func (stw *fsInterfaceWrapper) Glob(pattern string) ([]string, error) { - var prefix string - if pattern != "" && !strings.ContainsAny(pattern, `*?[\`) { - prefix = pattern - } - ls, err := stw.List(context.Background(), prefix) - if err != nil { - return nil, &fs.PathError{Op: "glob", Path: pattern, Err: err} - } - var ret []string - for _, b := range ls { - if ok, _ := path.Match(pattern, b.Name); ok { - ret = append(ret, b.Name) - } - } - return ret, nil -} - -// Sub satisfies fs.SubFS -func (stw *fsInterfaceWrapper) Sub(dir string) (fs.FS, error) { - if err := checkDirPath(dir, "sub"); err != nil { - return nil, err - } - return stw, nil -} - // fs.FileInfo implementation func (*fsBlobWrapper) IsDir() bool { return false } func (*fsBlobWrapper) ModTime() time.Time { return time.Time{} } -func (*fsBlobWrapper) Mode() fs.FileMode { return 0666 } +func (*fsBlobWrapper) Mode() fs.FileMode { return 0777 } func (bw *fsBlobWrapper) Name() string { return bw.b.Name } func (bw *fsBlobWrapper) Sys() interface{} { return bw.parent } func (bw *fsBlobWrapper) Size() int64 { return bw.b.Size } @@ -155,8 +71,3 @@ func (bw *fsBlobWrapper) Close() error { } return nil } - -// fs.DirEntry implementation - -func (bw *fsBlobWrapper) Type() fs.FileMode { return bw.Mode() } -func (bw *fsBlobWrapper) Info() (fs.FileInfo, error) { return bw, nil } diff --git a/tester/tester.go b/tester/tester.go index 43b621d..70aa77a 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -84,11 +84,6 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { // The backend is considered working from DoBackendTests. fsys := simpleblob.AsFS(b) - // Filesystem is empty - ls, err := fs.Glob(fsys, "*") - assert.NoError(t, err) - assert.Len(t, ls, 0) - // Opening random thing fails f, err := fsys.Open("something") assert.Error(t, err) @@ -104,17 +99,6 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { err = b.Store(ctx, "foo", fooData) require.NoError(t, err) - // Item isn't listed when prefix is bad - ls, err = fs.Glob(fsys, "bar") - assert.NoError(t, err) - assert.Len(t, ls, 0) - - // Item is listed - ls, err = fs.Glob(fsys, "*") - assert.NoError(t, err) - assert.Len(t, ls, 1) - assert.Contains(t, ls, "foo") - // Item can be loaded by name f, err = fsys.Open("foo") assert.NoError(t, err) @@ -132,7 +116,7 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { // Check file info info, err := f.Stat() assert.NoError(t, err) - assert.EqualValues(t, info.Mode(), 666) + assert.EqualValues(t, info.Mode(), 0777) assert.Equal(t, info.Name(), "foo") assert.EqualValues(t, info.Size(), 64) assert.Equal(t, info.Sys(), fsys) @@ -144,20 +128,4 @@ func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { p2, err = fs.ReadFile(fsys, "foo") assert.NoError(t, err) assert.Equal(t, p, p2) - - // fs.ReadDirFS is satisfied and allows only "." subdir - direntries, err := fs.ReadDir(fsys, "meh") - assert.Error(t, err) - assert.Nil(t, direntries) - direntries, err = fs.ReadDir(fsys, ".") - assert.NoError(t, err) - assert.Contains(t, direntries, f) - - // fs.SubFS is satisfied and allows only "." subdir - subfsys, err := fs.Sub(fsys, "anything") - assert.Error(t, err) - assert.Nil(t, subfsys) - subfsys, err = fs.Sub(fsys, ".") - assert.NoError(t, err) - assert.Equal(t, fsys, subfsys) } From 3c22efaf021151ba472e41a053080dbcdbedd403 Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 19 Jul 2022 17:06:26 +0200 Subject: [PATCH 7/8] bring fs tests together with backend tests, add context to wrapper --- backends/fs/fs_test.go | 10 --- backends/memory/memory_test.go | 1 - backends/s3/s3_test.go | 9 --- fs.go | 34 +++++++-- tester/tester.go | 122 ++++++++++++++++++--------------- 5 files changed, 96 insertions(+), 80 deletions(-) diff --git a/backends/fs/fs_test.go b/backends/fs/fs_test.go index d76844b..e608a7a 100644 --- a/backends/fs/fs_test.go +++ b/backends/fs/fs_test.go @@ -38,13 +38,3 @@ func TestBackend(t *testing.T) { assert.NoError(t, err) tester.DoBackendTests(t, b) } - -func TestFilesystem(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "simpleblob-test-") - assert.NoError(t, err) - t.Cleanup(cleanup(t, tmpDir)) - - b, err := New(Options{RootPath: tmpDir}) - assert.NoError(t, err) - tester.DoFSWrapperTests(t, b) -} diff --git a/backends/memory/memory_test.go b/backends/memory/memory_test.go index f3e663f..c5ebe98 100644 --- a/backends/memory/memory_test.go +++ b/backends/memory/memory_test.go @@ -8,5 +8,4 @@ import ( func TestBackend(t *testing.T) { tester.DoBackendTests(t, New()) - tester.DoFSWrapperTests(t, New()) } diff --git a/backends/s3/s3_test.go b/backends/s3/s3_test.go index 851ecb0..7ac5c07 100644 --- a/backends/s3/s3_test.go +++ b/backends/s3/s3_test.go @@ -103,12 +103,3 @@ func TestBackend_marker(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "bar-1", string(data)) } - -func TestFilesystem(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - t.Cleanup(cancel) - - b := getBackend(ctx, t) - tester.DoFSWrapperTests(t, b) - assert.Equal(t, "", b.lastMarker) -} diff --git a/fs.go b/fs.go index 37694b8..c9429bd 100644 --- a/fs.go +++ b/fs.go @@ -8,7 +8,10 @@ import ( ) // fsInterfaceWrapper wraps an Interface and implements fs.FS. -type fsInterfaceWrapper struct{ Interface } +type fsInterfaceWrapper struct { + Interface + ctx context.Context +} // fsBlobWrapper represents data upstream and implements both fs.File // and fs.FileInfo for convenience. @@ -20,25 +23,39 @@ type fsBlobWrapper struct { // AsFS casts the provided interface to a fs.FS interface if supported, // else it wraps it to replicate its functionalities. -func AsFS(st Interface) fs.FS { +func AsFS(ctx context.Context, st Interface) fs.FS { if fsys, ok := st.(fs.FS); ok { return fsys } - return &fsInterfaceWrapper{st} + return &fsInterfaceWrapper{st, ctx} } // Open retrieves a Blob, wrapped as a fs.File, from the underlying Interface. func (stw *fsInterfaceWrapper) Open(name string) (fs.File, error) { - b, err := stw.Load(context.Background(), name) + b, err := stw.Load(stw.ctx, name) if err != nil { return nil, &fs.PathError{Op: "open", Path: name, Err: err} } return &fsBlobWrapper{&Blob{name, int64(len(b))}, stw, nil}, nil } +// ReadDir satisfies fs.ReadDirFS +func (stw *fsInterfaceWrapper) ReadDir(name string) ([]fs.DirEntry, error) { + ls, err := stw.List(stw.ctx, "") + if err != nil { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} + } + ret := make([]fs.DirEntry, len(ls)) + for i, entry := range ls { + blob := entry + ret[i] = &fsBlobWrapper{&blob, stw, nil} + } + return ret, nil +} + // ReadFile implements fs.ReadFileFS on top of an Interface wrapped as a fs.FS. func (stw *fsInterfaceWrapper) ReadFile(name string) ([]byte, error) { - return stw.Load(context.Background(), name) + return stw.Load(stw.ctx, name) } // fs.FileInfo implementation @@ -57,7 +74,7 @@ func (bw *fsBlobWrapper) Stat() (fs.FileInfo, error) { } func (bw *fsBlobWrapper) Read(p []byte) (int, error) { if bw.r == nil { - b, err := bw.parent.Interface.Load(context.Background(), bw.b.Name) + b, err := bw.parent.Interface.Load(bw.parent.ctx, bw.b.Name) if err != nil { return 0, err } @@ -71,3 +88,8 @@ func (bw *fsBlobWrapper) Close() error { } return nil } + +// fs.DirEntry implementation + +func (bw *fsBlobWrapper) Type() fs.FileMode { return bw.Mode() } +func (bw *fsBlobWrapper) Info() (fs.FileInfo, error) { return bw, nil } diff --git a/tester/tester.go b/tester/tester.go index 70aa77a..cb4cd2b 100644 --- a/tester/tester.go +++ b/tester/tester.go @@ -2,7 +2,6 @@ package tester import ( "context" - "crypto/rand" "io" "io/fs" "os" @@ -10,7 +9,6 @@ import ( "github.com/PowerDNS/simpleblob" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // DoBackendTests tests a backend for conformance @@ -18,10 +16,19 @@ func DoBackendTests(t *testing.T, b simpleblob.Interface) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Wrap provided interface into a filesystem + // and use the backend to check operations on the filesystem. + // Such operations will be performed along the ones on the backend. + fsys := simpleblob.AsFS(context.Background(), b) + // Starts empty ls, err := b.List(ctx, "") assert.NoError(t, err) assert.Len(t, ls, 0) + // With FS + dirls, err := fs.ReadDir(fsys, ".") + assert.NoError(t, err) + assert.Len(t, dirls, 0) // Add items foo := []byte("foo") // will be modified later @@ -37,9 +44,17 @@ func DoBackendTests(t *testing.T, b simpleblob.Interface) { assert.NoError(t, err) // List all + expectedNames := []string{"bar-1", "bar-2", "foo-1"} // sorted ls, err = b.List(ctx, "") assert.NoError(t, err) - assert.Equal(t, ls.Names(), []string{"bar-1", "bar-2", "foo-1"}) // sorted + assert.Equal(t, ls.Names(), expectedNames) + // With FS + dirls, err = fs.ReadDir(fsys, ".") + assert.NoError(t, err) + assert.Len(t, dirls, len(expectedNames)) + for i, entry := range dirls { + assert.Equal(t, expectedNames[i], entry.Name()) + } // List with prefix ls, err = b.List(ctx, "foo-") @@ -54,78 +69,77 @@ func DoBackendTests(t *testing.T, b simpleblob.Interface) { data, err := b.Load(ctx, "foo-1") assert.NoError(t, err) assert.Equal(t, data, []byte("foo")) + // With FS + f, err := fsys.Open("foo-1") + assert.NoError(t, err) + dataf, err := io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, data, dataf) + assert.NoError(t, f.Close()) + // With ReadFileFS + datar, err := fs.ReadFile(fsys, "foo-1") + assert.NoError(t, err) + assert.Equal(t, data, datar) // Check overwritten data data, err = b.Load(ctx, "bar-1") assert.NoError(t, err) assert.Equal(t, data, []byte("bar1")) + // With FS + f, err = fsys.Open("bar-1") + assert.NoError(t, err) + dataf, err = io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, data, dataf) + assert.NoError(t, f.Close()) + // With ReadFileFS + datar, err = fs.ReadFile(fsys, "bar-1") + assert.NoError(t, err) + assert.Equal(t, data, datar) // Verify that Load makes a copy data[0] = '!' data, err = b.Load(ctx, "bar-1") assert.NoError(t, err) assert.Equal(t, data, []byte("bar1")) + // With FS + f, err = fsys.Open("bar-1") + assert.NoError(t, err) + dataf, err = io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, data, dataf) + assert.NoError(t, f.Close()) + // With ReadFileFS + datar, err = fs.ReadFile(fsys, "bar-1") + assert.NoError(t, err) + assert.Equal(t, data, datar) // Change foo buffer to verify that Store made a copy foo[0] = '!' data, err = b.Load(ctx, "foo-1") assert.NoError(t, err) assert.Equal(t, data, []byte("foo")) + // With FS + f, err = fsys.Open("foo-1") + assert.NoError(t, err) + dataf, err = io.ReadAll(f) + assert.NoError(t, err) + assert.Equal(t, data, dataf) + assert.NoError(t, f.Close()) + // With ReadFileFS + datar, err = fs.ReadFile(fsys, "foo-1") + assert.NoError(t, err) + assert.Equal(t, data, datar) // Load non-existing _, err = b.Load(ctx, "does-not-exist") assert.ErrorIs(t, err, os.ErrNotExist) -} - -// DoFSWrapperTests confronts Interface to its fs.FS implementations -func DoFSWrapperTests(t *testing.T, b simpleblob.Interface) { - // Wrap provided interface into a filesystem - // and use the backend to check operations on the filesystem. - // The backend is considered working from DoBackendTests. - fsys := simpleblob.AsFS(b) - - // Opening random thing fails - f, err := fsys.Open("something") - assert.Error(t, err) + // With FS + f, err = fsys.Open("something") + assert.ErrorIs(t, err, os.ErrNotExist) assert.Nil(t, f) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Test single item - fooData := make([]byte, 64) - _, err = rand.Read(fooData) - require.NoError(t, err) - err = b.Store(ctx, "foo", fooData) - require.NoError(t, err) - - // Item can be loaded by name - f, err = fsys.Open("foo") - assert.NoError(t, err) - assert.NotNil(t, f) - defer func() { - assert.NoError(t, f.Close()) - }() - - // Item has right content - var p []byte - p, err = io.ReadAll(f) - require.NoError(t, err) - assert.Equal(t, p, fooData) - - // Check file info - info, err := f.Stat() - assert.NoError(t, err) - assert.EqualValues(t, info.Mode(), 0777) - assert.Equal(t, info.Name(), "foo") - assert.EqualValues(t, info.Size(), 64) - assert.Equal(t, info.Sys(), fsys) - - // fs.ReadFileFS is satisfied - p2, err := fs.ReadFile(fsys, "meh") + // With ReadFileFS + datar, err = fs.ReadFile(fsys, "does-not-exist-either") assert.Error(t, err) - assert.Empty(t, p2) - p2, err = fs.ReadFile(fsys, "foo") - assert.NoError(t, err) - assert.Equal(t, p, p2) + assert.Empty(t, datar) } From be29704596a08626a0330c211e37a488602552f4 Mon Sep 17 00:00:00 2001 From: Rodrigue Geis Date: Tue, 19 Jul 2022 17:12:06 +0200 Subject: [PATCH 8/8] format fs.go --- fs.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fs.go b/fs.go index c9429bd..ccc469f 100644 --- a/fs.go +++ b/fs.go @@ -41,16 +41,16 @@ func (stw *fsInterfaceWrapper) Open(name string) (fs.File, error) { // ReadDir satisfies fs.ReadDirFS func (stw *fsInterfaceWrapper) ReadDir(name string) ([]fs.DirEntry, error) { - ls, err := stw.List(stw.ctx, "") - if err != nil { - return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} - } - ret := make([]fs.DirEntry, len(ls)) - for i, entry := range ls { - blob := entry - ret[i] = &fsBlobWrapper{&blob, stw, nil} - } - return ret, nil + ls, err := stw.List(stw.ctx, "") + if err != nil { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: err} + } + ret := make([]fs.DirEntry, len(ls)) + for i, entry := range ls { + blob := entry + ret[i] = &fsBlobWrapper{&blob, stw, nil} + } + return ret, nil } // ReadFile implements fs.ReadFileFS on top of an Interface wrapped as a fs.FS.