diff --git a/CHANGELOG.md b/CHANGELOG.md index ea4014fcdb3..11d84f1edf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [CHANGE] Remove S3 automatic bucket creation [#404](https://github.com/grafana/tempo/pull/404) * [CHANGE] Compactors should round robin tenants instead of choosing randomly [#420](https://github.com/grafana/tempo/issues/420) * [CHANGE] Switch distributor->ingester communication to more efficient PushBytes method. This is a **breaking change** when running in microservices mode with separate distributors and ingesters. To prevent errors ingesters must be fully upgraded first, then distributors. +* [CHANGE] Removed disk_cache. This is a **breaking change** b/c there is no disk cache. Please use redis or memcached. [#441](https://github.com/grafana/tempo/pull/441) * [ENHANCEMENT] Add docker-compose example for GCS along with new backend options [#397](https://github.com/grafana/tempo/pull/397) * [ENHANCEMENT] tempo-cli list blocks usability improvements [#403](https://github.com/grafana/tempo/pull/403) * [ENHANCEMENT] Add Query Frontend module to allow scaling the query path [#400](https://github.com/grafana/tempo/pull/400) diff --git a/go.mod b/go.mod index a07f5c4eadf..6509200ac4e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/hashicorp/go-hclog v0.14.0 github.com/jaegertracing/jaeger v1.18.2-0.20200707061226-97d2319ff2be github.com/jsternberg/zap-logfmt v1.0.0 - github.com/karrick/godirwalk v1.16.1 github.com/minio/minio-go/v7 v7.0.5 github.com/olekukonko/tablewriter v0.0.2 github.com/open-telemetry/opentelemetry-proto v0.4.0 diff --git a/go.sum b/go.sum index 5bba99aaafb..a95744b7196 100644 --- a/go.sum +++ b/go.sum @@ -1181,8 +1181,6 @@ github.com/karrick/godirwalk v1.7.7/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46s github.com/karrick/godirwalk v1.7.8/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= diff --git a/tempodb/backend/diskcache/cache.go b/tempodb/backend/diskcache/cache.go deleted file mode 100644 index c16eba95c6d..00000000000 --- a/tempodb/backend/diskcache/cache.go +++ /dev/null @@ -1,141 +0,0 @@ -package diskcache - -import ( - "context" - "fmt" - "os" - "strconv" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/grafana/tempo/tempodb/backend" - - "github.com/google/uuid" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -type bloomMissFunc func(ctx context.Context, blockID uuid.UUID, tenantID string, bloomShard int) ([]byte, error) -type indexMissFunc func(ctx context.Context, blockID uuid.UUID, tenantID string) ([]byte, error) - -const ( - typeBloom = "bloom" - typeIndex = "index" -) - -var ( - metricDiskCacheMiss = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tempodb", - Name: "disk_cache_miss_total", - Help: "Total number of times the disk cache missed.", - }, []string{"type"}) - metricDiskCache = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tempodb", - Name: "disk_cache_total", - Help: "Total number of times the disk cache was queried.", - }, []string{"type", "status"}) - metricDiskCacheClean = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tempodb", - Name: "disk_cache_clean_total", - Help: "Total number of times a disk clean has occurred.", - }, []string{"status"}) -) - -type reader struct { - cfg *Config - next backend.Reader - - logger log.Logger - stopCh chan struct{} -} - -func New(next backend.Reader, cfg *Config, logger log.Logger) (backend.Reader, error) { - // cleanup disk cache dir - err := os.RemoveAll(cfg.Path) - if err != nil { - return nil, err - } - err = os.MkdirAll(cfg.Path, os.ModePerm) - if err != nil { - return nil, err - } - - if cfg.DiskPruneCount == 0 { - return nil, fmt.Errorf("must specify disk prune count") - } - - if cfg.DiskCleanRate == 0 { - return nil, fmt.Errorf("must specify a clean rate") - } - - if cfg.MaxDiskMBs == 0 { - return nil, fmt.Errorf("must specify a maximum number of MBs to save") - } - - r := &reader{ - cfg: cfg, - next: next, - stopCh: make(chan struct{}), - logger: logger, - } - - r.startJanitor() - - return r, nil -} - -func (r *reader) Tenants(ctx context.Context) ([]string, error) { - return r.next.Tenants(ctx) -} - -func (r *reader) Blocks(ctx context.Context, tenantID string) ([]uuid.UUID, error) { - return r.next.Blocks(ctx, tenantID) -} - -func (r *reader) BlockMeta(ctx context.Context, blockID uuid.UUID, tenantID string) (*backend.BlockMeta, error) { - return r.next.BlockMeta(ctx, blockID, tenantID) -} - -func (r *reader) Bloom(ctx context.Context, blockID uuid.UUID, tenantID string, bloomShard int) ([]byte, error) { - b, skippableErr, err := r.readOrCacheBloom(ctx, blockID, tenantID, bloomShard, r.next.Bloom) - - if skippableErr != nil { - metricDiskCache.WithLabelValues(typeBloom, "error").Inc() - level.Error(r.logger).Log("err", skippableErr) - } else { - metricDiskCache.WithLabelValues(typeBloom, "success").Inc() - } - - return b, err -} - -func (r *reader) Index(ctx context.Context, blockID uuid.UUID, tenantID string) ([]byte, error) { - b, skippableErr, err := r.readOrCacheIndex(ctx, blockID, tenantID, r.next.Index) - - if skippableErr != nil { - metricDiskCache.WithLabelValues(typeIndex, "error").Inc() - level.Error(r.logger).Log("err", skippableErr) - } else { - metricDiskCache.WithLabelValues(typeIndex, "success").Inc() - } - - return b, err -} - -func (r *reader) Object(ctx context.Context, blockID uuid.UUID, tenantID string, start uint64, buffer []byte) error { - // not attempting to cache these...yet... - return r.next.Object(ctx, blockID, tenantID, start, buffer) -} - -func (r *reader) Shutdown() { - r.stopCh <- struct{}{} - r.next.Shutdown() -} - -func key(blockID uuid.UUID, tenantID string, t string) string { - return blockID.String() + ":" + tenantID + ":" + t -} - -func bloomKey(blockID uuid.UUID, tenantID string, t string, shardNum int) string { - return blockID.String() + ":" + tenantID + ":" + t + ":" + strconv.Itoa(shardNum) -} diff --git a/tempodb/backend/diskcache/config.go b/tempodb/backend/diskcache/config.go deleted file mode 100644 index 2ecb5fff6c8..00000000000 --- a/tempodb/backend/diskcache/config.go +++ /dev/null @@ -1,10 +0,0 @@ -package diskcache - -import "time" - -type Config struct { - Path string `yaml:"disk_path"` - MaxDiskMBs int `yaml:"disk_max_mbs"` - DiskPruneCount int `yaml:"disk_prune_count"` - DiskCleanRate time.Duration `yaml:"disk_clean_rate"` -} diff --git a/tempodb/backend/diskcache/disk_cache.go b/tempodb/backend/diskcache/disk_cache.go deleted file mode 100644 index c1e702b41b7..00000000000 --- a/tempodb/backend/diskcache/disk_cache.go +++ /dev/null @@ -1,204 +0,0 @@ -package diskcache - -import ( - "container/heap" - "context" - "io/ioutil" - "os" - "path/filepath" - "syscall" - "time" - - "github.com/go-kit/kit/log/level" - "github.com/google/uuid" - "github.com/karrick/godirwalk" -) - -// TODO: factor out common code with readOrCacheIndexToDisk into separate function -func (r *reader) readOrCacheBloom(ctx context.Context, blockID uuid.UUID, tenantID string, shardNum int, miss bloomMissFunc) ([]byte, error, error) { - var skippableError error - - k := bloomKey(blockID, tenantID, typeBloom, shardNum) - filename := filepath.Join(r.cfg.Path, k) - - bytes, err := ioutil.ReadFile(filename) - - if err != nil && !os.IsNotExist(err) { - skippableError = err - } - - if bytes != nil { - return bytes, nil, nil - } - - metricDiskCacheMiss.WithLabelValues(typeBloom).Inc() - bytes, err = miss(ctx, blockID, tenantID, shardNum) - if err != nil { - return nil, nil, err // backend store error. need to bubble this up - } - - if bytes != nil { - err = r.writeKeyToDisk(filename, bytes) - if err != nil { - skippableError = err - } - } - - return bytes, skippableError, nil -} - -func (r *reader) readOrCacheIndex(ctx context.Context, blockID uuid.UUID, tenantID string, miss indexMissFunc) ([]byte, error, error) { - var skippableError error - - k := key(blockID, tenantID, typeIndex) - filename := filepath.Join(r.cfg.Path, k) - - bytes, err := ioutil.ReadFile(filename) - - if err != nil && !os.IsNotExist(err) { - skippableError = err - } - - if bytes != nil { - return bytes, nil, nil - } - - metricDiskCacheMiss.WithLabelValues(typeIndex).Inc() - bytes, err = miss(ctx, blockID, tenantID) - if err != nil { - return nil, nil, err // backend store error. need to bubble this up - } - - if bytes != nil { - err = r.writeKeyToDisk(filename, bytes) - if err != nil { - skippableError = err - } - } - - return bytes, skippableError, nil -} - -func (r *reader) writeKeyToDisk(filename string, b []byte) error { - return ioutil.WriteFile(filename, b, 0644) -} - -func (r *reader) startJanitor() { - go func() { - ticker := time.NewTicker(r.cfg.DiskCleanRate) - for { - select { - case <-ticker.C: - // repeatedly clean until we don't need to - cleaned := true - for cleaned { - var err error - cleaned, err = clean(r.cfg.Path, r.cfg.MaxDiskMBs, r.cfg.DiskPruneCount) - if err != nil { - metricDiskCacheClean.WithLabelValues("error").Inc() - level.Error(r.logger).Log("msg", "error cleaning cache dir", "err", err) - } else { - metricDiskCacheClean.WithLabelValues("success").Inc() - } - } - case <-r.stopCh: - return - } - } - }() -} - -/* simplify */ -func clean(folder string, allowedMBs int, pruneCount int) (bool, error) { - - var totalSize int64 - fileInfoHeap := FileInfoHeap(make([]os.FileInfo, 0, pruneCount)) - heap.Init(&fileInfoHeap) - - err := godirwalk.Walk(folder, &godirwalk.Options{ - Callback: func(osPathname string, de *godirwalk.Dirent) error { - if de.IsDir() { - return nil - } - - info, err := os.Stat(osPathname) - if err != nil { - return err - } - - totalSize += info.Size() - - for len(fileInfoHeap) >= cap(fileInfoHeap) { - heap.Pop(&fileInfoHeap) - } - heap.Push(&fileInfoHeap, info) - return nil - }, - Unsorted: true, - }) - - if err != nil { - return false, err - } - - if totalSize < int64(allowedMBs*1024*1024) { - return false, nil - } - - // prune oldest files - for fileInfoHeap.Len() > 0 { - info := heap.Pop(&fileInfoHeap).(os.FileInfo) - if info == nil { - continue - } - - err = os.Remove(filepath.Join(folder, info.Name())) - } - - return true, err -} - -type FileInfoHeap []os.FileInfo - -func (h FileInfoHeap) Len() int { - return len(h) -} - -func (h FileInfoHeap) Less(i, j int) bool { - iInfo := h[i] - jInfo := h[j] - - if iInfo == nil { - return false - } - if jInfo == nil { - return true - } - - iStat, iOK := iInfo.Sys().(*syscall.Stat_t) - jStat, jOK := jInfo.Sys().(*syscall.Stat_t) - - if iOK && jOK { - return AtimeNano(iStat) > AtimeNano(jStat) - } - - return iInfo.ModTime().After(jInfo.ModTime()) -} - -func (h FileInfoHeap) Swap(i, j int) { - h[i], h[j] = h[j], h[i] -} - -func (h *FileInfoHeap) Push(x interface{}) { - item := x.(os.FileInfo) - *h = append(*h, item) -} - -func (h *FileInfoHeap) Pop() interface{} { - old := *h - n := len(old) - item := old[n-1] - old[n-1] = nil // avoid memory leak - *h = old[0 : n-1] - return item -} diff --git a/tempodb/backend/diskcache/disk_cache_test.go b/tempodb/backend/diskcache/disk_cache_test.go deleted file mode 100644 index 65e4de924c9..00000000000 --- a/tempodb/backend/diskcache/disk_cache_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package diskcache - -import ( - "context" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" -) - -const ( - testTenantID = "fake" -) - -func TestReadOrCache(t *testing.T) { - tempDir, err := ioutil.TempDir("/tmp", "") - defer os.RemoveAll(tempDir) - assert.NoError(t, err, "unexpected error creating temp dir") - - missBytes := []byte{0x01} - indexMissCalled := 0 - bloomMissCalled := 0 - - // indexMiss function to be called when the key is not cached - indexMiss := func(ctx context.Context, blockID uuid.UUID, tenantID string) ([]byte, error) { - indexMissCalled++ - return missBytes, nil - } - // bloomMiss function to be called when the bloomKey is not cached - bloomMiss := func(ctx context.Context, blockID uuid.UUID, tenantID string, shardNum int) ([]byte, error) { - bloomMissCalled++ - return missBytes, nil - } - - // create new cache - cache, err := New(nil, &Config{ - Path: tempDir, - MaxDiskMBs: 1024, - DiskPruneCount: 10, - DiskCleanRate: time.Hour, - }, nil) - assert.NoError(t, err) - - blockID := uuid.New() - tenantID := testTenantID - - // get key from cache - bytes, skippableErr, err := cache.(*reader).readOrCacheIndex(context.Background(), blockID, tenantID, indexMiss) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 1, indexMissCalled) - - // make sure the missFunc is not called again since the key is cached already - bytes, skippableErr, err = cache.(*reader).readOrCacheIndex(context.Background(), blockID, tenantID, indexMiss) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 1, indexMissCalled) - - // get key from cache - bytes, skippableErr, err = cache.(*reader).readOrCacheBloom(context.Background(), blockID, tenantID, 0, bloomMiss) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 1, bloomMissCalled) - - // make sure the missFunc is not called again since the key is cached already - bytes, skippableErr, err = cache.(*reader).readOrCacheBloom(context.Background(), blockID, tenantID, 0, bloomMiss) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 1, bloomMissCalled) -} - -func TestJanitor(t *testing.T) { - tempDir, err := ioutil.TempDir("/tmp", "") - defer os.RemoveAll(tempDir) - assert.NoError(t, err, "unexpected error creating temp dir") - - // 1KB per file - missBytes := make([]byte, 1024) - missFunc := func(ctx context.Context, blockID uuid.UUID, tenantID string) ([]byte, error) { - return missBytes, nil - } - - cache, err := New(nil, &Config{ - Path: tempDir, - MaxDiskMBs: 30, - DiskPruneCount: 10, - DiskCleanRate: time.Hour, - }, nil) - assert.NoError(t, err) - - // test - for i := 0; i < 10; i++ { - blockID := uuid.New() - tenantID := testTenantID - - bytes, skippableErr, err := cache.(*reader).readOrCacheIndex(context.Background(), blockID, tenantID, missFunc) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - } - - // force prune. should do nothing b/c we don't have enough files - cleaned, err := clean(tempDir, 1, 10) - assert.NoError(t, err) - assert.False(t, cleaned) - - // now make enough files to prune - fi, err := ioutil.ReadDir(tempDir) - assert.Len(t, fi, 10) - assert.NoError(t, err) - - // create 1024 files - for i := 0; i < 1024; i++ { - blockID := uuid.New() - tenantID := testTenantID - - bytes, skippableErr, err := cache.(*reader).readOrCacheIndex(context.Background(), blockID, tenantID, missFunc) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - } - - fi, err = ioutil.ReadDir(tempDir) - assert.Len(t, fi, 1034) - assert.NoError(t, err) - - // force clean at 1MB and see only 1033 (b/c prunecount = 1) - cleaned, err = clean(tempDir, 1, 1) - assert.NoError(t, err) - assert.True(t, cleaned) - - fi, err = ioutil.ReadDir(tempDir) - assert.Len(t, fi, 1033) - assert.NoError(t, err) - - // force clean at 1MB and see only 1023 (b/c prunecount = 10) - cleaned, err = clean(tempDir, 1, 10) - assert.NoError(t, err) - assert.True(t, cleaned) - - fi, err = ioutil.ReadDir(tempDir) - assert.Len(t, fi, 1023) - assert.NoError(t, err) - - // force clean at 1MB and see only 1023 (b/c we're less than 1MB) - cleaned, err = clean(tempDir, 1, 10) - assert.NoError(t, err) - assert.False(t, cleaned) - - fi, err = ioutil.ReadDir(tempDir) - assert.Len(t, fi, 1023) - assert.NoError(t, err) -} - -/*func TestJanitorCleanupOrder(t *testing.T) { - tempDir, err := ioutil.TempDir("/tmp", "") - defer os.RemoveAll(tempDir) - assert.NoError(t, err, "unexpected error creating temp dir") - - // 1MB per file - missCalled := 0 - missBytes := make([]byte, 1024*1024) - missFunc := func(blockID uuid.UUID, tenantID string) ([]byte, error) { - missCalled++ - return missBytes, nil - } - - cache, err := New(nil, &Config{ - Path: tempDir, - MaxDiskMBs: 30, - DiskPruneCount: 10, - DiskCleanRate: time.Hour, - }, nil) - assert.NoError(t, err) - - // add 3 files - tenantID := testTenantID - firstBlockID, _ := uuid.FromBytes([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) - secondBlockID, _ := uuid.FromBytes([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}) - thirdBlockID, _ := uuid.FromBytes([]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}) - - bytes, skippableErr, err := cache.(*reader).readOrCacheKeyToDisk(firstBlockID, tenantID, "type", missFunc) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 1, missCalled) - - bytes, skippableErr, err = cache.(*reader).readOrCacheKeyToDisk(secondBlockID, tenantID, "type", missFunc) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 2, missCalled) - - time.Sleep(time.Second) - - bytes, skippableErr, err = cache.(*reader).readOrCacheKeyToDisk(thirdBlockID, tenantID, "type", missFunc) - assert.NoError(t, err) - assert.NoError(t, skippableErr) - assert.Equal(t, missBytes, bytes) - assert.Equal(t, 3, missCalled) - - fi, err := ioutil.ReadDir(tempDir) - assert.Len(t, fi, 3) - assert.NoError(t, err) - - var newestFi os.FileInfo - fi, err = ioutil.ReadDir(tempDir) - assert.NoError(t, err) - for _, info := range fi { - if newestFi == nil { - newestFi = info - } - - if info.Sys().(*syscall.Stat_t).Atim.Nano() > newestFi.Sys().(*syscall.Stat_t).Atim.Nano() { - newestFi = info - } - } - - // force prune. should prune 2 - cleaned, err := clean(tempDir, 1, 2) - assert.NoError(t, err) - assert.True(t, cleaned) - - fi, err = ioutil.ReadDir(tempDir) - assert.Len(t, fi, 1) - assert.NoError(t, err) - - // confirm the third block is still in cache as it was created last - assert.Equal(t, newestFi.Name(), fi[0].Name()) -}*/ diff --git a/tempodb/backend/diskcache/file_darwin.go b/tempodb/backend/diskcache/file_darwin.go deleted file mode 100644 index d8e9a88e185..00000000000 --- a/tempodb/backend/diskcache/file_darwin.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build darwin - -package diskcache - -import "syscall" - -func AtimeNano(s *syscall.Stat_t) int64 { - return s.Atimespec.Nano() -} diff --git a/tempodb/backend/diskcache/file_linux.go b/tempodb/backend/diskcache/file_linux.go deleted file mode 100644 index 9ccdc832dff..00000000000 --- a/tempodb/backend/diskcache/file_linux.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build linux - -package diskcache - -import "syscall" - -func AtimeNano(s *syscall.Stat_t) int64 { - return s.Atim.Nano() -} diff --git a/tempodb/config.go b/tempodb/config.go index 5151bbd3a32..a6cfb977d18 100644 --- a/tempodb/config.go +++ b/tempodb/config.go @@ -4,7 +4,6 @@ import ( "time" "github.com/grafana/tempo/tempodb/backend/azure" - "github.com/grafana/tempo/tempodb/backend/diskcache" "github.com/grafana/tempo/tempodb/backend/gcs" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/backend/memcached" @@ -23,7 +22,6 @@ type Config struct { Pool *pool.Config `yaml:"pool,omitempty"` WAL *wal.Config `yaml:"wal"` - Diskcache *diskcache.Config `yaml:"disk_cache"` Memcached *memcached.Config `yaml:"memcached"` Redis *redis.Config `yaml:"redis"` diff --git a/tempodb/tempodb.go b/tempodb/tempodb.go index e930e31a7c2..86c23388a02 100644 --- a/tempodb/tempodb.go +++ b/tempodb/tempodb.go @@ -23,7 +23,6 @@ import ( "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/azure" - "github.com/grafana/tempo/tempodb/backend/diskcache" "github.com/grafana/tempo/tempodb/backend/gcs" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/backend/memcached" @@ -154,14 +153,6 @@ func New(cfg *Config, logger log.Logger) (Reader, Writer, Compactor, error) { return nil, nil, nil, err } - if cfg.Diskcache != nil { - r, err = diskcache.New(r, cfg.Diskcache, logger) - - if err != nil { - return nil, nil, nil, err - } - } - if cfg.Memcached != nil { r, w, err = memcached.New(r, w, cfg.Memcached, logger) diff --git a/vendor/github.com/karrick/godirwalk/.gitignore b/vendor/github.com/karrick/godirwalk/.gitignore deleted file mode 100644 index ea2d9ece601..00000000000 --- a/vendor/github.com/karrick/godirwalk/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.dll -*.so -*.dylib - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 -.glide/ - -examples/remove-empty-directories/remove-empty-directories -examples/sizes/sizes -examples/walk-fast/walk-fast -examples/walk-stdlib/walk-stdlib diff --git a/vendor/github.com/karrick/godirwalk/LICENSE b/vendor/github.com/karrick/godirwalk/LICENSE deleted file mode 100644 index 01ce194c80d..00000000000 --- a/vendor/github.com/karrick/godirwalk/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -BSD 2-Clause License - -Copyright (c) 2017, Karrick McDermott -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/karrick/godirwalk/README.md b/vendor/github.com/karrick/godirwalk/README.md deleted file mode 100644 index 0902cce5c9e..00000000000 --- a/vendor/github.com/karrick/godirwalk/README.md +++ /dev/null @@ -1,320 +0,0 @@ -# godirwalk - -`godirwalk` is a library for traversing a directory tree on a file -system. - -[![GoDoc](https://godoc.org/github.com/karrick/godirwalk?status.svg)](https://godoc.org/github.com/karrick/godirwalk) [![Build Status](https://dev.azure.com/microsoft0235/microsoft/_apis/build/status/karrick.godirwalk?branchName=master)](https://dev.azure.com/microsoft0235/microsoft/_build/latest?definitionId=1&branchName=master) - -In short, why do I use this library? - -1. It's faster than `filepath.Walk`. -1. It's more correct on Windows than `filepath.Walk`. -1. It's more easy to use than `filepath.Walk`. -1. It's more flexible than `filepath.Walk`. - -## Usage Example - -Additional examples are provided in the `examples/` subdirectory. - -This library will normalize the provided top level directory name -based on the os-specific path separator by calling `filepath.Clean` on -its first argument. However it always provides the pathname created by -using the correct os-specific path separator when invoking the -provided callback function. - -```Go - dirname := "some/directory/root" - err := godirwalk.Walk(dirname, &godirwalk.Options{ - Callback: func(osPathname string, de *godirwalk.Dirent) error { - // Following string operation is not most performant way - // of doing this, but common enough to warrant a simple - // example here: - if strings.Contains(osPathname, ".git") { - return godirwalk.SkipThis - } - fmt.Printf("%s %s\n", de.ModeType(), osPathname) - return nil - }, - Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc) - }) -``` - -This library not only provides functions for traversing a file system -directory tree, but also for obtaining a list of immediate descendants -of a particular directory, typically much more quickly than using -`os.ReadDir` or `os.ReadDirnames`. - -## Description - -Here's why I use `godirwalk` in preference to `filepath.Walk`, -`os.ReadDir`, and `os.ReadDirnames`. - -### It's faster than `filepath.Walk` - -When compared against `filepath.Walk` in benchmarks, it has been -observed to run between five and ten times the speed on darwin, at -speeds comparable to the that of the unix `find` utility; and about -twice the speed on linux; and about four times the speed on Windows. - -How does it obtain this performance boost? It does less work to give -you nearly the same output. This library calls the same `syscall` -functions to do the work, but it makes fewer calls, does not throw -away information that it might need, and creates less memory churn -along the way by reusing the same scratch buffer for reading from a -directory rather than reallocating a new buffer every time it reads -file system entry data from the operating system. - -While traversing a file system directory tree, `filepath.Walk` obtains -the list of immediate descendants of a directory, and throws away the -node type information for the file system entry that is provided by -the operating system that comes with the node's name. Then, -immediately prior to invoking the callback function, `filepath.Walk` -invokes `os.Stat` for each node, and passes the returned `os.FileInfo` -information to the callback. - -While the `os.FileInfo` information provided by `os.Stat` is extremely -helpful--and even includes the `os.FileMode` data--providing it -requires an additional system call for each node. - -Because most callbacks only care about what the node type is, this -library does not throw the type information away, but rather provides -that information to the callback function in the form of a -`os.FileMode` value. Note that the provided `os.FileMode` value that -this library provides only has the node type information, and does not -have the permission bits, sticky bits, or other information from the -file's mode. If the callback does care about a particular node's -entire `os.FileInfo` data structure, the callback can easiy invoke -`os.Stat` when needed, and only when needed. - -#### Benchmarks - -##### macOS - -```Bash -$ go test -bench=. -benchmem -goos: darwin -goarch: amd64 -pkg: github.com/karrick/godirwalk -BenchmarkReadDirnamesStandardLibrary-12 50000 26250 ns/op 10360 B/op 16 allocs/op -BenchmarkReadDirnamesThisLibrary-12 50000 24372 ns/op 5064 B/op 20 allocs/op -BenchmarkFilepathWalk-12 1 1099524875 ns/op 228415912 B/op 416952 allocs/op -BenchmarkGodirwalk-12 2 526754589 ns/op 103110464 B/op 451442 allocs/op -BenchmarkGodirwalkUnsorted-12 3 509219296 ns/op 100751400 B/op 378800 allocs/op -BenchmarkFlameGraphFilepathWalk-12 1 7478618820 ns/op 2284138176 B/op 4169453 allocs/op -BenchmarkFlameGraphGodirwalk-12 1 4977264058 ns/op 1031105328 B/op 4514423 allocs/op -PASS -ok github.com/karrick/godirwalk 21.219s -``` - -##### Linux - -```Bash -$ go test -bench=. -benchmem -goos: linux -goarch: amd64 -pkg: github.com/karrick/godirwalk -BenchmarkReadDirnamesStandardLibrary-12 100000 15458 ns/op 10360 B/op 16 allocs/op -BenchmarkReadDirnamesThisLibrary-12 100000 14646 ns/op 5064 B/op 20 allocs/op -BenchmarkFilepathWalk-12 2 631034745 ns/op 228210216 B/op 416939 allocs/op -BenchmarkGodirwalk-12 3 358714883 ns/op 102988664 B/op 451437 allocs/op -BenchmarkGodirwalkUnsorted-12 3 355363915 ns/op 100629234 B/op 378796 allocs/op -BenchmarkFlameGraphFilepathWalk-12 1 6086913991 ns/op 2282104720 B/op 4169417 allocs/op -BenchmarkFlameGraphGodirwalk-12 1 3456398824 ns/op 1029886400 B/op 4514373 allocs/op -PASS -ok github.com/karrick/godirwalk 19.179s -``` - -### It's more correct on Windows than `filepath.Walk` - -I did not previously care about this either, but humor me. We all love -how we can write once and run everywhere. It is essential for the -language's adoption, growth, and success, that the software we create -can run unmodified on all architectures and operating systems -supported by Go. - -When the traversed file system has a logical loop caused by symbolic -links to directories, on unix `filepath.Walk` ignores symbolic links -and traverses the entire directory tree without error. On Windows -however, `filepath.Walk` will continue following directory symbolic -links, even though it is not supposed to, eventually causing -`filepath.Walk` to terminate early and return an error when the -pathname gets too long from concatenating endless loops of symbolic -links onto the pathname. This error comes from Windows, passes through -`filepath.Walk`, and to the upstream client running `filepath.Walk`. - -The takeaway is that behavior is different based on which platform -`filepath.Walk` is running. While this is clearly not intentional, -until it is fixed in the standard library, it presents a compatibility -problem. - -This library fixes the above problem such that it will never follow -logical file sytem loops on either unix or Windows. Furthermore, it -will only follow symbolic links when `FollowSymbolicLinks` is set to -true. Behavior on Windows and other operating systems is identical. - -### It's more easy to use than `filepath.Walk` - -While this library strives to mimic the behavior of the incredibly -well-written `filepath.Walk` standard library, there are places where -it deviates a bit in order to provide a more easy or intuitive caller -interface. - -#### Callback interface does not send you an error to check - -Since this library does not invoke `os.Stat` on every file system node -it encounters, there is no possible error event for the callback -function to filter on. The third argument in the `filepath.WalkFunc` -function signature to pass the error from `os.Stat` to the callback -function is no longer necessary, and thus eliminated from signature of -the callback function from this library. - -Furthermore, this slight interface difference between -`filepath.WalkFunc` and this library's `WalkFunc` eliminates the -boilerplate code that callback handlers must write when they use -`filepath.Walk`. Rather than every callback function needing to check -the error value passed into it and branch accordingly, users of this -library do not even have an error value to check immediately upon -entry into the callback function. This is an improvement both in -runtime performance and code clarity. - -#### Callback function is invoked with OS specific file system path separator - -On every OS platform `filepath.Walk` invokes the callback function -with a solidus (`/`) delimited pathname. By contrast this library -invokes the callback with the os-specific pathname separator, -obviating a call to `filepath.Clean` in the callback function for each -node prior to actually using the provided pathname. - -In other words, even on Windows, `filepath.Walk` will invoke the -callback with `some/path/to/foo.txt`, requiring well written clients -to perform pathname normalization for every file prior to working with -the specified file. This is a hidden boilerplate requirement to create -truly os agnostic callback functions. In truth, many clients developed -on unix and not tested on Windows neglect this subtlety, and will -result in software bugs when someone tries to run that software on -Windows. - -This library invokes the callback function with `some\path\to\foo.txt` -for the same file when running on Windows, eliminating the need to -normalize the pathname by the client, and lessen the likelyhood that a -client will work on unix but not on Windows. - -This enhancement eliminates necessity for some more boilerplate code -in callback functions while improving the runtime performance of this -library. - -#### `godirwalk.SkipThis` is more intuitive to use than `filepath.SkipDir` - -One arguably confusing aspect of the `filepath.WalkFunc` interface -that this library must emulate is how a caller tells the `Walk` -function to skip file system entries. With both `filepath.Walk` and -this library's `Walk`, when a callback function wants to skip a -directory and not descend into its children, it returns -`filepath.SkipDir`. If the callback function returns -`filepath.SkipDir` for a non-directory, `filepath.Walk` and this -library will stop processing any more entries in the current -directory. This is not necessarily what most developers want or -expect. If you want to simply skip a particular non-directory entry -but continue processing entries in the directory, the callback -function must return nil. - -The implications of this interface design is when you want to walk a -file system hierarchy and skip an entry, you have to return a -different value based on what type of file system entry that node -is. To skip an entry, if the entry is a directory, you must return -`filepath.SkipDir`, and if entry is not a directory, you must return -`nil`. This is an unfortunate hurdle I have observed many developers -struggling with, simply because it is not an intuitive interface. - -Here is an example callback function that adheres to -`filepath.WalkFunc` interface to have it skip any file system entry -whose full pathname includes a particular substring, `optSkip`. Note -that this library still supports identical behavior of `filepath.Walk` -when the callback function returns `filepath.SkipDir`. - -```Go - func callback1(osPathname string, de *godirwalk.Dirent) error { - if optSkip != "" && strings.Contains(osPathname, optSkip) { - if b, err := de.IsDirOrSymlinkToDir(); b == true && err == nil { - return filepath.SkipDir - } - return nil - } - // Process file like normal... - return nil - } -``` - -This library attempts to eliminate some of that logic boilerplate -required in callback functions by providing a new token error value, -`SkipThis`, which a callback function may return to skip the current -file system entry regardless of what type of entry it is. If the -current entry is a directory, its children will not be enumerated, -exactly as if the callback had returned `filepath.SkipDir`. If the -current entry is a non-directory, the next file system entry in the -current directory will be enumerated, exactly as if the callback -returned `nil`. The following example callback function has identical -behavior as the previous, but has less boilerplate, and admittedly -logic that I find more simple to follow. - -```Go - func callback2(osPathname string, de *godirwalk.Dirent) error { - if optSkip != "" && strings.Contains(osPathname, optSkip) { - return godirwalk.SkipThis - } - // Process file like normal... - return nil - } -``` - -### It's more flexible than `filepath.Walk` - -#### Configurable Handling of Symbolic Links - -The default behavior of this library is to ignore symbolic links to -directories when walking a directory tree, just like `filepath.Walk` -does. However, it does invoke the callback function with each node it -finds, including symbolic links. If a particular use case exists to -follow symbolic links when traversing a directory tree, this library -can be invoked in manner to do so, by setting the -`FollowSymbolicLinks` config parameter to `true`. - -#### Configurable Sorting of Directory Children - -The default behavior of this library is to always sort the immediate -descendants of a directory prior to visiting each node, just like -`filepath.Walk` does. This is usually the desired behavior. However, -this does come at slight performance and memory penalties required to -sort the names when a directory node has many entries. Additionally if -caller specifies `Unsorted` enumeration in the configuration -parameter, reading directories is lazily performed as the caller -consumes entries. If a particular use case exists that does not -require sorting the directory's immediate descendants prior to -visiting its nodes, this library will skip the sorting step when the -`Unsorted` parameter is set to `true`. - -Here's an interesting read of the potential hazzards of traversing a -file system hierarchy in a non-deterministic order. If you know the -problem you are solving is not affected by the order files are -visited, then I encourage you to use `Unsorted`. Otherwise skip -setting this option. - -[Researchers find bug in Python script may have affected hundreds of studies](https://arstechnica.com/information-technology/2019/10/chemists-discover-cross-platform-python-scripts-not-so-cross-platform/) - -#### Configurable Post Children Callback - -This library provides upstream code with the ability to specify a -callback function to be invoked for each directory after its children -are processed. This has been used to recursively delete empty -directories after traversing the file system in a more efficient -manner. See the `examples/clean-empties` directory for an example of -this usage. - -#### Configurable Error Callback - -This library provides upstream code with the ability to specify a -callback to be invoked for errors that the operating system returns, -allowing the upstream code to determine the next course of action to -take, whether to halt walking the hierarchy, as it would do were no -error callback provided, or skip the node that caused the error. See -the `examples/walk-fast` directory for an example of this usage. diff --git a/vendor/github.com/karrick/godirwalk/azure-pipelines.yml b/vendor/github.com/karrick/godirwalk/azure-pipelines.yml deleted file mode 100644 index d56f2484d63..00000000000 --- a/vendor/github.com/karrick/godirwalk/azure-pipelines.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Go -# Build your Go project. -# Add steps that test, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/go - -trigger: -- master - -variables: - GOVERSION: 1.13 - -jobs: - - job: Linux - pool: - vmImage: 'ubuntu-latest' - steps: - - task: GoTool@0 - displayName: 'Use Go $(GOVERSION)' - inputs: - version: $(GOVERSION) - - task: Go@0 - inputs: - command: test - arguments: -race -v ./... - displayName: 'Execute Tests' - - - job: Mac - pool: - vmImage: 'macos-latest' - steps: - - task: GoTool@0 - displayName: 'Use Go $(GOVERSION)' - inputs: - version: $(GOVERSION) - - task: Go@0 - inputs: - command: test - arguments: -race -v ./... - displayName: 'Execute Tests' - - - job: Windows - pool: - vmImage: 'windows-latest' - steps: - - task: GoTool@0 - displayName: 'Use Go $(GOVERSION)' - inputs: - version: $(GOVERSION) - - task: Go@0 - inputs: - command: test - arguments: -race -v ./... - displayName: 'Execute Tests' diff --git a/vendor/github.com/karrick/godirwalk/bench.sh b/vendor/github.com/karrick/godirwalk/bench.sh deleted file mode 100644 index b2ba374cf28..00000000000 --- a/vendor/github.com/karrick/godirwalk/bench.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# for version in v1.9.1 v1.10.0 v1.10.3 v1.10.12 v1.11.2 v1.11.3 v1.12.0 v1.13.1 v1.14.0 v1.14.1 ; do -for version in v1.10.12 v1.14.1 v1.15.2 ; do - echo "### $version" > $version.txt - git checkout -- go.mod && git checkout $version && go test -run=NONE -bench=Benchmark2 >> $version.txt || exit 1 -done diff --git a/vendor/github.com/karrick/godirwalk/debug_development.go b/vendor/github.com/karrick/godirwalk/debug_development.go deleted file mode 100644 index 6e1cb0bf7ed..00000000000 --- a/vendor/github.com/karrick/godirwalk/debug_development.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build godirwalk_debug - -package godirwalk - -import ( - "fmt" - "os" -) - -// debug formats and prints arguments to stderr for development builds -func debug(f string, a ...interface{}) { - // fmt.Fprintf(os.Stderr, f, a...) - os.Stderr.Write([]byte("godirwalk: " + fmt.Sprintf(f, a...))) -} diff --git a/vendor/github.com/karrick/godirwalk/debug_release.go b/vendor/github.com/karrick/godirwalk/debug_release.go deleted file mode 100644 index 986178731fe..00000000000 --- a/vendor/github.com/karrick/godirwalk/debug_release.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build !godirwalk_debug - -package godirwalk - -// debug is a no-op for release builds -func debug(_ string, _ ...interface{}) {} diff --git a/vendor/github.com/karrick/godirwalk/dirent.go b/vendor/github.com/karrick/godirwalk/dirent.go deleted file mode 100644 index 38e141ba6b1..00000000000 --- a/vendor/github.com/karrick/godirwalk/dirent.go +++ /dev/null @@ -1,104 +0,0 @@ -package godirwalk - -import ( - "os" - "path/filepath" -) - -// Dirent stores the name and file system mode type of discovered file system -// entries. -type Dirent struct { - name string // base name of the file system entry. - path string // path name of the file system entry. - modeType os.FileMode // modeType is the type of file system entry. -} - -// NewDirent returns a newly initialized Dirent structure, or an error. This -// function does not follow symbolic links. -// -// This function is rarely used, as Dirent structures are provided by other -// functions in this library that read and walk directories, but is provided, -// however, for the occasion when a program needs to create a Dirent. -func NewDirent(osPathname string) (*Dirent, error) { - modeType, err := modeType(osPathname) - if err != nil { - return nil, err - } - return &Dirent{ - name: filepath.Base(osPathname), - path: filepath.Dir(osPathname), - modeType: modeType, - }, nil -} - -// IsDir returns true if and only if the Dirent represents a file system -// directory. Note that on some operating systems, more than one file mode bit -// may be set for a node. For instance, on Windows, a symbolic link that points -// to a directory will have both the directory and the symbolic link bits set. -func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } - -// IsDirOrSymlinkToDir returns true if and only if the Dirent represents a file -// system directory, or a symbolic link to a directory. Note that if the Dirent -// is not a directory but is a symbolic link, this method will resolve by -// sending a request to the operating system to follow the symbolic link. -func (de Dirent) IsDirOrSymlinkToDir() (bool, error) { - if de.IsDir() { - return true, nil - } - if !de.IsSymlink() { - return false, nil - } - // Does this symlink point to a directory? - info, err := os.Stat(filepath.Join(de.path, de.name)) - if err != nil { - return false, err - } - return info.IsDir(), nil -} - -// IsRegular returns true if and only if the Dirent represents a regular file. -// That is, it ensures that no mode type bits are set. -func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } - -// IsSymlink returns true if and only if the Dirent represents a file system -// symbolic link. Note that on some operating systems, more than one file mode -// bit may be set for a node. For instance, on Windows, a symbolic link that -// points to a directory will have both the directory and the symbolic link bits -// set. -func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } - -// IsDevice returns true if and only if the Dirent represents a device file. -func (de Dirent) IsDevice() bool { return de.modeType&os.ModeDevice != 0 } - -// ModeType returns the mode bits that specify the file system node type. We -// could make our own enum-like data type for encoding the file type, but Go's -// runtime already gives us architecture independent file modes, as discussed in -// `os/types.go`: -// -// Go's runtime FileMode type has same definition on all systems, so that -// information about files can be moved from one system to another portably. -func (de Dirent) ModeType() os.FileMode { return de.modeType } - -// Name returns the base name of the file system entry. -func (de Dirent) Name() string { return de.name } - -// reset releases memory held by entry err and name, and resets mode type to 0. -func (de *Dirent) reset() { - de.name = "" - de.path = "" - de.modeType = 0 -} - -// Dirents represents a slice of Dirent pointers, which are sortable by base -// name. This type satisfies the `sort.Interface` interface. -type Dirents []*Dirent - -// Len returns the count of Dirent structures in the slice. -func (l Dirents) Len() int { return len(l) } - -// Less returns true if and only if the base name of the element specified by -// the first index is lexicographically less than that of the second index. -func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } - -// Swap exchanges the two Dirent entries specified by the two provided indexes. -func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } diff --git a/vendor/github.com/karrick/godirwalk/doc.go b/vendor/github.com/karrick/godirwalk/doc.go deleted file mode 100644 index 440aa82a3a8..00000000000 --- a/vendor/github.com/karrick/godirwalk/doc.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Package godirwalk provides functions to read and traverse directory trees. - -In short, why do I use this library? - -* It's faster than `filepath.Walk`. - -* It's more correct on Windows than `filepath.Walk`. - -* It's more easy to use than `filepath.Walk`. - -* It's more flexible than `filepath.Walk`. - -USAGE - -This library will normalize the provided top level directory name based on the -os-specific path separator by calling `filepath.Clean` on its first -argument. However it always provides the pathname created by using the correct -os-specific path separator when invoking the provided callback function. - - dirname := "some/directory/root" - err := godirwalk.Walk(dirname, &godirwalk.Options{ - Callback: func(osPathname string, de *godirwalk.Dirent) error { - fmt.Printf("%s %s\n", de.ModeType(), osPathname) - return nil - }, - }) - -This library not only provides functions for traversing a file system directory -tree, but also for obtaining a list of immediate descendants of a particular -directory, typically much more quickly than using `os.ReadDir` or -`os.ReadDirnames`. - - scratchBuffer := make([]byte, godirwalk.MinimumScratchBufferSize) - - names, err := godirwalk.ReadDirnames("some/directory", scratchBuffer) - // ... - - entries, err := godirwalk.ReadDirents("another/directory", scratchBuffer) - // ... -*/ -package godirwalk diff --git a/vendor/github.com/karrick/godirwalk/go.mod b/vendor/github.com/karrick/godirwalk/go.mod deleted file mode 100644 index faf23fd98a4..00000000000 --- a/vendor/github.com/karrick/godirwalk/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/karrick/godirwalk - -go 1.13 diff --git a/vendor/github.com/karrick/godirwalk/go.sum b/vendor/github.com/karrick/godirwalk/go.sum deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/vendor/github.com/karrick/godirwalk/inoWithFileno.go b/vendor/github.com/karrick/godirwalk/inoWithFileno.go deleted file mode 100644 index 1dc04a717a7..00000000000 --- a/vendor/github.com/karrick/godirwalk/inoWithFileno.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build dragonfly freebsd openbsd netbsd - -package godirwalk - -import "syscall" - -func inoFromDirent(de *syscall.Dirent) uint64 { - return uint64(de.Fileno) -} diff --git a/vendor/github.com/karrick/godirwalk/inoWithIno.go b/vendor/github.com/karrick/godirwalk/inoWithIno.go deleted file mode 100644 index 842a6662d8a..00000000000 --- a/vendor/github.com/karrick/godirwalk/inoWithIno.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build aix darwin linux nacl solaris - -package godirwalk - -import "syscall" - -func inoFromDirent(de *syscall.Dirent) uint64 { - return uint64(de.Ino) -} diff --git a/vendor/github.com/karrick/godirwalk/modeType.go b/vendor/github.com/karrick/godirwalk/modeType.go deleted file mode 100644 index 6427a6852cd..00000000000 --- a/vendor/github.com/karrick/godirwalk/modeType.go +++ /dev/null @@ -1,22 +0,0 @@ -package godirwalk - -import ( - "os" -) - -// modeType returns the mode type of the file system entry identified by -// osPathname by calling os.LStat function, to intentionally not follow symbolic -// links. -// -// Even though os.LStat provides all file mode bits, we want to ensure same -// values returned to caller regardless of whether we obtained file mode bits -// from syscall or stat call. Therefore mask out the additional file mode bits -// that are provided by stat but not by the syscall, so users can rely on their -// values. -func modeType(osPathname string) (os.FileMode, error) { - fi, err := os.Lstat(osPathname) - if err == nil { - return fi.Mode() & os.ModeType, nil - } - return 0, err -} diff --git a/vendor/github.com/karrick/godirwalk/modeTypeWithType.go b/vendor/github.com/karrick/godirwalk/modeTypeWithType.go deleted file mode 100644 index 7890e7761b6..00000000000 --- a/vendor/github.com/karrick/godirwalk/modeTypeWithType.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build darwin dragonfly freebsd linux netbsd openbsd - -package godirwalk - -import ( - "os" - "path/filepath" - "syscall" -) - -// modeTypeFromDirent converts a syscall defined constant, which is in purview -// of OS, to a constant defined by Go, assumed by this project to be stable. -// -// When the syscall constant is not recognized, this function falls back to a -// Stat on the file system. -func modeTypeFromDirent(de *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { - switch de.Type { - case syscall.DT_REG: - return 0, nil - case syscall.DT_DIR: - return os.ModeDir, nil - case syscall.DT_LNK: - return os.ModeSymlink, nil - case syscall.DT_CHR: - return os.ModeDevice | os.ModeCharDevice, nil - case syscall.DT_BLK: - return os.ModeDevice, nil - case syscall.DT_FIFO: - return os.ModeNamedPipe, nil - case syscall.DT_SOCK: - return os.ModeSocket, nil - default: - // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), then - // resolve actual mode by reading file information. - return modeType(filepath.Join(osDirname, osBasename)) - } -} diff --git a/vendor/github.com/karrick/godirwalk/modeTypeWithoutType.go b/vendor/github.com/karrick/godirwalk/modeTypeWithoutType.go deleted file mode 100644 index 5299392ee90..00000000000 --- a/vendor/github.com/karrick/godirwalk/modeTypeWithoutType.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build aix js nacl solaris - -package godirwalk - -import ( - "os" - "path/filepath" - "syscall" -) - -// modeTypeFromDirent converts a syscall defined constant, which is in purview -// of OS, to a constant defined by Go, assumed by this project to be stable. -// -// Because some operating system syscall.Dirent structures do not include a Type -// field, fall back on Stat of the file system. -func modeTypeFromDirent(_ *syscall.Dirent, osDirname, osBasename string) (os.FileMode, error) { - return modeType(filepath.Join(osDirname, osBasename)) -} diff --git a/vendor/github.com/karrick/godirwalk/nameWithNamlen.go b/vendor/github.com/karrick/godirwalk/nameWithNamlen.go deleted file mode 100644 index 2c0231eea73..00000000000 --- a/vendor/github.com/karrick/godirwalk/nameWithNamlen.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build aix darwin dragonfly freebsd netbsd openbsd - -package godirwalk - -import ( - "reflect" - "syscall" - "unsafe" -) - -func nameFromDirent(de *syscall.Dirent) []byte { - // Because this GOOS' syscall.Dirent provides a Namlen field that says how - // long the name is, this function does not need to search for the NULL - // byte. - ml := int(de.Namlen) - - // Convert syscall.Dirent.Name, which is array of int8, to []byte, by - // overwriting Cap, Len, and Data slice header fields to values from - // syscall.Dirent fields. Setting the Cap, Len, and Data field values for - // the slice header modifies what the slice header points to, and in this - // case, the name buffer. - var name []byte - sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) - sh.Cap = ml - sh.Len = ml - sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) - - return name -} diff --git a/vendor/github.com/karrick/godirwalk/nameWithoutNamlen.go b/vendor/github.com/karrick/godirwalk/nameWithoutNamlen.go deleted file mode 100644 index f776fbc760e..00000000000 --- a/vendor/github.com/karrick/godirwalk/nameWithoutNamlen.go +++ /dev/null @@ -1,42 +0,0 @@ -// +build nacl linux js solaris - -package godirwalk - -import ( - "bytes" - "reflect" - "syscall" - "unsafe" -) - -// nameOffset is a compile time constant -const nameOffset = int(unsafe.Offsetof(syscall.Dirent{}.Name)) - -func nameFromDirent(de *syscall.Dirent) (name []byte) { - // Because this GOOS' syscall.Dirent does not provide a field that specifies - // the name length, this function must first calculate the max possible name - // length, and then search for the NULL byte. - ml := int(de.Reclen) - nameOffset - - // Convert syscall.Dirent.Name, which is array of int8, to []byte, by - // overwriting Cap, Len, and Data slice header fields to the max possible - // name length computed above, and finding the terminating NULL byte. - sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) - sh.Cap = ml - sh.Len = ml - sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) - - if index := bytes.IndexByte(name, 0); index >= 0 { - // Found NULL byte; set slice's cap and len accordingly. - sh.Cap = index - sh.Len = index - return - } - - // NOTE: This branch is not expected, but included for defensive - // programming, and provides a hard stop on the name based on the structure - // field array size. - sh.Cap = len(de.Name) - sh.Len = sh.Cap - return -} diff --git a/vendor/github.com/karrick/godirwalk/readdir.go b/vendor/github.com/karrick/godirwalk/readdir.go deleted file mode 100644 index c8fa8155e35..00000000000 --- a/vendor/github.com/karrick/godirwalk/readdir.go +++ /dev/null @@ -1,53 +0,0 @@ -package godirwalk - -// ReadDirents returns a sortable slice of pointers to Dirent structures, each -// representing the file system name and mode type for one of the immediate -// descendant of the specified directory. If the specified directory is a -// symbolic link, it will be resolved. -// -// If an optional scratch buffer is provided that is at least one page of -// memory, it will be used when reading directory entries from the file -// system. If you plan on calling this function in a loop, you will have -// significantly better performance if you allocate a scratch buffer and use it -// each time you call this function. -// -// children, err := godirwalk.ReadDirents(osDirname, nil) -// if err != nil { -// return nil, errors.Wrap(err, "cannot get list of directory children") -// } -// sort.Sort(children) -// for _, child := range children { -// fmt.Printf("%s %s\n", child.ModeType, child.Name) -// } -func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { - return readDirents(osDirname, scratchBuffer) -} - -// ReadDirnames returns a slice of strings, representing the immediate -// descendants of the specified directory. If the specified directory is a -// symbolic link, it will be resolved. -// -// If an optional scratch buffer is provided that is at least one page of -// memory, it will be used when reading directory entries from the file -// system. If you plan on calling this function in a loop, you will have -// significantly better performance if you allocate a scratch buffer and use it -// each time you call this function. -// -// Note that this function, depending on operating system, may or may not invoke -// the ReadDirents function, in order to prepare the list of immediate -// descendants. Therefore, if your program needs both the names and the file -// system mode types of descendants, it will always be faster to invoke -// ReadDirents directly, rather than calling this function, then looping over -// the results and calling os.Stat or os.LStat for each entry. -// -// children, err := godirwalk.ReadDirnames(osDirname, nil) -// if err != nil { -// return nil, errors.Wrap(err, "cannot get list of directory children") -// } -// sort.Strings(children) -// for _, child := range children { -// fmt.Printf("%s\n", child) -// } -func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { - return readDirnames(osDirname, scratchBuffer) -} diff --git a/vendor/github.com/karrick/godirwalk/readdir_unix.go b/vendor/github.com/karrick/godirwalk/readdir_unix.go deleted file mode 100644 index a993038ba23..00000000000 --- a/vendor/github.com/karrick/godirwalk/readdir_unix.go +++ /dev/null @@ -1,131 +0,0 @@ -// +build !windows - -package godirwalk - -import ( - "os" - "syscall" - "unsafe" -) - -// MinimumScratchBufferSize specifies the minimum size of the scratch buffer -// that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file -// entries from the operating system. During program startup it is initialized -// to the result from calling `os.Getpagesize()` for non Windows environments, -// and 0 for Windows. -var MinimumScratchBufferSize = os.Getpagesize() - -func newScratchBuffer() []byte { return make([]byte, MinimumScratchBufferSize) } - -func readDirents(osDirname string, scratchBuffer []byte) ([]*Dirent, error) { - var entries []*Dirent - var workBuffer []byte - - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - fd := int(dh.Fd()) - - if len(scratchBuffer) < MinimumScratchBufferSize { - scratchBuffer = newScratchBuffer() - } - - var sde syscall.Dirent - for { - if len(workBuffer) == 0 { - n, err := syscall.ReadDirent(fd, scratchBuffer) - // n, err := unix.ReadDirent(fd, scratchBuffer) - if err != nil { - if err == syscall.EINTR /* || err == unix.EINTR */ { - continue - } - _ = dh.Close() - return nil, err - } - if n <= 0 { // end of directory: normal exit - if err = dh.Close(); err != nil { - return nil, err - } - return entries, nil - } - workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read - } - - copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(&sde))[:], workBuffer) - workBuffer = workBuffer[reclen(&sde):] // advance buffer for next iteration through loop - - if inoFromDirent(&sde) == 0 { - continue // inode set to 0 indicates an entry that was marked as deleted - } - - nameSlice := nameFromDirent(&sde) - nameLength := len(nameSlice) - - if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { - continue - } - - childName := string(nameSlice) - mt, err := modeTypeFromDirent(&sde, osDirname, childName) - if err != nil { - _ = dh.Close() - return nil, err - } - entries = append(entries, &Dirent{name: childName, path: osDirname, modeType: mt}) - } -} - -func readDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { - var entries []string - var workBuffer []byte - var sde *syscall.Dirent - - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - fd := int(dh.Fd()) - - if len(scratchBuffer) < MinimumScratchBufferSize { - scratchBuffer = newScratchBuffer() - } - - for { - if len(workBuffer) == 0 { - n, err := syscall.ReadDirent(fd, scratchBuffer) - // n, err := unix.ReadDirent(fd, scratchBuffer) - if err != nil { - if err == syscall.EINTR /* || err == unix.EINTR */ { - continue - } - _ = dh.Close() - return nil, err - } - if n <= 0 { // end of directory: normal exit - if err = dh.Close(); err != nil { - return nil, err - } - return entries, nil - } - workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read - } - - sde = (*syscall.Dirent)(unsafe.Pointer(&workBuffer[0])) // point entry to first syscall.Dirent in buffer - // Handle first entry in the work buffer. - workBuffer = workBuffer[reclen(sde):] // advance buffer for next iteration through loop - - if inoFromDirent(sde) == 0 { - continue // inode set to 0 indicates an entry that was marked as deleted - } - - nameSlice := nameFromDirent(sde) - nameLength := len(nameSlice) - - if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { - continue - } - - entries = append(entries, string(nameSlice)) - } -} diff --git a/vendor/github.com/karrick/godirwalk/readdir_windows.go b/vendor/github.com/karrick/godirwalk/readdir_windows.go deleted file mode 100644 index 7dd76cbc350..00000000000 --- a/vendor/github.com/karrick/godirwalk/readdir_windows.go +++ /dev/null @@ -1,66 +0,0 @@ -// +build windows - -package godirwalk - -import "os" - -// MinimumScratchBufferSize specifies the minimum size of the scratch buffer -// that ReadDirents, ReadDirnames, Scanner, and Walk will use when reading file -// entries from the operating system. During program startup it is initialized -// to the result from calling `os.Getpagesize()` for non Windows environments, -// and 0 for Windows. -var MinimumScratchBufferSize = 0 - -func newScratchBuffer() []byte { return nil } - -func readDirents(osDirname string, _ []byte) ([]*Dirent, error) { - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - - fileinfos, err := dh.Readdir(-1) - if err != nil { - _ = dh.Close() - return nil, err - } - - entries := make([]*Dirent, len(fileinfos)) - - for i, fi := range fileinfos { - entries[i] = &Dirent{ - name: fi.Name(), - path: osDirname, - modeType: fi.Mode() & os.ModeType, - } - } - - if err = dh.Close(); err != nil { - return nil, err - } - return entries, nil -} - -func readDirnames(osDirname string, _ []byte) ([]string, error) { - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - - fileinfos, err := dh.Readdir(-1) - if err != nil { - _ = dh.Close() - return nil, err - } - - entries := make([]string, len(fileinfos)) - - for i, fi := range fileinfos { - entries[i] = fi.Name() - } - - if err = dh.Close(); err != nil { - return nil, err - } - return entries, nil -} diff --git a/vendor/github.com/karrick/godirwalk/reclenFromNamlen.go b/vendor/github.com/karrick/godirwalk/reclenFromNamlen.go deleted file mode 100644 index baeefcee3a5..00000000000 --- a/vendor/github.com/karrick/godirwalk/reclenFromNamlen.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build dragonfly - -package godirwalk - -import "syscall" - -func reclen(de *syscall.Dirent) uint64 { - return (16 + uint64(de.Namlen) + 1 + 7) &^ 7 -} diff --git a/vendor/github.com/karrick/godirwalk/reclenFromReclen.go b/vendor/github.com/karrick/godirwalk/reclenFromReclen.go deleted file mode 100644 index 99be34d8715..00000000000 --- a/vendor/github.com/karrick/godirwalk/reclenFromReclen.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build nacl linux js solaris aix darwin freebsd netbsd openbsd - -package godirwalk - -import "syscall" - -func reclen(de *syscall.Dirent) uint64 { - return uint64(de.Reclen) -} diff --git a/vendor/github.com/karrick/godirwalk/scandir_unix.go b/vendor/github.com/karrick/godirwalk/scandir_unix.go deleted file mode 100644 index 33250b61d8f..00000000000 --- a/vendor/github.com/karrick/godirwalk/scandir_unix.go +++ /dev/null @@ -1,166 +0,0 @@ -// +build !windows - -package godirwalk - -import ( - "os" - "syscall" - "unsafe" -) - -// Scanner is an iterator to enumerate the contents of a directory. -type Scanner struct { - scratchBuffer []byte // read directory bytes from file system into this buffer - workBuffer []byte // points into scratchBuffer, from which we chunk out directory entries - osDirname string - childName string - err error // err is the error associated with scanning directory - statErr error // statErr is any error return while attempting to stat an entry - dh *os.File // used to close directory after done reading - de *Dirent // most recently decoded directory entry - sde syscall.Dirent - fd int // file descriptor used to read entries from directory -} - -// NewScanner returns a new directory Scanner that lazily enumerates the -// contents of a single directory. -// -// scanner, err := godirwalk.NewScanner(dirname) -// if err != nil { -// fatal("cannot scan directory: %s", err) -// } -// -// for scanner.Scan() { -// dirent, err := scanner.Dirent() -// if err != nil { -// warning("cannot get dirent: %s", err) -// continue -// } -// name := dirent.Name() -// if name == "break" { -// break -// } -// if name == "continue" { -// continue -// } -// fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) -// } -// if err := scanner.Err(); err != nil { -// fatal("cannot scan directory: %s", err) -// } -func NewScanner(osDirname string) (*Scanner, error) { - return NewScannerWithScratchBuffer(osDirname, nil) -} - -// NewScannerWithScratchBuffer returns a new directory Scanner that lazily -// enumerates the contents of a single directory. On platforms other than -// Windows it uses the provided scratch buffer to read from the file system. On -// Windows the scratch buffer is ignored. -func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - if len(scratchBuffer) < MinimumScratchBufferSize { - scratchBuffer = newScratchBuffer() - } - scanner := &Scanner{ - scratchBuffer: scratchBuffer, - osDirname: osDirname, - dh: dh, - fd: int(dh.Fd()), - } - return scanner, nil -} - -// Dirent returns the current directory entry while scanning a directory. -func (s *Scanner) Dirent() (*Dirent, error) { - if s.de == nil { - s.de = &Dirent{name: s.childName, path: s.osDirname} - s.de.modeType, s.statErr = modeTypeFromDirent(&s.sde, s.osDirname, s.childName) - } - return s.de, s.statErr -} - -// done is called when directory scanner unable to continue, with either the -// triggering error, or nil when there are simply no more entries to read from -// the directory. -func (s *Scanner) done(err error) { - if s.dh == nil { - return - } - - if cerr := s.dh.Close(); err == nil { - s.err = cerr - } - - s.osDirname, s.childName = "", "" - s.scratchBuffer, s.workBuffer = nil, nil - s.dh, s.de, s.statErr = nil, nil, nil - s.sde = syscall.Dirent{} - s.fd = 0 -} - -// Err returns any error associated with scanning a directory. It is normal to -// call Err after Scan returns false, even though they both ensure Scanner -// resources are released. Do not call until done scanning a directory. -func (s *Scanner) Err() error { - s.done(nil) - return s.err -} - -// Name returns the base name of the current directory entry while scanning a -// directory. -func (s *Scanner) Name() string { return s.childName } - -// Scan potentially reads and then decodes the next directory entry from the -// file system. -// -// When it returns false, this releases resources used by the Scanner then -// returns any error associated with closing the file system directory resource. -func (s *Scanner) Scan() bool { - if s.dh == nil { - return false - } - - s.de = nil - - for { - // When the work buffer has nothing remaining to decode, we need to load - // more data from disk. - if len(s.workBuffer) == 0 { - n, err := syscall.ReadDirent(s.fd, s.scratchBuffer) - // n, err := unix.ReadDirent(s.fd, s.scratchBuffer) - if err != nil { - if err == syscall.EINTR /* || err == unix.EINTR */ { - continue - } - s.done(err) - return false - } - if n <= 0 { // end of directory: normal exit - s.done(nil) - return false - } - s.workBuffer = s.scratchBuffer[:n] // trim work buffer to number of bytes read - } - - // point entry to first syscall.Dirent in buffer - copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(&s.sde))[:], s.workBuffer) - s.workBuffer = s.workBuffer[reclen(&s.sde):] // advance buffer for next iteration through loop - - if inoFromDirent(&s.sde) == 0 { - continue // inode set to 0 indicates an entry that was marked as deleted - } - - nameSlice := nameFromDirent(&s.sde) - nameLength := len(nameSlice) - - if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { - continue - } - - s.childName = string(nameSlice) - return true - } -} diff --git a/vendor/github.com/karrick/godirwalk/scandir_windows.go b/vendor/github.com/karrick/godirwalk/scandir_windows.go deleted file mode 100644 index a2110618e0d..00000000000 --- a/vendor/github.com/karrick/godirwalk/scandir_windows.go +++ /dev/null @@ -1,133 +0,0 @@ -// +build windows - -package godirwalk - -import ( - "fmt" - "os" -) - -// Scanner is an iterator to enumerate the contents of a directory. -type Scanner struct { - osDirname string - childName string - dh *os.File // dh is handle to open directory - de *Dirent - err error // err is the error associated with scanning directory - childMode os.FileMode -} - -// NewScanner returns a new directory Scanner that lazily enumerates the -// contents of a single directory. -// -// scanner, err := godirwalk.NewScanner(dirname) -// if err != nil { -// fatal("cannot scan directory: %s", err) -// } -// -// for scanner.Scan() { -// dirent, err := scanner.Dirent() -// if err != nil { -// warning("cannot get dirent: %s", err) -// continue -// } -// name := dirent.Name() -// if name == "break" { -// break -// } -// if name == "continue" { -// continue -// } -// fmt.Printf("%v %v\n", dirent.ModeType(), dirent.Name()) -// } -// if err := scanner.Err(); err != nil { -// fatal("cannot scan directory: %s", err) -// } -func NewScanner(osDirname string) (*Scanner, error) { - dh, err := os.Open(osDirname) - if err != nil { - return nil, err - } - scanner := &Scanner{ - osDirname: osDirname, - dh: dh, - } - return scanner, nil -} - -// NewScannerWithScratchBuffer returns a new directory Scanner that lazily -// enumerates the contents of a single directory. On platforms other than -// Windows it uses the provided scratch buffer to read from the file system. On -// Windows the scratch buffer parameter is ignored. -func NewScannerWithScratchBuffer(osDirname string, scratchBuffer []byte) (*Scanner, error) { - return NewScanner(osDirname) -} - -// Dirent returns the current directory entry while scanning a directory. -func (s *Scanner) Dirent() (*Dirent, error) { - if s.de == nil { - s.de = &Dirent{ - name: s.childName, - path: s.osDirname, - modeType: s.childMode, - } - } - return s.de, nil -} - -// done is called when directory scanner unable to continue, with either the -// triggering error, or nil when there are simply no more entries to read from -// the directory. -func (s *Scanner) done(err error) { - if s.dh == nil { - return - } - - if cerr := s.dh.Close(); err == nil { - s.err = cerr - } - - s.childName, s.osDirname = "", "" - s.de, s.dh = nil, nil -} - -// Err returns any error associated with scanning a directory. It is normal to -// call Err after Scan returns false, even though they both ensure Scanner -// resources are released. Do not call until done scanning a directory. -func (s *Scanner) Err() error { - s.done(nil) - return s.err -} - -// Name returns the base name of the current directory entry while scanning a -// directory. -func (s *Scanner) Name() string { return s.childName } - -// Scan potentially reads and then decodes the next directory entry from the -// file system. -// -// When it returns false, this releases resources used by the Scanner then -// returns any error associated with closing the file system directory resource. -func (s *Scanner) Scan() bool { - if s.dh == nil { - return false - } - - s.de = nil - - fileinfos, err := s.dh.Readdir(1) - if err != nil { - s.done(err) - return false - } - - if l := len(fileinfos); l != 1 { - s.done(fmt.Errorf("expected a single entry rather than %d", l)) - return false - } - - fi := fileinfos[0] - s.childMode = fi.Mode() & os.ModeType - s.childName = fi.Name() - return true -} diff --git a/vendor/github.com/karrick/godirwalk/scanner.go b/vendor/github.com/karrick/godirwalk/scanner.go deleted file mode 100644 index c08179e1ed2..00000000000 --- a/vendor/github.com/karrick/godirwalk/scanner.go +++ /dev/null @@ -1,44 +0,0 @@ -package godirwalk - -import "sort" - -type scanner interface { - Dirent() (*Dirent, error) - Err() error - Name() string - Scan() bool -} - -// sortedScanner enumerates through a directory's contents after reading the -// entire directory and sorting the entries by name. Used by walk to simplify -// its implementation. -type sortedScanner struct { - dd []*Dirent - de *Dirent -} - -func newSortedScanner(osPathname string, scratchBuffer []byte) (*sortedScanner, error) { - deChildren, err := ReadDirents(osPathname, scratchBuffer) - if err != nil { - return nil, err - } - sort.Sort(deChildren) - return &sortedScanner{dd: deChildren}, nil -} - -func (d *sortedScanner) Err() error { - d.dd, d.de = nil, nil - return nil -} - -func (d *sortedScanner) Dirent() (*Dirent, error) { return d.de, nil } - -func (d *sortedScanner) Name() string { return d.de.name } - -func (d *sortedScanner) Scan() bool { - if len(d.dd) > 0 { - d.de, d.dd = d.dd[0], d.dd[1:] - return true - } - return false -} diff --git a/vendor/github.com/karrick/godirwalk/walk.go b/vendor/github.com/karrick/godirwalk/walk.go deleted file mode 100644 index 9d0235dfb81..00000000000 --- a/vendor/github.com/karrick/godirwalk/walk.go +++ /dev/null @@ -1,379 +0,0 @@ -package godirwalk - -import ( - "errors" - "fmt" - "os" - "path/filepath" -) - -// Options provide parameters for how the Walk function operates. -type Options struct { - // ErrorCallback specifies a function to be invoked in the case of an error - // that could potentially be ignored while walking a file system - // hierarchy. When set to nil or left as its zero-value, any error condition - // causes Walk to immediately return the error describing what took - // place. When non-nil, this user supplied function is invoked with the OS - // pathname of the file system object that caused the error along with the - // error that took place. The return value of the supplied ErrorCallback - // function determines whether the error will cause Walk to halt immediately - // as it would were no ErrorCallback value provided, or skip this file - // system node yet continue on with the remaining nodes in the file system - // hierarchy. - // - // ErrorCallback is invoked both for errors that are returned by the - // runtime, and for errors returned by other user supplied callback - // functions. - ErrorCallback func(string, error) ErrorAction - - // FollowSymbolicLinks specifies whether Walk will follow symbolic links - // that refer to directories. When set to false or left as its zero-value, - // Walk will still invoke the callback function with symbolic link nodes, - // but if the symbolic link refers to a directory, it will not recurse on - // that directory. When set to true, Walk will recurse on symbolic links - // that refer to a directory. - FollowSymbolicLinks bool - - // Unsorted controls whether or not Walk will sort the immediate descendants - // of a directory by their relative names prior to visiting each of those - // entries. - // - // When set to false or left at its zero-value, Walk will get the list of - // immediate descendants of a particular directory, sort that list by - // lexical order of their names, and then visit each node in the list in - // sorted order. This will cause Walk to always traverse the same directory - // tree in the same order, however may be inefficient for directories with - // many immediate descendants. - // - // When set to true, Walk skips sorting the list of immediate descendants - // for a directory, and simply visits each node in the order the operating - // system enumerated them. This will be more fast, but with the side effect - // that the traversal order may be different from one invocation to the - // next. - Unsorted bool - - // Callback is a required function that Walk will invoke for every file - // system node it encounters. - Callback WalkFunc - - // PostChildrenCallback is an option function that Walk will invoke for - // every file system directory it encounters after its children have been - // processed. - PostChildrenCallback WalkFunc - - // ScratchBuffer is an optional byte slice to use as a scratch buffer for - // Walk to use when reading directory entries, to reduce amount of garbage - // generation. Not all architectures take advantage of the scratch - // buffer. If omitted or the provided buffer has fewer bytes than - // MinimumScratchBufferSize, then a buffer with MinimumScratchBufferSize - // bytes will be created and used once per Walk invocation. - ScratchBuffer []byte - - // AllowNonDirectory causes Walk to bypass the check that ensures it is - // being called on a directory node, or when FollowSymbolicLinks is true, a - // symbolic link that points to a directory. Leave this value false to have - // Walk return an error when called on a non-directory. Set this true to - // have Walk run even when called on a non-directory node. - AllowNonDirectory bool -} - -// ErrorAction defines a set of actions the Walk function could take based on -// the occurrence of an error while walking the file system. See the -// documentation for the ErrorCallback field of the Options structure for more -// information. -type ErrorAction int - -const ( - // Halt is the ErrorAction return value when the upstream code wants to halt - // the walk process when a runtime error takes place. It matches the default - // action the Walk function would take were no ErrorCallback provided. - Halt ErrorAction = iota - - // SkipNode is the ErrorAction return value when the upstream code wants to - // ignore the runtime error for the current file system node, skip - // processing of the node that caused the error, and continue walking the - // file system hierarchy with the remaining nodes. - SkipNode -) - -// SkipThis is used as a return value from WalkFuncs to indicate that the file -// system entry named in the call is to be skipped. It is not returned as an -// error by any function. -var SkipThis = errors.New("skip this directory entry") - -// WalkFunc is the type of the function called for each file system node visited -// by Walk. The pathname argument will contain the argument to Walk as a prefix; -// that is, if Walk is called with "dir", which is a directory containing the -// file "a", the provided WalkFunc will be invoked with the argument "dir/a", -// using the correct os.PathSeparator for the Go Operating System architecture, -// GOOS. The directory entry argument is a pointer to a Dirent for the node, -// providing access to both the basename and the mode type of the file system -// node. -// -// If an error is returned by the Callback or PostChildrenCallback functions, -// and no ErrorCallback function is provided, processing stops. If an -// ErrorCallback function is provided, then it is invoked with the OS pathname -// of the node that caused the error along along with the error. The return -// value of the ErrorCallback function determines whether to halt processing, or -// skip this node and continue processing remaining file system nodes. -// -// The exception is when the function returns the special value -// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a -// directory, Walk skips the directory's contents entirely. If the function -// returns filepath.SkipDir when invoked on a non-directory file system node, -// Walk skips the remaining files in the containing directory. Note that any -// supplied ErrorCallback function is not invoked with filepath.SkipDir when the -// Callback or PostChildrenCallback functions return that special value. -// -// One arguably confusing aspect of the filepath.WalkFunc API that this library -// must emulate is how a caller tells Walk to skip file system entries or -// directories. With both filepath.Walk and this Walk, when a callback function -// wants to skip a directory and not descend into its children, it returns -// filepath.SkipDir. If the callback function returns filepath.SkipDir for a -// non-directory, filepath.Walk and this library will stop processing any more -// entries in the current directory, which is what many people do not want. If -// you want to simply skip a particular non-directory entry but continue -// processing entries in the directory, a callback function must return nil. The -// implications of this API is when you want to walk a file system hierarchy and -// skip an entry, when the entry is a directory, you must return one value, -// namely filepath.SkipDir, but when the entry is a non-directory, you must -// return a different value, namely nil. In other words, to get identical -// behavior for two file system entry types you need to send different token -// values. -// -// Here is an example callback function that adheres to filepath.Walk API to -// have it skip any file system entry whose full pathname includes a particular -// substring, optSkip: -// -// func callback1(osPathname string, de *godirwalk.Dirent) error { -// if optSkip != "" && strings.Contains(osPathname, optSkip) { -// if b, err := de.IsDirOrSymlinkToDir(); b == true && err == nil { -// return filepath.SkipDir -// } -// return nil -// } -// // Process file like normal... -// return nil -// } -// -// This library attempts to eliminate some of that logic boilerplate by -// providing a new token error value, SkipThis, which a callback function may -// return to skip the current file system entry regardless of what type of entry -// it is. If the current entry is a directory, its children will not be -// enumerated, exactly as if the callback returned filepath.SkipDir. If the -// current entry is a non-directory, the next file system entry in the current -// directory will be enumerated, exactly as if the callback returned nil. The -// following example callback function has identical behavior as the previous, -// but has less boilerplate, and admittedly more simple logic. -// -// func callback2(osPathname string, de *godirwalk.Dirent) error { -// if optSkip != "" && strings.Contains(osPathname, optSkip) { -// return godirwalk.SkipThis -// } -// // Process file like normal... -// return nil -// } -type WalkFunc func(osPathname string, directoryEntry *Dirent) error - -// Walk walks the file tree rooted at the specified directory, calling the -// specified callback function for each file system node in the tree, including -// root, symbolic links, and other node types. -// -// This function is often much faster than filepath.Walk because it does not -// invoke os.Stat for every node it encounters, but rather obtains the file -// system node type when it reads the parent directory. -// -// If a runtime error occurs, either from the operating system or from the -// upstream Callback or PostChildrenCallback functions, processing typically -// halts. However, when an ErrorCallback function is provided in the provided -// Options structure, that function is invoked with the error along with the OS -// pathname of the file system node that caused the error. The ErrorCallback -// function's return value determines the action that Walk will then take. -// -// func main() { -// dirname := "." -// if len(os.Args) > 1 { -// dirname = os.Args[1] -// } -// err := godirwalk.Walk(dirname, &godirwalk.Options{ -// Callback: func(osPathname string, de *godirwalk.Dirent) error { -// fmt.Printf("%s %s\n", de.ModeType(), osPathname) -// return nil -// }, -// ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { -// // Your program may want to log the error somehow. -// fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) -// -// // For the purposes of this example, a simple SkipNode will suffice, -// // although in reality perhaps additional logic might be called for. -// return godirwalk.SkipNode -// }, -// }) -// if err != nil { -// fmt.Fprintf(os.Stderr, "%s\n", err) -// os.Exit(1) -// } -// } -func Walk(pathname string, options *Options) error { - if options == nil || options.Callback == nil { - return errors.New("cannot walk without non-nil options and Callback function") - } - - pathname = filepath.Clean(pathname) - - var fi os.FileInfo - var err error - - if options.FollowSymbolicLinks { - fi, err = os.Stat(pathname) - } else { - fi, err = os.Lstat(pathname) - } - if err != nil { - return err - } - - mode := fi.Mode() - if !options.AllowNonDirectory && mode&os.ModeDir == 0 { - return fmt.Errorf("cannot Walk non-directory: %s", pathname) - } - - dirent := &Dirent{ - name: filepath.Base(pathname), - path: filepath.Dir(pathname), - modeType: mode & os.ModeType, - } - - if len(options.ScratchBuffer) < MinimumScratchBufferSize { - options.ScratchBuffer = newScratchBuffer() - } - - // If ErrorCallback is nil, set to a default value that halts the walk - // process on all operating system errors. This is done to allow error - // handling to be more succinct in the walk code. - if options.ErrorCallback == nil { - options.ErrorCallback = defaultErrorCallback - } - - err = walk(pathname, dirent, options) - switch err { - case nil, SkipThis, filepath.SkipDir: - // silence SkipThis and filepath.SkipDir for top level - debug("no error of significance: %v\n", err) - return nil - default: - return err - } -} - -// defaultErrorCallback always returns Halt because if the upstream code did not -// provide an ErrorCallback function, walking the file system hierarchy ought to -// halt upon any operating system error. -func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } - -// walk recursively traverses the file system node specified by pathname and the -// Dirent. -func walk(osPathname string, dirent *Dirent, options *Options) error { - err := options.Callback(osPathname, dirent) - if err != nil { - if err == SkipThis || err == filepath.SkipDir { - return err - } - if action := options.ErrorCallback(osPathname, err); action == SkipNode { - return nil - } - return err - } - - if dirent.IsSymlink() { - if !options.FollowSymbolicLinks { - return nil - } - // Does this symlink point to a directory? - info, err := os.Stat(osPathname) - if err != nil { - if action := options.ErrorCallback(osPathname, err); action == SkipNode { - return nil - } - return err - } - if !info.IsDir() { - return nil - } - } else if !dirent.IsDir() { - return nil - } - - // If get here, then specified pathname refers to a directory or a - // symbolic link to a directory. - - var ds scanner - - if options.Unsorted { - // When upstream does not request a sorted iteration, it's more memory - // efficient to read a single child at a time from the file system. - ds, err = NewScanner(osPathname) - } else { - // When upstream wants a sorted iteration, we must read the entire - // directory and sort through the child names, and then iterate on each - // child. - ds, err = newSortedScanner(osPathname, options.ScratchBuffer) - } - if err != nil { - if action := options.ErrorCallback(osPathname, err); action == SkipNode { - return nil - } - return err - } - - for ds.Scan() { - deChild, err := ds.Dirent() - osChildname := filepath.Join(osPathname, deChild.name) - if err != nil { - if action := options.ErrorCallback(osChildname, err); action == SkipNode { - return nil - } - return err - } - err = walk(osChildname, deChild, options) - debug("osChildname: %q; error: %v\n", osChildname, err) - if err == nil || err == SkipThis { - continue - } - if err != filepath.SkipDir { - return err - } - // When received SkipDir on a directory or a symbolic link to a - // directory, stop processing that directory but continue processing - // siblings. When received on a non-directory, stop processing - // remaining siblings. - isDir, err := deChild.IsDirOrSymlinkToDir() - if err != nil { - if action := options.ErrorCallback(osChildname, err); action == SkipNode { - continue // ignore and continue with next sibling - } - return err // caller does not approve of this error - } - if !isDir { - break // stop processing remaining siblings, but allow post children callback - } - // continue processing remaining siblings - } - if err = ds.Err(); err != nil { - return err - } - - if options.PostChildrenCallback == nil { - return nil - } - - err = options.PostChildrenCallback(osPathname, dirent) - if err == nil || err == filepath.SkipDir { - return err - } - - if action := options.ErrorCallback(osPathname, err); action == SkipNode { - return nil - } - return err -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 65414879f82..71e874f372d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -702,9 +702,6 @@ github.com/jstemmer/go-junit-report/parser github.com/jsternberg/zap-logfmt # github.com/julienschmidt/httprouter v1.3.0 github.com/julienschmidt/httprouter -# github.com/karrick/godirwalk v1.16.1 -## explicit -github.com/karrick/godirwalk # github.com/kisielk/gotool v1.0.0 github.com/kisielk/gotool github.com/kisielk/gotool/internal/load