Add cache for calculated etags for home and shares directory (#1359)

changelog/unreleased/gateway-etag-cache.md

@@ -0,0 +1,8 @@
+Enhancement: Add cache for calculated etags for home and shares directory
+
+Since we store the references in the shares directory instead of actual
+resources, we need to calculate the etag on every list/stat call. This is rather
+expensive so adding a cache would help to a great extent with regard to the
+performance.
+
+https://github.com/cs3org/reva/pull/1359

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
+	github.com/ReneKroon/ttlcache/v2 v2.1.0
 	github.com/aws/aws-sdk-go v1.35.27
 	github.com/c-bata/go-prompt v0.2.5
 	github.com/cheggaaa/pb v1.0.29

go.sum

@@ -39,6 +39,9 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk=
+github.com/ReneKroon/ttlcache/v2 v2.1.0 h1:5GQZXf7Pl68S4O/Mps4kAXQ+ObEqwHskC8WKT77mHiQ=
+github.com/ReneKroon/ttlcache/v2 v2.1.0/go.mod h1:0E5EIhFJrGmcn/niHrqnnvQHvihj+DdPzJlOEzBS3o4=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -422,6 +425,7 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -574,6 +578,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -591,6 +596,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200721223218-6123e77877b2/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/grpc/services/gateway/gateway.go

@@ -22,9 +22,11 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"time"
 
 	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 
+	"github.com/ReneKroon/ttlcache/v2"
 	"github.com/cs3org/reva/pkg/rgrpc"
 	"github.com/cs3org/reva/pkg/sharedconf"
 	"github.com/cs3org/reva/pkg/token"
@@ -62,6 +64,7 @@ type config struct {
 	ShareFolder   string                            `mapstructure:"share_folder"`
 	HomeMapping   string                            `mapstructure:"home_mapping"`
 	TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"`
+	EtagCacheTTL  int                               `mapstructure:"etag_cache_ttl"`
 }
 
 // sets defaults
@@ -106,6 +109,7 @@ type svc struct {
 	c              *config
 	dataGatewayURL url.URL
 	tokenmgr       token.Manager
+	etagCache      *ttlcache.Cache `mapstructure:"etag_cache"`
 }
 
 // New creates a new gateway svc that acts as a proxy for any grpc operation.
@@ -130,10 +134,15 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 		return nil, err
 	}
 
+	etagCache := ttlcache.NewCache()
+	_ = etagCache.SetTTL(time.Duration(c.EtagCacheTTL) * time.Second)
+	etagCache.SkipTTLExtensionOnHit(true)
+
 	s := &svc{
 		c:              c,
 		dataGatewayURL: *u,
 		tokenmgr:       tokenManager,
+		etagCache:      etagCache,
 	}
 
 	return s, nil
@@ -144,6 +153,7 @@ func (s *svc) Register(ss *grpc.Server) {
 }
 
 func (s *svc) Close() error {
+	s.etagCache.Close()
 	return nil
 }
 

internal/grpc/services/gateway/storageprovider.go

@@ -985,7 +985,15 @@ func (s *svc) statHome(ctx context.Context) (*provider.StatResponse, error) {
 		}, nil
 	}
 
-	statRes.Info.Etag = etag.GenerateEtagFromResources(statRes.Info, []*provider.ResourceInfo{statSharedFolder.Info})
+	if resEtag, err := s.etagCache.Get(statRes.Info.Owner.OpaqueId + ":" + statRes.Info.Path); err == nil {
+		statRes.Info.Etag = resEtag.(string)
+	} else {
+		statRes.Info.Etag = etag.GenerateEtagFromResources(statRes.Info, []*provider.ResourceInfo{statSharedFolder.Info})
+		if s.c.EtagCacheTTL > 0 {
+			_ = s.etagCache.Set(statRes.Info.Owner.OpaqueId+":"+statRes.Info.Path, statRes.Info.Etag)
+		}
+	}
+
 	return statRes, nil
 }
 
@@ -1020,7 +1028,15 @@ func (s *svc) statSharesFolder(ctx context.Context) (*provider.StatResponse, err
 			Status: lsRes.Status,
 		}, nil
 	}
-	statRes.Info.Etag = etag.GenerateEtagFromResources(statRes.Info, lsRes.Infos)
+
+	if resEtag, err := s.etagCache.Get(statRes.Info.Owner.OpaqueId + ":" + statRes.Info.Path); err == nil {
+		statRes.Info.Etag = resEtag.(string)
+	} else {
+		statRes.Info.Etag = etag.GenerateEtagFromResources(statRes.Info, lsRes.Infos)
+		if s.c.EtagCacheTTL > 0 {
+			_ = s.etagCache.Set(statRes.Info.Owner.OpaqueId+":"+statRes.Info.Path, statRes.Info.Etag)
+		}
+	}
 	return statRes, nil
 }
 

pkg/storage/utils/etag/etag.go

@@ -51,23 +51,23 @@ var (
 // GenerateEtagFromResources creates a unique etag for the root folder deriving
 // information from its multiple children
 func GenerateEtagFromResources(root *provider.ResourceInfo, children []*provider.ResourceInfo) string {
-	if m := getEtagParams(eosMtimeEtag, root.Etag); len(m) > 0 {
+	if params := getEtagParams(eosMtimeEtag, root.Etag); len(params) > 0 {
 		mtime := time.Unix(int64(root.Mtime.Seconds), int64(root.Mtime.Nanos))
 		for _, r := range children {
 			m := time.Unix(int64(r.Mtime.Seconds), int64(r.Mtime.Nanos))
 			if m.After(mtime) {
 				mtime = m
 			}
 		}
-		return fmt.Sprintf("\"%s:%d.%s\"", m["inode"], mtime.Unix(), strconv.FormatInt(mtime.UnixNano(), 10)[:3])
+		return fmt.Sprintf("\"%s:%d.%s\"", params["inode"], mtime.Unix(), strconv.FormatInt(mtime.UnixNano(), 10)[:3])
 	}
 
 	return combineEtags(children)
 }
 
 func combineEtags(resources []*provider.ResourceInfo) string {
 	sort.SliceStable(resources, func(i, j int) bool {
-		return resources[i].Etag < resources[j].Etag
+		return resources[i].Path < resources[j].Path
 	})
 
 	h := md5.New()