feat: add Features + datastore scoping

The motivation for this is to enable "dispatching" datastores that
dynamically implement the type of the datastore they are dispatching
to, so that type assertions behave equivalently on the dispatcher as
on the dispatchee. We also want this to be backwards-compatible with
existing code using type assertions.

At a high level, this works by generating a concrete implementation of
every possible combination of "features", and then picking the right
implementation at runtime. This is necessary due to language
constraints in Go--it is currently impossible to create a concrete
type dynamically with reflection that implements an interface.

"Features" are introduced here, which are supplemental, optional
interfaces that datastores may implement. These are
backwards-compatible with existing "features", which are:

* Batching
* CheckedDatastore
* GCDatastore
* PersistentDatastore
* ScrubbedDatastore
* TTLDatastore
* TxnDatastore

New features can also be added in a backwards-compatible way. E.g. if
datastore A is scoped down to datastore B, a new feature F is added,
and then implemented on B, then A will continue to implement the same
set of features since it hasn't implemented F yet (and vice versa if F
is implemented on A but not B).

Examples of things this enables:

* Allow us to deprecate ErrBatchUnsupported
* Allow existing dispatching datastores to support all
features (keytransform, retrystore, MutexDatastore, autobatch, etc.)
* Features supported by a Mount datastore could be scoped down to the
intersection of all children
* Communication with data about what functionality a datastore
supports (e.g. for cross-language/RPC support)

Some related issues:

* #160
* #88

datastore.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"time"
 
 	query "github.com/ipfs/go-datastore/query"
 )
@@ -103,8 +102,7 @@ type Read interface {
 // capabilities of a `Batch`, but the reverse is NOT true.
 type Batching interface {
 	Datastore
-
-	Batch(ctx context.Context) (Batch, error)
+	BatchingFeature
 }
 
 // ErrBatchUnsupported is returned if the by Batch if the Datastore doesn't
@@ -115,34 +113,29 @@ var ErrBatchUnsupported = errors.New("this datastore does not support batching")
 // which may need checking on-disk data integrity.
 type CheckedDatastore interface {
 	Datastore
-
-	Check(ctx context.Context) error
+	CheckedFeature
 }
 
 // ScrubbedDatastore is an interface that should be implemented by datastores
 // which want to provide a mechanism to check data integrity and/or
 // error correction.
 type ScrubbedDatastore interface {
 	Datastore
-
-	Scrub(ctx context.Context) error
+	ScrubbedFeature
 }
 
 // GCDatastore is an interface that should be implemented by datastores which
 // don't free disk space by just removing data from them.
 type GCDatastore interface {
 	Datastore
-
-	CollectGarbage(ctx context.Context) error
+	GCFeature
 }
 
 // PersistentDatastore is an interface that should be implemented by datastores
 // which can report disk usage.
 type PersistentDatastore interface {
 	Datastore
-
-	// DiskUsage returns the space used by a datastore, in bytes.
-	DiskUsage(ctx context.Context) (uint64, error)
+	PersistentFeature
 }
 
 // DiskUsage checks if a Datastore is a
@@ -163,13 +156,6 @@ type TTLDatastore interface {
 	TTL
 }
 
-// TTL encapulates the methods that deal with entries with time-to-live.
-type TTL interface {
-	PutWithTTL(ctx context.Context, key Key, value []byte, ttl time.Duration) error
-	SetTTL(ctx context.Context, key Key, ttl time.Duration) error
-	GetExpiration(ctx context.Context, key Key) (time.Time, error)
-}
-
 // Txn extends the Datastore type. Txns allow users to batch queries and
 // mutations to the Datastore into atomic groups, or transactions. Actions
 // performed on a transaction will not take hold until a successful call to
@@ -194,8 +180,7 @@ type Txn interface {
 // support transactions.
 type TxnDatastore interface {
 	Datastore
-
-	NewTransaction(ctx context.Context, readOnly bool) (Txn, error)
+	TxnFeature
 }
 
 // Errors

features.go

@@ -0,0 +1,142 @@
+package datastore
+
+import (
+	"context"
+	"reflect"
+	"time"
+)
+
+const (
+	FeatureNameBatching    = "Batching"
+	FeatureNameChecked     = "Checked"
+	FeatureNameGC          = "GC"
+	FeatureNamePersistent  = "Persistent"
+	FeatureNameScrubbed    = "Scrubbed"
+	FeatureNameTTL         = "TTL"
+	FeatureNameTransaction = "Transaction"
+)
+
+type BatchingFeature interface {
+	Batch(ctx context.Context) (Batch, error)
+}
+
+type CheckedFeature interface {
+	Check(ctx context.Context) error
+}
+
+type ScrubbedFeature interface {
+	Scrub(ctx context.Context) error
+}
+
+type GCFeature interface {
+	CollectGarbage(ctx context.Context) error
+}
+
+type PersistentFeature interface {
+	// DiskUsage returns the space used by a datastore, in bytes.
+	DiskUsage(ctx context.Context) (uint64, error)
+}
+
+// TTL encapulates the methods that deal with entries with time-to-live.
+type TTL interface {
+	PutWithTTL(ctx context.Context, key Key, value []byte, ttl time.Duration) error
+	SetTTL(ctx context.Context, key Key, ttl time.Duration) error
+	GetExpiration(ctx context.Context, key Key) (time.Time, error)
+}
+
+type TxnFeature interface {
+	NewTransaction(ctx context.Context, readOnly bool) (Txn, error)
+}
+
+// Feature contains metadata about a datastore Feature.
+type Feature struct {
+	Name string
+	// Interface is the nil interface of the feature.
+	Interface interface{}
+	// DatastoreInterface is the nil interface of the feature's corresponding datastore interface.
+	DatastoreInterface interface{}
+}
+
+var featuresByName map[string]Feature
+
+func init() {
+	featuresByName = map[string]Feature{}
+	for _, f := range Features() {
+		featuresByName[f.Name] = f
+	}
+}
+
+// Features returns a list of all known datastore features.
+// This serves both to provide an authoritative list of features,
+// and to define a canonical ordering of features.
+func Features() []Feature {
+	// for backwards compatibility, only append to this list
+	return []Feature{
+		{
+			Name:               FeatureNameBatching,
+			Interface:          (*BatchingFeature)(nil),
+			DatastoreInterface: (*Batching)(nil),
+		},
+		{
+			Name:               FeatureNameChecked,
+			Interface:          (*CheckedFeature)(nil),
+			DatastoreInterface: (*CheckedDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameGC,
+			Interface:          (*GCFeature)(nil),
+			DatastoreInterface: (*GCDatastore)(nil),
+		},
+		{
+			Name:               FeatureNamePersistent,
+			Interface:          (*PersistentFeature)(nil),
+			DatastoreInterface: (*PersistentDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameScrubbed,
+			Interface:          (*ScrubbedFeature)(nil),
+			DatastoreInterface: (*ScrubbedDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameTTL,
+			Interface:          (*TTL)(nil),
+			DatastoreInterface: (*TTLDatastore)(nil),
+		},
+		{
+			Name:               FeatureNameTransaction,
+			Interface:          (*TxnFeature)(nil),
+			DatastoreInterface: (*TxnDatastore)(nil),
+		},
+	}
+}
+
+// FeaturesByNames returns the features with the given names, if they are known.
+func FeaturesByNames(names ...string) (features []Feature) {
+	for _, n := range names {
+		if feat, ok := featuresByName[n]; ok {
+			features = append(features, feat)
+		}
+	}
+	return
+}
+
+// FeatureByName returns the feature with the given name, if known.
+func FeatureByName(name string) (Feature, bool) {
+	feat, known := featuresByName[name]
+	return feat, known
+}
+
+// FeaturesForDatastore returns the features supported by the given datastore.
+func FeaturesForDatastore(dstore Datastore) (features []Feature) {
+	if dstore == nil {
+		return nil
+	}
+	dstoreType := reflect.ValueOf(dstore).Type()
+	for _, f := range Features() {
+		fType := reflect.TypeOf(f.Interface).Elem()
+		if dstoreType.Implements(fType) {
+			features = append(features, f)
+		}
+	}
+	return
+}

features_test.go

@@ -0,0 +1,80 @@
+package datastore
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFeaturesByNames(t *testing.T) {
+	feats := FeaturesByNames()
+	if feats != nil {
+		t.Fatalf("expected nil features, got %v", feats)
+	}
+
+	feats = FeaturesByNames(FeatureNameBatching)
+	if len(feats) != 1 ||
+		feats[0].Name != FeatureNameBatching ||
+		feats[0].Interface != (*BatchingFeature)(nil) ||
+		feats[0].DatastoreInterface != (*Batching)(nil) {
+		t.Fatalf("expected a batching feature, got %v", feats)
+	}
+
+	feats = FeaturesByNames(FeatureNameBatching, "UnknownFeature")
+	if len(feats) != 1 || feats[0].Name != FeatureNameBatching {
+		t.Fatalf("expected a batching feature, got %v", feats)
+	}
+}
+
+func TestFeatureByName(t *testing.T) {
+	feat, ok := FeatureByName(FeatureNameBatching)
+	if !ok {
+		t.Fatalf("expected a batching feature")
+	}
+	if feat.Name != FeatureNameBatching ||
+		feat.Interface != (*BatchingFeature)(nil) ||
+		feat.DatastoreInterface != (*Batching)(nil) {
+		t.Fatalf("expected a batching feature, got %v", feat)
+	}
+
+	feat, ok = FeatureByName("UnknownFeature")
+	if ok {
+		t.Fatalf("expected UnknownFeature not to be found")
+	}
+}
+
+func TestFeaturesForDatastore(t *testing.T) {
+	cases := []struct {
+		name             string
+		d                Datastore
+		expectedFeatures []string
+	}{
+		{
+			name:             "MapDatastore",
+			d:                &MapDatastore{},
+			expectedFeatures: []string{"Batching"},
+		},
+		{
+			name:             "NullDatastore",
+			d:                &NullDatastore{},
+			expectedFeatures: []string{"Batching"},
+		},
+		{
+			name:             "LogDatastore",
+			d:                &LogDatastore{},
+			expectedFeatures: []string{"Batching", "Checked", "GC", "Persistent", "Scrubbed"},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			feats := FeaturesForDatastore(c.d)
+			if len(feats) != len(c.expectedFeatures) {
+				t.Fatalf("expected %d features, got %v", len(c.expectedFeatures), feats)
+			}
+			expectedFeats := FeaturesByNames(c.expectedFeatures...)
+			if !reflect.DeepEqual(expectedFeats, feats) {
+				t.Fatalf("expected features %v, got %v", c.expectedFeatures, feats)
+			}
+		})
+	}
+}

scoped/doc.go

@@ -0,0 +1,40 @@
+// Package scoped introduces a Datastore Shim that scopes down a source datastore
+// to the features supported by a target datastore. This is useful e.g. for dispatching
+// datastores, where the dispatcher needs to dynamically implement the same features
+// as the dispatchee, without knowing them statically.
+//
+// Use the Wrap function to wrap a datastore so that its interface is scoped down to
+// only those features supported both by it and its target datastore. Note that this
+// is a set intersection--if the target implements a feature not supported by the
+// wrapped datastore, then the resulting shim will not implement them either.
+//
+// For example:
+//
+//  import (
+//    "context"
+//    scopedds "github.com/ipfs/go-datastore/scoped"
+//    ds "github.com/ipfs/go-datastore"
+//  )
+//
+//  type BatchingDS struct { ds.Datastore }
+//
+//  func (b *BatchingDS) Batch(ctx context.Context) (ds.Batch, error) {
+//    // custom batching
+//    return nil, nil
+//  }
+//
+//  type BoringDS struct { ds.Datastore }
+//
+//  func Dispatcher(dstore ds.Datastore) ds.Datastore {
+//    dispatcher := &BatchingDS{Datastore: dstore}
+//    dispatchee := &BoringDS{Datastore: dstore}
+//
+//    // the dispatcher supports batching, but since the dispatchee
+//    // doesn't, the returned dispatcher does NOT implement ds.Batching
+//
+//    return scoped.Wrap(dispatcher, dispatchee)
+//  }
+
+package scoped
+
+//go:generate go run generate/main.go