add monotonic counter interface for rollback protection

db.go

@@ -501,8 +501,9 @@ type DB struct {
 	// compaction concurrency
 	openedAt time.Time
 
-	keyManager *edg.KeyManager
-	txLock     sync.Mutex
+	keyManager       *edg.KeyManager
+	txLock           sync.Mutex
+	monotonicCounter uint64
 }
 
 var _ Reader = (*DB)(nil)

edg.go

@@ -0,0 +1,60 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package estore
+
+import (
+	"encoding/binary"
+
+	"github.com/cockroachdb/errors"
+)
+
+var edgMonotonicCounterKey = []byte("!EDGELESS_MONOTONIC_COUNTER")
+
+func (d *DB) edgGetMonotonicCounterFromStore() (uint64, error) {
+	value, closer, err := d.Get(edgMonotonicCounterKey)
+	if errors.Is(err, ErrNotFound) {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	defer closer.Close()
+	return binary.LittleEndian.Uint64(value), nil
+}
+
+func (d *DB) edgSetMonotonicCounterOnStore(value uint64) error {
+	return d.Set(edgMonotonicCounterKey, binary.LittleEndian.AppendUint64(nil, value), nil)
+}
+
+func (d *DB) edgVerifyFreshness() error {
+	if d.opts.SetMonotonicCounter == nil {
+		return nil
+	}
+
+	// get counter from trusted source
+	sourceCount, err := d.opts.SetMonotonicCounter(0)
+	if err != nil {
+		return errors.Wrap(err, "getting monotonic counter from trusted source")
+	}
+
+	// get counter from store
+	storeCount, err := d.edgGetMonotonicCounterFromStore()
+	if err != nil {
+		return errors.Wrap(err, "getting monotonic counter from store")
+	}
+
+	if storeCount < sourceCount {
+		return errors.Newf("rollback detected: store counter: %v, trusted source counter: %v", storeCount, sourceCount)
+	}
+	if storeCount > sourceCount {
+		d.opts.Logger.Infof("WARNING: open: monotonic counter source lags behind: store counter: %v, source counter: %v", storeCount, sourceCount)
+		// will be synced on next tx commit
+	}
+
+	d.monotonicCounter = storeCount
+	return nil
+}

internal/edg/edg_test.go

@@ -12,6 +12,7 @@ import (
 	"math"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/edgelesssys/estore"
@@ -55,9 +56,10 @@ func TestConfidentiality(t *testing.T) {
 
 	fs := vfs.NewMem()
 	db, err := estore.Open("", &estore.Options{
-		EncryptionKey: testKey(),
-		FS:            fs,
-		Levels:        []estore.LevelOptions{{Compression: estore.NoCompression}},
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: (&fakeCounter{}).set,
+		FS:                  fs,
+		Levels:              []estore.LevelOptions{{Compression: estore.NoCompression}},
 	})
 	require.NoError(err)
 
@@ -239,8 +241,9 @@ func TestOldDB(t *testing.T) {
 	require := require.New(t)
 
 	opts := &estore.Options{
-		EncryptionKey: testKey(),
-		FS:            vfs.NewMem(),
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: (&fakeCounter{}).set,
+		FS:                  vfs.NewMem(),
 	}
 
 	ok, err := vfs.Clone(vfs.Default, opts.FS, "testdata/db-v1.0.0", "")
@@ -273,6 +276,206 @@ func TestOldDB(t *testing.T) {
 	require.NoError(db.Close())
 }
 
+func TestRollbackProtection(t *testing.T) {
+	require := require.New(t)
+
+	const dbdir = "db"
+	const olddir = "old"
+	fs := vfs.NewMem()
+	var counter fakeCounter
+
+	opts := &estore.Options{
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: counter.set,
+		FS:                  fs,
+	}
+
+	// create db
+	db, err := estore.Open(dbdir, opts)
+	require.NoError(err)
+	tx := db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val1"), nil))
+	require.NoError(tx.Commit())
+	require.NoError(db.Close())
+
+	// copy the db
+	ok, err := vfs.Clone(fs, fs, dbdir, olddir)
+	require.NoError(err)
+	require.True(ok)
+
+	// advance db
+	db, err = estore.Open(dbdir, opts)
+	require.NoError(err)
+	tx = db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val2"), nil))
+	require.NoError(tx.Commit())
+	require.NoError(db.Close())
+
+	// try to roll back the db
+	_, err = estore.Open(olddir, opts)
+	require.ErrorContains(err, "rollback detected")
+}
+
+func TestRollbackProtection_Open_CounterSourceFails(t *testing.T) {
+	require := require.New(t)
+
+	counter := fakeCounter{preErr: assert.AnError}
+
+	opts := &estore.Options{
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: counter.set,
+		FS:                  vfs.NewMem(),
+	}
+
+	_, err := estore.Open("", opts)
+	require.ErrorIs(err, counter.preErr)
+}
+
+func TestRollbackProtection_Open_NewDBWithExistingCounter(t *testing.T) {
+	require := require.New(t)
+
+	var counter fakeCounter
+
+	opts := &estore.Options{
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: counter.set,
+		FS:                  vfs.NewMem(),
+	}
+
+	counter.value = 2
+	_, err := estore.Open("", opts)
+	require.ErrorContains(err, "rollback detected")
+}
+
+func TestRollbackProtection_Open_CounterSourceRollbackCanBeHandled(t *testing.T) {
+	require := require.New(t)
+
+	var counter fakeCounter
+
+	opts := &estore.Options{
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: counter.set,
+		FS:                  vfs.NewMem(),
+	}
+
+	// create db
+	db, err := estore.Open("", opts)
+	require.NoError(err)
+	tx := db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val1"), nil))
+	require.NoError(tx.Commit())
+	tx = db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val2"), nil))
+	require.NoError(tx.Commit())
+	require.NoError(db.Close())
+
+	// roll back counter source
+	counter.value--
+
+	// advance db
+	db, err = estore.Open("", opts)
+	require.NoError(err)
+	tx = db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val3"), nil))
+	require.NoError(tx.Commit())
+	require.NoError(db.Close())
+
+	// counter is synced
+	require.EqualValues(3, counter.value)
+}
+
+func TestRollbackProtection_CounterSourceReturnsError(t *testing.T) {
+	testCases := map[string]struct {
+		preErr  error
+		postErr error
+	}{
+		"error before increment": {
+			preErr: assert.AnError,
+		},
+		"error after increment": {
+			postErr: assert.AnError,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			require := require.New(t)
+
+			var counter fakeCounter
+
+			opts := &estore.Options{
+				EncryptionKey:       testKey(),
+				SetMonotonicCounter: counter.set,
+				FS:                  vfs.NewMem(),
+			}
+
+			db, err := estore.Open("", opts)
+			require.NoError(err)
+
+			// tx fails
+			counter.preErr = tc.preErr
+			counter.postErr = tc.postErr
+			tx := db.NewTransaction(true)
+			require.NoError(tx.Set([]byte("key"), []byte("val1"), nil))
+			require.Error(tx.Commit())
+
+			// value was not written
+			_, _, err = db.Get([]byte("key"))
+			require.ErrorIs(err, estore.ErrNotFound)
+
+			// retry succeeds
+			counter.preErr = nil
+			counter.postErr = nil
+			tx = db.NewTransaction(true)
+			require.NoError(tx.Set([]byte("key"), []byte("val1"), nil))
+			require.NoError(tx.Commit())
+
+			require.NoError(db.Close())
+
+			// counter is synced
+			require.EqualValues(2, counter.value)
+		})
+	}
+}
+
+func TestRollbackProtection_AdvancingCounterSourceCausesFailure(t *testing.T) {
+	require := require.New(t)
+
+	var counter fakeCounter
+
+	opts := &estore.Options{
+		EncryptionKey:       testKey(),
+		SetMonotonicCounter: counter.set,
+		FS:                  vfs.NewMem(),
+		Logger:              &base.InMemLogger{}, // calls runtime.Goexit on Fatalf
+	}
+
+	// create db
+	db, err := estore.Open("", opts)
+	require.NoError(err)
+	tx := db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val1"), nil))
+	require.NoError(tx.Commit())
+
+	// advance counter source
+	counter.value++
+
+	tx = db.NewTransaction(true)
+	require.NoError(tx.Set([]byte("key"), []byte("val2"), nil))
+
+	// expect fatal exit on commit
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		tx.Commit()
+		panic("unreachable")
+	}()
+	wg.Wait()
+
+	require.NoError(db.Close())
+}
+
 func testKey() []byte {
 	return bytes.Repeat([]byte{2}, 16)
 }
@@ -312,3 +515,23 @@ func isCryptoError(err error) bool {
 
 	return false
 }
+
+type fakeCounter struct {
+	value   uint64
+	preErr  error
+	postErr error
+}
+
+func (c *fakeCounter) set(value uint64) (uint64, error) {
+	if c.preErr != nil {
+		return 0, c.preErr
+	}
+	prev := c.value
+	if value > c.value {
+		c.value = value
+	}
+	if c.postErr != nil {
+		return 0, c.postErr
+	}
+	return prev, nil
+}

internal/metamorphic/crossversion/crossversion_test.go

@@ -365,7 +365,7 @@ func (f *pebbleVersions) String() string {
 		if i > 0 {
 			fmt.Fprint(&buf, " ")
 		}
-		fmt.Fprintf(&buf, v.SHA)
+		fmt.Fprint(&buf, v.SHA)
 	}
 	return buf.String()
 }

iterator_test.go

@@ -2087,7 +2087,7 @@ func BenchmarkIteratorSeqSeekGEWithBounds(b *testing.B) {
 						valid = iter.Next()
 					}
 					if iter.Error() != nil {
-						b.Fatalf(iter.Error().Error())
+						b.Fatal(iter.Error().Error())
 					}
 				}
 				iter.Close()
@@ -2398,25 +2398,25 @@ func TestRangeKeyMaskingRandomized(t *testing.T) {
 			t.Fatalf("iteration didn't produce identical results")
 		}
 		if hasP1 && !bytes.Equal(iter1.Key(), iter2.Key()) {
-			t.Fatalf(fmt.Sprintf("iteration didn't produce identical point keys: %s, %s", iter1.Key(), iter2.Key()))
+			t.Fatalf("iteration didn't produce identical point keys: %s, %s", iter1.Key(), iter2.Key())
 		}
 		if hasR1 {
 			// Confirm that the range key is the same.
 			b1, e1 := iter1.RangeBounds()
 			b2, e2 := iter2.RangeBounds()
 			if !bytes.Equal(b1, b2) || !bytes.Equal(e1, e2) {
-				t.Fatalf(fmt.Sprintf(
+				t.Fatalf(
 					"iteration didn't produce identical range keys: [%s, %s], [%s, %s]",
 					b1, e1, b2, e2,
-				))
+				)
 			}
 
 		}
 
 		// Confirm that the returned point key wasn't hidden.
 		for j, pkey := range keys {
 			if bytes.Equal(iter1.Key(), pkey) && pointKeyHidden[j] {
-				t.Fatalf(fmt.Sprintf("hidden point key was exposed %s %d", pkey, keyTimeStamps[j]))
+				t.Fatalf("hidden point key was exposed %s %d", pkey, keyTimeStamps[j])
 			}
 		}
 	}