Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add the snapshot sync for v2 store #18

Closed
wants to merge 49 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
0b732a4
add store/v2 types, RootStore
roysc Aug 30, 2021
2ffea14
store/v2 dbadapter
roysc Sep 6, 2021
66aec81
store/v2 mem & transient stores
roysc Sep 14, 2021
4d1735d
db iterator adapter
roysc Sep 6, 2021
5f8d494
flat store additions
roysc Oct 14, 2021
4afd095
add RootStore implementation
roysc Oct 25, 2021
3b3f63e
rename Merkle* to StateCommitment*
roysc Nov 12, 2021
792e93f
move store/v2/flat.* => store/v2/root.*
roysc Nov 12, 2021
46bf6a3
update docs
roysc Nov 12, 2021
65105e7
rm StoreTypeDecoupled
roysc Nov 16, 2021
77effa4
smt.Store - raise smt.InvalidKeyError
roysc Nov 17, 2021
2ada818
Combine root/store & root/root_store
roysc Nov 12, 2021
fa85a72
RootStore uses per-substore SMTs
roysc Nov 16, 2021
d889e07
docs + changelog
roysc Nov 17, 2021
b2007bf
Revert changes to prefix store
roysc Nov 18, 2021
f1caf0f
try to fix rocksdb test case for CI
roysc Nov 18, 2021
d2f0ada
add checks to testcase
roysc Nov 18, 2021
2ae149f
nit
roysc Nov 18, 2021
62f1fc7
rootstore cleanup
roysc Nov 18, 2021
2677b2a
cleanup, comments
roysc Nov 24, 2021
0d5c769
PR revisions - godoc, cleanup
roysc Nov 25, 2021
35843aa
tests + mocks for v2/dbadapter
roysc Nov 25, 2021
5c9eb81
PR revisions
roysc Nov 26, 2021
21e3745
godoc, comments
roysc Nov 29, 2021
5f6b355
test commit - failed revert recovery
roysc Dec 1, 2021
856b5fa
update docs
roysc Dec 1, 2021
d28890b
cleanup
roysc Dec 1, 2021
f568704
godoc & code comments
roysc Dec 1, 2021
5a53647
PR revisions
roysc Dec 1, 2021
1432859
fix v2 mem & tran stores
roysc Dec 1, 2021
b046e20
rm npSubstoreCache
roysc Dec 1, 2021
1648aa6
impl, test trace & listen on rootstore
roysc Dec 1, 2021
91581f8
test fix
roysc Dec 1, 2021
282ea90
rename RootStore => MultiStore
roysc Dec 1, 2021
40584b7
make: don't build rocksdb unless ENABLE_ROCKSDB=true
roysc Dec 2, 2021
f4edb06
set rocksdb_build tag in CI submodule tests
roysc Dec 2, 2021
ad227e0
Update gorocksdb replace directive to use cosmos fork
roysc Dec 2, 2021
15b9b5c
Merge branch 'master' into roysc/adr-040-create-rootstore
roysc Dec 6, 2021
c4d6e42
feat: add the snapshot sync for v2 store
Dec 8, 2021
9eda661
address the pr comments
Dec 9, 2021
551dd3b
address the pr comments
Dec 11, 2021
8f68ced
removed prefix registry from restore
Dec 13, 2021
b024b20
addressed the pr comments
Dec 13, 2021
4284df5
test: add test cases for snapshot for v2 store
Dec 13, 2021
db80903
test: add test for checking restore for non-empty store
Dec 14, 2021
58e4795
test: add test for checking restore for store with existed schema
Dec 14, 2021
21715e9
address the pr comments
Dec 14, 2021
3431130
address the pr comments
Dec 14, 2021
59a7cbb
Merge the master
Dec 16, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions store/v2/root/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (rs *Store) Restore(height uint64, format uint32, chunks <-chan io.ReadClos
}

// Set up a restore stream pipeline
// chan io.ReadCloser -> chunkReader -> zlib -> delimited Protobuf -> ExportNode
// chan io.ReadCloser -> chunkReader -> zlib -> delimited Protobuf -> Exported KV Item
chunkReader := snapshots.NewChunkReader(chunks)
defer chunkReader.Close()
zReader, err := zlib.NewReader(chunkReader)
Expand All @@ -71,6 +71,10 @@ func (rs *Store) Restore(height uint64, format uint32, chunks <-chan io.ReadClos

switch item := item.Item.(type) {
case *storetypes.SnapshotItem_Schema:
if len(rs.schema) != 0 {
return sdkerrors.Wrap(sdkerrors.ErrLogic, "store schema is not empty")
}

storeSchemaReceived = true
schemaWriter := prefixdb.NewPrefixWriter(rs.stateTxn, schemaPrefix)
sKeys := item.Schema.GetKeys()
Expand Down Expand Up @@ -150,7 +154,7 @@ func (rs *Store) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, e
ch := make(chan io.ReadCloser)
go func() {
// Set up a stream pipeline to serialize snapshot nodes:
// ExportNode -> delimited Protobuf -> zlib -> buffer -> chunkWriter -> chan io.ReadCloser
// Export KV Item -> delimited Protobuf -> zlib -> buffer -> chunkWriter -> chan io.ReadCloser
chunkWriter := snapshots.NewChunkWriter(ch, snapshotChunkSize)
defer chunkWriter.Close()
bufWriter := bufio.NewWriterSize(chunkWriter, snapshotBufferSize)
Expand Down
17 changes: 14 additions & 3 deletions store/v2/root/snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,13 @@ func TestMultistoreSnapshot_Checksum(t *testing.T) {
}

func TestMultistoreSnapshotRestore(t *testing.T) {
source := newMultiStoreWithBasicData(t, memdb.NewDB(), 4)
source := newMultiStoreWithGeneratedData(t, memdb.NewDB(), 3, 4)
target := newMultiStore(t, memdb.NewDB(), 0)
require.Equal(t, source.LastCommitID().Version, int64(1))
version := uint64(source.LastCommitID().Version)
// check for target store restore
require.Equal(t, target.LastCommitID().Version, int64(0))

chunks, err := source.Snapshot(version, snapshottypes.CurrentFormat)
require.NoError(t, err)
ready := make(chan struct{})
Expand All @@ -191,11 +194,19 @@ func TestMultistoreSnapshotRestore(t *testing.T) {
require.Equal(t, sourceSubStore, targetSubStore)
}

// restore checking for non-empty store
target2 := newMultiStoreWithBasicData(t, memdb.NewDB(), 0)
// checking snapshot restore for non-empty store with existed saved version
gsk967 marked this conversation as resolved.
Show resolved Hide resolved
target2 := newMultiStoreWithBasicData(t, memdb.NewDB(), 4)
ready2 := make(chan struct{})
err = target2.Restore(version, snapshottypes.CurrentFormat, chunks, ready2)
require.Error(t, err)

// checking snapshot restoring for store with existed schema and without existed versions
target3 := newMultiStore(t, memdb.NewDB(), 4)
ready3 := make(chan struct{})
chunks, err = source.Snapshot(version, snapshottypes.CurrentFormat)
require.NoError(t, err)
err = target3.Restore(version, snapshottypes.CurrentFormat, chunks, ready3)
require.Error(t, err)
}

func BenchmarkMultistoreSnapshot100K(b *testing.B) {
Expand Down