From f1dab4f85c38b5c328eb4a35a0edff36e2b1b5a3 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 27 Mar 2023 16:28:50 +0200 Subject: [PATCH 1/2] refactor insertion index to be publicly accessible --- v2/blockstore/readwrite.go | 4 +-- .../store => index}/insertionindex.go | 29 +++++++++---------- v2/internal/store/index.go | 2 +- v2/internal/store/indexcheck.go | 5 ++-- v2/internal/store/resume.go | 3 +- 5 files changed, 22 insertions(+), 21 deletions(-) rename v2/{internal/store => index}/insertionindex.go (89%) diff --git a/v2/blockstore/readwrite.go b/v2/blockstore/readwrite.go index b6f0943e..f84ae9a8 100644 --- a/v2/blockstore/readwrite.go +++ b/v2/blockstore/readwrite.go @@ -37,7 +37,7 @@ type ReadWrite struct { f *os.File dataWriter *internalio.OffsetWriteSeeker - idx *store.InsertionIndex + idx *index.InsertionIndex header carv2.Header finalized bool // also protected by ronly.mu @@ -117,7 +117,7 @@ func OpenReadWriteFile(f *os.File, roots []cid.Cid, opts ...carv2.Option) (*Read // Set the header fileld before applying options since padding options may modify header. rwbs := &ReadWrite{ f: f, - idx: store.NewInsertionIndex(), + idx: index.NewInsertionIndex(), header: carv2.NewHeader(0), opts: carv2.ApplyOptions(opts...), finalized: false, diff --git a/v2/internal/store/insertionindex.go b/v2/index/insertionindex.go similarity index 89% rename from v2/internal/store/insertionindex.go rename to v2/index/insertionindex.go index bc25fffb..724f74ca 100644 --- a/v2/internal/store/insertionindex.go +++ b/v2/index/insertionindex.go @@ -1,4 +1,4 @@ -package store +package index import ( "bytes" @@ -8,7 +8,6 @@ import ( "io" "github.com/ipfs/go-cid" - "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/petar/GoLLRB/llrb" @@ -34,7 +33,7 @@ func NewInsertionIndex() *InsertionIndex { type recordDigest struct { digest []byte - index.Record + Record } func (r recordDigest) Less(than llrb.Item) bool { @@ -45,7 +44,7 @@ func (r recordDigest) Less(than llrb.Item) bool { return bytes.Compare(r.digest, other.digest) < 0 } -func newRecordDigest(r index.Record) recordDigest { +func newRecordDigest(r Record) recordDigest { d, err := multihash.Decode(r.Hash()) if err != nil { panic(err) @@ -60,7 +59,7 @@ func newRecordFromCid(c cid.Cid, at uint64) recordDigest { panic(err) } - return recordDigest{d.Digest, index.Record{Cid: c, Offset: at}} + return recordDigest{d.Digest, Record{Cid: c, Offset: at}} } func (ii *InsertionIndex) InsertNoReplace(key cid.Cid, n uint64) { @@ -75,19 +74,19 @@ func (ii *InsertionIndex) Get(c cid.Cid) (uint64, error) { return record.Offset, nil } -func (ii *InsertionIndex) getRecord(c cid.Cid) (index.Record, error) { +func (ii *InsertionIndex) getRecord(c cid.Cid) (Record, error) { d, err := multihash.Decode(c.Hash()) if err != nil { - return index.Record{}, err + return Record{}, err } entry := recordDigest{digest: d.Digest} e := ii.items.Get(entry) if e == nil { - return index.Record{}, index.ErrNotFound + return Record{}, ErrNotFound } r, ok := e.(recordDigest) if !ok { - return index.Record{}, errUnsupported + return Record{}, errUnsupported } return r.Record, nil @@ -112,7 +111,7 @@ func (ii *InsertionIndex) GetAll(c cid.Cid, fn func(uint64) bool) error { } ii.items.AscendGreaterOrEqual(entry, iter) if !any { - return index.ErrNotFound + return ErrNotFound } return nil } @@ -142,7 +141,7 @@ func (ii *InsertionIndex) Unmarshal(r io.Reader) error { } d := cbor.NewDecoder(r) for i := int64(0); i < length; i++ { - var rec index.Record + var rec Record if err := d.Decode(&rec); err != nil { return err } @@ -175,7 +174,7 @@ func (ii *InsertionIndex) Codec() multicodec.Code { return insertionIndexCodec } -func (ii *InsertionIndex) Load(rs []index.Record) error { +func (ii *InsertionIndex) Load(rs []Record) error { for _, r := range rs { rec := newRecordDigest(r) if rec.digest == nil { @@ -187,12 +186,12 @@ func (ii *InsertionIndex) Load(rs []index.Record) error { } // flatten returns a formatted index in the given codec for more efficient subsequent loading. -func (ii *InsertionIndex) Flatten(codec multicodec.Code) (index.Index, error) { - si, err := index.New(codec) +func (ii *InsertionIndex) Flatten(codec multicodec.Code) (Index, error) { + si, err := New(codec) if err != nil { return nil, err } - rcrds := make([]index.Record, ii.items.Len()) + rcrds := make([]Record, ii.items.Len()) idx := 0 iter := func(i llrb.Item) bool { diff --git a/v2/internal/store/index.go b/v2/internal/store/index.go index 62c9d2a0..8740603f 100644 --- a/v2/internal/store/index.go +++ b/v2/internal/store/index.go @@ -88,7 +88,7 @@ func FindCid( // Finalize will write the index to the writer at the offset specified in the header. It should only // be used for a CARv2 and when the CAR interface is being closed. -func Finalize(writer io.WriterAt, header carv2.Header, idx *InsertionIndex, dataSize uint64, storeIdentityCIDs bool, indexCodec multicodec.Code) error { +func Finalize(writer io.WriterAt, header carv2.Header, idx *index.InsertionIndex, dataSize uint64, storeIdentityCIDs bool, indexCodec multicodec.Code) error { // TODO check if add index option is set and don't write the index then set index offset to zero. header = header.WithDataSize(dataSize) header.Characteristics.SetFullyIndexed(storeIdentityCIDs) diff --git a/v2/internal/store/indexcheck.go b/v2/internal/store/indexcheck.go index 04b673e0..a11a01aa 100644 --- a/v2/internal/store/indexcheck.go +++ b/v2/internal/store/indexcheck.go @@ -3,6 +3,7 @@ package store import ( "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" ) // ShouldPut returns true if the block should be put into the CAR according to the options provided @@ -10,7 +11,7 @@ import ( // is an identity block and StoreIdentityCIDs is false, or because it already exists and // BlockstoreAllowDuplicatePuts is false. func ShouldPut( - idx *InsertionIndex, + idx *index.InsertionIndex, c cid.Cid, maxIndexCidSize uint64, storeIdentityCIDs bool, @@ -60,7 +61,7 @@ func ShouldPut( // rules associated with the options. Similar to ShouldPut, but for the simpler // Has() case. func Has( - idx *InsertionIndex, + idx *index.InsertionIndex, c cid.Cid, maxIndexCidSize uint64, storeIdentityCIDs bool, diff --git a/v2/internal/store/resume.go b/v2/internal/store/resume.go index ff472237..87ac8fa9 100644 --- a/v2/internal/store/resume.go +++ b/v2/internal/store/resume.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-cid" carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" "github.com/ipld/go-car/v2/internal/carv1" internalio "github.com/ipld/go-car/v2/internal/io" "github.com/multiformats/go-varint" @@ -51,7 +52,7 @@ func Resume( rw ReaderWriterAt, dataReader io.ReaderAt, dataWriter *internalio.OffsetWriteSeeker, - idx *InsertionIndex, + idx *index.InsertionIndex, roots []cid.Cid, dataOffset uint64, v1 bool, From f7dda8827dae29fcc6e74e213286132f877c93b8 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 27 Mar 2023 17:35:37 +0200 Subject: [PATCH 2/2] missing refactor --- v2/storage/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/storage/storage.go b/v2/storage/storage.go index 1a8b499b..50e566f0 100644 --- a/v2/storage/storage.go +++ b/v2/storage/storage.go @@ -93,7 +93,7 @@ func OpenReadable(reader io.ReaderAt, opts ...carv2.Option) (ReadableCar, error) sc.roots = header.Roots sc.reader = reader rr.Seek(0, io.SeekStart) - sc.idx = store.NewInsertionIndex() + sc.idx = index.NewInsertionIndex() if err := carv2.LoadIndex(sc.idx, rr, opts...); err != nil { return nil, err } @@ -120,7 +120,7 @@ func OpenReadable(reader io.ReaderAt, opts ...carv2.Option) (ReadableCar, error) if err != nil { return nil, err } - sc.idx = store.NewInsertionIndex() + sc.idx = index.NewInsertionIndex() if err := carv2.LoadIndex(sc.idx, dr, opts...); err != nil { return nil, err } @@ -169,7 +169,7 @@ func NewWritable(writer io.Writer, roots []cid.Cid, opts ...carv2.Option) (Writa func newWritable(writer io.Writer, roots []cid.Cid, opts ...carv2.Option) (*StorageCar, error) { sc := &StorageCar{ writer: &positionTrackingWriter{w: writer}, - idx: store.NewInsertionIndex(), + idx: index.NewInsertionIndex(), header: carv2.NewHeader(0), opts: carv2.ApplyOptions(opts...), roots: roots, @@ -260,7 +260,7 @@ func OpenReadableWritable(rw ReaderAtWriterAt, roots []cid.Cid, opts ...carv2.Op rw, sc.reader, sc.dataWriter, - sc.idx.(*store.InsertionIndex), + sc.idx.(*index.InsertionIndex), roots, sc.header.DataOffset, sc.opts.WriteAsCarV1, @@ -309,7 +309,7 @@ func (sc *StorageCar) Put(ctx context.Context, keyStr string, data []byte) error return errClosed } - idx, ok := sc.idx.(*store.InsertionIndex) + idx, ok := sc.idx.(*index.InsertionIndex) if !ok || sc.writer == nil { return fmt.Errorf("cannot put into a read-only CAR") } @@ -356,7 +356,7 @@ func (sc *StorageCar) Has(ctx context.Context, keyStr string) (bool, error) { return false, errClosed } - if idx, ok := sc.idx.(*store.InsertionIndex); ok && sc.writer != nil { + if idx, ok := sc.idx.(*index.InsertionIndex); ok && sc.writer != nil { // writable CAR, fast path using InsertionIndex return store.Has( idx, @@ -460,7 +460,7 @@ func (sc *StorageCar) GetStream(ctx context.Context, keyStr string) (io.ReadClos // payload location. This should be called on a writable StorageCar in order to // avoid data loss. func (sc *StorageCar) Finalize() error { - idx, ok := sc.idx.(*store.InsertionIndex) + idx, ok := sc.idx.(*index.InsertionIndex) if !ok || sc.writer == nil { // ignore this, it's not writable return nil