diff --git a/go.mod b/go.mod index e25cbbd09..f569c06fd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/filecoin-project/boostd-data => ./extern/boostd-data -// replace github.com/filcoin-project/boost-graphsync => ../boost-graphsync +// replace github.com/filecoin-project/boost-graphsync => ../boost-graphsync // replace github.com/filecoin-project/go-data-transfer => ../go-data-transfer @@ -334,7 +334,7 @@ require ( require ( github.com/filecoin-project/boost-gfm v1.26.7 - github.com/filecoin-project/boost-graphsync v0.13.8 + github.com/filecoin-project/boost-graphsync v0.13.9-0.20230920054848-cd7a9307c3f1 github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 github.com/filecoin-project/go-fil-markets v1.28.3 github.com/filecoin-project/lotus v1.23.2-0.20230622154405-168d022018ce diff --git a/go.sum b/go.sum index 7478462ea..84226b51f 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/filecoin-project/boost-gfm v1.26.7 h1:ENJEqx1OzY072QnUP37YrGVmUiCewRwHAjbtTxyW74Y= github.com/filecoin-project/boost-gfm v1.26.7/go.mod h1:OhG2y7WeDx3KU9DPjgWllS+3/ospPjm8/XDrvN6uOfk= -github.com/filecoin-project/boost-graphsync v0.13.8 h1:QqH15/qe4IUnQj5pc1WIuBDe2gq0kzWQ6l4dUirbmfs= -github.com/filecoin-project/boost-graphsync v0.13.8/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk= +github.com/filecoin-project/boost-graphsync v0.13.9-0.20230920054848-cd7a9307c3f1 h1:AMWmZa0tabhgJGXeBz3AY1YpaK6/OlZpEDlYZskoJ3s= +github.com/filecoin-project/boost-graphsync v0.13.9-0.20230920054848-cd7a9307c3f1/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk= github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0= github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= diff --git a/itests/markets_v1_identity_cid_test.go b/itests/markets_v1_identity_cid_test.go new file mode 100644 index 000000000..d1f0376e7 --- /dev/null +++ b/itests/markets_v1_identity_cid_test.go @@ -0,0 +1,141 @@ +package itests + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + + gstestutil "github.com/filecoin-project/boost-graphsync/testutil" + "github.com/filecoin-project/boost/itests/framework" + "github.com/filecoin-project/boost/testutil" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/fluent/qp" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + multihash "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +func TestMarketsV1DealAndRetrievalWithIdentityCID(t *testing.T) { + req := require.New(t) + ctx := context.Background() + log := framework.Log + + kit.QuietMiningLogs() + framework.SetLogLevel() + var opts []framework.FrameworkOpts + opts = append(opts, framework.EnableLegacyDeals(true)) + f := framework.NewTestFramework(ctx, t, opts...) + err := f.Start() + req.NoError(err) + defer f.Stop() + + // Create a CAR file + carPath := f.HomeDir + "/testfile.car" + log.Debugf("using test car %s", carPath) + carFile, err := os.Create(carPath) + req.NoError(err) + + carStore, err := storage.NewReadableWritable(carFile, []cid.Cid{cid.MustParse("bafyreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, car.WriteAsCarV1(true)) + req.NoError(err) + + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(carStore) + lsys.SetReadStorage(carStore) + + // Setup our DAG with an identity CID in the middle + testDag := gstestutil.SetupIdentityDAG(ctx, t, lsys) + expectedCids := make([]cid.Cid, 0) + + // Fill out the deal with a bunch of random data so we're not trying to do a + // tiny storage deal. + rseed := 1 + rnd := rand.New(rand.NewSource(int64(rseed))) + n, err := qp.BuildList(basicnode.Prototype.List, 8, func(la ipld.ListAssembler) { + for ii := 0; ii < 7; ii++ { + rndBytes := make([]byte, 1<<20) + _, err := rnd.Read(rndBytes) + req.NoError(err) + l, err := lsys.Store(ipld.LinkContext{}, rawlp, basicnode.NewBytes(rndBytes)) + req.NoError(err) + qp.ListEntry(la, qp.Link(l)) + expectedCids = append(expectedCids, l.(cidlink.Link).Cid) + } + qp.ListEntry(la, qp.Link(testDag.RootLink)) + }) + req.NoError(err) + expectedCids = append(expectedCids, toCids(testDag.AllLinksNoIdent)...) // all blocks BUT the identity block + rootLink, err := lsys.Store(ipld.LinkContext{}, dclp, n) + req.NoError(err) + expectedCids = append([]cid.Cid{rootLink.(cidlink.Link).Cid}, expectedCids...) + + req.NoError(carFile.Close()) + req.NoError(car.ReplaceRootsInFile(carPath, []cid.Cid{rootLink.(cidlink.Link).Cid})) + log.Debugw("filled car, replaced root with correct root", "root", rootLink.String()) + + // Import and make a deal to store + + res, err := f.FullNode.ClientImport(ctx, lapi.FileRef{Path: carPath, IsCAR: true}) + req.NoError(err) + + log.Debugw("imported data for deal") + + dp := f.DefaultMarketsV1DealParams() + dp.Data.Root = res.Root + + log.Debugw("starting deal", "root", res.Root) + dealProposalCid, err := f.FullNode.ClientStartDeal(ctx, &dp) + req.NoError(err) + + log.Debugw("got deal proposal cid", "cid", dealProposalCid) + + err = f.WaitDealSealed(ctx, dealProposalCid) + req.NoError(err) + + // Deal is stored and sealed, attempt different retrieval forms + + log.Debugw("deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root) + outPath := f.Retrieve(ctx, t, dealProposalCid, rootLink.(cidlink.Link).Cid, false, selectorparse.CommonSelector_ExploreAllRecursively) + + // Inspect what we got + gotCids, err := testutil.CidsInCar(outPath) + req.NoError(err) + req.Equal(toStr(expectedCids), toStr(gotCids)) + log.Debugw("successfully retrieved %d blocks, not including identity block", len(expectedCids)) +} + +func toCids(links []datamodel.Link) []cid.Cid { + var cids []cid.Cid + for _, link := range links { + cids = append(cids, link.(cidlink.Link).Cid) + fmt.Println("link.(cidlink.Link).Cid", link.(cidlink.Link).Cid) + } + return cids +} + +var rawlp = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: cid.Raw, + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} + +var dclp = cidlink.LinkPrototype{ + Prefix: cid.Prefix{ + Version: 1, + Codec: cid.DagCBOR, + MhType: multihash.SHA2_256, + MhLength: 32, + }, +} diff --git a/itests/markets_v1_offline_deal_test.go b/itests/markets_v1_offline_deal_test.go index 32859a223..45c21cbf2 100644 --- a/itests/markets_v1_offline_deal_test.go +++ b/itests/markets_v1_offline_deal_test.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/itests/framework" "github.com/filecoin-project/boost/testutil" - "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/itests/kit" "github.com/stretchr/testify/require" @@ -69,7 +68,7 @@ func TestMarketsV1OfflineDeal(t *testing.T) { // Create a CAR file from the raw file log.Debugw("generate out.car for miner") carFilePath := filepath.Join(f.HomeDir, "out.car") - err = f.FullNode.ClientGenCar(ctx, api.FileRef{Path: inPath}, carFilePath) + err = f.FullNode.ClientGenCar(ctx, lapi.FileRef{Path: inPath}, carFilePath) require.NoError(t, err) // Import the CAR file on the miner - this is the equivalent to diff --git a/retrievalmarket/server/gsunpaidretrieval_test.go b/retrievalmarket/server/gsunpaidretrieval_test.go index 42b5acfb2..ca92f338c 100644 --- a/retrievalmarket/server/gsunpaidretrieval_test.go +++ b/retrievalmarket/server/gsunpaidretrieval_test.go @@ -62,8 +62,7 @@ func TestGS(t *testing.T) { _ = logging.SetLogLevel("testgs", "info") //_ = logging.SetLogLevel("dt-impl", "debug") - missingCid, err := cid.Parse("bafkqaaa") - require.NoError(t, err) + missingCid := cid.MustParse("baguqeeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") testCases := []testCase{{ name: "happy path",