Merge branch 'main' of github.com:filecoin-project/boost into nonsens…

…e/piece-store

README.md

@@ -211,15 +211,15 @@ Note also that the provider address is `t01000` and you will need to supply an a
 1. Build images from the root of the Boost repository
 
 ```
-make docker/all
+make clean docker/all
 ```
 
 On ARM-based systems (*Apple M1/M2*) you need to force building Filecoin's Rust libraries from the source
 ```
-make docker/all ffi_from_source=1 build_lotus=1
+make clean docker/all ffi_from_source=1 build_lotus=1
 ```
 
-If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus).
+If you need to build containers using a specific version of lotus then provide the version as a parameter, e.g. `make clean docker/all lotus_version=v1.20.0-rc2 build_lotus=1`. The version must be a tag or a remote branch name of [Lotus git repo](https://github.com/filecoin-project/lotus).
 
 ### Start devnet docker stack
 
@@ -235,7 +235,7 @@ The initial setup could take up to 20 min or more as it needs to download Fileco
 
 2. Try opening the Boost GUI http://localhost:8080 . Devnet is ready to operate when the URL opens and indicates no errors on the startup page.
 
-You can inspect the status using `docker compose logs -f`.
+You can inspect the status using `cd docker/devnet && docker compose logs -f`.
 
 ### Start monitoring docker stack
 

api/common.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 )
 
 //                       MODIFYING THE API INTERFACE
@@ -38,8 +39,8 @@ type Common interface {
 	//// Version provides information about API provider
 	//Version(context.Context) (APIVersion, error) //perm:read
 
-	//// Discover returns an OpenRPC document describing an RPC API.
-	//Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
+	// Discover returns an OpenRPC document describing an RPC API.
+	Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) //perm:read
 
 	//// trigger graceful shutdown
 	//Shutdown(context.Context) error //perm:admin

api/docgen-openrpc/openrpc.go

@@ -102,7 +102,7 @@ func NewLotusOpenRPCDocument(Comments, GroupDocs map[string]string) *go_openrpc_
 		},
 		GetInfoFn: func() (info *meta_schema.InfoObject) {
 			info = &meta_schema.InfoObject{}
-			title := "Lotus RPC API"
+			title := "Boost RPC API"
 			info.Title = (*meta_schema.InfoObjectProperties)(&title)
 
 			version := build.BuildVersion

api/docgen/docgen.go

@@ -20,7 +20,7 @@ import (
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	graphsync "github.com/ipfs/go-graphsync"
+	"github.com/ipfs/go-graphsync"
 	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/metrics"
@@ -265,7 +265,7 @@ func init() {
 	addExample(apitypes.OpenRPCDocument{
 		"openrpc": "1.2.6",
 		"info": map[string]interface{}{
-			"title":   "Lotus RPC API",
+			"title":   "Boost RPC API",
 			"version": "1.2.1/generated=2020-11-22T08:22:42-06:00",
 		},
 		"methods": []interface{}{}},

build/openrpc.go

@@ -0,0 +1,39 @@
+package build
+
+import (
+	"bytes"
+	"compress/gzip"
+	"embed"
+	"encoding/json"
+	"log"
+
+	lotus_apitypes "github.com/filecoin-project/lotus/api/types"
+)
+
+//go:embed openrpc
+var openrpcfs embed.FS
+
+func mustReadGzippedOpenRPCDocument(data []byte) lotus_apitypes.OpenRPCDocument {
+	zr, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		log.Fatal(err)
+	}
+	m := lotus_apitypes.OpenRPCDocument{}
+	err = json.NewDecoder(zr).Decode(&m)
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = zr.Close()
+	if err != nil {
+		log.Fatal(err)
+	}
+	return m
+}
+
+func OpenRPCDiscoverJSON_Boost() lotus_apitypes.OpenRPCDocument {
+	data, err := openrpcfs.ReadFile("openrpc/boost.json.gz")
+	if err != nil {
+		panic(err)
+	}
+	return mustReadGzippedOpenRPCDocument(data)
+}

build/version.go

@@ -2,7 +2,7 @@ package build
 
 var CurrentCommit string
 
-const BuildVersion = "1.7.2"
+const BuildVersion = "1.7.3-rc1"
 
 func UserVersion() string {
 	return BuildVersion + CurrentCommit

cmd/boost/deal_cmd.go

@@ -44,11 +44,6 @@ var dealFlags = []cli.Flag{
 		Usage:    "size of the CAR file as a padded piece",
 		Required: true,
 	},
-	&cli.Uint64Flag{
-		Name:     "car-size",
-		Usage:    "size of the CAR file",
-		Required: true,
-	},
 	&cli.StringFlag{
 		Name:     "payload-cid",
 		Usage:    "root CID of the CAR file",
@@ -110,6 +105,11 @@ var dealCmd = &cli.Command{
 			Name:  "http-headers",
 			Usage: "http headers to be passed with the request (e.g key=value)",
 		},
+		&cli.Uint64Flag{
+			Name:     "car-size",
+			Usage:    "size of the CAR file: required for online deals",
+			Required: true,
+		},
 	}, dealFlags...),
 	Before: before,
 	Action: func(cctx *cli.Context) error {
@@ -192,15 +192,15 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error {
 		return fmt.Errorf("parsing payload cid %s: %w", payloadCidStr, err)
 	}
 
-	carFileSize := cctx.Uint64("car-size")
-	if carFileSize == 0 {
-		return fmt.Errorf("size of car file cannot be 0")
-	}
-
-	transfer := types.Transfer{
-		Size: carFileSize,
-	}
+	transfer := types.Transfer{}
 	if isOnline {
+
+		carFileSize := cctx.Uint64("car-size")
+		if carFileSize == 0 {
+			return fmt.Errorf("size of car file cannot be 0")
+		}
+
+		transfer.Size = carFileSize
 		// Store the path to the CAR file as a transfer parameter
 		transferParams := &types2.HttpRequest{URL: cctx.String("http-url")}
 

cmd/boostd/legacy_data_transfers.go

@@ -33,8 +33,9 @@ var dataTransfersCmd = &cli.Command{
 }
 
 var marketRestartTransfer = &cli.Command{
-	Name:  "restart",
-	Usage: "Force restart a stalled data transfer",
+	Name:      "restart",
+	Usage:     "Force restart a stalled data transfer",
+	ArgsUsage: "<transfer id>",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "peerid",
@@ -93,8 +94,9 @@ var marketRestartTransfer = &cli.Command{
 }
 
 var marketCancelTransfer = &cli.Command{
-	Name:  "cancel",
-	Usage: "Force cancel a data transfer",
+	Name:      "cancel",
+	Usage:     "Force cancel a data transfer",
+	ArgsUsage: "<transfer id>",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "peerid",
@@ -290,9 +292,10 @@ func toDTv2Logs(logs []*datatransfer.Log) []*datatransferv2.Log {
 }
 
 var transfersDiagnosticsCmd = &cli.Command{
-	Name:  "diagnostics",
-	Usage: "Get detailed diagnostics on active transfers with a specific peer",
-	Flags: []cli.Flag{},
+	Name:      "diagnostics",
+	Usage:     "Get detailed diagnostics on active transfers with a specific peer",
+	ArgsUsage: "<peer id>",
+	Flags:     []cli.Flag{},
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Args().Present() {
 			return cli.ShowCommandHelp(cctx, cctx.Command.Name)

documentation/en/api-v1-methods.md

@@ -1,4 +1,6 @@
 # Groups
+* [](#)
+  * [Discover](#discover)
 * [Actor](#actor)
   * [ActorSectorSize](#actorsectorsize)
 * [Auth](#auth)
@@ -89,6 +91,28 @@
   * [RuntimeSubsystems](#runtimesubsystems)
 * [Sectors](#sectors)
   * [SectorsRefs](#sectorsrefs)
+## 
+
+
+### Discover
+
+
+Perms: read
+
+Inputs: `null`
+
+Response:
+```json
+{
+  "info": {
+    "title": "Boost RPC API",
+    "version": "1.2.1/generated=2020-11-22T08:22:42-06:00"
+  },
+  "methods": [],
+  "openrpc": "1.2.6"
+}
+```
+
 ## Actor
 
 

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/filecoin-project/go-address v1.1.0
 	github.com/filecoin-project/go-bitfield v0.2.4
 	github.com/filecoin-project/go-cbor-util v0.0.1
-	github.com/filecoin-project/go-commp-utils v0.1.3
+	github.com/filecoin-project/go-commp-utils v0.1.4
 	github.com/filecoin-project/go-data-transfer v1.15.4-boost
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0

go.sum

@@ -365,8 +365,9 @@ github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQj
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-cbor-util v0.0.1 h1:E1LYZYTtjfAQwCReho0VXvbu8t3CYAVPiMx8EiV/VAs=
 github.com/filecoin-project/go-cbor-util v0.0.1/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
-github.com/filecoin-project/go-commp-utils v0.1.3 h1:rTxbkNXZU7FLgdkBk8RsQIEOuPONHykEoX3xGk41Fkw=
 github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9ANQrY3fDFoXdqyX04J+dWpK30=
+github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk=
+github.com/filecoin-project/go-commp-utils v0.1.4/go.mod h1:Sekocu5q9b4ECAUFu853GFUbm8I7upAluummHFe2kFo=
 github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 h1:4cITW0pwgvqLs86Q9bWQa34+jBfR1V687bDkmv2DgnA=
 github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837/go.mod h1:e2YBjSblNVoBckkbv3PPqsq71q98oFkFqL7s1etViGo=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=

gql/resolver.go

@@ -265,12 +265,22 @@ func (r *resolver) DealNew(ctx context.Context) (<-chan *dealNewResolver, error)
 }
 
 // mutation: dealCancel(id): ID
-func (r *resolver) DealCancel(_ context.Context, args struct{ ID graphql.ID }) (graphql.ID, error) {
+func (r *resolver) DealCancel(ctx context.Context, args struct{ ID graphql.ID }) (graphql.ID, error) {
 	dealUuid, err := toUuid(args.ID)
 	if err != nil {
 		return args.ID, err
 	}
 
+	deal, err := r.dealsDB.ByID(ctx, dealUuid)
+	if err != nil {
+		return args.ID, err
+	}
+
+	if deal.IsOffline {
+		err = r.provider.CancelOfflineDealAwaitingImport(dealUuid)
+		return args.ID, err
+	}
+
 	err = r.provider.CancelDealDataTransfer(dealUuid)
 	return args.ID, err
 }
@@ -605,8 +615,12 @@ func (dr *dealResolver) sealingState(ctx context.Context) string {
 		log.Warnw("error getting sealing status for sector", "sector", dr.SectorID, "error", err)
 		return "Sealer: Sealing"
 	}
-
-	return "Sealer: " + string(si.State)
+	for _, d := range si.Deals {
+		if d == dr.ProviderDealState.ChainDealID {
+			return "Sealer: " + string(si.State)
+		}
+	}
+	return fmt.Sprintf("Sealer: failed - deal not found in sector %d", si.SectorID)
 }
 
 func (dr *dealResolver) Logs(ctx context.Context) ([]*logsResolver, error) {

gql/resolver_mpool.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+
 	"github.com/filecoin-project/lotus/chain/consensus"
 
 	gqltypes "github.com/filecoin-project/boost/gql/types"
@@ -67,7 +68,7 @@ func (r *resolver) Mpool(ctx context.Context, args struct{ Local bool }) ([]*msg
 		method := m.Message.Method.String()
 		toact, err := r.fullNode.StateGetActor(ctx, m.Message.To, types.EmptyTSK)
 		if err == nil {
-			consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method].Params.Name()
+			method = consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method].Params.Name()
 		}
 
 		var params string

indexprovider/unsealedstatemanager.go

@@ -141,17 +141,12 @@ func (m *UnsealedStateManager) checkForUpdates(ctx context.Context) error {
 					VerifiedDeal:  deal.DealProposal.Proposal.VerifiedDeal,
 				}
 				announceCid, err := m.idxprov.announceBoostDealMetadata(ctx, md, propCid)
-				if err != nil {
-					// Check if the error is because the deal was already advertised
-					if !errors.Is(err, provider.ErrAlreadyAdvertised) {
-						// There was some other error, write it to the log
-						usmlog.Errorf("announcing deal %s to index provider: %w", deal.DealID, err)
-						continue
-					}
-				} else {
+				if err == nil {
 					usmlog.Infow("announced deal seal state to index provider",
 						"deal id", deal.DealID, "sector id", deal.SectorID.Number,
 						"seal state", sectorSealState, "announce cid", announceCid.String())
+				} else {
+					usmlog.Errorf("announcing deal %s to index provider: %w", deal.DealID, err)
 				}
 			}
 		}

indexprovider/wrapper.go

@@ -360,9 +360,13 @@ func (w *Wrapper) announceBoostDealMetadata(ctx context.Context, md metadata.Gra
 	fm := metadata.Default.New(&md)
 	annCid, err := w.prov.NotifyPut(ctx, nil, propCid.Bytes(), fm)
 	if err != nil {
-		return cid.Undef, fmt.Errorf("failed to announce deal to index provider: %w", err)
+		// Check if the error is because the deal was already advertised
+		// (we can safely ignore this error)
+		if !errors.Is(err, provider.ErrAlreadyAdvertised) {
+			return cid.Undef, fmt.Errorf("failed to announce deal to index provider: %w", err)
+		}
 	}
-	return annCid, err
+	return annCid, nil
 }
 
 func (w *Wrapper) AnnounceBoostDealRemoved(ctx context.Context, propCid cid.Cid) (cid.Cid, error) {