use forks of graphsync, go-data-transfer and go-fil-markets (#1333)

* refactor: use forks of graphsync, go-data-transfer and go-fil-markets * refactor: convert from data transfer v1 to v2 voucher type * fix: index provider validation voucher type * fix: pass index provider engine link system through to graphsync's transport configurer * feat: use tagged version of boost-gfm * fix: retrieval client imports * feat: tagged version of lotus * feat: require go 1.19 * lint: fix lint errors * fix: itests * fix: cbor-gen, docsgen * fix: update CI lint version * fix: lint * fix: docgen * fix: go mod tidy * fix: protocol proxy TestOutboundForwarding * fix: docsgen * fix: update filecoin-ffi submodule * fix: prometheus duplicate register panic * fix: cleanup imports * fix: legs voucher processing
filecoin-project · Mar 29, 2023 · c991eeb · c991eeb
1 parent 0b5cf47
commit c991eeb
Show file tree

Hide file tree

Showing 105 changed files with 1,825 additions and 1,278 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,7 +5,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.18.6
+      - image: cimg/go:1.19.7
     resource_class: 2xlarge
   ubuntu:
     docker:
@@ -231,7 +231,7 @@ jobs:
         default: golang
       golangci-lint-version:
         type: string
-        default: 1.46.2
+        default: 1.52.2
       concurrency:
         type: string
         default: '2'

diff --git a/api/api.go b/api/api.go
@@ -3,12 +3,12 @@ package api
 import (
 	"context"
 
+	"github.com/filecoin-project/boost-gfm/piecestore"
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
+	"github.com/filecoin-project/boost-gfm/storagemarket"
 	smtypes "github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/go-address"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -64,8 +64,8 @@ type Boost interface {
 	MarketGetRetrievalAsk(ctx context.Context) (*retrievalmarket.Ask, error)                                                                                                             //perm:read
 	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error //perm:admin
 	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           //perm:read
-	MarketListDataTransfers(ctx context.Context) ([]lapi.DataTransferChannel, error)                                                                                                     //perm:write
-	MarketDataTransferUpdates(ctx context.Context) (<-chan lapi.DataTransferChannel, error)                                                                                              //perm:write
+	MarketListDataTransfers(ctx context.Context) ([]DataTransferChannel, error)                                                                                                          //perm:write
+	MarketDataTransferUpdates(ctx context.Context) (<-chan DataTransferChannel, error)                                                                                                   //perm:write
 	MarketRestartDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                        //perm:write
 	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error                                                         //perm:write
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error                                                                                                        //perm:write

diff --git a/api/api_full.go b/api/api_full.go
@@ -14,8 +14,8 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
+	"github.com/filecoin-project/boost-gfm/storagemarket"
 
 	"github.com/filecoin-project/go-state-types/builtin/v8/paych"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"

diff --git a/api/docgen/docgen.go b/api/docgen/docgen.go
@@ -12,24 +12,23 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/filecoin-project/boost-gfm/filestore"
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
+	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-graphsync"
+	graphsync "github.com/ipfs/go-graphsync"
+	textselector "github.com/ipld/go-ipld-selector-text-lite"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/metrics"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/multiformats/go-multiaddr"
 
-	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/filestore"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-jsonrpc/auth"
-	textselector "github.com/ipld/go-ipld-selector-text-lite"
-
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 	"github.com/filecoin-project/go-state-types/crypto"
@@ -141,8 +140,8 @@ func init() {
 	addExample(map[string]int{"name": 42})
 	addExample(map[string]time.Time{"name": time.Unix(1615243938, 0).UTC()})
 	addExample(&types.ExecutionTrace{
-		Msg:    ExampleValue("init", reflect.TypeOf(&types.Message{}), nil).(*types.Message),
-		MsgRct: ExampleValue("init", reflect.TypeOf(&types.MessageReceipt{}), nil).(*types.MessageReceipt),
+		Msg:    ExampleValue("init", reflect.TypeOf(types.MessageTrace{}), nil).(types.MessageTrace),
+		MsgRct: ExampleValue("init", reflect.TypeOf(types.ReturnTrace{}), nil).(types.ReturnTrace),
 	})
 	addExample(map[string]types.Actor{
 		"t01236": ExampleValue("init", reflect.TypeOf(types.Actor{}), nil).(types.Actor),

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/api/types.go b/api/types.go
@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
 	"github.com/filecoin-project/lotus/chain/types"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer"

diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/cli/node/node.go b/cli/node/node.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -62,7 +61,7 @@ func Setup(cfgdir string) (*Node, error) {
 }
 
 func loadOrInitPeerKey(kf string) (crypto.PrivKey, error) {
-	data, err := ioutil.ReadFile(kf)
+	data, err := os.ReadFile(kf)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return nil, err
@@ -78,7 +77,7 @@ func loadOrInitPeerKey(kf string) (crypto.PrivKey, error) {
 			return nil, err
 		}
 
-		if err := ioutil.WriteFile(kf, data, 0600); err != nil {
+		if err := os.WriteFile(kf, data, 0600); err != nil {
 			return nil, err
 		}
 

diff --git a/cmd/boost/main.go b/cmd/boost/main.go
@@ -3,7 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 
 	"github.com/filecoin-project/boost/cmd"
@@ -19,7 +19,7 @@ import (
 var log = logging.Logger("boost")
 
 func init() {
-	llog.SetOutput(ioutil.Discard)
+	llog.SetOutput(io.Discard)
 }
 
 func main() {

diff --git a/cmd/boost/provider_cmd.go b/cmd/boost/provider_cmd.go
@@ -5,14 +5,14 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/filecoin-project/boost-gfm/retrievalmarket"
+	"github.com/filecoin-project/boost-gfm/storagemarket/network"
 	bcli "github.com/filecoin-project/boost/cli"
 	clinode "github.com/filecoin-project/boost/cli/node"
 	"github.com/filecoin-project/boost/cmd"
 	"github.com/filecoin-project/boost/retrievalmarket/lp2pimpl"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/storagemarket/network"
 	// TODO: This multiaddr util library should probably live in its own repo
 	multiaddrutil "github.com/filecoin-project/go-legs/httpsync/multiaddr"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -84,7 +84,11 @@ var libp2pInfoCmd = &cli.Command{
 		if err != nil {
 			return fmt.Errorf("getting protocols for peer %s: %w", addrInfo.ID, err)
 		}
-		sort.Strings(protos)
+		protostrs := make([]string, 0, len(protos))
+		for _, proto := range protos {
+			protostrs = append(protostrs, string(proto))
+		}
+		sort.Strings(protostrs)
 
 		agentVersionI, err := n.Host.Peerstore().Get(addrInfo.ID, "AgentVersion")
 		if err != nil {
@@ -98,7 +102,7 @@ var libp2pInfoCmd = &cli.Command{
 				"agent":      agentVersion,
 				"id":         addrInfo.ID.String(),
 				"multiaddrs": addrInfo.Addrs,
-				"protocols":  protos,
+				"protocols":  protostrs,
 			})
 		}
 
@@ -109,7 +113,7 @@ var libp2pInfoCmd = &cli.Command{
 		for _, addr := range addrInfo.Addrs {
 			fmt.Println("  " + addr.String())
 		}
-		fmt.Println("Protocols:\n" + "  " + strings.Join(protos, "\n  "))
+		fmt.Println("Protocols:\n" + "  " + strings.Join(protostrs, "\n  "))
 		return nil
 	},
 }

diff --git a/cmd/boost/retrieve_cmd.go b/cmd/boost/retrieve_cmd.go
@@ -24,7 +24,7 @@ import (
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
-	files "github.com/ipfs/go-ipfs-files"
+	"github.com/ipfs/go-libipfs/files"
 	"github.com/ipfs/go-merkledag"
 	unixfile "github.com/ipfs/go-unixfs/file"
 	"github.com/ipld/go-car"

diff --git a/cmd/boost/wallet_cmd.go b/cmd/boost/wallet_cmd.go
@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -395,7 +394,7 @@ var walletImport = &cli.Command{
 			inpdata = indata
 
 		} else {
-			fdata, err := ioutil.ReadFile(cctx.Args().First())
+			fdata, err := os.ReadFile(cctx.Args().First())
 			if err != nil {
 				return err
 			}

diff --git a/cmd/boostci/main.go b/cmd/boostci/main.go
@@ -1,7 +1,7 @@
 package main
 
 import (
-	"io/ioutil"
+	"io"
 	"log"
 	"os"
 
@@ -11,7 +11,7 @@ import (
 )
 
 func init() {
-	llog.SetOutput(ioutil.Discard)
+	llog.SetOutput(io.Discard)
 }
 
 func main() {

diff --git a/cmd/boostd/dummydeal.go b/cmd/boostd/dummydeal.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
@@ -239,12 +238,12 @@ func serveCarFile(dealUuid uuid.UUID, fpath string) (string, error) {
 	carName := dealUuid.String() + ".car"
 	destPath := path.Join(gql.DummyDealsDir, carName)
 
-	bytes, err := ioutil.ReadFile(fpath)
+	bytes, err := os.ReadFile(fpath)
 	if err != nil {
 		return "", fmt.Errorf("reading source car file: %w", err)
 	}
 
-	err = ioutil.WriteFile(destPath, bytes, 0644)
+	err = os.WriteFile(destPath, bytes, 0644)
 	if err != nil {
 		return "", fmt.Errorf("writing destination car file: %w", err)
 	}