diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 14ea24db19af..ec8c2297d990 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -179,6 +179,7 @@ var ( utils.TxGossipBroadcastDisabledFlag, utils.TxGossipReceivingDisabledFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABlobScanAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index fb5b25343d1f..7e1be6777fe0 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -237,6 +237,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.L1DisableMessageQueueV2Flag, utils.RollupVerifyEnabledFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlobScanAPIEndpointFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 25c6d568cd5b..a9bd855809b1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -908,6 +908,12 @@ var ( Name: "da.sync", Usage: "Enable node syncing from DA", } + DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ + Name: "da.missingheaderfields.baseurl", + Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks", + Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/", + } + DABlobScanAPIEndpointFlag = cli.StringFlag{ Name: "da.blob.blobscan", Usage: "BlobScan blob API endpoint", @@ -1396,6 +1402,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index 57a803198fa1..314368ee049d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1880,15 +1880,18 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() + // sanitize base fee + // Note: setting the base fee to 0 will cause problems as nil != 0 when serializing the header and thus block hash will be different. + if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { + header.BaseFee = nil + } + tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil) receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig) if err != nil { return nil, NonStatTy, fmt.Errorf("error processing block: %w", err) } - // TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique - // This should be done with https://github.com/scroll-tech/go-ethereum/pull/913. - if sign { // Prevent Engine from overriding timestamp. originalTime := header.Time @@ -1901,7 +1904,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types // finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble() header.GasUsed = gasUsed - header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + + // state root might be set from partial header. If it is not set, we calculate it. + if header.Root == (common.Hash{}) { + header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + } fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/eth/backend.go b/eth/backend.go index 0050d3853b65..99c8a1b5d5b6 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,9 @@ import ( "errors" "fmt" "math/big" + "net/url" + "path" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -61,6 +64,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/ccc" "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service" "github.com/scroll-tech/go-ethereum/rollup/sync_service" "github.com/scroll-tech/go-ethereum/rpc" @@ -241,7 +245,12 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if config.EnableDASyncing { // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { - eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA) + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) } @@ -339,6 +348,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether return eth, nil } +func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) { + downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) + } + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin") + + expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 + if expectedSHA256Checksum == nil { + return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") + } + + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) + return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/node/config.go b/node/config.go index 6e7133a95129..d7212389684e 100644 --- a/node/config.go +++ b/node/config.go @@ -201,6 +201,8 @@ type Config struct { L1DisableMessageQueueV2 bool `toml:",omitempty"` // Is daSyncingEnabled DaSyncingEnabled bool `toml:",omitempty"` + // Base URL for missing header fields file + DAMissingHeaderFieldsBaseURL string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/params/config.go b/params/config.go index 8d6d50ed5045..f41eb218adcf 100644 --- a/params/config.go +++ b/params/config.go @@ -30,16 +30,18 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") - ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") - ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") - ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") - ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") + ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") + ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") + ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") + ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xfa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xa02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924") ) func newUint64(val uint64) *uint64 { return &val } @@ -354,7 +356,8 @@ var ( ScrollChainAddress: common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"), L2SystemConfigAddress: common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"), }, - GenesisStateRoot: &ScrollSepoliaGenesisState, + GenesisStateRoot: &ScrollSepoliaGenesisState, + MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256, }, } @@ -406,7 +409,8 @@ var ( ScrollChainAddress: common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"), L2SystemConfigAddress: common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"), }, - GenesisStateRoot: &ScrollMainnetGenesisState, + GenesisStateRoot: &ScrollMainnetGenesisState, + MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256, }, } @@ -710,6 +714,9 @@ type ScrollConfig struct { // Genesis State Root for MPT clients GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"` + + // MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file. + MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"` } // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer @@ -760,8 +767,13 @@ func (s ScrollConfig) String() string { genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot) } - return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}", - s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot) + missingHeaderFieldsSHA256 := "" + if s.MissingHeaderFieldsSHA256 != nil { + missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256) + } + + return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}", + s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256) } // IsValidTxCount returns whether the given block's transaction count is below the limit. diff --git a/params/version.go b/params/version.go index 523d5ca47c51..555a1e557b27 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 5 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 65 // Patch version component of the current release + VersionPatch = 66 // Patch version component of the current release VersionMeta = "mainnet" // Version metadata to append to the version string ) diff --git a/rollup/da_syncer/block_queue.go b/rollup/da_syncer/block_queue.go index 630382f001c0..30d53bfcb6a5 100644 --- a/rollup/da_syncer/block_queue.go +++ b/rollup/da_syncer/block_queue.go @@ -6,19 +6,22 @@ import ( "github.com/scroll-tech/go-ethereum/core/rawdb" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and // provides them to the next stage one-by-one. type BlockQueue struct { - batchQueue *BatchQueue - blocks []*da.PartialBlock + batchQueue *BatchQueue + blocks []*da.PartialBlock + missingHeaderFieldsManager *missing_header_fields.Manager } -func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue { +func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue { return &BlockQueue{ - batchQueue: batchQueue, - blocks: make([]*da.PartialBlock, 0), + batchQueue: batchQueue, + blocks: make([]*da.PartialBlock, 0), + missingHeaderFieldsManager: missingHeaderFieldsManager, } } @@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error { return err } - bq.blocks, err = entryWithBlocks.Blocks() + bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager) if err != nil { return fmt.Errorf("failed to get blocks from entry: %w", err) } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index c99a474b256b..ca7322264a43 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type CommitBatchDAV0 struct { @@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int { return 0 } -func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) { l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped) if err != nil { return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err) @@ -120,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex := c.parentTotalL1MessagePopped for _, chunk := range c.chunks { - for blockId, daBlock := range chunk.Blocks { + for blockIndex, daBlock := range chunk.Blocks { // create txs txs := make(types.Transactions, 0, daBlock.NumTransactions()) // insert l1 msgs @@ -132,7 +133,12 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex += uint64(daBlock.NumL1Messages()) // insert l2 txs - txs = append(txs, chunk.Transactions[blockId]...) + txs = append(txs, chunk.Transactions[blockIndex]...) + + difficulty, stateRoot, coinbase, nonce, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + if err != nil { + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) + } block := NewPartialBlock( &PartialHeader{ @@ -140,8 +146,11 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { Time: daBlock.Timestamp(), BaseFee: daBlock.BaseFee(), GasLimit: daBlock.GasLimit(), - Difficulty: 10, // TODO: replace with real difficulty - ExtraData: []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data + Difficulty: difficulty, + ExtraData: extraData, + StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/commitV7.go b/rollup/da_syncer/da/commitV7.go index 6048b853aee7..37a4b948df6c 100644 --- a/rollup/da_syncer/da/commitV7.go +++ b/rollup/da_syncer/da/commitV7.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" @@ -113,7 +114,7 @@ func (c *CommitBatchDAV7) Event() l1.RollupEvent { return c.event } -func (c *CommitBatchDAV7) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV7) Blocks(_ *missing_header_fields.Manager) ([]*PartialBlock, error) { initialL1MessageIndex := c.parentTotalL1MessagePopped l1Txs, err := getL1MessagesV7(c.db, c.blobPayload.Blocks(), initialL1MessageIndex) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index fe72473451b6..f80ee631b51d 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -8,6 +8,7 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type Type int @@ -34,7 +35,7 @@ type Entry interface { type EntryWithBlocks interface { Entry - Blocks() ([]*PartialBlock, error) + Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) Version() encoding.CodecVersion Chunks() []*encoding.DAChunkRawTx BlobVersionedHashes() []common.Hash @@ -53,6 +54,9 @@ type PartialHeader struct { GasLimit uint64 Difficulty uint64 ExtraData []byte + StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce } func (h *PartialHeader) ToHeader() *types.Header { @@ -63,6 +67,9 @@ func (h *PartialHeader) ToHeader() *types.Header { GasLimit: h.GasLimit, Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, + Root: h.StateRoot, + Coinbase: h.Coinbase, + Nonce: h.Nonce, } } diff --git a/rollup/da_syncer/syncing_pipeline.go b/rollup/da_syncer/syncing_pipeline.go index e0ec4e4fd3c6..0490529a1aa4 100644 --- a/rollup/da_syncer/syncing_pipeline.go +++ b/rollup/da_syncer/syncing_pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // Config is the configuration parameters of data availability syncing. @@ -51,7 +52,7 @@ type SyncingPipeline struct { daQueue *DAQueue } -func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) { +func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config, missingHeaderFieldsManager *missing_header_fields.Manager) (*SyncingPipeline, error) { l1Reader, err := l1.NewReader(ctx, l1.Config{ ScrollChainAddress: genesisConfig.Scroll.L1Config.ScrollChainAddress, L1MessageQueueAddress: genesisConfig.Scroll.L1Config.L1MessageQueueAddress, @@ -128,7 +129,7 @@ func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesi daQueue := NewDAQueue(lastProcessedBatchMeta.L1BlockNumber, dataSourceFactory) batchQueue := NewBatchQueue(daQueue, db, lastProcessedBatchMeta) - blockQueue := NewBlockQueue(batchQueue) + blockQueue := NewBlockQueue(batchQueue, missingHeaderFieldsManager) daSyncer := NewDASyncer(blockchain, config.L2EndBlock) ctx, cancel := context.WithCancel(ctx) diff --git a/rollup/missing_header_fields/export-headers-toolkit/.gitignore b/rollup/missing_header_fields/export-headers-toolkit/.gitignore new file mode 100644 index 000000000000..adbb97d2d313 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/.gitignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/rollup/missing_header_fields/export-headers-toolkit/Dockerfile b/rollup/missing_header_fields/export-headers-toolkit/Dockerfile new file mode 100644 index 000000000000..04f729a05ef7 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.22 + +WORKDIR /app + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . . + +RUN go build -o main . + +ENTRYPOINT ["./main"] \ No newline at end of file diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md new file mode 100644 index 000000000000..e8c7a2abcc8b --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -0,0 +1,73 @@ +# Export missing block header fields toolkit + +A toolkit for exporting and transforming missing block header fields of Scroll before EuclidV2 upgrade. + +## Context +We are using the [Clique consensus](https://eips.ethereum.org/EIPS/eip-225) in Scroll L2. Amongst others, it requires the following header fields: +- `extraData` +- `difficulty` +- `coinbase` +- `nonce` + +However, before EuclidV2, these fields were not stored on L1/DA. +In order for nodes to be able to reconstruct the correct block hashes when only reading data from L1, +we need to provide the historical values of these fields to these nodes through a separate file. +Additionally, the `stateRoot` field is included in the file to ensure that the block headers can be reconstructed correctly, +independently of the state trie type used in the node (before EuclidV1 the state trie was ZK trie, after EuclidV1 it is a regular Merkle Patricia Trie). + +This toolkit provides commands to export the missing fields, deduplicate the data and create a file +with the missing fields that can be used to reconstruct the correct block hashes when only reading data from L1. + +The toolkit provides the following commands: +- `fetch` - Fetch missing block header fields from a running Scroll L2 node and store in a file +- `dedup` - Deduplicate the headers file, print unique values and create a new file with the deduplicated headers + +## Binary layout deduplicated missing header fields file +The deduplicated header file binary layout is as follows: + +```plaintext +...... + +Where: +- unique_vanity_count: number of unique vanities n +- unique_vanity_i: unique vanity i +- header_i: block header i +- header: + [][] + - flags: bitmask, lsb first + - bit 4: 1 if the header has a coinbase field + - bit 5: 1 if the header has a nonce field + - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 7: 0 if seal length is 65, 1 if seal length is 85 + - vanity_index: index of the vanity in the sorted vanities list (0-255) + - state_root: 32 bytes of state root data + - coinbase: 20 bytes of coinbase address (if present) + - nonce: 8 bytes of nonce (if present) + - seal: 65 or 85 bytes of seal data +``` + +## How to run +Each of the commands has its own set of flags and options. To display the help message run with `--help` flag. + +1. Fetch the missing block header fields from a running Scroll L2 node via RPC and store in a file (approx 40min for 5.5M blocks). +2. Deduplicate the headers file, print unique values and create a new file with the deduplicated headers + +```bash +go run main.go fetch --rpc=http://localhost:8545 --start=0 --end=100 --batch=10 --parallelism=10 --output=headers.bin --humanOutput=true +go run main.go dedup --input=headers.bin --output=headers-dedup.bin +``` + + +### With Docker +To run the toolkit with Docker, build the Docker image and run the commands inside the container. + +```bash +docker build -t export-headers-toolkit . + +# depending on the Docker config maybe finding the RPC container's IP with docker inspect is necessary. Potentially host IP works: http://172.17.0.1:8545 +docker run --rm -v "$(pwd)":/app/result export-headers-toolkit fetch --rpc=
--start=0 --end=5422047 --batch=10000 --parallelism=10 --output=/app/result/headers.bin --humanOutput=/app/result/headers.csv +docker run --rm -v "$(pwd)":/app/result export-headers-toolkit dedup --input=/app/result/headers.bin --output=/app/result/headers-dedup.bin +``` + + + diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go new file mode 100644 index 000000000000..69126a860a4c --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -0,0 +1,331 @@ +package cmd + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "fmt" + "io" + "log" + "os" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/scroll-tech/go-ethereum/common" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" +) + +// dedupCmd represents the dedup command +var dedupCmd = &cobra.Command{ + Use: "dedup", + Short: "Deduplicate the headers file, print unique values and create a new file with the deduplicated headers", + Long: `Deduplicate the headers file, print unique values and create a new file with the deduplicated headers. + +The binary layout of the deduplicated file is as follows: +- 1 byte for the count of unique vanity +- 32 bytes for each unique vanity +- for each header: + - 1 byte (bitmask, lsb first): + - bit 0-5: index of the vanity in the sorted vanities list + - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 7: 0 if seal length is 65, 1 if seal length is 85 + - 65 or 85 bytes for the seal`, + Run: func(cmd *cobra.Command, args []string) { + inputFile, err := cmd.Flags().GetString("input") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + outputFile, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + verifyFile, err := cmd.Flags().GetString("verify") + if err != nil { + log.Fatalf("Error reading verify flag: %v", err) + } + + // uncomment the following line to copy from the verify file to the input file. This is useful to generate a deduplicated header file for testing purposes. + // copyFromVerifyFile(verifyFile, inputFile) + + if verifyFile != "" { + verifyInputFile(verifyFile, inputFile) + } + + _, seenVanity, _ := runAnalysis(inputFile) + runDedup(inputFile, outputFile, seenVanity) + + if verifyFile != "" { + verifyOutputFile(verifyFile, outputFile) + } + + runSHA256(outputFile) + }, +} + +func init() { + rootCmd.AddCommand(dedupCmd) + + dedupCmd.Flags().String("input", "headers.bin", "headers file") + dedupCmd.Flags().String("output", "headers-dedup.bin", "deduplicated, binary formatted file") + dedupCmd.Flags().String("verify", "", "verify the input and output files with the given .csv file") +} + +func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) { + reader := newHeaderReader(inputFile) + defer reader.close() + + // track header fields we've seen + seenDifficulty = make(map[uint64]int) + seenVanity = make(map[[32]byte]bool) + seenSealLen = make(map[int]int) + + reader.read(func(header *types.Header) { + seenDifficulty[header.Difficulty]++ + seenVanity[header.Vanity()] = true + seenSealLen[header.SealLen()]++ + }) + + // Print distinct values and report + fmt.Println("--------------------------------------------------") + for diff, count := range seenDifficulty { + fmt.Printf("Difficulty %d: %d\n", diff, count) + } + + for vanity := range seenVanity { + fmt.Printf("Vanity: %x\n", vanity) + } + + for sealLen, count := range seenSealLen { + fmt.Printf("SealLen %d bytes: %d\n", sealLen, count) + } + + fmt.Println("--------------------------------------------------") + fmt.Printf("Unique values seen in the headers file (last seen block: %d):\n", reader.lastHeader.Number) + fmt.Printf("Distinct count: Difficulty:%d, Vanity:%d, SealLen:%d\n", len(seenDifficulty), len(seenVanity), len(seenSealLen)) + fmt.Printf("--------------------------------------------------\n\n") + + return seenDifficulty, seenVanity, seenSealLen +} + +func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) { + reader := newHeaderReader(inputFile) + defer reader.close() + + writer := newMissingHeaderFileWriter(outputFile, seenVanity) + defer writer.close() + + writer.missingHeaderWriter.writeVanities() + + reader.read(func(header *types.Header) { + writer.missingHeaderWriter.write(header) + }) +} + +func runSHA256(outputFile string) { + f, err := os.Open(outputFile) + if err != nil { + log.Fatalf("Error opening file: %v", err) + } + defer f.Close() + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + log.Fatalf("Error hashing file: %v", err) + } + + fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil)) +} + +type headerReader struct { + file *os.File + reader *bufio.Reader + lastHeader *types.Header +} + +func newHeaderReader(inputFile string) *headerReader { + f, err := os.Open(inputFile) + if err != nil { + log.Fatalf("Error opening input file: %v", err) + } + + h := &headerReader{ + file: f, + reader: bufio.NewReader(f), + } + + return h +} + +func (h *headerReader) read(callback func(header *types.Header)) { + headerSizeBytes := make([]byte, types.HeaderSizeSerialized) + + for { + _, err := io.ReadFull(h.reader, headerSizeBytes) + if err != nil { + if err == io.EOF { + break + } + log.Printf("Error reading headerSizeBytes: %v\n", err) + } + headerSize := binary.BigEndian.Uint16(headerSizeBytes) + + headerBytes := make([]byte, headerSize) + _, err = io.ReadFull(h.reader, headerBytes) + if err != nil { + if err == io.EOF { + break + } + log.Printf("Error reading headerBytes: %v\n", err) + } + header := new(types.Header).FromBytes(headerBytes) + + // sanity check: make sure headers are in order + if h.lastHeader != nil && header.Number != h.lastHeader.Number+1 { + fmt.Println("lastHeader:", h.lastHeader.String()) + log.Fatalf("Missing block: %d, got %d instead", h.lastHeader.Number+1, header.Number) + } + h.lastHeader = header + + callback(header) + } +} + +func (h *headerReader) close() { + h.file.Close() +} + +type csvHeaderReader struct { + file *os.File + reader *bufio.Reader +} + +func newCSVHeaderReader(verifyFile string) *csvHeaderReader { + f, err := os.Open(verifyFile) + if err != nil { + log.Fatalf("Error opening verify file: %v", err) + } + + h := &csvHeaderReader{ + file: f, + reader: bufio.NewReader(f), + } + + return h +} + +func (h *csvHeaderReader) readNext() *types.Header { + line, err := h.reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + log.Fatalf("Error reading line: %v", err) + } + + s := strings.Split(strings.TrimSpace(line), ",") + if len(s) != 6 { + log.Fatalf("Malformed CSV line: %q", line) + } + + num, err := strconv.ParseUint(s[0], 10, 64) + if err != nil { + log.Fatalf("Error parsing block number: %v", err) + } + difficulty, err := strconv.ParseUint(s[1], 10, 64) + if err != nil { + log.Fatalf("Error parsing difficulty: %v", err) + } + + stateRoot := common.HexToHash(s[2]) + coinbase := common.HexToAddress(s[3]) + nonceBytes := common.Hex2Bytes(s[4]) + extra := common.FromHex(strings.Split(s[5], "\n")[0]) + + header := types.NewHeader(num, difficulty, stateRoot, coinbase, coreTypes.BlockNonce(nonceBytes), extra) + return header +} + +func (h *csvHeaderReader) close() { + h.file.Close() +} + +func copyFromVerifyFile(verifyFile, inputFile string) { + fmt.Println("Copying from", verifyFile, "to", inputFile) + + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + writer := newFilesWriter(inputFile, "") + defer writer.close() + + for header := csvReader.readNext(); header != nil; header = csvReader.readNext() { + writer.write(header) + } +} + +func verifyInputFile(verifyFile, inputFile string) { + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + binaryReader := newHeaderReader(inputFile) + defer binaryReader.close() + + binaryReader.read(func(header *types.Header) { + csvHeader := csvReader.readNext() + + if !csvHeader.Equal(header) { + log.Fatalf("Header mismatch: %v != %v", csvHeader, header) + } + }) + + log.Printf("All headers match in %s and %s\n", verifyFile, inputFile) +} + +func verifyOutputFile(verifyFile, outputFile string) { + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + dedupReader, err := missing_header_fields.NewReader(outputFile) + if err != nil { + log.Fatalf("Error opening dedup file: %v", err) + } + defer dedupReader.Close() + + for { + header := csvReader.readNext() + if header == nil { + if err = dedupReader.ReadNext(); err == nil { + log.Fatalf("Expected EOF, got more headers") + } + break + } + + difficulty, stateRoot, coinbase, nonce, extraData, err := dedupReader.Read(header.Number) + if err != nil { + log.Fatalf("Error reading header: %v", err) + } + + if header.Difficulty != difficulty { + log.Fatalf("Difficulty mismatch: headerNum %d: %d != %d", header.Number, header.Difficulty, difficulty) + } + if header.StateRoot != stateRoot { + log.Fatalf("StateRoot mismatch: headerNum %d: %s != %s", header.Number, header.StateRoot, stateRoot) + } + if header.Coinbase != coinbase { + log.Fatalf("Coinbase mismatch: headerNum %d: %s != %s", header.Number, header.Coinbase.Hex(), coinbase.Hex()) + } + if header.Nonce != nonce { + log.Fatalf("Nonce mismatch: headerNum %d: %s != %s", header.Number, common.Bytes2Hex(header.Nonce[:]), common.Bytes2Hex(nonce[:])) + } + if !bytes.Equal(header.ExtraData, extraData) { + log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData) + } + } + + log.Printf("All headers match in %s and %s\n", verifyFile, outputFile) +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go new file mode 100644 index 000000000000..c4d216c1d7cf --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -0,0 +1,366 @@ +package cmd + +import ( + "bufio" + "container/heap" + "context" + "fmt" + "log" + "math/big" + "math/rand" + "os" + "strings" + "sync" + "time" + + "github.com/spf13/cobra" + "gorm.io/driver/postgres" + "gorm.io/gorm" + + "github.com/scroll-tech/go-ethereum/ethclient" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +var fetchCmd = &cobra.Command{ + Use: "fetch", + Short: "Fetch missing block header fields from a running Scroll L2 node via RPC and store in a file", + Long: `Fetch allows to retrieve the missing block header fields from a running Scroll L2 node via RPC. +It produces a binary file and optionally a human readable csv file with the missing fields.`, + Run: func(cmd *cobra.Command, args []string) { + rpcs, err := cmd.Flags().GetString("rpc") + if err != nil { + log.Fatalf("Error reading rpc flag: %v", err) + } + if rpcs == "" { + log.Fatal("No RPC URLs provided, please use the --rpc flag to specify at least one RPC URL.") + } + rpcNodes := strings.Split(rpcs, ",") + var clients []*ethclient.Client + for _, rpc := range rpcNodes { + client, err := ethclient.Dial(rpc) + if err != nil { + log.Fatalf("Error connecting to RPC: %v", err) + } + clients = append(clients, client) + } + startBlockNum, err := cmd.Flags().GetUint64("start") + if err != nil { + log.Fatalf("Error reading start flag: %v", err) + } + endBlockNum, err := cmd.Flags().GetUint64("end") + if err != nil { + log.Fatalf("Error reading end flag: %v", err) + } + batchSize, err := cmd.Flags().GetUint64("batch") + if err != nil { + log.Fatalf("Error reading batch flag: %v", err) + } + maxParallelGoroutines, err := cmd.Flags().GetInt("parallelism") + if err != nil { + log.Fatalf("Error reading parallelism flag: %v", err) + } + outputFile, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + humanReadableOutputFile, err := cmd.Flags().GetString("humanOutput") + if err != nil { + log.Fatalf("Error reading humanReadable flag: %v", err) + } + continueFile, err := cmd.Flags().GetString("continue") + if err != nil { + log.Fatalf("Error reading continue flag: %v", err) + } + dbDSN, err := cmd.Flags().GetString("db") + if err != nil { + log.Fatalf("Error reading db flag: %v", err) + } + + var db *gorm.DB + if dbDSN != "" { + db, err = gorm.Open(postgres.Open(dbDSN), &gorm.Config{}) + if err != nil { + log.Fatalf("Error connecting to database: %v", err) + } + } + + if continueFile != "" { + fmt.Println("Continue fetching block header fields from", continueFile) + + reader := newHeaderReader(continueFile) + defer reader.close() + + var lastSeenHeader uint64 + reader.read(func(header *types.Header) { + lastSeenHeader = header.Number + }) + fmt.Println("Last Seen Header:", lastSeenHeader) + + startBlockNum = lastSeenHeader + 1 + + fmt.Println("Overriding start block number to:", startBlockNum) + + if startBlockNum > endBlockNum { + log.Fatalf("Start block number %d exceeds end block number %d after continuing from file", startBlockNum, endBlockNum) + } + } + + runFetch(clients, db, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile) + }, +} + +func init() { + rootCmd.AddCommand(fetchCmd) + + fetchCmd.Flags().String("rpc", "http://localhost:8545,http://localhost:8546", "RPC URLs, separated by commas. Example: http://localhost:8545,http://localhost:8546") + fetchCmd.Flags().Uint64("start", 0, "start block number") + fetchCmd.Flags().Uint64("end", 1000, "end block number") + fetchCmd.Flags().Uint64("batch", 100, "batch size") + fetchCmd.Flags().Int("parallelism", 10, "max parallel goroutines each working on batch size blocks") + fetchCmd.Flags().String("output", "headers.bin", "output file") + fetchCmd.Flags().String("humanOutput", "", "additionally produce human readable csv file") + fetchCmd.Flags().String("continue", "", "continue fetching block header fields from the last seen block number in the specified continue file") + fetchCmd.Flags().String("db", "", "database to use instead of fetching from RPC") +} + +func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetries int) (*types.Header, error) { + var innerErr error + for i := 0; i < maxRetries; i++ { + header, err := client.HeaderByNumber(context.Background(), big.NewInt(int64(blockNum))) + if err == nil { + return types.NewHeader( + header.Number.Uint64(), + header.Difficulty.Uint64(), + header.Root, + header.Coinbase, + header.Nonce, + header.Extra, + ), nil + } + + innerErr = err // save the last error to return it if all retries fail + + // Wait before retrying + time.Sleep(time.Duration(i*200) * time.Millisecond) + log.Printf("Retrying header fetch for block %d, retry %d, error %v", blockNum, i+1, err) + } + + return nil, fmt.Errorf("error fetching header for block %d: %v", blockNum, innerErr) +} + +func fetchHeadersFromDB(db *gorm.DB, start, end uint64, headersChan chan<- *types.Header) { + blockORM := types.NewL2Block(db) + blocks, err := blockORM.GetL2BlocksInRange(context.Background(), start, end) + if err != nil { + log.Fatalf("Error fetching blocks from database: %v", err) + } + + for _, block := range blocks { + headersChan <- types.NewHeader( + block.Header.Number.Uint64(), + block.Header.Difficulty.Uint64(), + block.Header.Root, + block.Header.Coinbase, + block.Header.Nonce, + block.Header.Extra, + ) + } +} + +func fetchHeadersFromRPC(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { + // randomize client selection to distribute load + r := uint64(rand.Int()) + + // log time taken for fetching headers + startTime := time.Now() + var fetchTimeAvg, writeTimeAvg time.Duration + + for i := start; i <= end; i++ { + startTimeBlockFetch := time.Now() + client := clients[(r+i)%uint64(len(clients))] // round-robin client selection + header, err := headerByNumberWithRetry(client, i, 15) + if err != nil { + log.Fatalf("Error fetching header %d: %v", i, err) + } + fetchTimeAvg += time.Since(startTimeBlockFetch) + + startTimeHeaderWrite := time.Now() + headersChan <- header + writeTimeAvg += time.Since(startTimeHeaderWrite) + } + totalDuration := time.Since(startTime) + + fetchTimeAvg = fetchTimeAvg / time.Duration(end-start+1) + writeTimeAvg = writeTimeAvg / time.Duration(end-start+1) + log.Printf("Fetched %d header in %s (avg=%s, wrote to channel in avg %s", end-start+1, totalDuration, fetchTimeAvg, writeTimeAvg) +} + +func writeHeadersToFile(outputFile string, humanReadableOutputFile string, continueFile string, startBlockNum uint64, headersChan <-chan *types.Header) { + writer := newFilesWriter(outputFile, humanReadableOutputFile) + defer writer.close() + + if continueFile != "" { + reader := newHeaderReader(continueFile) + + var lastSeenHeader uint64 + var totalHeaders uint64 + reader.read(func(header *types.Header) { + writer.write(header) + totalHeaders++ + lastSeenHeader = header.Number + }) + + fmt.Println("Copied ", totalHeaders, "headers from continue file, last seen block number:", lastSeenHeader) + reader.close() + } + + headerHeap := &types.HeaderHeap{} + heap.Init(headerHeap) + + nextHeaderNum := startBlockNum + + // receive all headers and write them in order by using a sorted heap + for header := range headersChan { + heap.Push(headerHeap, header) + + // write all headers that are in order + for headerHeap.Len() > 0 && (*headerHeap)[0].Number == nextHeaderNum { + nextHeaderNum++ + sortedHeader := heap.Pop(headerHeap).(*types.Header) + writer.write(sortedHeader) + } + } + + fmt.Println("Finished writing headers to file, last block number:", nextHeaderNum-1) +} + +func runFetch(clients []*ethclient.Client, db *gorm.DB, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) { + headersChan := make(chan *types.Header, maxGoroutines*int(batchSize)) + tasks := make(chan task) + + var wgConsumer sync.WaitGroup + // start consumer goroutine to sort and write headers to file + wgConsumer.Add(1) + go func() { + writeHeadersToFile(outputFile, humanReadableOutputFile, continueFile, startBlockNum, headersChan) + wgConsumer.Done() + }() + + var wgProducers sync.WaitGroup + // start producer goroutines to fetch headers + for i := 0; i < maxGoroutines; i++ { + wgProducers.Add(1) + go func() { + for { + t, ok := <-tasks + if !ok { + break + } + log.Println("Received task", t.start, "to", t.end) + + // use DB if dbDSN is provided, otherwise fetch from RPC + if db != nil { + fetchHeadersFromDB(db, t.start, t.end, headersChan) + } else { + fetchHeadersFromRPC(clients, t.start, t.end, headersChan) + } + } + wgProducers.Done() + }() + } + + // need to fetch block 0 from RPC + if startBlockNum == 0 && db != nil { + fmt.Println("Fetching headers from database... and header 0 from RPC") + fetchHeadersFromRPC(clients, 0, 0, headersChan) + startBlockNum = 1 + } + + // create tasks/work packages for producer goroutines + for start := startBlockNum; start <= endBlockNum; start += batchSize { + end := start + batchSize - 1 + if end > endBlockNum { + end = endBlockNum + } + fmt.Println("Created task for blocks", start, "to", end) + + tasks <- task{start, end} + } + + close(tasks) + wgProducers.Wait() + close(headersChan) + wgConsumer.Wait() +} + +type task struct { + start uint64 + end uint64 +} + +// filesWriter is a helper struct to write headers to binary and human-readable csv files at the same time. +type filesWriter struct { + binaryFile *os.File + binaryWriter *bufio.Writer + + humanReadable bool + csvFile *os.File + csvWriter *bufio.Writer +} + +func newFilesWriter(outputFile string, humanReadableOutputFile string) *filesWriter { + binaryFile, err := os.Create(outputFile) + if err != nil { + log.Fatalf("Error creating binary file: %v", err) + } + + f := &filesWriter{ + binaryFile: binaryFile, + binaryWriter: bufio.NewWriter(binaryFile), + humanReadable: humanReadableOutputFile != "", + } + + if humanReadableOutputFile != "" { + csvFile, err := os.Create(humanReadableOutputFile) + if err != nil { + log.Fatalf("Error creating human readable file: %v", err) + } + f.csvFile = csvFile + f.csvWriter = bufio.NewWriter(csvFile) + } + + return f +} + +func (f *filesWriter) close() { + if err := f.binaryWriter.Flush(); err != nil { + log.Fatalf("Error flushing binary buffer: %v", err) + } + if f.humanReadable { + if err := f.csvWriter.Flush(); err != nil { + log.Fatalf("Error flushing csv buffer: %v", err) + } + } + + f.binaryFile.Close() + if f.humanReadable { + f.csvFile.Close() + } +} +func (f *filesWriter) write(header *types.Header) { + bytes, err := header.Bytes() + if err != nil { + log.Fatalf("Error converting header to bytes: %v", err) + } + + if _, err = f.binaryWriter.Write(bytes); err != nil { + log.Fatalf("Error writing to binary file: %v", err) + } + + if f.humanReadable { + if _, err = f.csvWriter.WriteString(header.String()); err != nil { + log.Fatalf("Error writing to human readable file: %v", err) + } + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go new file mode 100644 index 000000000000..0abade425127 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -0,0 +1,204 @@ +package cmd + +import ( + "bufio" + "bytes" + "io" + "log" + "math" + "os" + "sort" + + "github.com/scroll-tech/go-ethereum/common" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +// maxVanityCount is the maximum number of unique vanities that can be represented with a single byte. +const maxVanityCount = math.MaxUint8 + +type missingHeaderFileWriter struct { + file *os.File + writer *bufio.Writer + + missingHeaderWriter *missingHeaderWriter +} + +func newMissingHeaderFileWriter(filename string, seenVanity map[[32]byte]bool) *missingHeaderFileWriter { + if len(seenVanity) > maxVanityCount { + log.Fatalf("Number of unique vanities exceeds maximum: %d > %d", len(seenVanity), maxVanityCount) + } + + file, err := os.Create(filename) + if err != nil { + log.Fatalf("Error creating file: %v", err) + } + + writer := bufio.NewWriter(file) + return &missingHeaderFileWriter{ + file: file, + writer: writer, + missingHeaderWriter: newMissingHeaderWriter(writer, seenVanity), + } +} + +func (m *missingHeaderFileWriter) close() { + if err := m.writer.Flush(); err != nil { + log.Fatalf("Error flushing writer: %v", err) + } + if err := m.file.Close(); err != nil { + log.Fatalf("Error closing file: %v", err) + } +} + +type missingHeaderWriter struct { + writer io.Writer + + sortedVanities [][32]byte + sortedVanitiesMap map[[32]byte]int + seenDifficulty map[uint64]int + seenSealLen map[int]int +} + +func newMissingHeaderWriter(writer io.Writer, seenVanity map[[32]byte]bool) *missingHeaderWriter { + // sort the vanities and assign an index to each so that we can write the index of the vanity in the header + sortedVanities := make([][32]byte, 0, len(seenVanity)) + for vanity := range seenVanity { + sortedVanities = append(sortedVanities, vanity) + } + sort.Slice(sortedVanities, func(i, j int) bool { + return bytes.Compare(sortedVanities[i][:], sortedVanities[j][:]) < 0 + }) + sortedVanitiesMap := make(map[[32]byte]int) + for i, vanity := range sortedVanities { + sortedVanitiesMap[vanity] = i + } + + return &missingHeaderWriter{ + writer: writer, + sortedVanities: sortedVanities, + sortedVanitiesMap: sortedVanitiesMap, + } +} + +func (m *missingHeaderWriter) writeVanities() { + // write the count of unique vanities + if _, err := m.writer.Write([]byte{uint8(len(m.sortedVanitiesMap))}); err != nil { + log.Fatalf("Error writing unique vanity count: %v", err) + } + + // write the unique vanities + for _, vanity := range m.sortedVanities { + if _, err := m.writer.Write(vanity[:]); err != nil { + log.Fatalf("Error writing vanity: %v", err) + } + } +} + +func (m *missingHeaderWriter) write(header *types.Header) { + // 1. prepare the bitmask + hasCoinbase := header.Coinbase != (common.Address{}) + hasNonce := header.Nonce != (coreTypes.BlockNonce{}) + + bits := newBitMask(hasCoinbase, hasNonce, int(header.Difficulty), header.SealLen()) + vanityIndex := m.sortedVanitiesMap[header.Vanity()] + + if vanityIndex >= maxVanityCount { + log.Fatalf("Vanity index %d exceeds maximum allowed %d", vanityIndex, maxVanityCount-1) + } + + // 2. write the header: bitmask, optional coinbase, optional nonce, vanity index and seal + if _, err := m.writer.Write(bits.Bytes()); err != nil { + log.Fatalf("Error writing bitmask: %v", err) + } + if _, err := m.writer.Write([]byte{uint8(vanityIndex)}); err != nil { + log.Fatalf("Error writing vanity index: %v", err) + } + if _, err := m.writer.Write(header.StateRoot[:]); err != nil { + log.Fatalf("Error writing state root: %v", err) + } + if hasCoinbase { + if _, err := m.writer.Write(header.Coinbase[:]); err != nil { + log.Fatalf("Error writing coinbase: %v", err) + } + } + + if hasNonce { + if _, err := m.writer.Write(header.Nonce[:]); err != nil { + log.Fatalf("Error writing nonce: %v", err) + } + } + if _, err := m.writer.Write(header.Seal()); err != nil { + log.Fatalf("Error writing seal: %v", err) + } +} + +// bitMask is a bitmask that encodes the following information: +// +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + +func newBitMask(hasCoinbase bool, hasNonce bool, difficulty int, sealLen int) bitMask { + b := uint8(0) + + if hasCoinbase { + b |= 1 << 4 + } + + if hasNonce { + b |= 1 << 5 + } + if difficulty == 1 { + b |= 1 << 6 + } else if difficulty != 2 { + log.Fatalf("Invalid difficulty: %d", difficulty) + } + + if sealLen == 85 { + b |= 1 << 7 + } else if sealLen != 65 { + log.Fatalf("Invalid seal length: %d", sealLen) + } + + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} + +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go new file mode 100644 index 000000000000..8fee44df0177 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -0,0 +1,134 @@ +package cmd + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + + "github.com/scroll-tech/go-ethereum/common" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +func TestMissingHeaderWriter(t *testing.T) { + vanity1 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} + vanity2 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02} + vanity8 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08} + + stateRoot1 := common.HexToHash("0xabcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234") + stateRoot2 := common.HexToHash("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") + + var expectedBytes []byte + expectedBytes = append(expectedBytes, 0x03) + expectedBytes = append(expectedBytes, vanity1[:]...) + expectedBytes = append(expectedBytes, vanity2[:]...) + expectedBytes = append(expectedBytes, vanity8[:]...) + + seenVanity := map[[32]byte]bool{ + vanity8: true, + vanity1: true, + vanity2: true, + } + var buf []byte + bytesBuffer := bytes.NewBuffer(buf) + mhw := newMissingHeaderWriter(bytesBuffer, seenVanity) + + mhw.writeVanities() + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + + // Header0 + { + seal := randomSeal(65) + coinbase := common.Address{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + nonce := coreTypes.BlockNonce{0, 1, 2, 3, 4, 5, 6, 7} + header := types.NewHeader(0, 2, stateRoot1, coinbase, nonce, append(vanity1[:], seal...)) + mhw.write(header) + + // bit 6=0: difficulty 2, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b00110000) + expectedBytes = append(expectedBytes, 0x00) // vanity index0 + expectedBytes = append(expectedBytes, stateRoot1[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) + expectedBytes = append(expectedBytes, seal...) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header1 + { + coinbase := common.Address{} + nonce := coreTypes.BlockNonce{} + seal := randomSeal(65) + header := types.NewHeader(1, 1, stateRoot2, coinbase, nonce, append(vanity2[:], seal...)) + mhw.write(header) + + // bit 6=1: difficulty 1, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b01000000) + expectedBytes = append(expectedBytes, 0x01) // vanity index1 + expectedBytes = append(expectedBytes, stateRoot2[:]...) + expectedBytes = append(expectedBytes, seal...) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header2 + coinbase := common.Address{1} + nonce := coreTypes.BlockNonce{201} + { + seal := randomSeal(85) + header := types.NewHeader(2, 2, stateRoot1, coinbase, nonce, append(vanity2[:], seal...)) + mhw.write(header) + + // bit 6=0: difficulty 2, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b10110000) + expectedBytes = append(expectedBytes, 0x01) // vanity index1 + expectedBytes = append(expectedBytes, stateRoot1[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) + expectedBytes = append(expectedBytes, seal...) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header3 + { + seal := randomSeal(85) + header := types.NewHeader(3, 1, stateRoot2, coinbase, nonce, append(vanity8[:], seal...)) + mhw.write(header) + + // bit 6=1: difficulty 1, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b11110000) + expectedBytes = append(expectedBytes, 0x02) // vanity index2 + expectedBytes = append(expectedBytes, stateRoot2[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) + expectedBytes = append(expectedBytes, seal...) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header4 + { + stateRoot3 := common.Hash{123} + seal := randomSeal(65) + header := types.NewHeader(4, 2, stateRoot3, coinbase, nonce, append(vanity1[:], seal...)) + mhw.write(header) + + // bit 6=0: difficulty 2, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b00110000) + expectedBytes = append(expectedBytes, 0x00) // vanity index0 + expectedBytes = append(expectedBytes, stateRoot3[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) + expectedBytes = append(expectedBytes, seal...) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } +} + +func randomSeal(length int) []byte { + buf := make([]byte, length) + _, _ = rand.Read(buf) + return buf +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go new file mode 100644 index 000000000000..05c9dbb69b6e --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "export-headers-toolkit", + Short: "A toolkit for exporting and transforming missing block header fields of Scroll", + Long: `A toolkit for exporting and transforming missing block header fields of Scroll. + +The fields difficulty and extraData are missing from header data stored on L1 before EuclidV2. +This toolkit provides commands to export the missing fields, deduplicate the data and create a +file with the missing fields that can be used to reconstruct the correct block hashes when only reading +data from L1.`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.CompletionOptions.DisableDefaultCmd = true +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod new file mode 100644 index 000000000000..d10f0f5d0a80 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -0,0 +1,84 @@ +module github.com/scroll-tech/go-ethereum/export-headers-toolkit + +go 1.22 + +replace github.com/scroll-tech/go-ethereum => ../../.. + +require ( + github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 + github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 + github.com/spf13/cobra v1.9.1 + github.com/stretchr/testify v1.10.0 + gorm.io/driver/postgres v1.5.7 + gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde +) + +require ( + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect + github.com/fjl/memsize v0.0.2 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/uuid v1.1.5 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.3.0 // indirect + github.com/huin/goupnp v1.0.2 // indirect + github.com/iden3/go-iden3-crypto v0.0.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect + github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/tsdb v0.7.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rjeczalik/notify v0.9.1 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/scroll-tech/zktrie v0.8.4 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect + github.com/supranational/blst v0.3.12 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/urfave/cli.v1 v1.20.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum new file mode 100644 index 000000000000..9823b3e92427 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -0,0 +1,284 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38= +github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= +github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= +github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/iden3/go-iden3-crypto v0.0.16 h1:zN867xiz6HgErXVIV/6WyteGcOukE9gybYTorBMEdsk= +github.com/iden3/go-iden3-crypto v0.0.16/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 h1:Dhd58LE1D+dnoxpgLVeQBMF9uweL/fhQfZHWtWSiOlE= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY= +github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= +github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/supranational/blst v0.3.12 h1:Vfas2U2CFHhniv2QkUm2OVa1+pGTdqtpqm9NnhUUbZ8= +github.com/supranational/blst v0.3.12/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= +gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/rollup/missing_header_fields/export-headers-toolkit/main.go b/rollup/missing_header_fields/export-headers-toolkit/main.go new file mode 100644 index 000000000000..1d8e22abe8cb --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go new file mode 100644 index 000000000000..f9e1e61f9df8 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -0,0 +1,115 @@ +package types + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" +) + +const HeaderSizeSerialized = 2 +const VanitySize = 32 + +type Header struct { + Number uint64 + Difficulty uint64 + StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce + ExtraData []byte +} + +func NewHeader(number, difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte) *Header { + return &Header{ + Number: number, + Difficulty: difficulty, + StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, + ExtraData: extraData, + } +} + +func (h *Header) String() string { + return fmt.Sprintf("%d,%d,%s,%s,%s,%s\n", h.Number, h.Difficulty, h.StateRoot.Hex(), h.Coinbase.Hex(), common.Bytes2Hex(h.Nonce[:]), common.Bytes2Hex(h.ExtraData)) +} + +func (h *Header) Equal(other *Header) bool { + if h.Number != other.Number { + return false + } + if h.Difficulty != other.Difficulty { + return false + } + if h.StateRoot != other.StateRoot { + return false + } + if h.Coinbase != other.Coinbase { + return false + } + if h.Nonce != other.Nonce { + return false + } + if !bytes.Equal(h.ExtraData, other.ExtraData) { + return false + } + return true +} + +// Bytes returns the byte representation of the header including the initial 2 bytes for the size. +func (h *Header) Bytes() ([]byte, error) { + size := 8 + 8 + common.HashLength + common.AddressLength + 8 + len(h.ExtraData) + + buf := make([]byte, HeaderSizeSerialized+size) + binary.BigEndian.PutUint16(buf[:2], uint16(size)) + binary.BigEndian.PutUint64(buf[2:10], h.Number) + binary.BigEndian.PutUint64(buf[10:18], h.Difficulty) + copy(buf[18:50], h.StateRoot[:]) + copy(buf[50:70], h.Coinbase[:]) + copy(buf[70:78], h.Nonce[:]) + copy(buf[78:], h.ExtraData) + return buf, nil +} + +func (h *Header) Vanity() [VanitySize]byte { + return [VanitySize]byte(h.ExtraData[:VanitySize]) +} + +func (h *Header) Seal() []byte { + return h.ExtraData[VanitySize:] +} + +func (h *Header) SealLen() int { + return len(h.Seal()) +} + +// FromBytes reads the header from the byte representation excluding the initial 2 bytes for the size. +func (h *Header) FromBytes(buf []byte) *Header { + h.Number = binary.BigEndian.Uint64(buf[:8]) + h.Difficulty = binary.BigEndian.Uint64(buf[8:16]) + if len(buf) < 76 { + panic(fmt.Sprintf("buffer too short for header: %d bytes", len(buf))) + } + h.StateRoot = common.BytesToHash(buf[16:48]) + h.Coinbase = common.BytesToAddress(buf[48:68]) + h.Nonce = types.BlockNonce(buf[68:76]) + h.ExtraData = buf[76:] + + return h +} + +type HeaderHeap []*Header + +func (h HeaderHeap) Len() int { return len(h) } +func (h HeaderHeap) Less(i, j int) bool { return h[i].Number < h[j].Number } +func (h HeaderHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *HeaderHeap) Push(x interface{}) { *h = append(*h, x.(*Header)) } +func (h *HeaderHeap) Pop() interface{} { + old := *h + n := len(old) + item := old[n-1] + *h = old[0 : n-1] + return item +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go new file mode 100644 index 000000000000..2b2a18765090 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go @@ -0,0 +1,88 @@ +package types + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/scroll-tech/da-codec/encoding" + "gorm.io/gorm" + + "github.com/scroll-tech/go-ethereum/core/types" +) + +// L2Block represents a l2 block in the database. +type L2Block struct { + db *gorm.DB `gorm:"column:-"` + + // block + Number uint64 `json:"number" gorm:"number"` + Hash string `json:"hash" gorm:"hash"` + ParentHash string `json:"parent_hash" gorm:"parent_hash"` + Header string `json:"header" gorm:"header"` + Transactions string `json:"transactions" gorm:"transactions"` + WithdrawRoot string `json:"withdraw_root" gorm:"withdraw_root"` + StateRoot string `json:"state_root" gorm:"state_root"` + TxNum uint32 `json:"tx_num" gorm:"tx_num"` + GasUsed uint64 `json:"gas_used" gorm:"gas_used"` + BlockTimestamp uint64 `json:"block_timestamp" gorm:"block_timestamp"` + RowConsumption string `json:"row_consumption" gorm:"row_consumption"` + + // chunk + ChunkHash string `json:"chunk_hash" gorm:"chunk_hash;default:NULL"` + + // metadata + CreatedAt time.Time `json:"created_at" gorm:"column:created_at"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"` + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"column:deleted_at;default:NULL"` +} + +// NewL2Block creates a new L2Block instance +func NewL2Block(db *gorm.DB) *L2Block { + return &L2Block{db: db} +} + +// TableName returns the name of the "l2_block" table. +func (*L2Block) TableName() string { + return "l2_block" +} + +// GetL2BlocksInRange retrieves the L2 blocks within the specified range (inclusive). +// The range is closed, i.e., it includes both start and end block numbers. +// The returned blocks are sorted in ascending order by their block number. +func (o *L2Block) GetL2BlocksInRange(ctx context.Context, startBlockNumber uint64, endBlockNumber uint64) ([]*encoding.Block, error) { + if startBlockNumber > endBlockNumber { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: start block number should be less than or equal to end block number, start block: %v, end block: %v", startBlockNumber, endBlockNumber) + } + + db := o.db.WithContext(ctx) + db = db.Model(&L2Block{}) + db = db.Select("header") + db = db.Where("number >= ? AND number <= ?", startBlockNumber, endBlockNumber) + db = db.Order("number ASC") + + var l2Blocks []L2Block + if err := db.Find(&l2Blocks).Error; err != nil { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber) + } + + // sanity check + if uint64(len(l2Blocks)) != endBlockNumber-startBlockNumber+1 { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: unexpected number of results, expected: %v, got: %v", endBlockNumber-startBlockNumber+1, len(l2Blocks)) + } + + var blocks []*encoding.Block + for _, v := range l2Blocks { + var block encoding.Block + + block.Header = &types.Header{} + if err := json.Unmarshal([]byte(v.Header), block.Header); err != nil { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber) + } + + blocks = append(blocks, &block) + } + + return blocks, nil +} diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go new file mode 100644 index 000000000000..46804aa0830b --- /dev/null +++ b/rollup/missing_header_fields/manager.go @@ -0,0 +1,188 @@ +package missing_header_fields + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/log" +) + +const timeoutDownload = 30 * time.Minute + +// Manager is responsible for managing the missing header fields file. +// It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. +type Manager struct { + ctx context.Context + filePath string + downloadURL string + expectedChecksum common.Hash + + reader *Reader +} + +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum common.Hash) *Manager { + return &Manager{ + ctx: ctx, + filePath: filePath, + downloadURL: downloadURL, + expectedChecksum: expectedChecksum, + } +} + +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + // lazy initialization: if the reader is not initialized this is the first time we read from the file + if m.reader == nil { + if err = m.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + } + } + + return m.reader.Read(headerNum) +} + +func (m *Manager) initialize() error { + // if the file doesn't exist, download it + if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) { + if err = m.downloadFile(); err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + } + + // verify the expectedChecksum + f, err := os.Open(m.filePath) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + computedChecksum := h.Sum(nil) + if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x. Please delete %s to restart file download", m.expectedChecksum, computedChecksum, m.filePath) + } + + // finally initialize the reader + reader, err := NewReader(m.filePath) + if err != nil { + return fmt.Errorf("failed to create reader: %v", err) + } + + m.reader = reader + return nil +} + +func (m *Manager) Close() error { + if m.reader != nil { + return m.reader.Close() + } + return nil +} + +func (m *Manager) downloadFile() error { + log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL) + + downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload) + defer downloadCtxCancel() + + req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create download request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + // create a temporary file + tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return fmt.Errorf("failed to create temporary file: %v", err) + } + var ok bool + defer func() { + if !ok { + _ = os.Remove(tmpFilePath) + } + }() + + // copy the response body to the temporary file and print progress + writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength)) + if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil { + return fmt.Errorf("failed to copy response body: %v", err) + } + + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file: %v", err) + } + + // rename the temporary file to the final file path + if err = os.Rename(tmpFilePath, m.filePath); err != nil { + return fmt.Errorf("failed to rename temporary file: %v", err) + } + + ok = true + return nil +} + +type WriteCounter struct { + ctx context.Context + total uint64 + written uint64 + lastProgressPrinted time.Time +} + +func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter { + return &WriteCounter{ + ctx: ctx, + total: total, + } +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += uint64(n) + + // check if the context is done and return early + select { + case <-wc.ctx.Done(): + return n, wc.ctx.Err() + default: + } + + wc.printProgress() + + return n, nil +} + +func (wc *WriteCounter) printProgress() { + if time.Since(wc.lastProgressPrinted) < 5*time.Second { + return + } + wc.lastProgressPrinted = time.Now() + + log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total))) +} + +func toMB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go new file mode 100644 index 000000000000..ae0be08e043b --- /dev/null +++ b/rollup/missing_header_fields/manager_test.go @@ -0,0 +1,62 @@ +package missing_header_fields + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" + "github.com/scroll-tech/go-ethereum/params" +) + +func TestManagerDownload(t *testing.T) { + t.Skip("skipping test due to long runtime/downloading file") + log.Root().SetHandler(log.StdoutHandler) + + sha256 := *params.ScrollSepoliaChainConfig.Scroll.MissingHeaderFieldsSHA256 + downloadURL := "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/" + params.ScrollSepoliaChainConfig.ChainID.String() + ".bin" + filePath := filepath.Join(t.TempDir(), "test_file_path") + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + + // Check if the file was downloaded and tmp file was removed + _, err = os.Stat(filePath) + require.NoError(t, err) + _, err = os.Stat(filePath + ".tmp") + require.Error(t, err) +} + +func TestManagerChecksum(t *testing.T) { + downloadURL := "" // since the file exists we don't need to download it + filePath := filepath.Join("testdata", "missing-headers.bin") + + // Checksum doesn't match + { + sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.ErrorContains(t, err, "expectedChecksum mismatch") + } + + // Checksum matches + { + sha256 := [32]byte(common.FromHex("635c3f56bb66035bd99134a1e2bc23b34df376f4cd51a0c65e347ce3e65b5974")) + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + difficulty, stateRoot, coinbase, nonce, extra, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].coinbase, coinbase) + require.Equal(t, expectedMissingHeaders[0].nonce, nonce) + require.Equal(t, expectedMissingHeaders[0].extra, extra) + } +} diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go new file mode 100644 index 000000000000..fa8ffe5107b9 --- /dev/null +++ b/rollup/missing_header_fields/reader.go @@ -0,0 +1,221 @@ +package missing_header_fields + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" +) + +type missingHeader struct { + headerNum uint64 + difficulty uint64 + stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce + extraData []byte +} + +type Reader struct { + file *os.File + reader *bufio.Reader + sortedVanities map[int][32]byte + lastReadHeader *missingHeader +} + +func NewReader(filePath string) (*Reader, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + + r := &Reader{ + file: f, + reader: bufio.NewReader(f), + } + + if err = r.initialize(); err != nil { + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close file after initialization error: %w", err) + } + return nil, fmt.Errorf("failed to initialize reader: %w", err) + } + + return r, nil +} + +func (r *Reader) initialize() error { + // reset the reader and last read header + if _, err := r.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek to start: %w", err) + } + r.reader = bufio.NewReader(r.file) + r.lastReadHeader = nil + + // read the count of unique vanities + vanityCount, err := r.reader.ReadByte() + if err != nil { + return err + } + + // read the unique vanities + r.sortedVanities = make(map[int][32]byte) + for i := uint8(0); i < vanityCount; i++ { + var vanity [32]byte + if _, err = r.reader.Read(vanity[:]); err != nil { + return err + } + r.sortedVanities[int(i)] = vanity + } + + return nil +} + +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { + if err = r.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + } + } + + if r.lastReadHeader == nil { + if err = r.ReadNext(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err + } + } + + if headerNum > r.lastReadHeader.headerNum { + // skip the headers until the requested header number + for i := r.lastReadHeader.headerNum; i < headerNum; i++ { + if err = r.ReadNext(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err + } + } + } + + if headerNum == r.lastReadHeader.headerNum { + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil + } + + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) +} + +func (r *Reader) ReadNext() (err error) { + // read the bitmask + bitmaskByte, err := r.reader.ReadByte() + if err != nil { + return fmt.Errorf("failed to read bitmask: %v", err) + } + + bits := newBitMaskFromByte(bitmaskByte) + + // read the vanity index + vanityIndex, err := r.reader.ReadByte() + if err != nil { + return fmt.Errorf("failed to read vanity index: %v", err) + } + + stateRoot := make([]byte, common.HashLength) + if _, err := io.ReadFull(r.reader, stateRoot); err != nil { + return fmt.Errorf("failed to read state root: %v", err) + } + + var coinbase common.Address + if bits.hasCoinbase() { + if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { + return fmt.Errorf("failed to read coinbase: %v", err) + } + } + + var nonce types.BlockNonce + if bits.hasNonce() { + if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { + return fmt.Errorf("failed to read nonce: %v", err) + } + } + + seal := make([]byte, bits.sealLen()) + if _, err = io.ReadFull(r.reader, seal); err != nil { + return fmt.Errorf("failed to read seal: %v", err) + } + + // construct the extraData field + vanity := r.sortedVanities[int(vanityIndex)] + var b bytes.Buffer + b.Write(vanity[:]) + b.Write(seal) + + // we don't have the header number, so we'll just increment the last read header number + // we assume that the headers are written in order, starting from 0 + if r.lastReadHeader == nil { + r.lastReadHeader = &missingHeader{ + headerNum: 0, + difficulty: uint64(bits.difficulty()), + stateRoot: common.BytesToHash(stateRoot), + coinbase: coinbase, + nonce: nonce, + extraData: b.Bytes(), + } + } else { + r.lastReadHeader.headerNum++ + r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) + r.lastReadHeader.coinbase = coinbase + r.lastReadHeader.nonce = nonce + r.lastReadHeader.extraData = b.Bytes() + } + + return nil +} + +func (r *Reader) Close() error { + return r.file.Close() +} + +// bitMask is a bitmask that encodes the following information: +// +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} + +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go new file mode 100644 index 000000000000..4486491f6a17 --- /dev/null +++ b/rollup/missing_header_fields/reader_test.go @@ -0,0 +1,79 @@ +package missing_header_fields + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" +) + +type header struct { + number uint64 + difficulty uint64 + stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce + extra []byte +} + +var expectedMissingHeaders = []header{ + {0, 2, common.HexToHash("0x195dc9e93ed59fcd1d51e3262739761574b1d1518c6188e27a28357d9d93fb36"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x1c652497074d4a193fb16d61ffd6fc7727983e2d9e7010ac9ca31c241bdec3cb"), common.HexToAddress("0x687E0E85AD67ff71aC134CF61b65905b58Ab43b2"), types.BlockNonce(common.FromHex("ffffffffffffffff")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000228b5b48f65de89b35c77ae3791bfe26c1a3a91cffa2f30f50a58208f2b013ed1a86d1f5601e41b90a5345518e967985f2e949e53476ffc6d8781fa64e4e999101")}, + {2, 2, common.HexToHash("0x2ec6f0f086734c1d88f8ca915cbd2968804688c384aa6d95982435dc65ed476f"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000dbe3e10ae4c0e34d69bff96e86d4a0221361626e8cade7d9d69d870e37f7338f06a446b5461edd7f85d5dda1c54a41b9d0877e5fe9965c6a89a5f4aa3d0d6adc00")}, + {3, 2, common.HexToHash("0x16fc61ab25b479c4c367a85e3f537a598b92fb0e188464cbaca70b5dec08908d"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f649dccc68b8a96a7d06d4a69e9667c63f90c2a2819870c695c2ef95e6862df55f5953891a5f9a13dfafbdff4d23a53a5a6ded505fc87203ac78142bb787dcf200")}, + {4, 2, common.HexToHash("0x001b539a66e87624114117e4d643d5aac7c716b6468c82a8daffea747d16b1a4"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e7578000000000000009e49f8090a1e4941660730b5f11bc0d648da7c04cf3c608f5e0725395c9690f44e1f1c46a9ccd26d3d4db7db973749acb83b88b1564352fe70afc30245242b6c01")}, + {5, 2, common.HexToHash("0x264d833b19677ec62af0f34a4040b9f8e50b3914a16bf961b7d1e198902f127a"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f30ac95495ebb0a35a44657f0c361a7d1ea2d0a086ff7036f84875b47082094504809634f82e379b5ece2f43eead70b12e5d44c1befc4bb8763fac92b4fe8fbb01")}, + {6, 2, common.HexToHash("0x25ed6f6829966b24668510d4c828a596da881001470ab4e7e633761b6bdaba45"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000c9e8cf7bf35df3de8b7d0e98426a62a9e8ec302592f5622d527845ad78b53176558730f88dc79486ada7ed7aea3de4f8ca881b612440e31ab3b84ec7dff0cace01")}, + {7, 2, common.HexToHash("0x0e5e6c49b7c7cbcf3392c52f64b287c5a974b30f20f6695024231b5cf2155d0e"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d8ebea45d74ad882718f97488083530d2d968edf1d3ff6d18d2c757feb0e40206b158f9d4e829a795849a1e3148dd3a6f5251fa600631a74825e3ea041222f5001")}, + {8, 2, common.HexToHash("0x2f6d586a8ce1fc4f476887aaffd8b143d4f5604ce2242f945ae5e8874ede7084"), common.HexToAddress("0x48C3F81f3D998b6652900e1C3183736C238Fe429"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000cecebe754d3c81738e2e76e6a0b34756006b38cd1237728ecf2f41f9a6e325634941b1b29e0e4c1338d1c0fb1190da361a5880253b116493e7cff288f9f165e300")}, + {9, 2, common.HexToHash("0x2c7f91ed6610d3823da4ed730968c40d62a99b5b6245e3f0d4f83011c3be9422"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d6fa3b8ca99ca18fab9096d6791e70b7468a901f4cda0f48e7218e9807e79cee7b4fac8edeb23be5a593bbf9e0fc8ab678c0c58e7b4fb11869a1a4ac0f93657300")}, + {10, 2, common.HexToHash("0x1ae6b4a4b4f311a4f4a1b944445a32197f60d8070adcf7b92ed1f1cc42766504"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e75780000000000000016d5b03fbb592eb6cc5ec7ac98acc10b1b02184b69348b0ffb87817077e3066765573d90201d0b6690d49be1fda5a76f9edfddbdd45ebc69db3d59857e13bdf101")}, +} + +func TestReader_Read(t *testing.T) { + expectedVanities := map[int][32]byte{ + 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), + 1: [32]byte(common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000")), + } + + reader, err := NewReader("testdata/missing-headers.bin") + require.NoError(t, err) + + require.Len(t, reader.sortedVanities, len(expectedVanities)) + for i, expectedVanity := range expectedVanities { + require.Equal(t, expectedVanity, reader.sortedVanities[i]) + } + + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 5) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders, 10) + + // no data anymore + _, _, _, _, _, err = reader.Read(11) + require.Error(t, err) +} + +func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { + difficulty, stateRoot, coinbase, nonce, extra, err := reader.Read(headerNum) + require.NoError(t, err) + require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) + require.Equalf(t, expectedHeaders[headerNum].coinbase, coinbase, "expected coinbase %s, got %s", expectedHeaders[headerNum].coinbase.Hex(), coinbase.Hex()) + require.Equalf(t, expectedHeaders[headerNum].nonce, nonce, "expected nonce %s, got %s", common.Bytes2Hex(expectedHeaders[headerNum].nonce[:]), common.Bytes2Hex(nonce[:])) + require.Equal(t, expectedHeaders[headerNum].extra, extra) +} diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin new file mode 100644 index 000000000000..45674ccefc43 Binary files /dev/null and b/rollup/missing_header_fields/testdata/missing-headers.bin differ diff --git a/rollup/rollup_sync_service/rollup_sync_service_test.go b/rollup/rollup_sync_service/rollup_sync_service_test.go index cf8c3d535fca..749685b31566 100644 --- a/rollup/rollup_sync_service/rollup_sync_service_test.go +++ b/rollup/rollup_sync_service/rollup_sync_service_test.go @@ -21,6 +21,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) func TestGetCommittedBatchMetaCodecV0(t *testing.T) { @@ -185,7 +186,7 @@ func (m mockEntryWithBlocks) Event() l1.RollupEvent { panic("implement me") } -func (m mockEntryWithBlocks) Blocks() ([]*da.PartialBlock, error) { +func (m mockEntryWithBlocks) Blocks(_ *missing_header_fields.Manager) ([]*da.PartialBlock, error) { panic("implement me") }