diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c1fbb08746a5..459feca738de 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -177,6 +177,7 @@ var ( utils.RollupVerifyEnabledFlag, utils.ShadowforkPeersFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABlobScanAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1d299d339fe8..fbfc1b91fe56 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -237,6 +237,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.L1DisableMessageQueueV2Flag, utils.RollupVerifyEnabledFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlobScanAPIEndpointFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7371a3770074..5a7bdf80f0b7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -898,6 +898,12 @@ var ( Name: "da.sync", Usage: "Enable node syncing from DA", } + DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ + Name: "da.missingheaderfields.baseurl", + Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks", + Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/", + } + DABlobScanAPIEndpointFlag = cli.StringFlag{ Name: "da.blob.blobscan", Usage: "BlobScan blob API endpoint", @@ -1382,6 +1388,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index 57a803198fa1..314368ee049d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1880,15 +1880,18 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() + // sanitize base fee + // Note: setting the base fee to 0 will cause problems as nil != 0 when serializing the header and thus block hash will be different. + if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { + header.BaseFee = nil + } + tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil) receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig) if err != nil { return nil, NonStatTy, fmt.Errorf("error processing block: %w", err) } - // TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique - // This should be done with https://github.com/scroll-tech/go-ethereum/pull/913. - if sign { // Prevent Engine from overriding timestamp. originalTime := header.Time @@ -1901,7 +1904,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types // finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble() header.GasUsed = gasUsed - header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + + // state root might be set from partial header. If it is not set, we calculate it. + if header.Root == (common.Hash{}) { + header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + } fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/eth/backend.go b/eth/backend.go index ad765b835b0a..3fbefcfe63d2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,9 @@ import ( "errors" "fmt" "math/big" + "net/url" + "path" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -61,6 +64,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/ccc" "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service" "github.com/scroll-tech/go-ethereum/rollup/sync_service" "github.com/scroll-tech/go-ethereum/rpc" @@ -241,7 +245,12 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if config.EnableDASyncing { // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { - eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA) + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) } @@ -337,6 +346,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether return eth, nil } +func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) { + downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) + } + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin") + + expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 + if expectedSHA256Checksum == nil { + return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") + } + + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) + return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/node/config.go b/node/config.go index 6e7133a95129..d7212389684e 100644 --- a/node/config.go +++ b/node/config.go @@ -201,6 +201,8 @@ type Config struct { L1DisableMessageQueueV2 bool `toml:",omitempty"` // Is daSyncingEnabled DaSyncingEnabled bool `toml:",omitempty"` + // Base URL for missing header fields file + DAMissingHeaderFieldsBaseURL string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/params/config.go b/params/config.go index eda4340e80c4..b6b7da4b7fc0 100644 --- a/params/config.go +++ b/params/config.go @@ -30,16 +30,18 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") - ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") - ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") - ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") - ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") + ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") + ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") + ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") + ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xfa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xa02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924") ) func newUint64(val uint64) *uint64 { return &val } @@ -354,7 +356,8 @@ var ( ScrollChainAddress: common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"), L2SystemConfigAddress: common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"), }, - GenesisStateRoot: &ScrollSepoliaGenesisState, + GenesisStateRoot: &ScrollSepoliaGenesisState, + MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256, }, } @@ -406,7 +409,8 @@ var ( ScrollChainAddress: common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"), L2SystemConfigAddress: common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"), }, - GenesisStateRoot: &ScrollMainnetGenesisState, + GenesisStateRoot: &ScrollMainnetGenesisState, + MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256, }, } @@ -710,6 +714,9 @@ type ScrollConfig struct { // Genesis State Root for MPT clients GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"` + + // MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file. + MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"` } // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer @@ -760,8 +767,13 @@ func (s ScrollConfig) String() string { genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot) } - return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}", - s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot) + missingHeaderFieldsSHA256 := "" + if s.MissingHeaderFieldsSHA256 != nil { + missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256) + } + + return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}", + s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256) } // IsValidTxCount returns whether the given block's transaction count is below the limit. diff --git a/rollup/da_syncer/block_queue.go b/rollup/da_syncer/block_queue.go index 630382f001c0..30d53bfcb6a5 100644 --- a/rollup/da_syncer/block_queue.go +++ b/rollup/da_syncer/block_queue.go @@ -6,19 +6,22 @@ import ( "github.com/scroll-tech/go-ethereum/core/rawdb" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and // provides them to the next stage one-by-one. type BlockQueue struct { - batchQueue *BatchQueue - blocks []*da.PartialBlock + batchQueue *BatchQueue + blocks []*da.PartialBlock + missingHeaderFieldsManager *missing_header_fields.Manager } -func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue { +func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue { return &BlockQueue{ - batchQueue: batchQueue, - blocks: make([]*da.PartialBlock, 0), + batchQueue: batchQueue, + blocks: make([]*da.PartialBlock, 0), + missingHeaderFieldsManager: missingHeaderFieldsManager, } } @@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error { return err } - bq.blocks, err = entryWithBlocks.Blocks() + bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager) if err != nil { return fmt.Errorf("failed to get blocks from entry: %w", err) } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index c99a474b256b..ca7322264a43 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type CommitBatchDAV0 struct { @@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int { return 0 } -func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) { l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped) if err != nil { return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err) @@ -120,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex := c.parentTotalL1MessagePopped for _, chunk := range c.chunks { - for blockId, daBlock := range chunk.Blocks { + for blockIndex, daBlock := range chunk.Blocks { // create txs txs := make(types.Transactions, 0, daBlock.NumTransactions()) // insert l1 msgs @@ -132,7 +133,12 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex += uint64(daBlock.NumL1Messages()) // insert l2 txs - txs = append(txs, chunk.Transactions[blockId]...) + txs = append(txs, chunk.Transactions[blockIndex]...) + + difficulty, stateRoot, coinbase, nonce, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + if err != nil { + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) + } block := NewPartialBlock( &PartialHeader{ @@ -140,8 +146,11 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { Time: daBlock.Timestamp(), BaseFee: daBlock.BaseFee(), GasLimit: daBlock.GasLimit(), - Difficulty: 10, // TODO: replace with real difficulty - ExtraData: []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data + Difficulty: difficulty, + ExtraData: extraData, + StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/commitV7.go b/rollup/da_syncer/da/commitV7.go index 6048b853aee7..37a4b948df6c 100644 --- a/rollup/da_syncer/da/commitV7.go +++ b/rollup/da_syncer/da/commitV7.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" @@ -113,7 +114,7 @@ func (c *CommitBatchDAV7) Event() l1.RollupEvent { return c.event } -func (c *CommitBatchDAV7) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV7) Blocks(_ *missing_header_fields.Manager) ([]*PartialBlock, error) { initialL1MessageIndex := c.parentTotalL1MessagePopped l1Txs, err := getL1MessagesV7(c.db, c.blobPayload.Blocks(), initialL1MessageIndex) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index fe72473451b6..f80ee631b51d 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -8,6 +8,7 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type Type int @@ -34,7 +35,7 @@ type Entry interface { type EntryWithBlocks interface { Entry - Blocks() ([]*PartialBlock, error) + Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) Version() encoding.CodecVersion Chunks() []*encoding.DAChunkRawTx BlobVersionedHashes() []common.Hash @@ -53,6 +54,9 @@ type PartialHeader struct { GasLimit uint64 Difficulty uint64 ExtraData []byte + StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce } func (h *PartialHeader) ToHeader() *types.Header { @@ -63,6 +67,9 @@ func (h *PartialHeader) ToHeader() *types.Header { GasLimit: h.GasLimit, Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, + Root: h.StateRoot, + Coinbase: h.Coinbase, + Nonce: h.Nonce, } } diff --git a/rollup/da_syncer/syncing_pipeline.go b/rollup/da_syncer/syncing_pipeline.go index 080179107f8c..19b7bcbc07de 100644 --- a/rollup/da_syncer/syncing_pipeline.go +++ b/rollup/da_syncer/syncing_pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // Config is the configuration parameters of data availability syncing. @@ -50,7 +51,7 @@ type SyncingPipeline struct { daQueue *DAQueue } -func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) { +func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config, missingHeaderFieldsManager *missing_header_fields.Manager) (*SyncingPipeline, error) { l1Reader, err := l1.NewReader(ctx, l1.Config{ ScrollChainAddress: genesisConfig.Scroll.L1Config.ScrollChainAddress, L1MessageQueueAddress: genesisConfig.Scroll.L1Config.L1MessageQueueAddress, @@ -124,7 +125,7 @@ func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesi daQueue := NewDAQueue(lastProcessedBatchMeta.L1BlockNumber, dataSourceFactory) batchQueue := NewBatchQueue(daQueue, db, lastProcessedBatchMeta) - blockQueue := NewBlockQueue(batchQueue) + blockQueue := NewBlockQueue(batchQueue, missingHeaderFieldsManager) daSyncer := NewDASyncer(blockchain, config.L2EndBlock) ctx, cancel := context.WithCancel(ctx) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 3a155af6c5fd..69126a860a4c 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -16,8 +16,8 @@ import ( "github.com/scroll-tech/go-ethereum/common" coreTypes "github.com/scroll-tech/go-ethereum/core/types" - "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // dedupCmd represents the dedup command @@ -49,6 +49,9 @@ The binary layout of the deduplicated file is as follows: log.Fatalf("Error reading verify flag: %v", err) } + // uncomment the following line to copy from the verify file to the input file. This is useful to generate a deduplicated header file for testing purposes. + // copyFromVerifyFile(verifyFile, inputFile) + if verifyFile != "" { verifyInputFile(verifyFile, inputFile) } @@ -251,6 +254,20 @@ func (h *csvHeaderReader) close() { h.file.Close() } +func copyFromVerifyFile(verifyFile, inputFile string) { + fmt.Println("Copying from", verifyFile, "to", inputFile) + + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + writer := newFilesWriter(inputFile, "") + defer writer.close() + + for header := csvReader.readNext(); header != nil; header = csvReader.readNext() { + writer.write(header) + } +} + func verifyInputFile(verifyFile, inputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() @@ -273,7 +290,7 @@ func verifyOutputFile(verifyFile, outputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() - dedupReader, err := NewReader(outputFile) + dedupReader, err := missing_header_fields.NewReader(outputFile) if err != nil { log.Fatalf("Error opening dedup file: %v", err) } @@ -282,7 +299,7 @@ func verifyOutputFile(verifyFile, outputFile string) { for { header := csvReader.readNext() if header == nil { - if _, _, _, _, err = dedupReader.ReadNext(); err == nil { + if err = dedupReader.ReadNext(); err == nil { log.Fatalf("Expected EOF, got more headers") } break diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index 52700fc5f90c..8fee44df0177 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,10 +5,11 @@ import ( "crypto/rand" "testing" - coreTypes "github.com/scroll-tech/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index f1062d385a7b..d10f0f5d0a80 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -2,9 +2,11 @@ module github.com/scroll-tech/go-ethereum/export-headers-toolkit go 1.22 +replace github.com/scroll-tech/go-ethereum => ../../.. + require ( - github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 - github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 + github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 + github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 gorm.io/driver/postgres v1.5.7 @@ -12,7 +14,7 @@ require ( ) require ( - github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -23,7 +25,7 @@ require ( github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/fjl/memsize v0.0.2 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-stack/stack v1.8.1 // indirect @@ -43,6 +45,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -58,6 +61,7 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/scroll-tech/zktrie v0.8.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect github.com/supranational/blst v0.3.12 // indirect @@ -66,6 +70,8 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 8ef0f6fde0ce..9823b3e92427 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -1,7 +1,9 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= -github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38= +github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= @@ -41,8 +43,8 @@ github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -108,6 +110,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -166,14 +170,14 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 h1:CDrPMqifvAVyYqu0x1J5qickVV0b51tApPnOwDYLESI= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 h1:Jq4TTYcHVAIVPUHNYbOzNxEsWf+9Q3b30YQaMyl0TDI= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 h1:Dhd58LE1D+dnoxpgLVeQBMF9uweL/fhQfZHWtWSiOlE= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY= github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= @@ -199,6 +203,10 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -208,8 +216,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -229,7 +237,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go index 6e08b0621bc4..2b2a18765090 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go @@ -7,8 +7,9 @@ import ( "time" "github.com/scroll-tech/da-codec/encoding" - "github.com/scroll-tech/go-ethereum/core/types" "gorm.io/gorm" + + "github.com/scroll-tech/go-ethereum/core/types" ) // L2Block represents a l2 block in the database. diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go new file mode 100644 index 000000000000..46804aa0830b --- /dev/null +++ b/rollup/missing_header_fields/manager.go @@ -0,0 +1,188 @@ +package missing_header_fields + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/log" +) + +const timeoutDownload = 30 * time.Minute + +// Manager is responsible for managing the missing header fields file. +// It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. +type Manager struct { + ctx context.Context + filePath string + downloadURL string + expectedChecksum common.Hash + + reader *Reader +} + +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum common.Hash) *Manager { + return &Manager{ + ctx: ctx, + filePath: filePath, + downloadURL: downloadURL, + expectedChecksum: expectedChecksum, + } +} + +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + // lazy initialization: if the reader is not initialized this is the first time we read from the file + if m.reader == nil { + if err = m.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + } + } + + return m.reader.Read(headerNum) +} + +func (m *Manager) initialize() error { + // if the file doesn't exist, download it + if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) { + if err = m.downloadFile(); err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + } + + // verify the expectedChecksum + f, err := os.Open(m.filePath) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + computedChecksum := h.Sum(nil) + if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x. Please delete %s to restart file download", m.expectedChecksum, computedChecksum, m.filePath) + } + + // finally initialize the reader + reader, err := NewReader(m.filePath) + if err != nil { + return fmt.Errorf("failed to create reader: %v", err) + } + + m.reader = reader + return nil +} + +func (m *Manager) Close() error { + if m.reader != nil { + return m.reader.Close() + } + return nil +} + +func (m *Manager) downloadFile() error { + log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL) + + downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload) + defer downloadCtxCancel() + + req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create download request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + // create a temporary file + tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return fmt.Errorf("failed to create temporary file: %v", err) + } + var ok bool + defer func() { + if !ok { + _ = os.Remove(tmpFilePath) + } + }() + + // copy the response body to the temporary file and print progress + writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength)) + if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil { + return fmt.Errorf("failed to copy response body: %v", err) + } + + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file: %v", err) + } + + // rename the temporary file to the final file path + if err = os.Rename(tmpFilePath, m.filePath); err != nil { + return fmt.Errorf("failed to rename temporary file: %v", err) + } + + ok = true + return nil +} + +type WriteCounter struct { + ctx context.Context + total uint64 + written uint64 + lastProgressPrinted time.Time +} + +func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter { + return &WriteCounter{ + ctx: ctx, + total: total, + } +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += uint64(n) + + // check if the context is done and return early + select { + case <-wc.ctx.Done(): + return n, wc.ctx.Err() + default: + } + + wc.printProgress() + + return n, nil +} + +func (wc *WriteCounter) printProgress() { + if time.Since(wc.lastProgressPrinted) < 5*time.Second { + return + } + wc.lastProgressPrinted = time.Now() + + log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total))) +} + +func toMB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go new file mode 100644 index 000000000000..ae0be08e043b --- /dev/null +++ b/rollup/missing_header_fields/manager_test.go @@ -0,0 +1,62 @@ +package missing_header_fields + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" + "github.com/scroll-tech/go-ethereum/params" +) + +func TestManagerDownload(t *testing.T) { + t.Skip("skipping test due to long runtime/downloading file") + log.Root().SetHandler(log.StdoutHandler) + + sha256 := *params.ScrollSepoliaChainConfig.Scroll.MissingHeaderFieldsSHA256 + downloadURL := "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/" + params.ScrollSepoliaChainConfig.ChainID.String() + ".bin" + filePath := filepath.Join(t.TempDir(), "test_file_path") + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + + // Check if the file was downloaded and tmp file was removed + _, err = os.Stat(filePath) + require.NoError(t, err) + _, err = os.Stat(filePath + ".tmp") + require.Error(t, err) +} + +func TestManagerChecksum(t *testing.T) { + downloadURL := "" // since the file exists we don't need to download it + filePath := filepath.Join("testdata", "missing-headers.bin") + + // Checksum doesn't match + { + sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.ErrorContains(t, err, "expectedChecksum mismatch") + } + + // Checksum matches + { + sha256 := [32]byte(common.FromHex("635c3f56bb66035bd99134a1e2bc23b34df376f4cd51a0c65e347ce3e65b5974")) + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + difficulty, stateRoot, coinbase, nonce, extra, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].coinbase, coinbase) + require.Equal(t, expectedMissingHeaders[0].nonce, nonce) + require.Equal(t, expectedMissingHeaders[0].extra, extra) + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/reader.go similarity index 54% rename from rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go rename to rollup/missing_header_fields/reader.go index df88dc95032e..fa8ffe5107b9 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ b/rollup/missing_header_fields/reader.go @@ -1,4 +1,4 @@ -package cmd +package missing_header_fields import ( "bufio" @@ -11,8 +11,6 @@ import ( "github.com/scroll-tech/go-ethereum/core/types" ) -// TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit - type missingHeader struct { headerNum uint64 difficulty uint64 @@ -32,7 +30,7 @@ type Reader struct { func NewReader(filePath string) (*Reader, error) { f, err := os.Open(filePath) if err != nil { - return nil, fmt.Errorf("failed to open file: %v", err) + return nil, fmt.Errorf("failed to open file: %w", err) } r := &Reader{ @@ -40,10 +38,28 @@ func NewReader(filePath string) (*Reader, error) { reader: bufio.NewReader(f), } + if err = r.initialize(); err != nil { + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close file after initialization error: %w", err) + } + return nil, fmt.Errorf("failed to initialize reader: %w", err) + } + + return r, nil +} + +func (r *Reader) initialize() error { + // reset the reader and last read header + if _, err := r.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek to start: %w", err) + } + r.reader = bufio.NewReader(r.file) + r.lastReadHeader = nil + // read the count of unique vanities vanityCount, err := r.reader.ReadByte() if err != nil { - return nil, err + return err } // read the unique vanities @@ -51,17 +67,23 @@ func NewReader(filePath string) (*Reader, error) { for i := uint8(0); i < vanityCount; i++ { var vanity [32]byte if _, err = r.reader.Read(vanity[:]); err != nil { - return nil, err + return err } r.sortedVanities[int(i)] = vanity } - return r, nil + return nil } func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { + if err = r.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + } + } + if r.lastReadHeader == nil { - if _, _, _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } @@ -69,7 +91,7 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has if headerNum > r.lastReadHeader.headerNum { // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { - if _, _, _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } @@ -79,15 +101,14 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil } - // headerNum < r.lastReadHeader.headerNum is not supported - return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) } -func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { +func (r *Reader) ReadNext() (err error) { // read the bitmask bitmaskByte, err := r.reader.ReadByte() if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read bitmask: %v", err) + return fmt.Errorf("failed to read bitmask: %v", err) } bits := newBitMaskFromByte(bitmaskByte) @@ -95,30 +116,31 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t // read the vanity index vanityIndex, err := r.reader.ReadByte() if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read vanity index: %v", err) + return fmt.Errorf("failed to read vanity index: %v", err) } - var stateRoot common.Hash - if _, err := io.ReadFull(r.reader, stateRoot[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read state root: %v", err) + stateRoot := make([]byte, common.HashLength) + if _, err := io.ReadFull(r.reader, stateRoot); err != nil { + return fmt.Errorf("failed to read state root: %v", err) } + var coinbase common.Address if bits.hasCoinbase() { if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read coinbase: %v", err) + return fmt.Errorf("failed to read coinbase: %v", err) } } + var nonce types.BlockNonce if bits.hasNonce() { if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read nonce: %v", err) + return fmt.Errorf("failed to read nonce: %v", err) } } seal := make([]byte, bits.sealLen()) - if _, err = io.ReadFull(r.reader, seal); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read seal: %v", err) + return fmt.Errorf("failed to read seal: %v", err) } // construct the extraData field @@ -133,7 +155,7 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t r.lastReadHeader = &missingHeader{ headerNum: 0, difficulty: uint64(bits.difficulty()), - stateRoot: stateRoot, + stateRoot: common.BytesToHash(stateRoot), coinbase: coinbase, nonce: nonce, extraData: b.Bytes(), @@ -141,15 +163,59 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) - r.lastReadHeader.stateRoot = stateRoot + r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) r.lastReadHeader.coinbase = coinbase r.lastReadHeader.nonce = nonce r.lastReadHeader.extraData = b.Bytes() } - return difficulty, coinbase, nonce, b.Bytes(), nil + return nil } func (r *Reader) Close() error { return r.file.Close() } + +// bitMask is a bitmask that encodes the following information: +// +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} + +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go new file mode 100644 index 000000000000..4486491f6a17 --- /dev/null +++ b/rollup/missing_header_fields/reader_test.go @@ -0,0 +1,79 @@ +package missing_header_fields + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" +) + +type header struct { + number uint64 + difficulty uint64 + stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce + extra []byte +} + +var expectedMissingHeaders = []header{ + {0, 2, common.HexToHash("0x195dc9e93ed59fcd1d51e3262739761574b1d1518c6188e27a28357d9d93fb36"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x1c652497074d4a193fb16d61ffd6fc7727983e2d9e7010ac9ca31c241bdec3cb"), common.HexToAddress("0x687E0E85AD67ff71aC134CF61b65905b58Ab43b2"), types.BlockNonce(common.FromHex("ffffffffffffffff")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000228b5b48f65de89b35c77ae3791bfe26c1a3a91cffa2f30f50a58208f2b013ed1a86d1f5601e41b90a5345518e967985f2e949e53476ffc6d8781fa64e4e999101")}, + {2, 2, common.HexToHash("0x2ec6f0f086734c1d88f8ca915cbd2968804688c384aa6d95982435dc65ed476f"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000dbe3e10ae4c0e34d69bff96e86d4a0221361626e8cade7d9d69d870e37f7338f06a446b5461edd7f85d5dda1c54a41b9d0877e5fe9965c6a89a5f4aa3d0d6adc00")}, + {3, 2, common.HexToHash("0x16fc61ab25b479c4c367a85e3f537a598b92fb0e188464cbaca70b5dec08908d"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f649dccc68b8a96a7d06d4a69e9667c63f90c2a2819870c695c2ef95e6862df55f5953891a5f9a13dfafbdff4d23a53a5a6ded505fc87203ac78142bb787dcf200")}, + {4, 2, common.HexToHash("0x001b539a66e87624114117e4d643d5aac7c716b6468c82a8daffea747d16b1a4"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e7578000000000000009e49f8090a1e4941660730b5f11bc0d648da7c04cf3c608f5e0725395c9690f44e1f1c46a9ccd26d3d4db7db973749acb83b88b1564352fe70afc30245242b6c01")}, + {5, 2, common.HexToHash("0x264d833b19677ec62af0f34a4040b9f8e50b3914a16bf961b7d1e198902f127a"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f30ac95495ebb0a35a44657f0c361a7d1ea2d0a086ff7036f84875b47082094504809634f82e379b5ece2f43eead70b12e5d44c1befc4bb8763fac92b4fe8fbb01")}, + {6, 2, common.HexToHash("0x25ed6f6829966b24668510d4c828a596da881001470ab4e7e633761b6bdaba45"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000c9e8cf7bf35df3de8b7d0e98426a62a9e8ec302592f5622d527845ad78b53176558730f88dc79486ada7ed7aea3de4f8ca881b612440e31ab3b84ec7dff0cace01")}, + {7, 2, common.HexToHash("0x0e5e6c49b7c7cbcf3392c52f64b287c5a974b30f20f6695024231b5cf2155d0e"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d8ebea45d74ad882718f97488083530d2d968edf1d3ff6d18d2c757feb0e40206b158f9d4e829a795849a1e3148dd3a6f5251fa600631a74825e3ea041222f5001")}, + {8, 2, common.HexToHash("0x2f6d586a8ce1fc4f476887aaffd8b143d4f5604ce2242f945ae5e8874ede7084"), common.HexToAddress("0x48C3F81f3D998b6652900e1C3183736C238Fe429"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000cecebe754d3c81738e2e76e6a0b34756006b38cd1237728ecf2f41f9a6e325634941b1b29e0e4c1338d1c0fb1190da361a5880253b116493e7cff288f9f165e300")}, + {9, 2, common.HexToHash("0x2c7f91ed6610d3823da4ed730968c40d62a99b5b6245e3f0d4f83011c3be9422"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d6fa3b8ca99ca18fab9096d6791e70b7468a901f4cda0f48e7218e9807e79cee7b4fac8edeb23be5a593bbf9e0fc8ab678c0c58e7b4fb11869a1a4ac0f93657300")}, + {10, 2, common.HexToHash("0x1ae6b4a4b4f311a4f4a1b944445a32197f60d8070adcf7b92ed1f1cc42766504"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e75780000000000000016d5b03fbb592eb6cc5ec7ac98acc10b1b02184b69348b0ffb87817077e3066765573d90201d0b6690d49be1fda5a76f9edfddbdd45ebc69db3d59857e13bdf101")}, +} + +func TestReader_Read(t *testing.T) { + expectedVanities := map[int][32]byte{ + 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), + 1: [32]byte(common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000")), + } + + reader, err := NewReader("testdata/missing-headers.bin") + require.NoError(t, err) + + require.Len(t, reader.sortedVanities, len(expectedVanities)) + for i, expectedVanity := range expectedVanities { + require.Equal(t, expectedVanity, reader.sortedVanities[i]) + } + + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 5) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders, 10) + + // no data anymore + _, _, _, _, _, err = reader.Read(11) + require.Error(t, err) +} + +func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { + difficulty, stateRoot, coinbase, nonce, extra, err := reader.Read(headerNum) + require.NoError(t, err) + require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) + require.Equalf(t, expectedHeaders[headerNum].coinbase, coinbase, "expected coinbase %s, got %s", expectedHeaders[headerNum].coinbase.Hex(), coinbase.Hex()) + require.Equalf(t, expectedHeaders[headerNum].nonce, nonce, "expected nonce %s, got %s", common.Bytes2Hex(expectedHeaders[headerNum].nonce[:]), common.Bytes2Hex(nonce[:])) + require.Equal(t, expectedHeaders[headerNum].extra, extra) +} diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin new file mode 100644 index 000000000000..45674ccefc43 Binary files /dev/null and b/rollup/missing_header_fields/testdata/missing-headers.bin differ diff --git a/rollup/rollup_sync_service/rollup_sync_service_test.go b/rollup/rollup_sync_service/rollup_sync_service_test.go index e6f6d7339697..68c53f6ac8f5 100644 --- a/rollup/rollup_sync_service/rollup_sync_service_test.go +++ b/rollup/rollup_sync_service/rollup_sync_service_test.go @@ -21,6 +21,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) func TestGetCommittedBatchMetaCodecV0(t *testing.T) { @@ -185,7 +186,7 @@ func (m mockEntryWithBlocks) Event() l1.RollupEvent { panic("implement me") } -func (m mockEntryWithBlocks) Blocks() ([]*da.PartialBlock, error) { +func (m mockEntryWithBlocks) Blocks(_ *missing_header_fields.Manager) ([]*da.PartialBlock, error) { panic("implement me") }