From 9c51e74e00a38dec83ed48811c0b10774d1968b4 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:48:13 +0800 Subject: [PATCH 01/17] feat: add toolkit for exporting and transforming missing block headers fields --- .../export-headers-toolkit/.gitignore | 1 + .../export-headers-toolkit/Dockerfile | 13 + .../export-headers-toolkit/README.md | 64 +++++ .../export-headers-toolkit/cmd/dedup.go | 143 ++++++++++ .../export-headers-toolkit/cmd/fetch.go | 246 +++++++++++++++++ .../export-headers-toolkit/cmd/root.go | 32 +++ .../export-headers-toolkit/go.mod | 68 +++++ .../export-headers-toolkit/go.sum | 254 ++++++++++++++++++ .../export-headers-toolkit/main.go | 9 + .../export-headers-toolkit/types/header.go | 61 +++++ 10 files changed, 891 insertions(+) create mode 100644 rollup/missing_header_fields/export-headers-toolkit/.gitignore create mode 100644 rollup/missing_header_fields/export-headers-toolkit/Dockerfile create mode 100644 rollup/missing_header_fields/export-headers-toolkit/README.md create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/root.go create mode 100644 rollup/missing_header_fields/export-headers-toolkit/go.mod create mode 100644 rollup/missing_header_fields/export-headers-toolkit/go.sum create mode 100644 rollup/missing_header_fields/export-headers-toolkit/main.go create mode 100644 rollup/missing_header_fields/export-headers-toolkit/types/header.go diff --git a/rollup/missing_header_fields/export-headers-toolkit/.gitignore b/rollup/missing_header_fields/export-headers-toolkit/.gitignore new file mode 100644 index 000000000000..adbb97d2d313 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/.gitignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/rollup/missing_header_fields/export-headers-toolkit/Dockerfile b/rollup/missing_header_fields/export-headers-toolkit/Dockerfile new file mode 100644 index 000000000000..04f729a05ef7 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.22 + +WORKDIR /app + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY . . + +RUN go build -o main . + +ENTRYPOINT ["./main"] \ No newline at end of file diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md new file mode 100644 index 000000000000..e7f1a7af6ae6 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -0,0 +1,64 @@ +# Export missing block header fields toolkit + +A toolkit for exporting and transforming missing block header fields of Scroll before {{upgrade_name}} TODO: replace when upgrade is clear. + +## Context +We are using the [Clique consensus](https://eips.ethereum.org/EIPS/eip-225) in Scroll L2. Amongst others, it requires the following header fields: +- `extraData` +- `difficulty` + +However, before {{upgrade_name}}, these fields were not stored on L1/DA. +In order for nodes to be able to reconstruct the correct block hashes when only reading data from L1, +we need to provide the historical values of these fields to these nodes through a separate file. + +This toolkit provides commands to export the missing fields, deduplicate the data and create a file +with the missing fields that can be used to reconstruct the correct block hashes when only reading data from L1. + +The toolkit provides the following commands: +- `fetch` - Fetch missing block header fields from a running Scroll L2 node and store in a file +- `dedup` - Deduplicate the headers file, print unique values and create a new file with the deduplicated headers + +## Binary layout deduplicated missing header fields file +The deduplicated header file binary layout is as follows: + +```plaintext +...... + +Where: +- unique_vanity_count: number of unique vanities n +- unique_vanity_i: unique vanity i +- header_i: block header i +- header: + + - vanity_index: index of the vanity in the unique vanity list + - flags: bitmask, lsb first + - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 1: 0 if seal length is 65, 1 if seal length is 85 + - rest: 0 +``` + +## How to run +Each of the commands has its own set of flags and options. To display the help message run with `--help` flag. + +1. Fetch the missing block header fields from a running Scroll L2 node via RPC and store in a file (approx 40min for 5.5M blocks). +2. Deduplicate the headers file, print unique values and create a new file with the deduplicated headers + +```bash +go run main.go fetch --rpc=http://localhost:8545 --start=0 --end=100 --batch=10 --parallelism=10 --output=headers.bin --humanOutput=true +go run main.go dedup --input=headers.bin --output=headers-dedup.bin +``` + + +### With Docker +To run the toolkit with Docker, build the Docker image and run the commands inside the container. + +```bash +docker build -t export-headers-toolkit . + +# depending on the Docker config maybe finding the RPC container's IP with docker inspect is necessary. Potentially host IP works: http://172.17.0.1:8545 +docker run --rm -v "$(pwd)":/app/result export-headers-toolkit fetch --rpc=
--start=0 --end=5422047 --batch=10000 --parallelism=10 --output=/app/result/headers.bin --humanOutput=/app/result/headers.csv +docker run --rm -v "$(pwd)":/app/result export-headers-toolkit dedup --input=/app/result/headers.bin --output=/app/result/headers-dedup.bin +``` + + + diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go new file mode 100644 index 000000000000..8b67788eef77 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -0,0 +1,143 @@ +package cmd + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "log" + "os" + + "github.com/spf13/cobra" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +// dedupCmd represents the dedup command +var dedupCmd = &cobra.Command{ + Use: "dedup", + Short: "Deduplicate the headers file, print unique values and create a new file with the deduplicated headers", + Long: `Deduplicate the headers file, print unique values and create a new file with the deduplicated headers. + +The binary layout of the deduplicated file is as follows: +- 1 byte for the count of unique vanity +- 32 bytes for each unique vanity +- for each header: + - 1 byte for the index of the vanity in the unique vanity list + - 1 byte (bitmask, lsb first): + - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 1: 0 if seal length is 65, 1 if seal length is 85 + - rest: 0 + - 65 or 85 bytes for the seal`, + Run: func(cmd *cobra.Command, args []string) { + inputFile, err := cmd.Flags().GetString("input") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + outputFile, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + + runDedup(inputFile, outputFile) + }, +} + +func init() { + rootCmd.AddCommand(dedupCmd) + + dedupCmd.Flags().String("input", "headers.bin", "headers file") + dedupCmd.Flags().String("output", "headers-dedup.bin", "deduplicated, binary formatted file") +} + +func runDedup(inputFile, outputFile string) { + reader := newHeaderReader(inputFile) + defer reader.close() + + // track header fields we've seen + seenDifficulty := make(map[uint64]bool) + seenVanity := make(map[[32]byte]bool) + seenSealLen := make(map[int]bool) + + reader.read(func(header *types.Header) { + seenDifficulty[header.Difficulty] = true + seenVanity[header.Vanity()] = true + seenSealLen[header.SealLen()] = true + }) + + // Print report + fmt.Println("--------------------------------------------------") + fmt.Printf("Unique values seen in the headers file (last seen block: %d):\n", reader.lastHeader.Number) + fmt.Printf("Distinct count: Difficulty:%d, Vanity:%d, SealLen:%d\n", len(seenDifficulty), len(seenVanity), len(seenSealLen)) + fmt.Printf("--------------------------------------------------\n\n") + + for diff := range seenDifficulty { + fmt.Printf("Difficulty: %d\n", diff) + } + + for vanity := range seenVanity { + fmt.Printf("Vanity: %x\n", vanity) + } + + for sealLen := range seenSealLen { + fmt.Printf("SealLen: %d\n", sealLen) + } +} + +type headerReader struct { + file *os.File + reader *bufio.Reader + lastHeader *types.Header +} + +func newHeaderReader(inputFile string) *headerReader { + f, err := os.Open(inputFile) + if err != nil { + log.Fatalf("Error opening input file: %v", err) + } + + h := &headerReader{ + file: f, + reader: bufio.NewReader(f), + } + + return h +} + +func (h *headerReader) read(callback func(header *types.Header)) { + headerSizeBytes := make([]byte, types.HeaderSizeSerialized) + + for { + _, err := io.ReadFull(h.reader, headerSizeBytes) + if err != nil { + if err == io.EOF { + break + } + log.Fatalf("Error reading headerSizeBytes: %v", err) + } + headerSize := binary.BigEndian.Uint16(headerSizeBytes) + + headerBytes := make([]byte, headerSize) + _, err = io.ReadFull(h.reader, headerBytes) + if err != nil { + if err == io.EOF { + break + } + log.Fatalf("Error reading headerBytes: %v", err) + } + header := new(types.Header).FromBytes(headerBytes) + + // sanity check: make sure headers are in order + if h.lastHeader != nil && header.Number != h.lastHeader.Number+1 { + fmt.Println("lastHeader:", h.lastHeader.String()) + log.Fatalf("Missing block: %d, got %d instead", h.lastHeader.Number+1, header.Number) + } + h.lastHeader = header + + callback(header) + } +} + +func (h *headerReader) close() { + h.file.Close() +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go new file mode 100644 index 000000000000..e799c40cdc45 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -0,0 +1,246 @@ +package cmd + +import ( + "bufio" + "container/heap" + "context" + "fmt" + "log" + "math/big" + "os" + "sync" + "time" + + "github.com/scroll-tech/go-ethereum/ethclient" + "github.com/spf13/cobra" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +var fetchCmd = &cobra.Command{ + Use: "fetch", + Short: "Fetch missing block header fields from a running Scroll L2 node via RPC and store in a file", + Long: `Fetch allows to retrieve the missing block header fields from a running Scroll L2 node via RPC. +It produces a binary file and optionally a human readable csv file with the missing fields.`, + Run: func(cmd *cobra.Command, args []string) { + rpc, err := cmd.Flags().GetString("rpc") + if err != nil { + log.Fatalf("Error reading rpc flag: %v", err) + } + client, err := ethclient.Dial(rpc) + if err != nil { + log.Fatalf("Error connecting to RPC: %v", err) + } + startBlockNum, err := cmd.Flags().GetUint64("start") + if err != nil { + log.Fatalf("Error reading start flag: %v", err) + } + endBlockNum, err := cmd.Flags().GetUint64("end") + if err != nil { + log.Fatalf("Error reading end flag: %v", err) + } + batchSize, err := cmd.Flags().GetUint64("batch") + if err != nil { + log.Fatalf("Error reading batch flag: %v", err) + } + maxParallelGoroutines, err := cmd.Flags().GetInt("parallelism") + if err != nil { + log.Fatalf("Error reading parallelism flag: %v", err) + } + outputFile, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatalf("Error reading output flag: %v", err) + } + humanReadableOutputFile, err := cmd.Flags().GetString("humanOutput") + if err != nil { + log.Fatalf("Error reading humanReadable flag: %v", err) + } + + runFetch(client, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile) + }, +} + +func init() { + rootCmd.AddCommand(fetchCmd) + + fetchCmd.Flags().String("rpc", "http://localhost:8545", "RPC URL") + fetchCmd.Flags().Uint64("start", 0, "start block number") + fetchCmd.Flags().Uint64("end", 1000, "end block number") + fetchCmd.Flags().Uint64("batch", 100, "batch size") + fetchCmd.Flags().Int("parallelism", 10, "max parallel goroutines each working on batch size blocks") + fetchCmd.Flags().String("output", "headers.bin", "output file") + fetchCmd.Flags().String("humanOutput", "", "additionally produce human readable csv file") +} + +func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetries int) (*types.Header, error) { + var innerErr error + for i := 0; i < maxRetries; i++ { + header, err := client.HeaderByNumber(context.Background(), big.NewInt(int64(blockNum))) + if err == nil { + return &types.Header{ + Number: header.Number.Uint64(), + Difficulty: header.Difficulty.Uint64(), + ExtraData: header.Extra, + }, nil + } + + innerErr = err // save the last error to return it if all retries fail + + // Wait before retrying + time.Sleep(time.Duration(i*200) * time.Millisecond) + log.Printf("Retrying header fetch for block %d, retry %d, error %v", blockNum, i+1, err) + } + + return nil, fmt.Errorf("error fetching header for block %d: %v", blockNum, innerErr) +} + +func fetchHeaders(client *ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { + for i := start; i <= end; i++ { + header, err := headerByNumberWithRetry(client, i, 15) + if err != nil { + log.Fatalf("Error fetching header %d: %v", i, err) + } + + headersChan <- header + } +} + +func writeHeadersToFile(outputFile string, humanReadableOutputFile string, startBlockNum uint64, headersChan <-chan *types.Header) { + writer := newFilesWriter(outputFile, humanReadableOutputFile) + defer writer.close() + + headerHeap := &types.HeaderHeap{} + heap.Init(headerHeap) + + nextHeaderNum := startBlockNum + + // receive all headers and write them in order by using a sorted heap + for header := range headersChan { + heap.Push(headerHeap, header) + + // write all headers that are in order + for headerHeap.Len() > 0 && (*headerHeap)[0].Number == nextHeaderNum { + nextHeaderNum++ + sortedHeader := heap.Pop(headerHeap).(*types.Header) + writer.write(sortedHeader) + } + } + + fmt.Println("Finished writing headers to file, last block number:", nextHeaderNum-1) +} + +func runFetch(client *ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string) { + headersChan := make(chan *types.Header, maxGoroutines*int(batchSize)) + tasks := make(chan task, maxGoroutines) + + var wgConsumer sync.WaitGroup + // start consumer goroutine to sort and write headers to file + go func() { + wgConsumer.Add(1) + writeHeadersToFile(outputFile, humanReadableOutputFile, startBlockNum, headersChan) + wgConsumer.Done() + }() + + var wgProducers sync.WaitGroup + // start producer goroutines to fetch headers + for i := 0; i < maxGoroutines; i++ { + wgProducers.Add(1) + go func() { + for { + t, ok := <-tasks + if !ok { + break + } + fetchHeaders(client, t.start, t.end, headersChan) + } + wgProducers.Done() + }() + } + + // create tasks/work packages for producer goroutines + for start := startBlockNum; start <= endBlockNum; start += batchSize { + end := start + batchSize - 1 + if end > endBlockNum { + end = endBlockNum + } + fmt.Println("Fetching headers for blocks", start, "to", end) + + tasks <- task{start, end} + } + + close(tasks) + wgProducers.Wait() + close(headersChan) + wgConsumer.Wait() +} + +type task struct { + start uint64 + end uint64 +} + +// filesWriter is a helper struct to write headers to binary and human-readable csv files at the same time. +type filesWriter struct { + binaryFile *os.File + binaryWriter *bufio.Writer + + humanReadable bool + csvFile *os.File + csvWriter *bufio.Writer +} + +func newFilesWriter(outputFile string, humanReadableOutputFile string) *filesWriter { + binaryFile, err := os.Create(outputFile) + if err != nil { + log.Fatalf("Error creating binary file: %v", err) + } + + f := &filesWriter{ + binaryFile: binaryFile, + binaryWriter: bufio.NewWriter(binaryFile), + humanReadable: humanReadableOutputFile != "", + } + + if humanReadableOutputFile != "" { + csvFile, err := os.Create(humanReadableOutputFile) + if err != nil { + log.Fatalf("Error creating human readable file: %v", err) + } + f.csvFile = csvFile + f.csvWriter = bufio.NewWriter(csvFile) + } + + return f +} + +func (f *filesWriter) close() { + if err := f.binaryWriter.Flush(); err != nil { + log.Fatalf("Error flushing binary buffer: %v", err) + } + if f.humanReadable { + if err := f.csvWriter.Flush(); err != nil { + log.Fatalf("Error flushing csv buffer: %v", err) + } + } + + f.binaryFile.Close() + if f.humanReadable { + f.csvFile.Close() + } +} +func (f *filesWriter) write(header *types.Header) { + bytes, err := header.Bytes() + if err != nil { + log.Fatalf("Error converting header to bytes: %v", err) + } + + if _, err = f.binaryWriter.Write(bytes); err != nil { + log.Fatalf("Error writing to binary file: %v", err) + } + + if f.humanReadable { + if _, err = f.csvWriter.WriteString(header.String()); err != nil { + log.Fatalf("Error writing to human readable file: %v", err) + } + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go new file mode 100644 index 000000000000..9b485e96dd62 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "export-headers-toolkit", + Short: "A toolkit for exporting and transforming missing block header fields of Scroll", + Long: `A toolkit for exporting and transforming missing block header fields of Scroll. + +The fields difficulty and extraData are missing from header data stored on L1 before {{upgrade_name}}. +This toolkit provides commands to export the missing fields, deduplicate the data and create a +file with the missing fields that can be used to reconstruct the correct block hashes when only reading +data from L1.`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.CompletionOptions.DisableDefaultCmd = true +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod new file mode 100644 index 000000000000..7a843d32dbdb --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -0,0 +1,68 @@ +module github.com/scroll-tech/go-ethereum/export-headers-toolkit + +go 1.22 + +require ( + github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 + github.com/spf13/cobra v1.8.1 +) + +require ( + github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/btcsuite/btcd v0.20.1-beta // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect + github.com/edsrzf/mmap-go v1.0.0 // indirect + github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect + github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/uuid v1.1.5 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.3.0 // indirect + github.com/huin/goupnp v1.0.2 // indirect + github.com/iden3/go-iden3-crypto v0.0.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/tsdb v0.7.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rjeczalik/notify v0.9.1 // indirect + github.com/rs/cors v1.7.0 // indirect + github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 // indirect + github.com/scroll-tech/zktrie v0.8.4 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect + github.com/supranational/blst v0.3.12 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + gopkg.in/urfave/cli.v1 v1.20.0 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum new file mode 100644 index 000000000000..77acbbbbcb79 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -0,0 +1,254 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= +github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= +github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/iden3/go-iden3-crypto v0.0.16 h1:zN867xiz6HgErXVIV/6WyteGcOukE9gybYTorBMEdsk= +github.com/iden3/go-iden3-crypto v0.0.16/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 h1:CDrPMqifvAVyYqu0x1J5qickVV0b51tApPnOwDYLESI= +github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM= +github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 h1:Jq4TTYcHVAIVPUHNYbOzNxEsWf+9Q3b30YQaMyl0TDI= +github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ= +github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= +github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/supranational/blst v0.3.12 h1:Vfas2U2CFHhniv2QkUm2OVa1+pGTdqtpqm9NnhUUbZ8= +github.com/supranational/blst v0.3.12/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/rollup/missing_header_fields/export-headers-toolkit/main.go b/rollup/missing_header_fields/export-headers-toolkit/main.go new file mode 100644 index 000000000000..1d8e22abe8cb --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go new file mode 100644 index 000000000000..e60828250b69 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -0,0 +1,61 @@ +package types + +import ( + "encoding/binary" + "fmt" +) + +const HeaderSizeSerialized = 2 + +type Header struct { + Number uint64 + Difficulty uint64 + ExtraData []byte +} + +func (h *Header) String() string { + return fmt.Sprintf("%d,%d,0x%x\n", h.Number, h.Difficulty, h.ExtraData) +} + +// Bytes returns the byte representation of the header including the initial 2 bytes for the size. +func (h *Header) Bytes() ([]byte, error) { + size := 8 + 8 + len(h.ExtraData) + + buf := make([]byte, HeaderSizeSerialized+size) + binary.BigEndian.PutUint16(buf[:2], uint16(size)) + binary.BigEndian.PutUint64(buf[2:10], h.Number) + binary.BigEndian.PutUint64(buf[10:18], h.Difficulty) + copy(buf[18:], h.ExtraData) + return buf, nil +} + +func (h *Header) Vanity() [32]byte { + return [32]byte(h.ExtraData[:32]) +} + +func (h *Header) SealLen() int { + return len(h.ExtraData[32:]) +} + +// FromBytes reads the header from the byte representation excluding the initial 2 bytes for the size. +func (h *Header) FromBytes(buf []byte) *Header { + h.Number = binary.BigEndian.Uint64(buf[:8]) + h.Difficulty = binary.BigEndian.Uint64(buf[8:16]) + h.ExtraData = buf[16:] + + return h +} + +type HeaderHeap []*Header + +func (h HeaderHeap) Len() int { return len(h) } +func (h HeaderHeap) Less(i, j int) bool { return h[i].Number < h[j].Number } +func (h HeaderHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *HeaderHeap) Push(x interface{}) { *h = append(*h, x.(*Header)) } +func (h *HeaderHeap) Pop() interface{} { + old := *h + n := len(old) + item := old[n-1] + *h = old[0 : n-1] + return item +} From 5222fda7a94d5dda1721d839b408a066927aa93e Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Tue, 16 Jul 2024 08:19:35 +0800 Subject: [PATCH 02/17] feat: add missing header writer to write the missing header file with deduplicated headers --- .../export-headers-toolkit/cmd/dedup.go | 52 +++++--- .../cmd/missing_header_writer.go | 113 ++++++++++++++++++ .../cmd/missing_header_writer_test.go | 101 ++++++++++++++++ .../export-headers-toolkit/go.mod | 3 + .../export-headers-toolkit/go.sum | 8 ++ .../export-headers-toolkit/types/header.go | 19 ++- 6 files changed, 277 insertions(+), 19 deletions(-) create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 8b67788eef77..e759e877bd8f 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -39,7 +39,8 @@ The binary layout of the deduplicated file is as follows: log.Fatalf("Error reading output flag: %v", err) } - runDedup(inputFile, outputFile) + seenDifficulty, seenVanity, seenSealLen := runAnalysis(inputFile) + runDedup(inputFile, outputFile, seenDifficulty, seenVanity, seenSealLen) }, } @@ -50,38 +51,57 @@ func init() { dedupCmd.Flags().String("output", "headers-dedup.bin", "deduplicated, binary formatted file") } -func runDedup(inputFile, outputFile string) { +func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) { reader := newHeaderReader(inputFile) defer reader.close() // track header fields we've seen - seenDifficulty := make(map[uint64]bool) - seenVanity := make(map[[32]byte]bool) - seenSealLen := make(map[int]bool) + seenDifficulty = make(map[uint64]int) + seenVanity = make(map[[32]byte]bool) + seenSealLen = make(map[int]int) reader.read(func(header *types.Header) { - seenDifficulty[header.Difficulty] = true + seenDifficulty[header.Difficulty]++ seenVanity[header.Vanity()] = true - seenSealLen[header.SealLen()] = true + seenSealLen[header.SealLen()]++ }) - // Print report + // Print distinct values and report fmt.Println("--------------------------------------------------") - fmt.Printf("Unique values seen in the headers file (last seen block: %d):\n", reader.lastHeader.Number) - fmt.Printf("Distinct count: Difficulty:%d, Vanity:%d, SealLen:%d\n", len(seenDifficulty), len(seenVanity), len(seenSealLen)) - fmt.Printf("--------------------------------------------------\n\n") - - for diff := range seenDifficulty { - fmt.Printf("Difficulty: %d\n", diff) + for diff, count := range seenDifficulty { + fmt.Printf("Difficulty %d: %d\n", diff, count) } for vanity := range seenVanity { fmt.Printf("Vanity: %x\n", vanity) } - for sealLen := range seenSealLen { - fmt.Printf("SealLen: %d\n", sealLen) + for sealLen, count := range seenSealLen { + fmt.Printf("SealLen %d bytes: %d\n", sealLen, count) } + + fmt.Println("--------------------------------------------------") + fmt.Printf("Unique values seen in the headers file (last seen block: %d):\n", reader.lastHeader.Number) + fmt.Printf("Distinct count: Difficulty:%d, Vanity:%d, SealLen:%d\n", len(seenDifficulty), len(seenVanity), len(seenSealLen)) + fmt.Printf("--------------------------------------------------\n\n") + + return seenDifficulty, seenVanity, seenSealLen +} + +func runDedup(inputFile, outputFile string, seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) { + reader := newHeaderReader(inputFile) + defer reader.close() + + writer := newMissingHeaderFileWriter(outputFile, seenVanity) + writer.close() + + writer.missingHeaderWriter.writeVanities() + + reader.read(func(header *types.Header) { + writer.missingHeaderWriter.write(header) + }) + + fmt.Printf("Deduplicated headers written to %s\n", outputFile) } type headerReader struct { diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go new file mode 100644 index 000000000000..29524ab7247f --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -0,0 +1,113 @@ +package cmd + +import ( + "bufio" + "bytes" + "io" + "log" + "os" + "sort" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +type missingHeaderFileWriter struct { + file *os.File + writer *bufio.Writer + + missingHeaderWriter *missingHeaderWriter +} + +func newMissingHeaderFileWriter(filename string, seenVanity map[[32]byte]bool) *missingHeaderFileWriter { + file, err := os.Create(filename) + if err != nil { + log.Fatalf("Error creating file: %v", err) + } + + writer := bufio.NewWriter(file) + return &missingHeaderFileWriter{ + file: file, + writer: writer, + missingHeaderWriter: newMissingHeaderWriter(writer, seenVanity), + } +} + +func (m *missingHeaderFileWriter) close() { + if err := m.writer.Flush(); err != nil { + log.Fatalf("Error flushing writer: %v", err) + } + if err := m.file.Close(); err != nil { + log.Fatalf("Error closing file: %v", err) + } +} + +type missingHeaderWriter struct { + writer io.Writer + + sortedVanities [][32]byte + sortedVanitiesMap map[[32]byte]int + seenDifficulty map[uint64]int + seenSealLen map[int]int +} + +func newMissingHeaderWriter(writer io.Writer, seenVanity map[[32]byte]bool) *missingHeaderWriter { + // sort the vanities and assign an index to each so that we can write the index of the vanity in the header + sortedVanities := make([][32]byte, 0, len(seenVanity)) + for vanity := range seenVanity { + sortedVanities = append(sortedVanities, vanity) + } + sort.Slice(sortedVanities, func(i, j int) bool { + return bytes.Compare(sortedVanities[i][:], sortedVanities[j][:]) < 0 + }) + sortedVanitiesMap := make(map[[32]byte]int) + for i, vanity := range sortedVanities { + sortedVanitiesMap[vanity] = i + } + + return &missingHeaderWriter{ + writer: writer, + sortedVanities: sortedVanities, + sortedVanitiesMap: sortedVanitiesMap, + } +} + +func (m *missingHeaderWriter) writeVanities() { + // write the count of unique vanities + if _, err := m.writer.Write([]byte{uint8(len(m.sortedVanitiesMap))}); err != nil { + log.Fatalf("Error writing unique vanity count: %v", err) + } + + // write the unique vanities + for _, vanity := range m.sortedVanities { + if _, err := m.writer.Write(vanity[:]); err != nil { + log.Fatalf("Error writing vanity: %v", err) + } + } +} + +func (m *missingHeaderWriter) write(header *types.Header) { + // 1. write the index of the vanity in the unique vanity list + if _, err := m.writer.Write([]byte{uint8(m.sortedVanitiesMap[header.Vanity()])}); err != nil { + log.Fatalf("Error writing vanity index: %v", err) + } + + // 2. write the bitmask + // - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 + // - bit 1: 0 if seal length is 65, 1 if seal length is 85 + // - rest: 0 + bitmask := uint8(0) + if header.Difficulty == 1 { + bitmask |= 1 << 0 + } + if header.SealLen() == 85 { + bitmask |= 1 << 1 + } + + if _, err := m.writer.Write([]byte{bitmask}); err != nil { + log.Fatalf("Error writing bitmask: %v", err) + } + + if _, err := m.writer.Write(header.Seal()); err != nil { + log.Fatalf("Error writing seal: %v", err) + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go new file mode 100644 index 000000000000..f145a34089b5 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" +) + +func TestMissingHeaderWriter(t *testing.T) { + vanity1 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} + vanity2 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02} + vanity8 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08} + + var expectedBytes []byte + expectedBytes = append(expectedBytes, 0x03) + expectedBytes = append(expectedBytes, vanity1[:]...) + expectedBytes = append(expectedBytes, vanity2[:]...) + expectedBytes = append(expectedBytes, vanity8[:]...) + + seenVanity := map[[32]byte]bool{ + vanity8: true, + vanity1: true, + vanity2: true, + } + var buf []byte + bytesBuffer := bytes.NewBuffer(buf) + mhw := newMissingHeaderWriter(bytesBuffer, seenVanity) + + mhw.writeVanities() + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + + // Header0 + { + seal := randomSeal(65) + header := types.NewHeader(0, 2, append(vanity1[:], seal...)) + mhw.write(header) + + expectedBytes = append(expectedBytes, 0x00) // index 0 + expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65 + expectedBytes = append(expectedBytes, seal...) + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header1 + { + seal := randomSeal(65) + header := types.NewHeader(1, 1, append(vanity2[:], seal...)) + mhw.write(header) + + expectedBytes = append(expectedBytes, 0x01) // index 1 + expectedBytes = append(expectedBytes, 0x01) // difficulty 1, seal length 65 + expectedBytes = append(expectedBytes, seal...) + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header2 + { + seal := randomSeal(85) + header := types.NewHeader(2, 2, append(vanity2[:], seal...)) + mhw.write(header) + + expectedBytes = append(expectedBytes, 0x01) // index 1 + expectedBytes = append(expectedBytes, 0x02) // difficulty 2, seal length 85 + expectedBytes = append(expectedBytes, seal...) + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header3 + { + seal := randomSeal(85) + header := types.NewHeader(3, 1, append(vanity8[:], seal...)) + mhw.write(header) + + expectedBytes = append(expectedBytes, 0x02) // index 2 + expectedBytes = append(expectedBytes, 0x03) // difficulty 1, seal length 85 + expectedBytes = append(expectedBytes, seal...) + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } + + // Header4 + { + seal := randomSeal(65) + header := types.NewHeader(4, 2, append(vanity1[:], seal...)) + mhw.write(header) + + expectedBytes = append(expectedBytes, 0x00) // index 0 + expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65 + expectedBytes = append(expectedBytes, seal...) + assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + } +} + +func randomSeal(length int) []byte { + buf := make([]byte, length) + _, _ = rand.Read(buf) + return buf +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index 7a843d32dbdb..97806cc8252f 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -5,6 +5,7 @@ go 1.22 require ( github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 ) require ( @@ -42,6 +43,7 @@ require ( github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/tsdb v0.7.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rjeczalik/notify v0.9.1 // indirect @@ -64,5 +66,6 @@ require ( golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 77acbbbbcb79..c0fc07b58407 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -100,6 +100,10 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= @@ -147,6 +151,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -235,6 +241,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go index e60828250b69..80fcc6210fcf 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/header.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -6,6 +6,7 @@ import ( ) const HeaderSizeSerialized = 2 +const VanitySize = 32 type Header struct { Number uint64 @@ -13,6 +14,14 @@ type Header struct { ExtraData []byte } +func NewHeader(number, difficulty uint64, extraData []byte) *Header { + return &Header{ + Number: number, + Difficulty: difficulty, + ExtraData: extraData, + } +} + func (h *Header) String() string { return fmt.Sprintf("%d,%d,0x%x\n", h.Number, h.Difficulty, h.ExtraData) } @@ -29,12 +38,16 @@ func (h *Header) Bytes() ([]byte, error) { return buf, nil } -func (h *Header) Vanity() [32]byte { - return [32]byte(h.ExtraData[:32]) +func (h *Header) Vanity() [VanitySize]byte { + return [VanitySize]byte(h.ExtraData[:VanitySize]) +} + +func (h *Header) Seal() []byte { + return h.ExtraData[VanitySize:] } func (h *Header) SealLen() int { - return len(h.ExtraData[32:]) + return len(h.Seal()) } // FromBytes reads the header from the byte representation excluding the initial 2 bytes for the size. From 2b5c450a06da2ffe6084b292abfd5be8580d5df0 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:46:32 +0800 Subject: [PATCH 03/17] feat: add sha256 checksum generation --- .../export-headers-toolkit/cmd/dedup.go | 21 ++++++++++++++++--- .../export-headers-toolkit/cmd/fetch.go | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index e759e877bd8f..68ba381f98cd 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -2,6 +2,7 @@ package cmd import ( "bufio" + "crypto/sha256" "encoding/binary" "fmt" "io" @@ -41,9 +42,25 @@ The binary layout of the deduplicated file is as follows: seenDifficulty, seenVanity, seenSealLen := runAnalysis(inputFile) runDedup(inputFile, outputFile, seenDifficulty, seenVanity, seenSealLen) + runSHA256(outputFile) }, } +func runSHA256(outputFile string) { + f, err := os.Open(outputFile) + defer f.Close() + if err != nil { + log.Fatalf("Error opening file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + log.Fatalf("Error hashing file: %v", err) + } + + fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil)) +} + func init() { rootCmd.AddCommand(dedupCmd) @@ -93,15 +110,13 @@ func runDedup(inputFile, outputFile string, seenDifficulty map[uint64]int, seenV defer reader.close() writer := newMissingHeaderFileWriter(outputFile, seenVanity) - writer.close() + defer writer.close() writer.missingHeaderWriter.writeVanities() reader.read(func(header *types.Header) { writer.missingHeaderWriter.write(header) }) - - fmt.Printf("Deduplicated headers written to %s\n", outputFile) } type headerReader struct { diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index e799c40cdc45..b9799d12ba29 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -11,9 +11,10 @@ import ( "sync" "time" - "github.com/scroll-tech/go-ethereum/ethclient" "github.com/spf13/cobra" + "github.com/scroll-tech/go-ethereum/ethclient" + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) From 9eaa720f3a1a082da3d54d0cb4f6b9f239515bdd Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:01:34 +0800 Subject: [PATCH 04/17] add bitmask to encapsulate logic around header information in it. also add vanity count as part of the bitmask --- .../export-headers-toolkit/README.md | 9 +- .../export-headers-toolkit/cmd/dedup.go | 13 ++- .../cmd/missing_header_writer.go | 84 +++++++++++++++---- .../cmd/missing_header_writer_test.go | 20 ++--- 4 files changed, 88 insertions(+), 38 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index e7f1a7af6ae6..339bcada3d59 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -29,12 +29,11 @@ Where: - unique_vanity_i: unique vanity i - header_i: block header i - header: - - - vanity_index: index of the vanity in the unique vanity list + - flags: bitmask, lsb first - - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 - - bit 1: 0 if seal length is 65, 1 if seal length is 85 - - rest: 0 + - bit 0-5: index of the vanity in the sorted vanities list + - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 7: 0 if seal length is 65, 1 if seal length is 85 ``` ## How to run diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 68ba381f98cd..7ab1c10723ee 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -24,11 +24,10 @@ The binary layout of the deduplicated file is as follows: - 1 byte for the count of unique vanity - 32 bytes for each unique vanity - for each header: - - 1 byte for the index of the vanity in the unique vanity list - 1 byte (bitmask, lsb first): - - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 - - bit 1: 0 if seal length is 65, 1 if seal length is 85 - - rest: 0 + - bit 0-5: index of the vanity in the sorted vanities list + - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 7: 0 if seal length is 65, 1 if seal length is 85 - 65 or 85 bytes for the seal`, Run: func(cmd *cobra.Command, args []string) { inputFile, err := cmd.Flags().GetString("input") @@ -40,8 +39,8 @@ The binary layout of the deduplicated file is as follows: log.Fatalf("Error reading output flag: %v", err) } - seenDifficulty, seenVanity, seenSealLen := runAnalysis(inputFile) - runDedup(inputFile, outputFile, seenDifficulty, seenVanity, seenSealLen) + _, seenVanity, _ := runAnalysis(inputFile) + runDedup(inputFile, outputFile, seenVanity) runSHA256(outputFile) }, } @@ -105,7 +104,7 @@ func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity ma return seenDifficulty, seenVanity, seenSealLen } -func runDedup(inputFile, outputFile string, seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) { +func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) { reader := newHeaderReader(inputFile) defer reader.close() diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go index 29524ab7247f..bc9831d672d2 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -11,6 +11,9 @@ import ( "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) +// maxVanityCount is the maximum number of unique vanities that can be represented with 6 bits in the bitmask +const maxVanityCount = 1 << 6 + type missingHeaderFileWriter struct { file *os.File writer *bufio.Writer @@ -19,6 +22,10 @@ type missingHeaderFileWriter struct { } func newMissingHeaderFileWriter(filename string, seenVanity map[[32]byte]bool) *missingHeaderFileWriter { + if len(seenVanity) > maxVanityCount { + log.Fatalf("Number of unique vanities exceeds maximum: %d > %d", len(seenVanity), maxVanityCount) + } + file, err := os.Create(filename) if err != nil { log.Fatalf("Error creating file: %v", err) @@ -86,28 +93,73 @@ func (m *missingHeaderWriter) writeVanities() { } func (m *missingHeaderWriter) write(header *types.Header) { - // 1. write the index of the vanity in the unique vanity list - if _, err := m.writer.Write([]byte{uint8(m.sortedVanitiesMap[header.Vanity()])}); err != nil { - log.Fatalf("Error writing vanity index: %v", err) + // 1. prepare the bitmask + bits := newBitMask(m.sortedVanitiesMap[header.Vanity()], int(header.Difficulty), header.SealLen()) + + // 2. write the header: bitmask and seal + if _, err := m.writer.Write(bits.Bytes()); err != nil { + log.Fatalf("Error writing bitmask: %v", err) } - // 2. write the bitmask - // - bit 0: 0 if difficulty is 2, 1 if difficulty is 1 - // - bit 1: 0 if seal length is 65, 1 if seal length is 85 - // - rest: 0 - bitmask := uint8(0) - if header.Difficulty == 1 { - bitmask |= 1 << 0 + if _, err := m.writer.Write(header.Seal()); err != nil { + log.Fatalf("Error writing seal: %v", err) } - if header.SealLen() == 85 { - bitmask |= 1 << 1 +} + +// bitMask is a bitmask that encodes the following information: +// +// bit 0-5: index of the vanity in the sorted vanities list +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask { + b := uint8(0) + + if vanityIndex >= maxVanityCount { + log.Fatalf("Vanity index exceeds maximum: %d >= %d", vanityIndex, maxVanityCount) } + b |= uint8(vanityIndex) & 0b00111111 - if _, err := m.writer.Write([]byte{bitmask}); err != nil { - log.Fatalf("Error writing bitmask: %v", err) + if difficulty == 1 { + b |= 1 << 6 + } else if difficulty != 2 { + log.Fatalf("Invalid difficulty: %d", difficulty) } - if _, err := m.writer.Write(header.Seal()); err != nil { - log.Fatalf("Error writing seal: %v", err) + if sealLen == 85 { + b |= 1 << 7 + } else if sealLen != 65 { + log.Fatalf("Invalid seal length: %d", sealLen) } + + return bitMask{b} +} + +func (b bitMask) vanityIndex() int { + return int(b.b & 0b00111111) +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index f145a34089b5..25c5d039dd93 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -39,8 +39,8 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(0, 2, append(vanity1[:], seal...)) mhw.write(header) - expectedBytes = append(expectedBytes, 0x00) // index 0 - expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65 + // bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b00000000) expectedBytes = append(expectedBytes, seal...) assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -51,8 +51,8 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(1, 1, append(vanity2[:], seal...)) mhw.write(header) - expectedBytes = append(expectedBytes, 0x01) // index 1 - expectedBytes = append(expectedBytes, 0x01) // difficulty 1, seal length 65 + // bit 0-5:0x1 index1, bit 6=1: difficulty 1, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b01000001) expectedBytes = append(expectedBytes, seal...) assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -63,8 +63,8 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(2, 2, append(vanity2[:], seal...)) mhw.write(header) - expectedBytes = append(expectedBytes, 0x01) // index 1 - expectedBytes = append(expectedBytes, 0x02) // difficulty 2, seal length 85 + // bit 0-5:0x1 index1, bit 6=0: difficulty 2, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b10000001) expectedBytes = append(expectedBytes, seal...) assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -75,8 +75,8 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(3, 1, append(vanity8[:], seal...)) mhw.write(header) - expectedBytes = append(expectedBytes, 0x02) // index 2 - expectedBytes = append(expectedBytes, 0x03) // difficulty 1, seal length 85 + // bit 0-5:0x2 index2, bit 6=1: difficulty 1, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b11000010) expectedBytes = append(expectedBytes, seal...) assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -87,8 +87,8 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(4, 2, append(vanity1[:], seal...)) mhw.write(header) - expectedBytes = append(expectedBytes, 0x00) // index 0 - expectedBytes = append(expectedBytes, 0x00) // difficulty 2, seal length 65 + // bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b00000000) expectedBytes = append(expectedBytes, seal...) assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) } From dcdd6f05377fdd13447a03d48eaf89228f22d376 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:31:19 +0800 Subject: [PATCH 05/17] feat: add optional verification of input and output file with separate csv file --- .../export-headers-toolkit/cmd/dedup.go | 149 ++++++++++++++++-- .../cmd/missing_header_reader.go | 120 ++++++++++++++ .../cmd/missing_header_writer.go | 4 + .../export-headers-toolkit/types/header.go | 22 ++- 4 files changed, 279 insertions(+), 16 deletions(-) create mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 7ab1c10723ee..e2bbd7ee4ddd 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -2,15 +2,20 @@ package cmd import ( "bufio" + "bytes" "crypto/sha256" "encoding/binary" "fmt" "io" "log" "os" + "strconv" + "strings" "github.com/spf13/cobra" + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) @@ -38,26 +43,24 @@ The binary layout of the deduplicated file is as follows: if err != nil { log.Fatalf("Error reading output flag: %v", err) } + verifyFile, err := cmd.Flags().GetString("verify") + if err != nil { + log.Fatalf("Error reading verify flag: %v", err) + } + + if verifyFile != "" { + verifyInputFile(verifyFile, inputFile) + } _, seenVanity, _ := runAnalysis(inputFile) runDedup(inputFile, outputFile, seenVanity) - runSHA256(outputFile) - }, -} -func runSHA256(outputFile string) { - f, err := os.Open(outputFile) - defer f.Close() - if err != nil { - log.Fatalf("Error opening file: %v", err) - } - - h := sha256.New() - if _, err = io.Copy(h, f); err != nil { - log.Fatalf("Error hashing file: %v", err) - } + if verifyFile != "" { + verifyOutputFile(verifyFile, outputFile) + } - fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil)) + runSHA256(outputFile) + }, } func init() { @@ -65,6 +68,7 @@ func init() { dedupCmd.Flags().String("input", "headers.bin", "headers file") dedupCmd.Flags().String("output", "headers-dedup.bin", "deduplicated, binary formatted file") + dedupCmd.Flags().String("verify", "", "verify the input and output files with the given .csv file") } func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) { @@ -118,6 +122,21 @@ func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) { }) } +func runSHA256(outputFile string) { + f, err := os.Open(outputFile) + defer f.Close() + if err != nil { + log.Fatalf("Error opening file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + log.Fatalf("Error hashing file: %v", err) + } + + fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil)) +} + type headerReader struct { file *os.File reader *bufio.Reader @@ -175,3 +194,103 @@ func (h *headerReader) read(callback func(header *types.Header)) { func (h *headerReader) close() { h.file.Close() } + +type csvHeaderReader struct { + file *os.File + reader *bufio.Reader +} + +func newCSVHeaderReader(verifyFile string) *csvHeaderReader { + f, err := os.Open(verifyFile) + if err != nil { + log.Fatalf("Error opening verify file: %v", err) + } + + h := &csvHeaderReader{ + file: f, + reader: bufio.NewReader(f), + } + + return h +} + +func (h *csvHeaderReader) readNext() *types.Header { + line, err := h.reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + log.Fatalf("Error reading line: %v", err) + } + + s := strings.Split(line, ",") + extraString := strings.Split(s[2], "\n") + + num, err := strconv.ParseUint(s[0], 10, 64) + if err != nil { + log.Fatalf("Error parsing block number: %v", err) + } + difficulty, err := strconv.ParseUint(s[1], 10, 64) + if err != nil { + log.Fatalf("Error parsing difficulty: %v", err) + } + extra := common.FromHex(extraString[0]) + + header := types.NewHeader(num, difficulty, extra) + return header +} + +func (h *csvHeaderReader) close() { + h.file.Close() +} + +func verifyInputFile(verifyFile, inputFile string) { + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + binaryReader := newHeaderReader(inputFile) + defer binaryReader.close() + + binaryReader.read(func(header *types.Header) { + csvHeader := csvReader.readNext() + + if !csvHeader.Equal(header) { + log.Fatalf("Header mismatch: %v != %v", csvHeader, header) + } + }) + + log.Printf("All headers match in %s and %s\n", verifyFile, inputFile) +} + +func verifyOutputFile(verifyFile, outputFile string) { + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + dedupReader, err := NewReader(outputFile) + if err != nil { + log.Fatalf("Error opening dedup file: %v", err) + } + defer dedupReader.Close() + + for { + header := csvReader.readNext() + if header == nil { + if _, _, err = dedupReader.ReadNext(); err == nil { + log.Fatalf("Expected EOF, got more headers") + } + break + } + + difficulty, extraData, err := dedupReader.Read(header.Number) + if err != nil { + log.Fatalf("Error reading header: %v", err) + } + + if header.Difficulty != difficulty { + log.Fatalf("Difficulty mismatch: headerNum %d: %d != %d", header.Number, header.Difficulty, difficulty) + } + if !bytes.Equal(header.ExtraData, extraData) { + log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData) + } + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go new file mode 100644 index 000000000000..773b4e06451c --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go @@ -0,0 +1,120 @@ +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" +) + +// TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit + +type missingHeader struct { + headerNum uint64 + difficulty uint64 + extraData []byte +} + +type Reader struct { + file *os.File + reader *bufio.Reader + sortedVanities map[int][32]byte + lastReadHeader *missingHeader +} + +func NewReader(filePath string) (*Reader, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + + r := &Reader{ + file: f, + reader: bufio.NewReader(f), + } + + // read the count of unique vanities + vanityCount, err := r.reader.ReadByte() + if err != nil { + return nil, err + } + + // read the unique vanities + r.sortedVanities = make(map[int][32]byte) + for i := uint8(0); i < vanityCount; i++ { + var vanity [32]byte + if _, err = r.reader.Read(vanity[:]); err != nil { + return nil, err + } + r.sortedVanities[int(i)] = vanity + } + + return r, nil +} + +func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) { + if r.lastReadHeader == nil { + if _, _, err = r.ReadNext(); err != nil { + return 0, nil, err + } + } + + if headerNum > r.lastReadHeader.headerNum { + // skip the headers until the requested header number + for i := r.lastReadHeader.headerNum; i < headerNum; i++ { + if _, _, err = r.ReadNext(); err != nil { + return 0, nil, err + } + } + } + + if headerNum == r.lastReadHeader.headerNum { + return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil + } + + // headerNum < r.lastReadHeader.headerNum is not supported + return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) +} + +func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { + // read the bitmask + bitmaskByte, err := r.reader.ReadByte() + if err != nil { + return 0, nil, fmt.Errorf("failed to read bitmask: %v", err) + } + + bits := newBitMaskFromByte(bitmaskByte) + + seal := make([]byte, bits.sealLen()) + + if _, err = io.ReadFull(r.reader, seal); err != nil { + return 0, nil, fmt.Errorf("failed to read seal: %v", err) + } + + // construct the extraData field + vanity := r.sortedVanities[bits.vanityIndex()] + var b bytes.Buffer + b.Write(vanity[:]) + b.Write(seal) + + // we don't have the header number, so we'll just increment the last read header number + // we assume that the headers are written in order, starting from 0 + if r.lastReadHeader == nil { + r.lastReadHeader = &missingHeader{ + headerNum: 0, + difficulty: uint64(bits.difficulty()), + extraData: b.Bytes(), + } + } else { + r.lastReadHeader.headerNum++ + r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.extraData = b.Bytes() + } + + return difficulty, b.Bytes(), nil +} + +func (r *Reader) Close() error { + return r.file.Close() +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go index bc9831d672d2..5b3eed1c2cbd 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -115,6 +115,10 @@ type bitMask struct { b uint8 } +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask { b := uint8(0) diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go index 80fcc6210fcf..ffd76c8a8bb5 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/header.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -3,6 +3,8 @@ package types import ( "encoding/binary" "fmt" + + "github.com/scroll-tech/go-ethereum/common" ) const HeaderSizeSerialized = 2 @@ -23,7 +25,25 @@ func NewHeader(number, difficulty uint64, extraData []byte) *Header { } func (h *Header) String() string { - return fmt.Sprintf("%d,%d,0x%x\n", h.Number, h.Difficulty, h.ExtraData) + return fmt.Sprintf("%d,%d,%s\n", h.Number, h.Difficulty, common.Bytes2Hex(h.ExtraData)) +} + +func (h *Header) Equal(other *Header) bool { + if h.Number != other.Number { + return false + } + if h.Difficulty != other.Difficulty { + return false + } + if len(h.ExtraData) != len(other.ExtraData) { + return false + } + for i, b := range h.ExtraData { + if b != other.ExtraData[i] { + return false + } + } + return true } // Bytes returns the byte representation of the header including the initial 2 bytes for the size. From c8ce9d907abf142b38533da2604632ab6c6df63a Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 28 May 2025 13:48:39 +0200 Subject: [PATCH 06/17] add feature to continue already existing header file --- .../export-headers-toolkit/README.md | 4 +- .../export-headers-toolkit/cmd/dedup.go | 6 +- .../export-headers-toolkit/cmd/fetch.go | 91 ++++++++++++++++--- .../export-headers-toolkit/cmd/root.go | 2 +- 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index 339bcada3d59..a5accd15ff10 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -1,13 +1,13 @@ # Export missing block header fields toolkit -A toolkit for exporting and transforming missing block header fields of Scroll before {{upgrade_name}} TODO: replace when upgrade is clear. +A toolkit for exporting and transforming missing block header fields of Scroll before EuclidV2 upgrade. ## Context We are using the [Clique consensus](https://eips.ethereum.org/EIPS/eip-225) in Scroll L2. Amongst others, it requires the following header fields: - `extraData` - `difficulty` -However, before {{upgrade_name}}, these fields were not stored on L1/DA. +However, before EuclidV2, these fields were not stored on L1/DA. In order for nodes to be able to reconstruct the correct block hashes when only reading data from L1, we need to provide the historical values of these fields to these nodes through a separate file. diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index e2bbd7ee4ddd..a35937b569d5 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -124,10 +124,10 @@ func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) { func runSHA256(outputFile string) { f, err := os.Open(outputFile) - defer f.Close() if err != nil { log.Fatalf("Error opening file: %v", err) } + defer f.Close() h := sha256.New() if _, err = io.Copy(h, f); err != nil { @@ -166,7 +166,7 @@ func (h *headerReader) read(callback func(header *types.Header)) { if err == io.EOF { break } - log.Fatalf("Error reading headerSizeBytes: %v", err) + log.Printf("Error reading headerSizeBytes: %v\n", err) } headerSize := binary.BigEndian.Uint16(headerSizeBytes) @@ -176,7 +176,7 @@ func (h *headerReader) read(callback func(header *types.Header)) { if err == io.EOF { break } - log.Fatalf("Error reading headerBytes: %v", err) + log.Printf("Error reading headerBytes: %v\n", err) } header := new(types.Header).FromBytes(headerBytes) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index b9799d12ba29..1ed4e3083602 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -7,7 +7,9 @@ import ( "fmt" "log" "math/big" + "math/rand" "os" + "strings" "sync" "time" @@ -24,13 +26,21 @@ var fetchCmd = &cobra.Command{ Long: `Fetch allows to retrieve the missing block header fields from a running Scroll L2 node via RPC. It produces a binary file and optionally a human readable csv file with the missing fields.`, Run: func(cmd *cobra.Command, args []string) { - rpc, err := cmd.Flags().GetString("rpc") + rpcs, err := cmd.Flags().GetString("rpc") if err != nil { log.Fatalf("Error reading rpc flag: %v", err) } - client, err := ethclient.Dial(rpc) - if err != nil { - log.Fatalf("Error connecting to RPC: %v", err) + rpcNodes := strings.Split(rpcs, ",") + if len(rpcNodes) == 0 { + log.Fatal("No RPC URLs provided, please use the --rpcs flag to specify at least one RPC URL.") + } + var clients []*ethclient.Client + for _, rpc := range rpcNodes { + client, err := ethclient.Dial(rpc) + if err != nil { + log.Fatalf("Error connecting to RPC: %v", err) + } + clients = append(clients, client) } startBlockNum, err := cmd.Flags().GetUint64("start") if err != nil { @@ -56,21 +66,43 @@ It produces a binary file and optionally a human readable csv file with the miss if err != nil { log.Fatalf("Error reading humanReadable flag: %v", err) } + continueFile, err := cmd.Flags().GetString("continue") + if err != nil { + log.Fatalf("Error reading continue flag: %v", err) + } + + if continueFile != "" { + fmt.Println("Continue fetching block header fields from", continueFile) + + reader := newHeaderReader(continueFile) + defer reader.close() + + var lastSeenHeader uint64 + reader.read(func(header *types.Header) { + lastSeenHeader = header.Number + }) + fmt.Println("Last Seen Header:", lastSeenHeader) + + startBlockNum = lastSeenHeader + 1 + + fmt.Println("Overriding start block number to:", startBlockNum) + } - runFetch(client, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile) + runFetch(clients, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile) }, } func init() { rootCmd.AddCommand(fetchCmd) - fetchCmd.Flags().String("rpc", "http://localhost:8545", "RPC URL") + fetchCmd.Flags().String("rpc", "http://localhost:8545,http://localhost:8546", "RPC URLs, separated by commas. Example: http://localhost:8545,http://localhost:8546") fetchCmd.Flags().Uint64("start", 0, "start block number") fetchCmd.Flags().Uint64("end", 1000, "end block number") fetchCmd.Flags().Uint64("batch", 100, "batch size") fetchCmd.Flags().Int("parallelism", 10, "max parallel goroutines each working on batch size blocks") fetchCmd.Flags().String("output", "headers.bin", "output file") fetchCmd.Flags().String("humanOutput", "", "additionally produce human readable csv file") + fetchCmd.Flags().String("continue", "", "continue fetching block header fields from the last seen block number in the specified continue file") } func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetries int) (*types.Header, error) { @@ -95,21 +127,53 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri return nil, fmt.Errorf("error fetching header for block %d: %v", blockNum, innerErr) } -func fetchHeaders(client *ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { +func fetchHeaders(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { + // randomize client selection to distribute load + r := uint64(rand.Int()) + + // log time taken for fetching headers + startTime := time.Now() + var fetchTimeAvg, writeTimeAvg time.Duration + for i := start; i <= end; i++ { + startTimeBlockFetch := time.Now() + client := clients[(r+i)%uint64(len(clients))] // round-robin client selection header, err := headerByNumberWithRetry(client, i, 15) if err != nil { log.Fatalf("Error fetching header %d: %v", i, err) } + fetchTimeAvg += time.Since(startTimeBlockFetch) + startTimeHeaderWrite := time.Now() headersChan <- header + writeTimeAvg += time.Since(startTimeHeaderWrite) } + totalDuration := time.Since(startTime) + + fetchTimeAvg = fetchTimeAvg / time.Duration(end-start+1) + writeTimeAvg = writeTimeAvg / time.Duration(end-start+1) + log.Printf("Fetched %d header in %s (avg=%s, wrote to channel in avg %s", end-start+1, totalDuration, fetchTimeAvg, writeTimeAvg) } -func writeHeadersToFile(outputFile string, humanReadableOutputFile string, startBlockNum uint64, headersChan <-chan *types.Header) { +func writeHeadersToFile(outputFile string, humanReadableOutputFile string, continueFile string, startBlockNum uint64, headersChan <-chan *types.Header) { writer := newFilesWriter(outputFile, humanReadableOutputFile) defer writer.close() + if continueFile != "" { + reader := newHeaderReader(continueFile) + + var lastSeenHeader uint64 + var totalHeaders uint64 + reader.read(func(header *types.Header) { + writer.write(header) + totalHeaders++ + lastSeenHeader = header.Number + }) + + fmt.Println("Copied ", totalHeaders, "headers from continue file, last seen block number:", lastSeenHeader) + reader.close() + } + headerHeap := &types.HeaderHeap{} heap.Init(headerHeap) @@ -130,15 +194,15 @@ func writeHeadersToFile(outputFile string, humanReadableOutputFile string, start fmt.Println("Finished writing headers to file, last block number:", nextHeaderNum-1) } -func runFetch(client *ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string) { +func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) { headersChan := make(chan *types.Header, maxGoroutines*int(batchSize)) tasks := make(chan task, maxGoroutines) var wgConsumer sync.WaitGroup // start consumer goroutine to sort and write headers to file + wgConsumer.Add(1) go func() { - wgConsumer.Add(1) - writeHeadersToFile(outputFile, humanReadableOutputFile, startBlockNum, headersChan) + writeHeadersToFile(outputFile, humanReadableOutputFile, continueFile, startBlockNum, headersChan) wgConsumer.Done() }() @@ -152,7 +216,8 @@ func runFetch(client *ethclient.Client, startBlockNum uint64, endBlockNum uint64 if !ok { break } - fetchHeaders(client, t.start, t.end, headersChan) + log.Println("Received task", t.start, "to", t.end) + fetchHeaders(clients, t.start, t.end, headersChan) } wgProducers.Done() }() @@ -164,7 +229,7 @@ func runFetch(client *ethclient.Client, startBlockNum uint64, endBlockNum uint64 if end > endBlockNum { end = endBlockNum } - fmt.Println("Fetching headers for blocks", start, "to", end) + fmt.Println("Created task for blocks", start, "to", end) tasks <- task{start, end} } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go index 9b485e96dd62..05c9dbb69b6e 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/root.go @@ -12,7 +12,7 @@ var rootCmd = &cobra.Command{ Short: "A toolkit for exporting and transforming missing block header fields of Scroll", Long: `A toolkit for exporting and transforming missing block header fields of Scroll. -The fields difficulty and extraData are missing from header data stored on L1 before {{upgrade_name}}. +The fields difficulty and extraData are missing from header data stored on L1 before EuclidV2. This toolkit provides commands to export the missing fields, deduplicate the data and create a file with the missing fields that can be used to reconstruct the correct block hashes when only reading data from L1.`, From b41eb8ed1ec8465e995a30b6234d7ecd6763ee16 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 28 May 2025 14:08:05 +0200 Subject: [PATCH 07/17] change byte layout of deduplicated file due to too many vanities (>64) in Sepolia data --- .../export-headers-toolkit/README.md | 3 +- .../export-headers-toolkit/cmd/dedup.go | 2 ++ .../cmd/missing_header_reader.go | 8 ++++- .../cmd/missing_header_writer.go | 30 +++++++++---------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index a5accd15ff10..0fc59681ba43 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -29,9 +29,8 @@ Where: - unique_vanity_i: unique vanity i - header_i: block header i - header: - + - flags: bitmask, lsb first - - bit 0-5: index of the vanity in the sorted vanities list - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 - bit 7: 0 if seal length is 65, 1 if seal length is 85 ``` diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index a35937b569d5..9e8caef38977 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -293,4 +293,6 @@ func verifyOutputFile(verifyFile, outputFile string) { log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData) } } + + log.Printf("All headers match in %s and %s\n", verifyFile, outputFile) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go index 773b4e06451c..78f9ca2b2200 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go @@ -86,6 +86,12 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { bits := newBitMaskFromByte(bitmaskByte) + // read the vanity index + vanityIndex, err := r.reader.ReadByte() + if err != nil { + return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) + } + seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { @@ -93,7 +99,7 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { } // construct the extraData field - vanity := r.sortedVanities[bits.vanityIndex()] + vanity := r.sortedVanities[int(vanityIndex)] var b bytes.Buffer b.Write(vanity[:]) b.Write(seal) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go index 5b3eed1c2cbd..1c779d62c43e 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -5,14 +5,15 @@ import ( "bytes" "io" "log" + "math" "os" "sort" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) -// maxVanityCount is the maximum number of unique vanities that can be represented with 6 bits in the bitmask -const maxVanityCount = 1 << 6 +// maxVanityCount is the maximum number of unique vanities that can be represented with a single byte. +const maxVanityCount = math.MaxUint8 type missingHeaderFileWriter struct { file *os.File @@ -94,13 +95,20 @@ func (m *missingHeaderWriter) writeVanities() { func (m *missingHeaderWriter) write(header *types.Header) { // 1. prepare the bitmask - bits := newBitMask(m.sortedVanitiesMap[header.Vanity()], int(header.Difficulty), header.SealLen()) + bits := newBitMask(int(header.Difficulty), header.SealLen()) + vanityIndex := m.sortedVanitiesMap[header.Vanity()] - // 2. write the header: bitmask and seal + if vanityIndex >= maxVanityCount { + log.Fatalf("Vanity index %d exceeds maximum allowed %d", vanityIndex, maxVanityCount-1) + } + + // 2. write the header: bitmask, vanity index and seal if _, err := m.writer.Write(bits.Bytes()); err != nil { log.Fatalf("Error writing bitmask: %v", err) } - + if _, err := m.writer.Write([]byte{uint8(vanityIndex)}); err != nil { + log.Fatalf("Error writing vanity index: %v", err) + } if _, err := m.writer.Write(header.Seal()); err != nil { log.Fatalf("Error writing seal: %v", err) } @@ -108,7 +116,6 @@ func (m *missingHeaderWriter) write(header *types.Header) { // bitMask is a bitmask that encodes the following information: // -// bit 0-5: index of the vanity in the sorted vanities list // bit 6: 0 if difficulty is 2, 1 if difficulty is 1 // bit 7: 0 if seal length is 65, 1 if seal length is 85 type bitMask struct { @@ -119,14 +126,9 @@ func newBitMaskFromByte(b uint8) bitMask { return bitMask{b} } -func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask { +func newBitMask(difficulty int, sealLen int) bitMask { b := uint8(0) - if vanityIndex >= maxVanityCount { - log.Fatalf("Vanity index exceeds maximum: %d >= %d", vanityIndex, maxVanityCount) - } - b |= uint8(vanityIndex) & 0b00111111 - if difficulty == 1 { b |= 1 << 6 } else if difficulty != 2 { @@ -142,10 +144,6 @@ func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask { return bitMask{b} } -func (b bitMask) vanityIndex() int { - return int(b.b & 0b00111111) -} - func (b bitMask) difficulty() int { val := (b.b >> 6) & 0x01 if val == 0 { From 042377e6d35c05f597a5054ec44615a5acb9fc5d Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 28 May 2025 15:39:21 +0200 Subject: [PATCH 08/17] fix some stuff --- .../export-headers-toolkit/README.md | 6 ++-- .../export-headers-toolkit/cmd/dedup.go | 5 ++- .../export-headers-toolkit/cmd/fetch.go | 10 ++++-- .../cmd/missing_header_writer_test.go | 32 +++++++++++-------- .../export-headers-toolkit/go.mod | 6 ++-- .../export-headers-toolkit/go.sum | 14 ++++---- 6 files changed, 44 insertions(+), 29 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index 0fc59681ba43..10741edbb497 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -31,8 +31,10 @@ Where: - header: - flags: bitmask, lsb first - - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 - - bit 7: 0 if seal length is 65, 1 if seal length is 85 + - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 + - bit 7: 0 if seal length is 65, 1 if seal length is 85 + - vanity_index: index of the vanity in the sorted vanities list (0-255) + - seal: 65 or 85 bytes of seal data ``` ## How to run diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 9e8caef38977..efc233040de8 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -223,7 +223,10 @@ func (h *csvHeaderReader) readNext() *types.Header { log.Fatalf("Error reading line: %v", err) } - s := strings.Split(line, ",") + s := strings.Split(strings.TrimSpace(line), ",") + if len(s) < 3 { + log.Fatalf("Malformed CSV line: %q", line) + } extraString := strings.Split(s[2], "\n") num, err := strconv.ParseUint(s[0], 10, 64) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index 1ed4e3083602..90b726a20d88 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -30,10 +30,10 @@ It produces a binary file and optionally a human readable csv file with the miss if err != nil { log.Fatalf("Error reading rpc flag: %v", err) } - rpcNodes := strings.Split(rpcs, ",") - if len(rpcNodes) == 0 { - log.Fatal("No RPC URLs provided, please use the --rpcs flag to specify at least one RPC URL.") + if rpcs == "" { + log.Fatal("No RPC URLs provided, please use the --rpc flag to specify at least one RPC URL.") } + rpcNodes := strings.Split(rpcs, ",") var clients []*ethclient.Client for _, rpc := range rpcNodes { client, err := ethclient.Dial(rpc) @@ -86,6 +86,10 @@ It produces a binary file and optionally a human readable csv file with the miss startBlockNum = lastSeenHeader + 1 fmt.Println("Overriding start block number to:", startBlockNum) + + if startBlockNum > endBlockNum { + log.Fatalf("Start block number %d exceeds end block number %d after continuing from file", startBlockNum, endBlockNum) + } } runFetch(clients, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index 25c5d039dd93..825f34b3fb7b 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) @@ -39,10 +40,11 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(0, 2, append(vanity1[:], seal...)) mhw.write(header) - // bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65 + // bit 6=0: difficulty 2, bit 7=0: seal length 65 expectedBytes = append(expectedBytes, 0b00000000) + expectedBytes = append(expectedBytes, 0x00) // vanity index0 expectedBytes = append(expectedBytes, seal...) - assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header1 @@ -51,10 +53,11 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(1, 1, append(vanity2[:], seal...)) mhw.write(header) - // bit 0-5:0x1 index1, bit 6=1: difficulty 1, bit 7=0: seal length 65 - expectedBytes = append(expectedBytes, 0b01000001) + // bit 6=1: difficulty 1, bit 7=0: seal length 65 + expectedBytes = append(expectedBytes, 0b01000000) + expectedBytes = append(expectedBytes, 0x01) // vanity index1 expectedBytes = append(expectedBytes, seal...) - assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header2 @@ -63,10 +66,11 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(2, 2, append(vanity2[:], seal...)) mhw.write(header) - // bit 0-5:0x1 index1, bit 6=0: difficulty 2, bit 7=1: seal length 85 - expectedBytes = append(expectedBytes, 0b10000001) + // bit 6=0: difficulty 2, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b10000000) + expectedBytes = append(expectedBytes, 0x01) // vanity index1 expectedBytes = append(expectedBytes, seal...) - assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header3 @@ -75,10 +79,11 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(3, 1, append(vanity8[:], seal...)) mhw.write(header) - // bit 0-5:0x2 index2, bit 6=1: difficulty 1, bit 7=1: seal length 85 - expectedBytes = append(expectedBytes, 0b11000010) + // bit 6=1: difficulty 1, bit 7=1: seal length 85 + expectedBytes = append(expectedBytes, 0b11000000) + expectedBytes = append(expectedBytes, 0x02) // vanity index2 expectedBytes = append(expectedBytes, seal...) - assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header4 @@ -87,10 +92,11 @@ func TestMissingHeaderWriter(t *testing.T) { header := types.NewHeader(4, 2, append(vanity1[:], seal...)) mhw.write(header) - // bit 0-5:0x0 index0, bit 6=0: difficulty 2, bit 7=0: seal length 65 + // bit 6=0: difficulty 2, bit 7=0: seal length 65 expectedBytes = append(expectedBytes, 0b00000000) + expectedBytes = append(expectedBytes, 0x00) // vanity index0 expectedBytes = append(expectedBytes, seal...) - assert.Equal(t, expectedBytes, bytesBuffer.Bytes()) + require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } } diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index 97806cc8252f..e465c8617f26 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -4,8 +4,8 @@ go 1.22 require ( github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 - github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.9.0 + github.com/spf13/cobra v1.9.1 + github.com/stretchr/testify v1.10.0 ) require ( @@ -51,7 +51,7 @@ require ( github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 // indirect github.com/scroll-tech/zktrie v0.8.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect github.com/supranational/blst v0.3.12 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index c0fc07b58407..5c194d359597 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -27,7 +27,7 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -165,18 +165,18 @@ github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ7 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.12 h1:Vfas2U2CFHhniv2QkUm2OVa1+pGTdqtpqm9NnhUUbZ8= github.com/supranational/blst v0.3.12/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= From 12fdb4a1fc94f0ae3709e69a677e99392b70ec3f Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 19:42:31 +0200 Subject: [PATCH 09/17] add state root to header --- .../export-headers-toolkit/README.md | 7 +++++-- .../export-headers-toolkit/cmd/dedup.go | 15 +++++++++----- .../export-headers-toolkit/cmd/fetch.go | 11 +++++----- .../cmd/missing_header_reader.go | 20 ++++++++++++++----- .../cmd/missing_header_writer.go | 3 +++ .../cmd/missing_header_writer_test.go | 20 ++++++++++++++----- .../export-headers-toolkit/types/header.go | 20 ++++++++++++++----- 7 files changed, 69 insertions(+), 27 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index 10741edbb497..2148c2568565 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -9,7 +9,9 @@ We are using the [Clique consensus](https://eips.ethereum.org/EIPS/eip-225) in S However, before EuclidV2, these fields were not stored on L1/DA. In order for nodes to be able to reconstruct the correct block hashes when only reading data from L1, -we need to provide the historical values of these fields to these nodes through a separate file. +we need to provide the historical values of these fields to these nodes through a separate file. +Additionally, the `stateRoot` field is included in the file to ensure that the block headers can be reconstructed correctly, +independently of the state trie type used in the node (before EuclidV1 the state trie was ZK trie, after EuclidV1 it is a regular Merkle Patricia Trie). This toolkit provides commands to export the missing fields, deduplicate the data and create a file with the missing fields that can be used to reconstruct the correct block hashes when only reading data from L1. @@ -29,11 +31,12 @@ Where: - unique_vanity_i: unique vanity i - header_i: block header i - header: - + - flags: bitmask, lsb first - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 - bit 7: 0 if seal length is 65, 1 if seal length is 85 - vanity_index: index of the vanity in the sorted vanities list (0-255) + - state_root: 32 bytes of state root data - seal: 65 or 85 bytes of seal data ``` diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index efc233040de8..f22daef8d3fe 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -224,10 +224,9 @@ func (h *csvHeaderReader) readNext() *types.Header { } s := strings.Split(strings.TrimSpace(line), ",") - if len(s) < 3 { + if len(s) != 4 { log.Fatalf("Malformed CSV line: %q", line) } - extraString := strings.Split(s[2], "\n") num, err := strconv.ParseUint(s[0], 10, 64) if err != nil { @@ -237,9 +236,12 @@ func (h *csvHeaderReader) readNext() *types.Header { if err != nil { log.Fatalf("Error parsing difficulty: %v", err) } - extra := common.FromHex(extraString[0]) - header := types.NewHeader(num, difficulty, extra) + stateRoot := common.HexToHash(s[2]) + + extra := common.FromHex(strings.Split(s[3], "\n")[0]) + + header := types.NewHeader(num, difficulty, stateRoot, extra) return header } @@ -284,7 +286,7 @@ func verifyOutputFile(verifyFile, outputFile string) { break } - difficulty, extraData, err := dedupReader.Read(header.Number) + difficulty, stateRoot, extraData, err := dedupReader.Read(header.Number) if err != nil { log.Fatalf("Error reading header: %v", err) } @@ -292,6 +294,9 @@ func verifyOutputFile(verifyFile, outputFile string) { if header.Difficulty != difficulty { log.Fatalf("Difficulty mismatch: headerNum %d: %d != %d", header.Number, header.Difficulty, difficulty) } + if header.StateRoot != stateRoot { + log.Fatalf("StateRoot mismatch: headerNum %d: %s != %s", header.Number, header.StateRoot, stateRoot) + } if !bytes.Equal(header.ExtraData, extraData) { log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index 90b726a20d88..2c25a2d3f7ce 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -114,11 +114,12 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri for i := 0; i < maxRetries; i++ { header, err := client.HeaderByNumber(context.Background(), big.NewInt(int64(blockNum))) if err == nil { - return &types.Header{ - Number: header.Number.Uint64(), - Difficulty: header.Difficulty.Uint64(), - ExtraData: header.Extra, - }, nil + return types.NewHeader( + header.Number.Uint64(), + header.Difficulty.Uint64(), + header.Root, + header.Extra, + ), nil } innerErr = err // save the last error to return it if all retries fail diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go index 78f9ca2b2200..c4d86ab1d8e7 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "os" + + "github.com/scroll-tech/go-ethereum/common" ) // TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit @@ -13,6 +15,7 @@ import ( type missingHeader struct { headerNum uint64 difficulty uint64 + stateRoot common.Hash extraData []byte } @@ -53,10 +56,10 @@ func NewReader(filePath string) (*Reader, error) { return r, nil } -func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) { +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { if r.lastReadHeader == nil { if _, _, err = r.ReadNext(); err != nil { - return 0, nil, err + return 0, common.Hash{}, nil, err } } @@ -64,17 +67,17 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, er // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { if _, _, err = r.ReadNext(); err != nil { - return 0, nil, err + return 0, common.Hash{}, nil, err } } } if headerNum == r.lastReadHeader.headerNum { - return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil } // headerNum < r.lastReadHeader.headerNum is not supported - return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) } func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { @@ -92,6 +95,11 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) } + var stateRoot common.Hash + if _, err := io.ReadFull(r.reader, stateRoot[:]); err != nil { + return 0, nil, fmt.Errorf("failed to read state root: %v", err) + } + seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { @@ -110,11 +118,13 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { r.lastReadHeader = &missingHeader{ headerNum: 0, difficulty: uint64(bits.difficulty()), + stateRoot: stateRoot, extraData: b.Bytes(), } } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.stateRoot = stateRoot r.lastReadHeader.extraData = b.Bytes() } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go index 1c779d62c43e..cf10694a2ef0 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -109,6 +109,9 @@ func (m *missingHeaderWriter) write(header *types.Header) { if _, err := m.writer.Write([]byte{uint8(vanityIndex)}); err != nil { log.Fatalf("Error writing vanity index: %v", err) } + if _, err := m.writer.Write(header.StateRoot[:]); err != nil { + log.Fatalf("Error writing state root: %v", err) + } if _, err := m.writer.Write(header.Seal()); err != nil { log.Fatalf("Error writing seal: %v", err) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index 825f34b3fb7b..bf2897bfd433 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "testing" + "github.com/scroll-tech/go-ethereum/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,6 +17,9 @@ func TestMissingHeaderWriter(t *testing.T) { vanity2 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02} vanity8 := [32]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08} + stateRoot1 := common.HexToHash("0xabcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234") + stateRoot2 := common.HexToHash("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee") + var expectedBytes []byte expectedBytes = append(expectedBytes, 0x03) expectedBytes = append(expectedBytes, vanity1[:]...) @@ -37,12 +41,13 @@ func TestMissingHeaderWriter(t *testing.T) { // Header0 { seal := randomSeal(65) - header := types.NewHeader(0, 2, append(vanity1[:], seal...)) + header := types.NewHeader(0, 2, stateRoot1, append(vanity1[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=0: seal length 65 expectedBytes = append(expectedBytes, 0b00000000) expectedBytes = append(expectedBytes, 0x00) // vanity index0 + expectedBytes = append(expectedBytes, stateRoot1[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -50,12 +55,13 @@ func TestMissingHeaderWriter(t *testing.T) { // Header1 { seal := randomSeal(65) - header := types.NewHeader(1, 1, append(vanity2[:], seal...)) + header := types.NewHeader(1, 1, stateRoot2, append(vanity2[:], seal...)) mhw.write(header) // bit 6=1: difficulty 1, bit 7=0: seal length 65 expectedBytes = append(expectedBytes, 0b01000000) expectedBytes = append(expectedBytes, 0x01) // vanity index1 + expectedBytes = append(expectedBytes, stateRoot2[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -63,12 +69,13 @@ func TestMissingHeaderWriter(t *testing.T) { // Header2 { seal := randomSeal(85) - header := types.NewHeader(2, 2, append(vanity2[:], seal...)) + header := types.NewHeader(2, 2, stateRoot1, append(vanity2[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=1: seal length 85 expectedBytes = append(expectedBytes, 0b10000000) expectedBytes = append(expectedBytes, 0x01) // vanity index1 + expectedBytes = append(expectedBytes, stateRoot1[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -76,25 +83,28 @@ func TestMissingHeaderWriter(t *testing.T) { // Header3 { seal := randomSeal(85) - header := types.NewHeader(3, 1, append(vanity8[:], seal...)) + header := types.NewHeader(3, 1, stateRoot2, append(vanity8[:], seal...)) mhw.write(header) // bit 6=1: difficulty 1, bit 7=1: seal length 85 expectedBytes = append(expectedBytes, 0b11000000) expectedBytes = append(expectedBytes, 0x02) // vanity index2 + expectedBytes = append(expectedBytes, stateRoot2[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header4 { + stateRoot3 := common.Hash{123} seal := randomSeal(65) - header := types.NewHeader(4, 2, append(vanity1[:], seal...)) + header := types.NewHeader(4, 2, stateRoot3, append(vanity1[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=0: seal length 65 expectedBytes = append(expectedBytes, 0b00000000) expectedBytes = append(expectedBytes, 0x00) // vanity index0 + expectedBytes = append(expectedBytes, stateRoot3[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go index ffd76c8a8bb5..acbb4631e661 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/header.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -13,19 +13,21 @@ const VanitySize = 32 type Header struct { Number uint64 Difficulty uint64 + StateRoot common.Hash ExtraData []byte } -func NewHeader(number, difficulty uint64, extraData []byte) *Header { +func NewHeader(number, difficulty uint64, stateRoot common.Hash, extraData []byte) *Header { return &Header{ Number: number, Difficulty: difficulty, + StateRoot: stateRoot, ExtraData: extraData, } } func (h *Header) String() string { - return fmt.Sprintf("%d,%d,%s\n", h.Number, h.Difficulty, common.Bytes2Hex(h.ExtraData)) + return fmt.Sprintf("%d,%d,%s,%s\n", h.Number, h.Difficulty, h.StateRoot, common.Bytes2Hex(h.ExtraData)) } func (h *Header) Equal(other *Header) bool { @@ -35,6 +37,9 @@ func (h *Header) Equal(other *Header) bool { if h.Difficulty != other.Difficulty { return false } + if h.StateRoot != other.StateRoot { + return false + } if len(h.ExtraData) != len(other.ExtraData) { return false } @@ -48,13 +53,14 @@ func (h *Header) Equal(other *Header) bool { // Bytes returns the byte representation of the header including the initial 2 bytes for the size. func (h *Header) Bytes() ([]byte, error) { - size := 8 + 8 + len(h.ExtraData) + size := 8 + 8 + common.HashLength + len(h.ExtraData) buf := make([]byte, HeaderSizeSerialized+size) binary.BigEndian.PutUint16(buf[:2], uint16(size)) binary.BigEndian.PutUint64(buf[2:10], h.Number) binary.BigEndian.PutUint64(buf[10:18], h.Difficulty) - copy(buf[18:], h.ExtraData) + copy(buf[18:50], h.StateRoot[:]) + copy(buf[50:], h.ExtraData) return buf, nil } @@ -74,7 +80,11 @@ func (h *Header) SealLen() int { func (h *Header) FromBytes(buf []byte) *Header { h.Number = binary.BigEndian.Uint64(buf[:8]) h.Difficulty = binary.BigEndian.Uint64(buf[8:16]) - h.ExtraData = buf[16:] + if len(buf) < 48 { + panic(fmt.Sprintf("buffer too short for header: %d bytes", len(buf))) + } + h.StateRoot = common.BytesToHash(buf[16:48]) + h.ExtraData = buf[48:] return h } From 882f5fdeceb8260c04ef68bb52dfcb4d662cf3fe Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 20:28:27 +0200 Subject: [PATCH 10/17] goimports --- .../export-headers-toolkit/cmd/missing_header_writer_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index bf2897bfd433..e961fae1702c 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,10 +5,11 @@ import ( "crypto/rand" "testing" - "github.com/scroll-tech/go-ethereum/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) From 0f94e9c8401cc795b396f4e71da65311f9507d81 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 2 Jun 2025 08:09:15 +0200 Subject: [PATCH 11/17] only create 1 task at a time --- .../missing_header_fields/export-headers-toolkit/cmd/fetch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index 2c25a2d3f7ce..38866ba4de33 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -201,7 +201,7 @@ func writeHeadersToFile(outputFile string, humanReadableOutputFile string, conti func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) { headersChan := make(chan *types.Header, maxGoroutines*int(batchSize)) - tasks := make(chan task, maxGoroutines) + tasks := make(chan task) var wgConsumer sync.WaitGroup // start consumer goroutine to sort and write headers to file From fc20472fd55710afce386ea7d596b9e675ab2145 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:35:06 +0200 Subject: [PATCH 12/17] address review comments --- .../export-headers-toolkit/types/header.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go index acbb4631e661..ab475544aaf8 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/header.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -1,6 +1,7 @@ package types import ( + "bytes" "encoding/binary" "fmt" @@ -27,7 +28,7 @@ func NewHeader(number, difficulty uint64, stateRoot common.Hash, extraData []byt } func (h *Header) String() string { - return fmt.Sprintf("%d,%d,%s,%s\n", h.Number, h.Difficulty, h.StateRoot, common.Bytes2Hex(h.ExtraData)) + return fmt.Sprintf("%d,%d,%s,%s\n", h.Number, h.Difficulty, h.StateRoot.Hex(), common.Bytes2Hex(h.ExtraData)) } func (h *Header) Equal(other *Header) bool { @@ -40,14 +41,9 @@ func (h *Header) Equal(other *Header) bool { if h.StateRoot != other.StateRoot { return false } - if len(h.ExtraData) != len(other.ExtraData) { + if !bytes.Equal(h.ExtraData, other.ExtraData) { return false } - for i, b := range h.ExtraData { - if b != other.ExtraData[i] { - return false - } - } return true } From e02aeb37d66ebefd62f3609c4095fb9f2b38c918 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Tue, 17 Jun 2025 20:24:12 +0100 Subject: [PATCH 13/17] add optional coinbase and nonce to missing header --- .../export-headers-toolkit/README.md | 8 +++- .../export-headers-toolkit/cmd/dedup.go | 20 ++++++--- .../cmd/missing_header_reader.go | 45 +++++++++++++------ .../cmd/missing_header_writer.go | 44 +++++++++++++++--- .../cmd/missing_header_writer_test.go | 33 ++++++++++---- .../export-headers-toolkit/types/header.go | 27 ++++++++--- 6 files changed, 137 insertions(+), 40 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/README.md b/rollup/missing_header_fields/export-headers-toolkit/README.md index 2148c2568565..e8c7a2abcc8b 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/README.md +++ b/rollup/missing_header_fields/export-headers-toolkit/README.md @@ -6,6 +6,8 @@ A toolkit for exporting and transforming missing block header fields of Scroll b We are using the [Clique consensus](https://eips.ethereum.org/EIPS/eip-225) in Scroll L2. Amongst others, it requires the following header fields: - `extraData` - `difficulty` +- `coinbase` +- `nonce` However, before EuclidV2, these fields were not stored on L1/DA. In order for nodes to be able to reconstruct the correct block hashes when only reading data from L1, @@ -31,12 +33,16 @@ Where: - unique_vanity_i: unique vanity i - header_i: block header i - header: - + [][] - flags: bitmask, lsb first + - bit 4: 1 if the header has a coinbase field + - bit 5: 1 if the header has a nonce field - bit 6: 0 if difficulty is 2, 1 if difficulty is 1 - bit 7: 0 if seal length is 65, 1 if seal length is 85 - vanity_index: index of the vanity in the sorted vanities list (0-255) - state_root: 32 bytes of state root data + - coinbase: 20 bytes of coinbase address (if present) + - nonce: 8 bytes of nonce (if present) - seal: 65 or 85 bytes of seal data ``` diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index f22daef8d3fe..3a155af6c5fd 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/scroll-tech/go-ethereum/common" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) @@ -224,7 +225,7 @@ func (h *csvHeaderReader) readNext() *types.Header { } s := strings.Split(strings.TrimSpace(line), ",") - if len(s) != 4 { + if len(s) != 6 { log.Fatalf("Malformed CSV line: %q", line) } @@ -238,10 +239,11 @@ func (h *csvHeaderReader) readNext() *types.Header { } stateRoot := common.HexToHash(s[2]) + coinbase := common.HexToAddress(s[3]) + nonceBytes := common.Hex2Bytes(s[4]) + extra := common.FromHex(strings.Split(s[5], "\n")[0]) - extra := common.FromHex(strings.Split(s[3], "\n")[0]) - - header := types.NewHeader(num, difficulty, stateRoot, extra) + header := types.NewHeader(num, difficulty, stateRoot, coinbase, coreTypes.BlockNonce(nonceBytes), extra) return header } @@ -280,13 +282,13 @@ func verifyOutputFile(verifyFile, outputFile string) { for { header := csvReader.readNext() if header == nil { - if _, _, err = dedupReader.ReadNext(); err == nil { + if _, _, _, _, err = dedupReader.ReadNext(); err == nil { log.Fatalf("Expected EOF, got more headers") } break } - difficulty, stateRoot, extraData, err := dedupReader.Read(header.Number) + difficulty, stateRoot, coinbase, nonce, extraData, err := dedupReader.Read(header.Number) if err != nil { log.Fatalf("Error reading header: %v", err) } @@ -297,6 +299,12 @@ func verifyOutputFile(verifyFile, outputFile string) { if header.StateRoot != stateRoot { log.Fatalf("StateRoot mismatch: headerNum %d: %s != %s", header.Number, header.StateRoot, stateRoot) } + if header.Coinbase != coinbase { + log.Fatalf("Coinbase mismatch: headerNum %d: %s != %s", header.Number, header.Coinbase.Hex(), coinbase.Hex()) + } + if header.Nonce != nonce { + log.Fatalf("Nonce mismatch: headerNum %d: %s != %s", header.Number, common.Bytes2Hex(header.Nonce[:]), common.Bytes2Hex(nonce[:])) + } if !bytes.Equal(header.ExtraData, extraData) { log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go index c4d86ab1d8e7..df88dc95032e 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go @@ -8,6 +8,7 @@ import ( "os" "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" ) // TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit @@ -16,6 +17,8 @@ type missingHeader struct { headerNum uint64 difficulty uint64 stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce extraData []byte } @@ -56,35 +59,35 @@ func NewReader(filePath string) (*Reader, error) { return r, nil } -func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { if r.lastReadHeader == nil { - if _, _, err = r.ReadNext(); err != nil { - return 0, common.Hash{}, nil, err + if _, _, _, _, err = r.ReadNext(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } if headerNum > r.lastReadHeader.headerNum { // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { - if _, _, err = r.ReadNext(); err != nil { - return 0, common.Hash{}, nil, err + if _, _, _, _, err = r.ReadNext(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } } if headerNum == r.lastReadHeader.headerNum { - return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil } // headerNum < r.lastReadHeader.headerNum is not supported - return 0, common.Hash{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) } -func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { +func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { // read the bitmask bitmaskByte, err := r.reader.ReadByte() if err != nil { - return 0, nil, fmt.Errorf("failed to read bitmask: %v", err) + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read bitmask: %v", err) } bits := newBitMaskFromByte(bitmaskByte) @@ -92,18 +95,30 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { // read the vanity index vanityIndex, err := r.reader.ReadByte() if err != nil { - return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read vanity index: %v", err) } var stateRoot common.Hash if _, err := io.ReadFull(r.reader, stateRoot[:]); err != nil { - return 0, nil, fmt.Errorf("failed to read state root: %v", err) + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read state root: %v", err) + } + + if bits.hasCoinbase() { + if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read coinbase: %v", err) + } + } + + if bits.hasNonce() { + if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read nonce: %v", err) + } } seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { - return 0, nil, fmt.Errorf("failed to read seal: %v", err) + return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read seal: %v", err) } // construct the extraData field @@ -119,16 +134,20 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { headerNum: 0, difficulty: uint64(bits.difficulty()), stateRoot: stateRoot, + coinbase: coinbase, + nonce: nonce, extraData: b.Bytes(), } } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) r.lastReadHeader.stateRoot = stateRoot + r.lastReadHeader.coinbase = coinbase + r.lastReadHeader.nonce = nonce r.lastReadHeader.extraData = b.Bytes() } - return difficulty, b.Bytes(), nil + return difficulty, coinbase, nonce, b.Bytes(), nil } func (r *Reader) Close() error { diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go index cf10694a2ef0..0abade425127 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go @@ -9,6 +9,9 @@ import ( "os" "sort" + "github.com/scroll-tech/go-ethereum/common" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" ) @@ -95,14 +98,17 @@ func (m *missingHeaderWriter) writeVanities() { func (m *missingHeaderWriter) write(header *types.Header) { // 1. prepare the bitmask - bits := newBitMask(int(header.Difficulty), header.SealLen()) + hasCoinbase := header.Coinbase != (common.Address{}) + hasNonce := header.Nonce != (coreTypes.BlockNonce{}) + + bits := newBitMask(hasCoinbase, hasNonce, int(header.Difficulty), header.SealLen()) vanityIndex := m.sortedVanitiesMap[header.Vanity()] if vanityIndex >= maxVanityCount { log.Fatalf("Vanity index %d exceeds maximum allowed %d", vanityIndex, maxVanityCount-1) } - // 2. write the header: bitmask, vanity index and seal + // 2. write the header: bitmask, optional coinbase, optional nonce, vanity index and seal if _, err := m.writer.Write(bits.Bytes()); err != nil { log.Fatalf("Error writing bitmask: %v", err) } @@ -112,6 +118,17 @@ func (m *missingHeaderWriter) write(header *types.Header) { if _, err := m.writer.Write(header.StateRoot[:]); err != nil { log.Fatalf("Error writing state root: %v", err) } + if hasCoinbase { + if _, err := m.writer.Write(header.Coinbase[:]); err != nil { + log.Fatalf("Error writing coinbase: %v", err) + } + } + + if hasNonce { + if _, err := m.writer.Write(header.Nonce[:]); err != nil { + log.Fatalf("Error writing nonce: %v", err) + } + } if _, err := m.writer.Write(header.Seal()); err != nil { log.Fatalf("Error writing seal: %v", err) } @@ -119,8 +136,10 @@ func (m *missingHeaderWriter) write(header *types.Header) { // bitMask is a bitmask that encodes the following information: // -// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 -// bit 7: 0 if seal length is 65, 1 if seal length is 85 +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 type bitMask struct { b uint8 } @@ -129,9 +148,16 @@ func newBitMaskFromByte(b uint8) bitMask { return bitMask{b} } -func newBitMask(difficulty int, sealLen int) bitMask { +func newBitMask(hasCoinbase bool, hasNonce bool, difficulty int, sealLen int) bitMask { b := uint8(0) + if hasCoinbase { + b |= 1 << 4 + } + + if hasNonce { + b |= 1 << 5 + } if difficulty == 1 { b |= 1 << 6 } else if difficulty != 2 { @@ -165,6 +191,14 @@ func (b bitMask) sealLen() int { } } +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + func (b bitMask) Bytes() []byte { return []byte{b.b} } diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index e961fae1702c..52700fc5f90c 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "testing" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -42,21 +43,27 @@ func TestMissingHeaderWriter(t *testing.T) { // Header0 { seal := randomSeal(65) - header := types.NewHeader(0, 2, stateRoot1, append(vanity1[:], seal...)) + coinbase := common.Address{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + nonce := coreTypes.BlockNonce{0, 1, 2, 3, 4, 5, 6, 7} + header := types.NewHeader(0, 2, stateRoot1, coinbase, nonce, append(vanity1[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=0: seal length 65 - expectedBytes = append(expectedBytes, 0b00000000) + expectedBytes = append(expectedBytes, 0b00110000) expectedBytes = append(expectedBytes, 0x00) // vanity index0 expectedBytes = append(expectedBytes, stateRoot1[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } // Header1 { + coinbase := common.Address{} + nonce := coreTypes.BlockNonce{} seal := randomSeal(65) - header := types.NewHeader(1, 1, stateRoot2, append(vanity2[:], seal...)) + header := types.NewHeader(1, 1, stateRoot2, coinbase, nonce, append(vanity2[:], seal...)) mhw.write(header) // bit 6=1: difficulty 1, bit 7=0: seal length 65 @@ -68,15 +75,19 @@ func TestMissingHeaderWriter(t *testing.T) { } // Header2 + coinbase := common.Address{1} + nonce := coreTypes.BlockNonce{201} { seal := randomSeal(85) - header := types.NewHeader(2, 2, stateRoot1, append(vanity2[:], seal...)) + header := types.NewHeader(2, 2, stateRoot1, coinbase, nonce, append(vanity2[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=1: seal length 85 - expectedBytes = append(expectedBytes, 0b10000000) + expectedBytes = append(expectedBytes, 0b10110000) expectedBytes = append(expectedBytes, 0x01) // vanity index1 expectedBytes = append(expectedBytes, stateRoot1[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -84,13 +95,15 @@ func TestMissingHeaderWriter(t *testing.T) { // Header3 { seal := randomSeal(85) - header := types.NewHeader(3, 1, stateRoot2, append(vanity8[:], seal...)) + header := types.NewHeader(3, 1, stateRoot2, coinbase, nonce, append(vanity8[:], seal...)) mhw.write(header) // bit 6=1: difficulty 1, bit 7=1: seal length 85 - expectedBytes = append(expectedBytes, 0b11000000) + expectedBytes = append(expectedBytes, 0b11110000) expectedBytes = append(expectedBytes, 0x02) // vanity index2 expectedBytes = append(expectedBytes, stateRoot2[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } @@ -99,13 +112,15 @@ func TestMissingHeaderWriter(t *testing.T) { { stateRoot3 := common.Hash{123} seal := randomSeal(65) - header := types.NewHeader(4, 2, stateRoot3, append(vanity1[:], seal...)) + header := types.NewHeader(4, 2, stateRoot3, coinbase, nonce, append(vanity1[:], seal...)) mhw.write(header) // bit 6=0: difficulty 2, bit 7=0: seal length 65 - expectedBytes = append(expectedBytes, 0b00000000) + expectedBytes = append(expectedBytes, 0b00110000) expectedBytes = append(expectedBytes, 0x00) // vanity index0 expectedBytes = append(expectedBytes, stateRoot3[:]...) + expectedBytes = append(expectedBytes, coinbase[:]...) + expectedBytes = append(expectedBytes, nonce[:]...) expectedBytes = append(expectedBytes, seal...) require.Equal(t, expectedBytes, bytesBuffer.Bytes()) } diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go index ab475544aaf8..f9e1e61f9df8 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/header.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/header.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" ) const HeaderSizeSerialized = 2 @@ -15,20 +16,24 @@ type Header struct { Number uint64 Difficulty uint64 StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce ExtraData []byte } -func NewHeader(number, difficulty uint64, stateRoot common.Hash, extraData []byte) *Header { +func NewHeader(number, difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte) *Header { return &Header{ Number: number, Difficulty: difficulty, StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, ExtraData: extraData, } } func (h *Header) String() string { - return fmt.Sprintf("%d,%d,%s,%s\n", h.Number, h.Difficulty, h.StateRoot.Hex(), common.Bytes2Hex(h.ExtraData)) + return fmt.Sprintf("%d,%d,%s,%s,%s,%s\n", h.Number, h.Difficulty, h.StateRoot.Hex(), h.Coinbase.Hex(), common.Bytes2Hex(h.Nonce[:]), common.Bytes2Hex(h.ExtraData)) } func (h *Header) Equal(other *Header) bool { @@ -41,6 +46,12 @@ func (h *Header) Equal(other *Header) bool { if h.StateRoot != other.StateRoot { return false } + if h.Coinbase != other.Coinbase { + return false + } + if h.Nonce != other.Nonce { + return false + } if !bytes.Equal(h.ExtraData, other.ExtraData) { return false } @@ -49,14 +60,16 @@ func (h *Header) Equal(other *Header) bool { // Bytes returns the byte representation of the header including the initial 2 bytes for the size. func (h *Header) Bytes() ([]byte, error) { - size := 8 + 8 + common.HashLength + len(h.ExtraData) + size := 8 + 8 + common.HashLength + common.AddressLength + 8 + len(h.ExtraData) buf := make([]byte, HeaderSizeSerialized+size) binary.BigEndian.PutUint16(buf[:2], uint16(size)) binary.BigEndian.PutUint64(buf[2:10], h.Number) binary.BigEndian.PutUint64(buf[10:18], h.Difficulty) copy(buf[18:50], h.StateRoot[:]) - copy(buf[50:], h.ExtraData) + copy(buf[50:70], h.Coinbase[:]) + copy(buf[70:78], h.Nonce[:]) + copy(buf[78:], h.ExtraData) return buf, nil } @@ -76,11 +89,13 @@ func (h *Header) SealLen() int { func (h *Header) FromBytes(buf []byte) *Header { h.Number = binary.BigEndian.Uint64(buf[:8]) h.Difficulty = binary.BigEndian.Uint64(buf[8:16]) - if len(buf) < 48 { + if len(buf) < 76 { panic(fmt.Sprintf("buffer too short for header: %d bytes", len(buf))) } h.StateRoot = common.BytesToHash(buf[16:48]) - h.ExtraData = buf[48:] + h.Coinbase = common.BytesToAddress(buf[48:68]) + h.Nonce = types.BlockNonce(buf[68:76]) + h.ExtraData = buf[76:] return h } From 6ed9c95303bd6392dc76c17595f92891645d5bbd Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Tue, 17 Jun 2025 21:22:53 +0100 Subject: [PATCH 14/17] add flag to run fetch from rollup relayer DB --- .../export-headers-toolkit/cmd/fetch.go | 57 +++++++++++- .../export-headers-toolkit/go.mod | 7 ++ .../export-headers-toolkit/go.sum | 16 ++++ .../export-headers-toolkit/types/l2_block.go | 87 +++++++++++++++++++ 4 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go index 38866ba4de33..c4d216c1d7cf 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/fetch.go @@ -14,6 +14,8 @@ import ( "time" "github.com/spf13/cobra" + "gorm.io/driver/postgres" + "gorm.io/gorm" "github.com/scroll-tech/go-ethereum/ethclient" @@ -70,6 +72,18 @@ It produces a binary file and optionally a human readable csv file with the miss if err != nil { log.Fatalf("Error reading continue flag: %v", err) } + dbDSN, err := cmd.Flags().GetString("db") + if err != nil { + log.Fatalf("Error reading db flag: %v", err) + } + + var db *gorm.DB + if dbDSN != "" { + db, err = gorm.Open(postgres.Open(dbDSN), &gorm.Config{}) + if err != nil { + log.Fatalf("Error connecting to database: %v", err) + } + } if continueFile != "" { fmt.Println("Continue fetching block header fields from", continueFile) @@ -92,7 +106,7 @@ It produces a binary file and optionally a human readable csv file with the miss } } - runFetch(clients, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile) + runFetch(clients, db, startBlockNum, endBlockNum, batchSize, maxParallelGoroutines, outputFile, humanReadableOutputFile, continueFile) }, } @@ -107,6 +121,7 @@ func init() { fetchCmd.Flags().String("output", "headers.bin", "output file") fetchCmd.Flags().String("humanOutput", "", "additionally produce human readable csv file") fetchCmd.Flags().String("continue", "", "continue fetching block header fields from the last seen block number in the specified continue file") + fetchCmd.Flags().String("db", "", "database to use instead of fetching from RPC") } func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetries int) (*types.Header, error) { @@ -118,6 +133,8 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri header.Number.Uint64(), header.Difficulty.Uint64(), header.Root, + header.Coinbase, + header.Nonce, header.Extra, ), nil } @@ -132,7 +149,26 @@ func headerByNumberWithRetry(client *ethclient.Client, blockNum uint64, maxRetri return nil, fmt.Errorf("error fetching header for block %d: %v", blockNum, innerErr) } -func fetchHeaders(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { +func fetchHeadersFromDB(db *gorm.DB, start, end uint64, headersChan chan<- *types.Header) { + blockORM := types.NewL2Block(db) + blocks, err := blockORM.GetL2BlocksInRange(context.Background(), start, end) + if err != nil { + log.Fatalf("Error fetching blocks from database: %v", err) + } + + for _, block := range blocks { + headersChan <- types.NewHeader( + block.Header.Number.Uint64(), + block.Header.Difficulty.Uint64(), + block.Header.Root, + block.Header.Coinbase, + block.Header.Nonce, + block.Header.Extra, + ) + } +} + +func fetchHeadersFromRPC(clients []*ethclient.Client, start, end uint64, headersChan chan<- *types.Header) { // randomize client selection to distribute load r := uint64(rand.Int()) @@ -199,7 +235,7 @@ func writeHeadersToFile(outputFile string, humanReadableOutputFile string, conti fmt.Println("Finished writing headers to file, last block number:", nextHeaderNum-1) } -func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) { +func runFetch(clients []*ethclient.Client, db *gorm.DB, startBlockNum uint64, endBlockNum uint64, batchSize uint64, maxGoroutines int, outputFile string, humanReadableOutputFile string, continueFile string) { headersChan := make(chan *types.Header, maxGoroutines*int(batchSize)) tasks := make(chan task) @@ -222,12 +258,25 @@ func runFetch(clients []*ethclient.Client, startBlockNum uint64, endBlockNum uin break } log.Println("Received task", t.start, "to", t.end) - fetchHeaders(clients, t.start, t.end, headersChan) + + // use DB if dbDSN is provided, otherwise fetch from RPC + if db != nil { + fetchHeadersFromDB(db, t.start, t.end, headersChan) + } else { + fetchHeadersFromRPC(clients, t.start, t.end, headersChan) + } } wgProducers.Done() }() } + // need to fetch block 0 from RPC + if startBlockNum == 0 && db != nil { + fmt.Println("Fetching headers from database... and header 0 from RPC") + fetchHeadersFromRPC(clients, 0, 0, headersChan) + startBlockNum = 1 + } + // create tasks/work packages for producer goroutines for start := startBlockNum; start <= endBlockNum; start += batchSize { end := start + batchSize - 1 diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index e465c8617f26..f5bacf393cab 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -34,7 +34,12 @@ require ( github.com/huin/goupnp v1.0.2 // indirect github.com/iden3/go-iden3-crypto v0.0.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.4.3 // indirect github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -67,5 +72,7 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/postgres v1.5.7 // indirect + gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 5c194d359597..3e7939b39d95 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -93,9 +93,19 @@ github.com/iden3/go-iden3-crypto v0.0.16 h1:zN867xiz6HgErXVIV/6WyteGcOukE9gybYTo github.com/iden3/go-iden3-crypto v0.0.16/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -258,5 +268,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= +gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go new file mode 100644 index 000000000000..6e08b0621bc4 --- /dev/null +++ b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go @@ -0,0 +1,87 @@ +package types + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/scroll-tech/da-codec/encoding" + "github.com/scroll-tech/go-ethereum/core/types" + "gorm.io/gorm" +) + +// L2Block represents a l2 block in the database. +type L2Block struct { + db *gorm.DB `gorm:"column:-"` + + // block + Number uint64 `json:"number" gorm:"number"` + Hash string `json:"hash" gorm:"hash"` + ParentHash string `json:"parent_hash" gorm:"parent_hash"` + Header string `json:"header" gorm:"header"` + Transactions string `json:"transactions" gorm:"transactions"` + WithdrawRoot string `json:"withdraw_root" gorm:"withdraw_root"` + StateRoot string `json:"state_root" gorm:"state_root"` + TxNum uint32 `json:"tx_num" gorm:"tx_num"` + GasUsed uint64 `json:"gas_used" gorm:"gas_used"` + BlockTimestamp uint64 `json:"block_timestamp" gorm:"block_timestamp"` + RowConsumption string `json:"row_consumption" gorm:"row_consumption"` + + // chunk + ChunkHash string `json:"chunk_hash" gorm:"chunk_hash;default:NULL"` + + // metadata + CreatedAt time.Time `json:"created_at" gorm:"column:created_at"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"` + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"column:deleted_at;default:NULL"` +} + +// NewL2Block creates a new L2Block instance +func NewL2Block(db *gorm.DB) *L2Block { + return &L2Block{db: db} +} + +// TableName returns the name of the "l2_block" table. +func (*L2Block) TableName() string { + return "l2_block" +} + +// GetL2BlocksInRange retrieves the L2 blocks within the specified range (inclusive). +// The range is closed, i.e., it includes both start and end block numbers. +// The returned blocks are sorted in ascending order by their block number. +func (o *L2Block) GetL2BlocksInRange(ctx context.Context, startBlockNumber uint64, endBlockNumber uint64) ([]*encoding.Block, error) { + if startBlockNumber > endBlockNumber { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: start block number should be less than or equal to end block number, start block: %v, end block: %v", startBlockNumber, endBlockNumber) + } + + db := o.db.WithContext(ctx) + db = db.Model(&L2Block{}) + db = db.Select("header") + db = db.Where("number >= ? AND number <= ?", startBlockNumber, endBlockNumber) + db = db.Order("number ASC") + + var l2Blocks []L2Block + if err := db.Find(&l2Blocks).Error; err != nil { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber) + } + + // sanity check + if uint64(len(l2Blocks)) != endBlockNumber-startBlockNumber+1 { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange: unexpected number of results, expected: %v, got: %v", endBlockNumber-startBlockNumber+1, len(l2Blocks)) + } + + var blocks []*encoding.Block + for _, v := range l2Blocks { + var block encoding.Block + + block.Header = &types.Header{} + if err := json.Unmarshal([]byte(v.Header), block.Header); err != nil { + return nil, fmt.Errorf("L2Block.GetL2BlocksInRange error: %w, start block: %v, end block: %v", err, startBlockNumber, endBlockNumber) + } + + blocks = append(blocks, &block) + } + + return blocks, nil +} From 3b41bb8648fc8148de90a267999b89c6748ea6bd Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 07:42:50 +0100 Subject: [PATCH 15/17] go mod tidy --- rollup/missing_header_fields/export-headers-toolkit/go.mod | 6 +++--- rollup/missing_header_fields/export-headers-toolkit/go.sum | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index f5bacf393cab..f1062d385a7b 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -3,9 +3,12 @@ module github.com/scroll-tech/go-ethereum/export-headers-toolkit go 1.22 require ( + github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 + gorm.io/driver/postgres v1.5.7 + gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde ) require ( @@ -53,7 +56,6 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rjeczalik/notify v0.9.1 // indirect github.com/rs/cors v1.7.0 // indirect - github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 // indirect github.com/scroll-tech/zktrie v0.8.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spf13/pflag v1.0.6 // indirect @@ -72,7 +74,5 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/postgres v1.5.7 // indirect - gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 3e7939b39d95..8ef0f6fde0ce 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -270,8 +270,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= -gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= -gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg= gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= From cd1d5597ca711c77abd72fb1626a4ddbe9d92b20 Mon Sep 17 00:00:00 2001 From: Jonas Theis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:40:23 +0800 Subject: [PATCH 16/17] fix(l1 follower, rollup verifier): blockhash mismatch (#1192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implement missing header fields reader and manager * chore: auto version bump [bot] * increase download timeout * sanitize BaseFee when executing blocks from DA * initialize and pass missing header manager to DA syncing pipeline * add state root to deduplicated header * overwrite state root if given via missing header file * fix test * set correct links and missing header file hashes * allow reading of previous headers by resetting file and buffer to support reset of syncing pipeline * address review comments * add coinbase and nonce field to missing header reader * replace missing header reader in toolkit with actual implementation * update sync from DA pipeline to include coinbase and nonce from missing header fields file * update sha256 hashes for missing header fields files * lint * address review comments * address review comments --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 8 + core/blockchain.go | 15 +- eth/backend.go | 27 ++- node/config.go | 2 + params/config.go | 40 ++-- rollup/da_syncer/block_queue.go | 15 +- rollup/da_syncer/da/commitV0.go | 19 +- rollup/da_syncer/da/commitV7.go | 3 +- rollup/da_syncer/da/da.go | 9 +- rollup/da_syncer/syncing_pipeline.go | 5 +- .../export-headers-toolkit/cmd/dedup.go | 23 ++- .../cmd/missing_header_writer_test.go | 3 +- .../export-headers-toolkit/go.mod | 14 +- .../export-headers-toolkit/go.sum | 30 ++- .../export-headers-toolkit/types/l2_block.go | 3 +- rollup/missing_header_fields/manager.go | 188 ++++++++++++++++++ rollup/missing_header_fields/manager_test.go | 62 ++++++ .../missing_header_reader.go => reader.go} | 114 ++++++++--- rollup/missing_header_fields/reader_test.go | 79 ++++++++ .../testdata/missing-headers.bin | Bin 0 -> 1222 bytes .../rollup_sync_service_test.go | 3 +- 23 files changed, 585 insertions(+), 79 deletions(-) create mode 100644 rollup/missing_header_fields/manager.go create mode 100644 rollup/missing_header_fields/manager_test.go rename rollup/missing_header_fields/{export-headers-toolkit/cmd/missing_header_reader.go => reader.go} (54%) create mode 100644 rollup/missing_header_fields/reader_test.go create mode 100644 rollup/missing_header_fields/testdata/missing-headers.bin diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c1fbb08746a5..459feca738de 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -177,6 +177,7 @@ var ( utils.RollupVerifyEnabledFlag, utils.ShadowforkPeersFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABlobScanAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1d299d339fe8..fbfc1b91fe56 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -237,6 +237,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.L1DisableMessageQueueV2Flag, utils.RollupVerifyEnabledFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlobScanAPIEndpointFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7371a3770074..5a7bdf80f0b7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -898,6 +898,12 @@ var ( Name: "da.sync", Usage: "Enable node syncing from DA", } + DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ + Name: "da.missingheaderfields.baseurl", + Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks", + Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/", + } + DABlobScanAPIEndpointFlag = cli.StringFlag{ Name: "da.blob.blobscan", Usage: "BlobScan blob API endpoint", @@ -1382,6 +1388,8 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index 57a803198fa1..314368ee049d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1880,15 +1880,18 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() + // sanitize base fee + // Note: setting the base fee to 0 will cause problems as nil != 0 when serializing the header and thus block hash will be different. + if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { + header.BaseFee = nil + } + tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil) receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig) if err != nil { return nil, NonStatTy, fmt.Errorf("error processing block: %w", err) } - // TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique - // This should be done with https://github.com/scroll-tech/go-ethereum/pull/913. - if sign { // Prevent Engine from overriding timestamp. originalTime := header.Time @@ -1901,7 +1904,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types // finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble() header.GasUsed = gasUsed - header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + + // state root might be set from partial header. If it is not set, we calculate it. + if header.Root == (common.Hash{}) { + header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + } fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/eth/backend.go b/eth/backend.go index ad765b835b0a..3fbefcfe63d2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,9 @@ import ( "errors" "fmt" "math/big" + "net/url" + "path" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -61,6 +64,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/ccc" "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service" "github.com/scroll-tech/go-ethereum/rollup/sync_service" "github.com/scroll-tech/go-ethereum/rpc" @@ -241,7 +245,12 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if config.EnableDASyncing { // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { - eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA) + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) } @@ -337,6 +346,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether return eth, nil } +func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) { + downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) + } + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin") + + expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 + if expectedSHA256Checksum == nil { + return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") + } + + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) + return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/node/config.go b/node/config.go index 6e7133a95129..d7212389684e 100644 --- a/node/config.go +++ b/node/config.go @@ -201,6 +201,8 @@ type Config struct { L1DisableMessageQueueV2 bool `toml:",omitempty"` // Is daSyncingEnabled DaSyncingEnabled bool `toml:",omitempty"` + // Base URL for missing header fields file + DAMissingHeaderFieldsBaseURL string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/params/config.go b/params/config.go index eda4340e80c4..b6b7da4b7fc0 100644 --- a/params/config.go +++ b/params/config.go @@ -30,16 +30,18 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") - ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") - ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") - ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") - ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") + ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") + ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") + ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") + ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xfa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xa02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924") ) func newUint64(val uint64) *uint64 { return &val } @@ -354,7 +356,8 @@ var ( ScrollChainAddress: common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"), L2SystemConfigAddress: common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"), }, - GenesisStateRoot: &ScrollSepoliaGenesisState, + GenesisStateRoot: &ScrollSepoliaGenesisState, + MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256, }, } @@ -406,7 +409,8 @@ var ( ScrollChainAddress: common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"), L2SystemConfigAddress: common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"), }, - GenesisStateRoot: &ScrollMainnetGenesisState, + GenesisStateRoot: &ScrollMainnetGenesisState, + MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256, }, } @@ -710,6 +714,9 @@ type ScrollConfig struct { // Genesis State Root for MPT clients GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"` + + // MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file. + MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"` } // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer @@ -760,8 +767,13 @@ func (s ScrollConfig) String() string { genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot) } - return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}", - s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot) + missingHeaderFieldsSHA256 := "" + if s.MissingHeaderFieldsSHA256 != nil { + missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256) + } + + return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}", + s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256) } // IsValidTxCount returns whether the given block's transaction count is below the limit. diff --git a/rollup/da_syncer/block_queue.go b/rollup/da_syncer/block_queue.go index 630382f001c0..30d53bfcb6a5 100644 --- a/rollup/da_syncer/block_queue.go +++ b/rollup/da_syncer/block_queue.go @@ -6,19 +6,22 @@ import ( "github.com/scroll-tech/go-ethereum/core/rawdb" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and // provides them to the next stage one-by-one. type BlockQueue struct { - batchQueue *BatchQueue - blocks []*da.PartialBlock + batchQueue *BatchQueue + blocks []*da.PartialBlock + missingHeaderFieldsManager *missing_header_fields.Manager } -func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue { +func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue { return &BlockQueue{ - batchQueue: batchQueue, - blocks: make([]*da.PartialBlock, 0), + batchQueue: batchQueue, + blocks: make([]*da.PartialBlock, 0), + missingHeaderFieldsManager: missingHeaderFieldsManager, } } @@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error { return err } - bq.blocks, err = entryWithBlocks.Blocks() + bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager) if err != nil { return fmt.Errorf("failed to get blocks from entry: %w", err) } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index c99a474b256b..ca7322264a43 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type CommitBatchDAV0 struct { @@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int { return 0 } -func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) { l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped) if err != nil { return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err) @@ -120,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex := c.parentTotalL1MessagePopped for _, chunk := range c.chunks { - for blockId, daBlock := range chunk.Blocks { + for blockIndex, daBlock := range chunk.Blocks { // create txs txs := make(types.Transactions, 0, daBlock.NumTransactions()) // insert l1 msgs @@ -132,7 +133,12 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { curL1TxIndex += uint64(daBlock.NumL1Messages()) // insert l2 txs - txs = append(txs, chunk.Transactions[blockId]...) + txs = append(txs, chunk.Transactions[blockIndex]...) + + difficulty, stateRoot, coinbase, nonce, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + if err != nil { + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) + } block := NewPartialBlock( &PartialHeader{ @@ -140,8 +146,11 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { Time: daBlock.Timestamp(), BaseFee: daBlock.BaseFee(), GasLimit: daBlock.GasLimit(), - Difficulty: 10, // TODO: replace with real difficulty - ExtraData: []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data + Difficulty: difficulty, + ExtraData: extraData, + StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/commitV7.go b/rollup/da_syncer/da/commitV7.go index 6048b853aee7..37a4b948df6c 100644 --- a/rollup/da_syncer/da/commitV7.go +++ b/rollup/da_syncer/da/commitV7.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" @@ -113,7 +114,7 @@ func (c *CommitBatchDAV7) Event() l1.RollupEvent { return c.event } -func (c *CommitBatchDAV7) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV7) Blocks(_ *missing_header_fields.Manager) ([]*PartialBlock, error) { initialL1MessageIndex := c.parentTotalL1MessagePopped l1Txs, err := getL1MessagesV7(c.db, c.blobPayload.Blocks(), initialL1MessageIndex) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index fe72473451b6..f80ee631b51d 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -8,6 +8,7 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type Type int @@ -34,7 +35,7 @@ type Entry interface { type EntryWithBlocks interface { Entry - Blocks() ([]*PartialBlock, error) + Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) Version() encoding.CodecVersion Chunks() []*encoding.DAChunkRawTx BlobVersionedHashes() []common.Hash @@ -53,6 +54,9 @@ type PartialHeader struct { GasLimit uint64 Difficulty uint64 ExtraData []byte + StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce } func (h *PartialHeader) ToHeader() *types.Header { @@ -63,6 +67,9 @@ func (h *PartialHeader) ToHeader() *types.Header { GasLimit: h.GasLimit, Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, + Root: h.StateRoot, + Coinbase: h.Coinbase, + Nonce: h.Nonce, } } diff --git a/rollup/da_syncer/syncing_pipeline.go b/rollup/da_syncer/syncing_pipeline.go index 080179107f8c..19b7bcbc07de 100644 --- a/rollup/da_syncer/syncing_pipeline.go +++ b/rollup/da_syncer/syncing_pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // Config is the configuration parameters of data availability syncing. @@ -50,7 +51,7 @@ type SyncingPipeline struct { daQueue *DAQueue } -func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) { +func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config, missingHeaderFieldsManager *missing_header_fields.Manager) (*SyncingPipeline, error) { l1Reader, err := l1.NewReader(ctx, l1.Config{ ScrollChainAddress: genesisConfig.Scroll.L1Config.ScrollChainAddress, L1MessageQueueAddress: genesisConfig.Scroll.L1Config.L1MessageQueueAddress, @@ -124,7 +125,7 @@ func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesi daQueue := NewDAQueue(lastProcessedBatchMeta.L1BlockNumber, dataSourceFactory) batchQueue := NewBatchQueue(daQueue, db, lastProcessedBatchMeta) - blockQueue := NewBlockQueue(batchQueue) + blockQueue := NewBlockQueue(batchQueue, missingHeaderFieldsManager) daSyncer := NewDASyncer(blockchain, config.L2EndBlock) ctx, cancel := context.WithCancel(ctx) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 3a155af6c5fd..69126a860a4c 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -16,8 +16,8 @@ import ( "github.com/scroll-tech/go-ethereum/common" coreTypes "github.com/scroll-tech/go-ethereum/core/types" - "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // dedupCmd represents the dedup command @@ -49,6 +49,9 @@ The binary layout of the deduplicated file is as follows: log.Fatalf("Error reading verify flag: %v", err) } + // uncomment the following line to copy from the verify file to the input file. This is useful to generate a deduplicated header file for testing purposes. + // copyFromVerifyFile(verifyFile, inputFile) + if verifyFile != "" { verifyInputFile(verifyFile, inputFile) } @@ -251,6 +254,20 @@ func (h *csvHeaderReader) close() { h.file.Close() } +func copyFromVerifyFile(verifyFile, inputFile string) { + fmt.Println("Copying from", verifyFile, "to", inputFile) + + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + writer := newFilesWriter(inputFile, "") + defer writer.close() + + for header := csvReader.readNext(); header != nil; header = csvReader.readNext() { + writer.write(header) + } +} + func verifyInputFile(verifyFile, inputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() @@ -273,7 +290,7 @@ func verifyOutputFile(verifyFile, outputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() - dedupReader, err := NewReader(outputFile) + dedupReader, err := missing_header_fields.NewReader(outputFile) if err != nil { log.Fatalf("Error opening dedup file: %v", err) } @@ -282,7 +299,7 @@ func verifyOutputFile(verifyFile, outputFile string) { for { header := csvReader.readNext() if header == nil { - if _, _, _, _, err = dedupReader.ReadNext(); err == nil { + if err = dedupReader.ReadNext(); err == nil { log.Fatalf("Expected EOF, got more headers") } break diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index 52700fc5f90c..8fee44df0177 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,10 +5,11 @@ import ( "crypto/rand" "testing" - coreTypes "github.com/scroll-tech/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index f1062d385a7b..d10f0f5d0a80 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -2,9 +2,11 @@ module github.com/scroll-tech/go-ethereum/export-headers-toolkit go 1.22 +replace github.com/scroll-tech/go-ethereum => ../../.. + require ( - github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 - github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 + github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 + github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 gorm.io/driver/postgres v1.5.7 @@ -12,7 +14,7 @@ require ( ) require ( - github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -23,7 +25,7 @@ require ( github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/fjl/memsize v0.0.2 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-stack/stack v1.8.1 // indirect @@ -43,6 +45,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -58,6 +61,7 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/scroll-tech/zktrie v0.8.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect github.com/supranational/blst v0.3.12 // indirect @@ -66,6 +70,8 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 8ef0f6fde0ce..9823b3e92427 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -1,7 +1,9 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= -github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38= +github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= @@ -41,8 +43,8 @@ github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -108,6 +110,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -166,14 +170,14 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 h1:CDrPMqifvAVyYqu0x1J5qickVV0b51tApPnOwDYLESI= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 h1:Jq4TTYcHVAIVPUHNYbOzNxEsWf+9Q3b30YQaMyl0TDI= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 h1:Dhd58LE1D+dnoxpgLVeQBMF9uweL/fhQfZHWtWSiOlE= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY= github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= @@ -199,6 +203,10 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -208,8 +216,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -229,7 +237,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go index 6e08b0621bc4..2b2a18765090 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go @@ -7,8 +7,9 @@ import ( "time" "github.com/scroll-tech/da-codec/encoding" - "github.com/scroll-tech/go-ethereum/core/types" "gorm.io/gorm" + + "github.com/scroll-tech/go-ethereum/core/types" ) // L2Block represents a l2 block in the database. diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go new file mode 100644 index 000000000000..46804aa0830b --- /dev/null +++ b/rollup/missing_header_fields/manager.go @@ -0,0 +1,188 @@ +package missing_header_fields + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/log" +) + +const timeoutDownload = 30 * time.Minute + +// Manager is responsible for managing the missing header fields file. +// It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. +type Manager struct { + ctx context.Context + filePath string + downloadURL string + expectedChecksum common.Hash + + reader *Reader +} + +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum common.Hash) *Manager { + return &Manager{ + ctx: ctx, + filePath: filePath, + downloadURL: downloadURL, + expectedChecksum: expectedChecksum, + } +} + +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + // lazy initialization: if the reader is not initialized this is the first time we read from the file + if m.reader == nil { + if err = m.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + } + } + + return m.reader.Read(headerNum) +} + +func (m *Manager) initialize() error { + // if the file doesn't exist, download it + if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) { + if err = m.downloadFile(); err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + } + + // verify the expectedChecksum + f, err := os.Open(m.filePath) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + computedChecksum := h.Sum(nil) + if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x. Please delete %s to restart file download", m.expectedChecksum, computedChecksum, m.filePath) + } + + // finally initialize the reader + reader, err := NewReader(m.filePath) + if err != nil { + return fmt.Errorf("failed to create reader: %v", err) + } + + m.reader = reader + return nil +} + +func (m *Manager) Close() error { + if m.reader != nil { + return m.reader.Close() + } + return nil +} + +func (m *Manager) downloadFile() error { + log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL) + + downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload) + defer downloadCtxCancel() + + req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create download request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + // create a temporary file + tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return fmt.Errorf("failed to create temporary file: %v", err) + } + var ok bool + defer func() { + if !ok { + _ = os.Remove(tmpFilePath) + } + }() + + // copy the response body to the temporary file and print progress + writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength)) + if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil { + return fmt.Errorf("failed to copy response body: %v", err) + } + + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file: %v", err) + } + + // rename the temporary file to the final file path + if err = os.Rename(tmpFilePath, m.filePath); err != nil { + return fmt.Errorf("failed to rename temporary file: %v", err) + } + + ok = true + return nil +} + +type WriteCounter struct { + ctx context.Context + total uint64 + written uint64 + lastProgressPrinted time.Time +} + +func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter { + return &WriteCounter{ + ctx: ctx, + total: total, + } +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += uint64(n) + + // check if the context is done and return early + select { + case <-wc.ctx.Done(): + return n, wc.ctx.Err() + default: + } + + wc.printProgress() + + return n, nil +} + +func (wc *WriteCounter) printProgress() { + if time.Since(wc.lastProgressPrinted) < 5*time.Second { + return + } + wc.lastProgressPrinted = time.Now() + + log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total))) +} + +func toMB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go new file mode 100644 index 000000000000..ae0be08e043b --- /dev/null +++ b/rollup/missing_header_fields/manager_test.go @@ -0,0 +1,62 @@ +package missing_header_fields + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" + "github.com/scroll-tech/go-ethereum/params" +) + +func TestManagerDownload(t *testing.T) { + t.Skip("skipping test due to long runtime/downloading file") + log.Root().SetHandler(log.StdoutHandler) + + sha256 := *params.ScrollSepoliaChainConfig.Scroll.MissingHeaderFieldsSHA256 + downloadURL := "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/" + params.ScrollSepoliaChainConfig.ChainID.String() + ".bin" + filePath := filepath.Join(t.TempDir(), "test_file_path") + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + + // Check if the file was downloaded and tmp file was removed + _, err = os.Stat(filePath) + require.NoError(t, err) + _, err = os.Stat(filePath + ".tmp") + require.Error(t, err) +} + +func TestManagerChecksum(t *testing.T) { + downloadURL := "" // since the file exists we don't need to download it + filePath := filepath.Join("testdata", "missing-headers.bin") + + // Checksum doesn't match + { + sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) + require.ErrorContains(t, err, "expectedChecksum mismatch") + } + + // Checksum matches + { + sha256 := [32]byte(common.FromHex("635c3f56bb66035bd99134a1e2bc23b34df376f4cd51a0c65e347ce3e65b5974")) + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + difficulty, stateRoot, coinbase, nonce, extra, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].coinbase, coinbase) + require.Equal(t, expectedMissingHeaders[0].nonce, nonce) + require.Equal(t, expectedMissingHeaders[0].extra, extra) + } +} diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/reader.go similarity index 54% rename from rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go rename to rollup/missing_header_fields/reader.go index df88dc95032e..fa8ffe5107b9 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ b/rollup/missing_header_fields/reader.go @@ -1,4 +1,4 @@ -package cmd +package missing_header_fields import ( "bufio" @@ -11,8 +11,6 @@ import ( "github.com/scroll-tech/go-ethereum/core/types" ) -// TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit - type missingHeader struct { headerNum uint64 difficulty uint64 @@ -32,7 +30,7 @@ type Reader struct { func NewReader(filePath string) (*Reader, error) { f, err := os.Open(filePath) if err != nil { - return nil, fmt.Errorf("failed to open file: %v", err) + return nil, fmt.Errorf("failed to open file: %w", err) } r := &Reader{ @@ -40,10 +38,28 @@ func NewReader(filePath string) (*Reader, error) { reader: bufio.NewReader(f), } + if err = r.initialize(); err != nil { + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close file after initialization error: %w", err) + } + return nil, fmt.Errorf("failed to initialize reader: %w", err) + } + + return r, nil +} + +func (r *Reader) initialize() error { + // reset the reader and last read header + if _, err := r.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek to start: %w", err) + } + r.reader = bufio.NewReader(r.file) + r.lastReadHeader = nil + // read the count of unique vanities vanityCount, err := r.reader.ReadByte() if err != nil { - return nil, err + return err } // read the unique vanities @@ -51,17 +67,23 @@ func NewReader(filePath string) (*Reader, error) { for i := uint8(0); i < vanityCount; i++ { var vanity [32]byte if _, err = r.reader.Read(vanity[:]); err != nil { - return nil, err + return err } r.sortedVanities[int(i)] = vanity } - return r, nil + return nil } func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { + if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { + if err = r.initialize(); err != nil { + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + } + } + if r.lastReadHeader == nil { - if _, _, _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } @@ -69,7 +91,7 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has if headerNum > r.lastReadHeader.headerNum { // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { - if _, _, _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } @@ -79,15 +101,14 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil } - // headerNum < r.lastReadHeader.headerNum is not supported - return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) } -func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { +func (r *Reader) ReadNext() (err error) { // read the bitmask bitmaskByte, err := r.reader.ReadByte() if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read bitmask: %v", err) + return fmt.Errorf("failed to read bitmask: %v", err) } bits := newBitMaskFromByte(bitmaskByte) @@ -95,30 +116,31 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t // read the vanity index vanityIndex, err := r.reader.ReadByte() if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read vanity index: %v", err) + return fmt.Errorf("failed to read vanity index: %v", err) } - var stateRoot common.Hash - if _, err := io.ReadFull(r.reader, stateRoot[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read state root: %v", err) + stateRoot := make([]byte, common.HashLength) + if _, err := io.ReadFull(r.reader, stateRoot); err != nil { + return fmt.Errorf("failed to read state root: %v", err) } + var coinbase common.Address if bits.hasCoinbase() { if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read coinbase: %v", err) + return fmt.Errorf("failed to read coinbase: %v", err) } } + var nonce types.BlockNonce if bits.hasNonce() { if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read nonce: %v", err) + return fmt.Errorf("failed to read nonce: %v", err) } } seal := make([]byte, bits.sealLen()) - if _, err = io.ReadFull(r.reader, seal); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read seal: %v", err) + return fmt.Errorf("failed to read seal: %v", err) } // construct the extraData field @@ -133,7 +155,7 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t r.lastReadHeader = &missingHeader{ headerNum: 0, difficulty: uint64(bits.difficulty()), - stateRoot: stateRoot, + stateRoot: common.BytesToHash(stateRoot), coinbase: coinbase, nonce: nonce, extraData: b.Bytes(), @@ -141,15 +163,59 @@ func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce t } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) - r.lastReadHeader.stateRoot = stateRoot + r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) r.lastReadHeader.coinbase = coinbase r.lastReadHeader.nonce = nonce r.lastReadHeader.extraData = b.Bytes() } - return difficulty, coinbase, nonce, b.Bytes(), nil + return nil } func (r *Reader) Close() error { return r.file.Close() } + +// bitMask is a bitmask that encodes the following information: +// +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} + +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go new file mode 100644 index 000000000000..4486491f6a17 --- /dev/null +++ b/rollup/missing_header_fields/reader_test.go @@ -0,0 +1,79 @@ +package missing_header_fields + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" +) + +type header struct { + number uint64 + difficulty uint64 + stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce + extra []byte +} + +var expectedMissingHeaders = []header{ + {0, 2, common.HexToHash("0x195dc9e93ed59fcd1d51e3262739761574b1d1518c6188e27a28357d9d93fb36"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x1c652497074d4a193fb16d61ffd6fc7727983e2d9e7010ac9ca31c241bdec3cb"), common.HexToAddress("0x687E0E85AD67ff71aC134CF61b65905b58Ab43b2"), types.BlockNonce(common.FromHex("ffffffffffffffff")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000228b5b48f65de89b35c77ae3791bfe26c1a3a91cffa2f30f50a58208f2b013ed1a86d1f5601e41b90a5345518e967985f2e949e53476ffc6d8781fa64e4e999101")}, + {2, 2, common.HexToHash("0x2ec6f0f086734c1d88f8ca915cbd2968804688c384aa6d95982435dc65ed476f"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000dbe3e10ae4c0e34d69bff96e86d4a0221361626e8cade7d9d69d870e37f7338f06a446b5461edd7f85d5dda1c54a41b9d0877e5fe9965c6a89a5f4aa3d0d6adc00")}, + {3, 2, common.HexToHash("0x16fc61ab25b479c4c367a85e3f537a598b92fb0e188464cbaca70b5dec08908d"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f649dccc68b8a96a7d06d4a69e9667c63f90c2a2819870c695c2ef95e6862df55f5953891a5f9a13dfafbdff4d23a53a5a6ded505fc87203ac78142bb787dcf200")}, + {4, 2, common.HexToHash("0x001b539a66e87624114117e4d643d5aac7c716b6468c82a8daffea747d16b1a4"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e7578000000000000009e49f8090a1e4941660730b5f11bc0d648da7c04cf3c608f5e0725395c9690f44e1f1c46a9ccd26d3d4db7db973749acb83b88b1564352fe70afc30245242b6c01")}, + {5, 2, common.HexToHash("0x264d833b19677ec62af0f34a4040b9f8e50b3914a16bf961b7d1e198902f127a"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f30ac95495ebb0a35a44657f0c361a7d1ea2d0a086ff7036f84875b47082094504809634f82e379b5ece2f43eead70b12e5d44c1befc4bb8763fac92b4fe8fbb01")}, + {6, 2, common.HexToHash("0x25ed6f6829966b24668510d4c828a596da881001470ab4e7e633761b6bdaba45"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000c9e8cf7bf35df3de8b7d0e98426a62a9e8ec302592f5622d527845ad78b53176558730f88dc79486ada7ed7aea3de4f8ca881b612440e31ab3b84ec7dff0cace01")}, + {7, 2, common.HexToHash("0x0e5e6c49b7c7cbcf3392c52f64b287c5a974b30f20f6695024231b5cf2155d0e"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d8ebea45d74ad882718f97488083530d2d968edf1d3ff6d18d2c757feb0e40206b158f9d4e829a795849a1e3148dd3a6f5251fa600631a74825e3ea041222f5001")}, + {8, 2, common.HexToHash("0x2f6d586a8ce1fc4f476887aaffd8b143d4f5604ce2242f945ae5e8874ede7084"), common.HexToAddress("0x48C3F81f3D998b6652900e1C3183736C238Fe429"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000cecebe754d3c81738e2e76e6a0b34756006b38cd1237728ecf2f41f9a6e325634941b1b29e0e4c1338d1c0fb1190da361a5880253b116493e7cff288f9f165e300")}, + {9, 2, common.HexToHash("0x2c7f91ed6610d3823da4ed730968c40d62a99b5b6245e3f0d4f83011c3be9422"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d6fa3b8ca99ca18fab9096d6791e70b7468a901f4cda0f48e7218e9807e79cee7b4fac8edeb23be5a593bbf9e0fc8ab678c0c58e7b4fb11869a1a4ac0f93657300")}, + {10, 2, common.HexToHash("0x1ae6b4a4b4f311a4f4a1b944445a32197f60d8070adcf7b92ed1f1cc42766504"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e75780000000000000016d5b03fbb592eb6cc5ec7ac98acc10b1b02184b69348b0ffb87817077e3066765573d90201d0b6690d49be1fda5a76f9edfddbdd45ebc69db3d59857e13bdf101")}, +} + +func TestReader_Read(t *testing.T) { + expectedVanities := map[int][32]byte{ + 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), + 1: [32]byte(common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000")), + } + + reader, err := NewReader("testdata/missing-headers.bin") + require.NoError(t, err) + + require.Len(t, reader.sortedVanities, len(expectedVanities)) + for i, expectedVanity := range expectedVanities { + require.Equal(t, expectedVanity, reader.sortedVanities[i]) + } + + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 5) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) + + readAndAssertHeader(t, reader, expectedMissingHeaders, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders, 10) + + // no data anymore + _, _, _, _, _, err = reader.Read(11) + require.Error(t, err) +} + +func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { + difficulty, stateRoot, coinbase, nonce, extra, err := reader.Read(headerNum) + require.NoError(t, err) + require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) + require.Equalf(t, expectedHeaders[headerNum].coinbase, coinbase, "expected coinbase %s, got %s", expectedHeaders[headerNum].coinbase.Hex(), coinbase.Hex()) + require.Equalf(t, expectedHeaders[headerNum].nonce, nonce, "expected nonce %s, got %s", common.Bytes2Hex(expectedHeaders[headerNum].nonce[:]), common.Bytes2Hex(nonce[:])) + require.Equal(t, expectedHeaders[headerNum].extra, extra) +} diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin new file mode 100644 index 0000000000000000000000000000000000000000..45674ccefc43ebb2904b6cf7d1f5aaefa118bf0a GIT binary patch literal 1222 zcmZQ#AOLPOv$FEEq^Fi-bfo7S>KPg88MfwR=9N}J^))a^#-4m>cXj?**}%tY>Xv1q zB^xgW_9S*Zs?sp6ojdurnaAND^0qU((}E`O$rv^l=P36-(Ij9gVQB+KnN*eO?7m)- z_8W5(|6lu4u0F#~cV22I03-Z5O{L$T{xh3U&?bn^xKS>80mW zld}KEZdAxG^YfcIk&%H>@7RYAZN)ya9Y0P@jM=N1(cso`xMfxD)EO$KcT(TF=ih$( zkn72T$G(~Sf9ADaS)e4Gn3UJE_W8|gbKCjMzZ>_nEpglGCU>{K_3GV)N4*?(UTCk2 ze>p8Ct8?j>RkplYcNiEL#r`C&R^3v0tjwxqyRvNFwDe>46AmqEoKbLW>Y?{jpS9_JjgJiOl!~7ve1HAk|Gvsgt)g<@ z2E?BzVqQ}rqP@NS&L^N14AQ}~(q5FQ2s(;Cx#oOz)$!wE+uVAZR^0mks-#wIE@nmc35|8 z408_pSFrvtldFn$4ltn9e4DK$)9a3DefaF<;IQ+@Q*KL8O6nruL&NOE&W>e9H__QI?MRBpSs~jS3j3#fG&BeE>Q3vsFKhqpVy{kV{cAo4g>2FOxqeNvDkD4>J{IY{yzHy0{4$1Q zsgkBRy9JI)`T>jrjQY6|Sv?Q`_`7Gculj#uqw|%o2|kZh^ru8UebMfBub>56K00@9 zU#YK6V{xBe*|PLEzu%AN=XsR&n5H zU$y^6iOhvd*6>eGEe2X4^=!+MEuRIKd|9~D#U;u}vOeJkJJ+4>JM}JpJmXZB$|81k zgZ=JEy=`aWj<1=q<{-B;lZ1DsNjLxR_Qrzp$871T;kFYLWVzEOT$%mw@6zS@^X}i> SdnIm9=55=^);i(69~lAK Date: Wed, 9 Jul 2025 09:42:26 +0000 Subject: [PATCH 17/17] =?UTF-8?q?chore:=20auto=20version=20bump=E2=80=89[b?= =?UTF-8?q?ot]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 523d5ca47c51..555a1e557b27 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 5 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 65 // Patch version component of the current release + VersionPatch = 66 // Patch version component of the current release VersionMeta = "mainnet" // Version metadata to append to the version string )