From 4561826b67e41f581c238e635371a37b718c8325 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 09:37:32 +0200 Subject: [PATCH 01/18] implement missing header fields reader and manager --- rollup/missing_header_fields/manager.go | 185 ++++++++++++++++++ rollup/missing_header_fields/manager_test.go | 59 ++++++ rollup/missing_header_fields/reader.go | 153 +++++++++++++++ rollup/missing_header_fields/reader_test.go | 82 ++++++++ .../testdata/missing-headers.bin | Bin 0 -> 956 bytes 5 files changed, 479 insertions(+) create mode 100644 rollup/missing_header_fields/manager.go create mode 100644 rollup/missing_header_fields/manager_test.go create mode 100644 rollup/missing_header_fields/reader.go create mode 100644 rollup/missing_header_fields/reader_test.go create mode 100644 rollup/missing_header_fields/testdata/missing-headers.bin diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go new file mode 100644 index 000000000000..c0ea556ae1e1 --- /dev/null +++ b/rollup/missing_header_fields/manager.go @@ -0,0 +1,185 @@ +package missing_header_fields + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/scroll-tech/go-ethereum/log" +) + +const timeoutDownload = 10 * time.Minute + +// Manager is responsible for managing the missing header fields file. +// It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. +type Manager struct { + ctx context.Context + filePath string + downloadURL string + expectedChecksum [sha256.Size]byte + + reader *Reader +} + +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum [sha256.Size]byte) *Manager { + return &Manager{ + ctx: ctx, + filePath: filePath, + downloadURL: downloadURL, + expectedChecksum: expectedChecksum, + } +} + +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, extraData []byte, err error) { + // lazy initialization: if the reader is not initialized this is the first time we read from the file + if m.reader == nil { + if err = m.initialize(); err != nil { + return 0, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + } + } + + return m.reader.Read(headerNum) +} + +func (m *Manager) initialize() error { + // if the file doesn't exist, download it + if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) { + if err = m.downloadFile(); err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + } + + // verify the expectedChecksum + f, err := os.Open(m.filePath) + if err != nil { + return fmt.Errorf("failed to open file: %v", err) + } + + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + return fmt.Errorf("failed to copy file: %v", err) + } + if err = f.Close(); err != nil { + return fmt.Errorf("failed to close file: %v", err) + } + computedChecksum := h.Sum(nil) + if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x", m.expectedChecksum, computedChecksum) + } + + // finally initialize the reader + reader, err := NewReader(m.filePath) + if err != nil { + return fmt.Errorf("failed to create reader: %v", err) + } + + m.reader = reader + return nil +} +func (m *Manager) Close() error { + if m.reader != nil { + return m.reader.Close() + } + return nil +} + +func (m *Manager) downloadFile() error { + log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL) + + downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload) + defer downloadCtxCancel() + + req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create download request: %v", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %v", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server returned status code %d", resp.StatusCode) + } + + // create a temporary file + tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return fmt.Errorf("failed to create temporary file: %v", err) + } + var ok bool + defer func() { + if !ok { + _ = os.Remove(tmpFilePath) + } + }() + + // copy the response body to the temporary file and print progress + writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength)) + if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil { + return fmt.Errorf("failed to copy response body: %v", err) + } + + if err = tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temporary file: %v", err) + } + + // rename the temporary file to the final file path + if err = os.Rename(tmpFilePath, m.filePath); err != nil { + return fmt.Errorf("failed to rename temporary file: %v", err) + } + + ok = true + return nil +} + +type WriteCounter struct { + ctx context.Context + total uint64 + written uint64 + lastProgressPrinted time.Time +} + +func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter { + return &WriteCounter{ + ctx: ctx, + total: total, + } +} + +func (wc *WriteCounter) Write(p []byte) (int, error) { + n := len(p) + wc.written += uint64(n) + + // check if the context is done and return early + select { + case <-wc.ctx.Done(): + return n, wc.ctx.Err() + default: + } + + wc.printProgress() + + return n, nil +} + +func (wc *WriteCounter) printProgress() { + if time.Since(wc.lastProgressPrinted) < 5*time.Second { + return + } + wc.lastProgressPrinted = time.Now() + + log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total))) +} + +func toMB(bytes uint64) uint64 { + return bytes / 1024 / 1024 +} diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go new file mode 100644 index 000000000000..b9af30273eec --- /dev/null +++ b/rollup/missing_header_fields/manager_test.go @@ -0,0 +1,59 @@ +package missing_header_fields + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/log" +) + +func TestManagerDownload(t *testing.T) { + t.Skip("skipping test due to long runtime/downloading file") + log.Root().SetHandler(log.StdoutHandler) + + // TODO: replace with actual sha256 hash and downloadURL + sha256 := [32]byte(common.FromHex("0x250c097758924bc21d072e8dc57f4a2357ffaafb20e85eacea5c18dfe70e62b4")) + downloadURL := "https://ftp.halifax.rwth-aachen.de/ubuntu-releases/robots.txt" + filePath := filepath.Join(t.TempDir(), "test_file_path") + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + + // Check if the file was downloaded and tmp file was removed + _, err = os.Stat(filePath) + require.NoError(t, err) + _, err = os.Stat(filePath + ".tmp") + require.Error(t, err) +} + +func TestManagerChecksum(t *testing.T) { + downloadURL := "" // since the file exists we don't need to download it + filePath := filepath.Join("testdata", "missing-headers.bin") + + // Checksum doesn't match + { + sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + _, _, err := manager.GetMissingHeaderFields(0) + require.ErrorContains(t, err, "expectedChecksum mismatch") + } + + // Checksum matches + { + sha256 := [32]byte(common.FromHex("0xfa5d9de3dfdae76a9abd03f7c28274ab223d74a90ed1735e74be1f8fc9c2a435")) + manager := NewManager(context.Background(), filePath, downloadURL, sha256) + + difficulty, extra, err := manager.GetMissingHeaderFields(0) + require.NoError(t, err) + require.Equal(t, expectedMissingHeaders1[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders1[0].extra, extra) + } +} diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go new file mode 100644 index 000000000000..9d7c7d4bda94 --- /dev/null +++ b/rollup/missing_header_fields/reader.go @@ -0,0 +1,153 @@ +package missing_header_fields + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" +) + +type missingHeader struct { + headerNum uint64 + difficulty uint64 + extraData []byte +} + +type Reader struct { + file *os.File + reader *bufio.Reader + sortedVanities map[int][32]byte + lastReadHeader *missingHeader +} + +func NewReader(filePath string) (*Reader, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + + r := &Reader{ + file: f, + reader: bufio.NewReader(f), + } + + // read the count of unique vanities + vanityCount, err := r.reader.ReadByte() + if err != nil { + return nil, fmt.Errorf("failed to read vanity count: %v", err) + } + + // read the unique vanities + r.sortedVanities = make(map[int][32]byte) + for i := uint8(0); i < vanityCount; i++ { + var vanity [32]byte + if _, err = r.reader.Read(vanity[:]); err != nil { + return nil, fmt.Errorf("failed to read vanity: %v", err) + } + r.sortedVanities[int(i)] = vanity + } + + return r, nil +} + +func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) { + if r.lastReadHeader == nil { + if _, _, err = r.ReadNext(); err != nil { + return 0, nil, err + } + } + + if headerNum > r.lastReadHeader.headerNum { + // skip the headers until the requested header number + for i := r.lastReadHeader.headerNum; i < headerNum; i++ { + if _, _, err = r.ReadNext(); err != nil { + return 0, nil, err + } + } + } + + if headerNum == r.lastReadHeader.headerNum { + return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil + } + + // headerNum < r.lastReadHeader.headerNum is not supported + return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) +} + +func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { + // read the bitmask + bitmaskByte, err := r.reader.ReadByte() + if err != nil { + return 0, nil, fmt.Errorf("failed to read bitmask: %v", err) + } + + bits := newBitMask(bitmaskByte) + + // read the vanity index + vanityIndex, err := r.reader.ReadByte() + if err != nil { + return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) + } + + seal := make([]byte, bits.sealLen()) + if _, err = io.ReadFull(r.reader, seal); err != nil { + return 0, nil, fmt.Errorf("failed to read seal: %v", err) + } + + // construct the extraData field + vanity := r.sortedVanities[int(vanityIndex)] + var b bytes.Buffer + b.Write(vanity[:]) + b.Write(seal) + + // we don't have the header number, so we'll just increment the last read header number + // we assume that the headers are written in order, starting from 0 + if r.lastReadHeader == nil { + r.lastReadHeader = &missingHeader{ + headerNum: 0, + difficulty: uint64(bits.difficulty()), + extraData: b.Bytes(), + } + } else { + r.lastReadHeader.headerNum++ + r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.extraData = b.Bytes() + } + + return difficulty, b.Bytes(), nil +} + +func (r *Reader) Close() error { + return r.file.Close() +} + +// bitMask is a bitmask that encodes the following information: +// +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 +type bitMask struct { + b uint8 +} + +func newBitMask(b uint8) bitMask { + return bitMask{b} +} + +func (b bitMask) difficulty() int { + val := (b.b >> 6) & 0x01 + if val == 0 { + return 2 + } else { + return 1 + } +} + +func (b bitMask) sealLen() int { + val := (b.b >> 7) & 0x01 + if val == 0 { + return 65 + } else { + return 85 + } +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go new file mode 100644 index 000000000000..0c1766f2a4cc --- /dev/null +++ b/rollup/missing_header_fields/reader_test.go @@ -0,0 +1,82 @@ +package missing_header_fields + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/scroll-tech/go-ethereum/common" +) + +type header struct { + number uint64 + difficulty uint64 + extra []byte +} + +var expectedMissingHeaders1 = []header{ + {0, 1, common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, + {2, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, + {3, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000012388a2df0f522f96e67564d38be64b5ca7fb37ef9b3f88de875d08653871407584b180917a47dc4abec60bf8da462c617328b9d2da8c4bb9978e018b44ec07401")}, + {4, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000091e57e01b8ed1b433b2bd04e272f9eaf986f3fa728c8fc2b4112352101d24ba76ff1ee64e9a1f8a47c4c49e362e318b2b4767088514f72a7ba9bb7a45b4b447700")}, + {5, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000007ab6b6bd8d52c9beffe935e1bc805d9d4ad62d54485104e80943537d380d6f425ad055ad510c498d1e6efc2aa7e7cc7e1b6166f8421e94b13e291196cba1934a00")}, + {6, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000008f7175ed80593d395069afed2d970e505b076e15642e1f6e3bcb2f589a2a47fa5841868b80fcc70f18cd52de027bdb5ab881fdefc5ebf0d8034cb35e926e89f300")}, + {7, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000006224d2201ba60083743844ce0c2ec4b0ab3e79b69f64eae9a10055fe704380c6410c4f5119cb834f43705c1a785758170a868a38e536432e3a5a5805c83b13801")}, + {8, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000b5c1f5c8aa79582f4b9a66ae8a59561d6c357deaefc7353ea6ace017e76e4b367118e0fd55b9f4cd0235ee1f14222e9b558156b6253e84f71d8048e13643af4801")}, + {9, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000dff93471464bf856b2f633ac16b54c3ff88219a8c83067495ff9f16035c91fb56de8f6914eb4cbd8fbe8a54854e32d697a81408e20cdaa52fed9689d21f7ad0201")}, + {10, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000003c55c63554686f48d9e6dd78b8a7849152e33f169f843e623aa604f2c777eec9539d301fe5f1bea84e5cb7c40e74b723e7700b95eab08bc441d65092b40b548d01")}, + {11, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000013908919960db7ad7459bbeb172049e0dc71fb3a56c7e89bd3699b9ab2cd64d77acc2ba79bb4b9dc98baba0a8e32fd1ca791065b7bd535225f55e254359338d401")}, + {12, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000009b6d0158e0da8bb62c00ff406321b323caf26ffe8d0ea7f181080ea08e236afd63116dea58c752206ff4939240b80aaf1467bb7bb7ad2d8f7d8583f95eec725e01")}, +} + +func TestReader_Read(t *testing.T) { + expectedVanities := map[int][32]byte{ + 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), + 1: [32]byte(common.FromHex("0xd88304031d846765746888676f312e31392e31856c696e757800000000000000")), + } + + reader, err := NewReader("testdata/missing-headers.bin") + require.NoError(t, err) + + require.Len(t, reader.sortedVanities, len(expectedVanities)) + for i, expectedVanity := range expectedVanities { + require.Equal(t, expectedVanity, reader.sortedVanities[i]) + } + + readAndAssertHeader(t, reader, expectedMissingHeaders1, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) + + // we don't allow reading previous headers + _, _, err = reader.Read(5) + require.Error(t, err) + + readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) + + // we don't allow reading previous headers + _, _, err = reader.Read(5) + require.Error(t, err) + + // we don't allow reading previous headers + _, _, err = reader.Read(6) + require.Error(t, err) + + readAndAssertHeader(t, reader, expectedMissingHeaders1, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 10) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 11) + readAndAssertHeader(t, reader, expectedMissingHeaders1, 12) + + // no data anymore + _, _, err = reader.Read(13) + require.Error(t, err) +} + +func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { + difficulty, extra, err := reader.Read(headerNum) + require.NoError(t, err) + require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equal(t, expectedHeaders[headerNum].extra, extra) +} diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin new file mode 100644 index 0000000000000000000000000000000000000000..86506bd7ecb409e44c5e696b7866e8b9b134dc87 GIT binary patch literal 956 zcmZQ#AOLPOvoOoHq^Fi-bfo7S>KR(<8MfwR=9N}J^&MdFIQ&E2c4l{4&;&ji!{*`~ z<^Cs{1PmoYno+W8>+ulP%8s8N*`i&>1=C!MiRoEN}f4bS5DY79sl#zi^xn{Cy#eO#Zw#7Vm3OBrT`|z!`P>yfT zxz{pN?QV!&%5-*r_;i|TNQ{HSrv&d(m!8wX34c$Uw>uik$olY_Xw?CY5VGjf{qR-k zXI^@kuf@KUt*7cY*Zti5qxVJWg|^^!5%vgg2~P1PwMSOJN!Z`JBlk;um3Fq)zTl^>KX3hveEa1ZC;n(V3YjW0Uh-a^|M6YQ%Y{Fd z)cANlPI@e{X-io_N1%Vv@?EpHFNyYcDQ93{tlGA1Z*S1aeg9vYKHSp~JJ;))Ziq)9 z%L`8D;93jbe5a@jp=$$qJbUHx{%9?Kex^=3G3|$w+?0)Wnu60#FP!WJG@`$-^ld|= ztz|&w`nS5%`2wQZ^F&khqpGea{m?K*w)?f=QzK_*`Rw&)wiQ|H2!^m^!0}u z%s!jrCgpX01{%Sp+ZHQ^!XaNk1tp~rJSXCLJ?>#GRT~}n7Y>sK|tM|uE z?Ut>1ApShh+pJLH!QarGU(Pa_zLOVG(wiOH7`9E-uI0OIgU3TN=k*>yBkun+DRlGx z5w_`@@fxwMKK4JFBv+g;NcW8Y`7y!tr2N+07vCoOZ8?48_lu<-A&+%4s~R2p6wa;+ z`gb#9uHyH#Oh6-SLXVk-WaN9?e0H~D$MTkmL67al=C{}-SuJDvbiDlC$>6yL@=rhR zTj3Y8{Rm&lcID><+*4m|=sx0jEnw0X?vP%f5yBHXC8zOjUt1En`?a`&=YuNEN)vwR|3boEtJrTEZCA*Pcpt^kdgoy!>U z;8yoG9fto7$%>nmPkqY&*UPv3Vh{;RHvf!!QxwMt0C;$c4FCWD literal 0 HcmV?d00001 From a47ccd84f6fb3572983eb03cf05860f46d24a722 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 07:40:45 +0000 Subject: [PATCH 02/18] =?UTF-8?q?chore:=20auto=20version=20bump=E2=80=89[b?= =?UTF-8?q?ot]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index dd5c54baa4e8..7a5726b9fd67 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 5 // Major version component of the current release VersionMinor = 8 // Minor version component of the current release - VersionPatch = 51 // Patch version component of the current release + VersionPatch = 52 // Patch version component of the current release VersionMeta = "mainnet" // Version metadata to append to the version string ) From 5774441b9d76147b275112b8cef3845c55dc9db6 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 12:00:33 +0200 Subject: [PATCH 03/18] increase download timeout --- rollup/missing_header_fields/manager.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go index c0ea556ae1e1..e3becd0c225b 100644 --- a/rollup/missing_header_fields/manager.go +++ b/rollup/missing_header_fields/manager.go @@ -11,10 +11,11 @@ import ( "os" "time" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/log" ) -const timeoutDownload = 10 * time.Minute +const timeoutDownload = 30 * time.Minute // Manager is responsible for managing the missing header fields file. // It lazily downloads the file if it doesn't exist, verifies its expectedChecksum and provides the missing header fields. @@ -22,12 +23,12 @@ type Manager struct { ctx context.Context filePath string downloadURL string - expectedChecksum [sha256.Size]byte + expectedChecksum common.Hash reader *Reader } -func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum [sha256.Size]byte) *Manager { +func NewManager(ctx context.Context, filePath string, downloadURL string, expectedChecksum common.Hash) *Manager { return &Manager{ ctx: ctx, filePath: filePath, From 66f0aef05ecf70115783cdea4e8494e05183638d Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 12:01:22 +0200 Subject: [PATCH 04/18] sanitize BaseFee when executing blocks from DA --- core/blockchain.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 57a803198fa1..93bd5e1bc21e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1880,15 +1880,17 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() + // sanitize base fee + if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { + header.BaseFee = nil + } + tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil) receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig) if err != nil { return nil, NonStatTy, fmt.Errorf("error processing block: %w", err) } - // TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique - // This should be done with https://github.com/scroll-tech/go-ethereum/pull/913. - if sign { // Prevent Engine from overriding timestamp. originalTime := header.Time From 7a48a9207e4921e75cccf25479a3691ade9d5a49 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 12:03:11 +0200 Subject: [PATCH 05/18] initialize and pass missing header manager to DA syncing pipeline --- cmd/utils/flags.go | 11 ++++++++ eth/backend.go | 29 ++++++++++++++++++-- node/config.go | 2 ++ params/config.go | 40 ++++++++++++++++++---------- rollup/da_syncer/block_queue.go | 15 ++++++----- rollup/da_syncer/da/commitV0.go | 12 ++++++--- rollup/da_syncer/da/commitV7.go | 3 ++- rollup/da_syncer/da/da.go | 3 ++- rollup/da_syncer/syncing_pipeline.go | 5 ++-- 9 files changed, 91 insertions(+), 29 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7371a3770074..93548b10c746 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -898,6 +898,12 @@ var ( Name: "da.sync", Usage: "Enable node syncing from DA", } + DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ + Name: "da.missingheaderfields.baseurl", + Usage: "Base URL for fetching missing header fields", + Value: "https://missingheaderfields.scroll.io", // TODO: add actual base url + } + DABlobScanAPIEndpointFlag = cli.StringFlag{ Name: "da.blob.blobscan", Usage: "BlobScan blob API endpoint", @@ -1382,6 +1388,11 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } + cfg.DAMissingHeaderFieldsBaseURL = DAMissingHeaderFieldsBaseURLFlag.Value + if ctx.GlobalIsSet(DAMissingHeaderFieldsBaseURLFlag.Name) { + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) + } + if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) } diff --git a/eth/backend.go b/eth/backend.go index ad765b835b0a..4eebee9343c5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,9 @@ import ( "errors" "fmt" "math/big" + "net/url" + "path" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -61,6 +64,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/ccc" "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service" "github.com/scroll-tech/go-ethereum/rollup/sync_service" "github.com/scroll-tech/go-ethereum/rpc" @@ -239,9 +243,14 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether // simply let them run simultaneously. If messages are missing in DA syncing, it will be handled by the syncing pipeline // by waiting and retrying. if config.EnableDASyncing { + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { - eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA) + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) } @@ -255,7 +264,7 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if err != nil { return nil, fmt.Errorf("cannot initialize L1 sync service: %w", err) } - eth.syncService.Start() + //eth.syncService.Start() if config.EnableRollupVerify { // initialize and start rollup event sync service @@ -337,6 +346,22 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether return eth, nil } +func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.ChainConfig) (*missing_header_fields.Manager, error) { + downloadURL, err := url.Parse(stack.Config().DAMissingHeaderFieldsBaseURL) + if err != nil { + return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) + } + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()) + + expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 + if expectedSHA256Checksum == nil { + return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") + } + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) + fmt.Println(filePath) + return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/node/config.go b/node/config.go index 6e7133a95129..d7212389684e 100644 --- a/node/config.go +++ b/node/config.go @@ -201,6 +201,8 @@ type Config struct { L1DisableMessageQueueV2 bool `toml:",omitempty"` // Is daSyncingEnabled DaSyncingEnabled bool `toml:",omitempty"` + // Base URL for missing header fields file + DAMissingHeaderFieldsBaseURL string `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/params/config.go b/params/config.go index 96b77e04a7f4..582eea614005 100644 --- a/params/config.go +++ b/params/config.go @@ -30,16 +30,18 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") - ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") - ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") - ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") - ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + SepoliaGenesisHash = common.HexToHash("0x25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") + ScrollAlphaGenesisHash = common.HexToHash("0xa4fc62b9b0643e345bdcebe457b3ae898bef59c7203c3db269200055e037afda") + ScrollSepoliaGenesisHash = common.HexToHash("0xaa62d1a8b2bffa9e5d2368b63aae0d98d54928bd713125e3fd9e5c896c68592c") + ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") + ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") + ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0x33e75b9c794c90aacbbb14d784e72c9d63b5cb414e6d8229e11eb344dfa240bd") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0x4312a3496e5db202fc13e8a7daa9e85efde76c4a2f8f077e8777e55708d22e28") ) func newUint64(val uint64) *uint64 { return &val } @@ -353,7 +355,8 @@ var ( ScrollChainAddress: common.HexToAddress("0x2D567EcE699Eabe5afCd141eDB7A4f2D0D6ce8a0"), L2SystemConfigAddress: common.HexToAddress("0xF444cF06A3E3724e20B35c2989d3942ea8b59124"), }, - GenesisStateRoot: &ScrollSepoliaGenesisState, + GenesisStateRoot: &ScrollSepoliaGenesisState, + MissingHeaderFieldsSHA256: &ScrollSepoliaMissingHeaderFieldsSHA256, }, } @@ -404,7 +407,8 @@ var ( ScrollChainAddress: common.HexToAddress("0xa13BAF47339d63B743e7Da8741db5456DAc1E556"), L2SystemConfigAddress: common.HexToAddress("0x331A873a2a85219863d80d248F9e2978fE88D0Ea"), }, - GenesisStateRoot: &ScrollMainnetGenesisState, + GenesisStateRoot: &ScrollMainnetGenesisState, + MissingHeaderFieldsSHA256: &ScrollMainnetMissingHeaderFieldsSHA256, }, } @@ -706,6 +710,9 @@ type ScrollConfig struct { // Genesis State Root for MPT clients GenesisStateRoot *common.Hash `json:"genesisStateRoot,omitempty"` + + // MissingHeaderFieldsSHA256 is the SHA256 hash of the missing header fields file. + MissingHeaderFieldsSHA256 *common.Hash `json:"missingHeaderFieldsSHA256,omitempty"` } // L1Config contains the l1 parameters needed to sync l1 contract events (e.g., l1 messages, commit/revert/finalize batches) in the sequencer @@ -756,8 +763,13 @@ func (s ScrollConfig) String() string { genesisStateRoot = fmt.Sprintf("%v", *s.GenesisStateRoot) } - return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v}", - s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot) + missingHeaderFieldsSHA256 := "" + if s.MissingHeaderFieldsSHA256 != nil { + missingHeaderFieldsSHA256 = fmt.Sprintf("%v", *s.MissingHeaderFieldsSHA256) + } + + return fmt.Sprintf("{useZktrie: %v, maxTxPerBlock: %v, MaxTxPayloadBytesPerBlock: %v, feeVaultAddress: %v, l1Config: %v, genesisStateRoot: %v, missingHeaderFieldsSHA256: %v}", + s.UseZktrie, maxTxPerBlock, maxTxPayloadBytesPerBlock, s.FeeVaultAddress, s.L1Config.String(), genesisStateRoot, missingHeaderFieldsSHA256) } // IsValidTxCount returns whether the given block's transaction count is below the limit. diff --git a/rollup/da_syncer/block_queue.go b/rollup/da_syncer/block_queue.go index 630382f001c0..30d53bfcb6a5 100644 --- a/rollup/da_syncer/block_queue.go +++ b/rollup/da_syncer/block_queue.go @@ -6,19 +6,22 @@ import ( "github.com/scroll-tech/go-ethereum/core/rawdb" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and // provides them to the next stage one-by-one. type BlockQueue struct { - batchQueue *BatchQueue - blocks []*da.PartialBlock + batchQueue *BatchQueue + blocks []*da.PartialBlock + missingHeaderFieldsManager *missing_header_fields.Manager } -func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue { +func NewBlockQueue(batchQueue *BatchQueue, missingHeaderFieldsManager *missing_header_fields.Manager) *BlockQueue { return &BlockQueue{ - batchQueue: batchQueue, - blocks: make([]*da.PartialBlock, 0), + batchQueue: batchQueue, + blocks: make([]*da.PartialBlock, 0), + missingHeaderFieldsManager: missingHeaderFieldsManager, } } @@ -40,7 +43,7 @@ func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error { return err } - bq.blocks, err = entryWithBlocks.Blocks() + bq.blocks, err = entryWithBlocks.Blocks(bq.missingHeaderFieldsManager) if err != nil { return fmt.Errorf("failed to get blocks from entry: %w", err) } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index c99a474b256b..1a3ec0167f14 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/log" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type CommitBatchDAV0 struct { @@ -109,7 +110,7 @@ func (c *CommitBatchDAV0) CompareTo(other Entry) int { return 0 } -func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) { l1Txs, err := getL1Messages(c.db, c.parentTotalL1MessagePopped, c.skippedL1MessageBitmap, c.l1MessagesPopped) if err != nil { return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", c.batchIndex, err) @@ -134,14 +135,19 @@ func (c *CommitBatchDAV0) Blocks() ([]*PartialBlock, error) { // insert l2 txs txs = append(txs, chunk.Transactions[blockId]...) + difficulty, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + if err != nil { + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", blockId, err) + } + block := NewPartialBlock( &PartialHeader{ Number: daBlock.Number(), Time: daBlock.Timestamp(), BaseFee: daBlock.BaseFee(), GasLimit: daBlock.GasLimit(), - Difficulty: 10, // TODO: replace with real difficulty - ExtraData: []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data + Difficulty: difficulty, + ExtraData: extraData, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/commitV7.go b/rollup/da_syncer/da/commitV7.go index 6048b853aee7..37a4b948df6c 100644 --- a/rollup/da_syncer/da/commitV7.go +++ b/rollup/da_syncer/da/commitV7.go @@ -13,6 +13,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" @@ -113,7 +114,7 @@ func (c *CommitBatchDAV7) Event() l1.RollupEvent { return c.event } -func (c *CommitBatchDAV7) Blocks() ([]*PartialBlock, error) { +func (c *CommitBatchDAV7) Blocks(_ *missing_header_fields.Manager) ([]*PartialBlock, error) { initialL1MessageIndex := c.parentTotalL1MessagePopped l1Txs, err := getL1MessagesV7(c.db, c.blobPayload.Blocks(), initialL1MessageIndex) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index fe72473451b6..eaef6c8bf985 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -8,6 +8,7 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) type Type int @@ -34,7 +35,7 @@ type Entry interface { type EntryWithBlocks interface { Entry - Blocks() ([]*PartialBlock, error) + Blocks(manager *missing_header_fields.Manager) ([]*PartialBlock, error) Version() encoding.CodecVersion Chunks() []*encoding.DAChunkRawTx BlobVersionedHashes() []common.Hash diff --git a/rollup/da_syncer/syncing_pipeline.go b/rollup/da_syncer/syncing_pipeline.go index 080179107f8c..19b7bcbc07de 100644 --- a/rollup/da_syncer/syncing_pipeline.go +++ b/rollup/da_syncer/syncing_pipeline.go @@ -16,6 +16,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // Config is the configuration parameters of data availability syncing. @@ -50,7 +51,7 @@ type SyncingPipeline struct { daQueue *DAQueue } -func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) { +func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient l1.Client, l1DeploymentBlock uint64, config Config, missingHeaderFieldsManager *missing_header_fields.Manager) (*SyncingPipeline, error) { l1Reader, err := l1.NewReader(ctx, l1.Config{ ScrollChainAddress: genesisConfig.Scroll.L1Config.ScrollChainAddress, L1MessageQueueAddress: genesisConfig.Scroll.L1Config.L1MessageQueueAddress, @@ -124,7 +125,7 @@ func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesi daQueue := NewDAQueue(lastProcessedBatchMeta.L1BlockNumber, dataSourceFactory) batchQueue := NewBatchQueue(daQueue, db, lastProcessedBatchMeta) - blockQueue := NewBlockQueue(batchQueue) + blockQueue := NewBlockQueue(batchQueue, missingHeaderFieldsManager) daSyncer := NewDASyncer(blockchain, config.L2EndBlock) ctx, cancel := context.WithCancel(ctx) From 2ef390e5959c64334f4257a537b66abcf5f3e34b Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 20:03:07 +0200 Subject: [PATCH 06/18] add state root to deduplicated header --- rollup/missing_header_fields/manager.go | 4 +- rollup/missing_header_fields/manager_test.go | 9 +++-- rollup/missing_header_fields/reader.go | 28 ++++++++----- rollup/missing_header_fields/reader_test.go | 38 +++++++++--------- .../testdata/missing-headers.bin | Bin 956 -> 1174 bytes 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go index e3becd0c225b..17763c653b8d 100644 --- a/rollup/missing_header_fields/manager.go +++ b/rollup/missing_header_fields/manager.go @@ -37,11 +37,11 @@ func NewManager(ctx context.Context, filePath string, downloadURL string, expect } } -func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, extraData []byte, err error) { +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { // lazy initialization: if the reader is not initialized this is the first time we read from the file if m.reader == nil { if err = m.initialize(); err != nil { - return 0, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + return 0, common.Hash{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) } } diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go index b9af30273eec..e6f7dc8e5fa7 100644 --- a/rollup/missing_header_fields/manager_test.go +++ b/rollup/missing_header_fields/manager_test.go @@ -22,7 +22,7 @@ func TestManagerDownload(t *testing.T) { filePath := filepath.Join(t.TempDir(), "test_file_path") manager := NewManager(context.Background(), filePath, downloadURL, sha256) - _, _, err := manager.GetMissingHeaderFields(0) + _, _, _, err := manager.GetMissingHeaderFields(0) require.NoError(t, err) // Check if the file was downloaded and tmp file was removed @@ -42,18 +42,19 @@ func TestManagerChecksum(t *testing.T) { manager := NewManager(context.Background(), filePath, downloadURL, sha256) - _, _, err := manager.GetMissingHeaderFields(0) + _, _, _, err := manager.GetMissingHeaderFields(0) require.ErrorContains(t, err, "expectedChecksum mismatch") } // Checksum matches { - sha256 := [32]byte(common.FromHex("0xfa5d9de3dfdae76a9abd03f7c28274ab223d74a90ed1735e74be1f8fc9c2a435")) + sha256 := [32]byte(common.FromHex("e5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502")) manager := NewManager(context.Background(), filePath, downloadURL, sha256) - difficulty, extra, err := manager.GetMissingHeaderFields(0) + difficulty, stateRoot, extra, err := manager.GetMissingHeaderFields(0) require.NoError(t, err) require.Equal(t, expectedMissingHeaders1[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders1[0].stateRoot, stateRoot) require.Equal(t, expectedMissingHeaders1[0].extra, extra) } } diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go index 9d7c7d4bda94..673fae4996c2 100644 --- a/rollup/missing_header_fields/reader.go +++ b/rollup/missing_header_fields/reader.go @@ -6,11 +6,14 @@ import ( "fmt" "io" "os" + + "github.com/scroll-tech/go-ethereum/common" ) type missingHeader struct { headerNum uint64 difficulty uint64 + stateRoot common.Hash extraData []byte } @@ -35,7 +38,7 @@ func NewReader(filePath string) (*Reader, error) { // read the count of unique vanities vanityCount, err := r.reader.ReadByte() if err != nil { - return nil, fmt.Errorf("failed to read vanity count: %v", err) + return nil, err } // read the unique vanities @@ -43,7 +46,7 @@ func NewReader(filePath string) (*Reader, error) { for i := uint8(0); i < vanityCount; i++ { var vanity [32]byte if _, err = r.reader.Read(vanity[:]); err != nil { - return nil, fmt.Errorf("failed to read vanity: %v", err) + return nil, err } r.sortedVanities[int(i)] = vanity } @@ -51,10 +54,10 @@ func NewReader(filePath string) (*Reader, error) { return r, nil } -func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) { +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { if r.lastReadHeader == nil { if _, _, err = r.ReadNext(); err != nil { - return 0, nil, err + return 0, common.Hash{}, nil, err } } @@ -62,17 +65,17 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, er // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { if _, _, err = r.ReadNext(); err != nil { - return 0, nil, err + return 0, common.Hash{}, nil, err } } } if headerNum == r.lastReadHeader.headerNum { - return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil } // headerNum < r.lastReadHeader.headerNum is not supported - return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) } func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { @@ -82,7 +85,7 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { return 0, nil, fmt.Errorf("failed to read bitmask: %v", err) } - bits := newBitMask(bitmaskByte) + bits := newBitMaskFromBytes(bitmaskByte) // read the vanity index vanityIndex, err := r.reader.ReadByte() @@ -90,6 +93,11 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) } + stateRoot := make([]byte, common.HashLength) + if _, err := io.ReadFull(r.reader, stateRoot); err != nil { + return 0, nil, fmt.Errorf("failed to read state root: %v", err) + } + seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { return 0, nil, fmt.Errorf("failed to read seal: %v", err) @@ -107,11 +115,13 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { r.lastReadHeader = &missingHeader{ headerNum: 0, difficulty: uint64(bits.difficulty()), + stateRoot: common.BytesToHash(stateRoot), extraData: b.Bytes(), } } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) + r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) r.lastReadHeader.extraData = b.Bytes() } @@ -130,7 +140,7 @@ type bitMask struct { b uint8 } -func newBitMask(b uint8) bitMask { +func newBitMaskFromBytes(b uint8) bitMask { return bitMask{b} } diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go index 0c1766f2a4cc..aba64d9586fa 100644 --- a/rollup/missing_header_fields/reader_test.go +++ b/rollup/missing_header_fields/reader_test.go @@ -11,23 +11,22 @@ import ( type header struct { number uint64 difficulty uint64 + stateRoot common.Hash extra []byte } var expectedMissingHeaders1 = []header{ - {0, 1, common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, - {1, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, - {2, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, - {3, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000012388a2df0f522f96e67564d38be64b5ca7fb37ef9b3f88de875d08653871407584b180917a47dc4abec60bf8da462c617328b9d2da8c4bb9978e018b44ec07401")}, - {4, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000091e57e01b8ed1b433b2bd04e272f9eaf986f3fa728c8fc2b4112352101d24ba76ff1ee64e9a1f8a47c4c49e362e318b2b4767088514f72a7ba9bb7a45b4b447700")}, - {5, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000007ab6b6bd8d52c9beffe935e1bc805d9d4ad62d54485104e80943537d380d6f425ad055ad510c498d1e6efc2aa7e7cc7e1b6166f8421e94b13e291196cba1934a00")}, - {6, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000008f7175ed80593d395069afed2d970e505b076e15642e1f6e3bcb2f589a2a47fa5841868b80fcc70f18cd52de027bdb5ab881fdefc5ebf0d8034cb35e926e89f300")}, - {7, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000006224d2201ba60083743844ce0c2ec4b0ab3e79b69f64eae9a10055fe704380c6410c4f5119cb834f43705c1a785758170a868a38e536432e3a5a5805c83b13801")}, - {8, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000b5c1f5c8aa79582f4b9a66ae8a59561d6c357deaefc7353ea6ace017e76e4b367118e0fd55b9f4cd0235ee1f14222e9b558156b6253e84f71d8048e13643af4801")}, - {9, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000dff93471464bf856b2f633ac16b54c3ff88219a8c83067495ff9f16035c91fb56de8f6914eb4cbd8fbe8a54854e32d697a81408e20cdaa52fed9689d21f7ad0201")}, - {10, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000003c55c63554686f48d9e6dd78b8a7849152e33f169f843e623aa604f2c777eec9539d301fe5f1bea84e5cb7c40e74b723e7700b95eab08bc441d65092b40b548d01")}, - {11, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000013908919960db7ad7459bbeb172049e0dc71fb3a56c7e89bd3699b9ab2cd64d77acc2ba79bb4b9dc98baba0a8e32fd1ca791065b7bd535225f55e254359338d401")}, - {12, 2, common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000009b6d0158e0da8bb62c00ff406321b323caf26ffe8d0ea7f181080ea08e236afd63116dea58c752206ff4939240b80aaf1467bb7bb7ad2d8f7d8583f95eec725e01")}, + {0, 1, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, + {2, 2, common.HexToHash("0x11787ec3c17489215d0b13d594db83be55736f3933625aac0e1bba2812d49ffe"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, + {3, 2, common.HexToHash("0x2b4cff60622970fe15dfe6e7a8c8bf9619aea5790b57b9bc34811dd6670c16bf"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000012388a2df0f522f96e67564d38be64b5ca7fb37ef9b3f88de875d08653871407584b180917a47dc4abec60bf8da462c617328b9d2da8c4bb9978e018b44ec07401")}, + {4, 2, common.HexToHash("0x032c5535bc0684d25cd74a6bdc9f15052c9444ed8bfe5e9e4791d83f52b8dac1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000091e57e01b8ed1b433b2bd04e272f9eaf986f3fa728c8fc2b4112352101d24ba76ff1ee64e9a1f8a47c4c49e362e318b2b4767088514f72a7ba9bb7a45b4b447700")}, + {5, 2, common.HexToHash("0x0e13888dc93db27a55825d45b27d2b7d367ac9b6a302bd3480177253320f815a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000007ab6b6bd8d52c9beffe935e1bc805d9d4ad62d54485104e80943537d380d6f425ad055ad510c498d1e6efc2aa7e7cc7e1b6166f8421e94b13e291196cba1934a00")}, + {6, 2, common.HexToHash("0x29fee2024813a75d49d4670716e0f706eac9d72f091f64db1f802d346a1f2a58"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000008f7175ed80593d395069afed2d970e505b076e15642e1f6e3bcb2f589a2a47fa5841868b80fcc70f18cd52de027bdb5ab881fdefc5ebf0d8034cb35e926e89f300")}, + {7, 2, common.HexToHash("0x1d9ce9ddef4ab5c0d120715cab5b4eeef6ae8b94addb7a8bdfcb6c8f76f63e34"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000006224d2201ba60083743844ce0c2ec4b0ab3e79b69f64eae9a10055fe704380c6410c4f5119cb834f43705c1a785758170a868a38e536432e3a5a5805c83b13801")}, + {8, 2, common.HexToHash("0x1c9f95b06e3e65cdd0dae9d2b20de7760d9b6c768b05d043e0d0f493112d32f2"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000b5c1f5c8aa79582f4b9a66ae8a59561d6c357deaefc7353ea6ace017e76e4b367118e0fd55b9f4cd0235ee1f14222e9b558156b6253e84f71d8048e13643af4801")}, + {9, 2, common.HexToHash("0x19c478ece75320a6c4a3d1cecfd219f130a8b377e47b76adaa3eb574396bacb1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000dff93471464bf856b2f633ac16b54c3ff88219a8c83067495ff9f16035c91fb56de8f6914eb4cbd8fbe8a54854e32d697a81408e20cdaa52fed9689d21f7ad0201")}, + {10, 2, common.HexToHash("0x00a85e26b58e8294115b097a808ba9840326981efb8c4abb054eb40494e5bc0a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000003c55c63554686f48d9e6dd78b8a7849152e33f169f843e623aa604f2c777eec9539d301fe5f1bea84e5cb7c40e74b723e7700b95eab08bc441d65092b40b548d01")}, } func TestReader_Read(t *testing.T) { @@ -50,33 +49,32 @@ func TestReader_Read(t *testing.T) { readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) // we don't allow reading previous headers - _, _, err = reader.Read(5) + _, _, _, err = reader.Read(5) require.Error(t, err) readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) // we don't allow reading previous headers - _, _, err = reader.Read(5) + _, _, _, err = reader.Read(5) require.Error(t, err) // we don't allow reading previous headers - _, _, err = reader.Read(6) + _, _, _, err = reader.Read(6) require.Error(t, err) readAndAssertHeader(t, reader, expectedMissingHeaders1, 9) readAndAssertHeader(t, reader, expectedMissingHeaders1, 10) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 11) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 12) // no data anymore - _, _, err = reader.Read(13) + _, _, _, err = reader.Read(11) require.Error(t, err) } func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { - difficulty, extra, err := reader.Read(headerNum) + difficulty, stateRoot, extra, err := reader.Read(headerNum) require.NoError(t, err) require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) + require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) require.Equal(t, expectedHeaders[headerNum].extra, extra) } diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin index 86506bd7ecb409e44c5e696b7866e8b9b134dc87..96068b3f606f837ddbb24abfadb9c3f5cd9c0505 100644 GIT binary patch delta 410 zcmdnPK8y}!xJCZI{Y(DE{C%4;W;=h3U*@xQqH{Q5HM5)* z1S{$eA1vuqjO7--I^}lrzR==)OXH-dHGI;$G=#3q|2Ofjvx2tI|AZvXf`6j-pFLl3 zV*fPBbxSL`!*}j6X_UQ|&Lg&evI3*C0<%u2=^nP0OEK5IvhU0nW!0JD^0xb5+&uS* zH|&FU+&VZp8K{U)xTE)^?WU^GrdZcawc532RVTMCX4-4gAYK$~#NQY-c^OcV=D$Zw z9>U9GJ+Gv*i#_k$ zYxHSyK2VY5k%~9ZgB6w?S$y%_`Ad=?4OVO}e^Onxc9q@M63gs08z*l9Dq>g>r?$1P jX^LPpXH`S@$`)p|8FIgSymqttZDE=6bPw0$uS|{regnZ> delta 184 zcmbQnxrcp%^Tg!viMO05+cG*&UI4_*OwNo7Y{?g!#eKpe{ASkJ&HJfTx^ z8t?YCC6T*diz|3OxKsGsD(v`+*_ShC&)Rf0<$Bc_?d7w#?7TB$*DkI;qrWoCC$dFX zUo};V4}BD3I@#h1BLm~?T*in8x4O6KF#LB&R@|(7>Qnx|UcTiY8#(wE^eJckO%}|3 k6>&UBA^*$dNe(->){CU?uHL>@x4*WPz4>R{o1!>I0AGw!f&c&j From de651d9f39e5a8f00bda2e42852dbddcbc32178a Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 29 May 2025 20:26:43 +0200 Subject: [PATCH 07/18] overwrite state root if given via missing header file --- core/blockchain.go | 6 +++++- params/config.go | 4 ++-- rollup/da_syncer/da/commitV0.go | 9 +++++---- rollup/da_syncer/da/da.go | 2 ++ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 93bd5e1bc21e..2da580a26372 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1903,7 +1903,11 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types // finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble() header.GasUsed = gasUsed - header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + + // state root might be set from partial header. If it is not set, we calculate it. + if header.Root == (common.Hash{}) { + header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number)) + } fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/params/config.go b/params/config.go index 582eea614005..29c3922c0418 100644 --- a/params/config.go +++ b/params/config.go @@ -40,8 +40,8 @@ var ( ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") - ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0x33e75b9c794c90aacbbb14d784e72c9d63b5cb414e6d8229e11eb344dfa240bd") - ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0x4312a3496e5db202fc13e8a7daa9e85efde76c4a2f8f077e8777e55708d22e28") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xe5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502") // TODO: put the correct hash here + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xe5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502") // TODO: put the correct hash here ) func newUint64(val uint64) *uint64 { return &val } diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index 1a3ec0167f14..15f96ed4ed02 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -121,7 +121,7 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par curL1TxIndex := c.parentTotalL1MessagePopped for _, chunk := range c.chunks { - for blockId, daBlock := range chunk.Blocks { + for blockIndex, daBlock := range chunk.Blocks { // create txs txs := make(types.Transactions, 0, daBlock.NumTransactions()) // insert l1 msgs @@ -133,11 +133,11 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par curL1TxIndex += uint64(daBlock.NumL1Messages()) // insert l2 txs - txs = append(txs, chunk.Transactions[blockId]...) + txs = append(txs, chunk.Transactions[blockIndex]...) - difficulty, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + difficulty, stateRoot, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) if err != nil { - return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", blockId, err) + return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) } block := NewPartialBlock( @@ -148,6 +148,7 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par GasLimit: daBlock.GasLimit(), Difficulty: difficulty, ExtraData: extraData, + StateRoot: stateRoot, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index eaef6c8bf985..3a56d326d931 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -54,6 +54,7 @@ type PartialHeader struct { GasLimit uint64 Difficulty uint64 ExtraData []byte + StateRoot common.Hash } func (h *PartialHeader) ToHeader() *types.Header { @@ -64,6 +65,7 @@ func (h *PartialHeader) ToHeader() *types.Header { GasLimit: h.GasLimit, Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, + Root: h.StateRoot, } } From 7d8bb01fe2154cc381de930e6005839fb15c27d4 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Fri, 30 May 2025 07:55:35 +0200 Subject: [PATCH 08/18] fix test --- rollup/rollup_sync_service/rollup_sync_service_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rollup/rollup_sync_service/rollup_sync_service_test.go b/rollup/rollup_sync_service/rollup_sync_service_test.go index e6f6d7339697..68c53f6ac8f5 100644 --- a/rollup/rollup_sync_service/rollup_sync_service_test.go +++ b/rollup/rollup_sync_service/rollup_sync_service_test.go @@ -21,6 +21,7 @@ import ( "github.com/scroll-tech/go-ethereum/rollup/da_syncer" "github.com/scroll-tech/go-ethereum/rollup/da_syncer/da" "github.com/scroll-tech/go-ethereum/rollup/l1" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) func TestGetCommittedBatchMetaCodecV0(t *testing.T) { @@ -185,7 +186,7 @@ func (m mockEntryWithBlocks) Event() l1.RollupEvent { panic("implement me") } -func (m mockEntryWithBlocks) Blocks() ([]*da.PartialBlock, error) { +func (m mockEntryWithBlocks) Blocks(_ *missing_header_fields.Manager) ([]*da.PartialBlock, error) { panic("implement me") } From b90e909de85e947b32918d3508d811ffdf4014c7 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 2 Jun 2025 12:14:13 +0200 Subject: [PATCH 09/18] set correct links and missing header file hashes --- cmd/utils/flags.go | 4 ++-- eth/backend.go | 16 ++++++++-------- params/config.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 93548b10c746..a5a06faa9d37 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -900,8 +900,8 @@ var ( } DAMissingHeaderFieldsBaseURLFlag = cli.StringFlag{ Name: "da.missingheaderfields.baseurl", - Usage: "Base URL for fetching missing header fields", - Value: "https://missingheaderfields.scroll.io", // TODO: add actual base url + Usage: "Base URL for fetching missing header fields for pre-EuclidV2 blocks", + Value: "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/", } DABlobScanAPIEndpointFlag = cli.StringFlag{ diff --git a/eth/backend.go b/eth/backend.go index 4eebee9343c5..3fbefcfe63d2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -243,13 +243,13 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether // simply let them run simultaneously. If messages are missing in DA syncing, it will be handled by the syncing pipeline // by waiting and retrying. if config.EnableDASyncing { - missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) - if err != nil { - return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) - } - // Do not start syncing pipeline if we are producing blocks for permissionless batches. if !config.DA.ProduceBlocks { + missingHeaderFieldsManager, err := createMissingHeaderFieldsManager(stack, chainConfig) + if err != nil { + return nil, fmt.Errorf("cannot create missing header fields manager: %w", err) + } + eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA, missingHeaderFieldsManager) if err != nil { return nil, fmt.Errorf("cannot initialize da syncer: %w", err) @@ -264,7 +264,7 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client l1.Client) (*Ether if err != nil { return nil, fmt.Errorf("cannot initialize L1 sync service: %w", err) } - //eth.syncService.Start() + eth.syncService.Start() if config.EnableRollupVerify { // initialize and start rollup event sync service @@ -351,14 +351,14 @@ func createMissingHeaderFieldsManager(stack *node.Node, chainConfig *params.Chai if err != nil { return nil, fmt.Errorf("invalid DAMissingHeaderFieldsBaseURL: %w", err) } - downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()) + downloadURL.Path = path.Join(downloadURL.Path, chainConfig.ChainID.String()+".bin") expectedSHA256Checksum := chainConfig.Scroll.MissingHeaderFieldsSHA256 if expectedSHA256Checksum == nil { return nil, fmt.Errorf("missing expected SHA256 checksum for missing header fields file in chain config") } + filePath := filepath.Join(stack.Config().DataDir, fmt.Sprintf("missing-header-fields-%s-%s", chainConfig.ChainID, expectedSHA256Checksum.Hex())) - fmt.Println(filePath) return missing_header_fields.NewManager(context.Background(), filePath, downloadURL.String(), *expectedSHA256Checksum), nil } diff --git a/params/config.go b/params/config.go index 29c3922c0418..d85198b9e4aa 100644 --- a/params/config.go +++ b/params/config.go @@ -40,8 +40,8 @@ var ( ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") - ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xe5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502") // TODO: put the correct hash here - ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xe5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502") // TODO: put the correct hash here + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0x9062e2fa1200dca63bee1d18d429572f134f5f0c98cb4852f62fc394e33cf6e6") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0x3629f5e53250a526ffc46806c4d74b9c52c9209a6d45ecdfebdef5d596bb3f40") ) func newUint64(val uint64) *uint64 { return &val } From 51d5c7d4325650513baf38b98f159503c64e8c2d Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Mon, 2 Jun 2025 12:14:46 +0200 Subject: [PATCH 10/18] allow reading of previous headers by resetting file and buffer to support reset of syncing pipeline --- rollup/missing_header_fields/reader.go | 35 +++++++++++++++++---- rollup/missing_header_fields/reader_test.go | 14 +++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go index 673fae4996c2..803d3c483a69 100644 --- a/rollup/missing_header_fields/reader.go +++ b/rollup/missing_header_fields/reader.go @@ -27,7 +27,7 @@ type Reader struct { func NewReader(filePath string) (*Reader, error) { f, err := os.Open(filePath) if err != nil { - return nil, fmt.Errorf("failed to open file: %v", err) + return nil, fmt.Errorf("failed to open file: %w", err) } r := &Reader{ @@ -35,10 +35,28 @@ func NewReader(filePath string) (*Reader, error) { reader: bufio.NewReader(f), } + if err = r.initialize(); err != nil { + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close file after initialization error: %w", err) + } + return nil, fmt.Errorf("failed to initialize reader: %w", err) + } + + return r, nil +} + +func (r *Reader) initialize() error { + // reset the reader and last read header + if _, err := r.file.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("failed to seek to start: %w", err) + } + r.reader = bufio.NewReader(r.file) + r.lastReadHeader = nil + // read the count of unique vanities vanityCount, err := r.reader.ReadByte() if err != nil { - return nil, err + return err } // read the unique vanities @@ -46,15 +64,21 @@ func NewReader(filePath string) (*Reader, error) { for i := uint8(0); i < vanityCount; i++ { var vanity [32]byte if _, err = r.reader.Read(vanity[:]); err != nil { - return nil, err + return err } r.sortedVanities[int(i)] = vanity } - return r, nil + return nil } func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { + if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { + if err = r.initialize(); err != nil { + return 0, common.Hash{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + } + } + if r.lastReadHeader == nil { if _, _, err = r.ReadNext(); err != nil { return 0, common.Hash{}, nil, err @@ -74,8 +98,7 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil } - // headerNum < r.lastReadHeader.headerNum is not supported - return 0, common.Hash{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) } func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go index aba64d9586fa..53f5e08d2752 100644 --- a/rollup/missing_header_fields/reader_test.go +++ b/rollup/missing_header_fields/reader_test.go @@ -48,20 +48,14 @@ func TestReader_Read(t *testing.T) { readAndAssertHeader(t, reader, expectedMissingHeaders1, 1) readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) - // we don't allow reading previous headers - _, _, _, err = reader.Read(5) - require.Error(t, err) + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders1, 5) readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) - // we don't allow reading previous headers - _, _, _, err = reader.Read(5) - require.Error(t, err) - - // we don't allow reading previous headers - _, _, _, err = reader.Read(6) - require.Error(t, err) + // reading previous headers resets the file reader + readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) readAndAssertHeader(t, reader, expectedMissingHeaders1, 9) readAndAssertHeader(t, reader, expectedMissingHeaders1, 10) From 2ee91904498e0eb90a68f633447183f0c39db9ec Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Thu, 5 Jun 2025 08:48:51 +0200 Subject: [PATCH 11/18] address review comments --- rollup/missing_header_fields/manager_test.go | 12 +++++------ rollup/missing_header_fields/reader.go | 16 +++++++------- rollup/missing_header_fields/reader_test.go | 22 ++++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go index e6f7dc8e5fa7..f675c3077d38 100644 --- a/rollup/missing_header_fields/manager_test.go +++ b/rollup/missing_header_fields/manager_test.go @@ -10,15 +10,15 @@ import ( "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/log" + "github.com/scroll-tech/go-ethereum/params" ) func TestManagerDownload(t *testing.T) { t.Skip("skipping test due to long runtime/downloading file") log.Root().SetHandler(log.StdoutHandler) - // TODO: replace with actual sha256 hash and downloadURL - sha256 := [32]byte(common.FromHex("0x250c097758924bc21d072e8dc57f4a2357ffaafb20e85eacea5c18dfe70e62b4")) - downloadURL := "https://ftp.halifax.rwth-aachen.de/ubuntu-releases/robots.txt" + sha256 := *params.ScrollSepoliaChainConfig.Scroll.MissingHeaderFieldsSHA256 + downloadURL := "https://scroll-block-missing-metadata.s3.us-west-2.amazonaws.com/" + params.ScrollSepoliaChainConfig.ChainID.String() + ".bin" filePath := filepath.Join(t.TempDir(), "test_file_path") manager := NewManager(context.Background(), filePath, downloadURL, sha256) @@ -53,8 +53,8 @@ func TestManagerChecksum(t *testing.T) { difficulty, stateRoot, extra, err := manager.GetMissingHeaderFields(0) require.NoError(t, err) - require.Equal(t, expectedMissingHeaders1[0].difficulty, difficulty) - require.Equal(t, expectedMissingHeaders1[0].stateRoot, stateRoot) - require.Equal(t, expectedMissingHeaders1[0].extra, extra) + require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) + require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].extra, extra) } } diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go index 803d3c483a69..d078ecfdcb40 100644 --- a/rollup/missing_header_fields/reader.go +++ b/rollup/missing_header_fields/reader.go @@ -80,7 +80,7 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has } if r.lastReadHeader == nil { - if _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, nil, err } } @@ -88,7 +88,7 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has if headerNum > r.lastReadHeader.headerNum { // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { - if _, _, err = r.ReadNext(); err != nil { + if err = r.ReadNext(); err != nil { return 0, common.Hash{}, nil, err } } @@ -101,11 +101,11 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has return 0, common.Hash{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) } -func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { +func (r *Reader) ReadNext() (err error) { // read the bitmask bitmaskByte, err := r.reader.ReadByte() if err != nil { - return 0, nil, fmt.Errorf("failed to read bitmask: %v", err) + return fmt.Errorf("failed to read bitmask: %v", err) } bits := newBitMaskFromBytes(bitmaskByte) @@ -113,17 +113,17 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { // read the vanity index vanityIndex, err := r.reader.ReadByte() if err != nil { - return 0, nil, fmt.Errorf("failed to read vanity index: %v", err) + return fmt.Errorf("failed to read vanity index: %v", err) } stateRoot := make([]byte, common.HashLength) if _, err := io.ReadFull(r.reader, stateRoot); err != nil { - return 0, nil, fmt.Errorf("failed to read state root: %v", err) + return fmt.Errorf("failed to read state root: %v", err) } seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { - return 0, nil, fmt.Errorf("failed to read seal: %v", err) + return fmt.Errorf("failed to read seal: %v", err) } // construct the extraData field @@ -148,7 +148,7 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) { r.lastReadHeader.extraData = b.Bytes() } - return difficulty, b.Bytes(), nil + return nil } func (r *Reader) Close() error { diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go index 53f5e08d2752..1835c11ecba4 100644 --- a/rollup/missing_header_fields/reader_test.go +++ b/rollup/missing_header_fields/reader_test.go @@ -15,7 +15,7 @@ type header struct { extra []byte } -var expectedMissingHeaders1 = []header{ +var expectedMissingHeaders = []header{ {0, 1, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, {1, 2, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, {2, 2, common.HexToHash("0x11787ec3c17489215d0b13d594db83be55736f3933625aac0e1bba2812d49ffe"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, @@ -43,22 +43,22 @@ func TestReader_Read(t *testing.T) { require.Equal(t, expectedVanity, reader.sortedVanities[i]) } - readAndAssertHeader(t, reader, expectedMissingHeaders1, 0) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 0) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 1) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 0) + readAndAssertHeader(t, reader, expectedMissingHeaders, 1) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) // reading previous headers resets the file reader - readAndAssertHeader(t, reader, expectedMissingHeaders1, 5) + readAndAssertHeader(t, reader, expectedMissingHeaders, 5) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) + readAndAssertHeader(t, reader, expectedMissingHeaders, 8) // reading previous headers resets the file reader - readAndAssertHeader(t, reader, expectedMissingHeaders1, 6) + readAndAssertHeader(t, reader, expectedMissingHeaders, 6) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 9) - readAndAssertHeader(t, reader, expectedMissingHeaders1, 10) + readAndAssertHeader(t, reader, expectedMissingHeaders, 9) + readAndAssertHeader(t, reader, expectedMissingHeaders, 10) // no data anymore _, _, _, err = reader.Read(11) From ed1f352dd13e59654889eee477c8ca8f0b9f9786 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:17:41 +0100 Subject: [PATCH 12/18] add coinbase and nonce field to missing header reader --- rollup/da_syncer/da/commitV0.go | 2 +- rollup/missing_header_fields/manager.go | 5 +- rollup/missing_header_fields/manager_test.go | 10 ++- rollup/missing_header_fields/reader.go | 81 +++++++++++++++--- rollup/missing_header_fields/reader_test.go | 33 ++++--- .../testdata/missing-headers.bin | Bin 1174 -> 1222 bytes 6 files changed, 100 insertions(+), 31 deletions(-) diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index 15f96ed4ed02..ea190aa705ae 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -135,7 +135,7 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par // insert l2 txs txs = append(txs, chunk.Transactions[blockIndex]...) - difficulty, stateRoot, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + difficulty, stateRoot, _, _, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) if err != nil { return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) } diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go index 17763c653b8d..e26e52793c31 100644 --- a/rollup/missing_header_fields/manager.go +++ b/rollup/missing_header_fields/manager.go @@ -12,6 +12,7 @@ import ( "time" "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" "github.com/scroll-tech/go-ethereum/log" ) @@ -37,11 +38,11 @@ func NewManager(ctx context.Context, filePath string, downloadURL string, expect } } -func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { +func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { // lazy initialization: if the reader is not initialized this is the first time we read from the file if m.reader == nil { if err = m.initialize(); err != nil { - return 0, common.Hash{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to initialize missing header reader: %v", err) } } diff --git a/rollup/missing_header_fields/manager_test.go b/rollup/missing_header_fields/manager_test.go index f675c3077d38..ae0be08e043b 100644 --- a/rollup/missing_header_fields/manager_test.go +++ b/rollup/missing_header_fields/manager_test.go @@ -22,7 +22,7 @@ func TestManagerDownload(t *testing.T) { filePath := filepath.Join(t.TempDir(), "test_file_path") manager := NewManager(context.Background(), filePath, downloadURL, sha256) - _, _, _, err := manager.GetMissingHeaderFields(0) + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) require.NoError(t, err) // Check if the file was downloaded and tmp file was removed @@ -42,19 +42,21 @@ func TestManagerChecksum(t *testing.T) { manager := NewManager(context.Background(), filePath, downloadURL, sha256) - _, _, _, err := manager.GetMissingHeaderFields(0) + _, _, _, _, _, err := manager.GetMissingHeaderFields(0) require.ErrorContains(t, err, "expectedChecksum mismatch") } // Checksum matches { - sha256 := [32]byte(common.FromHex("e5a1e71338cd899e46ff28a9ae81b8f2579e429e18cec463104fb246a6e23502")) + sha256 := [32]byte(common.FromHex("635c3f56bb66035bd99134a1e2bc23b34df376f4cd51a0c65e347ce3e65b5974")) manager := NewManager(context.Background(), filePath, downloadURL, sha256) - difficulty, stateRoot, extra, err := manager.GetMissingHeaderFields(0) + difficulty, stateRoot, coinbase, nonce, extra, err := manager.GetMissingHeaderFields(0) require.NoError(t, err) require.Equal(t, expectedMissingHeaders[0].difficulty, difficulty) require.Equal(t, expectedMissingHeaders[0].stateRoot, stateRoot) + require.Equal(t, expectedMissingHeaders[0].coinbase, coinbase) + require.Equal(t, expectedMissingHeaders[0].nonce, nonce) require.Equal(t, expectedMissingHeaders[0].extra, extra) } } diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go index d078ecfdcb40..5704175fcb95 100644 --- a/rollup/missing_header_fields/reader.go +++ b/rollup/missing_header_fields/reader.go @@ -5,15 +5,19 @@ import ( "bytes" "fmt" "io" + "log" "os" "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" ) type missingHeader struct { headerNum uint64 difficulty uint64 stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce extraData []byte } @@ -72,16 +76,16 @@ func (r *Reader) initialize() error { return nil } -func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, extraData []byte, err error) { +func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { if r.lastReadHeader != nil && headerNum < r.lastReadHeader.headerNum { if err = r.initialize(); err != nil { - return 0, common.Hash{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to reinitialize reader due to requested header number being lower than last read header: %w", err) } } if r.lastReadHeader == nil { if err = r.ReadNext(); err != nil { - return 0, common.Hash{}, nil, err + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } @@ -89,16 +93,16 @@ func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Has // skip the headers until the requested header number for i := r.lastReadHeader.headerNum; i < headerNum; i++ { if err = r.ReadNext(); err != nil { - return 0, common.Hash{}, nil, err + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err } } } if headerNum == r.lastReadHeader.headerNum { - return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.extraData, nil + return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil } - return 0, common.Hash{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) + return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("error reading header number %d: last read header number is %d", headerNum, r.lastReadHeader.headerNum) } func (r *Reader) ReadNext() (err error) { @@ -108,7 +112,7 @@ func (r *Reader) ReadNext() (err error) { return fmt.Errorf("failed to read bitmask: %v", err) } - bits := newBitMaskFromBytes(bitmaskByte) + bits := newBitMaskFromByte(bitmaskByte) // read the vanity index vanityIndex, err := r.reader.ReadByte() @@ -121,6 +125,20 @@ func (r *Reader) ReadNext() (err error) { return fmt.Errorf("failed to read state root: %v", err) } + var coinbase common.Address + if bits.hasCoinbase() { + if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { + return fmt.Errorf("failed to read coinbase: %v", err) + } + } + + var nonce types.BlockNonce + if bits.hasNonce() { + if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { + return fmt.Errorf("failed to read nonce: %v", err) + } + } + seal := make([]byte, bits.sealLen()) if _, err = io.ReadFull(r.reader, seal); err != nil { return fmt.Errorf("failed to read seal: %v", err) @@ -139,12 +157,16 @@ func (r *Reader) ReadNext() (err error) { headerNum: 0, difficulty: uint64(bits.difficulty()), stateRoot: common.BytesToHash(stateRoot), + coinbase: coinbase, + nonce: nonce, extraData: b.Bytes(), } } else { r.lastReadHeader.headerNum++ r.lastReadHeader.difficulty = uint64(bits.difficulty()) r.lastReadHeader.stateRoot = common.BytesToHash(stateRoot) + r.lastReadHeader.coinbase = coinbase + r.lastReadHeader.nonce = nonce r.lastReadHeader.extraData = b.Bytes() } @@ -157,13 +179,40 @@ func (r *Reader) Close() error { // bitMask is a bitmask that encodes the following information: // -// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 -// bit 7: 0 if seal length is 65, 1 if seal length is 85 +// bit 4: 1 if the header has a coinbase field +// bit 5: 1 if the header has a nonce field +// bit 6: 0 if difficulty is 2, 1 if difficulty is 1 +// bit 7: 0 if seal length is 65, 1 if seal length is 85 type bitMask struct { b uint8 } -func newBitMaskFromBytes(b uint8) bitMask { +func newBitMaskFromByte(b uint8) bitMask { + return bitMask{b} +} + +func newBitMask(hasCoinbase bool, hasNonce bool, difficulty int, sealLen int) bitMask { + b := uint8(0) + + if hasCoinbase { + b |= 1 << 4 + } + + if hasNonce { + b |= 1 << 5 + } + if difficulty == 1 { + b |= 1 << 6 + } else if difficulty != 2 { + log.Fatalf("Invalid difficulty: %d", difficulty) + } + + if sealLen == 85 { + b |= 1 << 7 + } else if sealLen != 65 { + log.Fatalf("Invalid seal length: %d", sealLen) + } + return bitMask{b} } @@ -184,3 +233,15 @@ func (b bitMask) sealLen() int { return 85 } } + +func (b bitMask) hasCoinbase() bool { + return (b.b>>4)&0x01 == 1 +} + +func (b bitMask) hasNonce() bool { + return (b.b>>5)&0x01 == 1 +} + +func (b bitMask) Bytes() []byte { + return []byte{b.b} +} diff --git a/rollup/missing_header_fields/reader_test.go b/rollup/missing_header_fields/reader_test.go index 1835c11ecba4..4486491f6a17 100644 --- a/rollup/missing_header_fields/reader_test.go +++ b/rollup/missing_header_fields/reader_test.go @@ -6,33 +6,36 @@ import ( "github.com/stretchr/testify/require" "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/core/types" ) type header struct { number uint64 difficulty uint64 stateRoot common.Hash + coinbase common.Address + nonce types.BlockNonce extra []byte } var expectedMissingHeaders = []header{ - {0, 1, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, - {1, 2, common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000001982b5c754257988f9486b158a33709645735e8e965912c508aee9b0513cc2f22fe13f0835ce1e11abe666c9dba6a1259612b812783cc457e5b34b025980635501")}, - {2, 2, common.HexToHash("0x11787ec3c17489215d0b13d594db83be55736f3933625aac0e1bba2812d49ffe"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000237c933578bf062f86a30cdc71b0e946f0f685711e0e9cceeb1c953ed816d2694347e1e59625545c4040f2604b75448ccb5360fdcb378741331c1d4c0d342a7e01")}, - {3, 2, common.HexToHash("0x2b4cff60622970fe15dfe6e7a8c8bf9619aea5790b57b9bc34811dd6670c16bf"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000012388a2df0f522f96e67564d38be64b5ca7fb37ef9b3f88de875d08653871407584b180917a47dc4abec60bf8da462c617328b9d2da8c4bb9978e018b44ec07401")}, - {4, 2, common.HexToHash("0x032c5535bc0684d25cd74a6bdc9f15052c9444ed8bfe5e9e4791d83f52b8dac1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000091e57e01b8ed1b433b2bd04e272f9eaf986f3fa728c8fc2b4112352101d24ba76ff1ee64e9a1f8a47c4c49e362e318b2b4767088514f72a7ba9bb7a45b4b447700")}, - {5, 2, common.HexToHash("0x0e13888dc93db27a55825d45b27d2b7d367ac9b6a302bd3480177253320f815a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000007ab6b6bd8d52c9beffe935e1bc805d9d4ad62d54485104e80943537d380d6f425ad055ad510c498d1e6efc2aa7e7cc7e1b6166f8421e94b13e291196cba1934a00")}, - {6, 2, common.HexToHash("0x29fee2024813a75d49d4670716e0f706eac9d72f091f64db1f802d346a1f2a58"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000008f7175ed80593d395069afed2d970e505b076e15642e1f6e3bcb2f589a2a47fa5841868b80fcc70f18cd52de027bdb5ab881fdefc5ebf0d8034cb35e926e89f300")}, - {7, 2, common.HexToHash("0x1d9ce9ddef4ab5c0d120715cab5b4eeef6ae8b94addb7a8bdfcb6c8f76f63e34"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e75780000000000000006224d2201ba60083743844ce0c2ec4b0ab3e79b69f64eae9a10055fe704380c6410c4f5119cb834f43705c1a785758170a868a38e536432e3a5a5805c83b13801")}, - {8, 2, common.HexToHash("0x1c9f95b06e3e65cdd0dae9d2b20de7760d9b6c768b05d043e0d0f493112d32f2"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000b5c1f5c8aa79582f4b9a66ae8a59561d6c357deaefc7353ea6ace017e76e4b367118e0fd55b9f4cd0235ee1f14222e9b558156b6253e84f71d8048e13643af4801")}, - {9, 2, common.HexToHash("0x19c478ece75320a6c4a3d1cecfd219f130a8b377e47b76adaa3eb574396bacb1"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e757800000000000000dff93471464bf856b2f633ac16b54c3ff88219a8c83067495ff9f16035c91fb56de8f6914eb4cbd8fbe8a54854e32d697a81408e20cdaa52fed9689d21f7ad0201")}, - {10, 2, common.HexToHash("0x00a85e26b58e8294115b097a808ba9840326981efb8c4abb054eb40494e5bc0a"), common.FromHex("d88304031d846765746888676f312e31392e31856c696e7578000000000000003c55c63554686f48d9e6dd78b8a7849152e33f169f843e623aa604f2c777eec9539d301fe5f1bea84e5cb7c40e74b723e7700b95eab08bc441d65092b40b548d01")}, + {0, 2, common.HexToHash("0x195dc9e93ed59fcd1d51e3262739761574b1d1518c6188e27a28357d9d93fb36"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("000000000000000000000000000000000000000000000000000000000000000048c3f81f3d998b6652900e1c3183736c238fe4290000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")}, + {1, 2, common.HexToHash("0x1c652497074d4a193fb16d61ffd6fc7727983e2d9e7010ac9ca31c241bdec3cb"), common.HexToAddress("0x687E0E85AD67ff71aC134CF61b65905b58Ab43b2"), types.BlockNonce(common.FromHex("ffffffffffffffff")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000228b5b48f65de89b35c77ae3791bfe26c1a3a91cffa2f30f50a58208f2b013ed1a86d1f5601e41b90a5345518e967985f2e949e53476ffc6d8781fa64e4e999101")}, + {2, 2, common.HexToHash("0x2ec6f0f086734c1d88f8ca915cbd2968804688c384aa6d95982435dc65ed476f"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000dbe3e10ae4c0e34d69bff96e86d4a0221361626e8cade7d9d69d870e37f7338f06a446b5461edd7f85d5dda1c54a41b9d0877e5fe9965c6a89a5f4aa3d0d6adc00")}, + {3, 2, common.HexToHash("0x16fc61ab25b479c4c367a85e3f537a598b92fb0e188464cbaca70b5dec08908d"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f649dccc68b8a96a7d06d4a69e9667c63f90c2a2819870c695c2ef95e6862df55f5953891a5f9a13dfafbdff4d23a53a5a6ded505fc87203ac78142bb787dcf200")}, + {4, 2, common.HexToHash("0x001b539a66e87624114117e4d643d5aac7c716b6468c82a8daffea747d16b1a4"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e7578000000000000009e49f8090a1e4941660730b5f11bc0d648da7c04cf3c608f5e0725395c9690f44e1f1c46a9ccd26d3d4db7db973749acb83b88b1564352fe70afc30245242b6c01")}, + {5, 2, common.HexToHash("0x264d833b19677ec62af0f34a4040b9f8e50b3914a16bf961b7d1e198902f127a"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000f30ac95495ebb0a35a44657f0c361a7d1ea2d0a086ff7036f84875b47082094504809634f82e379b5ece2f43eead70b12e5d44c1befc4bb8763fac92b4fe8fbb01")}, + {6, 2, common.HexToHash("0x25ed6f6829966b24668510d4c828a596da881001470ab4e7e633761b6bdaba45"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000c9e8cf7bf35df3de8b7d0e98426a62a9e8ec302592f5622d527845ad78b53176558730f88dc79486ada7ed7aea3de4f8ca881b612440e31ab3b84ec7dff0cace01")}, + {7, 2, common.HexToHash("0x0e5e6c49b7c7cbcf3392c52f64b287c5a974b30f20f6695024231b5cf2155d0e"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d8ebea45d74ad882718f97488083530d2d968edf1d3ff6d18d2c757feb0e40206b158f9d4e829a795849a1e3148dd3a6f5251fa600631a74825e3ea041222f5001")}, + {8, 2, common.HexToHash("0x2f6d586a8ce1fc4f476887aaffd8b143d4f5604ce2242f945ae5e8874ede7084"), common.HexToAddress("0x48C3F81f3D998b6652900e1C3183736C238Fe429"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000cecebe754d3c81738e2e76e6a0b34756006b38cd1237728ecf2f41f9a6e325634941b1b29e0e4c1338d1c0fb1190da361a5880253b116493e7cff288f9f165e300")}, + {9, 2, common.HexToHash("0x2c7f91ed6610d3823da4ed730968c40d62a99b5b6245e3f0d4f83011c3be9422"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000d6fa3b8ca99ca18fab9096d6791e70b7468a901f4cda0f48e7218e9807e79cee7b4fac8edeb23be5a593bbf9e0fc8ab678c0c58e7b4fb11869a1a4ac0f93657300")}, + {10, 2, common.HexToHash("0x1ae6b4a4b4f311a4f4a1b944445a32197f60d8070adcf7b92ed1f1cc42766504"), common.HexToAddress("0x0000000000000000000000000000000000000000"), types.BlockNonce(common.FromHex("0000000000000000")), common.FromHex("d88305050c846765746888676f312e32312e31856c696e75780000000000000016d5b03fbb592eb6cc5ec7ac98acc10b1b02184b69348b0ffb87817077e3066765573d90201d0b6690d49be1fda5a76f9edfddbdd45ebc69db3d59857e13bdf101")}, } func TestReader_Read(t *testing.T) { expectedVanities := map[int][32]byte{ 0: [32]byte(common.FromHex("0000000000000000000000000000000000000000000000000000000000000000")), - 1: [32]byte(common.FromHex("0xd88304031d846765746888676f312e31392e31856c696e757800000000000000")), + 1: [32]byte(common.FromHex("d88305050c846765746888676f312e32312e31856c696e757800000000000000")), } reader, err := NewReader("testdata/missing-headers.bin") @@ -61,14 +64,16 @@ func TestReader_Read(t *testing.T) { readAndAssertHeader(t, reader, expectedMissingHeaders, 10) // no data anymore - _, _, _, err = reader.Read(11) + _, _, _, _, _, err = reader.Read(11) require.Error(t, err) } func readAndAssertHeader(t *testing.T, reader *Reader, expectedHeaders []header, headerNum uint64) { - difficulty, stateRoot, extra, err := reader.Read(headerNum) + difficulty, stateRoot, coinbase, nonce, extra, err := reader.Read(headerNum) require.NoError(t, err) require.Equalf(t, expectedHeaders[headerNum].difficulty, difficulty, "expected difficulty %d, got %d", expectedHeaders[headerNum].difficulty, difficulty) require.Equalf(t, expectedHeaders[headerNum].stateRoot, stateRoot, "expected state root %s, got %s", expectedHeaders[headerNum].stateRoot.Hex(), stateRoot.Hex()) + require.Equalf(t, expectedHeaders[headerNum].coinbase, coinbase, "expected coinbase %s, got %s", expectedHeaders[headerNum].coinbase.Hex(), coinbase.Hex()) + require.Equalf(t, expectedHeaders[headerNum].nonce, nonce, "expected nonce %s, got %s", common.Bytes2Hex(expectedHeaders[headerNum].nonce[:]), common.Bytes2Hex(nonce[:])) require.Equal(t, expectedHeaders[headerNum].extra, extra) } diff --git a/rollup/missing_header_fields/testdata/missing-headers.bin b/rollup/missing_header_fields/testdata/missing-headers.bin index 96068b3f606f837ddbb24abfadb9c3f5cd9c0505..45674ccefc43ebb2904b6cf7d1f5aaefa118bf0a 100644 GIT binary patch delta 1121 zcmV-n1fKhr3C0PKBR~ZO41{N8bZCfYZ!s=1F)lHMY-w(FcmMzZ00000fB+d?$>~1T zpUoXn<0dCLb`^B7(NTCzw7h zo^TMXoTD5h8{WgqXnqcbt!MvntP@Q38)cANSgS*_|NsC0|NsC0B8yu{_Fd?kHOG46 zc^m#F!K0}h|Dy8`P^E$h@~{)_8ivvJU>-ra3R6W6Tn-iKXEy0003N z{9&slw0XqCXQ*C3Q+ip8lKT!Agk;OCr+*7w><#@O(Uf`T5au6U&wL;taubFw};&F000318&jHQ z=yoI#K^NrKL)EIs#}>9mjDo1z|LSyo7O|wBN%#p09!WuF2Qanq8^G2`+I$4hJbz%1 zUI!&PT$YgZP9Gdbsm#)CJx#aUmp4hQxI2ikRzp($aIeDxMI#(C*L}h;rHX3~%qR^m*|8O?=NOiPu zf(b@X#g^>Yxh#O%fK;s&-xK794@XF2s009nOY)QAr%g-~C#eXkkvWLZ~ zbh8g2_GwTgBO6@u6*__$@5CJc3SZa*n{7*+{hpPYBu|w4LU`*m9 zFO*v4=!Z_;aD+(1_#Zu)i+^TPkPaL%gL7;nkK`%N&c1a`Jb`nLE_UXivqx3{YdFmk zH*${8FG2aH<0WHBL9w!)4onj`(ZKr=klHpHSb!xv5oD9+&+>@*@nz!x00At2k?m#> z(}F#u?Q;oe#0_Gpn_FT<v!(<6jPl`@}|B|J(W> z=%2o_Xq!XvU#3j88G^OPR3&+c`ABOOiZgJQMRQ(`mRSl~Fn*cQ@hLr3A|mL*hNKtS?f zOLaty%Tr+e%QuHXGaMaE4Kyl#0RRCjO#fhFDRBN3-{$A2$iJ2uuBCYkSGl}2fgRRo z3>Lo4>SW6fQ7o>f}tL$LEjih46 z7cz^TEvUr1nRwtBv`)Zu0RRC5ELAnU287aF*Gg;LpA`ixltk@|{$8F(k=Q>{xZ1&y z<$eLU?HfZoE6`3SFP^WMZ$GCf$owlo5;Y+K(o3gr@$O{lq4=bHOiANn;~28Ec7Je) zQBQKGx|_G8TT4WD0003F6Nrt;J+gXLf?Y+jeJg!7ddaq<0=+bV7jjcF4}n^Gwzj>E zQpvvm={4cJfL)zR)-6;>Q3U7-LsNY?4R1nP&{eHb3`vb1Zu}~z=gfW^VP^P39+a^@ zDG`>-p_57g00Alf;sQt$r(H?ZXMYD4;P(dV$=5FlA7tAffGsp?A1YXnadqv0Sv@&W zX|L@qmkv-{2W}N)E+1|?%P&}(Do6TQL57Qf{KpR%%~IY1d)r#Lf&K5r>+skEOtW5+ zZi({%00AAG>D}*2wZPFJaa^lgPVV-ui#&KJ~Rd*O(FrhV1EcVLxfD= z!t6^5v*(*>_D-&v5Cvc71UL+25XAKnoVYafHwD3`g>`{&sA!{(Q)DvZrKNyegRwXP z00A7Im9TC;WzEpq>C&Ag08l(az7(8SyZvvv=fscCD&DwRAaatg+wuG;u~t_*Sy^GprW1Oh5R78K}rG zXGvfA@nAK{AGK}h_K{As%h>zqrASocEopjzK#m~Is#5;hXq_SVtz`lM0097~UM97U zf|L&C#O1lM4v;>spyb3&3#x+!EZ%EnZ-FUdCgppF?KNg>aK4Lni q1oFps?#WY~FdyadzNk)Ix5N%~w Date: Wed, 18 Jun 2025 10:18:14 +0100 Subject: [PATCH 13/18] replace missing header reader in toolkit with actual implementation --- .../export-headers-toolkit/cmd/dedup.go | 23 ++- .../cmd/missing_header_reader.go | 155 ------------------ .../export-headers-toolkit/go.mod | 14 +- .../export-headers-toolkit/go.sum | 30 ++-- 4 files changed, 49 insertions(+), 173 deletions(-) delete mode 100644 rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go index 3a155af6c5fd..69126a860a4c 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go @@ -16,8 +16,8 @@ import ( "github.com/scroll-tech/go-ethereum/common" coreTypes "github.com/scroll-tech/go-ethereum/core/types" - "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" + "github.com/scroll-tech/go-ethereum/rollup/missing_header_fields" ) // dedupCmd represents the dedup command @@ -49,6 +49,9 @@ The binary layout of the deduplicated file is as follows: log.Fatalf("Error reading verify flag: %v", err) } + // uncomment the following line to copy from the verify file to the input file. This is useful to generate a deduplicated header file for testing purposes. + // copyFromVerifyFile(verifyFile, inputFile) + if verifyFile != "" { verifyInputFile(verifyFile, inputFile) } @@ -251,6 +254,20 @@ func (h *csvHeaderReader) close() { h.file.Close() } +func copyFromVerifyFile(verifyFile, inputFile string) { + fmt.Println("Copying from", verifyFile, "to", inputFile) + + csvReader := newCSVHeaderReader(verifyFile) + defer csvReader.close() + + writer := newFilesWriter(inputFile, "") + defer writer.close() + + for header := csvReader.readNext(); header != nil; header = csvReader.readNext() { + writer.write(header) + } +} + func verifyInputFile(verifyFile, inputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() @@ -273,7 +290,7 @@ func verifyOutputFile(verifyFile, outputFile string) { csvReader := newCSVHeaderReader(verifyFile) defer csvReader.close() - dedupReader, err := NewReader(outputFile) + dedupReader, err := missing_header_fields.NewReader(outputFile) if err != nil { log.Fatalf("Error opening dedup file: %v", err) } @@ -282,7 +299,7 @@ func verifyOutputFile(verifyFile, outputFile string) { for { header := csvReader.readNext() if header == nil { - if _, _, _, _, err = dedupReader.ReadNext(); err == nil { + if err = dedupReader.ReadNext(); err == nil { log.Fatalf("Expected EOF, got more headers") } break diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go deleted file mode 100644 index df88dc95032e..000000000000 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go +++ /dev/null @@ -1,155 +0,0 @@ -package cmd - -import ( - "bufio" - "bytes" - "fmt" - "io" - "os" - - "github.com/scroll-tech/go-ethereum/common" - "github.com/scroll-tech/go-ethereum/core/types" -) - -// TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit - -type missingHeader struct { - headerNum uint64 - difficulty uint64 - stateRoot common.Hash - coinbase common.Address - nonce types.BlockNonce - extraData []byte -} - -type Reader struct { - file *os.File - reader *bufio.Reader - sortedVanities map[int][32]byte - lastReadHeader *missingHeader -} - -func NewReader(filePath string) (*Reader, error) { - f, err := os.Open(filePath) - if err != nil { - return nil, fmt.Errorf("failed to open file: %v", err) - } - - r := &Reader{ - file: f, - reader: bufio.NewReader(f), - } - - // read the count of unique vanities - vanityCount, err := r.reader.ReadByte() - if err != nil { - return nil, err - } - - // read the unique vanities - r.sortedVanities = make(map[int][32]byte) - for i := uint8(0); i < vanityCount; i++ { - var vanity [32]byte - if _, err = r.reader.Read(vanity[:]); err != nil { - return nil, err - } - r.sortedVanities[int(i)] = vanity - } - - return r, nil -} - -func (r *Reader) Read(headerNum uint64) (difficulty uint64, stateRoot common.Hash, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { - if r.lastReadHeader == nil { - if _, _, _, _, err = r.ReadNext(); err != nil { - return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err - } - } - - if headerNum > r.lastReadHeader.headerNum { - // skip the headers until the requested header number - for i := r.lastReadHeader.headerNum; i < headerNum; i++ { - if _, _, _, _, err = r.ReadNext(); err != nil { - return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, err - } - } - } - - if headerNum == r.lastReadHeader.headerNum { - return r.lastReadHeader.difficulty, r.lastReadHeader.stateRoot, r.lastReadHeader.coinbase, r.lastReadHeader.nonce, r.lastReadHeader.extraData, nil - } - - // headerNum < r.lastReadHeader.headerNum is not supported - return 0, common.Hash{}, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum) -} - -func (r *Reader) ReadNext() (difficulty uint64, coinbase common.Address, nonce types.BlockNonce, extraData []byte, err error) { - // read the bitmask - bitmaskByte, err := r.reader.ReadByte() - if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read bitmask: %v", err) - } - - bits := newBitMaskFromByte(bitmaskByte) - - // read the vanity index - vanityIndex, err := r.reader.ReadByte() - if err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read vanity index: %v", err) - } - - var stateRoot common.Hash - if _, err := io.ReadFull(r.reader, stateRoot[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read state root: %v", err) - } - - if bits.hasCoinbase() { - if _, err = io.ReadFull(r.reader, coinbase[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read coinbase: %v", err) - } - } - - if bits.hasNonce() { - if _, err = io.ReadFull(r.reader, nonce[:]); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read nonce: %v", err) - } - } - - seal := make([]byte, bits.sealLen()) - - if _, err = io.ReadFull(r.reader, seal); err != nil { - return 0, common.Address{}, types.BlockNonce{}, nil, fmt.Errorf("failed to read seal: %v", err) - } - - // construct the extraData field - vanity := r.sortedVanities[int(vanityIndex)] - var b bytes.Buffer - b.Write(vanity[:]) - b.Write(seal) - - // we don't have the header number, so we'll just increment the last read header number - // we assume that the headers are written in order, starting from 0 - if r.lastReadHeader == nil { - r.lastReadHeader = &missingHeader{ - headerNum: 0, - difficulty: uint64(bits.difficulty()), - stateRoot: stateRoot, - coinbase: coinbase, - nonce: nonce, - extraData: b.Bytes(), - } - } else { - r.lastReadHeader.headerNum++ - r.lastReadHeader.difficulty = uint64(bits.difficulty()) - r.lastReadHeader.stateRoot = stateRoot - r.lastReadHeader.coinbase = coinbase - r.lastReadHeader.nonce = nonce - r.lastReadHeader.extraData = b.Bytes() - } - - return difficulty, coinbase, nonce, b.Bytes(), nil -} - -func (r *Reader) Close() error { - return r.file.Close() -} diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.mod b/rollup/missing_header_fields/export-headers-toolkit/go.mod index f1062d385a7b..d10f0f5d0a80 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.mod +++ b/rollup/missing_header_fields/export-headers-toolkit/go.mod @@ -2,9 +2,11 @@ module github.com/scroll-tech/go-ethereum/export-headers-toolkit go 1.22 +replace github.com/scroll-tech/go-ethereum => ../../.. + require ( - github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 - github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 + github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 + github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 gorm.io/driver/postgres v1.5.7 @@ -12,7 +14,7 @@ require ( ) require ( - github.com/VictoriaMetrics/fastcache v1.12.1 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -23,7 +25,7 @@ require ( github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 // indirect - github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect + github.com/fjl/memsize v0.0.2 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-stack/stack v1.8.1 // indirect @@ -43,6 +45,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect @@ -58,6 +61,7 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/scroll-tech/zktrie v0.8.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect github.com/supranational/blst v0.3.12 // indirect @@ -66,6 +70,8 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/rollup/missing_header_fields/export-headers-toolkit/go.sum b/rollup/missing_header_fields/export-headers-toolkit/go.sum index 8ef0f6fde0ce..9823b3e92427 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/go.sum +++ b/rollup/missing_header_fields/export-headers-toolkit/go.sum @@ -1,7 +1,9 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= -github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38= +github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= @@ -41,8 +43,8 @@ github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4 h1:B2mpK+MNqgPqk2/KNi1LbqwtZDy5F7iy0mynQiBr8VA= github.com/ethereum/c-kzg-4844/bindings/go v0.0.0-20230126171313-363c7d7593b4/go.mod h1:y4GA2JbAUama1S4QwYjC2hefgGLU8Ul0GMtL/ADMF1c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -108,6 +110,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -166,14 +170,14 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7 h1:CDrPMqifvAVyYqu0x1J5qickVV0b51tApPnOwDYLESI= -github.com/scroll-tech/da-codec v0.0.0-20240605080813-32bfc9fccde7/go.mod h1:1wWYii0OPwd5kw+xrz0PFgS420xNadrNF1x/ELJT+TM= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9 h1:Jq4TTYcHVAIVPUHNYbOzNxEsWf+9Q3b30YQaMyl0TDI= -github.com/scroll-tech/go-ethereum v1.10.14-0.20240624092647-7da0bd5480e9/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1 h1:Dhd58LE1D+dnoxpgLVeQBMF9uweL/fhQfZHWtWSiOlE= +github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY= github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= @@ -199,6 +203,10 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -208,8 +216,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -229,7 +237,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From b54c25f3035cbceef750c702cdf0f27a7ee078f1 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:48:29 +0100 Subject: [PATCH 14/18] update sync from DA pipeline to include coinbase and nonce from missing header fields file --- rollup/da_syncer/da/commitV0.go | 4 +++- rollup/da_syncer/da/da.go | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rollup/da_syncer/da/commitV0.go b/rollup/da_syncer/da/commitV0.go index ea190aa705ae..ca7322264a43 100644 --- a/rollup/da_syncer/da/commitV0.go +++ b/rollup/da_syncer/da/commitV0.go @@ -135,7 +135,7 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par // insert l2 txs txs = append(txs, chunk.Transactions[blockIndex]...) - difficulty, stateRoot, _, _, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) + difficulty, stateRoot, coinbase, nonce, extraData, err := manager.GetMissingHeaderFields(daBlock.Number()) if err != nil { return nil, fmt.Errorf("failed to get missing header fields for block %d: %w", daBlock.Number(), err) } @@ -149,6 +149,8 @@ func (c *CommitBatchDAV0) Blocks(manager *missing_header_fields.Manager) ([]*Par Difficulty: difficulty, ExtraData: extraData, StateRoot: stateRoot, + Coinbase: coinbase, + Nonce: nonce, }, txs) blocks = append(blocks, block) diff --git a/rollup/da_syncer/da/da.go b/rollup/da_syncer/da/da.go index 3a56d326d931..f80ee631b51d 100644 --- a/rollup/da_syncer/da/da.go +++ b/rollup/da_syncer/da/da.go @@ -55,6 +55,8 @@ type PartialHeader struct { Difficulty uint64 ExtraData []byte StateRoot common.Hash + Coinbase common.Address + Nonce types.BlockNonce } func (h *PartialHeader) ToHeader() *types.Header { @@ -66,6 +68,8 @@ func (h *PartialHeader) ToHeader() *types.Header { Difficulty: new(big.Int).SetUint64(h.Difficulty), Extra: h.ExtraData, Root: h.StateRoot, + Coinbase: h.Coinbase, + Nonce: h.Nonce, } } From 3b78f626eafc3703073c73b31fd29be8b3d6e9e1 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 10:59:43 +0100 Subject: [PATCH 15/18] update sha256 hashes for missing header fields files --- params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/config.go b/params/config.go index c73e1f8fcb13..b6b7da4b7fc0 100644 --- a/params/config.go +++ b/params/config.go @@ -40,8 +40,8 @@ var ( ScrollMainnetGenesisHash = common.HexToHash("0xbbc05efd412b7cd47a2ed0e5ddfcf87af251e414ea4c801d78b6784513180a80") ScrollSepoliaGenesisState = common.HexToHash("0x20695989e9038823e35f0e88fbc44659ffdbfa1fe89fbeb2689b43f15fa64cb5") ScrollMainnetGenesisState = common.HexToHash("0x08d535cc60f40af5dd3b31e0998d7567c2d568b224bed2ba26070aeb078d1339") - ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0x9062e2fa1200dca63bee1d18d429572f134f5f0c98cb4852f62fc394e33cf6e6") - ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0x3629f5e53250a526ffc46806c4d74b9c52c9209a6d45ecdfebdef5d596bb3f40") + ScrollMainnetMissingHeaderFieldsSHA256 = common.HexToHash("0xfa2746026ec9590e37e495cb20046e20a38fd0e7099abd2012640dddf6c88b25") + ScrollSepoliaMissingHeaderFieldsSHA256 = common.HexToHash("0xa02354c12ca0f918bf4768255af9ed13c137db7e56252348f304b17bb4088924") ) func newUint64(val uint64) *uint64 { return &val } From cc23f6dec94bffb1a3f93b13a71a0331d13e0b1c Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:04:09 +0100 Subject: [PATCH 16/18] lint --- .../cmd/missing_header_writer_test.go | 3 ++- .../export-headers-toolkit/types/l2_block.go | 3 ++- rollup/missing_header_fields/reader.go | 26 ------------------- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go index 52700fc5f90c..8fee44df0177 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go +++ b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer_test.go @@ -5,10 +5,11 @@ import ( "crypto/rand" "testing" - coreTypes "github.com/scroll-tech/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + coreTypes "github.com/scroll-tech/go-ethereum/core/types" + "github.com/scroll-tech/go-ethereum/common" "github.com/scroll-tech/go-ethereum/export-headers-toolkit/types" diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go index 6e08b0621bc4..2b2a18765090 100644 --- a/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go +++ b/rollup/missing_header_fields/export-headers-toolkit/types/l2_block.go @@ -7,8 +7,9 @@ import ( "time" "github.com/scroll-tech/da-codec/encoding" - "github.com/scroll-tech/go-ethereum/core/types" "gorm.io/gorm" + + "github.com/scroll-tech/go-ethereum/core/types" ) // L2Block represents a l2 block in the database. diff --git a/rollup/missing_header_fields/reader.go b/rollup/missing_header_fields/reader.go index 5704175fcb95..fa8ffe5107b9 100644 --- a/rollup/missing_header_fields/reader.go +++ b/rollup/missing_header_fields/reader.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "io" - "log" "os" "github.com/scroll-tech/go-ethereum/common" @@ -191,31 +190,6 @@ func newBitMaskFromByte(b uint8) bitMask { return bitMask{b} } -func newBitMask(hasCoinbase bool, hasNonce bool, difficulty int, sealLen int) bitMask { - b := uint8(0) - - if hasCoinbase { - b |= 1 << 4 - } - - if hasNonce { - b |= 1 << 5 - } - if difficulty == 1 { - b |= 1 << 6 - } else if difficulty != 2 { - log.Fatalf("Invalid difficulty: %d", difficulty) - } - - if sealLen == 85 { - b |= 1 << 7 - } else if sealLen != 65 { - log.Fatalf("Invalid seal length: %d", sealLen) - } - - return bitMask{b} -} - func (b bitMask) difficulty() int { val := (b.b >> 6) & 0x01 if val == 0 { From 055904272a7369e104002be33ca513117ccd4f17 Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:31:06 +0100 Subject: [PATCH 17/18] address review comments --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 5 +---- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c1fbb08746a5..459feca738de 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -177,6 +177,7 @@ var ( utils.RollupVerifyEnabledFlag, utils.ShadowforkPeersFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABlobScanAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1d299d339fe8..fbfc1b91fe56 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -237,6 +237,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.L1DisableMessageQueueV2Flag, utils.RollupVerifyEnabledFlag, utils.DASyncEnabledFlag, + utils.DAMissingHeaderFieldsBaseURLFlag, utils.DABlobScanAPIEndpointFlag, utils.DABlockNativeAPIEndpointFlag, utils.DABeaconNodeAPIEndpointFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a5a06faa9d37..5a7bdf80f0b7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1388,10 +1388,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name) } - cfg.DAMissingHeaderFieldsBaseURL = DAMissingHeaderFieldsBaseURLFlag.Value - if ctx.GlobalIsSet(DAMissingHeaderFieldsBaseURLFlag.Name) { - cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) - } + cfg.DAMissingHeaderFieldsBaseURL = ctx.GlobalString(DAMissingHeaderFieldsBaseURLFlag.Name) if ctx.GlobalIsSet(ExternalSignerFlag.Name) { cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) From 8217915b614db10ea320f021efbc7bd66f24936c Mon Sep 17 00:00:00 2001 From: jonastheis <4181434+jonastheis@users.noreply.github.com> Date: Wed, 9 Jul 2025 09:40:28 +0200 Subject: [PATCH 18/18] address review comments --- core/blockchain.go | 1 + rollup/missing_header_fields/manager.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 2da580a26372..314368ee049d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1881,6 +1881,7 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types header.ParentHash = parentBlock.Hash() // sanitize base fee + // Note: setting the base fee to 0 will cause problems as nil != 0 when serializing the header and thus block hash will be different. if header.BaseFee != nil && header.BaseFee.Cmp(common.Big0) == 0 { header.BaseFee = nil } diff --git a/rollup/missing_header_fields/manager.go b/rollup/missing_header_fields/manager.go index e26e52793c31..46804aa0830b 100644 --- a/rollup/missing_header_fields/manager.go +++ b/rollup/missing_header_fields/manager.go @@ -72,7 +72,7 @@ func (m *Manager) initialize() error { } computedChecksum := h.Sum(nil) if !bytes.Equal(computedChecksum, m.expectedChecksum[:]) { - return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x", m.expectedChecksum, computedChecksum) + return fmt.Errorf("expectedChecksum mismatch, expected %x, got %x. Please delete %s to restart file download", m.expectedChecksum, computedChecksum, m.filePath) } // finally initialize the reader @@ -84,6 +84,7 @@ func (m *Manager) initialize() error { m.reader = reader return nil } + func (m *Manager) Close() error { if m.reader != nil { return m.reader.Close()