From 833603ec143f066924762ee30e24d7fd1d96e225 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 10 Jun 2025 13:08:36 +0530 Subject: [PATCH 1/3] feat: add GetValidatorByIndex bulk query with appropriate checks --- .../keyperimpl/gnosis/validatorsyncer.go | 45 +++- .../validatorsyncer_integration_test.go | 25 ++- .../keyperimpl/gnosis/validatorsyncer_test.go | 196 +++++++++++++++++- .../medley/beaconapiclient/getvalidator.go | 13 +- 4 files changed, 254 insertions(+), 25 deletions(-) diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go index 334e2650..80cdf04a 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go @@ -164,7 +164,38 @@ func (v *ValidatorSyncer) filterEvents( } pubKeys := make([]*blst.P1Affine, 0) - for _, validatorIndex := range msg.ValidatorIndices() { + validatorIndices := msg.ValidatorIndices() + + // Split indices into chunks of 64 to respect GetValidatorByIndex limit + const maxIndicesPerRequest = 64 + validatorMap := make(map[int64]*beaconapiclient.ValidatorData) + + for i := 0; i < len(validatorIndices); i += maxIndicesPerRequest { + end := i + maxIndicesPerRequest + if end > len(validatorIndices) { + end = len(validatorIndices) + } + + chunk := validatorIndices[i:end] + validators, err := v.BeaconAPIClient.GetValidatorByIndex(ctx, "head", chunk) + if err != nil { + return nil, errors.Wrapf(err, "failed to get validators for chunk starting at index %d", i) + } + if validators == nil || len(validators.Data) == 0 { + evLog.Warn(). + Int("chunk-start", i). + Int("chunk-end", end). + Msg("ignoring registration message for unknown validators in chunk") + continue + } + + // Add validators from this chunk to our map + for _, validator := range validators.Data { + validatorMap[int64(validator.Index)] = &validator + } + } + + for _, validatorIndex := range validatorIndices { evLog = evLog.With().Int64("validator-index", validatorIndex).Logger() latestNonce, err := db.GetValidatorRegistrationNonceBefore(ctx, database.GetValidatorRegistrationNonceBeforeParams{ ValidatorIndex: validatorIndex, @@ -187,17 +218,15 @@ func (v *ValidatorSyncer) filterEvents( continue } - validator, err := v.BeaconAPIClient.GetValidatorByIndex(ctx, "head", uint64(validatorIndex)) - if err != nil { - return nil, errors.Wrapf(err, "failed to get validator %d", msg.ValidatorIndex) - } - if validator == nil { + validatorData, exists := validatorMap[validatorIndex] + if !exists { evLog.Warn().Msg("ignoring registration message for unknown validator") continue } - pubkey, err := validator.Data.Validator.GetPubkey() + + pubkey, err := validatorData.Validator.GetPubkey() if err != nil { - return nil, errors.Wrapf(err, "failed to get pubkey of validator %d", msg.ValidatorIndex) + return nil, errors.Wrapf(err, "failed to get pubkey of validator %d", validatorIndex) } pubKeys = append(pubKeys, pubkey) } diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_integration_test.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_integration_test.go index 8a5c1aed..29df752c 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_integration_test.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_integration_test.go @@ -8,7 +8,7 @@ import ( "net/http" "net/http/httptest" "os" - "strings" + "strconv" "testing" "github.com/ethereum/go-ethereum/common" @@ -71,15 +71,26 @@ func mockBeaconClientWithJSONData(t *testing.T) string { assert.NilError(t, err) return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - parts := strings.Split(r.URL.Path, "/") + // Get the id parameters from the query string + ids := r.URL.Query()["id"] + + validatorData := make([]beaconapiclient.ValidatorData, 0, len(ids)) + for _, id := range ids { + index, err := strconv.ParseUint(id, 10, 64) + assert.NilError(t, err) + if pubkey, exists := result[id]; exists { + validatorData = append(validatorData, beaconapiclient.ValidatorData{ + Index: index, + Validator: beaconapiclient.Validator{ + PubkeyHex: pubkey, + }, + }) + } + } x := beaconapiclient.GetValidatorByIndexResponse{ Finalized: true, - Data: beaconapiclient.ValidatorData{ - Validator: beaconapiclient.Validator{ - PubkeyHex: result[parts[len(parts)-1]], - }, - }, + Data: validatorData, } res, err := json.Marshal(x) assert.NilError(t, err) diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go index 5a499bd6..c248283b 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "testing" "github.com/ethereum/go-ethereum/common" @@ -39,7 +40,7 @@ func TestLegacyValidatorRegisterFilterEvent(t *testing.T) { pubkey := new(blst.P1Affine).From(privkey) sig := validatorregistry.CreateSignature(privkey, msg) - url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress())) + url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()), msg.ValidatorIndex) cl, err := beaconapiclient.New(url) assert.NilError(t, err) @@ -93,7 +94,7 @@ func TestAggregateValidatorRegisterFilterEvent(t *testing.T) { } sig := validatorregistry.CreateAggregateSignature(sks, msg) - url := mockBeaconClient(t, hex.EncodeToString(pks[0].Compress())) + url := mockBeaconClient(t, hex.EncodeToString(pks[0].Compress()), msg.ValidatorIndex) cl, err := beaconapiclient.New(url) assert.NilError(t, err) @@ -140,7 +141,7 @@ func TestValidatorRegisterWithInvalidNonce(t *testing.T) { pubkey := new(blst.P1Affine).From(privkey) sig := validatorregistry.CreateSignature(privkey, msg) - url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress())) + url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()), msg.ValidatorIndex) cl, err := beaconapiclient.New(url) assert.NilError(t, err) @@ -242,14 +243,195 @@ func TestValidatorRegisterWithUnknownValidator(t *testing.T) { assert.Equal(t, len(finalEvents), 0) } -func mockBeaconClient(t *testing.T, pubKeyHex string) string { +func TestValidatorRegisterWithUnorderedIndices(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + msg := &validatorregistry.AggregateRegistrationMessage{ + Version: 1, + ChainID: 2, + ValidatorRegistryAddress: common.HexToAddress("0x1234567890123456789012345678901234567890"), + ValidatorIndex: 3, + Nonce: 0, + Count: 2, + IsRegistration: true, + } + ctx := context.Background() + + var ikm [32]byte + var sks []*blst.SecretKey + var pks []*blst.P1Affine + for i := 0; i < int(msg.Count); i++ { + privkey := blst.KeyGen(ikm[:]) + pubkey := new(blst.P1Affine).From(privkey) + sks = append(sks, privkey) + pks = append(pks, pubkey) + } + + sig := validatorregistry.CreateAggregateSignature(sks, msg) + + // Create a mock beacon client that returns validators in a different order + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The message requests indices [3, 4] but we return them in reverse order [4, 3] + x := beaconapiclient.GetValidatorByIndexResponse{ + Finalized: true, + Data: []beaconapiclient.ValidatorData{ + { + Index: 4, + Validator: beaconapiclient.Validator{ + PubkeyHex: hex.EncodeToString(pks[1].Compress()), + }, + }, + { + Index: 3, + Validator: beaconapiclient.Validator{ + PubkeyHex: hex.EncodeToString(pks[0].Compress()), + }, + }, + }, + } + res, err := json.Marshal(x) + assert.NilError(t, err) + w.WriteHeader(http.StatusOK) + _, err = w.Write(res) + assert.NilError(t, err) + })) + defer server.Close() + + cl, err := beaconapiclient.New(server.URL) + assert.NilError(t, err) + + dbpool, dbclose := testsetup.NewTestDBPool(ctx, t, database.Definition) + t.Cleanup(dbclose) + + vs := ValidatorSyncer{ + BeaconAPIClient: cl, + DBPool: dbpool, + ChainID: msg.ChainID, + } + + events := []*validatorRegistryBindings.ValidatorregistryUpdated{{ + Signature: sig.Compress(), + Message: msg.Marshal(), + Raw: types.Log{ + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + }, + }} + + finalEvents, err := vs.filterEvents(ctx, events) + assert.NilError(t, err) + + // The event should still be accepted despite the different order + assert.DeepEqual(t, finalEvents, events) +} + +func TestValidatorRegisterWithManyIndices(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + msg := &validatorregistry.AggregateRegistrationMessage{ + Version: 1, + ChainID: 2, + ValidatorRegistryAddress: common.HexToAddress("0x1234567890123456789012345678901234567890"), + ValidatorIndex: 3, + Nonce: 0, + Count: 100, // More than 64 indices + IsRegistration: true, + } + ctx := context.Background() + + var ikm [32]byte + var sks []*blst.SecretKey + var pks []*blst.P1Affine + for i := 0; i < int(msg.Count); i++ { + privkey := blst.KeyGen(ikm[:]) + pubkey := new(blst.P1Affine).From(privkey) + sks = append(sks, privkey) + pks = append(pks, pubkey) + } + + sig := validatorregistry.CreateAggregateSignature(sks, msg) + + // Create a mock beacon client that handles multiple chunks of indices + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Parse the indices from the query parameters + query := r.URL.Query() + indices := query["id"] + + // Create response data for this chunk + var data []beaconapiclient.ValidatorData + for _, indexStr := range indices { + index, err := strconv.ParseUint(indexStr, 10, 64) + assert.NilError(t, err) + + // Use the index to determine which pubkey to use + pubkeyIndex := int(index) - 3 // Since we start from index 3 + data = append(data, beaconapiclient.ValidatorData{ + Index: index, + Validator: beaconapiclient.Validator{ + PubkeyHex: hex.EncodeToString(pks[pubkeyIndex].Compress()), + }, + }) + } + + x := beaconapiclient.GetValidatorByIndexResponse{ + Finalized: true, + Data: data, + } + res, err := json.Marshal(x) + assert.NilError(t, err) + w.WriteHeader(http.StatusOK) + _, err = w.Write(res) + assert.NilError(t, err) + + requestCount++ + })) + defer server.Close() + + cl, err := beaconapiclient.New(server.URL) + assert.NilError(t, err) + + dbpool, dbclose := testsetup.NewTestDBPool(ctx, t, database.Definition) + t.Cleanup(dbclose) + + vs := ValidatorSyncer{ + BeaconAPIClient: cl, + DBPool: dbpool, + ChainID: msg.ChainID, + } + + events := []*validatorRegistryBindings.ValidatorregistryUpdated{{ + Signature: sig.Compress(), + Message: msg.Marshal(), + Raw: types.Log{ + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + }, + }} + + finalEvents, err := vs.filterEvents(ctx, events) + assert.NilError(t, err) + + // The event should be accepted + assert.DeepEqual(t, finalEvents, events) + + // Verify that we made the expected number of requests + // For 100 indices with max 64 per request, we should make 2 requests + expectedRequests := 2 + assert.Equal(t, requestCount, expectedRequests, "Expected %d requests for %d indices", expectedRequests, msg.Count) +} + +func mockBeaconClient(t *testing.T, pubKeyHex string, index uint64) string { t.Helper() return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { x := beaconapiclient.GetValidatorByIndexResponse{ Finalized: true, - Data: beaconapiclient.ValidatorData{ - Validator: beaconapiclient.Validator{ - PubkeyHex: pubKeyHex, + Data: []beaconapiclient.ValidatorData{ + { + Index: index, + Validator: beaconapiclient.Validator{ + PubkeyHex: pubKeyHex, + }, }, }, } diff --git a/rolling-shutter/medley/beaconapiclient/getvalidator.go b/rolling-shutter/medley/beaconapiclient/getvalidator.go index fb52cd92..2a7860b0 100644 --- a/rolling-shutter/medley/beaconapiclient/getvalidator.go +++ b/rolling-shutter/medley/beaconapiclient/getvalidator.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/url" "github.com/pkg/errors" blst "github.com/supranational/blst/bindings/go" @@ -15,7 +16,7 @@ import ( type GetValidatorByIndexResponse struct { ExecutionOptimistic bool `json:"execution_optimistic"` Finalized bool `json:"finalized"` - Data ValidatorData + Data []ValidatorData } type ValidatorData struct { @@ -39,9 +40,15 @@ type Validator struct { func (c *Client) GetValidatorByIndex( ctx context.Context, stateID string, - validatorIndex uint64, + validatorIndices []int64, ) (*GetValidatorByIndexResponse, error) { - path := c.url.JoinPath("/eth/v1/beacon/states/", stateID, "/validators/", fmt.Sprint(validatorIndex)) + path := c.url.JoinPath("/eth/v1/beacon/states/", stateID, "/validators/") + query := url.Values{} + for _, index := range validatorIndices { + query.Add("id", fmt.Sprint(index)) + } + path.RawQuery = query.Encode() + req, err := http.NewRequestWithContext(ctx, "GET", path.String(), http.NoBody) if err != nil { return nil, err From 7547440b17e613f2059d40e61d1eb6f7932e3aae Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 10 Jun 2025 14:12:45 +0530 Subject: [PATCH 2/3] fix: linter errors: ValidatorSyncer --- rolling-shutter/keyperimpl/gnosis/validatorsyncer.go | 6 ++++++ .../keyperimpl/gnosis/validatorsyncer_test.go | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go index 80cdf04a..21e4881a 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go @@ -191,6 +191,12 @@ func (v *ValidatorSyncer) filterEvents( // Add validators from this chunk to our map for _, validator := range validators.Data { + if validator.Index > math.MaxInt64 { + evLog.Warn(). + Uint64("validator-index", validator.Index). + Msg("ignoring validator with index larger than MaxInt64") + continue + } validatorMap[int64(validator.Index)] = &validator } } diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go index c248283b..8358e3fd 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "math" "net/http" "net/http/httptest" "strconv" @@ -271,7 +272,7 @@ func TestValidatorRegisterWithUnorderedIndices(t *testing.T) { sig := validatorregistry.CreateAggregateSignature(sks, msg) // Create a mock beacon client that returns validators in a different order - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { // The message requests indices [3, 4] but we return them in reverse order [4, 3] x := beaconapiclient.GetValidatorByIndexResponse{ Finalized: true, @@ -366,7 +367,13 @@ func TestValidatorRegisterWithManyIndices(t *testing.T) { assert.NilError(t, err) // Use the index to determine which pubkey to use + if index > math.MaxInt { + t.Fatalf("validator index %d exceeds MaxInt", index) + } pubkeyIndex := int(index) - 3 // Since we start from index 3 + if pubkeyIndex < 0 || pubkeyIndex >= len(pks) { + t.Fatalf("invalid pubkey index %d for validator index %d", pubkeyIndex, index) + } data = append(data, beaconapiclient.ValidatorData{ Index: index, Validator: beaconapiclient.Validator{ From e3bd520bf0ce64b79126ec8f82b5727cf54ebae9 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 10 Jun 2025 14:18:33 +0530 Subject: [PATCH 3/3] chore: rename GetValidatorByIndex->GetValidatorIndices and remove redundant check --- rolling-shutter/keyperimpl/gnosis/validatorsyncer.go | 11 ++--------- .../medley/beaconapiclient/getvalidator.go | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go index 21e4881a..3682d7f2 100644 --- a/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go +++ b/rolling-shutter/keyperimpl/gnosis/validatorsyncer.go @@ -166,7 +166,7 @@ func (v *ValidatorSyncer) filterEvents( pubKeys := make([]*blst.P1Affine, 0) validatorIndices := msg.ValidatorIndices() - // Split indices into chunks of 64 to respect GetValidatorByIndex limit + // Split indices into chunks of 64 to respect GetValidatorByIndices limit const maxIndicesPerRequest = 64 validatorMap := make(map[int64]*beaconapiclient.ValidatorData) @@ -177,17 +177,10 @@ func (v *ValidatorSyncer) filterEvents( } chunk := validatorIndices[i:end] - validators, err := v.BeaconAPIClient.GetValidatorByIndex(ctx, "head", chunk) + validators, err := v.BeaconAPIClient.GetValidatorByIndices(ctx, "head", chunk) if err != nil { return nil, errors.Wrapf(err, "failed to get validators for chunk starting at index %d", i) } - if validators == nil || len(validators.Data) == 0 { - evLog.Warn(). - Int("chunk-start", i). - Int("chunk-end", end). - Msg("ignoring registration message for unknown validators in chunk") - continue - } // Add validators from this chunk to our map for _, validator := range validators.Data { diff --git a/rolling-shutter/medley/beaconapiclient/getvalidator.go b/rolling-shutter/medley/beaconapiclient/getvalidator.go index 2a7860b0..f3cb26bd 100644 --- a/rolling-shutter/medley/beaconapiclient/getvalidator.go +++ b/rolling-shutter/medley/beaconapiclient/getvalidator.go @@ -37,7 +37,7 @@ type Validator struct { WithdrawalEpoch uint64 `json:"withdrawal_epoch,string"` } -func (c *Client) GetValidatorByIndex( +func (c *Client) GetValidatorByIndices( ctx context.Context, stateID string, validatorIndices []int64,