feat: add GetValidatorByIndex bulk query with appropriate checks

rolling-shutter/keyperimpl/gnosis/validatorsyncer.go

@@ -164,7 +164,37 @@ func (v *ValidatorSyncer) filterEvents(
 		}
 
 		pubKeys := make([]*blst.P1Affine, 0)
-		for _, validatorIndex := range msg.ValidatorIndices() {
+		validatorIndices := msg.ValidatorIndices()
+
+		// Split indices into chunks of 64 to respect GetValidatorByIndices limit
+		const maxIndicesPerRequest = 64
+		validatorMap := make(map[int64]*beaconapiclient.ValidatorData)
+
+		for i := 0; i < len(validatorIndices); i += maxIndicesPerRequest {
+			end := i + maxIndicesPerRequest
+			if end > len(validatorIndices) {
+				end = len(validatorIndices)
+			}
+
+			chunk := validatorIndices[i:end]
+			validators, err := v.BeaconAPIClient.GetValidatorByIndices(ctx, "head", chunk)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get validators for chunk starting at index %d", i)
+			}
+
+			// Add validators from this chunk to our map
+			for _, validator := range validators.Data {
+				if validator.Index > math.MaxInt64 {
+					evLog.Warn().
+						Uint64("validator-index", validator.Index).
+						Msg("ignoring validator with index larger than MaxInt64")
+					continue
+				}
+				validatorMap[int64(validator.Index)] = &validator
+			}
+		}
+
+		for _, validatorIndex := range validatorIndices {
 			evLog = evLog.With().Int64("validator-index", validatorIndex).Logger()
 			latestNonce, err := db.GetValidatorRegistrationNonceBefore(ctx, database.GetValidatorRegistrationNonceBeforeParams{
 				ValidatorIndex: validatorIndex,
@@ -187,17 +217,15 @@ func (v *ValidatorSyncer) filterEvents(
 				continue
 			}
 
-			validator, err := v.BeaconAPIClient.GetValidatorByIndex(ctx, "head", uint64(validatorIndex))
-			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get validator %d", msg.ValidatorIndex)
-			}
-			if validator == nil {
+			validatorData, exists := validatorMap[validatorIndex]
+			if !exists {
 				evLog.Warn().Msg("ignoring registration message for unknown validator")
 				continue
 			}
-			pubkey, err := validator.Data.Validator.GetPubkey()
+
+			pubkey, err := validatorData.Validator.GetPubkey()
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get pubkey of validator %d", msg.ValidatorIndex)
+				return nil, errors.Wrapf(err, "failed to get pubkey of validator %d", validatorIndex)
 			}
 			pubKeys = append(pubKeys, pubkey)
 		}

rolling-shutter/keyperimpl/gnosis/validatorsyncer_integration_test.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
-	"strings"
+	"strconv"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -71,15 +71,26 @@ func mockBeaconClientWithJSONData(t *testing.T) string {
 	assert.NilError(t, err)
 
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		parts := strings.Split(r.URL.Path, "/")
+		// Get the id parameters from the query string
+		ids := r.URL.Query()["id"]
+
+		validatorData := make([]beaconapiclient.ValidatorData, 0, len(ids))
+		for _, id := range ids {
+			index, err := strconv.ParseUint(id, 10, 64)
+			assert.NilError(t, err)
+			if pubkey, exists := result[id]; exists {
+				validatorData = append(validatorData, beaconapiclient.ValidatorData{
+					Index: index,
+					Validator: beaconapiclient.Validator{
+						PubkeyHex: pubkey,
+					},
+				})
+			}
+		}
 
 		x := beaconapiclient.GetValidatorByIndexResponse{
 			Finalized: true,
-			Data: beaconapiclient.ValidatorData{
-				Validator: beaconapiclient.Validator{
-					PubkeyHex: result[parts[len(parts)-1]],
-				},
-			},
+			Data:      validatorData,
 		}
 		res, err := json.Marshal(x)
 		assert.NilError(t, err)

rolling-shutter/keyperimpl/gnosis/validatorsyncer_test.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"math"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -39,7 +41,7 @@ func TestLegacyValidatorRegisterFilterEvent(t *testing.T) {
 	pubkey := new(blst.P1Affine).From(privkey)
 
 	sig := validatorregistry.CreateSignature(privkey, msg)
-	url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()))
+	url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()), msg.ValidatorIndex)
 
 	cl, err := beaconapiclient.New(url)
 	assert.NilError(t, err)
@@ -93,7 +95,7 @@ func TestAggregateValidatorRegisterFilterEvent(t *testing.T) {
 	}
 
 	sig := validatorregistry.CreateAggregateSignature(sks, msg)
-	url := mockBeaconClient(t, hex.EncodeToString(pks[0].Compress()))
+	url := mockBeaconClient(t, hex.EncodeToString(pks[0].Compress()), msg.ValidatorIndex)
 
 	cl, err := beaconapiclient.New(url)
 	assert.NilError(t, err)
@@ -140,7 +142,7 @@ func TestValidatorRegisterWithInvalidNonce(t *testing.T) {
 	pubkey := new(blst.P1Affine).From(privkey)
 
 	sig := validatorregistry.CreateSignature(privkey, msg)
-	url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()))
+	url := mockBeaconClient(t, hex.EncodeToString(pubkey.Compress()), msg.ValidatorIndex)
 
 	cl, err := beaconapiclient.New(url)
 	assert.NilError(t, err)
@@ -242,14 +244,201 @@ func TestValidatorRegisterWithUnknownValidator(t *testing.T) {
 	assert.Equal(t, len(finalEvents), 0)
 }
 
-func mockBeaconClient(t *testing.T, pubKeyHex string) string {
+func TestValidatorRegisterWithUnorderedIndices(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	msg := &validatorregistry.AggregateRegistrationMessage{
+		Version:                  1,
+		ChainID:                  2,
+		ValidatorRegistryAddress: common.HexToAddress("0x1234567890123456789012345678901234567890"),
+		ValidatorIndex:           3,
+		Nonce:                    0,
+		Count:                    2,
+		IsRegistration:           true,
+	}
+	ctx := context.Background()
+
+	var ikm [32]byte
+	var sks []*blst.SecretKey
+	var pks []*blst.P1Affine
+	for i := 0; i < int(msg.Count); i++ {
+		privkey := blst.KeyGen(ikm[:])
+		pubkey := new(blst.P1Affine).From(privkey)
+		sks = append(sks, privkey)
+		pks = append(pks, pubkey)
+	}
+
+	sig := validatorregistry.CreateAggregateSignature(sks, msg)
+
+	// Create a mock beacon client that returns validators in a different order
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		// The message requests indices [3, 4] but we return them in reverse order [4, 3]
+		x := beaconapiclient.GetValidatorByIndexResponse{
+			Finalized: true,
+			Data: []beaconapiclient.ValidatorData{
+				{
+					Index: 4,
+					Validator: beaconapiclient.Validator{
+						PubkeyHex: hex.EncodeToString(pks[1].Compress()),
+					},
+				},
+				{
+					Index: 3,
+					Validator: beaconapiclient.Validator{
+						PubkeyHex: hex.EncodeToString(pks[0].Compress()),
+					},
+				},
+			},
+		}
+		res, err := json.Marshal(x)
+		assert.NilError(t, err)
+		w.WriteHeader(http.StatusOK)
+		_, err = w.Write(res)
+		assert.NilError(t, err)
+	}))
+	defer server.Close()
+
+	cl, err := beaconapiclient.New(server.URL)
+	assert.NilError(t, err)
+
+	dbpool, dbclose := testsetup.NewTestDBPool(ctx, t, database.Definition)
+	t.Cleanup(dbclose)
+
+	vs := ValidatorSyncer{
+		BeaconAPIClient: cl,
+		DBPool:          dbpool,
+		ChainID:         msg.ChainID,
+	}
+
+	events := []*validatorRegistryBindings.ValidatorregistryUpdated{{
+		Signature: sig.Compress(),
+		Message:   msg.Marshal(),
+		Raw: types.Log{
+			Address: common.HexToAddress("0x1234567890123456789012345678901234567890"),
+		},
+	}}
+
+	finalEvents, err := vs.filterEvents(ctx, events)
+	assert.NilError(t, err)
+
+	// The event should still be accepted despite the different order
+	assert.DeepEqual(t, finalEvents, events)
+}
+
+func TestValidatorRegisterWithManyIndices(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	msg := &validatorregistry.AggregateRegistrationMessage{
+		Version:                  1,
+		ChainID:                  2,
+		ValidatorRegistryAddress: common.HexToAddress("0x1234567890123456789012345678901234567890"),
+		ValidatorIndex:           3,
+		Nonce:                    0,
+		Count:                    100, // More than 64 indices
+		IsRegistration:           true,
+	}
+	ctx := context.Background()
+
+	var ikm [32]byte
+	var sks []*blst.SecretKey
+	var pks []*blst.P1Affine
+	for i := 0; i < int(msg.Count); i++ {
+		privkey := blst.KeyGen(ikm[:])
+		pubkey := new(blst.P1Affine).From(privkey)
+		sks = append(sks, privkey)
+		pks = append(pks, pubkey)
+	}
+
+	sig := validatorregistry.CreateAggregateSignature(sks, msg)
+
+	// Create a mock beacon client that handles multiple chunks of indices
+	requestCount := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Parse the indices from the query parameters
+		query := r.URL.Query()
+		indices := query["id"]
+
+		// Create response data for this chunk
+		var data []beaconapiclient.ValidatorData
+		for _, indexStr := range indices {
+			index, err := strconv.ParseUint(indexStr, 10, 64)
+			assert.NilError(t, err)
+
+			// Use the index to determine which pubkey to use
+			if index > math.MaxInt {
+				t.Fatalf("validator index %d exceeds MaxInt", index)
+			}
+			pubkeyIndex := int(index) - 3 // Since we start from index 3
+			if pubkeyIndex < 0 || pubkeyIndex >= len(pks) {
+				t.Fatalf("invalid pubkey index %d for validator index %d", pubkeyIndex, index)
+			}
+			data = append(data, beaconapiclient.ValidatorData{
+				Index: index,
+				Validator: beaconapiclient.Validator{
+					PubkeyHex: hex.EncodeToString(pks[pubkeyIndex].Compress()),
+				},
+			})
+		}
+
+		x := beaconapiclient.GetValidatorByIndexResponse{
+			Finalized: true,
+			Data:      data,
+		}
+		res, err := json.Marshal(x)
+		assert.NilError(t, err)
+		w.WriteHeader(http.StatusOK)
+		_, err = w.Write(res)
+		assert.NilError(t, err)
+
+		requestCount++
+	}))
+	defer server.Close()
+
+	cl, err := beaconapiclient.New(server.URL)
+	assert.NilError(t, err)
+
+	dbpool, dbclose := testsetup.NewTestDBPool(ctx, t, database.Definition)
+	t.Cleanup(dbclose)
+
+	vs := ValidatorSyncer{
+		BeaconAPIClient: cl,
+		DBPool:          dbpool,
+		ChainID:         msg.ChainID,
+	}
+
+	events := []*validatorRegistryBindings.ValidatorregistryUpdated{{
+		Signature: sig.Compress(),
+		Message:   msg.Marshal(),
+		Raw: types.Log{
+			Address: common.HexToAddress("0x1234567890123456789012345678901234567890"),
+		},
+	}}
+
+	finalEvents, err := vs.filterEvents(ctx, events)
+	assert.NilError(t, err)
+
+	// The event should be accepted
+	assert.DeepEqual(t, finalEvents, events)
+
+	// Verify that we made the expected number of requests
+	// For 100 indices with max 64 per request, we should make 2 requests
+	expectedRequests := 2
+	assert.Equal(t, requestCount, expectedRequests, "Expected %d requests for %d indices", expectedRequests, msg.Count)
+}
+
+func mockBeaconClient(t *testing.T, pubKeyHex string, index uint64) string {
 	t.Helper()
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		x := beaconapiclient.GetValidatorByIndexResponse{
 			Finalized: true,
-			Data: beaconapiclient.ValidatorData{
-				Validator: beaconapiclient.Validator{
-					PubkeyHex: pubKeyHex,
+			Data: []beaconapiclient.ValidatorData{
+				{
+					Index: index,
+					Validator: beaconapiclient.Validator{
+						PubkeyHex: pubKeyHex,
+					},
 				},
 			},
 		}

rolling-shutter/medley/beaconapiclient/getvalidator.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/pkg/errors"
 	blst "github.com/supranational/blst/bindings/go"
@@ -15,7 +16,7 @@ import (
 type GetValidatorByIndexResponse struct {
 	ExecutionOptimistic bool `json:"execution_optimistic"`
 	Finalized           bool `json:"finalized"`
-	Data                ValidatorData
+	Data                []ValidatorData
 }
 
 type ValidatorData struct {
@@ -36,12 +37,18 @@ type Validator struct {
 	WithdrawalEpoch            uint64 `json:"withdrawal_epoch,string"`
 }
 
-func (c *Client) GetValidatorByIndex(
+func (c *Client) GetValidatorByIndices(
 	ctx context.Context,
 	stateID string,
-	validatorIndex uint64,
+	validatorIndices []int64,
 ) (*GetValidatorByIndexResponse, error) {
-	path := c.url.JoinPath("/eth/v1/beacon/states/", stateID, "/validators/", fmt.Sprint(validatorIndex))
+	path := c.url.JoinPath("/eth/v1/beacon/states/", stateID, "/validators/")
+	query := url.Values{}
+	for _, index := range validatorIndices {
+		query.Add("id", fmt.Sprint(index))
+	}
+	path.RawQuery = query.Encode()
+
 	req, err := http.NewRequestWithContext(ctx, "GET", path.String(), http.NoBody)
 	if err != nil {
 		return nil, err