Add test to re-execute specified range of mainnet C-Chain blocks

.github/workflows/c-chain-reexecution-benchmark.yml

@@ -0,0 +1,73 @@
+name: C-Chain Re-Execution Benchmark
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      start-block:
+        description: 'The start block for the benchmark.'
+        required: false
+        default: 101
+      end-block:
+        description: 'The end block for the benchmark.'
+        required: false
+        default: 250000
+      source-block-dir:
+        description: 'The source block directory. Supports S3 directory/zip and local directories.'
+        required: false
+        default: s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb.zip
+      current-state-dir:
+        description: 'The current state directory. Supports S3 directory/zip and local directories.'
+        required: false
+        default: s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100.zip
+  schedule:
+    - cron: '0 6 * * 0' # Runs every Sunday at 06:00 UTC
+
+jobs:
+    c-chain-reexecution:
+      permissions:
+        id-token: write
+        contents: write
+      runs-on: ubuntu-latest
+      steps:
+        - name: Configure AWS Credentials
+          uses: aws-actions/configure-aws-credentials@v4
+          with:
+            role-to-assume: ${{ secrets.AWS_S3_READ_ONLY_ROLE }}
+            aws-region: us-east-2
+        - name: Set task env via GITHUB_ENV
+          id: set-params
+          run: |
+            {
+              echo "START_BLOCK=${{ github.event.inputs.start-block || 101 }}"
+              echo "END_BLOCK=${{ github.event.inputs.end-block || 250000 }}"
+              echo "SOURCE_BLOCK_DIR=${{ github.event.inputs.source-block-dir || 's3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb.zip' }}"
+              echo "CURRENT_STATE_DIR=${{ github.event.inputs.current-state-dir || 's3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100.zip' }}"
+            } >> "$GITHUB_ENV"
+        - uses: actions/checkout@v4
+        - uses: ./.github/actions/setup-go-for-project
+        - name: Run C-Chain Re-Execution
+          uses: ./.github/actions/run-monitored-tmpnet-cmd
+          with:
+            run: ./scripts/run_task.sh reexecute-cchain-range-with-copied-data EXECUTION_DATA_DIR=${{ github.workspace }}/reexecution-data BENCHMARK_OUTPUT_FILE=${{ github.workspace }}/reexecute-cchain-range-benchmark-res.txt
+            prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }}
+            prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
+            grafana_dashboard_id: 'Gl1I20mnk/c-chain'
+            loki_username: ${{ secrets.LOKI_ID || '' }}
+            loki_password: ${{ secrets.LOKI_PASSWORD || '' }}
+            runtime: "" # Set runtime input to empty string to disable log collection
+        - name: Download Previous Benchmark Result
+          uses: actions/cache@v4
+          with:
+            path: ./cache
+            key: ${{ runner.os }}-reexecute-cchain-range-benchmark.json
+        - name: Compare Benchmark Results
+          uses: benchmark-action/github-action-benchmark@v1
+          with:
+            tool: 'go'
+            output-file-path: ${{ github.workspace }}/reexecute-cchain-range-benchmark-res.txt
+            external-data-json-path: ./cache/${{ runner.os }}-reexecute-cchain-range-benchmark.json
+            fail-on-alert: true
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            summary-always: true
+            comment-on-alert: true

Taskfile.yml

@@ -85,6 +85,24 @@ tasks:
       - cmd: go mod tidy
       - task: check-clean-branch
 
+  export-cchain-block-range:
+    desc: Export range of C-Chain blocks from source to target directory.
+    vars:
+      SOURCE_BLOCK_DIR: '{{.SOURCE_BLOCK_DIR}}'
+      TARGET_BLOCK_DIR: '{{.TARGET_BLOCK_DIR}}'
+      START_BLOCK: '{{.START_BLOCK}}'
+      END_BLOCK: '{{.END_BLOCK}}'
+    cmds:
+      - cmd: go test -timeout=0 -run=TestExportBlockRange github.com/ava-labs/avalanchego/tests/reexecute/c --source-block-dir={{.SOURCE_BLOCK_DIR}} --target-block-dir={{.TARGET_BLOCK_DIR}} --start-block={{.START_BLOCK}} --end-block={{.END_BLOCK}}
+
+  export-dir-to-s3:
+    desc: Copies a directory to s3
+    vars:
+      LOCAL_SRC: '{{.LOCAL_SRC}}'
+      S3_DST: '{{.S3_DST}}'
+    cmds:
+      - cmd: s5cmd cp {{.LOCAL_SRC}} {{.S3_DST}}
+
   generate-mocks:
     desc: Generates testing mocks
     cmds:
@@ -109,6 +127,30 @@ tasks:
     desc: Runs ginkgo against the current working directory
     cmd: ./bin/ginkgo build {{.USER_WORKING_DIR}}
 
+  import-cchain-reexecute-range:
+    desc: Imports the C-Chain block and state data to re-execute. Defaults to import the first 200 and the current state created with the default config of the C-Chain (hashdb).
+    vars:
+      EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}'
+      SOURCE_BLOCK_DIR: '{{.SOURCE_BLOCK_DIR | default "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-200.zip"}}'
+      CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR | default "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100.zip"}}'
+    cmds:
+      - task: import-s3-to-dir
+        vars:
+          SRC: '{{.SOURCE_BLOCK_DIR}}'
+          DST: '{{.EXECUTION_DATA_DIR}}/blocks'
+      - task: import-s3-to-dir
+        vars:
+          SRC: '{{.CURRENT_STATE_DIR}}'
+          DST: '{{.EXECUTION_DATA_DIR}}/current-state'
+
+  import-s3-to-dir:
+    desc: Imports an S3 path to a local directory. Unzipping if needed.
+    vars:
+      SRC: '{{.SRC}}'
+      DST: '{{.DST}}'
+    cmds:
+      - cmd: bash -x ./scripts/copy_dir.sh {{.SRC}} {{.DST}}
+
   install-nix:
     desc: Installs nix with the determinate systems installer
     cmd: curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
@@ -139,6 +181,49 @@ tasks:
     desc: Runs shellcheck to check sanity of shell scripts
     cmd: ./scripts/shellcheck.sh
 
+  reexecute-cchain-range:
+    desc: Re-execute a range of C-Chain blocks.
+    vars:
+      CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR}}'
+      SOURCE_BLOCK_DIR: '{{.SOURCE_BLOCK_DIR}}'
+      START_BLOCK: '{{.START_BLOCK}}'
+      END_BLOCK: '{{.END_BLOCK}}'
+      LABELS: '{{.LABELS | default ""}}'
+      BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE | default ""}}'
+    cmd: |
+      CURRENT_STATE_DIR={{.CURRENT_STATE_DIR}} \
+      SOURCE_BLOCK_DIR={{.SOURCE_BLOCK_DIR}} \
+      START_BLOCK={{.START_BLOCK}} \
+      END_BLOCK={{.END_BLOCK}} \
+      LABELS={{.LABELS}} \
+      BENCHMARK_OUTPUT_FILE={{.BENCHMARK_OUTPUT_FILE}} \
+      bash -x ./scripts/benchmark_cchain_range.sh
+
+  reexecute-cchain-range-with-copied-data:
+    desc: Combines import-cchain-reexecute-range and reexecute-cchain-range
+    vars:
+      EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}'
+      SOURCE_BLOCK_DIR: '{{.SOURCE_BLOCK_DIR | default "s3://avalanchego-bootstrap-testing/cchain-mainnet-blocks-1m-ldb.zip"}}'
+      CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR | default "s3://avalanchego-bootstrap-testing/cchain-current-state-hashdb-full-100.zip"}}'
+      START_BLOCK: '{{.START_BLOCK | default "101"}}'
+      END_BLOCK: '{{.END_BLOCK | default "250000"}}'
+      LABELS: '{{.LABELS | default ""}}'
+      BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE | default ""}}'
+    cmds:
+      - task: import-cchain-reexecute-range
+        vars:
+          SOURCE_BLOCK_DIR: '{{.SOURCE_BLOCK_DIR}}'
+          CURRENT_STATE_DIR: '{{.CURRENT_STATE_DIR}}'
+          EXECUTION_DATA_DIR: '{{.EXECUTION_DATA_DIR}}'
+      - task: reexecute-cchain-range
+        vars:
+          SOURCE_BLOCK_DIR: '{{.EXECUTION_DATA_DIR}}/blocks'
+          CURRENT_STATE_DIR: '{{.EXECUTION_DATA_DIR}}/current-state'
+          START_BLOCK: '{{.START_BLOCK}}'
+          END_BLOCK: '{{.END_BLOCK}}'
+          LABELS: '{{.LABELS}}'
+          BENCHMARK_OUTPUT_FILE: '{{.BENCHMARK_OUTPUT_FILE}}'
+
   test-bootstrap-monitor-e2e:
     desc: Runs bootstrap monitor e2e tests
     cmd: bash -x ./scripts/tests.e2e.bootstrap_monitor.sh

flake.nix

@@ -61,6 +61,9 @@
 
             # Solidity compiler
             solc
+
+            # s5cmd for rapid s3 interactions
+            s5cmd
           ] ++ lib.optionals stdenv.isDarwin [
             # macOS-specific frameworks
             darwin.apple_sdk.frameworks.Security

scripts/benchmark_cchain_range.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This script runs the C-Chain re-execution benchmark with a single iteration.
+# It expects the following environment variables to be set:
+#   SOURCE_BLOCK_DIR: Path or S3 URL to the source block directory or zip.
+#   CURRENT_STATE_DIR: Path or S3 URL to the current state directory or zip.
+#   START_BLOCK: The starting block height (exclusive).
+#   END_BLOCK: The ending block height (inclusive).
+#   LABELS (optional): Comma-separated key=value pairs for metric labels.
+#   BENCHMARK_OUTPUT_FILE (optional): If set, benchmark output is also written to this file.
+
+: "${SOURCE_BLOCK_DIR:?SOURCE_BLOCK_DIR must be set}"
+: "${CURRENT_STATE_DIR:?CURRENT_STATE_DIR must be set}"
+: "${START_BLOCK:?START_BLOCK must be set}"
+: "${END_BLOCK:?END_BLOCK must be set}"
+
+cmd="go test -timeout=0 -v -benchtime=1x -bench=BenchmarkReexecuteRange -run=^$ github.com/ava-labs/avalanchego/tests/reexecute/c --source-block-dir=\"${SOURCE_BLOCK_DIR}\" --target-dir=\"${CURRENT_STATE_DIR}\" --start-block=\"${START_BLOCK}\" --end-block=\"${END_BLOCK}\" ${LABELS:+--labels=\"${LABELS}\"}"
+
+if [ -n "${BENCHMARK_OUTPUT_FILE:-}" ]; then
+  eval "$cmd" | tee "${BENCHMARK_OUTPUT_FILE}"
+else
+  eval "$cmd"
+fi

scripts/build_test.sh

@@ -7,7 +7,7 @@ AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 # Load the constants
 source "$AVALANCHE_PATH"/scripts/constants.sh
 
-EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/load/c | grep -v tests/upgrade | grep -v tests/fixture/bootstrapmonitor/e2e"
+EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/load/c | grep -v tests/upgrade | grep -v tests/fixture/bootstrapmonitor/e2e | grep -v tests/reexecute"
 
 if [[ "$(go env GOOS)" == "windows" ]]; then
   # Test discovery for the antithesis test setups is broken due to

scripts/copy_dir.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Usage: ./scripts/copy_dir.sh source_directory destination_directory
+# Sources can be S3 URLs (s3://bucket/path) or a local file path
+# Assumes s5cmd has been installed and is available in the PATH.
+# s5cmd is included in the nix dev shell.
+
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <source_directory> <destination_directory>"
+    echo "S3 Example: $0 's3://bucket1/path1' /dest/dir"
+    echo "Local Example: $0 '/local/path1' /dest/dir"
+    exit 1
+fi
+
+SRC="$1"
+DST="$2"
+
+# Ensure destination directory exists
+mkdir -p "$DST"
+
+# Function to copy from a single source to destination
+copy_source() {
+    local source="$1"
+    local dest="$2"
+
+    # Check if source starts with s3://
+    if [[ "$source" == s3://* ]]; then
+        echo "Copying from S3: $source -> $dest"
+        # Use s5cmd to copy from S3
+        time s5cmd cp "$source" "$dest"
+
+        # If we copied a zip, extract it in place
+        if [[ "$source" == *.zip ]]; then
+            echo "Extracting zip file in place"
+            time unzip "$dest"/*.zip -d "$dest"
+            rm "$dest"/*.zip
+        fi
+    else
+        echo "Copying from local filesystem: $source -> $dest"
+        # Use cp for local filesystem with recursive support
+        if [ -d "$source" ]; then
+            time cp -r "$source"/* "$dest/"
+        elif [ -f "$source" ]; then
+            time cp "$source" "$dest/"
+        else
+            echo "Warning: Source not found: $source"
+            return 1
+        fi
+    fi
+}
+
+copy_source "$SRC" "$DST"