test/flakey head prev 10

.github/actions/setup-create-base64-upgrade-config/action.yml

@@ -92,7 +92,6 @@ runs:
         [ChainlinkUpgradeImage]
         image="$UPGRADE_IMAGE"
         version="$UPGRADE_VERSION"
-        postgres_version="$CHAINLINK_POSTGRES_VERSION"
 
         [Logging]
         test_log_collect=$test_log_collect

.github/workflows/integration-tests.yml

@@ -218,6 +218,40 @@ jobs:
           AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           dep_evm_sha: ${{ inputs.evm-ref }}
 
+  build-test-image:
+    if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
+    environment: integration
+    permissions:
+      id-token: write
+      contents: read
+    name: Build Test Image
+    runs-on: ubuntu22.04-16cores-64GB
+    needs: [changes]
+    steps:
+      - name: Collect Metrics
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
+        with:
+          id: ${{ env.COLLECTION_ID }}-build-test-image
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Build Test Image
+        continue-on-error: true
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          repository: smartcontractkit/chainlink
+          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Build Test Image
+        if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
+        uses: ./.github/actions/build-test-image
+        with:
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+
   compare-tests:
     needs: [changes]
     runs-on: ubuntu-latest
@@ -692,7 +726,7 @@ jobs:
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ""
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           should_tidy: "false"
       - name: Show Otel-Collector Logs
         if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins'
@@ -796,7 +830,6 @@ jobs:
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
   eth-smoke-go-mod-cache:
-
     environment: integration
     needs: [eth-smoke-tests]
     runs-on: ubuntu-latest
@@ -830,7 +863,7 @@ jobs:
       id-token: write
       contents: read
     runs-on: ubuntu-latest
-    needs: [build-chainlink, changes]
+    needs: [build-chainlink, changes, build-test-image]
     # Only run migration tests on new tags
     if: startsWith(github.ref, 'refs/tags/')
     env:
@@ -843,17 +876,6 @@ jobs:
       TEST_LOG_LEVEL: debug
       TEST_SUITE: migration
     steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
-        with:
-          id: ${{ env.COLLECTION_ID }}-migration-tests
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: Version Migration Tests
-          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
-        continue-on-error: true
       - name: Checkout the repo
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
@@ -864,12 +886,7 @@ jobs:
         run: |
           untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name)
           latest_version="${untrimmed_ver:1}"
-          # Check if latest_version is empty
-          if [ -z "$latest_version" ]; then
-              echo "Error: The latest_version is empty. The migration tests need a verison to run."
-              exit 1
-          fi
-          echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT"
+          echo "latest_version=${latest_version} | tee -a $GITHUB_OUTPUT"
       - name: Name Versions
         run: |
           echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'"
@@ -881,22 +898,13 @@ jobs:
           chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }}
           upgradeImage: ${{ env.UPGRADE_IMAGE }}
           upgradeVersion: ${{ env.UPGRADE_VERSION }}
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          lokiEndpoint: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
-          lokiTenantId: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          lokiBasicAuth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
       - name: Run Migration Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
+          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }}
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: node-migration-test-logs
           artifacts_location: |
             ./integration-tests/migration/logs
@@ -908,24 +916,28 @@ jobs:
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ""
+          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
-          should_tidy: "false"
       - name: Upload Coverage Data
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-migration-tests
           path: .covdata
           retention-days: 1          
-      - name: Notify Slack
-        if: failure() && github.event_name != 'workflow_dispatch'
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
+
+      - name: Collect Metrics
+        if: always()
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0
         with:
-          channel-id: "#team-test-tooling-internal"
-          slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: ${{ job.html_url }}\n${{ format('https://github.com/smartcontractkit/chainlink/actions/runs/{0}', github.run_id) }}"
+          id: ${{ env.COLLECTION_ID }}-migration-tests
+          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
+          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+          this-job-name: Version Migration Tests
+          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
+        continue-on-error: true
 
   ## Solana Section
   get_solana_sha:

common/client/models.go

@@ -28,35 +28,6 @@ var sendTxSevereErrors = []SendTxReturnCode{Fatal, Underpriced, Unsupported, Exc
 // sendTxSuccessfulCodes - error codes which signal that transaction was accepted by the node
 var sendTxSuccessfulCodes = []SendTxReturnCode{Successful, TransactionAlreadyKnown}
 
-func (c SendTxReturnCode) String() string {
-	switch c {
-	case Successful:
-		return "Successful"
-	case Fatal:
-		return "Fatal"
-	case Retryable:
-		return "Retryable"
-	case Underpriced:
-		return "Underpriced"
-	case Unknown:
-		return "Unknown"
-	case Unsupported:
-		return "Unsupported"
-	case TransactionAlreadyKnown:
-		return "TransactionAlreadyKnown"
-	case InsufficientFunds:
-		return "InsufficientFunds"
-	case ExceedsMaxFee:
-		return "ExceedsMaxFee"
-	case FeeOutOfValidRange:
-		return "FeeOutOfValidRange"
-	case OutOfCounters:
-		return "OutOfCounters"
-	default:
-		return fmt.Sprintf("SendTxReturnCode(%d)", c)
-	}
-}
-
 type NodeTier int
 
 const (

common/client/multi_node.go

@@ -561,13 +561,6 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	return n.RPC().PendingSequenceAt(ctx, addr)
 }
 
-type sendTxErrors map[SendTxReturnCode][]error
-
-// String - returns string representation of the errors map. Required by logger to properly represent the value
-func (errs sendTxErrors) String() string {
-	return fmt.Sprint(map[SendTxReturnCode][]error(errs))
-}
-
 func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM]) SendEmptyTransaction(
 	ctx context.Context,
 	newTxAttempt func(seq SEQ, feeLimit uint32, fee FEE, fromAddress ADDR) (attempt any, err error),
@@ -609,7 +602,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	ctx, cancel := c.chStop.Ctx(ctx)
 	defer cancel()
 	requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum))
-	errorsByCode := sendTxErrors{}
+	errorsByCode := map[SendTxReturnCode][]error{}
 	var softTimeoutChan <-chan time.Time
 	var resultsCount int
 loop:
@@ -646,7 +639,7 @@ loop:
 
 func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM]) reportSendTxAnomalies(tx TX, txResults <-chan sendTxResult) {
 	defer c.wg.Done()
-	resultsByCode := sendTxErrors{}
+	resultsByCode := map[SendTxReturnCode][]error{}
 	// txResults eventually will be closed
 	for txResult := range txResults {
 		resultsByCode[txResult.ResultCode] = append(resultsByCode[txResult.ResultCode], txResult.Err)
@@ -660,7 +653,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 	}
 }
 
-func aggregateTxResults(resultsByCode sendTxErrors) (txResult error, err error) {
+func aggregateTxResults(resultsByCode map[SendTxReturnCode][]error) (txResult error, err error) {
 	severeErrors, hasSevereErrors := findFirstIn(resultsByCode, sendTxSevereErrors)
 	successResults, hasSuccess := findFirstIn(resultsByCode, sendTxSuccessfulCodes)
 	if hasSuccess {

common/client/multi_node_test.go

@@ -796,13 +796,13 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 		Name                string
 		ExpectedTxResult    string
 		ExpectedCriticalErr string
-		ResultsByCode       sendTxErrors
+		ResultsByCode       map[SendTxReturnCode][]error
 	}{
 		{
 			Name:                "Returns success and logs critical error on success and Fatal",
 			ExpectedTxResult:    "success",
 			ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				Successful: {errors.New("success")},
 				Fatal:      {errors.New("fatal")},
 			},
@@ -811,7 +811,7 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 			Name:                "Returns TransactionAlreadyKnown and logs critical error on TransactionAlreadyKnown and Fatal",
 			ExpectedTxResult:    "tx_already_known",
 			ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				TransactionAlreadyKnown: {errors.New("tx_already_known")},
 				Unsupported:             {errors.New("unsupported")},
 			},
@@ -820,7 +820,7 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 			Name:                "Prefers sever error to temporary",
 			ExpectedTxResult:    "underpriced",
 			ExpectedCriticalErr: "",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				Retryable:   {errors.New("retryable")},
 				Underpriced: {errors.New("underpriced")},
 			},
@@ -829,15 +829,15 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 			Name:                "Returns temporary error",
 			ExpectedTxResult:    "retryable",
 			ExpectedCriticalErr: "",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				Retryable: {errors.New("retryable")},
 			},
 		},
 		{
 			Name:                "Insufficient funds is treated as  error",
 			ExpectedTxResult:    "",
 			ExpectedCriticalErr: "",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				Successful:        {nil},
 				InsufficientFunds: {errors.New("insufficientFunds")},
 			},
@@ -846,13 +846,13 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 			Name:                "Logs critical error on empty ResultsByCode",
 			ExpectedTxResult:    "expected at least one response on SendTransaction",
 			ExpectedCriticalErr: "expected at least one response on SendTransaction",
-			ResultsByCode:       sendTxErrors{},
+			ResultsByCode:       map[SendTxReturnCode][]error{},
 		},
 		{
 			Name:                "Zk out of counter error",
 			ExpectedTxResult:    "not enough keccak counters to continue the execution",
 			ExpectedCriticalErr: "",
-			ResultsByCode: sendTxErrors{
+			ResultsByCode: map[SendTxReturnCode][]error{
 				OutOfCounters: {errors.New("not enough keccak counters to continue the execution")},
 			},
 		},
@@ -870,9 +870,6 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 				assert.EqualError(t, txResult, testCase.ExpectedTxResult)
 			}
 
-			logger.Sugared(logger.Test(t)).Info("Map: " + fmt.Sprint(testCase.ResultsByCode))
-			logger.Sugared(logger.Test(t)).Criticalw("observed invariant violation on SendTransaction", "resultsByCode", testCase.ResultsByCode, "err", err)
-
 			if testCase.ExpectedCriticalErr == "" {
 				assert.NoError(t, err)
 			} else {
@@ -887,4 +884,5 @@ func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
 		delete(codesToCover, codeToIgnore)
 	}
 	assert.Empty(t, codesToCover, "all of the SendTxReturnCode must be covered by this test")
+
 }