Merge branch 'main' into 162950-test-action-variables

.buildkite/ftr_configs.yml

@@ -100,6 +100,7 @@ enabled:
   - test/functional/apps/dashboard_elements/controls/options_list/config.ts
   - test/functional/apps/dashboard_elements/image_embeddable/config.ts
   - test/functional/apps/dashboard_elements/input_control_vis/config.ts
+  - test/functional/apps/dashboard_elements/links/config.ts
   - test/functional/apps/dashboard_elements/markdown/config.ts
   - test/functional/apps/dashboard/group1/config.ts
   - test/functional/apps/dashboard/group2/config.ts
@@ -278,7 +279,6 @@ enabled:
   - x-pack/test/functional/apps/home/config.ts
   - x-pack/test/functional/apps/index_lifecycle_management/config.ts
   - x-pack/test/functional/apps/index_management/config.ts
-  - x-pack/test/functional/apps/index_management/index_details_page/config.ts
   - x-pack/test/functional/apps/infra/config.ts
   - x-pack/test/functional/apps/ingest_pipelines/config.ts
   - x-pack/test/functional/apps/lens/group1/config.ts
@@ -311,6 +311,7 @@ enabled:
   - x-pack/test/functional/apps/reporting_management/config.ts
   - x-pack/test/functional/apps/rollup_job/config.ts
   - x-pack/test/functional/apps/saved_objects_management/config.ts
+  - x-pack/test/functional/apps/saved_query_management/config.ts
   - x-pack/test/functional/apps/security/config.ts
   - x-pack/test/functional/apps/snapshot_restore/config.ts
   - x-pack/test/functional/apps/spaces/config.ts
@@ -358,6 +359,7 @@ enabled:
   - x-pack/test/saved_object_api_integration/spaces_only/config.ts
   - x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts
   - x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts
+  - x-pack/test/saved_object_tagging/api_integration/tagging_usage_collection/config.ts
   - x-pack/test/saved_object_tagging/functional/config.ts
   - x-pack/test/saved_objects_field_count/config.ts
   - x-pack/test/search_sessions_integration/config.ts
@@ -401,16 +403,22 @@ enabled:
   - x-pack/test/upgrade_assistant_integration/config.js
   - x-pack/test/usage_collection/config.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
-  - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts
+  - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
+  - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
   - x-pack/performance/journeys/ecommerce_dashboard.ts
   - x-pack/performance/journeys/ecommerce_dashboard_map_only.ts
   - x-pack/performance/journeys/flight_dashboard.ts

.buildkite/pipelines/on_merge.yml

@@ -154,16 +154,15 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-#  TODO: Enable in #166813 after fixing types
-#  - command: .buildkite/scripts/steps/check_types.sh
-#    label: 'Check Types'
-#    agents:
-#      queue: n2-16-spot
-#    timeout_in_minutes: 60
-#    retry:
-#      automatic:
-#        - exit_status: '-1'
-#          limit: 3
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/pipelines/pull_request/base.yml

@@ -136,13 +136,12 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-# TODO: Enable in #166813 after fixing types
-#  - command: .buildkite/scripts/steps/check_types.sh
-#    label: 'Check Types'
-#    agents:
-#      queue: n2-16-spot
-#    timeout_in_minutes: 60
-#    retry:
-#      automatic:
-#        - exit_status: '-1'
-#          limit: 3
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -25,17 +25,16 @@ steps:
     artifact_paths:
       - "target/kibana-osquery/**/*"
 
-  # Error: self-signed certificate in certificate chain
-  # - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-  #   label: 'Serverless Osquery Cypress Tests'
-  #   agents:
-  #     queue: n2-4-spot
-  #   depends_on: build
-  #   timeout_in_minutes: 50
-  #   parallelism: 6
-  #   retry:
-  #     automatic:
-  #       - exit_status: '*'
-  #         limit: 1
-  #   artifact_paths:
-  #     - "target/kibana-osquery/**/*"
+  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
+    label: 'Serverless Osquery Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 50
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+    artifact_paths:
+      - "target/kibana-osquery/**/*"

.buildkite/pipelines/pull_request/security_solution.yml

@@ -28,8 +28,8 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
-    parallelism: 6
+    timeout_in_minutes: 60
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'
@@ -40,7 +40,7 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
     parallelism: 1
     retry:
       automatic: false

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -21,6 +21,8 @@ steps:
     build:
       env:
         ENVIRONMENT: ${ENVIRONMENT}
+        EC_ENV: qa
+        EC_REGION: aws-eu-west-1
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
   - group: ":female-detective: Security Solution Tests"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -21,6 +21,16 @@ steps:
         NAME_PREFIX: ci_test_kibana-promotion_
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
 
+  - label: ":pipeline::kibana::seedling: Trigger Kibana Serverless Tests for ${ENVIRONMENT}"
+    trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
+    soft_fail: true # Remove this before release or when tests stabilize
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+        EC_ENV: staging
+        EC_REGION: aws-us-east-1
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -59,12 +59,6 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:hard-typecheck')) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check.yml'));
-    } else {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check_selective.yml'));
-    }
-
     if (
       (await doAnyChangesMatch([
         /^src\/plugins\/controls/,

.buildkite/scripts/steps/check_types_commits.sh

@@ -2,25 +2,38 @@
 
 set -euo pipefail
 
+# This script will collect typescript projects and run typecheck on projects between the given 2 parameters
+# Could be used for selective typechecking on projects that might be affected for a given PR.
+# (The accuracy for finding related projects is not a 100%)
 
 if [[ "${CI-}" == "true" ]]; then
   .buildkite/scripts/bootstrap.sh
 
   sha1=$(git merge-base $GITHUB_PR_TARGET_BRANCH $GITHUB_PR_TRIGGERED_SHA)
   sha2="${GITHUB_PR_TRIGGERED_SHA-}"
 else
-  # Script take between 0 and 2 arguments representing two commit SHA's:
-  # If 0, it will diff HEAD and HEAD^
-  # If 1 (SHA1), it will diff SHA1 and SHA1^
-  # If 2 (SHA1, SHA2), it will diff SHA1 and SHA2
-  sha1="${1-HEAD}"
-  sha2="${2-$sha1^}"
+  if [[ "${1-}" == "--cached" ]]; then
+    # Only check staged files
+    sha1=$1
+    sha2=""
+  else
+    # Script take between 0 and 2 arguments representing two commit SHA's:
+    # If 0, it will diff HEAD and HEAD^
+    # If 1 (SHA1), it will diff SHA1 and SHA1^
+    # If 2 (SHA1, SHA2), it will diff SHA1 and SHA2
+    sha1="${1-HEAD}"
+    sha2="${2-$sha1^}"
+  fi
 fi
 
 uniq_dirs=()
 uniq_tsconfigs=()
 
-echo "Detecting files changed between $sha1 and $sha2..."
+if [[ "$sha1" == "--cached" ]]; then
+  echo "Detecting files changed in staging area..."
+else
+  echo "Detecting files changed between $sha1 and $sha2..."
+fi
 
 files=($(git diff --name-only $sha1 $sha2))
 
@@ -96,19 +109,35 @@ done
 
 echo "Looking for related tsconfig.json files..."
 
-for dir in "${uniq_dirs[@]}"
-do
-  find_tsconfig $dir
-done
+if [ ${#uniq_dirs[@]} -gt 0 ]; then
+  for dir in "${uniq_dirs[@]}"
+  do
+    find_tsconfig $dir
+  done
+fi
 
 if [ ${#uniq_tsconfigs[@]} -eq 0 ]; then
-  echo "No tsconfig.json files found for changes in $sha1 $sha2"
+  if [[ "$sha1" == "--cached" ]]; then
+    echo "No tsconfig.json files found for staged changes"
+  else
+    echo "No tsconfig.json files found for changes between $sha1 and $sha2"
+  fi
   exit
 fi
 
 echo "Running scripts/type_check for each found tsconfig.json file..."
 
+finalExitCode=0
+
 for tsconfig in "${uniq_tsconfigs[@]}"
 do
+  set +e
   node scripts/type_check --project $tsconfig
-done
+  exitCode=$?
+  set -e
+  if [ "$exitCode" -gt "$finalExitCode" ]; then
+    finalExitCode=$exitCode
+  fi
+done
+
+exit $finalExitCode

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -24,11 +24,37 @@ echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
 echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
 
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-docker pull "$SOURCE_IMAGE"
-docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"
-docker push "$TARGET_IMAGE"
 
-ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE")
+docker manifest inspect "$SOURCE_IMAGE" | tee manifests.json
+
+ARM_64_DIGEST=$(jq -r '.manifests[] | select(.platform.architecture == "arm64") | .digest' manifests.json)
+AMD_64_DIGEST=$(jq -r '.manifests[] | select(.platform.architecture == "amd64") | .digest' manifests.json)
+
+echo docker pull --platform linux/arm64 "$SOURCE_IMAGE@$ARM_64_DIGEST"
+docker pull --platform linux/arm64 "$SOURCE_IMAGE@$ARM_64_DIGEST"
+echo linux/arm64 image pulled, with digest: $ARM_64_DIGEST
+
+echo docker pull --platform linux/amd64 "$SOURCE_IMAGE@$AMD_64_DIGEST"
+docker pull --platform linux/amd64 "$SOURCE_IMAGE@$AMD_64_DIGEST"
+echo linux/amd64 image pulled, with digest: $AMD_64_DIGEST
+
+docker tag "$SOURCE_IMAGE@$ARM_64_DIGEST" "$TARGET_IMAGE-arm64"
+docker tag "$SOURCE_IMAGE@$AMD_64_DIGEST" "$TARGET_IMAGE-amd64"
+
+docker push "$TARGET_IMAGE-arm64"
+docker push "$TARGET_IMAGE-amd64"
+
+docker manifest rm "$TARGET_IMAGE" || echo "Nothing to delete"
+
+docker manifest create "$TARGET_IMAGE" \
+--amend "$TARGET_IMAGE-arm64" \
+--amend "$TARGET_IMAGE-amd64"
+
+docker manifest push "$TARGET_IMAGE"
+
+docker manifest inspect "$TARGET_IMAGE"
+
+ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE@$ARM_64_DIGEST")
 ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]')
 
 docker logout docker.elastic.co
@@ -37,7 +63,7 @@ echo "Image push to $TARGET_IMAGE successful."
 echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
 
 MANIFEST_UPLOAD_PATH="Skipped"
-if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+if [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Uploading latest-verified manifest to GCS"
   cat << EOT >> $MANIFEST_FILE_NAME
 {
@@ -58,10 +84,12 @@ EOT
   gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
   MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
 
-elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then
+elif [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
 elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"
+else
+  echo "--- Skipping upload of latest-verified manifest to GCS, no flag and hash provided"
 fi
 
 echo "--- Annotating build with info"

.eslintrc.js

@@ -908,6 +908,7 @@ module.exports = {
     },
     {
       files: [
+        'x-pack/plugins/aiops/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/apm/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/exploratory_view/**/*.{js,mjs,ts,tsx}',
         'x-pack/plugins/infra/**/*.{js,mjs,ts,tsx}',