From 3c12119cb6ec7a8f56590401d0c6538c1fec1d95 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 11:30:46 -0400 Subject: [PATCH 01/35] [buildkite] Add uptime playwright tests to PR pipeline (#115590) (#115727) Co-authored-by: Brian Seeders --- .buildkite/pipelines/pull_request/uptime.yml | 11 +++++++++++ .../scripts/pipelines/pull_request/pipeline.js | 4 ++++ .buildkite/scripts/steps/functional/common.sh | 2 ++ .buildkite/scripts/steps/functional/uptime.sh | 17 +++++++++++++++++ x-pack/plugins/uptime/e2e/config.ts | 2 +- x-pack/plugins/uptime/e2e/playwright_start.ts | 15 +++++---------- x-pack/plugins/uptime/scripts/e2e.js | 18 +++++++++++++----- 7 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 .buildkite/pipelines/pull_request/uptime.yml create mode 100755 .buildkite/scripts/steps/functional/uptime.sh diff --git a/.buildkite/pipelines/pull_request/uptime.yml b/.buildkite/pipelines/pull_request/uptime.yml new file mode 100644 index 00000000000000..60fdea1add04c6 --- /dev/null +++ b/.buildkite/pipelines/pull_request/uptime.yml @@ -0,0 +1,11 @@ +steps: + - command: .buildkite/scripts/steps/functional/uptime.sh + label: 'Uptime @elastic/synthetics Tests' + agents: + queue: ci-group-6 + depends_on: build + timeout_in_minutes: 120 + retry: + automatic: + - exit_status: '*' + limit: 1 diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.js b/.buildkite/scripts/pipelines/pull_request/pipeline.js index 028c90020a0b8d..7b5c944d31c1c0 100644 --- a/.buildkite/scripts/pipelines/pull_request/pipeline.js +++ b/.buildkite/scripts/pipelines/pull_request/pipeline.js @@ -73,6 +73,10 @@ const uploadPipeline = (pipelineContent) => { // pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml')); // } + if (await doAnyChangesMatch([/^x-pack\/plugins\/uptime/])) { + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/uptime.yml')); + } + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml')); uploadPipeline(pipeline.join('\n')); diff --git a/.buildkite/scripts/steps/functional/common.sh b/.buildkite/scripts/steps/functional/common.sh index b60ed835799e57..bedd22c53c7ec8 100755 --- a/.buildkite/scripts/steps/functional/common.sh +++ b/.buildkite/scripts/steps/functional/common.sh @@ -2,6 +2,8 @@ set -euo pipefail +# Note, changes here might also need to be made in other scripts, e.g. uptime.sh + source .buildkite/scripts/common/util.sh .buildkite/scripts/bootstrap.sh diff --git a/.buildkite/scripts/steps/functional/uptime.sh b/.buildkite/scripts/steps/functional/uptime.sh new file mode 100755 index 00000000000000..5a59f4dfa48bd7 --- /dev/null +++ b/.buildkite/scripts/steps/functional/uptime.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +.buildkite/scripts/bootstrap.sh +.buildkite/scripts/download_build_artifacts.sh + +export JOB=kibana-uptime-playwright + +echo "--- Uptime @elastic/synthetics Tests" + +cd "$XPACK_DIR" + +checks-reporter-with-killswitch "Uptime @elastic/synthetics Tests" \ + node plugins/uptime/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" diff --git a/x-pack/plugins/uptime/e2e/config.ts b/x-pack/plugins/uptime/e2e/config.ts index 70cc57247d4907..c5d573afccd967 100644 --- a/x-pack/plugins/uptime/e2e/config.ts +++ b/x-pack/plugins/uptime/e2e/config.ts @@ -39,7 +39,7 @@ async function config({ readConfigFile }: FtrConfigProviderContext) { '--csp.warnLegacyBrowsers=false', // define custom kibana server args here `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, - `--elasticsearch.ignoreVersionMismatch=true`, + `--elasticsearch.ignoreVersionMismatch=${process.env.CI ? 'false' : 'true'}`, `--uiSettings.overrides.theme:darkMode=true`, `--elasticsearch.username=kibana_system`, `--elasticsearch.password=changeme`, diff --git a/x-pack/plugins/uptime/e2e/playwright_start.ts b/x-pack/plugins/uptime/e2e/playwright_start.ts index aedb255b058be4..5949339c1ba259 100644 --- a/x-pack/plugins/uptime/e2e/playwright_start.ts +++ b/x-pack/plugins/uptime/e2e/playwright_start.ts @@ -15,15 +15,10 @@ import './journeys'; export function playwrightRunTests() { return async ({ getService }: any) => { - try { - const result = await playwrightStart(getService); - - if (result && result.uptime.status !== 'succeeded') { - process.exit(1); - } - } catch (error) { - console.error('errors: ', error); - process.exit(1); + const result = await playwrightStart(getService); + + if (result && result.uptime.status !== 'succeeded') { + throw new Error('Tests failed'); } }; } @@ -42,7 +37,7 @@ async function playwrightStart(getService: any) { const res = await playwrightRun({ params: { kibanaUrl }, - playwrightOptions: { chromiumSandbox: false, timeout: 60 * 1000 }, + playwrightOptions: { headless: true, chromiumSandbox: false, timeout: 60 * 1000 }, }); console.log('Removing esArchiver...'); diff --git a/x-pack/plugins/uptime/scripts/e2e.js b/x-pack/plugins/uptime/scripts/e2e.js index e2a8dfaf25c93e..e7c0cb612646d2 100644 --- a/x-pack/plugins/uptime/scripts/e2e.js +++ b/x-pack/plugins/uptime/scripts/e2e.js @@ -28,9 +28,14 @@ const { argv } = yargs(process.argv.slice(2)) type: 'boolean', description: 'Opens the Playwright Test Runner', }) + .option('kibana-install-dir', { + default: '', + type: 'string', + description: 'Path to the Kibana install directory', + }) .help(); -const { server, runner, open } = argv; +const { server, runner, open, kibanaInstallDir } = argv; const e2eDir = path.join(__dirname, '../e2e'); @@ -44,9 +49,12 @@ if (server) { const config = './playwright_run.ts'; function executeRunner() { - childProcess.execSync(`node ../../../scripts/${ftrScript} --config ${config}`, { - cwd: e2eDir, - stdio: 'inherit', - }); + childProcess.execSync( + `node ../../../scripts/${ftrScript} --config ${config} --kibana-install-dir '${kibanaInstallDir}'`, + { + cwd: e2eDir, + stdio: 'inherit', + } + ); } executeRunner(); From 470b39a1f558bf61e7d66984ce87153c17991377 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 12:00:13 -0400 Subject: [PATCH 02/35] [TSVB] Fixes the long text problem that appears behind the gauge chart (#115516) (#115745) Co-authored-by: Stratoula Kalafateli --- .../timeseries/public/application/visualizations/views/gauge.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/vis_types/timeseries/public/application/visualizations/views/gauge.js b/src/plugins/vis_types/timeseries/public/application/visualizations/views/gauge.js index ca5021a8829325..0e4322a7ba82c7 100644 --- a/src/plugins/vis_types/timeseries/public/application/visualizations/views/gauge.js +++ b/src/plugins/vis_types/timeseries/public/application/visualizations/views/gauge.js @@ -75,6 +75,7 @@ export class Gauge extends Component { top: this.state.top || 0, left: this.state.left || 0, transform: `matrix(${scale}, 0, 0, ${scale}, ${translateX}, ${translateY})`, + zIndex: 1, }, valueColor: { color: this.props.valueColor, From 408168ad7bd6466bda1d4b1cae15dc9545a5f8bb Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Wed, 20 Oct 2021 18:01:13 +0200 Subject: [PATCH 03/35] [ftr] update webdriver dependency to 4.0 (#115649) (#115767) * [ftr] update webdriver dependency to 4.0 * [ftr] cast options on assign Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- package.json | 4 +- test/functional/services/remote/webdriver.ts | 124 +++++++++---------- yarn.lock | 59 ++++----- 3 files changed, 87 insertions(+), 100 deletions(-) diff --git a/package.json b/package.json index 256163fff50781..2a80c3171c097b 100644 --- a/package.json +++ b/package.json @@ -605,7 +605,7 @@ "@types/reduce-reducers": "^1.0.0", "@types/redux-actions": "^2.6.1", "@types/seedrandom": ">=2.0.0 <4.0.0", - "@types/selenium-webdriver": "^4.0.9", + "@types/selenium-webdriver": "^4.0.15", "@types/semver": "^7", "@types/set-value": "^2.0.0", "@types/sinon": "^7.0.13", @@ -787,7 +787,7 @@ "rxjs-marbles": "^5.0.6", "sass-loader": "^10.2.0", "sass-resources-loader": "^2.0.1", - "selenium-webdriver": "^4.0.0-alpha.7", + "selenium-webdriver": "^4.0.0", "serve-static": "1.14.1", "shelljs": "^0.8.4", "simple-git": "1.116.0", diff --git a/test/functional/services/remote/webdriver.ts b/test/functional/services/remote/webdriver.ts index b501acb296b35c..063c3527d4da3f 100644 --- a/test/functional/services/remote/webdriver.ts +++ b/test/functional/services/remote/webdriver.ts @@ -71,6 +71,60 @@ export interface BrowserConfig { acceptInsecureCerts: boolean; } +function initChromiumOptions(browserType: Browsers, acceptInsecureCerts: boolean) { + const options = browserType === Browsers.Chrome ? new chrome.Options() : new edge.Options(); + + options.addArguments( + // Disables the sandbox for all process types that are normally sandboxed. + 'no-sandbox', + // Launches URL in new browser window. + 'new-window', + // By default, file:// URIs cannot read other file:// URIs. This is an override for developers who need the old behavior for testing. + 'allow-file-access-from-files', + // Use fake device for Media Stream to replace actual camera and microphone. + 'use-fake-device-for-media-stream', + // Bypass the media stream infobar by selecting the default device for media streams (e.g. WebRTC). Works with --use-fake-device-for-media-stream. + 'use-fake-ui-for-media-stream' + ); + + if (process.platform === 'linux') { + // The /dev/shm partition is too small in certain VM environments, causing + // Chrome to fail or crash. Use this flag to work-around this issue + // (a temporary directory will always be used to create anonymous shared memory files). + options.addArguments('disable-dev-shm-usage'); + } + + if (headlessBrowser === '1') { + // Use --disable-gpu to avoid an error from a missing Mesa library, as per + // See: https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md + options.headless(); + options.addArguments('disable-gpu'); + } + + if (certValidation === '0') { + options.addArguments('ignore-certificate-errors'); + } + + if (remoteDebug === '1') { + // Visit chrome://inspect in chrome to remotely view/debug + options.headless(); + options.addArguments('disable-gpu', 'remote-debugging-port=9222'); + } + + if (browserBinaryPath) { + options.setChromeBinaryPath(browserBinaryPath); + } + + const prefs = new logging.Preferences(); + prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL); + options.setUserPreferences(chromiumUserPrefs); + options.setLoggingPrefs(prefs); + options.set('unexpectedAlertBehaviour', 'accept'); + options.setAcceptInsecureCerts(acceptInsecureCerts); + + return options; +} + let attemptCounter = 0; let edgePaths: { driverPath: string | undefined; browserPath: string | undefined }; async function attemptToCreateCommand( @@ -86,55 +140,10 @@ async function attemptToCreateCommand( const buildDriverInstance = async () => { switch (browserType) { case 'chrome': { - const chromeOptions = new chrome.Options(); - chromeOptions.addArguments( - // Disables the sandbox for all process types that are normally sandboxed. - 'no-sandbox', - // Launches URL in new browser window. - 'new-window', - // By default, file:// URIs cannot read other file:// URIs. This is an override for developers who need the old behavior for testing. - 'allow-file-access-from-files', - // Use fake device for Media Stream to replace actual camera and microphone. - 'use-fake-device-for-media-stream', - // Bypass the media stream infobar by selecting the default device for media streams (e.g. WebRTC). Works with --use-fake-device-for-media-stream. - 'use-fake-ui-for-media-stream' - ); - - if (process.platform === 'linux') { - // The /dev/shm partition is too small in certain VM environments, causing - // Chrome to fail or crash. Use this flag to work-around this issue - // (a temporary directory will always be used to create anonymous shared memory files). - chromeOptions.addArguments('disable-dev-shm-usage'); - } - - if (headlessBrowser === '1') { - // Use --disable-gpu to avoid an error from a missing Mesa library, as per - // See: https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md - chromeOptions.headless(); - chromeOptions.addArguments('disable-gpu'); - } - - if (certValidation === '0') { - chromeOptions.addArguments('ignore-certificate-errors'); - } - - if (remoteDebug === '1') { - // Visit chrome://inspect in chrome to remotely view/debug - chromeOptions.headless(); - chromeOptions.addArguments('disable-gpu', 'remote-debugging-port=9222'); - } - - if (browserBinaryPath) { - chromeOptions.setChromeBinaryPath(browserBinaryPath); - } - - const prefs = new logging.Preferences(); - prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL); - chromeOptions.setUserPreferences(chromiumUserPrefs); - chromeOptions.setLoggingPrefs(prefs); - chromeOptions.set('unexpectedAlertBehaviour', 'accept'); - chromeOptions.setAcceptInsecureCerts(config.acceptInsecureCerts); - + const chromeOptions = initChromiumOptions( + browserType, + config.acceptInsecureCerts + ) as chrome.Options; let session; if (remoteSessionUrl) { session = await new Builder() @@ -164,19 +173,10 @@ async function attemptToCreateCommand( case 'msedge': { if (edgePaths && edgePaths.browserPath && edgePaths.driverPath) { - const edgeOptions = new edge.Options(); - if (headlessBrowser === '1') { - // @ts-ignore internal modules are not typed - edgeOptions.headless(); - } - // @ts-ignore internal modules are not typed - edgeOptions.setEdgeChromium(true); - // @ts-ignore internal modules are not typed - edgeOptions.setBinaryPath(edgePaths.browserPath); - const options = edgeOptions.get('ms:edgeOptions'); - // overriding options to include preferences - Object.assign(options, { prefs: chromiumUserPrefs }); - edgeOptions.set('ms:edgeOptions', options); + const edgeOptions = initChromiumOptions( + browserType, + config.acceptInsecureCerts + ) as edge.Options; const session = await new Builder() .forBrowser('MicrosoftEdge') .setEdgeOptions(edgeOptions) diff --git a/yarn.lock b/yarn.lock index ee3cc08cc3c3c9..ba8c04e1fc122c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7013,10 +7013,10 @@ resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.28.tgz#9ce8fa048c1e8c85cb71d7fe4d704e000226036f" integrity sha512-SMA+fUwULwK7sd/ZJicUztiPs8F1yCPwF3O23Z9uQ32ME5Ha0NmDK9+QTsYE4O2tHXChzXomSWWeIhCnoN1LqA== -"@types/selenium-webdriver@^4.0.9": - version "4.0.9" - resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-4.0.9.tgz#12621e55b2ef8f6c98bd17fe23fa720c6cba16bd" - integrity sha512-HopIwBE7GUXsscmt/J0DhnFXLSmO04AfxT6b8HAprknwka7pqEWquWDMXxCjd+NUHK9MkCe1SDKKsMiNmCItbQ== +"@types/selenium-webdriver@^4.0.15": + version "4.0.15" + resolved "https://registry.yarnpkg.com/@types/selenium-webdriver/-/selenium-webdriver-4.0.15.tgz#03012b84155cf6bbae2f64aa9dccf2a84c78c7c8" + integrity sha512-5760PIZkzhPejy3hsKAdCKe5LJygGdxLKOLxmZL9GEUcFlO5OgzM6G2EbdbvOnaw4xvUSa9Uip6Ipwkih12BPA== "@types/semver@^7": version "7.3.4" @@ -18924,10 +18924,10 @@ jsts@^1.6.2: array-includes "^3.1.2" object.assign "^4.1.2" -jszip@^3.2.2: - version "3.3.0" - resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.3.0.tgz#29d72c21a54990fa885b11fc843db320640d5271" - integrity sha512-EJ9k766htB1ZWnsV5ZMDkKLgA+201r/ouFF8R2OigVjVdcm2rurcBrrdXaeqBJbqnUVMko512PYmlncBKE1Huw== +jszip@^3.6.0: + version "3.7.1" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.1.tgz#bd63401221c15625a1228c556ca8a68da6fda3d9" + integrity sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg== dependencies: lie "~3.3.0" pako "~1.0.2" @@ -21904,7 +21904,7 @@ os-shim@^0.1.2: resolved "https://registry.yarnpkg.com/os-shim/-/os-shim-0.1.3.tgz#6b62c3791cf7909ea35ed46e17658bb417cb3917" integrity sha1-a2LDeRz3kJ6jXtRuF2WLtBfLORc= -os-tmpdir@^1.0.0, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2: +os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= @@ -25597,13 +25597,6 @@ rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.2, rimraf@^2.6.3: dependencies: glob "^7.1.3" -rimraf@^2.7.1: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== - dependencies: - glob "^7.1.3" - rimraf@~2.4.0: version "2.4.5" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.4.5.tgz#ee710ce5d93a8fdb856fb5ea8ff0e2d75934b2da" @@ -25909,14 +25902,15 @@ select-hose@^2.0.0: resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo= -selenium-webdriver@^4.0.0-alpha.7: - version "4.0.0-alpha.7" - resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.0.0-alpha.7.tgz#e3879d8457fd7ad8e4424094b7dc0540d99e6797" - integrity sha512-D4qnTsyTr91jT8f7MfN+OwY0IlU5+5FmlO5xlgRUV6hDEV8JyYx2NerdTEqDDkNq7RZDYc4VoPALk8l578RBHw== +selenium-webdriver@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/selenium-webdriver/-/selenium-webdriver-4.0.0.tgz#7dc8969facee3be634459e173f557b7e34308e73" + integrity sha512-tOlu6FnTjPq2FKpd153pl8o2cB7H40Rvl/ogiD2sapMv4IDjQqpIxbd+swDJe9UDLdszeh5CDis6lgy4e9UG1w== dependencies: - jszip "^3.2.2" - rimraf "^2.7.1" - tmp "0.0.30" + jszip "^3.6.0" + rimraf "^3.0.2" + tmp "^0.2.1" + ws ">=7.4.6" selfsigned@^1.10.7: version "1.10.8" @@ -28083,13 +28077,6 @@ title-case@^2.1.1: no-case "^2.2.0" upper-case "^1.0.3" -tmp@0.0.30: - version "0.0.30" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.30.tgz#72419d4a8be7d6ce75148fd8b324e593a711c2ed" - integrity sha1-ckGdSovn1s51FI/YsyTlk6cRwu0= - dependencies: - os-tmpdir "~1.0.1" - tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" @@ -28097,7 +28084,7 @@ tmp@^0.0.33: dependencies: os-tmpdir "~1.0.2" -tmp@~0.2.1: +tmp@^0.2.1, tmp@~0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== @@ -30508,6 +30495,11 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" +ws@>=7.4.6, ws@^7.4.6: + version "7.5.5" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" + integrity sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w== + ws@^6.1.2, ws@^6.2.1: version "6.2.2" resolved "https://registry.yarnpkg.com/ws/-/ws-6.2.2.tgz#dd5cdbd57a9979916097652d78f1cc5faea0c32e" @@ -30520,11 +30512,6 @@ ws@^7.2.3: resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== -ws@^7.4.6: - version "7.5.5" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" - integrity sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w== - x-is-function@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/x-is-function/-/x-is-function-1.0.4.tgz#5d294dc3d268cbdd062580e0c5df77a391d1fa1e" From 0fd2c80d6afec31f891766506db5a44ee76bcfd2 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 12:43:25 -0400 Subject: [PATCH 04/35] chore(NA): adds backport configs for renaming 7.x into 7.16 (#115783) (#115814) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tiago Costa --- .backportrc.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.backportrc.json b/.backportrc.json index 8ad858d37f840b..0162ef6085ab52 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -2,7 +2,7 @@ "upstream": "elastic/kibana", "targetBranchChoices": [ { "name": "master", "checked": true }, - { "name": "7.x", "checked": true }, + { "name": "7.16", "checked": true }, "7.15", "7.14", "7.13", @@ -33,7 +33,6 @@ "targetPRLabels": ["backport"], "branchLabelMapping": { "^v8.0.0$": "master", - "^v7.16.0$": "7.x", "^v(\\d+).(\\d+).\\d+$": "$1.$2" } } \ No newline at end of file From 2dffd8a143b3896d7680c5a9f58253022d4ed9e3 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 12:52:10 -0400 Subject: [PATCH 05/35] [backports] 7.16 should not be checked by default (#115819) (#115822) Signed-off-by: Tyler Smalley Co-authored-by: Tyler Smalley --- .backportrc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.backportrc.json b/.backportrc.json index 0162ef6085ab52..c5644d68996545 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -2,7 +2,7 @@ "upstream": "elastic/kibana", "targetBranchChoices": [ { "name": "master", "checked": true }, - { "name": "7.16", "checked": true }, + "7.16", "7.15", "7.14", "7.13", From 18a137c07570465470f9c0f0fa98d1846cd81621 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Wed, 20 Oct 2021 09:53:16 -0700 Subject: [PATCH 06/35] docs: update links to APM docs (#115664) (#115818) # Conflicts: # docs/apm/getting-started.asciidoc --- docs/apm/agent-configuration.asciidoc | 10 ------- docs/apm/api.asciidoc | 4 +-- docs/apm/apm-app-users.asciidoc | 16 +++++----- docs/apm/errors.asciidoc | 2 +- docs/apm/getting-started.asciidoc | 4 +++ docs/apm/service-maps.asciidoc | 2 +- docs/apm/spans.asciidoc | 2 +- .../apm-app-reader/widget.asciidoc | 30 +++++++++---------- .../central-config-users/widget.asciidoc | 30 +++++++++---------- docs/apm/transactions.asciidoc | 2 +- docs/apm/troubleshooting.asciidoc | 16 +++++----- docs/settings/apm-settings.asciidoc | 12 ++++---- .../how-to-secure-access-to-kibana.asciidoc | 2 +- 13 files changed, 62 insertions(+), 70 deletions(-) diff --git a/docs/apm/agent-configuration.asciidoc b/docs/apm/agent-configuration.asciidoc index 4e4a37067ea10a..ac9ce84e78c3eb 100644 --- a/docs/apm/agent-configuration.asciidoc +++ b/docs/apm/agent-configuration.asciidoc @@ -23,16 +23,6 @@ However, if APM Server is slow to respond, is offline, reports an error, etc., APM agents will use local defaults until they're able to update the configuration. For this reason, it is still essential to set custom default configurations locally in each of your agents. -[float] -==== APM Server setup - -This feature requires {apm-server-ref}/setup-kibana-endpoint.html[Kibana endpoint configuration] in APM Server. -In addition, if an APM agent is using {apm-server-ref}/configuration-anonymous.html[anonymous authentication] to communicate with the APM Server, -the agent's service name must be included in the `apm-server.auth.anonymous.allow_service` list. - -APM Server acts as a proxy between the agents and Kibana. -Kibana communicates any changed settings to APM Server so that your agents only need to poll APM Server to determine which settings have changed. - [float] ==== Supported configurations diff --git a/docs/apm/api.asciidoc b/docs/apm/api.asciidoc index 5f81a41e93df89..8bf1b38920141b 100644 --- a/docs/apm/api.asciidoc +++ b/docs/apm/api.asciidoc @@ -563,9 +563,7 @@ More information on Kibana's API is available in <>. === RUM source map API IMPORTANT: This endpoint is only compatible with the -{apm-server-ref}/apm-integration.html[APM integration for Elastic Agent]. -Users with a standalone APM Server should instead use the APM Server -{apm-server-ref}/sourcemap-api.html[source map upload API]. +{apm-guide-ref}/index.html[APM integration for Elastic Agent]. A source map allows minified files to be mapped back to original source code -- allowing you to maintain the speed advantage of minified code, diff --git a/docs/apm/apm-app-users.asciidoc b/docs/apm/apm-app-users.asciidoc index 7c2cef5b6b39ac..41ad67b1696e6f 100644 --- a/docs/apm/apm-app-users.asciidoc +++ b/docs/apm/apm-app-users.asciidoc @@ -56,8 +56,8 @@ To create an APM reader user: include::./tab-widgets/apm-app-reader/widget.asciidoc[] -- + -TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? -Add the privileges under the **Data streams** tab. +TIP: Using the deprecated APM Server binaries? +Add the privileges under the **Classic APM indices** tab above. . Assign the `read-apm` role created in the previous step, and the following built-in roles to any APM reader users: @@ -84,8 +84,8 @@ In some instances, you may wish to restrict certain Kibana apps that a user has include::./tab-widgets/apm-app-reader/widget.asciidoc[] -- + -TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? -Add the privileges under the **Data streams** tab. +TIP: Using the deprecated APM Server binaries? +Add the privileges under the **Classic APM indices** tab above. . Assign feature privileges to any Kibana feature that the user needs access to. Here are two examples: @@ -184,8 +184,8 @@ Central configuration users need to be able to view, create, update, and delete include::./tab-widgets/central-config-users/widget.asciidoc[] -- + -TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? -Add the privileges under the **Data streams** tab. +TIP: Using the deprecated APM Server binaries? +Add the privileges under the **Classic APM indices** tab above. . Assign the `central-config-manager` role created in the previous step, and the following Kibana feature privileges to anyone who needs to manage central configurations: @@ -211,8 +211,8 @@ but not create, update, or delete them. include::./tab-widgets/central-config-users/widget.asciidoc[] -- + -TIP: Using the {apm-server-ref-v}/apm-integration.html[APM integration for Elastic Agent]? -Add the privileges under the **Data streams** tab. +TIP: Using the deprecated APM Server binaries? +Add the privileges under the **Classic APM indices** tab above. . Assign the `central-config-reader` role created in the previous step, and the following Kibana feature privileges to anyone who needs to read central configurations: diff --git a/docs/apm/errors.asciidoc b/docs/apm/errors.asciidoc index d8fc75bf503409..c47604df03c992 100644 --- a/docs/apm/errors.asciidoc +++ b/docs/apm/errors.asciidoc @@ -2,7 +2,7 @@ [[errors]] === Errors -TIP: {apm-overview-ref-v}/errors.html[Errors] are groups of exceptions with a similar exception or log message. +TIP: {apm-guide-ref}/data-model-errors.html[Errors] are groups of exceptions with a similar exception or log message. The *Errors* overview provides a high-level view of the exceptions that APM agents catch, or that users manually report with APM agent APIs. diff --git a/docs/apm/getting-started.asciidoc b/docs/apm/getting-started.asciidoc index c0cb89b51fcc16..7c7321dc37e8c8 100644 --- a/docs/apm/getting-started.asciidoc +++ b/docs/apm/getting-started.asciidoc @@ -41,7 +41,11 @@ Notice something awry? Select a service or trace and dive deeper with: * <> TIP: Want to learn more about the Elastic APM ecosystem? +<<<<<<< HEAD See the {apm-overview-ref-v}/overview.html[APM Overview]. +======= +See the {apm-guide-ref}/apm-overview.html[APM Overview]. +>>>>>>> 2daadc0d74d (docs: update links to APM docs (#115664)) include::services.asciidoc[] diff --git a/docs/apm/service-maps.asciidoc b/docs/apm/service-maps.asciidoc index f43253d8194290..f76b9976dd1d2e 100644 --- a/docs/apm/service-maps.asciidoc +++ b/docs/apm/service-maps.asciidoc @@ -41,7 +41,7 @@ We currently surface two types of service maps: === How do service maps work? Service maps rely on distributed traces to draw connections between services. -As {apm-overview-ref-v}/distributed-tracing.html[distributed tracing] is enabled out-of-the-box for supported technologies, so are service maps. +As {apm-guide-ref}/apm-distributed-tracing.html[distributed tracing] is enabled out-of-the-box for supported technologies, so are service maps. However, if a service isn't instrumented, or a `traceparent` header isn't being propagated to it, distributed tracing will not work, and the connection will not be drawn on the map. diff --git a/docs/apm/spans.asciidoc b/docs/apm/spans.asciidoc index 7f29b1f003f1cf..afe87efc0df1e0 100644 --- a/docs/apm/spans.asciidoc +++ b/docs/apm/spans.asciidoc @@ -16,7 +16,7 @@ You also get a stack trace, which shows the SQL query in your code. Finally, APM knows which files are your code and which are just modules or libraries that you've installed. These library frames will be minimized by default in order to show you the most relevant stack trace. -TIP: A {apm-overview-ref-v}/transaction-spans.html[span] is the duration of a single event. +TIP: A {apm-guide-ref}/data-model-spans.html[span] is the duration of a single event. Spans are automatically captured by APM agents, and you can also define custom spans. Each span has a type and is defined by a different color in the timeline/waterfall visualization. diff --git a/docs/apm/tab-widgets/apm-app-reader/widget.asciidoc b/docs/apm/tab-widgets/apm-app-reader/widget.asciidoc index 51c01367786b68..090cb002bcf27b 100644 --- a/docs/apm/tab-widgets/apm-app-reader/widget.asciidoc +++ b/docs/apm/tab-widgets/apm-app-reader/widget.asciidoc @@ -2,37 +2,37 @@
+ id="data-streams-tab" + aria-labelledby="data-streams" + hidden=""> ++++ -include::content.asciidoc[tag=classic-indices] +include::content.asciidoc[tag=data-streams] ++++
diff --git a/docs/apm/tab-widgets/central-config-users/widget.asciidoc b/docs/apm/tab-widgets/central-config-users/widget.asciidoc index 68bef4e50c5495..4a36e91e031ef7 100644 --- a/docs/apm/tab-widgets/central-config-users/widget.asciidoc +++ b/docs/apm/tab-widgets/central-config-users/widget.asciidoc @@ -2,37 +2,37 @@
+ id="data-streams-tab" + aria-labelledby="data-streams" + hidden=""> ++++ -include::content.asciidoc[tag=classic-indices] +include::content.asciidoc[tag=data-streams] ++++
diff --git a/docs/apm/transactions.asciidoc b/docs/apm/transactions.asciidoc index c0850e4e9d5078..e7555a6c3e3d6e 100644 --- a/docs/apm/transactions.asciidoc +++ b/docs/apm/transactions.asciidoc @@ -2,7 +2,7 @@ [[transactions]] === Transactions -TIP: A {apm-overview-ref-v}/transactions.html[transaction] describes an event captured by an Elastic APM agent instrumenting a service. +TIP: A {apm-guide-ref}/data-model-transactions.html[transaction] describes an event captured by an Elastic APM agent instrumenting a service. APM agents automatically collect performance metrics on HTTP requests, database queries, and much more. [role="screenshot"] diff --git a/docs/apm/troubleshooting.asciidoc b/docs/apm/troubleshooting.asciidoc index 84cdb9876dc630..d44de3c2efe2f5 100644 --- a/docs/apm/troubleshooting.asciidoc +++ b/docs/apm/troubleshooting.asciidoc @@ -12,7 +12,7 @@ https://github.com/elastic/kibana/pulls[pull request] with your proposed changes If your issue is potentially related to other components of the APM ecosystem, don't forget to check our other troubleshooting guides or discussion forum: -* {apm-server-ref}/troubleshooting.html[APM Server troubleshooting] +* {apm-guide-ref}/troubleshoot-apm.html[APM Server troubleshooting] * {apm-dotnet-ref}/troubleshooting.html[.NET agent troubleshooting] * {apm-go-ref}/troubleshooting.html[Go agent troubleshooting] * {apm-ios-ref}/troubleshooting.html[iOS agent troubleshooting] @@ -53,7 +53,7 @@ By default, this index template is created by APM Server on startup. However, this only happens if `setup.template.enabled` is `true` in `apm-server.yml`. You can create the index template manually by running `apm-server setup`. Take note that index templates *cannot* be applied retroactively -- they are only applied at index creation time. -More information is available in {apm-server-ref}/apm-server-configuration.html[Set up and configure]. +More information is available in {apm-guide-ref}/apm-server-configuration.html[Set up and configure]. You can check for the existence of an APM index template using the {ref}/indices-get-template.html[Get index template API]. @@ -68,12 +68,12 @@ GET /_template/apm-{version} *Using Logstash, Kafka, etc.* If you're not outputting data directly from APM Server to Elasticsearch (perhaps you're using Logstash or Kafka), then the index template will not be set up automatically. Instead, you'll need to -{apm-server-ref}/apm-server-template.html[load the template manually]. +{apm-guide-ref}/apm-server-template.html[load the template manually]. *Using a custom index names* This problem can also occur if you've customized the index name that you write APM data to. If you change the default, you must also configure the `setup.template.name` and `setup.template.pattern` options. -See {apm-server-ref}/configuration-template.html[Load the Elasticsearch index template]. +See {apm-guide-ref}/configuration-template.html[Load the Elasticsearch index template]. If the Elasticsearch index template has already been successfully loaded to the index, you can customize the indices that the APM app uses to display data. Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.indices.*` values to @@ -118,8 +118,8 @@ Instead, we should strip away the unique information and group our transactions In this case, that means naming all blog transactions, `/blog`, and all documentation transactions, `/guide`. If you feel like you'd be losing valuable information by following this naming convention, don't fret! -You can always add additional metadata to your transactions using {apm-overview-ref-v}/metadata.html#labels-fields[labels] (indexed) or -{apm-overview-ref-v}/metadata.html#custom-fields[custom context] (non-indexed). +You can always add additional metadata to your transactions using {apm-guide-ref-v}/metadata.html#labels-fields[labels] (indexed) or +{apm-guide-ref-v}/metadata.html#custom-fields[custom context] (non-indexed). After ensuring you've correctly named your transactions, you might still see an error in the APM app related to too many transaction names. @@ -182,10 +182,10 @@ Selecting the `apm-*` index pattern shows a listing of every field defined in th *Ensure a field is searchable* There are two things you can do to if you'd like to ensure a field is searchable: -1. Index your additional data as {apm-overview-ref-v}/metadata.html[labels] instead. +1. Index your additional data as {apm-guide-ref}/metadata.html[labels] instead. These are dynamic by default, which means they will be indexed and become searchable and aggregatable. -2. Use the {apm-server-ref}/configuration-template.html[`append_fields`] feature. As an example, +2. Use the {apm-guide-ref}/configuration-template.html[`append_fields`] feature. As an example, adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`: [source,yml] diff --git a/docs/settings/apm-settings.asciidoc b/docs/settings/apm-settings.asciidoc index ad19b38e46b3cf..76d39d63545981 100644 --- a/docs/settings/apm-settings.asciidoc +++ b/docs/settings/apm-settings.asciidoc @@ -84,7 +84,7 @@ Changing these settings may disable features of the APM App. | `xpack.apm.searchAggregatedTransactions` {ess-icon} | experimental[] Enables Transaction histogram metrics. Defaults to `never` and aggregated transactions are not used. When set to `auto`, the UI will use metric indices over transaction indices for transactions if aggregated transactions are found. When set to `always`, additional configuration in APM Server is required. - See {apm-server-ref-v}/transaction-metrics.html[Configure transaction metrics] for more information. + See {apm-guide-ref}/transaction-metrics.html[Configure transaction metrics] for more information. | `xpack.apm.metricsInterval` {ess-icon} | Sets a `fixed_interval` for date histograms in metrics aggregations. Defaults to `30`. @@ -93,22 +93,22 @@ Changing these settings may disable features of the APM App. | Set to `false` to disable cloud APM migrations. Defaults to `true`. | `xpack.apm.indices.error` {ess-icon} - | Matcher for all {apm-server-ref}/error-indices.html[error indices]. Defaults to `logs-apm*,apm-*`. + | Matcher for all error indices. Defaults to `logs-apm*,apm-*`. | `xpack.apm.indices.onboarding` {ess-icon} | Matcher for all onboarding indices. Defaults to `apm-*`. | `xpack.apm.indices.span` {ess-icon} - | Matcher for all {apm-server-ref}/span-indices.html[span indices]. Defaults to `traces-apm*,apm-*`. + | Matcher for all span indices. Defaults to `traces-apm*,apm-*`. | `xpack.apm.indices.transaction` {ess-icon} - | Matcher for all {apm-server-ref}/transaction-indices.html[transaction indices]. Defaults to `traces-apm*,apm-*`. + | Matcher for all transaction indices. Defaults to `traces-apm*,apm-*`. | `xpack.apm.indices.metric` {ess-icon} - | Matcher for all {apm-server-ref}/metricset-indices.html[metrics indices]. Defaults to `metrics-apm*,apm-*`. + | Matcher for all metrics indices. Defaults to `metrics-apm*,apm-*`. | `xpack.apm.indices.sourcemap` {ess-icon} - | Matcher for all {apm-server-ref}/sourcemap-indices.html[source map indices]. Defaults to `apm-*`. + | Matcher for all source map indices. Defaults to `apm-*`. |=== diff --git a/docs/user/security/tutorials/how-to-secure-access-to-kibana.asciidoc b/docs/user/security/tutorials/how-to-secure-access-to-kibana.asciidoc index c1a46e65894da4..b041c6da266170 100644 --- a/docs/user/security/tutorials/how-to-secure-access-to-kibana.asciidoc +++ b/docs/user/security/tutorials/how-to-secure-access-to-kibana.asciidoc @@ -13,7 +13,7 @@ This guide introduces you to three of {kib}'s security features: spaces, roles, Do you have multiple teams or tenants using {kib}? Do you want a “playground” to experiment with new visualizations or alerts? If so, then <> can help. -Think of a space as another instance of {kib}. A space allows you to organize your <>, <>, <>, and much more into their own categories. For example, you might have a Marketing space for your marketeers to track the results of their campaigns, and an Engineering space for your developers to {apm-get-started-ref}/overview.html[monitor application performance]. +Think of a space as another instance of {kib}. A space allows you to organize your <>, <>, <>, and much more into their own categories. For example, you might have a Marketing space for your marketeers to track the results of their campaigns, and an Engineering space for your developers to {apm-guide-ref}/apm-overview.html[monitor application performance]. The assets you create in one space are isolated from other spaces, so when you enter a space, you only see the assets that belong to that space. From 6044b8c0decd67df96e60bbd557c14aa793c70f7 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Wed, 20 Oct 2021 18:00:44 +0100 Subject: [PATCH 07/35] [7.16] chore(NA): generic config changes for renaming 7.x into 7.16 (#115779) * [7.16] chore(NA): generic config changes for renaming 7.x into 7.16 * chore(NA): removes packer cache from 7.16 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .ci/packer_cache.sh | 60 --------------------------------------------- package.json | 2 +- 2 files changed, 1 insertion(+), 61 deletions(-) delete mode 100755 .ci/packer_cache.sh diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh deleted file mode 100755 index 5f091cc2e11a60..00000000000000 --- a/.ci/packer_cache.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -e - -branch="$(git rev-parse --abbrev-ref HEAD 2> /dev/null)" - -# run setup script that gives us node, yarn, and bootstraps the project -source src/dev/ci_setup/setup.sh; - -# download es snapshots -node scripts/es snapshot --download-only; -node scripts/es snapshot --license=oss --download-only; - -# download reporting browsers -(cd "x-pack" && yarn gulp prepare); - -# cache the chromedriver archive -chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")" -chromedriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")" -if [ -z "$chromedriverDistVersion" ] || [ -z "$chromedriverPkgVersion" ]; then - echo "UNABLE TO DETERMINE CHROMEDRIVER VERSIONS" - exit 1 -fi -mkdir -p .chromedriver -curl "https://chromedriver.storage.googleapis.com/$chromedriverDistVersion/chromedriver_linux64.zip" > .chromedriver/chromedriver.zip -echo "$chromedriverPkgVersion" > .chromedriver/pkgVersion - -# cache the geckodriver archive -geckodriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.geckodriver)")" -if [ -z "$geckodriverPkgVersion" ]; then - echo "UNABLE TO DETERMINE geckodriver VERSIONS" - exit 1 -fi -mkdir -p ".geckodriver" -cp "node_modules/geckodriver/geckodriver.tar.gz" .geckodriver/geckodriver.tar.gz -echo "$geckodriverPkgVersion" > .geckodriver/pkgVersion - -# archive cacheable directories -mkdir -p "$HOME/.kibana/bootstrap_cache" -tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ - node_modules \ - packages/*/node_modules \ - x-pack/node_modules \ - x-pack/legacy/plugins/*/node_modules \ - x-pack/plugins/reporting/.chromium \ - test/plugin_functional/plugins/*/node_modules \ - examples/*/node_modules \ - .es \ - .chromedriver \ - .geckodriver; - -echo "created $HOME/.kibana/bootstrap_cache/$branch.tar" - -if [ "$branch" == "master" ]; then - echo "Creating bootstrap cache for 7.x"; - - git clone https://github.com/elastic/kibana.git --branch 7.x --depth 1 /tmp/kibana-7.x - (cd /tmp/kibana-7.x && ./.ci/packer_cache.sh); - rm -rf /tmp/kibana-7.x; -fi diff --git a/package.json b/package.json index 2a80c3171c097b..fe9b8ec09c5c8d 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ ], "private": true, "version": "7.16.0", - "branch": "7.x", + "branch": "7.16", "types": "./kibana.d.ts", "tsdocMetadata": "./build/tsdoc-metadata.json", "build": { From 606f4f10352c1223332c4c28548f0916beb9f1cb Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:10:45 -0400 Subject: [PATCH 08/35] [7.x] [Security Solution][Timelines] - Resolve UI (#114350) (#115671) Co-authored-by: Michael Olorunnisola --- .../template_wrapper/bottom_bar/index.tsx | 2 + .../common/components/url_state/index.tsx | 1 + .../url_state/initialize_redux_by_url.tsx | 2 +- ...query_timeline_by_id_on_url_change.test.ts | 140 +++++++++++++++ .../query_timeline_by_id_on_url_change.ts | 84 +++++++++ .../common/components/url_state/types.ts | 1 + .../components/url_state/use_url_state.tsx | 16 +- .../hooks/use_resolve_conflict.test.tsx | 166 ++++++++++++++++++ .../common/hooks/use_resolve_conflict.tsx | 100 +++++++++++ .../common/hooks/use_resolve_redirect.test.ts | 140 +++++++++++++++ .../common/hooks/use_resolve_redirect.ts | 89 ++++++++++ .../open_timeline/__mocks__/index.ts | 6 +- .../components/open_timeline/helpers.test.ts | 28 +-- .../components/open_timeline/helpers.ts | 21 ++- .../components/open_timeline/types.ts | 7 + .../components/timeline/index.test.tsx | 6 + .../timelines/components/timeline/index.tsx | 4 +- .../public/timelines/containers/api.ts | 13 -- .../timelines/store/timeline/actions.ts | 2 + .../timelines/store/timeline/defaults.ts | 121 ++++++------- .../timelines/store/timeline/helpers.ts | 4 + .../public/timelines/store/timeline/model.ts | 2 + .../timelines/store/timeline/reducer.ts | 9 +- .../plugins/security_solution/public/types.ts | 2 +- .../plugins/security_solution/tsconfig.json | 5 +- 25 files changed, 868 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.test.ts create mode 100644 x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.test.tsx create mode 100644 x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.tsx create mode 100644 x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.test.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.ts diff --git a/x-pack/plugins/security_solution/public/app/home/template_wrapper/bottom_bar/index.tsx b/x-pack/plugins/security_solution/public/app/home/template_wrapper/bottom_bar/index.tsx index eb606cd8ff583b..8e972b92c2fa14 100644 --- a/x-pack/plugins/security_solution/public/app/home/template_wrapper/bottom_bar/index.tsx +++ b/x-pack/plugins/security_solution/public/app/home/template_wrapper/bottom_bar/index.tsx @@ -16,6 +16,7 @@ import { useSourcererScope, getScopeFromPath } from '../../../../common/containe import { TimelineId } from '../../../../../common/types/timeline'; import { AutoSaveWarningMsg } from '../../../../timelines/components/timeline/auto_save_warning'; import { Flyout } from '../../../../timelines/components/flyout'; +import { useResolveRedirect } from '../../../../common/hooks/use_resolve_redirect'; export const BOTTOM_BAR_CLASSNAME = 'timeline-bottom-bar'; @@ -26,6 +27,7 @@ export const SecuritySolutionBottomBar = React.memo( const [showTimeline] = useShowTimeline(); const { indicesExist } = useSourcererScope(getScopeFromPath(pathname)); + useResolveRedirect(); return indicesExist && showTimeline ? ( <> diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/index.tsx b/x-pack/plugins/security_solution/public/common/components/url_state/index.tsx index 281db88ebd0572..2e04bbc5f1daf2 100644 --- a/x-pack/plugins/security_solution/public/common/components/url_state/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/url_state/index.tsx @@ -25,6 +25,7 @@ export const UseUrlStateMemo = React.memo( prevProps.pathName === nextProps.pathName && deepEqual(prevProps.urlState, nextProps.urlState) && deepEqual(prevProps.indexPattern, nextProps.indexPattern) && + prevProps.search === nextProps.search && deepEqual(prevProps.navTabs, nextProps.navTabs) ); diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx b/x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx index 4a448e9064090f..f04cf30da61f54 100644 --- a/x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx +++ b/x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx @@ -124,7 +124,7 @@ export const useSetInitialStateFromUrl = () => { [dispatch, updateTimeline, updateTimelineIsLoading] ); - return setInitialStateFromUrl; + return Object.freeze({ setInitialStateFromUrl, updateTimeline, updateTimelineIsLoading }); }; const updateTimerange = (newUrlStateString: string, dispatch: Dispatch) => { diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.test.ts b/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.test.ts new file mode 100644 index 00000000000000..5cc4f8e8b80f93 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { queryTimelineById } from '../../../timelines/components/open_timeline/helpers'; +import { queryTimelineByIdOnUrlChange } from './query_timeline_by_id_on_url_change'; +import * as urlHelpers from './helpers'; + +jest.mock('../../../timelines/components/open_timeline/helpers'); + +describe('queryTimelineByIdOnUrlChange', () => { + const oldTestTimelineId = '04e8ffb0-2c2a-11ec-949c-39005af91f70'; + const newTestTimelineId = `${oldTestTimelineId}-newId`; + const oldTimelineRisonSearchString = `?timeline=(activeTab:query,graphEventId:%27%27,id:%27${oldTestTimelineId}%27,isOpen:!t)`; + const newTimelineRisonSearchString = `?timeline=(activeTab:query,graphEventId:%27%27,id:%27${newTestTimelineId}%27,isOpen:!t)`; + const mockUpdateTimeline = jest.fn(); + const mockUpdateTimelineIsLoading = jest.fn(); + const mockQueryTimelineById = jest.fn(); + beforeEach(() => { + (queryTimelineById as jest.Mock).mockImplementation(mockQueryTimelineById); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('when search strings are empty', () => { + it('should not call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: '', + search: '', + timelineIdFromReduxStore: 'current-timeline-id', + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + describe('when search string has not changed', () => { + it('should not call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: oldTimelineRisonSearchString, + search: oldTimelineRisonSearchString, + timelineIdFromReduxStore: 'timeline-id', + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + describe('when decode rison fails', () => { + it('should not call queryTimelineById', () => { + jest.spyOn(urlHelpers, 'decodeRisonUrlState').mockImplementationOnce(() => { + throw new Error('Unable to decode'); + }); + + queryTimelineByIdOnUrlChange({ + oldSearch: oldTimelineRisonSearchString, + search: newTimelineRisonSearchString, + timelineIdFromReduxStore: '', + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + describe('when new id is not provided', () => { + it('should not call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: oldTimelineRisonSearchString, + search: '?timeline=(activeTab:query)', // no id + timelineIdFromReduxStore: newTestTimelineId, + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + describe('when new id matches the data in redux', () => { + it('should not call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: oldTimelineRisonSearchString, + search: newTimelineRisonSearchString, + timelineIdFromReduxStore: newTestTimelineId, + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + // You can only redirect or run into conflict scenarios when already viewing a timeline + describe('when not actively on a page with timeline in the search field', () => { + it('should not call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: '?random=foo', + search: newTimelineRisonSearchString, + timelineIdFromReduxStore: oldTestTimelineId, + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).not.toBeCalled(); + }); + }); + + describe('when an old timeline id exists, but a new id is given', () => { + it('should call queryTimelineById', () => { + queryTimelineByIdOnUrlChange({ + oldSearch: oldTimelineRisonSearchString, + search: newTimelineRisonSearchString, + timelineIdFromReduxStore: oldTestTimelineId, + updateTimeline: mockUpdateTimeline, + updateTimelineIsLoading: mockUpdateTimelineIsLoading, + }); + + expect(queryTimelineById).toBeCalledWith({ + activeTimelineTab: 'query', + duplicate: false, + graphEventId: '', + timelineId: newTestTimelineId, + openTimeline: true, + updateIsLoading: mockUpdateTimelineIsLoading, + updateTimeline: mockUpdateTimeline, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.ts b/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.ts new file mode 100644 index 00000000000000..2778cefdc79530 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/url_state/query_timeline_by_id_on_url_change.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Action } from 'typescript-fsa'; +import { DispatchUpdateTimeline } from '../../../timelines/components/open_timeline/types'; +import { queryTimelineById } from '../../../timelines/components/open_timeline/helpers'; +import { TimelineTabs } from '../../../../common/types/timeline'; +import { + decodeRisonUrlState, + getQueryStringFromLocation, + getParamFromQueryString, +} from './helpers'; +import { TimelineUrl } from '../../../timelines/store/timeline/model'; +import { CONSTANTS } from './constants'; + +const getQueryStringKeyValue = ({ search, urlKey }: { search: string; urlKey: string }) => + getParamFromQueryString(getQueryStringFromLocation(search), urlKey); + +interface QueryTimelineIdOnUrlChange { + oldSearch?: string; + search: string; + timelineIdFromReduxStore: string; + updateTimeline: DispatchUpdateTimeline; + updateTimelineIsLoading: (status: { id: string; isLoading: boolean }) => Action<{ + id: string; + isLoading: boolean; + }>; +} + +/** + * After the initial load of the security solution, timeline is not updated when the timeline url search value is changed + * This is because those state changes happen in place and doesn't lead to a requerying of data for the new id. + * To circumvent this for the sake of the redirects needed for the saved object Id changes happening in 8.0 + * We are actively pulling the id changes that take place for timeline in the url and calling the query below + * to request the new data. + */ +export const queryTimelineByIdOnUrlChange = ({ + oldSearch, + search, + timelineIdFromReduxStore, + updateTimeline, + updateTimelineIsLoading, +}: QueryTimelineIdOnUrlChange) => { + const oldUrlStateString = getQueryStringKeyValue({ + urlKey: CONSTANTS.timeline, + search: oldSearch ?? '', + }); + + const newUrlStateString = getQueryStringKeyValue({ urlKey: CONSTANTS.timeline, search }); + + if (oldUrlStateString != null && newUrlStateString != null) { + let newTimeline = null; + let oldTimeline = null; + try { + newTimeline = decodeRisonUrlState(newUrlStateString); + } catch (error) { + // do nothing as timeline is defaulted to null + } + + try { + oldTimeline = decodeRisonUrlState(oldUrlStateString); + } catch (error) { + // do nothing as timeline is defaulted to null + } + const newId = newTimeline?.id; + const oldId = oldTimeline?.id; + + if (newId && newId !== oldId && newId !== timelineIdFromReduxStore) { + queryTimelineById({ + activeTimelineTab: newTimeline?.activeTab ?? TimelineTabs.query, + duplicate: false, + graphEventId: newTimeline?.graphEventId, + timelineId: newId, + openTimeline: true, + updateIsLoading: updateTimelineIsLoading, + updateTimeline, + }); + } + } +}; diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/types.ts b/x-pack/plugins/security_solution/public/common/components/url_state/types.ts index e803c091423be0..06ed33ac69f6e7 100644 --- a/x-pack/plugins/security_solution/public/common/components/url_state/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/url_state/types.ts @@ -79,6 +79,7 @@ export interface PreviousLocationUrlState { pathName: string | undefined; pageName: string | undefined; urlState: UrlState; + search: string | undefined; } export interface UrlStateToRedux { diff --git a/x-pack/plugins/security_solution/public/common/components/url_state/use_url_state.tsx b/x-pack/plugins/security_solution/public/common/components/url_state/use_url_state.tsx index bc47ba9d8ae990..3245d647227ad9 100644 --- a/x-pack/plugins/security_solution/public/common/components/url_state/use_url_state.tsx +++ b/x-pack/plugins/security_solution/public/common/components/url_state/use_url_state.tsx @@ -39,6 +39,7 @@ import { } from './types'; import { TimelineUrl } from '../../../timelines/store/timeline/model'; import { UrlInputsModel } from '../../store/inputs/model'; +import { queryTimelineByIdOnUrlChange } from './query_timeline_by_id_on_url_change'; function usePrevious(value: PreviousLocationUrlState) { const ref = useRef(value); @@ -60,9 +61,10 @@ export const useUrlStateHooks = ({ const [isFirstPageLoad, setIsFirstPageLoad] = useState(true); const { filterManager, savedQueries } = useKibana().services.data.query; const { pathname: browserPathName } = useLocation(); - const prevProps = usePrevious({ pathName, pageName, urlState }); + const prevProps = usePrevious({ pathName, pageName, urlState, search }); - const setInitialStateFromUrl = useSetInitialStateFromUrl(); + const { setInitialStateFromUrl, updateTimeline, updateTimelineIsLoading } = + useSetInitialStateFromUrl(); const handleInitialize = useCallback( (type: UrlStateType) => { @@ -190,6 +192,16 @@ export const useUrlStateHooks = ({ document.title = `${getTitle(pageName, navTabs)} - Kibana`; }, [pageName, navTabs]); + useEffect(() => { + queryTimelineByIdOnUrlChange({ + oldSearch: prevProps.search, + search, + timelineIdFromReduxStore: urlState.timeline.id, + updateTimeline, + updateTimelineIsLoading, + }); + }, [search, prevProps.search, urlState.timeline.id, updateTimeline, updateTimelineIsLoading]); + return null; }; diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.test.tsx b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.test.tsx new file mode 100644 index 00000000000000..bafbe078cdbdb7 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.test.tsx @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useLocation } from 'react-router-dom'; +import { renderHook } from '@testing-library/react-hooks'; +import { useDeepEqualSelector } from './use_selector'; +import { useKibana } from '../lib/kibana'; +import { useResolveConflict } from './use_resolve_conflict'; +import * as urlHelpers from '../components/url_state/helpers'; + +jest.mock('react-router-dom', () => { + const original = jest.requireActual('react-router-dom'); + + return { + ...original, + useLocation: jest.fn(), + }; +}); +jest.mock('../lib/kibana'); +jest.mock('./use_selector'); +jest.mock('../../timelines/store/timeline/', () => ({ + timelineSelectors: { + getTimelineByIdSelector: () => jest.fn(), + }, +})); + +describe('useResolveConflict', () => { + const mockGetLegacyUrlConflict = jest.fn().mockReturnValue('Test!'); + beforeEach(() => { + jest.resetAllMocks(); + // Mock rison format in actual url + (useLocation as jest.Mock).mockReturnValue({ + pathname: 'my/cool/path', + search: + 'timeline=(activeTab:query,graphEventId:%27%27,id:%2704e8ffb0-2c2a-11ec-949c-39005af91f70%27,isOpen:!t)', + }); + (useKibana as jest.Mock).mockReturnValue({ + services: { + spaces: { + ui: { + components: { + getLegacyUrlConflict: mockGetLegacyUrlConflict, + }, + }, + }, + }, + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('resolve object is not provided', () => { + it('should not show the conflict message', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + const { result } = renderHook<{}, JSX.Element | null>(() => useResolveConflict()); + expect(mockGetLegacyUrlConflict).not.toHaveBeenCalled(); + expect(result.current).toEqual(null); + }); + }); + + describe('outcome is exactMatch', () => { + it('should not show the conflict message', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'exactMatch', + }, + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + const { result } = renderHook<{}, JSX.Element | null>(() => useResolveConflict()); + expect(mockGetLegacyUrlConflict).not.toHaveBeenCalled(); + expect(result.current).toEqual(null); + }); + }); + + describe('outcome is aliasMatch', () => { + it('should not show the conflict message', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'aliasMatch', + alias_target_id: 'new-id', + }, + })); + const { result } = renderHook<{}, JSX.Element | null>(() => useResolveConflict()); + expect(mockGetLegacyUrlConflict).not.toHaveBeenCalled(); + expect(result.current).toEqual(null); + }); + }); + + describe('outcome is conflict', () => { + const mockTextContent = 'I am the visible conflict message'; + it('should show the conflict message', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'conflict', + alias_target_id: 'new-id', + }, + })); + mockGetLegacyUrlConflict.mockImplementation(() => mockTextContent); + const { result } = renderHook<{}, JSX.Element | null>(() => useResolveConflict()); + expect(mockGetLegacyUrlConflict).toHaveBeenCalledWith({ + objectNoun: 'timeline', + currentObjectId: '04e8ffb0-2c2a-11ec-949c-39005af91f70', + otherObjectId: 'new-id', + otherObjectPath: + 'my/cool/path?timeline=%28activeTab%3Aquery%2CgraphEventId%3A%27%27%2Cid%3Anew-id%2CisOpen%3A%21t%29', + }); + expect(result.current).toMatchInlineSnapshot(` + + I am the visible conflict message + + + `); + }); + + describe('rison is unable to be decoded', () => { + it('should use timeline values from redux to create the otherObjectPath', async () => { + jest.spyOn(urlHelpers, 'decodeRisonUrlState').mockImplementation(() => { + throw new Error('Unable to decode'); + }); + (useLocation as jest.Mock).mockReturnValue({ + pathname: 'my/cool/path', + search: '?foo=bar', + }); + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'conflict', + alias_target_id: 'new-id', + }, + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + mockGetLegacyUrlConflict.mockImplementation(() => mockTextContent); + renderHook(() => useResolveConflict()); + const { result } = renderHook<{}, JSX.Element | null>(() => useResolveConflict()); + expect(mockGetLegacyUrlConflict).toHaveBeenCalledWith({ + objectNoun: 'timeline', + currentObjectId: 'current-saved-object-id', + otherObjectId: 'new-id', + otherObjectPath: + 'my/cool/path?foo=bar&timeline=%28activeTab%3Asome-tab%2CgraphEventId%3Acurrent-graph-event-id%2Cid%3Anew-id%2CisOpen%3A%21f%29', + }); + expect(result.current).toMatchInlineSnapshot(` + + I am the visible conflict message + + + `); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.tsx b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.tsx new file mode 100644 index 00000000000000..6a493d944ecdac --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_conflict.tsx @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useMemo } from 'react'; +import { useLocation } from 'react-router-dom'; +import { EuiSpacer } from '@elastic/eui'; +import { useDeepEqualSelector } from './use_selector'; +import { TimelineId } from '../../../common/types/timeline'; +import { timelineSelectors } from '../../timelines/store/timeline'; +import { TimelineUrl } from '../../timelines/store/timeline/model'; +import { timelineDefaults } from '../../timelines/store/timeline/defaults'; +import { decodeRisonUrlState, encodeRisonUrlState } from '../components/url_state/helpers'; +import { useKibana } from '../lib/kibana'; +import { CONSTANTS } from '../components/url_state/constants'; + +/** + * Unfortunately the url change initiated when clicking the button to otherObjectPath doesn't seem to be + * respected by the useSetInitialStateFromUrl here: x-pack/plugins/security_solution/public/common/components/url_state/initialize_redux_by_url.tsx + * + * FYI: It looks like the routing causes replaceStateInLocation to be called instead: + * x-pack/plugins/security_solution/public/common/components/url_state/helpers.ts + * + * Potentially why the markdown component needs a click handler as well for timeline? + * see: /x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/timeline/processor.tsx + */ +export const useResolveConflict = () => { + const { search, pathname } = useLocation(); + const { spaces } = useKibana().services; + const getTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); + const { resolveTimelineConfig, savedObjectId, show, graphEventId, activeTab } = + useDeepEqualSelector((state) => getTimeline(state, TimelineId.active) ?? timelineDefaults); + + const getLegacyUrlConflictCallout = useCallback(() => { + // This function returns a callout component *if* we have encountered a "legacy URL conflict" scenario + if ( + !spaces || + resolveTimelineConfig?.outcome !== 'conflict' || + resolveTimelineConfig?.alias_target_id == null + ) { + return null; + } + + const searchQuery = new URLSearchParams(search); + const timelineRison = searchQuery.get(CONSTANTS.timeline) ?? undefined; + // Try to get state on URL, but default to what's in Redux in case of decodeRisonFailure + const currentTimelineState = { + id: savedObjectId ?? '', + isOpen: !!show, + activeTab, + graphEventId, + }; + let timelineSearch: TimelineUrl = currentTimelineState; + try { + timelineSearch = decodeRisonUrlState(timelineRison) ?? currentTimelineState; + } catch (error) { + // do nothing as it's already defaulted on line 77 + } + // We have resolved to one object, but another object has a legacy URL alias associated with this ID/page. We should display a + // callout with a warning for the user, and provide a way for them to navigate to the other object. + const currentObjectId = timelineSearch?.id; + const newSavedObjectId = resolveTimelineConfig?.alias_target_id ?? ''; // This is always defined if outcome === 'conflict' + + const newTimelineSearch: TimelineUrl = { + ...timelineSearch, + id: newSavedObjectId, + }; + const newTimelineRison = encodeRisonUrlState(newTimelineSearch); + searchQuery.set(CONSTANTS.timeline, newTimelineRison); + + const newPath = `${pathname}?${searchQuery.toString()}${window.location.hash}`; + + return ( + <> + {spaces.ui.components.getLegacyUrlConflict({ + objectNoun: CONSTANTS.timeline, + currentObjectId, + otherObjectId: newSavedObjectId, + otherObjectPath: newPath, + })} + + + ); + }, [ + activeTab, + graphEventId, + pathname, + resolveTimelineConfig?.alias_target_id, + resolveTimelineConfig?.outcome, + savedObjectId, + search, + show, + spaces, + ]); + + return useMemo(() => getLegacyUrlConflictCallout(), [getLegacyUrlConflictCallout]); +}; diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.test.ts b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.test.ts new file mode 100644 index 00000000000000..c9a0eedefd0af5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useLocation } from 'react-router-dom'; +import { renderHook } from '@testing-library/react-hooks'; +import { useDeepEqualSelector } from './use_selector'; +import { useKibana } from '../lib/kibana'; +import { useResolveRedirect } from './use_resolve_redirect'; +import * as urlHelpers from '../components/url_state/helpers'; + +jest.mock('react-router-dom', () => { + const original = jest.requireActual('react-router-dom'); + + return { + ...original, + useLocation: jest.fn(), + }; +}); +jest.mock('../lib/kibana'); +jest.mock('./use_selector'); +jest.mock('../../timelines/store/timeline/', () => ({ + timelineSelectors: { + getTimelineByIdSelector: () => jest.fn(), + }, +})); + +describe('useResolveRedirect', () => { + const mockRedirectLegacyUrl = jest.fn(); + beforeEach(() => { + jest.resetAllMocks(); + // Mock rison format in actual url + (useLocation as jest.Mock).mockReturnValue({ + pathname: 'my/cool/path', + search: + 'timeline=(activeTab:query,graphEventId:%27%27,id:%2704e8ffb0-2c2a-11ec-949c-39005af91f70%27,isOpen:!t)', + }); + (useKibana as jest.Mock).mockReturnValue({ + services: { + spaces: { + ui: { + redirectLegacyUrl: mockRedirectLegacyUrl, + }, + }, + }, + }); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('resolve object is not provided', () => { + it('should not redirect', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + renderHook(() => useResolveRedirect()); + expect(mockRedirectLegacyUrl).not.toHaveBeenCalled(); + }); + }); + + describe('outcome is exactMatch', () => { + it('should not redirect', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'exactMatch', + }, + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + renderHook(() => useResolveRedirect()); + expect(mockRedirectLegacyUrl).not.toHaveBeenCalled(); + }); + }); + + describe('outcome is aliasMatch', () => { + it('should redirect to url with id:new-id if outcome is aliasMatch', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'aliasMatch', + alias_target_id: 'new-id', + }, + })); + renderHook(() => useResolveRedirect()); + expect(mockRedirectLegacyUrl).toHaveBeenCalledWith( + 'my/cool/path?timeline=%28activeTab%3Aquery%2CgraphEventId%3A%27%27%2Cid%3Anew-id%2CisOpen%3A%21t%29', + 'timeline' + ); + }); + + describe('rison is unable to be decoded', () => { + it('should use timeline values from redux to create the redirect path', async () => { + jest.spyOn(urlHelpers, 'decodeRisonUrlState').mockImplementation(() => { + throw new Error('Unable to decode'); + }); + (useLocation as jest.Mock).mockReturnValue({ + pathname: 'my/cool/path', + search: '?foo=bar', + }); + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'aliasMatch', + alias_target_id: 'new-id', + }, + savedObjectId: 'current-saved-object-id', + activeTab: 'some-tab', + graphEventId: 'current-graph-event-id', + show: false, + })); + renderHook(() => useResolveRedirect()); + expect(mockRedirectLegacyUrl).toHaveBeenCalledWith( + 'my/cool/path?foo=bar&timeline=%28activeTab%3Asome-tab%2CgraphEventId%3Acurrent-graph-event-id%2Cid%3Anew-id%2CisOpen%3A%21f%29', + 'timeline' + ); + }); + }); + }); + + describe('outcome is conflict', () => { + it('should not redirect', async () => { + (useDeepEqualSelector as jest.Mock).mockImplementation(() => ({ + resolveTimelineConfig: { + outcome: 'conflict', + alias_target_id: 'new-id', + }, + })); + renderHook(() => useResolveRedirect()); + expect(mockRedirectLegacyUrl).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.ts b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.ts new file mode 100644 index 00000000000000..a6ba0b24828e78 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/use_resolve_redirect.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useLocation } from 'react-router-dom'; +import { useDeepEqualSelector } from './use_selector'; +import { TimelineId } from '../../../common/types/timeline'; +import { timelineSelectors } from '../../timelines/store/timeline/'; +import { timelineDefaults } from '../../timelines/store/timeline/defaults'; +import { decodeRisonUrlState, encodeRisonUrlState } from '../components/url_state/helpers'; +import { useKibana } from '../lib/kibana'; +import { TimelineUrl } from '../../timelines/store/timeline/model'; +import { CONSTANTS } from '../components/url_state/constants'; + +/** + * This hooks is specifically for use with the resolve api that was introduced as part of 7.16 + * If a deep link id has been migrated to a new id, this hook will cause a redirect to a url with + * the new ID. + */ + +export const useResolveRedirect = () => { + const { search, pathname } = useLocation(); + const [hasRedirected, updateHasRedirected] = useState(false); + const { spaces } = useKibana().services; + + const getTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); + const { resolveTimelineConfig, savedObjectId, show, activeTab, graphEventId } = + useDeepEqualSelector((state) => getTimeline(state, TimelineId.active) ?? timelineDefaults); + + const redirect = useCallback(() => { + const searchQuery = new URLSearchParams(search); + const timelineRison = searchQuery.get(CONSTANTS.timeline) ?? undefined; + + // Try to get state on URL, but default to what's in Redux in case of decodeRisonFailure + const currentTimelineState = { + id: savedObjectId ?? '', + isOpen: !!show, + activeTab, + graphEventId, + }; + let timelineSearch: TimelineUrl = currentTimelineState; + try { + timelineSearch = decodeRisonUrlState(timelineRison) ?? currentTimelineState; + } catch (error) { + // do nothing as it's already defaulted on line 77 + } + + if ( + hasRedirected || + !spaces || + resolveTimelineConfig?.outcome !== 'aliasMatch' || + resolveTimelineConfig?.alias_target_id == null + ) { + return null; + } + + // We found this object by a legacy URL alias from its old ID; redirect the user to the page with its new ID, preserving any URL hash + const newObjectId = resolveTimelineConfig?.alias_target_id ?? ''; // This is always defined if outcome === 'aliasMatch' + const newTimelineSearch = { + ...timelineSearch, + id: newObjectId, + }; + const newTimelineRison = encodeRisonUrlState(newTimelineSearch); + searchQuery.set(CONSTANTS.timeline, newTimelineRison); + const newPath = `${pathname}?${searchQuery.toString()}`; + spaces.ui.redirectLegacyUrl(newPath, CONSTANTS.timeline); + // Prevent the effect from being called again as the url change takes place in location rather than a true redirect + updateHasRedirected(true); + }, [ + activeTab, + graphEventId, + hasRedirected, + pathname, + resolveTimelineConfig?.outcome, + resolveTimelineConfig?.alias_target_id, + savedObjectId, + search, + show, + spaces, + ]); + + useEffect(() => { + redirect(); + }, [redirect]); +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/__mocks__/index.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/__mocks__/index.ts index f1e1c42539effd..2521d14481ca83 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/__mocks__/index.ts @@ -9,7 +9,7 @@ import { TimelineStatus, TimelineType } from '../../../../../common/types/timeli export const mockTimeline = { data: { - getOneTimeline: { + timeline: { savedObjectId: 'eb2781c0-1df5-11eb-8589-2f13958b79f7', columns: [ { @@ -163,6 +163,7 @@ export const mockTimeline = { version: 'WzQ4NSwxXQ==', __typename: 'TimelineResult', }, + outcome: 'exactMatch', }, loading: false, networkStatus: 7, @@ -171,7 +172,7 @@ export const mockTimeline = { export const mockTemplate = { data: { - getOneTimeline: { + timeline: { savedObjectId: '0c70a200-1de0-11eb-885c-6fc13fca1850', columns: [ { @@ -416,6 +417,7 @@ export const mockTemplate = { version: 'WzQwMywxXQ==', __typename: 'TimelineResult', }, + outcome: 'exactMatch', }, loading: false, networkStatus: 7, diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.test.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.test.ts index 5d52d2c8a4d48c..1b93f1556a95ca 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.test.ts @@ -50,7 +50,7 @@ import { mockTimeline as mockSelectedTimeline, mockTemplate as mockSelectedTemplate, } from './__mocks__'; -import { getTimeline } from '../../containers/api'; +import { resolveTimeline } from '../../containers/api'; import { defaultHeaders } from '../timeline/body/column_headers/default_headers'; jest.mock('../../../common/store/inputs/actions'); @@ -951,7 +951,7 @@ describe('helpers', () => { }; beforeAll(async () => { - (getTimeline as jest.Mock).mockRejectedValue(mockError); + (resolveTimeline as jest.Mock).mockRejectedValue(mockError); queryTimelineById<{}>(args as unknown as QueryTimelineById<{}>); }); @@ -986,7 +986,7 @@ describe('helpers', () => { }; beforeAll(async () => { - (getTimeline as jest.Mock).mockResolvedValue(selectedTimeline); + (resolveTimeline as jest.Mock).mockResolvedValue(selectedTimeline); await queryTimelineById<{}>(args as unknown as QueryTimelineById<{}>); }); @@ -1002,7 +1002,7 @@ describe('helpers', () => { }); test('get timeline by Id', () => { - expect(getTimeline).toHaveBeenCalled(); + expect(resolveTimeline).toHaveBeenCalled(); }); test('it does not call onError when an error does not occur', () => { @@ -1011,7 +1011,7 @@ describe('helpers', () => { test('Do not override daterange if TimelineStatus is active', () => { const { timeline } = formatTimelineResultToModel( - omitTypenameInTimeline(getOr({}, 'data.getOneTimeline', selectedTimeline)), + omitTypenameInTimeline(getOr({}, 'data.timeline', selectedTimeline)), args.duplicate, args.timelineType ); @@ -1044,7 +1044,7 @@ describe('helpers', () => { }; beforeAll(async () => { - (getTimeline as jest.Mock).mockResolvedValue(selectedTimeline); + (resolveTimeline as jest.Mock).mockResolvedValue(selectedTimeline); await queryTimelineById<{}>(args as unknown as QueryTimelineById<{}>); }); @@ -1060,12 +1060,12 @@ describe('helpers', () => { }); test('get timeline by Id', () => { - expect(getTimeline).toHaveBeenCalled(); + expect(resolveTimeline).toHaveBeenCalled(); }); test('should not override daterange if TimelineStatus is active', () => { const { timeline } = formatTimelineResultToModel( - omitTypenameInTimeline(getOr({}, 'data.getOneTimeline', selectedTimeline)), + omitTypenameInTimeline(getOr({}, 'data.timeline', selectedTimeline)), args.duplicate, args.timelineType ); @@ -1085,6 +1085,10 @@ describe('helpers', () => { to: '2020-07-08T08:20:18.966Z', notes: [], id: TimelineId.active, + resolveTimelineConfig: { + outcome: 'exactMatch', + alias_target_id: undefined, + }, }); }); @@ -1112,12 +1116,12 @@ describe('helpers', () => { }; beforeAll(async () => { - (getTimeline as jest.Mock).mockResolvedValue(template); + (resolveTimeline as jest.Mock).mockResolvedValue(template); await queryTimelineById<{}>(args as unknown as QueryTimelineById<{}>); }); afterAll(() => { - (getTimeline as jest.Mock).mockReset(); + (resolveTimeline as jest.Mock).mockReset(); jest.clearAllMocks(); }); @@ -1129,12 +1133,12 @@ describe('helpers', () => { }); test('get timeline by Id', () => { - expect(getTimeline).toHaveBeenCalled(); + expect(resolveTimeline).toHaveBeenCalled(); }); test('override daterange if TimelineStatus is immutable', () => { const { timeline } = formatTimelineResultToModel( - omitTypenameInTimeline(getOr({}, 'data.getOneTimeline', template)), + omitTypenameInTimeline(getOr({}, 'data.timeline', template)), args.duplicate, args.timelineType ); diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts index 1fbddf61f8cd3b..f325ab34e88d5a 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts @@ -20,6 +20,7 @@ import { TimelineType, TimelineTabs, TimelineResult, + SingleTimelineResolveResponse, ColumnHeaderResult, FilterTimelineResult, DataProviderResult, @@ -65,7 +66,7 @@ import { DEFAULT_FROM_MOMENT, DEFAULT_TO_MOMENT, } from '../../../common/utils/default_date_settings'; -import { getTimeline } from '../../containers/api'; +import { resolveTimeline } from '../../containers/api'; import { PinnedEvent } from '../../../../common/types/timeline/pinned_event'; import { NoteResult } from '../../../../common/types/timeline/note'; @@ -346,11 +347,12 @@ export const queryTimelineById = ({ updateTimeline, }: QueryTimelineById) => { updateIsLoading({ id: TimelineId.active, isLoading: true }); - Promise.resolve(getTimeline(timelineId)) + Promise.resolve(resolveTimeline(timelineId)) .then((result) => { - const timelineToOpen: TimelineResult = omitTypenameInTimeline( - getOr({}, 'data.getOneTimeline', result) - ); + const data: SingleTimelineResolveResponse['data'] | null = getOr(null, 'data', result); + if (!data) return; + + const timelineToOpen = omitTypenameInTimeline(data.timeline); const { timeline, notes } = formatTimelineResultToModel( timelineToOpen, @@ -370,6 +372,10 @@ export const queryTimelineById = ({ from, id: TimelineId.active, notes, + resolveTimelineConfig: { + outcome: data.outcome, + alias_target_id: data.alias_target_id, + }, timeline: { ...timeline, activeTab: activeTimelineTab, @@ -399,6 +405,7 @@ export const dispatchUpdateTimeline = forceNotes = false, from, notes, + resolveTimelineConfig, timeline, to, ruleNote, @@ -429,7 +436,9 @@ export const dispatchUpdateTimeline = } else { dispatch(dispatchSetTimelineRangeDatePicker({ from, to })); } - dispatch(dispatchAddTimeline({ id, timeline, savedTimeline: duplicate })); + dispatch( + dispatchAddTimeline({ id, timeline, resolveTimelineConfig, savedTimeline: duplicate }) + ); if ( timeline.kqlQuery != null && timeline.kqlQuery.filterQuery != null && diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/types.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/types.ts index 79a700856c00fb..4c9ce991252dc2 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/types.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/types.ts @@ -16,6 +16,7 @@ import { TemplateTimelineTypeLiteral, RowRendererId, TimelineStatusLiteralWithNull, + SingleTimelineResolveResponse, } from '../../../../common/types/timeline'; /** The users who added a timeline to favorites */ @@ -194,12 +195,17 @@ export interface OpenTimelineProps { hideActions?: ActionTimelineToShow[]; } +export interface ResolveTimelineConfig { + alias_target_id: SingleTimelineResolveResponse['data']['alias_target_id']; + outcome: SingleTimelineResolveResponse['data']['outcome']; +} export interface UpdateTimeline { duplicate: boolean; id: string; forceNotes?: boolean; from: string; notes: NoteResult[] | null | undefined; + resolveTimelineConfig?: ResolveTimelineConfig; timeline: TimelineModel; to: string; ruleNote?: string; @@ -210,6 +216,7 @@ export type DispatchUpdateTimeline = ({ id, from, notes, + resolveTimelineConfig, timeline, to, ruleNote, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx index db7a3cc3c9900b..c91673e5f931c6 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx @@ -40,6 +40,12 @@ const mockUseResizeObserver: jest.Mock = useResizeObserver as jest.Mock; jest.mock('use-resize-observer/polyfilled'); mockUseResizeObserver.mockImplementation(() => ({})); +jest.mock('../../../common/hooks/use_resolve_conflict', () => { + return { + useResolveConflict: jest.fn().mockImplementation(() => null), + }; +}); + jest.mock('react-router-dom', () => { const original = jest.requireActual('react-router-dom'); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx index 3ac92194766b1b..efd1977fd351f3 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx @@ -28,6 +28,7 @@ import { TabsContent } from './tabs_content'; import { HideShowContainer, TimelineContainer } from './styles'; import { useTimelineFullScreen } from '../../../common/containers/use_full_screen'; import { EXIT_FULL_SCREEN_CLASS_NAME } from '../../../common/components/exit_full_screen'; +import { useResolveConflict } from '../../../common/hooks/use_resolve_conflict'; const TimelineTemplateBadge = styled.div` background: ${({ theme }) => theme.eui.euiColorVis3_behindText}; @@ -119,6 +120,7 @@ const StatefulTimelineComponent: React.FC = ({ [containerElement, onSkipFocusBeforeEventsTable, onSkipFocusAfterEventsTable] ); const timelineContext = useMemo(() => ({ timelineId }), [timelineId]); + const resolveConflictComponent = useResolveConflict(); return ( @@ -132,7 +134,7 @@ const StatefulTimelineComponent: React.FC = ({ {timelineType === TimelineType.template && ( {i18n.TIMELINE_TEMPLATE} )} - + {resolveConflictComponent} { return decodeSingleTimelineResponse(response); }; -export const getResolvedTimelineTemplate = async (templateTimelineId: string) => { - const response = await KibanaServices.get().http.get( - TIMELINE_RESOLVE_URL, - { - query: { - template_timeline_id: templateTimelineId, - }, - } - ); - - return decodeResolvedSingleTimelineResponse(response); -}; - export const getAllTimelines = async (args: GetTimelinesArgs, abortSignal: AbortSignal) => { const response = await KibanaServices.get().http.fetch(TIMELINES_URL, { method: 'GET', diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/actions.ts b/x-pack/plugins/security_solution/public/timelines/store/timeline/actions.ts index 95ad6c5d44ca35..f3a70bd1390ae3 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/actions.ts +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/actions.ts @@ -25,6 +25,7 @@ import type { SerializedFilterQuery, } from '../../../../common/types/timeline'; import { tGridActions } from '../../../../../timelines/public'; +import { ResolveTimelineConfig } from '../../components/open_timeline/types'; export const { applyDeltaToColumnWidth, clearEventsDeleted, @@ -91,6 +92,7 @@ export const updateTimeline = actionCreator<{ export const addTimeline = actionCreator<{ id: string; timeline: TimelineModel; + resolveTimelineConfig?: ResolveTimelineConfig; savedTimeline?: boolean; }>('ADD_TIMELINE'); diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/defaults.ts b/x-pack/plugins/security_solution/public/timelines/store/timeline/defaults.ts index 4691872bfb927b..4c2b8d2992d3d6 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/defaults.ts +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/defaults.ts @@ -14,64 +14,65 @@ import { SubsetTimelineModel, TimelineModel } from './model'; // normalizeTimeRange uses getTimeRangeSettings which cannot be used outside Kibana context if the uiSettings is not false const { from: start, to: end } = normalizeTimeRange({ from: '', to: '' }, false); -export const timelineDefaults: SubsetTimelineModel & Pick = - { - activeTab: TimelineTabs.query, - prevActiveTab: TimelineTabs.query, - columns: defaultHeaders, - documentType: '', - defaultColumns: defaultHeaders, - dataProviders: [], - dateRange: { start, end }, - deletedEventIds: [], - description: '', - eqlOptions: { - eventCategoryField: 'event.category', - tiebreakerField: '', - timestampField: '@timestamp', - query: '', - size: 100, +export const timelineDefaults: SubsetTimelineModel & + Pick = { + activeTab: TimelineTabs.query, + prevActiveTab: TimelineTabs.query, + columns: defaultHeaders, + documentType: '', + defaultColumns: defaultHeaders, + dataProviders: [], + dateRange: { start, end }, + deletedEventIds: [], + description: '', + eqlOptions: { + eventCategoryField: 'event.category', + tiebreakerField: '', + timestampField: '@timestamp', + query: '', + size: 100, + }, + eventType: 'all', + eventIdToNoteIds: {}, + excludedRowRendererIds: [], + expandedDetail: {}, + highlightedDropAndProviderId: '', + historyIds: [], + filters: [], + indexNames: [], + isFavorite: false, + isLive: false, + isSelectAllChecked: false, + isLoading: false, + isSaving: false, + itemsPerPage: 25, + itemsPerPageOptions: [10, 25, 50, 100], + kqlMode: 'filter', + kqlQuery: { + filterQuery: null, + }, + loadingEventIds: [], + resolveTimelineConfig: undefined, + queryFields: [], + title: '', + timelineType: TimelineType.default, + templateTimelineId: null, + templateTimelineVersion: null, + noteIds: [], + pinnedEventIds: {}, + pinnedEventsSaveObject: {}, + savedObjectId: null, + selectAll: false, + selectedEventIds: {}, + show: false, + showCheckboxes: false, + sort: [ + { + columnId: '@timestamp', + columnType: 'number', + sortDirection: 'desc', }, - eventType: 'all', - eventIdToNoteIds: {}, - excludedRowRendererIds: [], - expandedDetail: {}, - highlightedDropAndProviderId: '', - historyIds: [], - filters: [], - indexNames: [], - isFavorite: false, - isLive: false, - isSelectAllChecked: false, - isLoading: false, - isSaving: false, - itemsPerPage: 25, - itemsPerPageOptions: [10, 25, 50, 100], - kqlMode: 'filter', - kqlQuery: { - filterQuery: null, - }, - loadingEventIds: [], - queryFields: [], - title: '', - timelineType: TimelineType.default, - templateTimelineId: null, - templateTimelineVersion: null, - noteIds: [], - pinnedEventIds: {}, - pinnedEventsSaveObject: {}, - savedObjectId: null, - selectAll: false, - selectedEventIds: {}, - show: false, - showCheckboxes: false, - sort: [ - { - columnId: '@timestamp', - columnType: 'number', - sortDirection: 'desc', - }, - ], - status: TimelineStatus.draft, - version: null, - }; + ], + status: TimelineStatus.draft, + version: null, +}; diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/helpers.ts b/x-pack/plugins/security_solution/public/timelines/store/timeline/helpers.ts index 6ee844958aeed6..b7af561ae2a04b 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/helpers.ts +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/helpers.ts @@ -47,6 +47,7 @@ import { RESIZED_COLUMN_MIN_WITH, } from '../../components/timeline/body/constants'; import { activeTimeline } from '../../containers/active_timeline_context'; +import { ResolveTimelineConfig } from '../../components/open_timeline/types'; export const isNotNull = (value: T | null): value is T => value !== null; @@ -124,6 +125,7 @@ export const addTimelineNoteToEvent = ({ interface AddTimelineParams { id: string; + resolveTimelineConfig?: ResolveTimelineConfig; timeline: TimelineModel; timelineById: TimelineById; } @@ -145,6 +147,7 @@ export const shouldResetActiveTimelineContext = ( */ export const addTimelineToStore = ({ id, + resolveTimelineConfig, timeline, timelineById, }: AddTimelineParams): TimelineById => { @@ -159,6 +162,7 @@ export const addTimelineToStore = ({ filterManager: timelineById[id].filterManager, isLoading: timelineById[id].isLoading, initialized: timelineById[id].initialized, + resolveTimelineConfig, dateRange: timeline.status === TimelineStatus.immutable && timeline.timelineType === TimelineType.template diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/model.ts b/x-pack/plugins/security_solution/public/timelines/store/timeline/model.ts index b53da997c08cb3..29b49197ef7978 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/model.ts +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/model.ts @@ -15,6 +15,7 @@ import type { } from '../../../../common/types/timeline'; import { PinnedEvent } from '../../../../common/types/timeline/pinned_event'; import type { TGridModelForTimeline } from '../../../../../timelines/public'; +import { ResolveTimelineConfig } from '../../components/open_timeline/types'; export const DEFAULT_PAGE_COUNT = 2; // Eui Pager will not render unless this is a minimum of 2 pages export type KqlMode = 'filter' | 'search'; @@ -59,6 +60,7 @@ export type TimelineModel = TGridModelForTimeline & { /** Events pinned to this timeline */ pinnedEventIds: Record; pinnedEventsSaveObject: Record; + resolveTimelineConfig?: ResolveTimelineConfig; showSaveModal?: boolean; savedQueryId?: string | null; /** When true, show the timeline flyover */ diff --git a/x-pack/plugins/security_solution/public/timelines/store/timeline/reducer.ts b/x-pack/plugins/security_solution/public/timelines/store/timeline/reducer.ts index 97fa72667a3c6f..e997bbd848d506 100644 --- a/x-pack/plugins/security_solution/public/timelines/store/timeline/reducer.ts +++ b/x-pack/plugins/security_solution/public/timelines/store/timeline/reducer.ts @@ -94,9 +94,14 @@ export const initialTimelineState: TimelineState = { /** The reducer for all timeline actions */ export const timelineReducer = reducerWithInitialState(initialTimelineState) - .case(addTimeline, (state, { id, timeline }) => ({ + .case(addTimeline, (state, { id, timeline, resolveTimelineConfig }) => ({ ...state, - timelineById: addTimelineToStore({ id, timeline, timelineById: state.timelineById }), + timelineById: addTimelineToStore({ + id, + timeline, + resolveTimelineConfig, + timelineById: state.timelineById, + }), })) .case(createTimeline, (state, { id, timelineType = TimelineType.default, ...timelineProps }) => { return { diff --git a/x-pack/plugins/security_solution/public/types.ts b/x-pack/plugins/security_solution/public/types.ts index bee0e9b3a3d1d4..61813d1a122b4e 100644 --- a/x-pack/plugins/security_solution/public/types.ts +++ b/x-pack/plugins/security_solution/public/types.ts @@ -9,7 +9,6 @@ import { CoreStart } from '../../../../src/core/public'; import { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; import { DataPublicPluginStart } from '../../../../src/plugins/data/public'; import { EmbeddableStart } from '../../../../src/plugins/embeddable/public'; -import { SpacesPluginStart } from '../../../plugins/spaces/public'; import { LensPublicStart } from '../../../plugins/lens/public'; import { NewsfeedPublicPluginStart } from '../../../../src/plugins/newsfeed/public'; import { Start as InspectorStart } from '../../../../src/plugins/inspector/public'; @@ -18,6 +17,7 @@ import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/p import { Storage } from '../../../../src/plugins/kibana_utils/public'; import { FleetStart } from '../../fleet/public'; import { PluginStart as ListsPluginStart } from '../../lists/public'; +import { SpacesPluginStart } from '../../spaces/public'; import { TriggersAndActionsUIPublicPluginSetup as TriggersActionsSetup, TriggersAndActionsUIPublicPluginStart as TriggersActionsStart, diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 5c87d58199df4e..8f4d602e26461a 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -1,4 +1,3 @@ - { "extends": "../../../tsconfig.base.json", "compilerOptions": { @@ -40,7 +39,7 @@ { "path": "../maps/tsconfig.json" }, { "path": "../ml/tsconfig.json" }, { "path": "../spaces/tsconfig.json" }, - { "path": "../security/tsconfig.json"}, - { "path": "../timelines/tsconfig.json"}, + { "path": "../security/tsconfig.json" }, + { "path": "../timelines/tsconfig.json" } ] } From 4f0614f9bad9de79f21255b7ebb6ec99ed51c772 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:14:37 -0400 Subject: [PATCH 09/35] [Security Solution] Analyze event moved outside of overflow popover (#115478) (#115555) Co-authored-by: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> --- .../timeline_actions/alert_context_menu.tsx | 16 +----- .../timeline/body/actions/index.test.tsx | 29 ++++++++++ .../timeline/body/actions/index.tsx | 54 ++++++++++++++++++- .../__snapshots__/index.test.tsx.snap | 2 +- .../timeline/body/control_columns/index.tsx | 2 +- .../components/timeline/body/helpers.tsx | 6 ++- 6 files changed, 90 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx index 06d61b3f0b2847..a9b6eabecff864 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx @@ -34,7 +34,6 @@ import { useExceptionActions } from './use_add_exception_actions'; import { useEventFilterModal } from './use_event_filter_modal'; import { Status } from '../../../../../common/detection_engine/schemas/common/schemas'; import { useKibana } from '../../../../common/lib/kibana'; -import { useInvestigateInResolverContextItem } from './investigate_in_resolver'; import { ATTACH_ALERT_TO_CASE_FOR_ROW } from '../../../../timelines/components/timeline/body/translations'; import { useEventFilterAction } from './use_event_filter_action'; import { useAddToCaseActions } from './use_add_to_case_actions'; @@ -163,30 +162,19 @@ const AlertContextMenuComponent: React.FC !isEvent && ruleId - ? [ - ...investigateInResolverActionItems, - ...addToCaseActionItems, - ...statusActionItems, - ...exceptionActionItems, - ] - : [...investigateInResolverActionItems, ...addToCaseActionItems, ...eventFilterActionItems], + ? [...addToCaseActionItems, ...statusActionItems, ...exceptionActionItems] + : [...addToCaseActionItems, ...eventFilterActionItems], [ statusActionItems, addToCaseActionItems, eventFilterActionItems, exceptionActionItems, - investigateInResolverActionItems, isEvent, ruleId, ] diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx index 5ed9398a621e8e..1da09bcf4e25fe 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.test.tsx @@ -185,5 +185,34 @@ describe('Actions', () => { wrapper.find('[data-test-subj="timeline-context-menu-button"]').first().prop('isDisabled') ).toBe(false); }); + test('it shows the analyze event button when the event is from an endpoint', () => { + const ecsData = { + ...mockTimelineData[0].ecs, + event: { kind: ['alert'] }, + agent: { type: ['endpoint'] }, + process: { entity_id: ['1'] }, + }; + const wrapper = mount( + + + + ); + + expect(wrapper.find('[data-test-subj="view-in-analyzer"]').exists()).toBe(true); + }); + test('it does not render the analyze event button when the event is from an unsupported source', () => { + const ecsData = { + ...mockTimelineData[0].ecs, + event: { kind: ['alert'] }, + agent: { type: ['notendpoint'] }, + }; + const wrapper = mount( + + + + ); + + expect(wrapper.find('[data-test-subj="view-in-analyzer"]').exists()).toBe(false); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx index 73650bd320f326..c4dae739cb251e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx @@ -21,9 +21,23 @@ import { EventsTdContent } from '../../styles'; import * as i18n from '../translations'; import { DEFAULT_ICON_BUTTON_WIDTH } from '../../helpers'; import { useShallowEqualSelector } from '../../../../../common/hooks/use_selector'; -import { TimelineId, ActionProps, OnPinEvent } from '../../../../../../common/types/timeline'; +import { + setActiveTabTimeline, + updateTimelineGraphEventId, +} from '../../../../store/timeline/actions'; +import { + useGlobalFullScreen, + useTimelineFullScreen, +} from '../../../../../common/containers/use_full_screen'; +import { + TimelineId, + ActionProps, + OnPinEvent, + TimelineTabs, +} from '../../../../../../common/types/timeline'; import { timelineActions, timelineSelectors } from '../../../../store/timeline'; import { timelineDefaults } from '../../../../store/timeline/defaults'; +import { isInvestigateInResolverActionEnabled } from '../../../../../detections/components/alerts_table/timeline_actions/investigate_in_resolver'; const ActionsContainer = styled.div` align-items: center; @@ -100,6 +114,24 @@ const ActionsComponent: React.FC = ({ [eventType, ecsData.event?.kind, ecsData.agent?.type] ); + const isDisabled = useMemo(() => !isInvestigateInResolverActionEnabled(ecsData), [ecsData]); + const { setGlobalFullScreen } = useGlobalFullScreen(); + const { setTimelineFullScreen } = useTimelineFullScreen(); + const handleClick = useCallback(() => { + const dataGridIsFullScreen = document.querySelector('.euiDataGrid--fullScreen'); + dispatch(updateTimelineGraphEventId({ id: timelineId, graphEventId: ecsData._id })); + if (timelineId === TimelineId.active) { + if (dataGridIsFullScreen) { + setTimelineFullScreen(true); + } + dispatch(setActiveTabTimeline({ id: timelineId, activeTab: TimelineTabs.graph })); + } else { + if (dataGridIsFullScreen) { + setGlobalFullScreen(true); + } + } + }, [dispatch, ecsData._id, timelineId, setGlobalFullScreen, setTimelineFullScreen]); + return ( {showCheckboxes && !tGridEnabled && ( @@ -171,6 +203,26 @@ const ActionsComponent: React.FC = ({ refetch={refetch ?? noop} onRuleChange={onRuleChange} /> + {isDisabled === false ? ( +
+ + + + + +
+ ) : null} ); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap index 6bc2dc089494dd..25d5104a98d955 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap @@ -521,7 +521,7 @@ exports[`ColumnHeaders rendering renders correctly against snapshot 1`] = ` "compare": null, "type": [Function], }, - "width": 108, + "width": 140, }, ] } diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/control_columns/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/control_columns/index.tsx index d38bf2136513e4..2cdc8d5f4e284e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/control_columns/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/control_columns/index.tsx @@ -9,7 +9,7 @@ import { ControlColumnProps } from '../../../../../../common/types/timeline'; import { Actions } from '../actions'; import { HeaderActions } from '../actions/header_actions'; -const DEFAULT_CONTROL_COLUMN_WIDTH = 108; +const DEFAULT_CONTROL_COLUMN_WIDTH = 140; export const defaultControlColumn: ControlColumnProps = { id: 'default-timeline-control-column', diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/helpers.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/helpers.tsx index 5b993110d38b58..7032319b593330 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/helpers.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/helpers.tsx @@ -17,8 +17,10 @@ import { import { OnPinEvent, OnUnPinEvent } from '../events'; import * as i18n from './translations'; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const omitTypenameAndEmpty = (k: string, v: any): any | undefined => +export const omitTypenameAndEmpty = ( + k: string, + v: string | object | Array +): string | object | Array | undefined => k !== '__typename' && v != null ? v : undefined; export const stringifyEvent = (ecs: Ecs): string => JSON.stringify(ecs, omitTypenameAndEmpty, 2); From d71a2500a5632891047f469a2485eca2ce0fe866 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:15:18 -0400 Subject: [PATCH 10/35] [ES client] Rename deprecated params (#115528) (#115789) * filterPath --> filter_path * ignoreUnavailable --> ignore_unavailable * ignoreUnavailable --> ignore_unavailable * trackScores --> track_scores * trackTotalHits --> track_total_hits * rollback unnecessary changes Co-authored-by: Mikhail Shustov --- .../server/fetcher/lib/es_api.test.js | 4 +-- .../get_saved_object_counts.test.ts | 12 +++---- .../get_saved_object_counts.ts | 4 +-- .../collectors/custom_element_collector.ts | 4 +-- .../server/collectors/workpad_collector.ts | 4 +-- .../log_entries/kibana_log_entries_adapter.ts | 8 ++--- .../adapters/metrics/lib/check_valid_node.ts | 4 +-- .../elasticsearch_source_status_adapter.ts | 4 +-- .../queries/log_entry_datasets.ts | 8 ++--- .../server/lib/infra_ml/queries/common.ts | 8 ++--- .../queries/metrics_host_anomalies.test.ts | 8 ++--- .../queries/metrics_k8s_anomalies.test.ts | 8 ++--- .../server/lib/log_analysis/queries/common.ts | 8 ++--- .../plugins/infra/server/lib/metrics/index.ts | 4 +-- .../infra/server/lib/sources/has_data.ts | 4 +-- .../lib/get_cloud_metadata.ts | 4 +-- .../metadata/lib/get_cloud_metric_metadata.ts | 4 +-- .../metadata/lib/get_metric_metadata.ts | 4 +-- .../routes/metadata/lib/get_node_info.ts | 4 +-- .../routes/metadata/lib/get_pod_node_name.ts | 4 +-- .../lib/get_dataset_for_field.ts | 4 +-- .../lib/query_total_groupings.ts | 4 +-- .../server/utils/calculate_metric_interval.ts | 4 +-- .../lib/elasticsearch/get_last_recovery.ts | 2 +- .../actions/all/query.all_actions.dsl.ts | 4 +-- .../details/query.action_details.dsl.ts | 4 +-- .../results/query.action_results.dsl.ts | 4 +-- .../factory/agents/query.all_agents.dsl.ts | 4 +-- .../factory/results/query.all_results.dsl.ts | 4 +-- .../server/usage/get_reporting_usage.ts | 2 +- .../rollup/server/collectors/helpers.ts | 4 +-- .../factory/cti/event_enrichment/query.ts | 4 +-- .../cti/event_enrichment/response.test.ts | 4 +-- .../factory/hosts/all/__mocks__/index.ts | 8 ++--- .../factory/hosts/all/query.all_hosts.dsl.ts | 4 +-- .../hosts/all/query.all_hosts_entities.dsl.ts | 4 +-- .../hosts/authentications/__mocks__/index.ts | 8 ++--- .../hosts/authentications/dsl/query.dsl.ts | 4 +-- .../authentications/dsl/query_entities.dsl.ts | 4 +-- .../factory/hosts/details/__mocks__/index.ts | 8 ++--- .../hosts/details/query.host_details.dsl.ts | 4 +-- .../query.hosts_kpi_authentications.dsl.ts | 4 +-- ....hosts_kpi_authentications_entities.dsl.ts | 4 +-- .../kpi/hosts/query.hosts_kpi_hosts.dsl.ts | 4 +-- .../query.hosts_kpi_hosts_entities.dsl.ts | 4 +-- .../query.hosts_kpi_unique_ips.dsl.ts | 4 +-- ...query.hosts_kpi_unique_ips_entities.dsl.ts | 4 +-- .../hosts/last_first_seen/__mocks__/index.ts | 12 +++---- .../query.first_or_last_seen_host.dsl.ts | 4 +-- .../factory/hosts/overview/__mocks__/index.ts | 8 ++--- .../hosts/overview/query.overview_host.dsl.ts | 4 +-- .../hosts/risk_score/query.hosts_risk.dsl.ts | 4 +-- .../uncommon_processes/__mocks__/index.ts | 8 ++--- .../hosts/uncommon_processes/dsl/query.dsl.ts | 4 +-- .../matrix_histogram/__mocks__/index.ts | 24 +++++++------- .../alerts/__mocks__/index.ts | 4 +-- .../alerts/query.alerts_histogram.dsl.ts | 4 +-- .../anomalies/__mocks__/index.ts | 4 +-- .../query.anomalies_histogram.dsl.ts | 4 +-- .../authentications/__mocks__/index.ts | 4 +-- .../query.authentications_histogram.dsl.ts | 4 +-- ....authentications_histogram_entities.dsl.ts | 4 +-- .../matrix_histogram/dns/__mocks__/index.ts | 4 +-- .../dns/query.dns_histogram.dsl.ts | 4 +-- .../events/__mocks__/index.ts | 32 +++++++++---------- .../events/query.events_histogram.dsl.ts | 4 +-- .../network/details/__mocks__/index.ts | 8 ++--- .../details/query.details_network.dsl.ts | 4 +-- .../factory/network/dns/__mocks__/index.ts | 8 ++--- .../network/dns/query.dns_network.dsl.ts | 4 +-- .../factory/network/http/__mocks__/index.ts | 8 ++--- .../network/http/query.http_network.dsl.ts | 4 +-- .../dns/query.network_kip_dns_entities.dsl.ts | 4 +-- .../kpi/dns/query.network_kpi_dns.dsl.ts | 4 +-- .../query.network_kpi_network_events.dsl.ts | 4 +-- ...network_kpi_network_events_entities.dsl.ts | 4 +-- .../query.network_kpi_tls_handshakes.dsl.ts | 4 +-- ...network_kpi_tls_handshakes_entities.dsl.ts | 4 +-- .../query.network_kpi_unique_flows.dsl.ts | 4 +-- ...uery.network_kpi_unique_private_ips.dsl.ts | 4 +-- ...ork_kpi_unique_private_ips_entities.dsl.ts | 4 +-- .../network/overview/__mocks__/index.ts | 8 ++--- .../overview/query.overview_network.dsl.ts | 4 +-- .../factory/network/tls/__mocks__/index.ts | 8 ++--- .../network/tls/query.tls_network.dsl.ts | 4 +-- .../network/top_countries/__mocks__/index.ts | 8 ++--- .../query.top_countries_network.dsl.ts | 4 +-- ...uery.top_countries_network_entities.dsl.ts | 4 +-- .../network/top_n_flow/__mocks__/index.ts | 8 ++--- .../query.top_n_flow_network.dsl.ts | 4 +-- .../query.top_n_flow_network_entities.dsl.ts | 4 +-- .../factory/network/users/__mocks__/index.ts | 8 ++--- .../network/users/query.users_network.dsl.ts | 4 +-- .../ueba/host_rules/query.host_rules.dsl.ts | 4 +-- .../host_tactics/query.host_tactics.dsl.ts | 4 +-- .../ueba/risk_score/query.risk_score.dsl.ts | 4 +-- .../ueba/user_rules/query.user_rules.dsl.ts | 4 +-- .../detections/detection_rule_helpers.ts | 4 +-- .../server/usage/detections/types.ts | 4 +-- .../events/all/query.events_all.dsl.ts | 4 +-- .../details/query.events_details.dsl.test.ts | 4 +-- .../details/query.events_details.dsl.ts | 4 +-- .../factory/events/kpi/query.kpi.dsl.ts | 4 +-- .../query.events_last_event_time.dsl.ts | 12 +++---- 104 files changed, 280 insertions(+), 280 deletions(-) diff --git a/src/plugins/data_views/server/fetcher/lib/es_api.test.js b/src/plugins/data_views/server/fetcher/lib/es_api.test.js index 2893fde671bb2c..085add965b921f 100644 --- a/src/plugins/data_views/server/fetcher/lib/es_api.test.js +++ b/src/plugins/data_views/server/fetcher/lib/es_api.test.js @@ -61,7 +61,7 @@ describe('server/index_patterns/service/lib/es_api', () => { expect(resp).toBe(football); }); - it('sets ignoreUnavailable and allowNoIndices params', async () => { + it('sets ignore_unavailable and allow_no_indices params', async () => { const getAlias = sinon.stub(); const callCluster = { indices: { @@ -149,7 +149,7 @@ describe('server/index_patterns/service/lib/es_api', () => { expect(resp).toBe(football); }); - it('sets ignoreUnavailable, allowNoIndices, and fields params', async () => { + it('sets ignore_unavailable, allow_no_indices, and fields params', async () => { const fieldCaps = sinon.stub(); const callCluster = { indices: { diff --git a/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.test.ts b/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.test.ts index 94c618354fc37c..4b3421efed96ee 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.test.ts @@ -26,8 +26,8 @@ describe('getSavedObjectsCounts', () => { expect(results).toStrictEqual([]); expect(esClient.search).toHaveBeenCalledWith({ index: '.kibana', - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', + ignore_unavailable: true, + filter_path: 'aggregations.types.buckets', body: { size: 0, query: { match_all: {} }, @@ -41,8 +41,8 @@ describe('getSavedObjectsCounts', () => { await getSavedObjectsCounts(esClient, '.kibana'); expect(esClient.search).toHaveBeenCalledWith({ index: '.kibana', - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', + ignore_unavailable: true, + filter_path: 'aggregations.types.buckets', body: { size: 0, query: { match_all: {} }, @@ -56,8 +56,8 @@ describe('getSavedObjectsCounts', () => { await getSavedObjectsCounts(esClient, '.kibana', ['type_one', 'type_two']); expect(esClient.search).toHaveBeenCalledWith({ index: '.kibana', - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', + ignore_unavailable: true, + filter_path: 'aggregations.types.buckets', body: { size: 0, query: { terms: { type: ['type_one', 'type_two'] } }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.ts b/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.ts index eeaeed67e753f3..f82a803b763c59 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts.ts @@ -17,8 +17,8 @@ export async function getSavedObjectsCounts( const savedObjectCountSearchParams = { index: kibanaIndex, - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', + ignore_unavailable: true, + filter_path: 'aggregations.types.buckets', body: { size: 0, query, diff --git a/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts b/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts index 18cfe1a3df56c5..b0e219a4d886d7 100644 --- a/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts +++ b/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts @@ -147,8 +147,8 @@ const customElementCollector: TelemetryCollector = async function customElementC const customElementParams = { size: 10000, index: kibanaIndex, - ignoreUnavailable: true, - filterPath: [`hits.hits._source.${CUSTOM_ELEMENT_TYPE}.content`], + ignore_unavailable: true, + filter_path: [`hits.hits._source.${CUSTOM_ELEMENT_TYPE}.content`], body: { query: { bool: { filter: { term: { type: CUSTOM_ELEMENT_TYPE } } } } }, }; diff --git a/x-pack/plugins/canvas/server/collectors/workpad_collector.ts b/x-pack/plugins/canvas/server/collectors/workpad_collector.ts index d42b8480ed7c75..62b0ce200e320d 100644 --- a/x-pack/plugins/canvas/server/collectors/workpad_collector.ts +++ b/x-pack/plugins/canvas/server/collectors/workpad_collector.ts @@ -381,8 +381,8 @@ const workpadCollector: TelemetryCollector = async function (kibanaIndex, esClie const searchParams = { size: 10000, // elasticsearch index.max_result_window default value index: kibanaIndex, - ignoreUnavailable: true, - filterPath: ['hits.hits._source.canvas-workpad', '-hits.hits._source.canvas-workpad.assets'], + ignore_unavailable: true, + filter_path: ['hits.hits._source.canvas-workpad', '-hits.hits._source.canvas-workpad.assets'], body: { query: { bool: { filter: { term: { type: CANVAS_TYPE } } } } }, }; diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index ab98de7901a3d1..524658559eadff 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -69,9 +69,9 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { }; const esQuery = { - allowNoIndices: true, + allow_no_indices: true, index: resolvedLogSourceConfiguration.indices, - ignoreUnavailable: true, + ignore_unavailable: true, body: { size: size + 1, // Extra one to test if it has more before or after track_total_hits: false, @@ -139,9 +139,9 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { ); const query = { - allowNoIndices: true, + allow_no_indices: true, index: resolvedLogSourceConfiguration.indices, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { count_by_date: { diff --git a/x-pack/plugins/infra/server/lib/adapters/metrics/lib/check_valid_node.ts b/x-pack/plugins/infra/server/lib/adapters/metrics/lib/check_valid_node.ts index 30f77dd096d8d7..31cc16a3f9eb60 100644 --- a/x-pack/plugins/infra/server/lib/adapters/metrics/lib/check_valid_node.ts +++ b/x-pack/plugins/infra/server/lib/adapters/metrics/lib/check_valid_node.ts @@ -14,8 +14,8 @@ export const checkValidNode = async ( id: string ): Promise => { const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: indexPattern, terminateAfter: 1, body: { diff --git a/x-pack/plugins/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts index 0aa305a580ffa2..9c15d4746cdc54 100644 --- a/x-pack/plugins/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/source_status/elasticsearch_source_status_adapter.ts @@ -18,13 +18,13 @@ export class InfraElasticsearchSourceStatusAdapter implements InfraSourceStatusA this.framework .callWithRequest(requestContext, 'indices.getAlias', { name: aliasName, - filterPath: '*.settings.index.uuid', // to keep the response size as small as possible + filter_path: '*.settings.index.uuid', // to keep the response size as small as possible }) .catch(withDefaultIfNotFound({})), this.framework .callWithRequest(requestContext, 'indices.get', { index: aliasName, - filterPath: '*.settings.index.uuid', // to keep the response size as small as possible + filter_path: '*.settings.index.uuid', // to keep the response size as small as possible }) .catch(withDefaultIfNotFound({})), ]); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/queries/log_entry_datasets.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/queries/log_entry_datasets.ts index 3431f3bfb0c8c7..9eae8daa3e74f2 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/queries/log_entry_datasets.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/queries/log_entry_datasets.ts @@ -65,10 +65,10 @@ export const createLogEntryDatasetsQuery = ( }); const defaultRequestParameters = { - allowNoIndices: true, - ignoreUnavailable: true, - trackScores: false, - trackTotalHits: false, + allow_no_indices: true, + ignore_unavailable: true, + track_scores: false, + track_total_hits: false, }; const compositeDatasetKeyRT = rt.type({ diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts index f4088c56303e23..594dc3371f3a28 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/common.ts @@ -6,10 +6,10 @@ */ export const defaultRequestParameters = { - allowNoIndices: true, - ignoreUnavailable: true, - trackScores: false, - trackTotalHits: false, + allow_no_indices: true, + ignore_unavailable: true, + track_scores: false, + track_total_hits: false, }; export const createJobIdFilters = (jobId: string) => [ diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_host_anomalies.test.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_host_anomalies.test.ts index 7c281963e5717c..5e7cb78790de0b 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_host_anomalies.test.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_host_anomalies.test.ts @@ -27,10 +27,10 @@ describe('createMetricsHostAnomaliesQuery', () => { pagination, }) ).toMatchObject({ - allowNoIndices: true, - ignoreUnavailable: true, - trackScores: false, - trackTotalHits: false, + allow_no_indices: true, + ignore_unavailable: true, + track_scores: false, + track_total_hits: false, body: { query: { bool: { diff --git a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.test.ts b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.test.ts index 61efb896a6c891..959b20a4825441 100644 --- a/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.test.ts +++ b/x-pack/plugins/infra/server/lib/infra_ml/queries/metrics_k8s_anomalies.test.ts @@ -27,10 +27,10 @@ describe('createMetricsK8sAnomaliesQuery', () => { pagination, }) ).toMatchObject({ - allowNoIndices: true, - ignoreUnavailable: true, - trackScores: false, - trackTotalHits: false, + allow_no_indices: true, + ignore_unavailable: true, + track_scores: false, + track_total_hits: false, body: { query: { bool: { diff --git a/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts b/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts index 4c3ba5612c443a..b05e4ce00fe9c0 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts @@ -6,10 +6,10 @@ */ export const defaultRequestParameters = { - allowNoIndices: true, - ignoreUnavailable: true, - trackScores: false, - trackTotalHits: false, + allow_no_indices: true, + ignore_unavailable: true, + track_scores: false, + track_total_hits: false, }; export const createJobIdFilters = (jobId: string) => [ diff --git a/x-pack/plugins/infra/server/lib/metrics/index.ts b/x-pack/plugins/infra/server/lib/metrics/index.ts index 131f6944b04844..d291dbf88b49a7 100644 --- a/x-pack/plugins/infra/server/lib/metrics/index.ts +++ b/x-pack/plugins/infra/server/lib/metrics/index.ts @@ -47,8 +47,8 @@ export const query = async ( ]; const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: options.indexPattern, body: { size: 0, diff --git a/x-pack/plugins/infra/server/lib/sources/has_data.ts b/x-pack/plugins/infra/server/lib/sources/has_data.ts index b036124344ea90..d56512918f11ae 100644 --- a/x-pack/plugins/infra/server/lib/sources/has_data.ts +++ b/x-pack/plugins/infra/server/lib/sources/has_data.ts @@ -10,9 +10,9 @@ import { ESSearchClient } from '../metrics/types'; export const hasData = async (index: string, client: ESSearchClient) => { const params = { index, - allowNoIndices: true, + allow_no_indices: true, terminate_after: 1, - ignoreUnavailable: true, + ignore_unavailable: true, body: { size: 0, }, diff --git a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts index 57f99aee6d1874..c721ca75ea9788 100644 --- a/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts +++ b/x-pack/plugins/infra/server/routes/inventory_metadata/lib/get_cloud_metadata.ts @@ -40,8 +40,8 @@ export const getCloudMetadata = async ( } const metricQuery = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: sourceConfiguration.metricAlias, body: { query: { diff --git a/x-pack/plugins/infra/server/routes/metadata/lib/get_cloud_metric_metadata.ts b/x-pack/plugins/infra/server/routes/metadata/lib/get_cloud_metric_metadata.ts index be2a39b84ef1c6..d9da7bce2246fe 100644 --- a/x-pack/plugins/infra/server/routes/metadata/lib/get_cloud_metric_metadata.ts +++ b/x-pack/plugins/infra/server/routes/metadata/lib/get_cloud_metric_metadata.ts @@ -26,8 +26,8 @@ export const getCloudMetricsMetadata = async ( timeRange: { from: number; to: number } ): Promise => { const metricQuery = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: sourceConfiguration.metricAlias, body: { query: { diff --git a/x-pack/plugins/infra/server/routes/metadata/lib/get_metric_metadata.ts b/x-pack/plugins/infra/server/routes/metadata/lib/get_metric_metadata.ts index 26bcf267ad6ccc..bfa0884bfe1999 100644 --- a/x-pack/plugins/infra/server/routes/metadata/lib/get_metric_metadata.ts +++ b/x-pack/plugins/infra/server/routes/metadata/lib/get_metric_metadata.ts @@ -32,8 +32,8 @@ export const getMetricMetadata = async ( ): Promise => { const fields = findInventoryFields(nodeType, sourceConfiguration.fields); const metricQuery = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: sourceConfiguration.metricAlias, body: { query: { diff --git a/x-pack/plugins/infra/server/routes/metadata/lib/get_node_info.ts b/x-pack/plugins/infra/server/routes/metadata/lib/get_node_info.ts index fccbead2832c90..06035ed40adf18 100644 --- a/x-pack/plugins/infra/server/routes/metadata/lib/get_node_info.ts +++ b/x-pack/plugins/infra/server/routes/metadata/lib/get_node_info.ts @@ -53,8 +53,8 @@ export const getNodeInfo = async ( const fields = findInventoryFields(nodeType, sourceConfiguration.fields); const timestampField = sourceConfiguration.fields.timestamp; const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, terminateAfter: 1, index: sourceConfiguration.metricAlias, body: { diff --git a/x-pack/plugins/infra/server/routes/metadata/lib/get_pod_node_name.ts b/x-pack/plugins/infra/server/routes/metadata/lib/get_pod_node_name.ts index 8e4c707d9b4672..9bf809ba3b3f44 100644 --- a/x-pack/plugins/infra/server/routes/metadata/lib/get_pod_node_name.ts +++ b/x-pack/plugins/infra/server/routes/metadata/lib/get_pod_node_name.ts @@ -22,8 +22,8 @@ export const getPodNodeName = async ( const fields = findInventoryFields(nodeType, sourceConfiguration.fields); const timestampField = sourceConfiguration.fields.timestamp; const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, terminateAfter: 1, index: sourceConfiguration.metricAlias, body: { diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts index b88472fbb4b14d..be25bbbf022ee9 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/get_dataset_for_field.ts @@ -22,8 +22,8 @@ export const getDatasetForField = async ( timerange: { field: string; to: number; from: number } ) => { const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, terminateAfter: 1, index: indexPattern, body: { diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/query_total_groupings.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/query_total_groupings.ts index b871fa21c111d7..a2bf778d5016dd 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/query_total_groupings.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/query_total_groupings.ts @@ -41,8 +41,8 @@ export const queryTotalGroupings = async ( } const params = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: options.indexPattern, body: { size: 0, diff --git a/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts b/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts index 13a108b76b6a48..3357b1a842183f 100644 --- a/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts +++ b/x-pack/plugins/infra/server/utils/calculate_metric_interval.ts @@ -35,9 +35,9 @@ export const calculateMetricInterval = async ( from = options.timerange.to - inventoryModel.metrics.defaultTimeRangeInSeconds * 1000; } const query = { - allowNoIndices: true, + allow_no_indices: true, index: options.indexPattern, - ignoreUnavailable: true, + ignore_unavailable: true, body: { query: { bool: { diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts index 43527f875cf729..974ffad7745d97 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts @@ -107,7 +107,7 @@ export async function getLastRecovery(req: LegacyRequest, esIndexPattern: string const mbParams = { index: esIndexPattern, size, - ignoreUnavailable: true, + ignore_unavailable: true, body: { _source: ['elasticsearch.index.recovery', '@timestamp'], sort: { timestamp: { order: 'desc', unmapped_type: 'long' } }, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/all/query.all_actions.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/all/query.all_actions.dsl.ts index 55bec687d3e2cc..8dc8fad02a7c13 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/all/query.all_actions.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/all/query.all_actions.dsl.ts @@ -20,9 +20,9 @@ export const buildActionsQuery = ({ // const filter = [...createQueryFilterClauses(filterQuery)]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: '.fleet-actions', - ignoreUnavailable: true, + ignore_unavailable: true, body: { // query: { bool: { filter } }, query: { diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts index bc9d63e6193385..41f2875b1d4c80 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts @@ -23,9 +23,9 @@ export const buildActionDetailsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: '.fleet-actions', - ignoreUnavailable: true, + ignore_unavailable: true, body: { query: { bool: { filter } }, size: 1, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts index d74067bff02519..109e2609119330 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts @@ -25,9 +25,9 @@ export const buildActionResultsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: '.fleet-actions-results*', - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggs: { aggs: { diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts index 52101462270c73..314de574f0a70a 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts @@ -21,9 +21,9 @@ export const buildAgentsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: '.fleet-agents', - ignoreUnavailable: true, + ignore_unavailable: true, body: { query: { bool: { diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts index 406ff26991f0e2..59326d6c501559 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts @@ -36,9 +36,9 @@ export const buildResultsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: `logs-${OSQUERY_INTEGRATION_NAME}.result*`, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggs: { count_by_agent_id: { diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts index 9a452943ff6996..4598aadd808f7f 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage.ts @@ -147,7 +147,7 @@ export async function getReportingUsage( const reportingIndex = config.get('index'); const params = { index: `${reportingIndex}-*`, - filterPath: 'aggregations.*.buckets', + filter_path: 'aggregations.*.buckets', body: { size: 0, aggs: { diff --git a/x-pack/plugins/rollup/server/collectors/helpers.ts b/x-pack/plugins/rollup/server/collectors/helpers.ts index b6e5bc190d972d..b007bbbff7e4aa 100644 --- a/x-pack/plugins/rollup/server/collectors/helpers.ts +++ b/x-pack/plugins/rollup/server/collectors/helpers.ts @@ -32,8 +32,8 @@ export async function fetchRollupIndexPatterns(kibanaIndex: string, esClient: El const searchParams = { size: ES_MAX_RESULT_WINDOW_DEFAULT_VALUE, index: kibanaIndex, - ignoreUnavailable: true, - filterPath: ['hits.hits._id'], + ignore_unavailable: true, + filter_path: ['hits.hits._id'], body: { query: { bool: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts index 786f9a20f77e54..ea015ea145b3f7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -28,8 +28,8 @@ export const buildEventEnrichmentQuery: SecuritySolutionFactory { const parsedResponse = await parseEventEnrichmentResponse(options, response); const expectedInspect = expect.objectContaining({ - allowNoIndices: true, + allow_no_indices: true, body: { _source: false, fields: ['*'], @@ -57,7 +57,7 @@ describe('parseEventEnrichmentResponse', () => { }, }, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: ['filebeat-*'], }); const parsedInspect = JSON.parse(parsedResponse.inspect.dsl[0]); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts index 0369f182a4c753..d6e456e7066739 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/__mocks__/index.ts @@ -611,7 +611,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -622,7 +622,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { docvalue_fields: mockOptions.docValueFields, @@ -783,7 +783,7 @@ export const mockBuckets: HostAggEsItem = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, track_total_hits: false, body: { aggregations: { @@ -821,7 +821,7 @@ export const expectedDsl = { docvalue_fields: mockOptions.docValueFields, size: 0, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts index 5d8540f8860772..bc405e89bf7a6c 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts.dsl.ts @@ -40,9 +40,9 @@ export const buildHostsQuery = ({ const agg = { host_count: { cardinality: { field: 'host.name' } } }; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts index 1c338998e3b65f..94124bc9567b74 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/query.all_hosts_entities.dsl.ts @@ -40,9 +40,9 @@ export const buildHostsQueryEntities = ({ const agg = { host_count: { cardinality: { field: 'host.name' } } }; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts index 1dd3dc8ee4cff5..bdc4755c6a02bf 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/__mocks__/index.ts @@ -2149,7 +2149,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -2160,7 +2160,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { docvalue_fields: mockOptions.docValueFields, aggregations: { @@ -2371,7 +2371,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -2382,7 +2382,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { docvalue_fields: mockOptions.docValueFields, aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query.dsl.ts index 325d45e04b2b03..1057ace837b43f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query.dsl.ts @@ -61,9 +61,9 @@ export const buildQuery = ({ }; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts index d320130115588d..a17bb2ecf9c8fb 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/dsl/query_entities.dsl.ts @@ -41,9 +41,9 @@ export const buildQueryEntities = ({ }; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts index e0473c9501b0cc..ec8c7d7ca68664 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts @@ -1301,7 +1301,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -1312,7 +1312,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { @@ -1415,7 +1415,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -1426,7 +1426,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts index 45afed2526aa36..003149a9501bbf 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts @@ -35,9 +35,9 @@ export const buildHostDetailsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts index 01473a4368dbca..738a9db6837281 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications.dsl.ts @@ -39,8 +39,8 @@ export const buildHostsKpiAuthenticationsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts index cff09f2354d315..fd1104345babcc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/authentications/query.hosts_kpi_authentications_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildHostsKpiAuthenticationsQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts index 5ea2d1aa407805..ed1d0e8edb1070 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts.dsl.ts @@ -28,8 +28,8 @@ export const buildHostsKpiHostsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts index 972ead9a6538eb..785f6aa5b69800 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/hosts/query.hosts_kpi_hosts_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildHostsKpiHostsQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts index 0471644d11bbea..c875e23b523e5d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips.dsl.ts @@ -28,8 +28,8 @@ export const buildHostsKpiUniqueIpsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts index 2a55c34238d70d..e323de78a0459a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/kpi/unique_ips/query.hosts_kpi_unique_ips_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildHostsKpiUniqueIpsQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts index 443e7e96a3c7f8..54403f9c392e75 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/__mocks__/index.ts @@ -124,7 +124,7 @@ export const formattedSearchStrategyFirstResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -135,7 +135,7 @@ export const formattedSearchStrategyFirstResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { query: { bool: { filter: [{ term: { 'host.name': 'siem-kibana' } }] } }, @@ -190,7 +190,7 @@ export const formattedSearchStrategyLastResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -201,7 +201,7 @@ export const formattedSearchStrategyLastResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { query: { bool: { filter: [{ term: { 'host.name': 'siem-kibana' } }] } }, @@ -225,7 +225,7 @@ export const formattedSearchStrategyLastResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -236,7 +236,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { _source: ['@timestamp'], diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.first_or_last_seen_host.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.first_or_last_seen_host.dsl.ts index 21876b9aad11be..8f9f7462b9dcb9 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.first_or_last_seen_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/last_first_seen/query.first_or_last_seen_host.dsl.ts @@ -17,9 +17,9 @@ export const buildFirstOrLastSeenHostQuery = ({ const filter = [{ term: { 'host.name': hostName } }]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts index 2b4e4b8291401a..85bce797f52aec 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/__mocks__/index.ts @@ -117,7 +117,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -128,7 +128,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { @@ -330,7 +330,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -341,7 +341,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts index a0ae368ac0fe28..92d194e78284b0 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/overview/query.overview_host.dsl.ts @@ -28,9 +28,9 @@ export const buildOverviewHostQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/risk_score/query.hosts_risk.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/risk_score/query.hosts_risk.dsl.ts index 43930ab3de2efe..5bbc9b7726002c 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/risk_score/query.hosts_risk.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/risk_score/query.hosts_risk.dsl.ts @@ -32,8 +32,8 @@ export const buildHostsRiskScoreQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: false, - ignoreUnavailable: true, + allow_no_indices: false, + ignore_unavailable: true, track_total_hits: false, body: { query: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts index 0ad976a0f498c2..2730465323c19f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/__mocks__/index.ts @@ -4300,7 +4300,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -4311,7 +4311,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { process_count: { cardinality: { field: 'process.name' } }, @@ -4435,7 +4435,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -4446,7 +4446,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { process_count: { cardinality: { field: 'process.name' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/dsl/query.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/dsl/query.dsl.ts index 5d4f45c68160a8..c5a78354ed8667 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/dsl/query.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/uncommon_processes/dsl/query.dsl.ts @@ -48,9 +48,9 @@ export const buildQuery = ({ }; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...agg, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts index 7f36e3551e5bee..d5478dad15d502 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/__mocks__/index.ts @@ -41,8 +41,8 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -127,7 +127,7 @@ export const formattedAlertsSearchStrategyResponse: MatrixHistogramStrategyRespo }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, track_total_hits: false, body: { aggregations: { @@ -164,7 +164,7 @@ export const expectedDsl = { }, size: 0, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -209,8 +209,8 @@ export const formattedAnomaliesSearchStrategyResponse: MatrixHistogramStrategyRe 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggs: { @@ -392,8 +392,8 @@ export const formattedAuthenticationsSearchStrategyResponse: MatrixHistogramStra 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -959,8 +959,8 @@ export const formattedEventsSearchStrategyResponse: MatrixHistogramStrategyRespo 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -1927,7 +1927,7 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -1938,7 +1938,7 @@ export const formattedDnsSearchStrategyResponse: MatrixHistogramStrategyResponse 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { dns_count: { cardinality: { field: 'dns.question.registered_domain' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts index 82531f35b09abc..4f6b910fbccf7f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/__mocks__/index.ts @@ -36,8 +36,8 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts index 54ee066b64119b..60df6023a13d00 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/alerts/query.alerts_histogram.dsl.ts @@ -83,8 +83,8 @@ export const buildAlertsHistogramQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: getHistogramAggregation(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts index ab76d54dee11ff..700580655f1b0d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/__mocks__/index.ts @@ -36,8 +36,8 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts index 78fc0a30d04778..b82e7823fd8470 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/anomalies/query.anomalies_histogram.dsl.ts @@ -64,8 +64,8 @@ export const buildAnomaliesHistogramQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggs: getHistogramAggregation(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts index 1fd7b85242df64..b917da12c9ad77 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/__mocks__/index.ts @@ -35,8 +35,8 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts index 8661fff574b4a7..b16efcd8301e0f 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram.dsl.ts @@ -76,8 +76,8 @@ export const buildAuthenticationsHistogramQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: getHistogramAggregation(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts index c66a0d6c11b94f..886dcf11123bd2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/authentications/query.authentications_histogram_entities.dsl.ts @@ -59,8 +59,8 @@ export const buildAuthenticationsHistogramQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: getHistogramAggregation(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts index 4d97fba3cb80cf..d4e721a5ebe80a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/__mocks__/index.ts @@ -26,7 +26,7 @@ export const mockOptions = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -37,7 +37,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { dns_count: { cardinality: { field: 'dns.question.registered_domain' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts index d9dfc57a264a86..7a7b4b49d17c1c 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/dns/query.dns_histogram.dsl.ts @@ -77,9 +77,9 @@ export const buildDnsHistogramQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts index 5dab2bcd5cf9de..d2ee38ddd66a82 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/__mocks__/index.ts @@ -40,8 +40,8 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -95,8 +95,8 @@ export const expectedThresholdDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -152,8 +152,8 @@ export const expectedThresholdMissingFieldDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -197,7 +197,7 @@ export const expectedThresholdMissingFieldDsl = { }; export const expectedThresholdWithCardinalityDsl = { - allowNoIndices: true, + allow_no_indices: true, body: { aggregations: { eventActionGroup: { @@ -244,7 +244,7 @@ export const expectedThresholdWithCardinalityDsl = { }, size: 0, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -269,8 +269,8 @@ export const expectedThresholdWithGroupFieldsAndCardinalityDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -316,7 +316,7 @@ export const expectedThresholdWithGroupFieldsAndCardinalityDsl = { }; export const expectedThresholdGroupWithCardinalityDsl = { - allowNoIndices: true, + allow_no_indices: true, body: { aggregations: { eventActionGroup: { @@ -365,7 +365,7 @@ export const expectedThresholdGroupWithCardinalityDsl = { }, size: 0, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -390,8 +390,8 @@ export const expectedIpIncludingMissingDataDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { @@ -453,8 +453,8 @@ export const expectedIpNotIncludingMissingDataDsl = { 'packetbeat-*', 'winlogbeat-*', ], - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts index 16bcb3cdbfcb13..91aeb50448d4e2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram/events/query.events_histogram.dsl.ts @@ -152,8 +152,8 @@ export const buildEventsHistogramQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { aggregations: getHistogramAggregation(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts index 7f71906bcaa97f..158b63e6e8455b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/__mocks__/index.ts @@ -304,7 +304,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -315,7 +315,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { docvalue_fields: mockOptions.docValueFields, @@ -446,7 +446,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -457,7 +457,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts index e5a508663a2e0b..5684c7685231e6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/details/query.details_network.dsl.ts @@ -103,9 +103,9 @@ export const buildNetworkDetailsQuery = ({ ip, }: NetworkDetailsRequestOptions) => { const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts index cc01450e5bec56..f1bae35f53ebb6 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/__mocks__/index.ts @@ -131,7 +131,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -142,7 +142,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { dns_count: { cardinality: { field: 'dns.question.registered_domain' } }, @@ -203,7 +203,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -214,7 +214,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { dns_count: { cardinality: { field: 'dns.question.registered_domain' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts index 90e8ee02ee6848..612d83e81660de 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts @@ -88,9 +88,9 @@ export const buildDnsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts index 9d52eb638eac6a..f831b9f20e8ca0 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/__mocks__/index.ts @@ -613,7 +613,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -624,7 +624,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { http_count: { cardinality: { field: 'url.path' } }, @@ -671,7 +671,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -682,7 +682,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { http_count: { cardinality: { field: 'url.path' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts index 85db56cd721674..8882d178042611 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts @@ -36,9 +36,9 @@ export const buildHttpQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...getCountAgg(), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts index 75b32af4b01f53..c0317772e5fa97 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kip_dns_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildDnsQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts index 1d1aa50cc3ee2b..8d27f7d289d031 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/dns/query.network_kpi_dns.dsl.ts @@ -56,8 +56,8 @@ export const buildDnsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { query: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts index 3d5607c8b443a8..4d5ca88fe383aa 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events.dsl.ts @@ -30,8 +30,8 @@ export const buildNetworkEventsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { query: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts index 6311bb6ea20390..bb40da0a064fba 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/network_events/query.network_kpi_network_events_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildNetworkEventsQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts index 0a826938e95b86..eae7f7a29ce72b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes.dsl.ts @@ -56,8 +56,8 @@ export const buildTlsHandshakeQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: true, body: { query: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts index 5b0ac92b35049d..bd7a464e1a0909 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/tls_handshakes/query.network_kpi_tls_handshakes_entities.dsl.ts @@ -28,8 +28,8 @@ export const buildTlsHandshakeQueryEntities = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts index ec8de30cfff852..3cb04caf5afe54 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_flows/query.network_kpi_unique_flows.dsl.ts @@ -30,8 +30,8 @@ export const buildUniqueFlowsQuery = ({ const dslQuery = { index: defaultIndex, - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts index 590e7117826d73..c915cd4fb58d62 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips.dsl.ts @@ -84,9 +84,9 @@ export const buildUniquePrivateIpsQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts index a56cf4c3d1cedd..922a082439807d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/kpi/unique_private_ips/query.network_kpi_unique_private_ips_entities.dsl.ts @@ -84,9 +84,9 @@ export const buildUniquePrivateIpsQueryEntities = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts index 74b201e9a2294b..0b5019d6fec9a7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/__mocks__/index.ts @@ -101,7 +101,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -112,7 +112,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { @@ -206,8 +206,8 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, - ignoreUnavailable: true, + allow_no_indices: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts index 7e35ae2fd4308f..f911850c8cd948 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/overview/query.overview_network.dsl.ts @@ -28,9 +28,9 @@ export const buildOverviewNetworkQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggregations: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts index e46c2c20aa1a8e..c34ec2225ed952 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/__mocks__/index.ts @@ -59,7 +59,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -70,7 +70,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { @@ -114,7 +114,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -125,7 +125,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts index 4f724cf9b15f94..297643fe569526 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts @@ -75,9 +75,9 @@ export const buildNetworkTlsQuery = ({ const filter = ip ? [...defaultFilter, { term: { [`${flowTarget}.ip`]: ip } }] : defaultFilter; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts index ba5db90df82452..490ade26ad2a5e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/__mocks__/index.ts @@ -58,7 +58,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -69,7 +69,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { top_countries_count: { cardinality: { field: 'destination.geo.country_iso_code' } }, @@ -118,7 +118,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -129,7 +129,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { top_countries_count: { cardinality: { field: 'destination.geo.country_iso_code' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts index 880cd4002086a8..463d3c9b11bd2e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts @@ -42,9 +42,9 @@ export const buildTopCountriesQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...getCountAgg(flowTarget), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts index d661bfa0d67075..7f1d27db45d2dc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network_entities.dsl.ts @@ -47,9 +47,9 @@ export const buildTopCountriesQueryEntities = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...getCountAgg(flowTarget), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts index e881a9ef93949c..fa759661772d55 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/__mocks__/index.ts @@ -810,7 +810,7 @@ export const formattedSearchStrategyResponse: NetworkTopNFlowStrategyResponse = dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -821,7 +821,7 @@ export const formattedSearchStrategyResponse: NetworkTopNFlowStrategyResponse = 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { top_n_flow_count: { cardinality: { field: 'source.ip' } }, @@ -878,7 +878,7 @@ export const formattedSearchStrategyResponse: NetworkTopNFlowStrategyResponse = }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -889,7 +889,7 @@ export const expectedDsl = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { top_n_flow_count: { cardinality: { field: 'source.ip' } }, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts index f1ac87e59d392f..52efb50f00b4b8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts @@ -42,9 +42,9 @@ export const buildTopNFlowQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...getCountAgg(flowTarget), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts index 3ea3c6f363de08..d11a2debf3cff7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network_entities.dsl.ts @@ -47,9 +47,9 @@ export const buildTopNFlowQueryEntities = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggregations: { ...getCountAgg(flowTarget), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts index 686730dbe79275..acb98b7e347bc4 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/__mocks__/index.ts @@ -119,7 +119,7 @@ export const formattedSearchStrategyResponse = { dsl: [ JSON.stringify( { - allowNoIndices: true, + allow_no_indices: true, index: [ 'apm-*-transaction*', 'traces-apm*', @@ -130,7 +130,7 @@ export const formattedSearchStrategyResponse = { 'packetbeat-*', 'winlogbeat-*', ], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { @@ -175,7 +175,7 @@ export const formattedSearchStrategyResponse = { }; export const expectedDsl = { - allowNoIndices: true, + allow_no_indices: true, track_total_hits: false, body: { aggs: { @@ -209,7 +209,7 @@ export const expectedDsl = { }, size: 0, }, - ignoreUnavailable: true, + ignore_unavailable: true, index: [ 'apm-*-transaction*', 'traces-apm*', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts index 2b02b25292a32c..0c35c967c2ac52 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/users/query.users_network.dsl.ts @@ -34,9 +34,9 @@ export const buildUsersQuery = ({ ]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { aggs: { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_rules/query.host_rules.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_rules/query.host_rules.dsl.ts index 4c116104b3e14d..d2aeb63b743f5a 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_rules/query.host_rules.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_rules/query.host_rules.dsl.ts @@ -30,9 +30,9 @@ export const buildHostRulesQuery = ({ ]; return { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, // can stop getting this from sourcerer and assume default detections index if we want - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_tactics/query.host_tactics.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_tactics/query.host_tactics.dsl.ts index ec1afe247011b2..f9bbcc80772784 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_tactics/query.host_tactics.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/host_tactics/query.host_tactics.dsl.ts @@ -30,9 +30,9 @@ export const buildHostTacticsQuery = ({ ]; return { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, // can stop getting this from sourcerer and assume default detections index if we want - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/risk_score/query.risk_score.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/risk_score/query.risk_score.dsl.ts index 79c50d84e3c92e..90d2e51c8ff9ea 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/risk_score/query.risk_score.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/risk_score/query.risk_score.dsl.ts @@ -31,9 +31,9 @@ export const buildRiskScoreQuery = ({ ]; return { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/user_rules/query.user_rules.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/user_rules/query.user_rules.dsl.ts index c2242ff00a6c14..d3111eed4aef83 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/user_rules/query.user_rules.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/ueba/user_rules/query.user_rules.dsl.ts @@ -30,9 +30,9 @@ export const buildUserRulesQuery = ({ ]; return { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, // can stop getting this from sourcerer and assume default detections index if we want - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts index 8d5a2efc7fae11..eaeceb8ab57ee9 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts @@ -188,8 +188,8 @@ export const getDetectionRuleMetrics = async ( let rulesUsage: DetectionRulesTypeUsage = initialDetectionRulesUsage; const ruleSearchOptions: RuleSearchParams = { body: { query: { bool: { filter: { term: { 'alert.alertTypeId': SIGNALS_ID } } } } }, - filterPath: [], - ignoreUnavailable: true, + filter_path: [], + ignore_unavailable: true, index: kibanaIndex, size: MAX_RESULTS_WINDOW, }; diff --git a/x-pack/plugins/security_solution/server/usage/detections/types.ts b/x-pack/plugins/security_solution/server/usage/detections/types.ts index 0e3ba97ca0f7c2..430a524f3f03ae 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/types.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/types.ts @@ -17,8 +17,8 @@ interface RuleSearchBody { export interface RuleSearchParams { body: RuleSearchBody; - filterPath: string[]; - ignoreUnavailable: boolean; + filter_path: string[]; + ignore_unavailable: boolean; index: string; size: number; } diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts index 72a7d6e2692e8a..3653cb60d36533 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/query.events_all.dsl.ts @@ -63,9 +63,9 @@ export const buildTimelineEventsAllQuery = ({ }); const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), aggregations: { diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts index 3572a2b9c497ed..9066861cdb8189 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts @@ -22,7 +22,7 @@ describe('buildTimelineDetailsQuery', () => { expect(query).toMatchInlineSnapshot(` Object { - "allowNoIndices": true, + "allow_no_indices": true, "body": Object { "_source": true, "docvalue_fields": Array [ @@ -53,7 +53,7 @@ describe('buildTimelineDetailsQuery', () => { }, }, }, - "ignoreUnavailable": true, + "ignore_unavailable": true, "index": ".siem-signals-default", "size": 1, } diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts index 4297cd595e2616..70b592440468e3 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts @@ -33,9 +33,9 @@ export const buildTimelineDetailsQuery = ( }; return { - allowNoIndices: true, + allow_no_indices: true, index: indexName, - ignoreUnavailable: true, + ignore_unavailable: true, body: { docvalue_fields: docValueFields, query, diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/kpi/query.kpi.dsl.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/kpi/query.kpi.dsl.ts index 41eed7cbb4fa32..dd87ef177bfe62 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/kpi/query.kpi.dsl.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/kpi/query.kpi.dsl.ts @@ -44,9 +44,9 @@ export const buildTimelineKpiQuery = ({ const filter = [...filterClause, ...getTimerangeFilter(timerange), { match_all: {} }]; const dslQuery = { - allowNoIndices: true, + allow_no_indices: true, index: defaultIndex, - ignoreUnavailable: true, + ignore_unavailable: true, body: { aggs: { userCount: { diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts index 354c682377bac7..4ff86c9a9f290e 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/last_event_time/query.events_last_event_time.dsl.ts @@ -38,9 +38,9 @@ export const buildLastEventTimeQuery = ({ case LastEventIndexKey.ipDetails: if (details.ip) { return { - allowNoIndices: true, + allow_no_indices: true, index: indicesToQuery.network, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), @@ -61,9 +61,9 @@ export const buildLastEventTimeQuery = ({ case LastEventIndexKey.hostDetails: if (details.hostName) { return { - allowNoIndices: true, + allow_no_indices: true, index: indicesToQuery.hosts, - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), @@ -85,9 +85,9 @@ export const buildLastEventTimeQuery = ({ case LastEventIndexKey.network: case LastEventIndexKey.ueba: return { - allowNoIndices: true, + allow_no_indices: true, index: indicesToQuery[indexKey], - ignoreUnavailable: true, + ignore_unavailable: true, track_total_hits: false, body: { ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), From 6f3400379b89caa0c6cbac8c021e7fd7f1816a2d Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Wed, 20 Oct 2021 15:16:19 -0400 Subject: [PATCH 11/35] [7.x] - Deprecate excluding ML from base privileges (#115445) Co-authored-by: Joe Portner <5295965+jportner@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../common/licensing/license_features.ts | 5 + .../common/licensing/license_service.test.ts | 17 +- .../common/licensing/license_service.ts | 14 ++ .../security/common/model/deprecations.ts | 12 +- x-pack/plugins/security/common/model/index.ts | 4 +- .../nav_control/nav_control_service.test.ts | 2 +- .../security/server/deprecations/index.ts | 1 + .../server/deprecations/ml_privileges.test.ts | 234 ++++++++++++++++++ .../server/deprecations/ml_privileges.ts | 124 ++++++++++ .../privilege_deprecations.test.ts | 75 +++++- .../deprecations/privilege_deprecations.ts | 20 +- x-pack/plugins/security/server/mocks.ts | 2 +- x-pack/plugins/security/server/plugin.test.ts | 2 +- x-pack/plugins/security/server/plugin.ts | 8 + .../server/routes/views/login.test.ts | 1 + .../deprecation_privileges/index.test.ts | 18 +- .../server/deprecation_privileges/index.ts | 8 +- .../security_solution/server/plugin.ts | 3 +- 18 files changed, 503 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugins/security/server/deprecations/ml_privileges.test.ts create mode 100644 x-pack/plugins/security/server/deprecations/ml_privileges.ts diff --git a/x-pack/plugins/security/common/licensing/license_features.ts b/x-pack/plugins/security/common/licensing/license_features.ts index ac80c89ae7be39..edd6a7ea2084bc 100644 --- a/x-pack/plugins/security/common/licensing/license_features.ts +++ b/x-pack/plugins/security/common/licensing/license_features.ts @@ -65,6 +65,11 @@ export interface SecurityLicenseFeatures { */ readonly allowRbac: boolean; + /** + * Indicates if Machine Learning features are available. + */ + readonly allowML: boolean; + /** * Indicates whether we allow sub-feature privileges. */ diff --git a/x-pack/plugins/security/common/licensing/license_service.test.ts b/x-pack/plugins/security/common/licensing/license_service.test.ts index cdc80c1a038f15..5d7ce321f9ef6f 100644 --- a/x-pack/plugins/security/common/licensing/license_service.test.ts +++ b/x-pack/plugins/security/common/licensing/license_service.test.ts @@ -28,6 +28,7 @@ describe('license features', function () { allowRbac: false, allowSubFeaturePrivileges: false, allowAuditLogging: false, + allowML: false, allowLegacyAuditLogging: false, }); }); @@ -51,6 +52,7 @@ describe('license features', function () { allowRbac: false, allowSubFeaturePrivileges: false, allowAuditLogging: false, + allowML: false, allowLegacyAuditLogging: false, }); }); @@ -75,6 +77,7 @@ describe('license features', function () { "allowAuditLogging": false, "allowLegacyAuditLogging": false, "allowLogin": false, + "allowML": false, "allowRbac": false, "allowRoleDocumentLevelSecurity": false, "allowRoleFieldLevelSecurity": false, @@ -97,6 +100,7 @@ describe('license features', function () { "allowAuditLogging": true, "allowLegacyAuditLogging": true, "allowLogin": true, + "allowML": true, "allowRbac": true, "allowRoleDocumentLevelSecurity": true, "allowRoleFieldLevelSecurity": true, @@ -134,10 +138,12 @@ describe('license features', function () { allowRbac: true, allowSubFeaturePrivileges: false, allowAuditLogging: false, + allowML: false, allowLegacyAuditLogging: false, }); - expect(getFeatureSpy).toHaveBeenCalledTimes(1); + expect(getFeatureSpy).toHaveBeenCalledTimes(2); expect(getFeatureSpy).toHaveBeenCalledWith('security'); + expect(getFeatureSpy).toHaveBeenCalledWith('ml'); }); it('should not show login page or other security elements if security is disabled in Elasticsearch.', () => { @@ -160,6 +166,7 @@ describe('license features', function () { allowRbac: false, allowSubFeaturePrivileges: false, allowAuditLogging: false, + allowML: false, allowLegacyAuditLogging: false, }); }); @@ -185,6 +192,7 @@ describe('license features', function () { allowRbac: true, allowSubFeaturePrivileges: false, allowAuditLogging: false, + allowML: false, allowLegacyAuditLogging: true, }); }); @@ -210,6 +218,7 @@ describe('license features', function () { allowRbac: true, allowSubFeaturePrivileges: true, allowAuditLogging: true, + allowML: false, allowLegacyAuditLogging: true, }); }); @@ -217,7 +226,10 @@ describe('license features', function () { it('should allow to login, allow RBAC, role mappings, access agreement, sub-feature privileges, and DLS if license >= platinum', () => { const mockRawLicense = licenseMock.createLicense({ license: { mode: 'platinum', type: 'platinum' }, - features: { security: { isEnabled: true, isAvailable: true } }, + features: { + security: { isEnabled: true, isAvailable: true }, + ml: { isEnabled: true, isAvailable: true }, + }, }); const serviceSetup = new SecurityLicenseService().setup({ @@ -235,6 +247,7 @@ describe('license features', function () { allowRbac: true, allowSubFeaturePrivileges: true, allowAuditLogging: true, + allowML: true, allowLegacyAuditLogging: true, }); }); diff --git a/x-pack/plugins/security/common/licensing/license_service.ts b/x-pack/plugins/security/common/licensing/license_service.ts index 51093428e84a03..252b91c2f1b0f5 100644 --- a/x-pack/plugins/security/common/licensing/license_service.ts +++ b/x-pack/plugins/security/common/licensing/license_service.ts @@ -68,6 +68,15 @@ export class SecurityLicenseService { ); } + private isMLEnabledFromRawLicense(rawLicense: Readonly | undefined) { + if (!rawLicense) { + return false; + } + + const mlFeature = rawLicense.getFeature('ml'); + return mlFeature !== undefined && mlFeature.isAvailable && mlFeature.isEnabled; + } + private calculateFeaturesFromRawLicense( rawLicense: Readonly | undefined ): SecurityLicenseFeatures { @@ -85,6 +94,7 @@ export class SecurityLicenseService { allowRoleDocumentLevelSecurity: false, allowRoleFieldLevelSecurity: false, allowRbac: false, + allowML: false, allowSubFeaturePrivileges: false, layout: rawLicense !== undefined && !rawLicense?.isAvailable @@ -93,6 +103,8 @@ export class SecurityLicenseService { }; } + const allowML = this.isMLEnabledFromRawLicense(rawLicense); + if (!this.isSecurityEnabledFromRawLicense(rawLicense)) { return { showLogin: false, @@ -105,6 +117,7 @@ export class SecurityLicenseService { allowRoleDocumentLevelSecurity: false, allowRoleFieldLevelSecurity: false, allowRbac: false, + allowML, allowSubFeaturePrivileges: false, }; } @@ -124,6 +137,7 @@ export class SecurityLicenseService { // Only platinum and trial licenses are compliant with field- and document-level security. allowRoleDocumentLevelSecurity: isLicensePlatinumOrBetter, allowRoleFieldLevelSecurity: isLicensePlatinumOrBetter, + allowML, allowRbac: true, }; } diff --git a/x-pack/plugins/security/common/model/deprecations.ts b/x-pack/plugins/security/common/model/deprecations.ts index e990f370c51736..5b12f2c41f8ff9 100644 --- a/x-pack/plugins/security/common/model/deprecations.ts +++ b/x-pack/plugins/security/common/model/deprecations.ts @@ -9,17 +9,17 @@ import type { DeprecationsDetails, GetDeprecationsContext } from '../../../../../src/core/server'; import type { Role } from './role'; -export interface PrivilegeDeprecationsRolesByFeatureIdResponse { +export interface PrivilegeDeprecationsRolesResponse { roles?: Role[]; errors?: DeprecationsDetails[]; } -export interface PrivilegeDeprecationsRolesByFeatureIdRequest { +export interface PrivilegeDeprecationsRolesRequest { context: GetDeprecationsContext; - featureId: string; + featureId?: string; } export interface PrivilegeDeprecationsService { - getKibanaRolesByFeatureId: ( - args: PrivilegeDeprecationsRolesByFeatureIdRequest - ) => Promise; + getKibanaRoles: ( + args: PrivilegeDeprecationsRolesRequest + ) => Promise; } diff --git a/x-pack/plugins/security/common/model/index.ts b/x-pack/plugins/security/common/model/index.ts index 082e6bdc12cd07..1d351da114cf9c 100644 --- a/x-pack/plugins/security/common/model/index.ts +++ b/x-pack/plugins/security/common/model/index.ts @@ -34,7 +34,7 @@ export { RoleMapping, } from './role_mapping'; export { - PrivilegeDeprecationsRolesByFeatureIdRequest, - PrivilegeDeprecationsRolesByFeatureIdResponse, + PrivilegeDeprecationsRolesRequest, + PrivilegeDeprecationsRolesResponse, PrivilegeDeprecationsService, } from './deprecations'; diff --git a/x-pack/plugins/security/public/nav_control/nav_control_service.test.ts b/x-pack/plugins/security/public/nav_control/nav_control_service.test.ts index 9839d29291629a..325c20a65e1766 100644 --- a/x-pack/plugins/security/public/nav_control/nav_control_service.test.ts +++ b/x-pack/plugins/security/public/nav_control/nav_control_service.test.ts @@ -19,7 +19,7 @@ import { SecurityNavControlService } from './nav_control_service'; const validLicense = { isAvailable: true, getFeature: (feature) => { - expect(feature).toEqual('security'); + expect(['security', 'ml']).toContain(feature); return { isAvailable: true, diff --git a/x-pack/plugins/security/server/deprecations/index.ts b/x-pack/plugins/security/server/deprecations/index.ts index 0bbeccf1588b43..d5a1cc52a71872 100644 --- a/x-pack/plugins/security/server/deprecations/index.ts +++ b/x-pack/plugins/security/server/deprecations/index.ts @@ -15,3 +15,4 @@ export { KIBANA_ADMIN_ROLE_NAME, KIBANA_USER_ROLE_NAME, } from './kibana_user_role'; +export { registerMLPrivilegesDeprecation } from './ml_privileges'; diff --git a/x-pack/plugins/security/server/deprecations/ml_privileges.test.ts b/x-pack/plugins/security/server/deprecations/ml_privileges.test.ts new file mode 100644 index 00000000000000..e8dfcb40a80e8d --- /dev/null +++ b/x-pack/plugins/security/server/deprecations/ml_privileges.test.ts @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import type { SecurityGetRoleRole } from '@elastic/elasticsearch/api/types'; + +import type { PackageInfo, RegisterDeprecationsConfig } from 'src/core/server'; +import { + deprecationsServiceMock, + elasticsearchServiceMock, + loggingSystemMock, + savedObjectsClientMock, +} from 'src/core/server/mocks'; + +import { licenseMock } from '../../common/licensing/index.mock'; +import { securityMock } from '../mocks'; +import { registerMLPrivilegesDeprecation } from './ml_privileges'; + +function getDepsMock() { + return { + logger: loggingSystemMock.createLogger(), + deprecationsService: deprecationsServiceMock.createSetupContract(), + license: licenseMock.create({ + allowML: true, + }), + packageInfo: { + branch: 'some-branch', + buildSha: 'sha', + dist: true, + version: '8.0.0', + buildNum: 1, + } as PackageInfo, + applicationName: 'kibana-.kibana', + }; +} + +function getContextMock() { + return { + esClient: elasticsearchServiceMock.createScopedClusterClient(), + savedObjectsClient: savedObjectsClientMock.create(), + }; +} + +function createMockRole(role: Partial = {}) { + return { + name: 'role', + cluster: [], + indices: [], + run_as: [], + applications: [], + metadata: {}, + transient_metadata: { enabled: true }, + ...role, + }; +} + +describe('Machine Learning privileges deprecations', () => { + let mockDeps: ReturnType; + let mockContext: ReturnType; + let deprecationHandler: RegisterDeprecationsConfig; + beforeEach(() => { + mockContext = getContextMock(); + mockDeps = getDepsMock(); + registerMLPrivilegesDeprecation(mockDeps); + + expect(mockDeps.deprecationsService.registerDeprecations).toHaveBeenCalledTimes(1); + deprecationHandler = mockDeps.deprecationsService.registerDeprecations.mock.calls[0][0]; + }); + + it('does not return any deprecations if security is not enabled', async () => { + mockDeps.license.isEnabled.mockReturnValue(false); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toEqual([]); + expect(mockContext.esClient.asCurrentUser.security.getRole).not.toHaveBeenCalled(); + }); + + it('does not return any deprecations if ML is not enabled', async () => { + mockDeps.license.isEnabled.mockReturnValue(true); + mockDeps.license.getFeatures.mockReturnValue({ + allowRbac: true, + showLogin: true, + allowLogin: true, + showLinks: true, + showRoleMappingsManagement: true, + allowAccessAgreement: true, + allowAuditLogging: true, + allowLegacyAuditLogging: true, + allowSubFeaturePrivileges: true, + allowRoleDocumentLevelSecurity: true, + allowRoleFieldLevelSecurity: true, + allowML: false, + }); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toEqual([]); + expect(mockContext.esClient.asCurrentUser.security.getRole).not.toHaveBeenCalled(); + }); + + it('does not return any deprecations if none of the custom roles grant base privileges', async () => { + mockContext.esClient.asCurrentUser.security.getRole.mockResolvedValue( + securityMock.createApiResponse({ body: { roleA: createMockRole() } }) + ); + + mockContext.esClient.asCurrentUser.security.getRoleMapping.mockResolvedValue( + securityMock.createApiResponse({ + body: { + mappingA: { enabled: true, roles: ['roleA'], rules: {}, metadata: {} }, + }, + }) + ); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toEqual([]); + }); + + it('returns deprecations even if cannot retrieve roles due to permission error', async () => { + mockContext.esClient.asCurrentUser.security.getRole.mockRejectedValue( + new errors.ResponseError(securityMock.createApiResponse({ statusCode: 403, body: {} })) + ); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "A user with the \\"manage_security\\" cluster privilege is required to perform this check.", + ], + }, + "level": "fetch_error", + "message": "You must have the 'manage_security' cluster privilege to fix role deprecations.", + "title": "Error in privilege deprecations services", + }, + ] + `); + }); + + it('returns deprecations even if cannot retrieve roles due to unknown error', async () => { + mockContext.esClient.asCurrentUser.security.getRole.mockRejectedValue( + new errors.ResponseError(securityMock.createApiResponse({ statusCode: 500, body: {} })) + ); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "A user with the \\"manage_security\\" cluster privilege is required to perform this check.", + ], + }, + "level": "fetch_error", + "message": "Error retrieving roles for privilege deprecations: {}", + "title": "Error in privilege deprecations services", + }, + ] + `); + }); + + it('returns role-related deprecations', async () => { + mockContext.esClient.asCurrentUser.security.getRole.mockResolvedValue( + securityMock.createApiResponse({ + body: { + roleA: createMockRole({ + applications: [ + { + application: 'kibana-.kibana', + privileges: ['all'], + resources: ['*'], + }, + ], + }), + roleB: createMockRole({ + applications: [ + { + application: 'kibana-.kibana', + privileges: ['space_all'], + resources: ['space:b'], + }, + ], + }), + roleC: createMockRole({ + applications: [ + { + application: 'kibana-.kibana', + privileges: ['space_read'], + resources: ['space:b'], + }, + ], + }), + roleD: createMockRole({ + applications: [ + { + // This shouldn't trigger a deprecation because of a mismatched applicaiton name + application: 'NOT_kibana-.kibana', + privileges: ['space_read'], + resources: ['space:b'], + }, + ], + }), + roleE: createMockRole({ + applications: [ + { + // This shouldn't trigger a deprecation because feature privileges are granted instead + application: 'kibana-.kibana', + privileges: ['feature_discover.all'], + resources: ['*'], + }, + ], + }), + }, + }) + ); + + await expect(deprecationHandler.getDeprecations(mockContext)).resolves.toMatchInlineSnapshot(` + Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Change the affected roles to use feature privileges that grant access to only the desired features instead.", + "If you don't make any changes, affected roles will grant access to the Machine Learning feature in 8.0.", + "The affected roles are: roleA, roleB, roleC", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/some-branch/kibana-privileges.html", + "level": "warning", + "message": "Roles that use base privileges will include the Machine Learning feature in 8.0.", + "title": "The Machine Learning feature is changing", + }, + ] + `); + }); +}); diff --git a/x-pack/plugins/security/server/deprecations/ml_privileges.ts b/x-pack/plugins/security/server/deprecations/ml_privileges.ts new file mode 100644 index 00000000000000..5272f4e5e425b0 --- /dev/null +++ b/x-pack/plugins/security/server/deprecations/ml_privileges.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import type { + DeprecationsDetails, + DeprecationsServiceSetup, + GetDeprecationsContext, + Logger, + PackageInfo, +} from 'src/core/server'; + +import type { SecurityLicense } from '../../common'; +import type { + PrivilegeDeprecationsRolesResponse, + PrivilegeDeprecationsService, +} from '../../common/model'; +import { isRoleReserved } from '../../common/model'; +import { getPrivilegeDeprecationsService } from './privilege_deprecations'; + +interface Deps { + deprecationsService: DeprecationsServiceSetup; + license: SecurityLicense; + logger: Logger; + packageInfo: PackageInfo; + applicationName: string; +} + +function getDeprecationTitle() { + return i18n.translate('xpack.security.deprecations.mlPrivileges.deprecationTitle', { + defaultMessage: 'The Machine Learning feature is changing', + }); +} + +function getDeprecationMessage() { + return i18n.translate('xpack.security.deprecations.mlPrivileges.deprecationMessage', { + defaultMessage: + 'Roles that use base privileges will include the Machine Learning feature in 8.0.', + }); +} + +export const registerMLPrivilegesDeprecation = ({ + deprecationsService, + logger, + license, + packageInfo, + applicationName, +}: Deps) => { + deprecationsService.registerDeprecations({ + getDeprecations: async (context) => { + // Nothing to do if security or ml is disabled + if (!license.isEnabled() || !license.getFeatures().allowML) { + return []; + } + + const privilegeDeprecationService = getPrivilegeDeprecationsService( + { + applicationName, + }, + license, + logger + ); + + return [...(await getRolesDeprecations(context, privilegeDeprecationService, packageInfo))]; + }, + }); +}; + +async function getRolesDeprecations( + context: GetDeprecationsContext, + privilegeDeprecationService: PrivilegeDeprecationsService, + packageInfo: PackageInfo +): Promise { + const response: PrivilegeDeprecationsRolesResponse = + await privilegeDeprecationService.getKibanaRoles({ context }); + if (response.errors) { + return response.errors; + } + + const rolesWithBasePrivileges = (response.roles ?? []) + .filter((role) => { + const hasBasePrivileges = role.kibana.some( + (kp) => kp.base.includes('all') || kp.base.includes('read') + ); + return !isRoleReserved(role) && hasBasePrivileges; + }) + .map((role) => role.name); + + if (rolesWithBasePrivileges.length === 0) { + return []; + } + + return [ + { + title: getDeprecationTitle(), + message: getDeprecationMessage(), + level: 'warning', + deprecationType: 'feature', + documentationUrl: `https://www.elastic.co/guide/en/kibana/${packageInfo.branch}/kibana-privileges.html`, + correctiveActions: { + manualSteps: [ + i18n.translate('xpack.security.deprecations.mlPrivileges.manualSteps1', { + defaultMessage: + 'Change the affected roles to use feature privileges that grant access to only the desired features instead.', + }), + i18n.translate('xpack.security.deprecations.mlPrivileges.manualSteps2', { + defaultMessage: + "If you don't make any changes, affected roles will grant access to the Machine Learning feature in 8.0.", + }), + i18n.translate('xpack.security.deprecations.mlPrivileges.manualSteps3', { + defaultMessage: 'The affected roles are: {roles}', + values: { + roles: rolesWithBasePrivileges.join(', '), + }, + }), + ], + }, + }, + ]; +} diff --git a/x-pack/plugins/security/server/deprecations/privilege_deprecations.test.ts b/x-pack/plugins/security/server/deprecations/privilege_deprecations.test.ts index e889eb17d5af9b..f795f3e0a46d52 100644 --- a/x-pack/plugins/security/server/deprecations/privilege_deprecations.test.ts +++ b/x-pack/plugins/security/server/deprecations/privilege_deprecations.test.ts @@ -15,17 +15,70 @@ const kibanaIndexName = '.a-kibana-index'; const application = `kibana-${kibanaIndexName}`; describe('#getPrivilegeDeprecationsService', () => { - describe('#getKibanaRolesByFeatureId', () => { + describe('#getKibanaRoles', () => { const mockAsCurrentUser = elasticsearchServiceMock.createScopedClusterClient(); const mockLicense = licenseMock.create(); const mockLogger = loggingSystemMock.createLogger(); const authz = { applicationName: application }; - const { getKibanaRolesByFeatureId } = getPrivilegeDeprecationsService( - authz, - mockLicense, - mockLogger - ); + const { getKibanaRoles } = getPrivilegeDeprecationsService(authz, mockLicense, mockLogger); + + it('returns all roles when the "feature" parameter is not provided', async () => { + mockAsCurrentUser.asCurrentUser.security.getRole.mockResolvedValue( + elasticsearchServiceMock.createSuccessTransportRequestPromise({ + first_role: { + cluster: [], + indices: [], + applications: [ + { + application, + privileges: ['feature_siem.all', 'feature_siem.cases_read'], + resources: ['space:securitySolutions'], + }, + ], + run_as: [], + metadata: { + _reserved: true, + }, + transient_metadata: { + enabled: true, + }, + }, + second_role: { + cluster: [], + indices: [], + applications: [ + { + application, + privileges: ['all'], + resources: ['*'], + }, + ], + run_as: [], + metadata: { + _reserved: true, + }, + transient_metadata: { + enabled: true, + }, + }, + }) + ); + + const mockContext = { + esClient: mockAsCurrentUser, + savedObjectsClient: jest.fn(), + } as unknown as GetDeprecationsContext; + + const resp = await getKibanaRoles({ context: mockContext }); + expect(resp).not.toHaveProperty('errors'); + expect(resp.roles?.map((r) => r.name)).toMatchInlineSnapshot(` + Array [ + "first_role", + "second_role", + ] + `); + }); it('happy path to find siem roles with feature_siem privileges', async () => { mockAsCurrentUser.asCurrentUser.security.getRole.mockResolvedValue( @@ -56,7 +109,7 @@ describe('#getPrivilegeDeprecationsService', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; - const resp = await getKibanaRolesByFeatureId({ context: mockContext, featureId: 'siem' }); + const resp = await getKibanaRoles({ context: mockContext, featureId: 'siem' }); expect(resp).toMatchInlineSnapshot(` Object { "roles": Array [ @@ -130,7 +183,7 @@ describe('#getPrivilegeDeprecationsService', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; - const resp = await getKibanaRolesByFeatureId({ context: mockContext, featureId: 'siem' }); + const resp = await getKibanaRoles({ context: mockContext, featureId: 'siem' }); expect(resp).toMatchInlineSnapshot(` Object { "roles": Array [ @@ -209,7 +262,7 @@ describe('#getPrivilegeDeprecationsService', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; - const resp = await getKibanaRolesByFeatureId({ context: mockContext, featureId: 'siem' }); + const resp = await getKibanaRoles({ context: mockContext, featureId: 'siem' }); expect(resp).toMatchInlineSnapshot(` Object { "roles": Array [], @@ -230,7 +283,7 @@ describe('#getPrivilegeDeprecationsService', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; - const resp = await getKibanaRolesByFeatureId({ context: mockContext, featureId: 'siem' }); + const resp = await getKibanaRoles({ context: mockContext, featureId: 'siem' }); expect(resp).toMatchInlineSnapshot(` Object { "errors": Array [ @@ -262,7 +315,7 @@ describe('#getPrivilegeDeprecationsService', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; - const resp = await getKibanaRolesByFeatureId({ context: mockContext, featureId: 'siem' }); + const resp = await getKibanaRoles({ context: mockContext, featureId: 'siem' }); expect(resp).toMatchInlineSnapshot(` Object { "errors": Array [ diff --git a/x-pack/plugins/security/server/deprecations/privilege_deprecations.ts b/x-pack/plugins/security/server/deprecations/privilege_deprecations.ts index df212d5c7bde39..7a35a0099e7ac6 100644 --- a/x-pack/plugins/security/server/deprecations/privilege_deprecations.ts +++ b/x-pack/plugins/security/server/deprecations/privilege_deprecations.ts @@ -10,8 +10,8 @@ import type { Logger } from 'src/core/server'; import type { SecurityLicense } from '../../common/licensing'; import type { - PrivilegeDeprecationsRolesByFeatureIdRequest, - PrivilegeDeprecationsRolesByFeatureIdResponse, + PrivilegeDeprecationsRolesRequest, + PrivilegeDeprecationsRolesResponse, } from '../../common/model'; import { transformElasticsearchRoleToRole } from '../authorization'; import type { AuthorizationServiceSetupInternal, ElasticsearchRole } from '../authorization'; @@ -22,10 +22,10 @@ export const getPrivilegeDeprecationsService = ( license: SecurityLicense, logger: Logger ) => { - const getKibanaRolesByFeatureId = async ({ + const getKibanaRoles = async ({ context, featureId, - }: PrivilegeDeprecationsRolesByFeatureIdRequest): Promise => { + }: PrivilegeDeprecationsRolesRequest): Promise => { // Nothing to do if security is disabled if (!license.isEnabled()) { return { @@ -95,12 +95,16 @@ export const getPrivilegeDeprecationsService = ( }; } return { - roles: kibanaRoles.filter((role) => - role.kibana.find((privilege) => Object.hasOwnProperty.call(privilege.feature, featureId)) - ), + roles: featureId + ? kibanaRoles.filter((role) => + role.kibana.find((privilege) => + Object.hasOwnProperty.call(privilege.feature, featureId) + ) + ) + : kibanaRoles, }; }; return Object.freeze({ - getKibanaRolesByFeatureId, + getKibanaRoles, }); }; diff --git a/x-pack/plugins/security/server/mocks.ts b/x-pack/plugins/security/server/mocks.ts index 7cae0d29bf9430..5869ae67c1faa1 100644 --- a/x-pack/plugins/security/server/mocks.ts +++ b/x-pack/plugins/security/server/mocks.ts @@ -29,7 +29,7 @@ function createSetupMock() { registerSpacesService: jest.fn(), license: licenseMock.create(), privilegeDeprecationsService: { - getKibanaRolesByFeatureId: jest.fn(), + getKibanaRoles: jest.fn(), }, }; } diff --git a/x-pack/plugins/security/server/plugin.test.ts b/x-pack/plugins/security/server/plugin.test.ts index 4784e14a11fb48..596fddd0c416fb 100644 --- a/x-pack/plugins/security/server/plugin.test.ts +++ b/x-pack/plugins/security/server/plugin.test.ts @@ -124,7 +124,7 @@ describe('Security Plugin', () => { "isLicenseAvailable": [Function], }, "privilegeDeprecationsService": Object { - "getKibanaRolesByFeatureId": [Function], + "getKibanaRoles": [Function], }, } `); diff --git a/x-pack/plugins/security/server/plugin.ts b/x-pack/plugins/security/server/plugin.ts index bf540d5d4ddc82..ffe8793448a746 100644 --- a/x-pack/plugins/security/server/plugin.ts +++ b/x-pack/plugins/security/server/plugin.ts @@ -46,6 +46,7 @@ import { getPrivilegeDeprecationsService, registerKibanaDashboardOnlyRoleDeprecation, registerKibanaUserRoleDeprecation, + registerMLPrivilegesDeprecation, } from './deprecations'; import { ElasticsearchService } from './elasticsearch'; import type { SecurityFeatureUsageServiceStart } from './feature_usage'; @@ -434,5 +435,12 @@ export class SecurityPlugin logger, packageInfo: this.initializerContext.env.packageInfo, }); + registerMLPrivilegesDeprecation({ + deprecationsService: core.deprecations, + license, + logger, + packageInfo: this.initializerContext.env.packageInfo, + applicationName: this.authorizationSetup?.applicationName!, + }); } } diff --git a/x-pack/plugins/security/server/routes/views/login.test.ts b/x-pack/plugins/security/server/routes/views/login.test.ts index 6def1b7d77df3d..ab6d8f6fdf8a99 100644 --- a/x-pack/plugins/security/server/routes/views/login.test.ts +++ b/x-pack/plugins/security/server/routes/views/login.test.ts @@ -172,6 +172,7 @@ describe('Login view routes', () => { allowSubFeaturePrivileges: true, allowAuditLogging: true, allowLegacyAuditLogging: true, + allowML: true, showLogin: true, }); diff --git a/x-pack/plugins/security_solution/server/deprecation_privileges/index.test.ts b/x-pack/plugins/security_solution/server/deprecation_privileges/index.test.ts index 213beb62071841..aeee6f15a950f6 100644 --- a/x-pack/plugins/security_solution/server/deprecation_privileges/index.test.ts +++ b/x-pack/plugins/security_solution/server/deprecation_privileges/index.test.ts @@ -324,7 +324,7 @@ describe('deprecations', () => { savedObjectsClient: jest.fn(), } as unknown as GetDeprecationsContext; const getDeprecations = jest.fn(); - const getKibanaRolesByFeatureId = jest.fn(); + const getKibanaRoles = jest.fn(); const mockDeprecationsService: DeprecationsServiceSetup = { registerDeprecations: (deprecationContext: RegisterDeprecationsConfig) => { getDeprecations.mockImplementation(deprecationContext.getDeprecations); @@ -334,15 +334,15 @@ describe('deprecations', () => { beforeAll(() => { registerPrivilegeDeprecations({ deprecationsService: mockDeprecationsService, - getKibanaRolesByFeatureId, + getKibanaRoles, logger: loggingSystemMock.createLogger(), }); }); beforeEach(() => { - getKibanaRolesByFeatureId.mockReset(); + getKibanaRoles.mockReset(); }); - test('getDeprecations return the errors from getKibanaRolesByFeatureId', async () => { + test('getDeprecations return the errors from getKibanaRoles', async () => { const errorResponse = { errors: [ { @@ -357,13 +357,13 @@ describe('deprecations', () => { }, ], }; - getKibanaRolesByFeatureId.mockResolvedValue(errorResponse); + getKibanaRoles.mockResolvedValue(errorResponse); const response = await getDeprecations(mockContext); expect(response).toEqual(errorResponse.errors); }); test('getDeprecations return empty array when securitySolutionCases privileges are already set up', async () => { - getKibanaRolesByFeatureId.mockResolvedValue({ + getKibanaRoles.mockResolvedValue({ roles: [ { _transform_error: [], @@ -399,7 +399,7 @@ describe('deprecations', () => { }); test('happy path build securitySolutionCases privileges from siem privileges', async () => { - getKibanaRolesByFeatureId.mockResolvedValue({ + getKibanaRoles.mockResolvedValue({ roles: [ { _transform_error: [], @@ -481,7 +481,7 @@ describe('deprecations', () => { }); test('getDeprecations handles multiple roles and filters out any that have already been updated', async () => { - getKibanaRolesByFeatureId.mockResolvedValue({ + getKibanaRoles.mockResolvedValue({ roles: [ { _transform_error: [], @@ -616,7 +616,7 @@ describe('deprecations', () => { }); test('getDeprecations handles multiple roles and filters out any that do not grant access to Cases', async () => { - getKibanaRolesByFeatureId.mockResolvedValue({ + getKibanaRoles.mockResolvedValue({ roles: [ { _transform_error: [], diff --git a/x-pack/plugins/security_solution/server/deprecation_privileges/index.ts b/x-pack/plugins/security_solution/server/deprecation_privileges/index.ts index b56583d26261f6..af2d46851affc6 100644 --- a/x-pack/plugins/security_solution/server/deprecation_privileges/index.ts +++ b/x-pack/plugins/security_solution/server/deprecation_privileges/index.ts @@ -14,7 +14,7 @@ import { CASES_FEATURE_ID, SERVER_APP_ID } from '../../common/constants'; interface Deps { deprecationsService: DeprecationsServiceSetup; - getKibanaRolesByFeatureId?: PrivilegeDeprecationsService['getKibanaRolesByFeatureId']; + getKibanaRoles?: PrivilegeDeprecationsService['getKibanaRoles']; logger: Logger; } @@ -70,16 +70,16 @@ function outdatedSiemRolePredicate(role: Role) { export const registerPrivilegeDeprecations = ({ deprecationsService, - getKibanaRolesByFeatureId, + getKibanaRoles, logger, }: Deps) => { deprecationsService.registerDeprecations({ getDeprecations: async (context) => { let deprecatedRoles: DeprecationsDetails[] = []; - if (!getKibanaRolesByFeatureId) { + if (!getKibanaRoles) { return deprecatedRoles; } - const responseRoles = await getKibanaRolesByFeatureId({ + const responseRoles = await getKibanaRoles({ context, featureId: 'siem', }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index c3a359a0c6cf8e..53d145c7ab6357 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -327,8 +327,7 @@ export class Plugin implements ISecuritySolutionPlugin { registerPrivilegeDeprecations({ deprecationsService: core.deprecations, - getKibanaRolesByFeatureId: - plugins.security?.privilegeDeprecationsService.getKibanaRolesByFeatureId, + getKibanaRoles: plugins.security?.privilegeDeprecationsService.getKibanaRoles, logger: this.logger.get('deprecations'), }); From 51c7ef5a9c019c7f5a3260c98fdf7e04cb49ff70 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:19:13 -0400 Subject: [PATCH 12/35] [ML] Functional tests - adjust test retries and waiting conditions (#115592) (#115722) This PR fixes a few test stability issues, mostly by adding/adjusting retries. * Stabilize data viz full time range selection * Stabilize DFA wizard source data loading * Stabilize feature importance service methods * Stabilize DFA table row existence assertion * Stabilize dashboard embeddable service methods Co-authored-by: Robert Oskamp --- .../services/ml/dashboard_embeddables.ts | 10 ++++----- .../ml/data_frame_analytics_creation.ts | 5 ++++- .../ml/data_frame_analytics_results.ts | 22 ++++++++++--------- .../services/ml/data_frame_analytics_table.ts | 5 +++++ .../ml/data_visualizer_index_based.ts | 7 ++++-- x-pack/test/functional/services/ml/index.ts | 1 - 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/x-pack/test/functional/services/ml/dashboard_embeddables.ts b/x-pack/test/functional/services/ml/dashboard_embeddables.ts index c4fa9f643e69e3..0dc5cc8fae2d5e 100644 --- a/x-pack/test/functional/services/ml/dashboard_embeddables.ts +++ b/x-pack/test/functional/services/ml/dashboard_embeddables.ts @@ -7,12 +7,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { MlCommonUI } from './common_ui'; import { MlDashboardJobSelectionTable } from './dashboard_job_selection_table'; export function MachineLearningDashboardEmbeddablesProvider( { getService }: FtrProviderContext, - mlCommonUI: MlCommonUI, mlDashboardJobSelectionTable: MlDashboardJobSelectionTable ) { const retry = getService('retry'); @@ -22,14 +20,14 @@ export function MachineLearningDashboardEmbeddablesProvider( return { async assertAnomalyChartsEmbeddableInitializerExists() { - await retry.tryForTime(5000, async () => { - await testSubjects.existOrFail('mlAnomalyChartsEmbeddableInitializer'); + await retry.tryForTime(10 * 1000, async () => { + await testSubjects.existOrFail('mlAnomalyChartsEmbeddableInitializer', { timeout: 1000 }); }); }, async assertAnomalyChartsEmbeddableInitializerNotExists() { - await retry.tryForTime(5000, async () => { - await testSubjects.missingOrFail('mlAnomalyChartsEmbeddableInitializer'); + await retry.tryForTime(10 * 1000, async () => { + await testSubjects.missingOrFail('mlAnomalyChartsEmbeddableInitializer', { timeout: 1000 }); }); }, diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index 45ba4c5c348330..2c375d47b0b3b0 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -18,10 +18,11 @@ import { } from '../../../../plugins/ml/common/util/analytics_utils'; export function MachineLearningDataFrameAnalyticsCreationProvider( - { getService }: FtrProviderContext, + { getPageObject, getService }: FtrProviderContext, mlCommonUI: MlCommonUI, mlApi: MlApi ) { + const headerPage = getPageObject('header'); const testSubjects = getService('testSubjects'); const comboBox = getService('comboBox'); const retry = getService('retry'); @@ -111,10 +112,12 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( }, async assertSourceDataPreviewExists() { + await headerPage.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('mlAnalyticsCreationDataGrid loaded', { timeout: 5000 }); }, async assertIndexPreviewHistogramChartButtonExists() { + await headerPage.waitUntilLoadingHasFinished(); await testSubjects.existOrFail('mlAnalyticsCreationDataGridHistogramButton'); }, diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_results.ts b/x-pack/test/functional/services/ml/data_frame_analytics_results.ts index ac728e6b88303d..a1bf8c6a65d70a 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_results.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_results.ts @@ -73,7 +73,7 @@ export function MachineLearningDataFrameAnalyticsResultsProvider( async assertTotalFeatureImportanceEvaluatePanelExists() { await testSubjects.existOrFail('mlDFExpandableSection-FeatureImportanceSummary'); - await testSubjects.existOrFail('mlTotalFeatureImportanceChart', { timeout: 5000 }); + await testSubjects.existOrFail('mlTotalFeatureImportanceChart', { timeout: 30 * 1000 }); }, async assertFeatureImportanceDecisionPathElementsExists() { @@ -167,17 +167,19 @@ export function MachineLearningDataFrameAnalyticsResultsProvider( async openFeatureImportancePopover() { this.assertResultsTableNotEmpty(); - const featureImportanceCell = await this.getFirstFeatureImportanceCell(); - await featureImportanceCell.focus(); - const interactionButton = await featureImportanceCell.findByTagName('button'); + await retry.tryForTime(30 * 1000, async () => { + const featureImportanceCell = await this.getFirstFeatureImportanceCell(); + await featureImportanceCell.focus(); + const interactionButton = await featureImportanceCell.findByTagName('button'); - // simulate hover and wait for button to appear - await featureImportanceCell.moveMouseTo(); - await this.waitForInteractionButtonToDisplay(interactionButton); + // simulate hover and wait for button to appear + await featureImportanceCell.moveMouseTo(); + await this.waitForInteractionButtonToDisplay(interactionButton); - // open popover - await interactionButton.click(); - await testSubjects.existOrFail('mlDFAFeatureImportancePopover'); + // open popover + await interactionButton.click(); + await testSubjects.existOrFail('mlDFAFeatureImportancePopover', { timeout: 1000 }); + }); }, async getFirstFeatureImportanceCell(): Promise { diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_table.ts b/x-pack/test/functional/services/ml/data_frame_analytics_table.ts index 5850919f2adc31..0bfb37c6c94f8d 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_table.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_table.ts @@ -196,6 +196,11 @@ export function MachineLearningDataFrameAnalyticsTableProvider({ getService }: F analyticsId: string, shouldBeDisplayed: boolean ) { + await this.waitForRefreshButtonLoaded(); + await testSubjects.click('~mlAnalyticsRefreshListButton'); + await this.waitForRefreshButtonLoaded(); + await testSubjects.existOrFail('mlAnalyticsJobList', { timeout: 30 * 1000 }); + if (shouldBeDisplayed) { await this.filterWithSearchString(analyticsId, 1); } else { diff --git a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts index 7f32968ec43269..68839464526292 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_index_based.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_index_based.ts @@ -33,8 +33,11 @@ export function MachineLearningDataVisualizerIndexBasedProvider({ }, async clickUseFullDataButton(expectedFormattedTotalDocCount: string) { - await testSubjects.clickWhenNotDisabled('dataVisualizerButtonUseFullData'); - await this.assertTotalDocumentCount(expectedFormattedTotalDocCount); + await retry.tryForTime(30 * 1000, async () => { + await testSubjects.clickWhenNotDisabled('dataVisualizerButtonUseFullData'); + await testSubjects.clickWhenNotDisabled('superDatePickerApplyTimeButton'); + await this.assertTotalDocumentCount(expectedFormattedTotalDocCount); + }); }, async assertTotalDocCountHeaderExist() { diff --git a/x-pack/test/functional/services/ml/index.ts b/x-pack/test/functional/services/ml/index.ts index d50ec371d7c23d..17302b27822237 100644 --- a/x-pack/test/functional/services/ml/index.ts +++ b/x-pack/test/functional/services/ml/index.ts @@ -67,7 +67,6 @@ export function MachineLearningProvider(context: FtrProviderContext) { const dashboardJobSelectionTable = MachineLearningDashboardJobSelectionTableProvider(context); const dashboardEmbeddables = MachineLearningDashboardEmbeddablesProvider( context, - commonUI, dashboardJobSelectionTable ); From c96ca68b711a798b4034c7817ef4398133c330cc Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:19:26 -0400 Subject: [PATCH 13/35] [App Search] Load curation settings at root /curations/ path (#115690) (#115803) Co-authored-by: Byron Hulcher --- .../curations/curations_logic.test.ts | 11 -- .../components/curations/curations_logic.ts | 1 - .../curations/curations_router.test.tsx | 100 ++++++++++++++++++ .../components/curations/curations_router.tsx | 35 +++++- .../curations/views/curations.test.tsx | 6 +- .../components/curations/views/curations.tsx | 7 +- .../views/curations_overview.test.tsx | 49 ++++++++- .../curations/views/curations_overview.tsx | 8 +- .../curations_settings.test.tsx | 61 ----------- .../curations_settings/curations_settings.tsx | 26 +---- 10 files changed, 195 insertions(+), 109 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.test.ts index 0d02fbe4138708..42c3985e4dcf1f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.test.ts @@ -107,17 +107,6 @@ describe('CurationsLogic', () => { describe('listeners', () => { describe('loadCurations', () => { - it('should set dataLoading state', () => { - mount({ dataLoading: false }); - - CurationsLogic.actions.loadCurations(); - - expect(CurationsLogic.values).toEqual({ - ...DEFAULT_VALUES, - dataLoading: true, - }); - }); - it('should make an API call and set curations & meta state', async () => { http.get.mockReturnValueOnce(Promise.resolve(MOCK_CURATIONS_RESPONSE)); mount(); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts index 04d04b297050ad..4419603efddf0c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_logic.ts @@ -61,7 +61,6 @@ export const CurationsLogic = kea true, onCurationsLoad: () => false, }, ], diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.test.tsx index 9598212d3e0c98..a0fd778ac7ddea 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.test.tsx @@ -5,6 +5,9 @@ * 2.0. */ +import '../../../__mocks__/shallow_useeffect.mock'; +import '../../../__mocks__/react_router'; +import { setMockActions, setMockValues } from '../../../__mocks__/kea_logic'; import '../../__mocks__/engine_logic.mock'; import React from 'react'; @@ -12,13 +15,110 @@ import { Route, Switch } from 'react-router-dom'; import { shallow } from 'enzyme'; +import { LogRetentionOptions } from '../log_retention'; + import { CurationsRouter } from './'; +const MOCK_VALUES = { + // CurationsSettingsLogic + dataLoading: false, + curationsSettings: { + enabled: true, + mode: 'automatic', + }, + // LogRetentionLogic + logRetention: { + [LogRetentionOptions.Analytics]: { + enabled: true, + }, + }, + // LicensingLogic + hasPlatinumLicense: true, +}; + +const MOCK_ACTIONS = { + // CurationsSettingsLogic + loadCurationsSettings: jest.fn(), + onSkipLoadingCurationsSettings: jest.fn(), + // LogRetentionLogic + fetchLogRetention: jest.fn(), +}; + describe('CurationsRouter', () => { + beforeEach(() => { + jest.clearAllMocks(); + setMockActions(MOCK_ACTIONS); + }); + it('renders', () => { const wrapper = shallow(); expect(wrapper.find(Switch)).toHaveLength(1); expect(wrapper.find(Route)).toHaveLength(4); }); + + it('loads log retention settings', () => { + setMockValues(MOCK_VALUES); + shallow(); + + expect(MOCK_ACTIONS.fetchLogRetention).toHaveBeenCalled(); + }); + + describe('when the user has no platinum license', () => { + beforeEach(() => { + setMockValues({ + ...MOCK_VALUES, + hasPlatinumLicense: false, + }); + }); + + it('it does not fetch log retention', () => { + shallow(); + expect(MOCK_ACTIONS.fetchLogRetention).toHaveBeenCalledTimes(0); + }); + }); + + describe('loading curation settings based on log retention', () => { + it('loads curation settings when log retention is enabled', () => { + setMockValues({ + ...MOCK_VALUES, + logRetention: { + [LogRetentionOptions.Analytics]: { + enabled: true, + }, + }, + }); + + shallow(); + + expect(MOCK_ACTIONS.loadCurationsSettings).toHaveBeenCalledTimes(1); + }); + + it('skips loading curation settings when log retention is enabled', () => { + setMockValues({ + ...MOCK_VALUES, + logRetention: { + [LogRetentionOptions.Analytics]: { + enabled: false, + }, + }, + }); + + shallow(); + + expect(MOCK_ACTIONS.onSkipLoadingCurationsSettings).toHaveBeenCalledTimes(1); + }); + + it('takes no action if log retention has not yet been loaded', () => { + setMockValues({ + ...MOCK_VALUES, + logRetention: null, + }); + + shallow(); + + expect(MOCK_ACTIONS.loadCurationsSettings).toHaveBeenCalledTimes(0); + expect(MOCK_ACTIONS.onSkipLoadingCurationsSettings).toHaveBeenCalledTimes(0); + }); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.tsx index 693e5406b714b3..a3b000ea5054ac 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curations_router.tsx @@ -5,20 +5,53 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { Route, Switch } from 'react-router-dom'; +import { useValues, useActions } from 'kea'; + +import { LicensingLogic } from '../../../shared/licensing'; import { ENGINE_CURATIONS_PATH, ENGINE_CURATIONS_NEW_PATH, ENGINE_CURATION_PATH, ENGINE_CURATION_SUGGESTION_PATH, } from '../../routes'; +import { LogRetentionLogic, LogRetentionOptions } from '../log_retention'; import { Curation } from './curation'; import { Curations, CurationCreation, CurationSuggestion } from './views'; +import { CurationsSettingsLogic } from './views/curations_settings'; export const CurationsRouter: React.FC = () => { + // We need to loadCurationsSettings here so they are available across all views + + const { hasPlatinumLicense } = useValues(LicensingLogic); + + const { loadCurationsSettings, onSkipLoadingCurationsSettings } = + useActions(CurationsSettingsLogic); + + const { logRetention } = useValues(LogRetentionLogic); + const { fetchLogRetention } = useActions(LogRetentionLogic); + + const analyticsDisabled = !logRetention?.[LogRetentionOptions.Analytics].enabled; + + useEffect(() => { + if (hasPlatinumLicense) { + fetchLogRetention(); + } + }, [hasPlatinumLicense]); + + useEffect(() => { + if (logRetention) { + if (!analyticsDisabled) { + loadCurationsSettings(); + } else { + onSkipLoadingCurationsSettings(); + } + } + }, [logRetention]); + return ( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx index 42d808da6d9eeb..aacabf0ac73036 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.test.tsx @@ -126,14 +126,14 @@ describe('Curations', () => { describe('loading state', () => { it('renders a full-page loading state on initial page load', () => { - setMockValues({ ...values, dataLoading: true, curations: [] }); + setMockValues({ ...values, dataLoading: true }); const wrapper = shallow(); expect(wrapper.prop('isLoading')).toEqual(true); }); - it('does not re-render a full-page loading state after initial page load (uses component-level loading state instead)', () => { - setMockValues({ ...values, dataLoading: true, curations: [{}] }); + it('does not re-render a full-page loading state when data is loaded', () => { + setMockValues({ ...values, dataLoading: false }); const wrapper = shallow(); expect(wrapper.prop('isLoading')).toEqual(false); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx index 7440e0cf42b446..3d4751fcb343f3 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx @@ -25,12 +25,13 @@ import { getCurationsBreadcrumbs } from '../utils'; import { CurationsHistory } from './curations_history/curations_history'; import { CurationsOverview } from './curations_overview'; -import { CurationsSettings } from './curations_settings'; +import { CurationsSettings, CurationsSettingsLogic } from './curations_settings'; export const Curations: React.FC = () => { - const { dataLoading, curations, meta, selectedPageTab } = useValues(CurationsLogic); + const { dataLoading: curationsDataLoading, meta, selectedPageTab } = useValues(CurationsLogic); const { loadCurations, onSelectPageTab } = useActions(CurationsLogic); const { hasPlatinumLicense } = useValues(LicensingLogic); + const { dataLoading: curationsSettingsDataLoading } = useValues(CurationsSettingsLogic); const OVERVIEW_TAB = { label: i18n.translate( @@ -92,7 +93,7 @@ export const Curations: React.FC = () => { ], tabs: pageTabs, }} - isLoading={dataLoading && !curations.length} + isLoading={curationsSettingsDataLoading || curationsDataLoading} > {selectedPageTab === 'overview' && } {selectedPageTab === 'history' && } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.test.tsx index ff6ee66d8cb10f..809157704a14e9 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.test.tsx @@ -19,13 +19,32 @@ import { SuggestionsTable } from '../components/suggestions_table'; import { CurationsOverview } from './curations_overview'; +const MOCK_VALUES = { + // CurationsSettingsLogic + curationsSettings: { + enabled: true, + }, + // CurationsLogic + curations: [ + { + id: 'cur-id-1', + }, + { + id: 'cur-id-2', + }, + ], + // LicensingLogics + hasPlatinumLicense: true, +}; + describe('CurationsOverview', () => { beforeEach(() => { jest.clearAllMocks(); + setMockValues(MOCK_VALUES); }); it('renders an empty message when there are no curations', () => { - setMockValues({ curations: [] }); + setMockValues({ ...MOCK_VALUES, curations: [] }); const wrapper = shallow(); expect(wrapper.find(EmptyState).exists()).toBe(true); @@ -33,6 +52,7 @@ describe('CurationsOverview', () => { it('renders a curations table when there are curations present', () => { setMockValues({ + ...MOCK_VALUES, curations: [ { id: 'cur-id-1', @@ -47,15 +67,36 @@ describe('CurationsOverview', () => { expect(wrapper.find(CurationsTable)).toHaveLength(1); }); - it('renders a suggestions table when the user has a platinum license', () => { - setMockValues({ curations: [], hasPlatinumLicense: true }); + it('renders a suggestions table when the user has a platinum license and curations suggestions enabled', () => { + setMockValues({ + ...MOCK_VALUES, + hasPlatinumLicense: true, + curationsSettings: { + enabled: true, + }, + }); const wrapper = shallow(); expect(wrapper.find(SuggestionsTable).exists()).toBe(true); }); it('doesn\t render a suggestions table when the user has no platinum license', () => { - setMockValues({ curations: [], hasPlatinumLicense: false }); + setMockValues({ + ...MOCK_VALUES, + hasPlatinumLicense: false, + }); + const wrapper = shallow(); + + expect(wrapper.find(SuggestionsTable).exists()).toBe(false); + }); + + it('doesn\t render a suggestions table when the user has disabled suggestions', () => { + setMockValues({ + ...MOCK_VALUES, + curationsSettings: { + enabled: false, + }, + }); const wrapper = shallow(); expect(wrapper.find(SuggestionsTable).exists()).toBe(false); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.tsx index 079f0046cb9bf9..00593403b08cf0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_overview.tsx @@ -16,11 +16,17 @@ import { CurationsTable, EmptyState } from '../components'; import { SuggestionsTable } from '../components/suggestions_table'; import { CurationsLogic } from '../curations_logic'; +import { CurationsSettingsLogic } from './curations_settings'; + export const CurationsOverview: React.FC = () => { const { curations } = useValues(CurationsLogic); const { hasPlatinumLicense } = useValues(LicensingLogic); - const shouldShowSuggestions = hasPlatinumLicense; + const { + curationsSettings: { enabled }, + } = useValues(CurationsSettingsLogic); + + const shouldShowSuggestions = enabled && hasPlatinumLicense; return ( <> diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.test.tsx index 4b4e11c31d4b88..3b01d1e41c2718 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.test.tsx @@ -17,8 +17,6 @@ import { shallow, ShallowWrapper } from 'enzyme'; import { EuiButtonEmpty, EuiCallOut, EuiSwitch } from '@elastic/eui'; -import { mountWithIntl } from '@kbn/test/jest'; - import { Loading } from '../../../../../shared/loading'; import { EuiButtonTo } from '../../../../../shared/react_router_helpers'; import { DataPanel } from '../../../data_panel'; @@ -46,8 +44,6 @@ const MOCK_VALUES = { const MOCK_ACTIONS = { // CurationsSettingsLogic - loadCurationsSettings: jest.fn(), - onSkipLoadingCurationsSettings: jest.fn(), toggleCurationsEnabled: jest.fn(), toggleCurationsMode: jest.fn(), // LogRetentionLogic @@ -60,14 +56,6 @@ describe('CurationsSettings', () => { setMockActions(MOCK_ACTIONS); }); - it('loads curations and log retention settings on load', () => { - setMockValues(MOCK_VALUES); - mountWithIntl(); - - expect(MOCK_ACTIONS.loadCurationsSettings).toHaveBeenCalled(); - expect(MOCK_ACTIONS.fetchLogRetention).toHaveBeenCalled(); - }); - it('contains a switch to toggle curations settings', () => { let wrapper: ShallowWrapper; @@ -166,50 +154,6 @@ describe('CurationsSettings', () => { expect(wrapper.is(Loading)).toBe(true); }); - describe('loading curation settings based on log retention', () => { - it('loads curation settings when log retention is enabled', () => { - setMockValues({ - ...MOCK_VALUES, - logRetention: { - [LogRetentionOptions.Analytics]: { - enabled: true, - }, - }, - }); - - shallow(); - - expect(MOCK_ACTIONS.loadCurationsSettings).toHaveBeenCalledTimes(1); - }); - - it('skips loading curation settings when log retention is enabled', () => { - setMockValues({ - ...MOCK_VALUES, - logRetention: { - [LogRetentionOptions.Analytics]: { - enabled: false, - }, - }, - }); - - shallow(); - - expect(MOCK_ACTIONS.onSkipLoadingCurationsSettings).toHaveBeenCalledTimes(1); - }); - - it('takes no action if log retention has not yet been loaded', () => { - setMockValues({ - ...MOCK_VALUES, - logRetention: null, - }); - - shallow(); - - expect(MOCK_ACTIONS.loadCurationsSettings).toHaveBeenCalledTimes(0); - expect(MOCK_ACTIONS.onSkipLoadingCurationsSettings).toHaveBeenCalledTimes(0); - }); - }); - describe('when the user has no platinum license', () => { beforeEach(() => { setMockValues({ @@ -218,11 +162,6 @@ describe('CurationsSettings', () => { }); }); - it('it does not fetch log retention', () => { - shallow(); - expect(MOCK_ACTIONS.fetchLogRetention).toHaveBeenCalledTimes(0); - }); - it('shows a CTA to upgrade your license when the user when the user', () => { const wrapper = shallow(); expect(wrapper.is(DataPanel)).toBe(true); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.tsx index de669298b11d95..a5d4a33d8b870b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations_settings/curations_settings.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React from 'react'; import { useActions, useValues } from 'kea'; @@ -43,34 +43,12 @@ export const CurationsSettings: React.FC = () => { curationsSettings: { enabled, mode }, dataLoading, } = useValues(CurationsSettingsLogic); - const { - loadCurationsSettings, - onSkipLoadingCurationsSettings, - toggleCurationsEnabled, - toggleCurationsMode, - } = useActions(CurationsSettingsLogic); + const { toggleCurationsEnabled, toggleCurationsMode } = useActions(CurationsSettingsLogic); const { isLogRetentionUpdating, logRetention } = useValues(LogRetentionLogic); - const { fetchLogRetention } = useActions(LogRetentionLogic); const analyticsDisabled = !logRetention?.[LogRetentionOptions.Analytics].enabled; - useEffect(() => { - if (hasPlatinumLicense) { - fetchLogRetention(); - } - }, [hasPlatinumLicense]); - - useEffect(() => { - if (logRetention) { - if (!analyticsDisabled) { - loadCurationsSettings(); - } else { - onSkipLoadingCurationsSettings(); - } - } - }, [logRetention]); - if (!hasPlatinumLicense) return ( Date: Wed, 20 Oct 2021 15:19:37 -0400 Subject: [PATCH 14/35] [App Search] Improve visual design of Promoted Documents panel (#115683) (#115768) Co-authored-by: Byron Hulcher --- .../curation/documents/hidden_documents.tsx | 29 ++++---- .../curation/documents/organic_documents.tsx | 47 +++++++------ .../documents/promoted_documents.scss | 7 ++ .../curation/documents/promoted_documents.tsx | 66 ++++++++++-------- .../curation/results/add_result_flyout.tsx | 67 ++++++++++--------- .../curation/results/curation_result.tsx | 21 +++--- .../components/data_panel/data_panel.tsx | 10 ++- 7 files changed, 133 insertions(+), 114 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.scss diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/hidden_documents.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/hidden_documents.tsx index 519808940fa1b0..5e6123efa988e3 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/hidden_documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/hidden_documents.tsx @@ -52,19 +52,22 @@ export const HiddenDocuments: React.FC = () => { isLoading={hiddenDocumentsLoading} > {hasDocuments ? ( - documents.map((document, index) => ( - removeHiddenId(document.id), - }, - ]} - /> - )) + + {documents.map((document, index) => ( + + removeHiddenId(document.id), + }, + ]} + /> + + ))} + ) : ( { } > {hasDocuments ? ( - documents.map((document: Result, index) => ( - addHiddenId(document.id.raw), - }, - { - ...PROMOTE_DOCUMENT_ACTION, - onClick: () => addPromotedId(document.id.raw), - }, - ] - } - /> - )) + + {documents.map((document: Result, index) => ( + + addHiddenId(document.id.raw), + }, + { + ...PROMOTE_DOCUMENT_ACTION, + onClick: () => addPromotedId(document.id.raw), + }, + ] + } + /> + + ))} + ) : organicDocumentsLoading ? ( ) : ( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.scss b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.scss new file mode 100644 index 00000000000000..59346b66af6079 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.scss @@ -0,0 +1,7 @@ +.promotedDocuments { + &--results { + border-radius: $euiSizeM; + border: $euiBorderWidthThick solid $euiColorPrimary; + background-color: tintOrShade($euiColorPrimary, 90%, 70%); // Copied from @elastit/eui/src/global_styling/variables/_panels.scss + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.tsx index c953fb210c03d6..ab5bf2876ef0b5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/documents/promoted_documents.tsx @@ -31,6 +31,8 @@ import { PROMOTED_DOCUMENTS_TITLE } from '../constants'; import { CurationLogic } from '../curation_logic'; import { AddResultButton, CurationResult, convertToResultFormat } from '../results'; +import './promoted_documents.scss'; + export const PromotedDocuments: React.FC = () => { const { curation, isAutomated, promotedIds, promotedDocumentsLoading } = useValues(CurationLogic); const documents = curation.promoted; @@ -89,36 +91,42 @@ export const PromotedDocuments: React.FC = () => { > {hasDocuments ? ( - - {documents.map((document, index) => ( - - {(provided) => ( - + + {documents.map((document, index) => ( + + removePromotedId(document.id), - }, - ] - } - dragHandleProps={provided.dragHandleProps} - /> - )} - - ))} + draggableId={document.id} + customDragHandle + spacing="none" + isDragDisabled={isAutomated} + > + {(provided) => ( + removePromotedId(document.id), + }, + ] + } + dragHandleProps={provided.dragHandleProps} + /> + )} + + + ))} + ) : ( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/results/add_result_flyout.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/results/add_result_flyout.tsx index 9d341cc2f5520e..4c5ba8f1447fd4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/results/add_result_flyout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/curation/results/add_result_flyout.tsx @@ -19,6 +19,8 @@ import { EuiSpacer, EuiFieldSearch, EuiEmptyPrompt, + EuiFlexGroup, + EuiFlexItem, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -76,38 +78,41 @@ export const AddResultFlyout: React.FC = () => { {searchResults.length > 0 ? ( - searchResults.map((result) => { - const id = result.id.raw; - const isPromoted = promotedIds.includes(id); - const isHidden = hiddenIds.includes(id); + + {searchResults.map((result, index) => { + const id = result.id.raw; + const isPromoted = promotedIds.includes(id); + const isHidden = hiddenIds.includes(id); - return ( - removeHiddenId(id), - } - : { - ...HIDE_DOCUMENT_ACTION, - onClick: () => addHiddenId(id), - }, - isPromoted - ? { - ...DEMOTE_DOCUMENT_ACTION, - onClick: () => removePromotedId(id), - } - : { - ...PROMOTE_DOCUMENT_ACTION, - onClick: () => addPromotedId(id), - }, - ]} - /> - ); - }) + return ( + + removeHiddenId(id), + } + : { + ...HIDE_DOCUMENT_ACTION, + onClick: () => addHiddenId(id), + }, + isPromoted + ? { + ...DEMOTE_DOCUMENT_ACTION, + onClick: () => removePromotedId(id), + } + : { + ...PROMOTE_DOCUMENT_ACTION, + onClick: () => addPromotedId(id), + }, + ]} + /> + + ); + })} + ) : ( = ({ actions, dragHandleProps, resu } = useValues(EngineLogic); return ( - <> - - - + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx index 8dc43d08e5e096..d199d2a6d3edd9 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx @@ -20,12 +20,13 @@ import { EuiTitle, EuiTitleProps, } from '@elastic/eui'; +import { _EuiPanelDivlike } from '@elastic/eui/src/components/panel/panel'; import { LoadingOverlay } from '../../../shared/loading'; import './data_panel.scss'; -interface Props { +type Props = Omit<_EuiPanelDivlike, 'title'> & { title: React.ReactElement; // e.g., h2 tag titleSize?: EuiTitleProps['size']; subtitle?: React.ReactNode; @@ -33,10 +34,9 @@ interface Props { action?: React.ReactNode; responsive?: boolean; filled?: boolean; - hasBorder?: boolean; isLoading?: boolean; className?: string; -} +}; export const DataPanel: React.FC = ({ title, @@ -46,7 +46,6 @@ export const DataPanel: React.FC = ({ action, responsive = false, filled, - hasBorder, isLoading, className, children, @@ -58,12 +57,11 @@ export const DataPanel: React.FC = ({ return ( From d48316c1b31e76398522f927d28b2d8b65d5837c Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:19:45 -0400 Subject: [PATCH 15/35] [Bug] fix memory info extraction for fullstory (#115756) (#115799) * fix memory info extraction * ts Co-authored-by: Liza Katz --- x-pack/plugins/cloud/public/plugin.test.ts | 18 ++++++++++++++---- x-pack/plugins/cloud/public/plugin.ts | 16 ++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/cloud/public/plugin.test.ts b/x-pack/plugins/cloud/public/plugin.test.ts index a19a28d6c47133..c1c94375d70638 100644 --- a/x-pack/plugins/cloud/public/plugin.test.ts +++ b/x-pack/plugins/cloud/public/plugin.test.ts @@ -138,14 +138,22 @@ describe('Cloud Plugin', () => { describe('with memory', () => { beforeAll(() => { - // @ts-expect-error + // @ts-expect-error 2339 window.performance.memory = { - someMetric: 1, + get jsHeapSizeLimit() { + return 3; + }, + get totalJSHeapSize() { + return 2; + }, + get usedJSHeapSize() { + return 1; + }, }; }); afterAll(() => { - // @ts-expect-error + // @ts-expect-error 2339 delete window.performance.memory; }); @@ -159,7 +167,9 @@ describe('Cloud Plugin', () => { expect(fullStoryApiMock.event).toHaveBeenCalledWith('Loaded Kibana', { kibana_version_str: initContext.env.packageInfo.version, - some_metric_int: 1, + memory_js_heap_size_limit_int: 3, + memory_js_heap_size_total_int: 2, + memory_js_heap_size_used_int: 1, }); }); }); diff --git a/x-pack/plugins/cloud/public/plugin.ts b/x-pack/plugins/cloud/public/plugin.ts index 82fabea8b5a56d..64b03acdc3ffd7 100644 --- a/x-pack/plugins/cloud/public/plugin.ts +++ b/x-pack/plugins/cloud/public/plugin.ts @@ -16,7 +16,6 @@ import { } from 'src/core/public'; import { i18n } from '@kbn/i18n'; import { Subscription } from 'rxjs'; -import { mapKeys, snakeCase } from 'lodash'; import type { AuthenticatedUser, SecurityPluginSetup, @@ -250,11 +249,16 @@ export class CloudPlugin implements Plugin { } // Get performance information from the browser (non standard property - const memoryInfo = mapKeys( - // @ts-expect-error - window.performance.memory || {}, - (_, key) => `${snakeCase(key)}_int` - ); + // @ts-expect-error 2339 + const memory = window.performance.memory; + let memoryInfo = {}; + if (memory) { + memoryInfo = { + memory_js_heap_size_limit_int: memory.jsHeapSizeLimit, + memory_js_heap_size_total_int: memory.totalJSHeapSize, + memory_js_heap_size_used_int: memory.usedJSHeapSize, + }; + } // Record an event that Kibana was opened so we can easily search for sessions that use Kibana fullStory.event('Loaded Kibana', { // `str` suffix is required, see docs: https://help.fullstory.com/hc/en-us/articles/360020623234 From cecc4e5b29f9a4b18eba9a4af15aa518fa5dece7 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 15:20:05 -0400 Subject: [PATCH 16/35] IM rule default interval timeout and lookback - 1h (#115185) (#115518) * Make 1h default value for IM rule interval and lookback time * Fix test name * Move value to cosntants * Update lookback * Change lookback to 5 minutes Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Khristinin Nikita --- .../indicator_match_rule.spec.ts | 19 +++++++++++++++++++ .../cypress/screens/create_new_rule.ts | 6 ++++++ .../rules/step_schedule_rule/index.tsx | 19 +++++++++++++++---- .../detection_engine/rules/create/index.tsx | 1 + 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts index 871e50821b58c5..8735b8d49974c2 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts @@ -102,6 +102,12 @@ import { waitForAlertsToPopulate, waitForTheRuleToBeExecuted, } from '../../tasks/create_new_rule'; +import { + SCHEDULE_INTERVAL_AMOUNT_INPUT, + SCHEDULE_INTERVAL_UNITS_INPUT, + SCHEDULE_LOOKBACK_AMOUNT_INPUT, + SCHEDULE_LOOKBACK_UNITS_INPUT, +} from '../../screens/create_new_rule'; import { goBackToRuleDetails, waitForKibana } from '../../tasks/edit_rule'; import { esArchiverLoad, esArchiverUnload } from '../../tasks/es_archiver'; import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; @@ -383,6 +389,19 @@ describe('indicator match', () => { getIndicatorMappingComboField(2).should('not.exist'); }); }); + + describe('Schedule', () => { + it('IM rule has 1h time interval and lookback by default', () => { + selectIndicatorMatchType(); + fillDefineIndicatorMatchRuleAndContinue(getNewThreatIndicatorRule()); + fillAboutRuleAndContinue(getNewThreatIndicatorRule()); + + cy.get(SCHEDULE_INTERVAL_AMOUNT_INPUT).invoke('val').should('eql', '1'); + cy.get(SCHEDULE_INTERVAL_UNITS_INPUT).invoke('val').should('eql', 'h'); + cy.get(SCHEDULE_LOOKBACK_AMOUNT_INPUT).invoke('val').should('eql', '5'); + cy.get(SCHEDULE_LOOKBACK_UNITS_INPUT).invoke('val').should('eql', 'm'); + }); + }); }); describe('Generating signals', () => { diff --git a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts index 3510df6186870a..aadaa5dfa0d88a 100644 --- a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts @@ -201,6 +201,12 @@ export const SCHEDULE_INTERVAL_AMOUNT_INPUT = export const SCHEDULE_INTERVAL_UNITS_INPUT = '[data-test-subj="detectionEngineStepScheduleRuleInterval"] [data-test-subj="timeType"]'; +export const SCHEDULE_LOOKBACK_AMOUNT_INPUT = + '[data-test-subj="detectionEngineStepScheduleRuleFrom"] [data-test-subj="interval"]'; + +export const SCHEDULE_LOOKBACK_UNITS_INPUT = + '[data-test-subj="detectionEngineStepScheduleRuleFrom"] [data-test-subj="timeType"]'; + export const SEVERITY_DROPDOWN = '[data-test-subj="detectionEngineStepAboutRuleSeverity"] [data-test-subj="select"]'; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_schedule_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_schedule_rule/index.tsx index 528494c9331ace..9d7c2b76b385f4 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_schedule_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_schedule_rule/index.tsx @@ -6,6 +6,7 @@ */ import React, { FC, memo, useCallback, useEffect } from 'react'; +import { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { RuleStep, @@ -16,16 +17,25 @@ import { StepRuleDescription } from '../description_step'; import { ScheduleItem } from '../schedule_item_form'; import { Form, UseField, useForm } from '../../../../shared_imports'; import { StepContentWrapper } from '../step_content_wrapper'; +import { isThreatMatchRule } from '../../../../../common/detection_engine/utils'; import { NextStep } from '../next_step'; import { schema } from './schema'; interface StepScheduleRuleProps extends RuleStepProps { defaultValues?: ScheduleStepRule | null; + ruleType?: Type; } -const stepScheduleDefaultValue: ScheduleStepRule = { - interval: '5m', - from: '1m', +const DEFAULT_INTERVAL = '5m'; +const DEFAULT_FROM = '1m'; +const THREAT_MATCH_INTERVAL = '1h'; +const THREAT_MATCH_FROM = '5m'; + +const getStepScheduleDefaultValue = (ruleType: Type | undefined): ScheduleStepRule => { + return { + interval: isThreatMatchRule(ruleType) ? THREAT_MATCH_INTERVAL : DEFAULT_INTERVAL, + from: isThreatMatchRule(ruleType) ? THREAT_MATCH_FROM : DEFAULT_FROM, + }; }; const StepScheduleRuleComponent: FC = ({ @@ -37,8 +47,9 @@ const StepScheduleRuleComponent: FC = ({ isUpdateView = false, onSubmit, setForm, + ruleType, }) => { - const initialState = defaultValues ?? stepScheduleDefaultValue; + const initialState = defaultValues ?? getStepScheduleDefaultValue(ruleType); const { form } = useForm({ defaultValue: initialState, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx index a2f4385aeeb862..d37acaeb0ffee0 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/index.tsx @@ -401,6 +401,7 @@ const CreateRulePageComponent: React.FC = () => { > Date: Wed, 20 Oct 2021 15:21:06 -0400 Subject: [PATCH 17/35] [Osquery] Cypress automation for osquery manager integration (#108759) (#115777) Co-authored-by: Bryan Clement --- x-pack/plugins/osquery/cypress/.gitignore | 2 + x-pack/plugins/osquery/cypress/README.md | 12 +- .../integration/osquery_manager.spec.ts | 14 +- .../osquery/cypress/screens/integrations.ts | 1 + .../osquery/cypress/screens/live_query.ts | 11 + .../osquery/cypress/tasks/integrations.ts | 22 +- .../osquery/cypress/tasks/live_query.ts | 25 +++ .../osquery/cypress/tasks/navigation.ts | 10 +- .../public/live_queries/form/index.tsx | 1 + x-pack/test/osquery_cypress/agent.ts | 126 +++++++++++ .../test/osquery_cypress/artifact_manager.ts | 125 +++++++++++ x-pack/test/osquery_cypress/fleet_server.ts | 65 ++++++ x-pack/test/osquery_cypress/fleet_server.yml | 17 ++ .../test/osquery_cypress/resource_manager.ts | 24 ++ x-pack/test/osquery_cypress/runner.ts | 205 +++++++++++++----- x-pack/test/osquery_cypress/users.ts | 101 +++++++++ 16 files changed, 679 insertions(+), 82 deletions(-) create mode 100644 x-pack/plugins/osquery/cypress/.gitignore create mode 100644 x-pack/plugins/osquery/cypress/screens/live_query.ts create mode 100644 x-pack/plugins/osquery/cypress/tasks/live_query.ts create mode 100644 x-pack/test/osquery_cypress/agent.ts create mode 100644 x-pack/test/osquery_cypress/artifact_manager.ts create mode 100644 x-pack/test/osquery_cypress/fleet_server.ts create mode 100644 x-pack/test/osquery_cypress/fleet_server.yml create mode 100644 x-pack/test/osquery_cypress/resource_manager.ts create mode 100644 x-pack/test/osquery_cypress/users.ts diff --git a/x-pack/plugins/osquery/cypress/.gitignore b/x-pack/plugins/osquery/cypress/.gitignore new file mode 100644 index 00000000000000..adaba54810395d --- /dev/null +++ b/x-pack/plugins/osquery/cypress/.gitignore @@ -0,0 +1,2 @@ +videos +screenshots diff --git a/x-pack/plugins/osquery/cypress/README.md b/x-pack/plugins/osquery/cypress/README.md index 0df311ebc0a05b..72d558b6e59802 100644 --- a/x-pack/plugins/osquery/cypress/README.md +++ b/x-pack/plugins/osquery/cypress/README.md @@ -20,7 +20,7 @@ A headless browser is a browser simulation program that does not have a user int #### FTR (CI) -This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/security_solution_cypress` +This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/osquery_cypress` ### Test Execution: Examples @@ -36,7 +36,7 @@ yarn kbn bootstrap node scripts/build_kibana_platform_plugins # launch the cypress test runner -cd x-pack/plugins/security_solution +cd x-pack/plugins/osquery yarn cypress:run-as-ci ``` #### FTR + Interactive @@ -51,7 +51,7 @@ yarn kbn bootstrap node scripts/build_kibana_platform_plugins # launch the cypress test runner -cd x-pack/plugins/security_solution +cd x-pack/plugins/osquery yarn cypress:open-as-ci ``` @@ -98,16 +98,16 @@ We use es_archiver to manage the data that our Cypress tests need. 1. Set up a clean instance of kibana and elasticsearch (if this is not possible, try to clean/minimize the data that you are going to archive). 2. With the kibana and elasticsearch instance up and running, create the data that you need for your test. -3. When you are sure that you have all the data you need run the following command from: `x-pack/plugins/security_solution` +3. When you are sure that you have all the data you need run the following command from: `x-pack/plugins/osquery` ```sh -node ../../../scripts/es_archiver save --dir ../../test/security_solution_cypress/es_archives --config ../../../test/functional/config.js --es-url http://:@: +node ../../../scripts/es_archiver save --dir ../../test/osquery_cypress/es_archives --config ../../../test/functional/config.js --es-url http://:@: ``` Example: ```sh -node ../../../scripts/es_archiver save custom_rules ".kibana",".siem-signal*" --dir ../../test/security_solution_cypress/es_archives --config ../../../test/functional/config.js --es-url http://elastic:changeme@localhost:9220 +node ../../../scripts/es_archiver save custom_rules ".kibana",".siem-signal*" --dir ../../test/osquery_cypress/es_archives --config ../../../test/functional/config.js --es-url http://elastic:changeme@localhost:9220 ``` Note that the command will create the folder if it does not exist. diff --git a/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts b/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts index 0babfd2f10a8e6..367de59b3e1fcc 100644 --- a/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts +++ b/x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts @@ -8,13 +8,19 @@ import { HEADER } from '../screens/osquery'; import { OSQUERY_NAVIGATION_LINK } from '../screens/navigation'; -import { INTEGRATIONS, OSQUERY, openNavigationFlyout, navigateTo } from '../tasks/navigation'; +import { OSQUERY, NEW_LIVE_QUERY, openNavigationFlyout, navigateTo } from '../tasks/navigation'; import { addIntegration } from '../tasks/integrations'; +import { checkResults, inputQuery, selectAllAgents, submitQuery } from '../tasks/live_query'; describe('Osquery Manager', () => { - before(() => { - navigateTo(INTEGRATIONS); - addIntegration('Osquery Manager'); + before(() => addIntegration(Cypress.env('OSQUERY_POLICY'))); + + it('Runs live queries', () => { + navigateTo(NEW_LIVE_QUERY); + selectAllAgents(); + inputQuery(); + submitQuery(); + checkResults(); }); it('Displays Osquery on the navigation flyout once installed ', () => { diff --git a/x-pack/plugins/osquery/cypress/screens/integrations.ts b/x-pack/plugins/osquery/cypress/screens/integrations.ts index 0b29e857f46ee4..6a2951c85bf74f 100644 --- a/x-pack/plugins/osquery/cypress/screens/integrations.ts +++ b/x-pack/plugins/osquery/cypress/screens/integrations.ts @@ -8,3 +8,4 @@ export const ADD_POLICY_BTN = '[data-test-subj="addIntegrationPolicyButton"]'; export const CREATE_PACKAGE_POLICY_SAVE_BTN = '[data-test-subj="createPackagePolicySaveButton"]'; export const INTEGRATIONS_CARD = '.euiCard__titleAnchor'; +export const SAVE_PACKAGE_CONFIRM = '[data-test-subj=confirmModalConfirmButton]'; diff --git a/x-pack/plugins/osquery/cypress/screens/live_query.ts b/x-pack/plugins/osquery/cypress/screens/live_query.ts new file mode 100644 index 00000000000000..1a521fe1cd6512 --- /dev/null +++ b/x-pack/plugins/osquery/cypress/screens/live_query.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const AGENT_FIELD = '[data-test-subj="comboBoxInput"]'; +export const ALL_AGENTS_OPTION = '[title="All agents"]'; +export const LIVE_QUERY_EDITOR = '#osquery_editor'; +export const SUBMIT_BUTTON = '#submit-button'; diff --git a/x-pack/plugins/osquery/cypress/tasks/integrations.ts b/x-pack/plugins/osquery/cypress/tasks/integrations.ts index f85ef56550af50..4de42ffa95bb50 100644 --- a/x-pack/plugins/osquery/cypress/tasks/integrations.ts +++ b/x-pack/plugins/osquery/cypress/tasks/integrations.ts @@ -5,16 +5,18 @@ * 2.0. */ -import { - ADD_POLICY_BTN, - CREATE_PACKAGE_POLICY_SAVE_BTN, - INTEGRATIONS_CARD, -} from '../screens/integrations'; +import { CREATE_PACKAGE_POLICY_SAVE_BTN, SAVE_PACKAGE_CONFIRM } from '../screens/integrations'; -export const addIntegration = (integration: string) => { - cy.get(INTEGRATIONS_CARD).contains(integration).click(); - cy.get(ADD_POLICY_BTN).click(); +import { navigateTo, OSQUERY_INTEGRATION_PAGE } from './navigation'; + +// TODO: allow adding integration version strings to this +export const addIntegration = (policyId: string) => { + navigateTo(OSQUERY_INTEGRATION_PAGE, { qs: { policyId } }); cy.get(CREATE_PACKAGE_POLICY_SAVE_BTN).click(); - cy.get(CREATE_PACKAGE_POLICY_SAVE_BTN).should('not.exist'); - cy.reload(); + cy.get(SAVE_PACKAGE_CONFIRM).click(); + // XXX: there is a race condition between the test going to the ui powered by the agent, and the agent having the integration ready to go + // so we wait. + // TODO: actually make this wait til the agent has been updated with the proper integration + cy.wait(5000); + return cy.reload(); }; diff --git a/x-pack/plugins/osquery/cypress/tasks/live_query.ts b/x-pack/plugins/osquery/cypress/tasks/live_query.ts new file mode 100644 index 00000000000000..c2b97c885cad69 --- /dev/null +++ b/x-pack/plugins/osquery/cypress/tasks/live_query.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + AGENT_FIELD, + ALL_AGENTS_OPTION, + LIVE_QUERY_EDITOR, + SUBMIT_BUTTON, +} from '../screens/live_query'; + +export const selectAllAgents = () => { + cy.get(AGENT_FIELD).first().click(); + return cy.get(ALL_AGENTS_OPTION).contains('All agents').click(); +}; + +export const inputQuery = () => cy.get(LIVE_QUERY_EDITOR).type('select * from processes;'); + +export const submitQuery = () => cy.get(SUBMIT_BUTTON).contains('Submit').click(); + +export const checkResults = () => + cy.get('[data-test-subj="dataGridRowCell"]').should('have.lengthOf.above', 0); diff --git a/x-pack/plugins/osquery/cypress/tasks/navigation.ts b/x-pack/plugins/osquery/cypress/tasks/navigation.ts index 63d6b205b433bf..7528f318a2fa4b 100644 --- a/x-pack/plugins/osquery/cypress/tasks/navigation.ts +++ b/x-pack/plugins/osquery/cypress/tasks/navigation.ts @@ -7,11 +7,13 @@ import { TOGGLE_NAVIGATION_BTN } from '../screens/navigation'; -export const INTEGRATIONS = 'app/integrations#/'; export const OSQUERY = 'app/osquery/live_queries'; - -export const navigateTo = (page: string) => { - cy.visit(page); +export const NEW_LIVE_QUERY = 'app/osquery/live_queries/new'; +export const OSQUERY_INTEGRATION_PAGE = '/app/fleet/integrations/osquery_manager/add-integration'; +export const navigateTo = (page: string, opts?: Partial) => { + cy.visit(page, opts); + // There's a security warning toast that seemingly makes ui elements in the bottom right unavailable, so we close it + return cy.get('[data-test-subj="toastCloseButton"]').click(); }; export const openNavigationFlyout = () => { diff --git a/x-pack/plugins/osquery/public/live_queries/form/index.tsx b/x-pack/plugins/osquery/public/live_queries/form/index.tsx index 86323b7ab43150..6d13c76d9d5928 100644 --- a/x-pack/plugins/osquery/public/live_queries/form/index.tsx +++ b/x-pack/plugins/osquery/public/live_queries/form/index.tsx @@ -328,6 +328,7 @@ const LiveQueryFormComponent: React.FC = ({ )} diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts new file mode 100644 index 00000000000000..802a7caa66d5f6 --- /dev/null +++ b/x-pack/test/osquery_cypress/agent.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/dev-utils'; +import axios, { AxiosRequestConfig } from 'axios'; +import { copyFile } from 'fs/promises'; +import { ChildProcess, execFileSync, spawn } from 'child_process'; +import { resolve } from 'path'; +import { unlinkSync } from 'fs'; +import { Manager } from './resource_manager'; + +interface AgentManagerParams { + user: string; + password: string; + kibanaUrl: string; + esHost: string; +} + +export class AgentManager extends Manager { + private directoryPath: string; + private params: AgentManagerParams; + private log: ToolingLog; + private agentProcess?: ChildProcess; + private requestOptions: AxiosRequestConfig; + constructor(directoryPath: string, params: AgentManagerParams, log: ToolingLog) { + super(); + // TODO: check if the file exists + this.directoryPath = directoryPath; + this.log = log; + this.params = params; + this.requestOptions = { + headers: { + 'kbn-xsrf': 'kibana', + }, + auth: { + username: this.params.user, + password: this.params.password, + }, + }; + } + + public getBinaryPath() { + return resolve(this.directoryPath, 'elastic-agent'); + } + + public async setup() { + this.log.info('Running agent preconfig'); + await axios.post(`${this.params.kibanaUrl}/api/fleet/agents/setup`, {}, this.requestOptions); + + this.log.info('Updating the default agent output'); + const { + data: { + items: [defaultOutput], + }, + } = await axios.get(this.params.kibanaUrl + '/api/fleet/outputs', this.requestOptions); + + await axios.put( + `${this.params.kibanaUrl}/api/fleet/outputs/${defaultOutput.id}`, + { hosts: [this.params.esHost] }, + this.requestOptions + ); + + this.log.info('Getting agent enrollment key'); + const { data: apiKeys } = await axios.get( + this.params.kibanaUrl + '/api/fleet/enrollment-api-keys', + this.requestOptions + ); + const policy = apiKeys.list[1]; + + this.log.info('Enrolling the agent'); + const args = [ + 'enroll', + '--insecure', + '-f', + // TODO: parse the host/port out of the logs for the fleet server + '--url=http://localhost:8220', + `--enrollment-token=${policy.api_key}`, + ]; + const agentBinPath = this.getBinaryPath(); + execFileSync(agentBinPath, args, { stdio: 'inherit' }); + + // Copy the config file + const configPath = resolve(__dirname, this.directoryPath, 'elastic-agent.yml'); + this.log.info(`Copying agent config from ${configPath}`); + await copyFile(configPath, resolve('.', 'elastic-agent.yml')); + + this.log.info('Running the agent'); + this.agentProcess = spawn(agentBinPath, ['run', '-v'], { stdio: 'inherit' }); + + // Wait til we see the agent is online + let done = false; + let retries = 0; + while (!done) { + await new Promise((r) => setTimeout(r, 5000)); + const { data: agents } = await axios.get( + `${this.params.kibanaUrl}/api/fleet/agents`, + this.requestOptions + ); + done = agents.list[0]?.status === 'online'; + if (++retries > 12) { + this.log.error('Giving up on enrolling the agent after a minute'); + throw new Error('Agent timed out while coming online'); + } + } + return { policyId: policy.policy_id as string }; + } + + protected _cleanup() { + this.log.info('Cleaning up the agent process'); + if (this.agentProcess) { + if (!this.agentProcess.kill(9)) { + this.log.warning('Unable to kill agent process'); + } + + this.agentProcess.on('close', () => { + this.log.info('Agent process closed'); + }); + delete this.agentProcess; + } + unlinkSync(resolve('.', 'elastic-agent.yml')); + } +} diff --git a/x-pack/test/osquery_cypress/artifact_manager.ts b/x-pack/test/osquery_cypress/artifact_manager.ts new file mode 100644 index 00000000000000..89d6f349870073 --- /dev/null +++ b/x-pack/test/osquery_cypress/artifact_manager.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import axios, { AxiosResponse } from 'axios'; +import { get } from 'lodash'; +import { execSync } from 'child_process'; +import { writeFileSync, unlinkSync, rmdirSync } from 'fs'; +import { resolve } from 'path'; +import { ToolingLog } from '@kbn/dev-utils'; +import { Manager } from './resource_manager'; + +const archMap: { [key: string]: string } = { + x64: 'x86_64', +}; + +type ArtifactName = 'elastic-agent' | 'fleet-server'; + +async function getArtifact( + artifact: string, + urlExtractor: (data: AxiosResponse, filename: string) => string, + log: ToolingLog, + version: string +) { + log.info(`Fetching ${version} of ${artifact}`); + const agents = await axios( + `https://artifacts-api.elastic.co/v1/versions/${version}/builds/latest` + ); + const arch = archMap[process.arch] ?? process.arch; + const dirName = `${artifact}-${version}-${process.platform}-${arch}`; + const filename = dirName + '.tar.gz'; + const url = urlExtractor(agents.data, filename); + if (!url) { + log.error(`Could not find url for ${artifact}: ${url}`); + throw new Error(`Unable to fetch ${artifact}`); + } + log.info(`Fetching ${filename} from ${url}`); + const agent = await axios(url as string, { responseType: 'arraybuffer' }); + writeFileSync(filename, agent.data); + execSync(`tar xvf ${filename}`); + return resolve(filename); +} + +// There has to be a better way to represent partial function application +type ArtifactFetcher = ( + log: Parameters[2], + version: Parameters[3] +) => ReturnType; +type ArtifactFetchers = { + [artifactName in ArtifactName]: ArtifactFetcher; +}; + +const fetchers: ArtifactFetchers = { + 'elastic-agent': getArtifact.bind(null, 'elastic-agent', (data, filename) => + get(data, ['build', 'projects', 'beats', 'packages', filename, 'url']) + ), + 'fleet-server': getArtifact.bind(null, 'fleet-server', (data, filename) => + get(data, ['build', 'projects', 'fleet-server', 'packages', filename, 'url']) + ), +}; + +export type FetchArtifactsParams = { + [artifactName in ArtifactName]?: string; +}; + +type ArtifactPaths = FetchArtifactsParams; +export class ArtifactManager extends Manager { + private artifacts: ArtifactPaths; + private versions: FetchArtifactsParams; + private log: ToolingLog; + + constructor(versions: FetchArtifactsParams, log: ToolingLog) { + super(); + this.versions = versions; + this.log = log; + this.artifacts = {}; + } + + public fetchArtifacts = async () => { + this.log.info('Fetching artifacts'); + await Promise.all( + Object.keys(this.versions).map(async (name: string) => { + const artifactName = name as ArtifactName; + const version = this.versions[artifactName]; + if (!version) { + this.log.warning(`No version is specified for ${artifactName}, skipping`); + return; + } + const fetcher = fetchers[artifactName]; + if (!fetcher) { + this.log.warning(`No fetcher is defined for ${artifactName}, skipping`); + } + + this.artifacts[artifactName] = await fetcher(this.log, version); + }) + ); + }; + + public getArtifactDirectory(artifactName: string) { + const file = this.artifacts[artifactName as ArtifactName]; + // this will break if the tarball name diverges from the directory that gets untarred + if (!file) { + throw new Error(`Unknown artifact ${artifactName}, unable to retreive directory`); + } + return file.replace('.tar.gz', ''); + } + + protected _cleanup() { + this.log.info('Cleaning up artifacts'); + if (this.artifacts) { + for (const artifactName of Object.keys(this.artifacts)) { + const file = this.artifacts[artifactName as ArtifactName]; + if (!file) { + this.log.warning(`Unknown artifact ${artifactName} encountered during cleanup, skipping`); + continue; + } + unlinkSync(file); + rmdirSync(this.getArtifactDirectory(artifactName), { recursive: true }); + } + } + } +} diff --git a/x-pack/test/osquery_cypress/fleet_server.ts b/x-pack/test/osquery_cypress/fleet_server.ts new file mode 100644 index 00000000000000..3c520233bc9b01 --- /dev/null +++ b/x-pack/test/osquery_cypress/fleet_server.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChildProcess, spawn } from 'child_process'; +import { copyFile } from 'fs/promises'; +import { unlinkSync } from 'fs'; +import { resolve } from 'path'; +import { ToolingLog } from '@kbn/dev-utils'; +import { Manager } from './resource_manager'; +export interface ElasticsearchConfig { + esHost: string; + user: string; + password: string; +} + +export class FleetManager extends Manager { + private directoryPath: string; + private fleetProcess?: ChildProcess; + private esConfig: ElasticsearchConfig; + private log: ToolingLog; + constructor(directoryPath: string, esConfig: ElasticsearchConfig, log: ToolingLog) { + super(); + // TODO: check if the file exists + this.esConfig = esConfig; + this.directoryPath = directoryPath; + this.log = log; + } + public async setup(): Promise { + this.log.info('Setting fleet up'); + await copyFile(resolve(__dirname, 'fleet_server.yml'), resolve('.', 'fleet-server.yml')); + return new Promise((res, rej) => { + const env = { + ELASTICSEARCH_HOSTS: this.esConfig.esHost, + ELASTICSEARCH_USERNAME: this.esConfig.user, + ELASTICSEARCH_PASSWORD: this.esConfig.password, + }; + const file = resolve(this.directoryPath, 'fleet-server'); + // TODO: handle logging properly + this.fleetProcess = spawn(file, [], { stdio: 'inherit', env }); + this.fleetProcess.on('error', rej); + // TODO: actually wait for the fleet server to start listening + setTimeout(res, 15000); + }); + } + + protected _cleanup() { + this.log.info('Removing old fleet config'); + if (this.fleetProcess) { + this.log.info('Closing fleet process'); + if (!this.fleetProcess.kill(9)) { + this.log.warning('Unable to kill fleet server process'); + } + + this.fleetProcess.on('close', () => { + this.log.info('Fleet server process closed'); + }); + delete this.fleetProcess; + } + unlinkSync(resolve('.', 'fleet-server.yml')); + } +} diff --git a/x-pack/test/osquery_cypress/fleet_server.yml b/x-pack/test/osquery_cypress/fleet_server.yml new file mode 100644 index 00000000000000..70ac62b018a25e --- /dev/null +++ b/x-pack/test/osquery_cypress/fleet_server.yml @@ -0,0 +1,17 @@ +# mostly a stub config +output: + elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:localhost:9220}' + username: '${ELASTICSEARCH_USERNAME:elastic}' + password: '${ELASTICSEARCH_PASSWORD:changeme}' + +fleet: + agent: + id: 1e4954ce-af37-4731-9f4a-407b08e69e42 + logging: + level: '${LOG_LEVEL:DEBUG}' + +logging: + to_stderr: true + +http.enabled: true diff --git a/x-pack/test/osquery_cypress/resource_manager.ts b/x-pack/test/osquery_cypress/resource_manager.ts new file mode 100644 index 00000000000000..e892021155417d --- /dev/null +++ b/x-pack/test/osquery_cypress/resource_manager.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const CLEANUP_EVENTS = ['SIGINT', 'exit', 'uncaughtException', 'unhandledRejection']; +export class Manager { + private cleaned = false; + constructor() { + const cleanup = () => this.cleanup(); + CLEANUP_EVENTS.forEach((ev) => process.on(ev, cleanup)); + } + // This must be a synchronous method because it is used in the unhandledException and exit event handlers + public cleanup() { + // Since this can be called multiple places we proxy it with some protection + if (this._cleanup && !this.cleaned) { + this.cleaned = true; + this._cleanup(); + } + } + protected _cleanup?(): void; +} diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts index 32c84af5faf76d..bd0a97ad5feacb 100644 --- a/x-pack/test/osquery_cypress/runner.ts +++ b/x-pack/test/osquery_cypress/runner.ts @@ -12,70 +12,159 @@ import { withProcRunner } from '@kbn/dev-utils'; import { FtrProviderContext } from './ftr_provider_context'; -export async function OsqueryCypressCliTestRunner({ getService }: FtrProviderContext) { +import { ArtifactManager, FetchArtifactsParams } from './artifact_manager'; +import { setupUsers } from './users'; +import { AgentManager } from './agent'; +import { FleetManager } from './fleet_server'; + +interface SetupParams { + artifacts: FetchArtifactsParams; +} + +async function withFleetAgent( + { getService }: FtrProviderContext, + params: SetupParams, + runner: (runnerEnv: Record) => Promise +) { const log = getService('log'); const config = getService('config'); - await withProcRunner(log, async (procs) => { - await procs.run('cypress', { - cmd: 'yarn', - args: ['cypress:run'], - cwd: resolve(__dirname, '../../plugins/osquery'), - env: { - FORCE_COLOR: '1', - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_baseUrl: Url.format(config.get('servers.kibana')), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_protocol: config.get('servers.kibana.protocol'), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_hostname: config.get('servers.kibana.hostname'), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_configport: config.get('servers.kibana.port'), - CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')), - CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'), - CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'), - CYPRESS_KIBANA_URL: Url.format({ - protocol: config.get('servers.kibana.protocol'), - hostname: config.get('servers.kibana.hostname'), - port: config.get('servers.kibana.port'), - }), - ...process.env, - }, - wait: true, - }); + const artifactManager = new ArtifactManager(params.artifacts, log); + await artifactManager.fetchArtifacts(); + + const esHost = Url.format(config.get('servers.elasticsearch')); + const esConfig = { + user: config.get('servers.elasticsearch.username'), + password: config.get('servers.elasticsearch.password'), + esHost, + }; + const fleetManager = new FleetManager( + artifactManager.getArtifactDirectory('fleet-server'), + esConfig, + log + ); + + const agentManager = new AgentManager( + artifactManager.getArtifactDirectory('elastic-agent'), + { + ...esConfig, + kibanaUrl: Url.format({ + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + port: config.get('servers.kibana.port'), + }), + }, + log + ); + + // Since the managers will create uncaughtException event handlers we need to exit manually + process.on('uncaughtException', (err) => { + // eslint-disable-next-line no-console + console.error('Encountered error; exiting after cleanup.', err); + process.exit(1); }); + + await fleetManager.setup(); + const { policyId } = await agentManager.setup(); + await setupUsers(esConfig); + try { + await runner({ + CYPRESS_OSQUERY_POLICY: policyId, + }); + } finally { + fleetManager.cleanup(); + agentManager.cleanup(); + artifactManager.cleanup(); + } } -export async function OsqueryCypressVisualTestRunner({ getService }: FtrProviderContext) { - const log = getService('log'); - const config = getService('config'); +export async function OsqueryCypressCliTestRunner(context: FtrProviderContext) { + const log = context.getService('log'); + const config = context.getService('config'); + await withFleetAgent( + context, + { + artifacts: { + 'elastic-agent': '7.15.0-SNAPSHOT', + 'fleet-server': '7.15.0-SNAPSHOT', + }, + }, + (runnerEnv) => + withProcRunner(log, async (procs) => { + await procs.run('cypress', { + cmd: 'yarn', + args: ['cypress:run'], + cwd: resolve(__dirname, '../../plugins/osquery'), + env: { + FORCE_COLOR: '1', + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_baseUrl: Url.format(config.get('servers.kibana')), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_protocol: config.get('servers.kibana.protocol'), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_hostname: config.get('servers.kibana.hostname'), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_configport: config.get('servers.kibana.port'), + CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')), + CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'), + CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'), + CYPRESS_KIBANA_URL: Url.format({ + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + port: config.get('servers.kibana.port'), + }), + ...runnerEnv, + ...process.env, + }, + wait: true, + }); + }) + ); +} + +export async function OsqueryCypressVisualTestRunner(context: FtrProviderContext) { + const log = context.getService('log'); + const config = context.getService('config'); - await withProcRunner(log, async (procs) => { - await procs.run('cypress', { - cmd: 'yarn', - args: ['cypress:open'], - cwd: resolve(__dirname, '../../plugins/osquery'), - env: { - FORCE_COLOR: '1', - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_baseUrl: Url.format(config.get('servers.kibana')), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_protocol: config.get('servers.kibana.protocol'), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_hostname: config.get('servers.kibana.hostname'), - // eslint-disable-next-line @typescript-eslint/naming-convention - CYPRESS_configport: config.get('servers.kibana.port'), - CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')), - CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'), - CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'), - CYPRESS_KIBANA_URL: Url.format({ - protocol: config.get('servers.kibana.protocol'), - hostname: config.get('servers.kibana.hostname'), - port: config.get('servers.kibana.port'), - }), - ...process.env, + await withFleetAgent( + context, + { + artifacts: { + 'elastic-agent': '7.15.0-SNAPSHOT', + 'fleet-server': '7.15.0-SNAPSHOT', }, - wait: true, - }); - }); + }, + (runnerEnv) => + withProcRunner( + log, + async (procs) => + await procs.run('cypress', { + cmd: 'yarn', + args: ['cypress:open'], + cwd: resolve(__dirname, '../../plugins/osquery'), + env: { + FORCE_COLOR: '1', + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_baseUrl: Url.format(config.get('servers.kibana')), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_protocol: config.get('servers.kibana.protocol'), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_hostname: config.get('servers.kibana.hostname'), + // eslint-disable-next-line @typescript-eslint/naming-convention + CYPRESS_configport: config.get('servers.kibana.port'), + CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')), + CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'), + CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'), + CYPRESS_KIBANA_URL: Url.format({ + protocol: config.get('servers.kibana.protocol'), + hostname: config.get('servers.kibana.hostname'), + port: config.get('servers.kibana.port'), + }), + ...runnerEnv, + ...process.env, + }, + wait: true, + }) + ) + ); } diff --git a/x-pack/test/osquery_cypress/users.ts b/x-pack/test/osquery_cypress/users.ts new file mode 100644 index 00000000000000..bfe8abfd8452fe --- /dev/null +++ b/x-pack/test/osquery_cypress/users.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import axios from 'axios'; +import { ElasticsearchConfig } from './fleet_server'; + +interface Roles { + [roleName: string]: { + indices: [ + { + names: string[]; + privileges: string[]; + allow_restricted_indices: boolean; + } + ]; + applications: [ + { + application: string; + privileges: string[]; + resources: string[]; + } + ]; + transient_metadata: { + enabled: boolean; + }; + }; +} + +interface Users { + [username: string]: { roles: string[] }; +} + +export const ROLES: Roles = { + read: { + indices: [ + { + names: ['logs-*'], + privileges: ['read'], + allow_restricted_indices: false, + }, + ], + applications: [ + { + application: 'kibana-.kibana', + privileges: ['feature_osquery.read'], + resources: ['*'], + }, + ], + transient_metadata: { + enabled: true, + }, + }, + all: { + indices: [ + { + names: ['logs-*'], + privileges: ['read'], + allow_restricted_indices: false, + }, + ], + applications: [ + { + application: 'kibana-.kibana', + privileges: ['feature_osquery.all'], + resources: ['*'], + }, + ], + transient_metadata: { + enabled: true, + }, + }, +}; + +export const USERS: Users = { + osqueryRead: { + roles: ['osquery_read'], + }, + osqueryAll: { + roles: ['osquery_all'], + }, +}; + +export const setupUsers = async (config: ElasticsearchConfig) => { + const { esHost, user: username, password } = config; + const params = { + auth: { username, password }, + }; + await Promise.all( + Object.keys(ROLES).map((role) => + axios.put(`${esHost}/_security/role/osquery_${role}`, ROLES[role], params) + ) + ); + await Promise.all( + Object.keys(USERS).map((newUsername) => + axios.put(`${esHost}/_security/user/${newUsername}`, { password, ...USERS[username] }, params) + ) + ); +}; From f7dec63a1b64e298186d71b4fa8a8e1377cb6f8f Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 20 Oct 2021 12:31:02 -0700 Subject: [PATCH 18/35] skip flaky suite (#115473) --- .../public/management/users/edit_user/edit_user_page.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security/public/management/users/edit_user/edit_user_page.test.tsx b/x-pack/plugins/security/public/management/users/edit_user/edit_user_page.test.tsx index fb01ea0e618651..5b5e74cb2c618f 100644 --- a/x-pack/plugins/security/public/management/users/edit_user/edit_user_page.test.tsx +++ b/x-pack/plugins/security/public/management/users/edit_user/edit_user_page.test.tsx @@ -34,7 +34,8 @@ const userMock = { roles: ['superuser'], }; -describe('EditUserPage', () => { +// Failing: See https://github.com/elastic/kibana/issues/115473 +describe.skip('EditUserPage', () => { it('warns when viewing deactivated user', async () => { const coreStart = coreMock.createStart(); const history = createMemoryHistory({ initialEntries: ['/edit/jdoe'] }); From 8374a7e53f99ff485263c9c254f5b41576505ba1 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Wed, 20 Oct 2021 23:00:28 +0300 Subject: [PATCH 19/35] [7.16] [Telemetry] Switch to v3 endpoint (#113525) (#115815) * [Telemetry] Switch to v3 endpoint (#113525) # Conflicts: # x-pack/test/functional/apps/infra/logs_source_configuration.ts * push uncommited file * telemetry config * update test --- src/plugins/telemetry/common/constants.ts | 33 ++++++--- .../get_telemetry_channel_endpoint.test.ts | 69 +++++++++++++++--- .../get_telemetry_channel_endpoint.ts | 51 ++++++++----- .../common/telemetry_config/index.ts | 6 +- src/plugins/telemetry/common/types.ts | 10 +++ .../public/services/telemetry_sender.test.ts | 28 +++++--- .../public/services/telemetry_sender.ts | 20 ++++-- .../public/services/telemetry_service.test.ts | 45 +++++++----- .../public/services/telemetry_service.ts | 51 ++++++++----- src/plugins/telemetry/server/config/config.ts | 10 +-- .../server/config/deprecations.test.ts | 8 +-- src/plugins/telemetry/server/fetcher.test.ts | 11 +-- src/plugins/telemetry/server/fetcher.ts | 36 ++++++---- src/plugins/telemetry/server/plugin.ts | 5 +- .../routes/telemetry_opt_in_stats.test.ts | 47 ++++++++---- .../server/routes/telemetry_opt_in_stats.ts | 29 +++++--- .../server/encryption/encrypt.ts | 8 +-- .../server/plugin.test.ts | 36 ++++++++-- .../server/plugin.ts | 72 +++++++++++++------ .../server/types.ts | 5 ++ .../apis/telemetry/telemetry.ts | 16 +++-- .../apis/telemetry/telemetry_local.ts | 2 +- .../apis/fleet_telemetry.ts | 2 +- .../tagging_api/apis/usage_collection.ts | 4 +- 24 files changed, 425 insertions(+), 179 deletions(-) create mode 100644 src/plugins/telemetry/common/types.ts diff --git a/src/plugins/telemetry/common/constants.ts b/src/plugins/telemetry/common/constants.ts index f6b99badca492c..7da944c7a1556f 100644 --- a/src/plugins/telemetry/common/constants.ts +++ b/src/plugins/telemetry/common/constants.ts @@ -45,21 +45,32 @@ export const PATH_TO_ADVANCED_SETTINGS = '/app/management/kibana/settings'; */ export const PRIVACY_STATEMENT_URL = `https://www.elastic.co/legal/privacy-statement`; +/** + * The telemetry payload content encryption encoding + */ +export const PAYLOAD_CONTENT_ENCODING = 'aes256gcm'; + /** * The endpoint version when hitting the remote telemetry service */ -export const ENDPOINT_VERSION = 'v2'; +export const ENDPOINT_VERSION = 'v3'; + +/** + * The staging telemetry endpoint for the remote telemetry service. + */ + +export const ENDPOINT_STAGING = 'https://telemetry-staging.elastic.co/'; + +/** + * The production telemetry endpoint for the remote telemetry service. + */ + +export const ENDPOINT_PROD = 'https://telemetry.elastic.co/'; /** - * The telemetry endpoints for the remote telemetry service. + * The telemetry channels for the remote telemetry service. */ -export const TELEMETRY_ENDPOINT = { - MAIN_CHANNEL: { - PROD: `https://telemetry.elastic.co/xpack/${ENDPOINT_VERSION}/send`, - STAGING: `https://telemetry-staging.elastic.co/xpack/${ENDPOINT_VERSION}/send`, - }, - OPT_IN_STATUS_CHANNEL: { - PROD: `https://telemetry.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`, - STAGING: `https://telemetry-staging.elastic.co/opt_in_status/${ENDPOINT_VERSION}/send`, - }, +export const TELEMETRY_CHANNELS = { + SNAPSHOT_CHANNEL: 'kibana-snapshot', + OPT_IN_STATUS_CHANNEL: 'kibana-opt_in_status', }; diff --git a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts index 74d45f6a9f7d40..c7f9984269581b 100644 --- a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts +++ b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.test.ts @@ -6,14 +6,55 @@ * Side Public License, v 1. */ -import { getTelemetryChannelEndpoint } from './get_telemetry_channel_endpoint'; -import { TELEMETRY_ENDPOINT } from '../constants'; +import { + getTelemetryChannelEndpoint, + getChannel, + getBaseUrl, +} from './get_telemetry_channel_endpoint'; + +describe('getBaseUrl', () => { + it('throws on unknown env', () => { + expect(() => + // @ts-expect-error + getBaseUrl('ANY') + ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry endpoint env ANY."`); + }); + + it('returns correct prod base url', () => { + const baseUrl = getBaseUrl('prod'); + expect(baseUrl).toMatchInlineSnapshot(`"https://telemetry.elastic.co/"`); + }); + + it('returns correct staging base url', () => { + const baseUrl = getBaseUrl('staging'); + expect(baseUrl).toMatchInlineSnapshot(`"https://telemetry-staging.elastic.co/"`); + }); +}); + +describe('getChannel', () => { + it('throws on unknown channel', () => { + expect(() => + // @ts-expect-error + getChannel('ANY') + ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry channel ANY."`); + }); + + it('returns correct snapshot channel name', () => { + const channelName = getChannel('snapshot'); + expect(channelName).toMatchInlineSnapshot(`"kibana-snapshot"`); + }); + + it('returns correct optInStatus channel name', () => { + const channelName = getChannel('optInStatus'); + expect(channelName).toMatchInlineSnapshot(`"kibana-opt_in_status"`); + }); +}); describe('getTelemetryChannelEndpoint', () => { it('throws on unknown env', () => { expect(() => // @ts-expect-error - getTelemetryChannelEndpoint({ env: 'ANY', channelName: 'main' }) + getTelemetryChannelEndpoint({ env: 'ANY', channelName: 'snapshot' }) ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry endpoint env ANY."`); }); @@ -24,25 +65,33 @@ describe('getTelemetryChannelEndpoint', () => { ).toThrowErrorMatchingInlineSnapshot(`"Unknown telemetry channel ANY."`); }); - describe('main channel', () => { + describe('snapshot channel', () => { it('returns correct prod endpoint', () => { - const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'main' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD); + const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'snapshot' }); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-snapshot"` + ); }); it('returns correct staging endpoint', () => { - const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'main' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING); + const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'snapshot' }); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-snapshot"` + ); }); }); describe('optInStatus channel', () => { it('returns correct prod endpoint', () => { const endpoint = getTelemetryChannelEndpoint({ env: 'prod', channelName: 'optInStatus' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-opt_in_status"` + ); }); it('returns correct staging endpoint', () => { const endpoint = getTelemetryChannelEndpoint({ env: 'staging', channelName: 'optInStatus' }); - expect(endpoint).toBe(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING); + expect(endpoint).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status"` + ); }); }); }); diff --git a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts index a0af7878afef64..75d83611b8c8d8 100644 --- a/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts +++ b/src/plugins/telemetry/common/telemetry_config/get_telemetry_channel_endpoint.ts @@ -6,29 +6,48 @@ * Side Public License, v 1. */ -import { TELEMETRY_ENDPOINT } from '../constants'; +import { + ENDPOINT_VERSION, + ENDPOINT_STAGING, + ENDPOINT_PROD, + TELEMETRY_CHANNELS, +} from '../constants'; +export type ChannelName = 'snapshot' | 'optInStatus'; +export type TelemetryEnv = 'staging' | 'prod'; export interface GetTelemetryChannelEndpointConfig { - channelName: 'main' | 'optInStatus'; - env: 'staging' | 'prod'; + channelName: ChannelName; + env: TelemetryEnv; } -export function getTelemetryChannelEndpoint({ - channelName, - env, -}: GetTelemetryChannelEndpointConfig): string { - if (env !== 'staging' && env !== 'prod') { - throw new Error(`Unknown telemetry endpoint env ${env}.`); - } - - const endpointEnv = env === 'staging' ? 'STAGING' : 'PROD'; - +export function getChannel(channelName: ChannelName): string { switch (channelName) { - case 'main': - return TELEMETRY_ENDPOINT.MAIN_CHANNEL[endpointEnv]; + case 'snapshot': + return TELEMETRY_CHANNELS.SNAPSHOT_CHANNEL; case 'optInStatus': - return TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL[endpointEnv]; + return TELEMETRY_CHANNELS.OPT_IN_STATUS_CHANNEL; default: throw new Error(`Unknown telemetry channel ${channelName}.`); } } + +export function getBaseUrl(env: TelemetryEnv): string { + switch (env) { + case 'prod': + return ENDPOINT_PROD; + case 'staging': + return ENDPOINT_STAGING; + default: + throw new Error(`Unknown telemetry endpoint env ${env}.`); + } +} + +export function getTelemetryChannelEndpoint({ + channelName, + env, +}: GetTelemetryChannelEndpointConfig): string { + const baseUrl = getBaseUrl(env); + const channelPath = getChannel(channelName); + + return `${baseUrl}${ENDPOINT_VERSION}/send/${channelPath}`; +} diff --git a/src/plugins/telemetry/common/telemetry_config/index.ts b/src/plugins/telemetry/common/telemetry_config/index.ts index eb268639cad919..b15475280fe853 100644 --- a/src/plugins/telemetry/common/telemetry_config/index.ts +++ b/src/plugins/telemetry/common/telemetry_config/index.ts @@ -12,4 +12,8 @@ export { getTelemetryAllowChangingOptInStatus } from './get_telemetry_allow_chan export { getTelemetryFailureDetails } from './get_telemetry_failure_details'; export type { TelemetryFailureDetails } from './get_telemetry_failure_details'; export { getTelemetryChannelEndpoint } from './get_telemetry_channel_endpoint'; -export type { GetTelemetryChannelEndpointConfig } from './get_telemetry_channel_endpoint'; +export type { + GetTelemetryChannelEndpointConfig, + ChannelName, + TelemetryEnv, +} from './get_telemetry_channel_endpoint'; diff --git a/src/plugins/telemetry/common/types.ts b/src/plugins/telemetry/common/types.ts new file mode 100644 index 00000000000000..aefbbd2358861f --- /dev/null +++ b/src/plugins/telemetry/common/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export type EncryptedTelemetryPayload = Array<{ clusterUuid: string; stats: string }>; +export type UnencryptedTelemetryPayload = Array<{ clusterUuid: string; stats: object }>; diff --git a/src/plugins/telemetry/public/services/telemetry_sender.test.ts b/src/plugins/telemetry/public/services/telemetry_sender.test.ts index 50738b11e508d4..10da46fe2761d6 100644 --- a/src/plugins/telemetry/public/services/telemetry_sender.test.ts +++ b/src/plugins/telemetry/public/services/telemetry_sender.test.ts @@ -171,8 +171,11 @@ describe('TelemetrySender', () => { }); it('sends report if due', async () => { + const mockClusterUuid = 'mk_uuid'; const mockTelemetryUrl = 'telemetry_cluster_url'; - const mockTelemetryPayload = ['hashed_cluster_usage_data1']; + const mockTelemetryPayload = [ + { clusterUuid: mockClusterUuid, stats: 'hashed_cluster_usage_data1' }, + ]; const telemetryService = mockTelemetryService(); const telemetrySender = new TelemetrySender(telemetryService); @@ -184,14 +187,21 @@ describe('TelemetrySender', () => { expect(telemetryService.fetchTelemetry).toBeCalledTimes(1); expect(mockFetch).toBeCalledTimes(1); - expect(mockFetch).toBeCalledWith(mockTelemetryUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': telemetryService.currentKibanaVersion, - }, - body: mockTelemetryPayload[0], - }); + expect(mockFetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "telemetry_cluster_url", + Object { + "body": "hashed_cluster_usage_data1", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mockKibanaVersion", + }, + "method": "POST", + }, + ] + `); }); it('sends report separately for every cluster', async () => { diff --git a/src/plugins/telemetry/public/services/telemetry_sender.ts b/src/plugins/telemetry/public/services/telemetry_sender.ts index fa973344951222..87287a420e7251 100644 --- a/src/plugins/telemetry/public/services/telemetry_sender.ts +++ b/src/plugins/telemetry/public/services/telemetry_sender.ts @@ -6,9 +6,14 @@ * Side Public License, v 1. */ -import { REPORT_INTERVAL_MS, LOCALSTORAGE_KEY } from '../../common/constants'; +import { + REPORT_INTERVAL_MS, + LOCALSTORAGE_KEY, + PAYLOAD_CONTENT_ENCODING, +} from '../../common/constants'; import { TelemetryService } from './telemetry_service'; import { Storage } from '../../../kibana_utils/public'; +import type { EncryptedTelemetryPayload } from '../../common/types'; export class TelemetrySender { private readonly telemetryService: TelemetryService; @@ -57,18 +62,21 @@ export class TelemetrySender { this.isSending = true; try { const telemetryUrl = this.telemetryService.getTelemetryUrl(); - const telemetryData: string | string[] = await this.telemetryService.fetchTelemetry(); - const clusters: string[] = ([] as string[]).concat(telemetryData); + const telemetryPayload: EncryptedTelemetryPayload = + await this.telemetryService.fetchTelemetry(); + await Promise.all( - clusters.map( - async (cluster) => + telemetryPayload.map( + async ({ clusterUuid, stats }) => await fetch(telemetryUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Elastic-Stack-Version': this.telemetryService.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, }, - body: cluster, + body: stats, }) ) ); diff --git a/src/plugins/telemetry/public/services/telemetry_service.test.ts b/src/plugins/telemetry/public/services/telemetry_service.test.ts index b23ba127c1522c..ca4af0a9034006 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.test.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.test.ts @@ -10,7 +10,7 @@ /* eslint-disable dot-notation */ import { mockTelemetryService } from '../mocks'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; + describe('TelemetryService', () => { describe('fetchTelemetry', () => { it('calls expected URL with 20 minutes - now', async () => { @@ -142,7 +142,9 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'staging' }, }); - expect(telemetryService.getTelemetryUrl()).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING); + expect(telemetryService.getTelemetryUrl()).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-snapshot"` + ); }); it('should return prod endpoint when sendUsageTo is set to prod', async () => { @@ -150,7 +152,9 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'prod' }, }); - expect(telemetryService.getTelemetryUrl()).toBe(TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD); + expect(telemetryService.getTelemetryUrl()).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-snapshot"` + ); }); }); @@ -160,8 +164,8 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'staging' }, }); - expect(telemetryService.getOptInStatusUrl()).toBe( - TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING + expect(telemetryService.getOptInStatusUrl()).toMatchInlineSnapshot( + `"https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status"` ); }); @@ -170,8 +174,8 @@ describe('TelemetryService', () => { config: { sendUsageTo: 'prod' }, }); - expect(telemetryService.getOptInStatusUrl()).toBe( - TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD + expect(telemetryService.getOptInStatusUrl()).toMatchInlineSnapshot( + `"https://telemetry.elastic.co/v3/send/kibana-opt_in_status"` ); }); }); @@ -247,7 +251,7 @@ describe('TelemetryService', () => { const telemetryService = mockTelemetryService({ config: { userCanChangeSettings: undefined }, }); - const mockPayload = ['mock_hashed_opt_in_status_payload']; + const mockPayload = [{ clusterUuid: 'mk_uuid', stats: 'mock_hashed_opt_in_status_payload' }]; const mockUrl = 'mock_telemetry_optin_status_url'; const mockGetOptInStatusUrl = jest.fn().mockReturnValue(mockUrl); @@ -257,21 +261,28 @@ describe('TelemetryService', () => { expect(mockGetOptInStatusUrl).toBeCalledTimes(1); expect(mockFetch).toBeCalledTimes(1); - expect(mockFetch).toBeCalledWith(mockUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': 'mockKibanaVersion', - }, - body: JSON.stringify(mockPayload), - }); + expect(mockFetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "mock_telemetry_optin_status_url", + Object { + "body": "mock_hashed_opt_in_status_payload", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mockKibanaVersion", + }, + "method": "POST", + }, + ] + `); }); it('swallows errors if fetch fails', async () => { const telemetryService = mockTelemetryService({ config: { userCanChangeSettings: undefined }, }); - const mockPayload = ['mock_hashed_opt_in_status_payload']; + const mockPayload = [{ clusterUuid: 'mk_uuid', stats: 'mock_hashed_opt_in_status_payload' }]; const mockUrl = 'mock_telemetry_optin_status_url'; const mockGetOptInStatusUrl = jest.fn().mockReturnValue(mockUrl); diff --git a/src/plugins/telemetry/public/services/telemetry_service.ts b/src/plugins/telemetry/public/services/telemetry_service.ts index 4e52ec3a7e6edf..63e9b66a49a92c 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.ts @@ -10,6 +10,8 @@ import { i18n } from '@kbn/i18n'; import { CoreStart } from 'kibana/public'; import { TelemetryPluginConfig } from '../plugin'; import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; +import type { UnencryptedTelemetryPayload, EncryptedTelemetryPayload } from '../../common/types'; +import { PAYLOAD_CONTENT_ENCODING } from '../../common/constants'; interface TelemetryServiceConstructor { config: TelemetryPluginConfig; @@ -101,7 +103,7 @@ export class TelemetryService { /** Retrieve the URL to report telemetry **/ public getTelemetryUrl = () => { const { sendUsageTo } = this.config; - return getTelemetryChannelEndpoint({ channelName: 'main', env: sendUsageTo }); + return getTelemetryChannelEndpoint({ channelName: 'snapshot', env: sendUsageTo }); }; /** @@ -137,7 +139,7 @@ export class TelemetryService { }; /** Fetches an unencrypted telemetry payload so we can show it to the user **/ - public fetchExample = async () => { + public fetchExample = async (): Promise => { return await this.fetchTelemetry({ unencrypted: true }); }; @@ -145,11 +147,11 @@ export class TelemetryService { * Fetches telemetry payload * @param unencrypted Default `false`. Whether the returned payload should be encrypted or not. */ - public fetchTelemetry = async ({ unencrypted = false } = {}) => { + public fetchTelemetry = async ({ + unencrypted = false, + } = {}): Promise => { return this.http.post('/api/telemetry/v2/clusters/_stats', { - body: JSON.stringify({ - unencrypted, - }), + body: JSON.stringify({ unencrypted }), }); }; @@ -167,13 +169,16 @@ export class TelemetryService { try { // Report the option to the Kibana server to store the settings. // It returns the encrypted update to send to the telemetry cluster [{cluster_uuid, opt_in_status}] - const optInPayload = await this.http.post('/api/telemetry/v2/optIn', { - body: JSON.stringify({ enabled: optedIn }), - }); + const optInStatusPayload = await this.http.post( + '/api/telemetry/v2/optIn', + { + body: JSON.stringify({ enabled: optedIn }), + } + ); if (this.reportOptInStatusChange) { // Use the response to report about the change to the remote telemetry cluster. // If it's opt-out, this will be the last communication to the remote service. - await this.reportOptInStatus(optInPayload); + await this.reportOptInStatus(optInStatusPayload); } this.isOptedIn = optedIn; } catch (err) { @@ -216,18 +221,26 @@ export class TelemetryService { * Pushes the encrypted payload [{cluster_uuid, opt_in_status}] to the remote telemetry service * @param optInPayload [{cluster_uuid, opt_in_status}] encrypted by the server into an array of strings */ - private reportOptInStatus = async (optInPayload: string[]): Promise => { + private reportOptInStatus = async ( + optInStatusPayload: EncryptedTelemetryPayload + ): Promise => { const telemetryOptInStatusUrl = this.getOptInStatusUrl(); try { - await fetch(telemetryOptInStatusUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Elastic-Stack-Version': this.currentKibanaVersion, - }, - body: JSON.stringify(optInPayload), - }); + await Promise.all( + optInStatusPayload.map(async ({ clusterUuid, stats }) => { + return await fetch(telemetryOptInStatusUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Elastic-Stack-Version': this.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + body: stats, + }); + }) + ); } catch (err) { // Sending the ping is best-effort. Telemetry tries to send the ping once and discards it immediately if sending fails. // swallow any errors diff --git a/src/plugins/telemetry/server/config/config.ts b/src/plugins/telemetry/server/config/config.ts index 8d75f0aba1726c..bfcdefe316c093 100644 --- a/src/plugins/telemetry/server/config/config.ts +++ b/src/plugins/telemetry/server/config/config.ts @@ -9,8 +9,8 @@ import { schema, TypeOf, Type } from '@kbn/config-schema'; import { getConfigPath } from '@kbn/utils'; import { PluginConfigDescriptor } from 'kibana/server'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; import { deprecateEndpointConfigs } from './deprecations'; +import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; const clusterEnvSchema: [Type<'prod'>, Type<'staging'>] = [ schema.literal('prod'), @@ -44,10 +44,10 @@ const configSchema = schema.object({ schema.contextRef('dist'), schema.literal(false), // Point to staging if it's not a distributable release schema.string({ - defaultValue: TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING, + defaultValue: getTelemetryChannelEndpoint({ channelName: 'snapshot', env: 'staging' }), }), schema.string({ - defaultValue: TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD, + defaultValue: getTelemetryChannelEndpoint({ channelName: 'snapshot', env: 'prod' }), }) ), /** @@ -58,10 +58,10 @@ const configSchema = schema.object({ schema.contextRef('dist'), schema.literal(false), // Point to staging if it's not a distributable release schema.string({ - defaultValue: TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING, + defaultValue: getTelemetryChannelEndpoint({ channelName: 'optInStatus', env: 'staging' }), }), schema.string({ - defaultValue: TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD, + defaultValue: getTelemetryChannelEndpoint({ channelName: 'optInStatus', env: 'prod' }), }) ), sendUsageFrom: schema.oneOf([schema.literal('server'), schema.literal('browser')], { diff --git a/src/plugins/telemetry/server/config/deprecations.test.ts b/src/plugins/telemetry/server/config/deprecations.test.ts index 1243ae67120572..b0c17b8c79c983 100644 --- a/src/plugins/telemetry/server/config/deprecations.test.ts +++ b/src/plugins/telemetry/server/config/deprecations.test.ts @@ -9,7 +9,7 @@ import { configDeprecationsMock } from '../../../../core/server/mocks'; import { deprecateEndpointConfigs } from './deprecations'; import type { TelemetryConfigType } from './config'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; +import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; describe('deprecateEndpointConfigs', () => { const fromPath = 'telemetry'; @@ -43,7 +43,7 @@ describe('deprecateEndpointConfigs', () => { it('sets "telemetryConfig.sendUsageTo: staging" if "telemetry.url" uses the staging endpoint', () => { const rawConfig = createMockRawConfig({ - url: TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING, + url: getTelemetryChannelEndpoint({ channelName: 'snapshot', env: 'staging' }), }); const result = deprecateEndpointConfigs( rawConfig, @@ -97,7 +97,7 @@ describe('deprecateEndpointConfigs', () => { it('sets "telemetryConfig.sendUsageTo: staging" if "telemetry.optInStatusUrl" uses the staging endpoint', () => { const rawConfig = createMockRawConfig({ - optInStatusUrl: TELEMETRY_ENDPOINT.MAIN_CHANNEL.STAGING, + optInStatusUrl: getTelemetryChannelEndpoint({ channelName: 'snapshot', env: 'staging' }), }); const result = deprecateEndpointConfigs( rawConfig, @@ -151,7 +151,7 @@ describe('deprecateEndpointConfigs', () => { it('registers deprecation when "telemetry.url" is set', () => { const rawConfig = createMockRawConfig({ - url: TELEMETRY_ENDPOINT.MAIN_CHANNEL.PROD, + url: getTelemetryChannelEndpoint({ channelName: 'snapshot', env: 'prod' }), }); deprecateEndpointConfigs(rawConfig, fromPath, mockAddDeprecation, deprecationContext); expect(mockAddDeprecation).toBeCalledTimes(1); diff --git a/src/plugins/telemetry/server/fetcher.test.ts b/src/plugins/telemetry/server/fetcher.test.ts index 15b40d2b3e4e5c..8d427808bb5e14 100644 --- a/src/plugins/telemetry/server/fetcher.test.ts +++ b/src/plugins/telemetry/server/fetcher.test.ts @@ -71,7 +71,11 @@ describe('FetcherTask', () => { const initializerContext = coreMock.createPluginInitializerContext({}); const fetcherTask = new FetcherTask(initializerContext); const mockTelemetryUrl = 'mock_telemetry_url'; - const mockClusters = ['cluster_1', 'cluster_2']; + const mockClusters = [ + { clusterUuid: 'mk_uuid_1', stats: 'cluster_1' }, + { clusterUuid: 'mk_uuid_2', stats: 'cluster_2' }, + ]; + const getCurrentConfigs = jest.fn().mockResolvedValue({ telemetryUrl: mockTelemetryUrl, }); @@ -95,9 +99,8 @@ describe('FetcherTask', () => { expect(areAllCollectorsReady).toBeCalledTimes(1); expect(fetchTelemetry).toBeCalledTimes(1); - expect(sendTelemetry).toBeCalledTimes(2); - expect(sendTelemetry).toHaveBeenNthCalledWith(1, mockTelemetryUrl, mockClusters[0]); - expect(sendTelemetry).toHaveBeenNthCalledWith(2, mockTelemetryUrl, mockClusters[1]); + expect(sendTelemetry).toBeCalledTimes(1); + expect(sendTelemetry).toHaveBeenNthCalledWith(1, mockTelemetryUrl, mockClusters); expect(updateReportFailure).toBeCalledTimes(0); }); }); diff --git a/src/plugins/telemetry/server/fetcher.ts b/src/plugins/telemetry/server/fetcher.ts index e15b5be2604ec4..02ac428b076679 100644 --- a/src/plugins/telemetry/server/fetcher.ts +++ b/src/plugins/telemetry/server/fetcher.ts @@ -25,7 +25,8 @@ import { getTelemetryFailureDetails, } from '../common/telemetry_config'; import { getTelemetrySavedObject, updateTelemetrySavedObject } from './telemetry_repository'; -import { REPORT_INTERVAL_MS } from '../common/constants'; +import { REPORT_INTERVAL_MS, PAYLOAD_CONTENT_ENCODING } from '../common/constants'; +import type { EncryptedTelemetryPayload } from '../common/types'; import { TelemetryConfigType } from './config'; export interface FetcherTaskDepsStart { @@ -103,7 +104,7 @@ export class FetcherTask { return; } - let clusters: string[] = []; + let clusters: EncryptedTelemetryPayload = []; this.isSending = true; try { @@ -120,9 +121,7 @@ export class FetcherTask { try { const { telemetryUrl } = telemetryConfig; - for (const cluster of clusters) { - await this.sendTelemetry(telemetryUrl, cluster); - } + await this.sendTelemetry(telemetryUrl, clusters); await this.updateLastReported(); } catch (err) { @@ -141,7 +140,7 @@ export class FetcherTask { const allowChangingOptInStatus = config.allowChangingOptInStatus; const configTelemetryOptIn = typeof config.optIn === 'undefined' ? null : config.optIn; const telemetryUrl = getTelemetryChannelEndpoint({ - channelName: 'main', + channelName: 'snapshot', env: config.sendUsageTo, }); const { failureCount, failureVersion } = getTelemetryFailureDetails({ @@ -206,13 +205,16 @@ export class FetcherTask { return false; } - private async fetchTelemetry() { + private async fetchTelemetry(): Promise { return await this.telemetryCollectionManager!.getStats({ unencrypted: false, }); } - private async sendTelemetry(telemetryUrl: string, cluster: string): Promise { + private async sendTelemetry( + telemetryUrl: string, + payload: EncryptedTelemetryPayload + ): Promise { this.logger.debug(`Sending usage stats.`); /** * send OPTIONS before sending usage data. @@ -222,10 +224,18 @@ export class FetcherTask { method: 'options', }); - await fetch(telemetryUrl, { - method: 'post', - body: cluster, - headers: { 'X-Elastic-Stack-Version': this.currentKibanaVersion }, - }); + await Promise.all( + payload.map(async ({ clusterUuid, stats }) => { + await fetch(telemetryUrl, { + method: 'post', + body: stats, + headers: { + 'X-Elastic-Stack-Version': this.currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + }); + }) + ); } } diff --git a/src/plugins/telemetry/server/plugin.ts b/src/plugins/telemetry/server/plugin.ts index d38f054a4402e0..a5707cc0ffeeaa 100644 --- a/src/plugins/telemetry/server/plugin.ts +++ b/src/plugins/telemetry/server/plugin.ts @@ -118,7 +118,10 @@ export class TelemetryPlugin implements Plugin { const { sendUsageTo } = await config$.pipe(take(1)).toPromise(); - const telemetryUrl = getTelemetryChannelEndpoint({ env: sendUsageTo, channelName: 'main' }); + const telemetryUrl = getTelemetryChannelEndpoint({ + env: sendUsageTo, + channelName: 'snapshot', + }); return new URL(telemetryUrl); }, diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts index acc9a863af61b4..edf9cf5b5e18c7 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.test.ts @@ -10,11 +10,14 @@ jest.mock('node-fetch'); import fetch from 'node-fetch'; import { sendTelemetryOptInStatus } from './telemetry_opt_in_stats'; import { StatsGetterConfig } from 'src/plugins/telemetry_collection_manager/server'; -import { TELEMETRY_ENDPOINT } from '../../common/constants'; + describe('sendTelemetryOptInStatus', () => { + const mockClusterUuid = 'mk_uuid'; const mockStatsGetterConfig = { unencrypted: false } as StatsGetterConfig; const mockTelemetryCollectionManager = { - getOptInStats: jest.fn().mockResolvedValue(['mock_opt_in_hashed_value']), + getOptInStats: jest + .fn() + .mockResolvedValue([{ clusterUuid: mockClusterUuid, stats: 'mock_opt_in_hashed_value' }]), }; beforeEach(() => { @@ -35,11 +38,21 @@ describe('sendTelemetryOptInStatus', () => { ); expect(result).toBeUndefined(); expect(fetch).toBeCalledTimes(1); - expect(fetch).toBeCalledWith(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.PROD, { - method: 'post', - body: '["mock_opt_in_hashed_value"]', - headers: { 'X-Elastic-Stack-Version': mockConfig.currentKibanaVersion }, - }); + expect((fetch as jest.MockedFunction).mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "https://telemetry.elastic.co/v3/send/kibana-opt_in_status", + Object { + "body": "mock_opt_in_hashed_value", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mock_kibana_version", + }, + "method": "post", + }, + ] + `); }); it('sends to staging endpoint on "sendUsageTo: staging"', async () => { @@ -56,10 +69,20 @@ describe('sendTelemetryOptInStatus', () => { ); expect(fetch).toBeCalledTimes(1); - expect(fetch).toBeCalledWith(TELEMETRY_ENDPOINT.OPT_IN_STATUS_CHANNEL.STAGING, { - method: 'post', - body: '["mock_opt_in_hashed_value"]', - headers: { 'X-Elastic-Stack-Version': mockConfig.currentKibanaVersion }, - }); + expect((fetch as jest.MockedFunction).mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "https://telemetry-staging.elastic.co/v3/send/kibana-opt_in_status", + Object { + "body": "mock_opt_in_hashed_value", + "headers": Object { + "Content-Type": "application/json", + "X-Elastic-Cluster-ID": "mk_uuid", + "X-Elastic-Content-Encoding": "aes256gcm", + "X-Elastic-Stack-Version": "mock_kibana_version", + }, + "method": "post", + }, + ] + `); }); }); diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts index f6b7eddcbe765c..2a956656621944 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts @@ -15,6 +15,8 @@ import { StatsGetterConfig, } from 'src/plugins/telemetry_collection_manager/server'; import { getTelemetryChannelEndpoint } from '../../common/telemetry_config'; +import { PAYLOAD_CONTENT_ENCODING } from '../../common/constants'; +import type { UnencryptedTelemetryPayload } from '../../common/types'; interface SendTelemetryOptInStatusConfig { sendUsageTo: 'staging' | 'prod'; @@ -26,23 +28,30 @@ export async function sendTelemetryOptInStatus( telemetryCollectionManager: Pick, config: SendTelemetryOptInStatusConfig, statsGetterConfig: StatsGetterConfig -) { +): Promise { const { sendUsageTo, newOptInStatus, currentKibanaVersion } = config; const optInStatusUrl = getTelemetryChannelEndpoint({ env: sendUsageTo, channelName: 'optInStatus', }); - const optInStatus = await telemetryCollectionManager.getOptInStats( - newOptInStatus, - statsGetterConfig - ); + const optInStatusPayload: UnencryptedTelemetryPayload = + await telemetryCollectionManager.getOptInStats(newOptInStatus, statsGetterConfig); - await fetch(optInStatusUrl, { - method: 'post', - body: JSON.stringify(optInStatus), - headers: { 'X-Elastic-Stack-Version': currentKibanaVersion }, - }); + await Promise.all( + optInStatusPayload.map(async ({ clusterUuid, stats }) => { + return await fetch(optInStatusUrl, { + method: 'post', + body: typeof stats === 'string' ? stats : JSON.stringify(stats), + headers: { + 'Content-Type': 'application/json', + 'X-Elastic-Stack-Version': currentKibanaVersion, + 'X-Elastic-Cluster-ID': clusterUuid, + 'X-Elastic-Content-Encoding': PAYLOAD_CONTENT_ENCODING, + }, + }); + }) + ); } export function registerTelemetryOptInStatsRoutes( diff --git a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts index 1b80a2c29b362e..2ed69c2f8a944e 100644 --- a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts +++ b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts @@ -14,11 +14,11 @@ export function getKID(useProdKey = false): string { } export async function encryptTelemetry( - payload: Payload | Payload[], + payload: Payload, { useProdKey = false } = {} -): Promise { +): Promise { const kid = getKID(useProdKey); const encryptor = await createRequestEncryptor(telemetryJWKS); - const clusters = ([] as Payload[]).concat(payload); - return Promise.all(clusters.map((cluster) => encryptor.encrypt(kid, cluster))); + + return await encryptor.encrypt(kid, payload); } diff --git a/src/plugins/telemetry_collection_manager/server/plugin.test.ts b/src/plugins/telemetry_collection_manager/server/plugin.test.ts index d05799f82c3544..6e37ef5ffd4f57 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.test.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.test.ts @@ -91,7 +91,9 @@ describe('Telemetry Collection Manager', () => { cluster_uuid: 'clusterUuid', cluster_name: 'clusterName', timestamp: new Date().toISOString(), - cluster_stats: {}, + cluster_stats: { + cluster_uuid: 'clusterUuid', + }, stack_stats: {}, version: 'version', }; @@ -120,7 +122,12 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); collectionStrategy.statsGetter.mockResolvedValue([basicStats]); - await expect(setupApi.getStats(config)).resolves.toStrictEqual([expect.any(String)]); + await expect(setupApi.getStats(config)).resolves.toStrictEqual([ + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, + ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient ).toBeInstanceOf(TelemetrySavedObjectsClient); @@ -141,7 +148,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(true, config)).resolves.toStrictEqual([ - expect.any(String), + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -153,7 +163,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(false, config)).resolves.toStrictEqual([ - expect.any(String), + { + clusterUuid: 'clusterUuid', + stats: expect.any(String), + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -181,7 +194,10 @@ describe('Telemetry Collection Manager', () => { ]); collectionStrategy.statsGetter.mockResolvedValue([basicStats]); await expect(setupApi.getStats(config)).resolves.toStrictEqual([ - { ...basicStats, collectionSource: 'test_collection' }, + { + clusterUuid: 'clusterUuid', + stats: { ...basicStats, collectionSource: 'test_collection' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -203,7 +219,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(true, config)).resolves.toStrictEqual([ - { cluster_uuid: 'clusterUuid', opt_in_status: true }, + { + clusterUuid: 'clusterUuid', + stats: { opt_in_status: true, cluster_uuid: 'clusterUuid' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient @@ -215,7 +234,10 @@ describe('Telemetry Collection Manager', () => { { clusterUuid: 'clusterUuid' }, ]); await expect(setupApi.getOptInStats(false, config)).resolves.toStrictEqual([ - { cluster_uuid: 'clusterUuid', opt_in_status: false }, + { + clusterUuid: 'clusterUuid', + stats: { opt_in_status: false, cluster_uuid: 'clusterUuid' }, + }, ]); expect( collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 9770395e0ec0cc..6dd1de65a8bdc8 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -28,6 +28,7 @@ import type { StatsGetterConfig, StatsCollectionConfig, UsageStatsPayload, + OptInStatsPayload, StatsCollectionContext, UnencryptedStatsGetterConfig, EncryptedStatsGetterConfig, @@ -163,6 +164,14 @@ export class TelemetryCollectionManagerPlugin } } + private async getOptInStats( + optInStatus: boolean, + config: UnencryptedStatsGetterConfig + ): Promise>; + private async getOptInStats( + optInStatus: boolean, + config: EncryptedStatsGetterConfig + ): Promise>; private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { if (!this.usageCollection) { return []; @@ -179,13 +188,23 @@ export class TelemetryCollectionManagerPlugin optInStatus, statsCollectionConfig ); - if (optInStats && optInStats.length) { - this.logger.debug(`Got Opt In stats using ${collection.title} collection.`); - if (config.unencrypted) { - return optInStats; - } - return encryptTelemetry(optInStats, { useProdKey: this.isDistributable }); - } + + this.logger.debug(`Received Opt In stats using ${collection.title} collection.`); + + return await Promise.all( + optInStats.map(async (clusterStats) => { + const clusterUuid = clusterStats.cluster_uuid; + + return { + clusterUuid, + stats: config.unencrypted + ? clusterStats + : await encryptTelemetry(clusterStats, { + useProdKey: this.isDistributable, + }), + }; + }) + ); } catch (err) { this.logger.debug( `Failed to collect any opt in stats with collection ${collection.title}.` @@ -205,7 +224,7 @@ export class TelemetryCollectionManagerPlugin collection: CollectionStrategy, optInStatus: boolean, statsCollectionConfig: StatsCollectionConfig - ) => { + ): Promise => { const context: StatsCollectionContext = { logger: this.logger.get(collection.title), version: this.version, @@ -218,8 +237,12 @@ export class TelemetryCollectionManagerPlugin })); }; - private async getStats(config: UnencryptedStatsGetterConfig): Promise; - private async getStats(config: EncryptedStatsGetterConfig): Promise; + private async getStats( + config: UnencryptedStatsGetterConfig + ): Promise>; + private async getStats( + config: EncryptedStatsGetterConfig + ): Promise>; private async getStats(config: StatsGetterConfig) { if (!this.usageCollection) { return []; @@ -231,16 +254,25 @@ export class TelemetryCollectionManagerPlugin if (statsCollectionConfig) { try { const usageData = await this.getUsageForCollection(collection, statsCollectionConfig); - if (usageData.length) { - this.logger.debug(`Got Usage using ${collection.title} collection.`); - if (config.unencrypted) { - return usageData; - } - - return await encryptTelemetry(usageData, { - useProdKey: this.isDistributable, - }); - } + this.logger.debug(`Received Usage using ${collection.title} collection.`); + + return await Promise.all( + usageData.map(async (clusterStats) => { + const { cluster_uuid: clusterUuid } = clusterStats.cluster_stats as Record< + string, + string + >; + + return { + clusterUuid, + stats: config.unencrypted + ? clusterStats + : await encryptTelemetry(clusterStats, { + useProdKey: this.isDistributable, + }), + }; + }) + ); } catch (err) { this.logger.debug( `Failed to collect any usage with registered collection ${collection.title}.` diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index 985eff409c1de6..648e457f9a2381 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -74,6 +74,11 @@ export interface UsageStatsPayload extends BasicStatsPayload { collectionSource: string; } +export interface OptInStatsPayload { + cluster_uuid: string; + opt_in_status: boolean; +} + export interface StatsCollectionContext { logger: Logger | Console; version: string; diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.ts b/x-pack/test/api_integration/apis/telemetry/telemetry.ts index c5b8b40368302f..527d755123f264 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.ts @@ -19,6 +19,7 @@ import monitoringRootTelemetrySchema from '../../../../plugins/telemetry_collect import ossPluginsTelemetrySchema from '../../../../../src/plugins/telemetry/schema/oss_plugins.json'; import xpackPluginsTelemetrySchema from '../../../../plugins/telemetry_collection_xpack/schema/xpack_plugins.json'; import { assertTelemetryPayload } from '../../../../../test/api_integration/apis/telemetry/utils'; +import type { UnencryptedTelemetryPayload } from '../../../../../src/plugins/telemetry/common/types'; /** * Update the .monitoring-* documents loaded via the archiver to the recent `timestamp` @@ -92,15 +93,16 @@ export default function ({ getService }: FtrProviderContext) { await esArchiver.load(archive); await updateMonitoringDates(esSupertest, fromTimestamp, toTimestamp, timestamp); - const { body } = await supertest + const { body }: { body: UnencryptedTelemetryPayload } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') .send({ unencrypted: true }) .expect(200); expect(body.length).to.be.greaterThan(1); - localXPack = body.shift(); - monitoring = body; + const telemetryStats = body.map(({ stats }) => stats); + localXPack = telemetryStats.shift() as Record; + monitoring = telemetryStats as Array>; }); after(() => esArchiver.unload(archive)); @@ -142,15 +144,17 @@ export default function ({ getService }: FtrProviderContext) { }); after(() => esArchiver.unload(archive)); it('should load non-expiring basic cluster', async () => { - const { body } = await supertest + const { body }: { body: UnencryptedTelemetryPayload } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') .send({ unencrypted: true }) .expect(200); expect(body).length(2); - const [localXPack, ...monitoring] = body; - expect(localXPack.collectionSource).to.eql('local_xpack'); + const telemetryStats = body.map(({ stats }) => stats); + + const [localXPack, ...monitoring] = telemetryStats; + expect((localXPack as Record).collectionSource).to.eql('local_xpack'); expect(monitoring).to.eql(basicClusterFixture.map((item) => ({ ...item, timestamp }))); }); }); diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index 508a6584e92463..e34e0fff25888d 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -47,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); expect(body.length).to.be(1); - stats = body[0]; + stats = body[0].stats; }); it('should pass the schema validation', () => { diff --git a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts index ed79d7200c4ede..0d8f38c55c7f87 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts @@ -107,7 +107,7 @@ export default function (providerContext: FtrProviderContext) { it('should return the correct telemetry values for fleet', async () => { const { - body: [apiResponse], + body: [{ stats: apiResponse }], } = await supertest .post(`/api/telemetry/v2/clusters/_stats`) .set('kbn-xsrf', 'xxxx') diff --git a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts index b6ec4aa8dcfa5c..03494edccd648a 100644 --- a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts +++ b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/usage_collection.ts @@ -40,11 +40,11 @@ export default function ({ getService }: FtrProviderContext) { * - vis-3: ref to tag-3 */ it('collects the expected data', async () => { - const telemetryStats = (await usageAPI.getTelemetryStats({ + const [{ stats: telemetryStats }] = (await usageAPI.getTelemetryStats({ unencrypted: true, })) as any; - const taggingStats = telemetryStats[0].stack_stats.kibana.plugins.saved_objects_tagging; + const taggingStats = telemetryStats.stack_stats.kibana.plugins.saved_objects_tagging; expect(taggingStats).to.eql({ usedTags: 4, taggedObjects: 5, From e66f3fe508dfda32bc4aa76154e37f49a198935f Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:01:20 -0400 Subject: [PATCH 20/35] Fix breadcrumbs not showing when opening a timeline from case details (#115728) (#115834) Co-authored-by: Pablo Machado --- .../cases/components/case_view/index.tsx | 304 +++++++++--------- .../public/cases/pages/case_details.tsx | 19 +- 2 files changed, 163 insertions(+), 160 deletions(-) diff --git a/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx b/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx index c5b866129df499..47ba9e1e9cb8ff 100644 --- a/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useRef, useState } from 'react'; +import React, { useCallback, useRef } from 'react'; import { useDispatch } from 'react-redux'; import { getCaseDetailsUrl, @@ -28,7 +28,6 @@ import { InvestigateInTimelineAction } from '../../../detections/components/aler import { useFetchAlertData } from './helpers'; import { SEND_ALERT_TO_TIMELINE } from './translations'; import { useInsertTimeline } from '../use_insert_timeline'; -import { SpyRoute } from '../../../common/utils/route/spy_routes'; import * as timelineMarkdownPlugin from '../../../common/components/markdown_editor/plugins/timeline'; import { CaseDetailsRefreshContext } from '../../../common/components/endpoint/host_isolation/endpoint_host_isolation_cases_context'; import { getEndpointDetailsPath } from '../../../management/common/routing'; @@ -37,6 +36,7 @@ interface Props { caseId: string; subCaseId?: string; userCanCrud: boolean; + onCaseDataSuccess: (data: Case) => void; } export interface OnUpdateFields { @@ -78,175 +78,163 @@ const InvestigateInTimelineActionComponent = (alertIds: string[]) => { ); }; -export const CaseView = React.memo(({ caseId, subCaseId, userCanCrud }: Props) => { - const [spyState, setSpyState] = useState<{ caseTitle: string | undefined }>({ - caseTitle: undefined, - }); - - const onCaseDataSuccess = useCallback( - (data: Case) => { - if (spyState.caseTitle === undefined || spyState.caseTitle !== data.title) { - setSpyState({ caseTitle: data.title }); - } - }, - [spyState.caseTitle] - ); - - const { - cases: casesUi, - application: { navigateToApp }, - } = useKibana().services; - const dispatch = useDispatch(); - const { formatUrl, search } = useFormatUrl(SecurityPageName.case); - const { formatUrl: detectionsFormatUrl, search: detectionsUrlSearch } = useFormatUrl( - SecurityPageName.rules - ); - - const allCasesLink = getCaseUrl(search); - const formattedAllCasesLink = formatUrl(allCasesLink); - const configureCasesHref = formatUrl(getConfigureCasesUrl()); +export const CaseView = React.memo( + ({ caseId, subCaseId, userCanCrud, onCaseDataSuccess }: Props) => { + const { + cases: casesUi, + application: { navigateToApp }, + } = useKibana().services; + const dispatch = useDispatch(); + const { formatUrl, search } = useFormatUrl(SecurityPageName.case); + const { formatUrl: detectionsFormatUrl, search: detectionsUrlSearch } = useFormatUrl( + SecurityPageName.rules + ); - const caseDetailsLink = formatUrl(getCaseDetailsUrl({ id: caseId }), { absolute: true }); - const getCaseDetailHrefWithCommentId = (commentId: string) => - formatUrl(getCaseDetailsUrlWithCommentId({ id: caseId, commentId, subCaseId }), { - absolute: true, - }); + const allCasesLink = getCaseUrl(search); + const formattedAllCasesLink = formatUrl(allCasesLink); + const configureCasesHref = formatUrl(getConfigureCasesUrl()); - const getDetectionsRuleDetailsHref = useCallback( - (ruleId) => detectionsFormatUrl(getRuleDetailsUrl(ruleId ?? '', detectionsUrlSearch)), - [detectionsFormatUrl, detectionsUrlSearch] - ); - - const showAlertDetails = useCallback( - (alertId: string, index: string) => { - dispatch( - timelineActions.toggleDetailPanel({ - panelView: 'eventDetail', - timelineId: TimelineId.casePage, - params: { - eventId: alertId, - indexName: index, - }, - }) - ); - }, - [dispatch] - ); + const caseDetailsLink = formatUrl(getCaseDetailsUrl({ id: caseId }), { absolute: true }); + const getCaseDetailHrefWithCommentId = (commentId: string) => + formatUrl(getCaseDetailsUrlWithCommentId({ id: caseId, commentId, subCaseId }), { + absolute: true, + }); - const endpointDetailsHref = (endpointId: string) => - formatUrl( - getEndpointDetailsPath({ - name: 'endpointActivityLog', - selected_endpoint: endpointId, - }) + const getDetectionsRuleDetailsHref = useCallback( + (ruleId) => detectionsFormatUrl(getRuleDetailsUrl(ruleId ?? '', detectionsUrlSearch)), + [detectionsFormatUrl, detectionsUrlSearch] ); - const onComponentInitialized = useCallback(() => { - dispatch( - timelineActions.createTimeline({ - id: TimelineId.casePage, - columns: [], - indexNames: [], - expandedDetail: {}, - show: false, - }) + const showAlertDetails = useCallback( + (alertId: string, index: string) => { + dispatch( + timelineActions.toggleDetailPanel({ + panelView: 'eventDetail', + timelineId: TimelineId.casePage, + params: { + eventId: alertId, + indexName: index, + }, + }) + ); + }, + [dispatch] ); - }, [dispatch]); - const refreshRef = useRef(null); + const endpointDetailsHref = (endpointId: string) => + formatUrl( + getEndpointDetailsPath({ + name: 'endpointActivityLog', + selected_endpoint: endpointId, + }) + ); - return ( - - {casesUi.getCaseView({ - refreshRef, - allCasesNavigation: { - href: formattedAllCasesLink, - onClick: async (e) => { - if (e) { - e.preventDefault(); - } - return navigateToApp(APP_ID, { - deepLinkId: SecurityPageName.case, - path: allCasesLink, - }); - }, - }, - caseDetailsNavigation: { - href: caseDetailsLink, - onClick: async (e) => { - if (e) { - e.preventDefault(); - } - return navigateToApp(APP_ID, { - deepLinkId: SecurityPageName.case, - path: getCaseDetailsUrl({ id: caseId }), - }); - }, - }, - caseId, - configureCasesNavigation: { - href: configureCasesHref, - onClick: async (e) => { - if (e) { - e.preventDefault(); - } - return navigateToApp(APP_ID, { - deepLinkId: SecurityPageName.case, - path: getConfigureCasesUrl(search), - }); + const onComponentInitialized = useCallback(() => { + dispatch( + timelineActions.createTimeline({ + id: TimelineId.casePage, + columns: [], + indexNames: [], + expandedDetail: {}, + show: false, + }) + ); + }, [dispatch]); + + const refreshRef = useRef(null); + + return ( + + {casesUi.getCaseView({ + refreshRef, + allCasesNavigation: { + href: formattedAllCasesLink, + onClick: async (e) => { + if (e) { + e.preventDefault(); + } + return navigateToApp(APP_ID, { + deepLinkId: SecurityPageName.case, + path: allCasesLink, + }); + }, }, - }, - getCaseDetailHrefWithCommentId, - onCaseDataSuccess, - onComponentInitialized, - actionsNavigation: { - href: endpointDetailsHref, - onClick: (endpointId: string, e) => { - if (e) { - e.preventDefault(); - } - return navigateToApp(APP_ID, { - path: getEndpointDetailsPath({ - name: 'endpointActivityLog', - selected_endpoint: endpointId, - }), - }); + caseDetailsNavigation: { + href: caseDetailsLink, + onClick: async (e) => { + if (e) { + e.preventDefault(); + } + return navigateToApp(APP_ID, { + deepLinkId: SecurityPageName.case, + path: getCaseDetailsUrl({ id: caseId }), + }); + }, }, - }, - ruleDetailsNavigation: { - href: getDetectionsRuleDetailsHref, - onClick: async (ruleId: string | null | undefined, e) => { - if (e) { - e.preventDefault(); - } - return navigateToApp(APP_ID, { - deepLinkId: SecurityPageName.rules, - path: getRuleDetailsUrl(ruleId ?? ''), - }); + caseId, + configureCasesNavigation: { + href: configureCasesHref, + onClick: async (e) => { + if (e) { + e.preventDefault(); + } + return navigateToApp(APP_ID, { + deepLinkId: SecurityPageName.case, + path: getConfigureCasesUrl(search), + }); + }, }, - }, - showAlertDetails, - subCaseId, - timelineIntegration: { - editor_plugins: { - parsingPlugin: timelineMarkdownPlugin.parser, - processingPluginRenderer: timelineMarkdownPlugin.renderer, - uiPlugin: timelineMarkdownPlugin.plugin, + getCaseDetailHrefWithCommentId, + onCaseDataSuccess, + onComponentInitialized, + actionsNavigation: { + href: endpointDetailsHref, + onClick: (endpointId: string, e) => { + if (e) { + e.preventDefault(); + } + return navigateToApp(APP_ID, { + path: getEndpointDetailsPath({ + name: 'endpointActivityLog', + selected_endpoint: endpointId, + }), + }); + }, }, - hooks: { - useInsertTimeline, + ruleDetailsNavigation: { + href: getDetectionsRuleDetailsHref, + onClick: async (ruleId: string | null | undefined, e) => { + if (e) { + e.preventDefault(); + } + return navigateToApp(APP_ID, { + deepLinkId: SecurityPageName.rules, + path: getRuleDetailsUrl(ruleId ?? ''), + }); + }, }, - ui: { - renderInvestigateInTimelineActionComponent: InvestigateInTimelineActionComponent, - renderTimelineDetailsPanel: TimelineDetailsPanel, + showAlertDetails, + subCaseId, + timelineIntegration: { + editor_plugins: { + parsingPlugin: timelineMarkdownPlugin.parser, + processingPluginRenderer: timelineMarkdownPlugin.renderer, + uiPlugin: timelineMarkdownPlugin.plugin, + }, + hooks: { + useInsertTimeline, + }, + ui: { + renderInvestigateInTimelineActionComponent: InvestigateInTimelineActionComponent, + renderTimelineDetailsPanel: TimelineDetailsPanel, + }, }, - }, - useFetchAlertData, - userCanCrud, - })} - - - ); -}); + useFetchAlertData, + userCanCrud, + })} + + ); + } +); CaseView.displayName = 'CaseView'; diff --git a/x-pack/plugins/security_solution/public/cases/pages/case_details.tsx b/x-pack/plugins/security_solution/public/cases/pages/case_details.tsx index e8680b148f9400..ea8205cddad599 100644 --- a/x-pack/plugins/security_solution/public/cases/pages/case_details.tsx +++ b/x-pack/plugins/security_solution/public/cases/pages/case_details.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { useParams } from 'react-router-dom'; import { SecurityPageName } from '../../app/types'; @@ -17,6 +17,7 @@ import { getCaseUrl } from '../../common/components/link_to'; import { navTabs } from '../../app/home/home_navigations'; import { CaseView } from '../components/case_view'; import { APP_ID } from '../../../common/constants'; +import { Case } from '../../../../cases/common'; export const CaseDetailsPage = React.memo(() => { const { @@ -38,6 +39,19 @@ export const CaseDetailsPage = React.memo(() => { } }, [userPermissions, navigateToApp, search]); + const [spyState, setSpyState] = useState<{ caseTitle: string | undefined }>({ + caseTitle: undefined, + }); + + const onCaseDataSuccess = useCallback( + (data: Case) => { + if (spyState.caseTitle === undefined || spyState.caseTitle !== data.title) { + setSpyState({ caseTitle: data.title }); + } + }, + [spyState.caseTitle] + ); + return caseId != null ? ( <> @@ -45,9 +59,10 @@ export const CaseDetailsPage = React.memo(() => { caseId={caseId} subCaseId={subCaseId} userCanCrud={userPermissions?.crud ?? false} + onCaseDataSuccess={onCaseDataSuccess} /> - + ) : null; }); From dc70113a3406e8a6d27c08d0724714ed0962745f Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:06:26 -0400 Subject: [PATCH 21/35] [Security] Realign CCS functional tests (#115795) (#115836) * Realign CCS functional tests * Reduce the diff with plain functional tests Co-authored-by: Domenico Andreoli --- .../detection_alerts/alerts_details.spec.ts | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts b/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts index 825abc6185947e..78bb13395f79b6 100644 --- a/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts +++ b/x-pack/plugins/security_solution/cypress/ccs_integration/detection_alerts/alerts_details.spec.ts @@ -5,18 +5,18 @@ * 2.0. */ -import { ALERT_FLYOUT, JSON_LINES } from '../../screens/alerts_details'; +import { JSON_TEXT } from '../../screens/alerts_details'; import { expandFirstAlert, waitForAlertsIndexToBeCreated, waitForAlertsPanelToBeLoaded, } from '../../tasks/alerts'; -import { openJsonView, scrollJsonViewToBottom } from '../../tasks/alerts_details'; +import { openJsonView } from '../../tasks/alerts_details'; import { createCustomRuleActivated } from '../../tasks/api_calls/rules'; import { cleanKibana } from '../../tasks/common'; +import { esArchiverCCSLoad } from '../../tasks/es_archiver'; import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; -import { esArchiverCCSLoad, esArchiverCCSUnload } from '../../tasks/es_archiver'; import { getUnmappedCCSRule } from '../../objects/rule'; @@ -35,24 +35,14 @@ describe('Alert details with unmapped fields', () => { expandFirstAlert(); }); - afterEach(() => { - esArchiverCCSUnload('unmapped_fields'); - }); - it('Displays the unmapped field on the JSON view', () => { - const expectedUnmappedField = { line: 2, text: ' "unmapped": "This is the unmapped field"' }; + const expectedUnmappedValue = 'This is the unmapped field'; openJsonView(); - scrollJsonViewToBottom(); - - cy.get(ALERT_FLYOUT) - .find(JSON_LINES) - .then((elements) => { - const length = elements.length; - cy.wrap(elements) - .eq(length - expectedUnmappedField.line) - .invoke('text') - .should('include', expectedUnmappedField.text); - }); + + cy.get(JSON_TEXT).then((x) => { + const parsed = JSON.parse(x.text()); + expect(parsed._source.unmapped).to.equal(expectedUnmappedValue); + }); }); }); From 845bf5c2b45d7af51cd234a86ef9fefcb23fe405 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:10:03 -0400 Subject: [PATCH 22/35] Keep page load transaction open (#115410) (#115835) * keep page load transaction open * cleanup * code review Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Liza Katz --- src/core/public/apm_system.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/core/public/apm_system.ts b/src/core/public/apm_system.ts index 1a653636c54d42..c64c1923f1131d 100644 --- a/src/core/public/apm_system.ts +++ b/src/core/public/apm_system.ts @@ -30,6 +30,7 @@ interface StartDeps { export class ApmSystem { private readonly enabled: boolean; + private pageLoadTransaction?: Transaction; /** * `apmConfig` would be populated with relevant APM RUM agent * configuration if server is started with elastic.apm.* config. @@ -49,10 +50,23 @@ export class ApmSystem { this.addHttpRequestNormalization(apm); init(apmConfig); + this.pageLoadTransaction = apm.getCurrentTransaction(); + + // Keep the page load transaction open until all resources finished loading + if (this.pageLoadTransaction && this.pageLoadTransaction.type === 'page-load') { + // @ts-expect-error 2339 + this.pageLoadTransaction.block(true); + this.pageLoadTransaction.mark('apm-setup'); + } } async start(start?: StartDeps) { if (!this.enabled || !start) return; + + if (this.pageLoadTransaction && this.pageLoadTransaction.type === 'page-load') { + this.pageLoadTransaction.mark('apm-start'); + } + /** * Register listeners for navigation changes and capture them as * route-change transactions after Kibana app is bootstrapped @@ -60,6 +74,11 @@ export class ApmSystem { start.application.currentAppId$.subscribe((appId) => { const apmInstance = (window as any).elasticApm; if (appId && apmInstance && typeof apmInstance.startTransaction === 'function') { + // Close the page load transaction + if (this.pageLoadTransaction && this.pageLoadTransaction.type === 'page-load') { + this.pageLoadTransaction.end(); + this.pageLoadTransaction = undefined; + } apmInstance.startTransaction(`/app/${appId}`, 'route-change', { managed: true, canReuse: true, From 0c22993ad66967f25eca316821f4d7f271dbb7c8 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 16:27:48 -0400 Subject: [PATCH 23/35] Fixed callouts (#115631) (#115838) Co-authored-by: Jason Stoltzfus --- .../components/suggested_curations_callout.test.tsx | 12 ++++++++++++ .../components/suggested_curations_callout.tsx | 8 +++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.test.tsx index 38e57fa0483e69..58e2cd8cf4c9b0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.test.tsx @@ -26,6 +26,8 @@ const MOCK_VALUES = { }, }, }, + // LicensingLogic + hasPlatinumLicense: true, }; describe('SuggestedCurationsCallout', () => { @@ -56,4 +58,14 @@ describe('SuggestedCurationsCallout', () => { expect(wrapper.isEmptyRender()).toBe(true); }); + + it('is empty when the user has no platinum license', () => { + // This would happen if the user *had* suggestions and then downgraded from platinum to gold or something + const values = set('hasPlatinumLicense', false, MOCK_VALUES); + setMockValues(values); + + const wrapper = shallow(); + + expect(wrapper.isEmptyRender()).toBe(true); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.tsx index a7155b7d2b161d..046cc2d744b009 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/suggested_curations_callout.tsx @@ -10,6 +10,7 @@ import { useValues } from 'kea'; import { i18n } from '@kbn/i18n'; +import { LicensingLogic } from '../../../../shared/licensing'; import { ENGINE_CURATIONS_PATH } from '../../../routes'; import { SuggestionsCallout } from '../../curations/components/suggestions_callout'; import { EngineLogic, generateEnginePath } from '../../engine'; @@ -18,10 +19,15 @@ export const SuggestedCurationsCallout: React.FC = () => { const { engine: { search_relevance_suggestions: searchRelevanceSuggestions }, } = useValues(EngineLogic); + const { hasPlatinumLicense } = useValues(LicensingLogic); const pendingCount = searchRelevanceSuggestions?.curation.pending; - if (typeof searchRelevanceSuggestions === 'undefined' || pendingCount === 0) { + if ( + typeof searchRelevanceSuggestions === 'undefined' || + pendingCount === 0 || + hasPlatinumLicense === false + ) { return null; } From b47374fe59767124df8106f352f506c19f9152d7 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 20 Oct 2021 14:29:43 -0600 Subject: [PATCH 24/35] [saved objects] Remove migrations `enableV2` config. (#115655) (#115839) # Conflicts: # src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts # src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_same_v1.test.ts --- docs/setup/settings.asciidoc | 3 - .../deprecations/unknown_object_types.test.ts | 12 - .../deprecations/unknown_object_types.ts | 10 - .../migrations/kibana/kibana_migrator.mock.ts | 2 - .../migrations/kibana/kibana_migrator.test.ts | 218 ++++++------------ .../migrations/kibana/kibana_migrator.ts | 75 ++---- .../7.7.2_xpack_100k.test.ts | 1 - .../7_13_0_failed_action_tasks.test.ts | 1 - .../7_13_0_transform_failures.test.ts | 1 - .../7_13_0_unknown_types.test.ts | 1 - .../batch_size_bytes.test.ts | 1 - ...ze_bytes_exceeds_es_content_length.test.ts | 1 - .../integration_tests/cleanup.test.ts | 1 - .../collects_corrupt_docs.test.ts | 1 - .../corrupt_outdated_docs.test.ts | 1 - .../migration_from_older_v1.test.ts | 1 - .../multiple_es_nodes.test.ts | 1 - .../multiple_kibana_nodes.test.ts | 1 - .../integration_tests/outdated_docs.test.ts | 1 - .../integration_tests/rewriting_id.test.ts | 1 - .../migrations_state_action_machine.test.ts | 1 - .../deprecations/delete_unknown_types.ts | 5 +- src/core/server/saved_objects/routes/index.ts | 2 +- .../delete_unknown_types.test.ts | 9 - .../saved_objects_config.test.ts | 4 +- .../saved_objects/saved_objects_config.ts | 21 +- .../service/lib/get_index_for_type.test.ts | 76 ++---- .../service/lib/get_index_for_type.ts | 13 +- .../service/lib/repository.test.js | 20 +- .../saved_objects/service/lib/repository.ts | 1 - .../resources/base/bin/kibana-docker | 1 - x-pack/plugins/uptime/e2e/config.ts | 1 - .../test/apm_api_integration/configs/index.ts | 9 - 33 files changed, 129 insertions(+), 368 deletions(-) diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index ddf52ccad768d9..09d0c43d6b6c17 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -497,9 +497,6 @@ override this parameter to use their own Tile Map Service. For example: | `migrations.maxBatchSizeBytes:` | Defines the maximum payload size for indexing batches of upgraded saved objects to avoid migrations failing due to a 413 Request Entity Too Large response from Elasticsearch. This value should be lower than or equal to your Elasticsearch cluster's `http.max_content_length` configuration option. *Default: `100mb`* -| `migrations.enableV2:` - | experimental[]. Enables the new Saved Objects migration algorithm. For information about the migration algorithm, refer to <>. When `migrations v2` is stable, the setting will be removed in an upcoming release without any further notice. Setting the value to `false` causes {kib} to use the legacy migration algorithm, which shipped in 7.11 and earlier versions. *Default: `true`* - | `migrations.retryAttempts:` | The number of times migrations retry temporary failures, such as a network timeout, 503 status code, or `snapshot_in_progress_exception`. When upgrade migrations frequently fail after exhausting all retry attempts with a message such as `Unable to complete the [...] step after 15 attempts, terminating.`, increase the setting value. *Default: `15`* diff --git a/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts b/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts index d7ea73456e236a..1f9ca741691d10 100644 --- a/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts +++ b/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts @@ -13,7 +13,6 @@ import { deleteUnknownTypeObjects, getUnknownTypesDeprecations } from './unknown import { typeRegistryMock } from '../saved_objects_type_registry.mock'; import { elasticsearchClientMock } from '../../elasticsearch/client/mocks'; import type { KibanaConfigType } from '../../kibana_config'; -import type { SavedObjectConfig } from '../saved_objects_config'; import { SavedObjectsType } from 'kibana/server'; const createSearchResponse = (count: number): estypes.SearchResponse => { @@ -32,7 +31,6 @@ describe('unknown saved object types deprecation', () => { let typeRegistry: ReturnType; let esClient: ReturnType; let kibanaConfig: KibanaConfigType; - let savedObjectsConfig: SavedObjectConfig; beforeEach(() => { typeRegistry = typeRegistryMock.create(); @@ -48,12 +46,6 @@ describe('unknown saved object types deprecation', () => { index: '.kibana', enabled: true, }; - - savedObjectsConfig = { - migration: { - enableV2: true, - }, - } as SavedObjectConfig; }); afterEach(() => { @@ -69,7 +61,6 @@ describe('unknown saved object types deprecation', () => { it('calls `esClient.asInternalUser.search` with the correct parameters', async () => { await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -96,7 +87,6 @@ describe('unknown saved object types deprecation', () => { ); const deprecations = await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -112,7 +102,6 @@ describe('unknown saved object types deprecation', () => { ); const deprecations = await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -141,7 +130,6 @@ describe('unknown saved object types deprecation', () => { describe('deleteUnknownTypeObjects', () => { it('calls `esClient.asInternalUser.search` with the correct parameters', async () => { await deleteUnknownTypeObjects({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, diff --git a/src/core/server/saved_objects/deprecations/unknown_object_types.ts b/src/core/server/saved_objects/deprecations/unknown_object_types.ts index c966e621ca6055..8cd650bac8a2d5 100644 --- a/src/core/server/saved_objects/deprecations/unknown_object_types.ts +++ b/src/core/server/saved_objects/deprecations/unknown_object_types.ts @@ -13,14 +13,12 @@ import { IScopedClusterClient } from '../../elasticsearch'; import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { SavedObjectsRawDocSource } from '../serialization'; import type { KibanaConfigType } from '../../kibana_config'; -import type { SavedObjectConfig } from '../saved_objects_config'; import { getIndexForType } from '../service/lib'; interface UnknownTypesDeprecationOptions { typeRegistry: ISavedObjectTypeRegistry; esClient: IScopedClusterClient; kibanaConfig: KibanaConfigType; - savedObjectsConfig: SavedObjectConfig; kibanaVersion: string; } @@ -32,11 +30,9 @@ const getTargetIndices = ({ typeRegistry, kibanaVersion, kibanaConfig, - savedObjectsConfig, }: { types: string[]; typeRegistry: ISavedObjectTypeRegistry; - savedObjectsConfig: SavedObjectConfig; kibanaConfig: KibanaConfigType; kibanaVersion: string; }) => { @@ -46,7 +42,6 @@ const getTargetIndices = ({ getIndexForType({ type, typeRegistry, - migV2Enabled: savedObjectsConfig.migration.enableV2, kibanaVersion, defaultIndex: kibanaConfig.index, }) @@ -69,7 +64,6 @@ const getUnknownSavedObjects = async ({ typeRegistry, esClient, kibanaConfig, - savedObjectsConfig, kibanaVersion, }: UnknownTypesDeprecationOptions) => { const knownTypes = getKnownTypes(typeRegistry); @@ -78,7 +72,6 @@ const getUnknownSavedObjects = async ({ typeRegistry, kibanaConfig, kibanaVersion, - savedObjectsConfig, }); const query = getUnknownTypesQuery(knownTypes); @@ -141,7 +134,6 @@ interface DeleteUnknownTypesOptions { typeRegistry: ISavedObjectTypeRegistry; esClient: IScopedClusterClient; kibanaConfig: KibanaConfigType; - savedObjectsConfig: SavedObjectConfig; kibanaVersion: string; } @@ -149,7 +141,6 @@ export const deleteUnknownTypeObjects = async ({ esClient, typeRegistry, kibanaConfig, - savedObjectsConfig, kibanaVersion, }: DeleteUnknownTypesOptions) => { const knownTypes = getKnownTypes(typeRegistry); @@ -158,7 +149,6 @@ export const deleteUnknownTypeObjects = async ({ typeRegistry, kibanaConfig, kibanaVersion, - savedObjectsConfig, }); const query = getUnknownTypesQuery(knownTypes); diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts index 9471bbc1b87a63..660300ea867ff2 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts @@ -42,8 +42,6 @@ const createMigrator = ( scrollDuration: '15m', pollInterval: 1500, skip: false, - // TODO migrationsV2: remove/deprecate once we remove migrations v1 - enableV2: false, retryAttempts: 10, }, runMigrations: jest.fn(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index 6e10349f4b57c3..c397559b52570c 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -7,7 +7,7 @@ */ import { take } from 'rxjs/operators'; -import { estypes, errors as esErrors } from '@elastic/elasticsearch'; +import { estypes } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; import { KibanaMigratorOptions, KibanaMigrator } from './kibana_migrator'; @@ -125,13 +125,6 @@ describe('KibanaMigrator', () => { it('only runs migrations once if called multiple times', async () => { const options = mockOptions(); - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); options.client.indices.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) ); @@ -144,159 +137,79 @@ describe('KibanaMigrator', () => { migrator.prepareMigrations(); await migrator.runMigrations(); await migrator.runMigrations(); + await migrator.runMigrations(); - expect(options.client.cat.templates).toHaveBeenCalledTimes(1); + // indices.get is called twice during a single migration + expect(options.client.indices.get).toHaveBeenCalledTimes(2); }); - describe('when enableV2 = false', () => { - it('when enableV2 = false creates an IndexMigrator which retries NoLivingConnectionsError errors from ES client', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - options.client.indices.create = jest - .fn() - .mockReturnValueOnce( - elasticsearchClientMock.createErrorTransportRequestPromise( - new esErrors.NoLivingConnectionsError('reason', {} as any) - ) - ) - .mockImplementationOnce(() => - elasticsearchClientMock.createSuccessTransportRequestPromise('success') - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - - migrator.prepareMigrations(); - await migrator.runMigrations(); + it('emits results on getMigratorResult$()', async () => { + const options = mockV2MigrationOptions(); + const migrator = new KibanaMigrator(options); + const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); + migrator.prepareMigrations(); + await migrator.runMigrations(); - expect(options.client.indices.create).toHaveBeenCalledTimes(3); - const { status } = await migratorStatus; - return expect(status).toEqual('completed'); + const { status, result } = await migratorStatus; + expect(status).toEqual('completed'); + expect(result![0]).toMatchObject({ + destIndex: '.my-index_8.2.3_001', + sourceIndex: '.my-index_pre8.2.3_001', + elapsedMs: expect.any(Number), + status: 'migrated', }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_1', - elapsedMs: expect.any(Number), - sourceIndex: '.my-index', - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_1', - elapsedMs: expect.any(Number), - sourceIndex: 'other-index', - status: 'migrated', - }); + expect(result![1]).toMatchObject({ + destIndex: 'other-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', }); }); - describe('when enableV2 = true', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockV2MigrationOptions(); - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_8.2.3_001', - sourceIndex: '.my-index_pre8.2.3_001', - elapsedMs: expect.any(Number), - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_8.2.3_001', - elapsedMs: expect.any(Number), - status: 'patched', - }); - }); - it('rejects when the migration state machine terminates in a FATAL state', () => { - const options = mockV2MigrationOptions(); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - { - '.my-index_8.2.4_001': { - aliases: { - '.my-index': {}, - '.my-index_8.2.4': {}, - }, - mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, - settings: {}, + it('rejects when the migration state machine terminates in a FATAL state', () => { + const options = mockV2MigrationOptions(); + options.client.indices.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise( + { + '.my-index_8.2.4_001': { + aliases: { + '.my-index': {}, + '.my-index_8.2.4': {}, }, + mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, + settings: {}, }, - { statusCode: 200 } - ) - ); + }, + { statusCode: 200 } + ) + ); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` - ); - }); - it('rejects when an unexpected exception occurs in an action', async () => { - const options = mockV2MigrationOptions(); - options.client.tasks.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, - failures: [], - task: { description: 'task description' } as any, - }) - ); + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( + `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` + ); + }); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` - [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] - `); - expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` - [Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] - `); - }); + it('rejects when an unexpected exception occurs in an action', async () => { + const options = mockV2MigrationOptions(); + options.client.tasks.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise({ + completed: true, + error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, + failures: [], + task: { description: 'task description' } as any, + }) + ); + + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` + [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] + `); + expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` + [Error: Reindex failed with the following error: + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] + `); }); }); }); @@ -306,7 +219,7 @@ type MockedOptions = KibanaMigratorOptions & { }; const mockV2MigrationOptions = () => { - const options = mockOptions({ enableV2: true }); + const options = mockOptions(); options.client.indices.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise( @@ -362,7 +275,7 @@ const mockV2MigrationOptions = () => { return options; }; -const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) => { +const mockOptions = () => { const options: MockedOptions = { logger: loggingSystemMock.create().get(), kibanaVersion: '8.2.3', @@ -401,7 +314,6 @@ const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) pollInterval: 20000, scrollDuration: '10m', skip: false, - enableV2, retryAttempts: 20, }, client: elasticsearchClientMock.createElasticsearchClient(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index 572b2934e49b82..d3755f8c7e666c 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -22,13 +22,7 @@ import { SavedObjectsSerializer, SavedObjectsRawDoc, } from '../../serialization'; -import { - buildActiveMappings, - createMigrationEsClient, - IndexMigrator, - MigrationResult, - MigrationStatus, -} from '../core'; +import { buildActiveMappings, MigrationResult, MigrationStatus } from '../core'; import { DocumentMigrator, VersionedTransformer } from '../core/document_migrator'; import { createIndexMap } from '../core/build_index_map'; import { SavedObjectsMigrationConfigType } from '../../saved_objects_config'; @@ -71,7 +65,6 @@ export class KibanaMigrator { status: 'waiting_to_start', }); private readonly activeMappings: IndexMapping; - private migrationsRetryDelay?: number; // TODO migrationsV2: make private once we remove migrations v1 public readonly kibanaVersion: string; // TODO migrationsV2: make private once we remove migrations v1 @@ -105,7 +98,6 @@ export class KibanaMigrator { // Building the active mappings (and associated md5sums) is an expensive // operation so we cache the result this.activeMappings = buildActiveMappings(this.mappingProperties); - this.migrationsRetryDelay = migrationsRetryDelay; } /** @@ -173,49 +165,28 @@ export class KibanaMigrator { }); const migrators = Object.keys(indexMap).map((index) => { - // TODO migrationsV2: remove old migrations algorithm - if (this.soMigrationsConfig.enableV2) { - return { - migrate: (): Promise => { - return runResilientMigrator({ - client: this.client, - kibanaVersion: this.kibanaVersion, - targetMappings: buildActiveMappings(indexMap[index].typeMappings), - logger: this.log, - preMigrationScript: indexMap[index].script, - transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => - migrateRawDocsSafely({ - serializer: this.serializer, - knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)), - migrateDoc: this.documentMigrator.migrateAndConvert, - rawDocs, - }), - migrationVersionPerType: this.documentMigrator.migrationVersion, - indexPrefix: index, - migrationsConfig: this.soMigrationsConfig, - typeRegistry: this.typeRegistry, - }); - }, - }; - } else { - return new IndexMigrator({ - batchSize: this.soMigrationsConfig.batchSize, - client: createMigrationEsClient(this.client, this.log, this.migrationsRetryDelay), - documentMigrator: this.documentMigrator, - index, - kibanaVersion: this.kibanaVersion, - log: this.log, - mappingProperties: indexMap[index].typeMappings, - setStatus: (status) => this.status$.next(status), - pollInterval: this.soMigrationsConfig.pollInterval, - scrollDuration: this.soMigrationsConfig.scrollDuration, - serializer: this.serializer, - // Only necessary for the migrator of the kibana index. - obsoleteIndexTemplatePattern: - index === kibanaIndexName ? 'kibana_index_template*' : undefined, - convertToAliasScript: indexMap[index].script, - }); - } + return { + migrate: (): Promise => { + return runResilientMigrator({ + client: this.client, + kibanaVersion: this.kibanaVersion, + targetMappings: buildActiveMappings(indexMap[index].typeMappings), + logger: this.log, + preMigrationScript: indexMap[index].script, + transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => + migrateRawDocsSafely({ + serializer: this.serializer, + knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)), + migrateDoc: this.documentMigrator.migrateAndConvert, + rawDocs, + }), + migrationVersionPerType: this.documentMigrator.migrationVersion, + indexPrefix: index, + migrationsConfig: this.soMigrationsConfig, + typeRegistry: this.typeRegistry, + }); + }, + }; }); return Promise.all(migrators.map((migrator) => migrator.migrate())); diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts index d072ccc5c976ed..3dcfb34c47a17b 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts @@ -49,7 +49,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts index d70e034703158a..a4ce95a9e05843 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts @@ -113,7 +113,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 250, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts index fb40bda81cba59..c8e17a64a3fa3f 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts @@ -155,7 +155,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts index f15e0c5d684d05..31c802bb561f97 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts @@ -218,7 +218,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts index ead46627e324db..9daae7eddafe05 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts @@ -118,7 +118,6 @@ function createRoot(options: { maxBatchSizeBytes?: number }) { { migrations: { skip: false, - enableV2: true, batchSize: 1000, maxBatchSizeBytes: options.maxBatchSizeBytes, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts index 4f456b0bc741f8..d61426d92d390d 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts @@ -88,7 +88,6 @@ function createRoot(options: { maxBatchSizeBytes?: number }) { { migrations: { skip: false, - enableV2: true, batchSize: 1000, maxBatchSizeBytes: options.maxBatchSizeBytes, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts index d76bbc786cffca..c84f72b1842618 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts @@ -28,7 +28,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts index 9738650b0db883..0f59d194614954 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts @@ -150,7 +150,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts index d19ae51315745b..7e4b52dea185d4 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts @@ -154,7 +154,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts index 8e01e11eaccfb7..12d04d89dee78b 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts @@ -77,7 +77,6 @@ describe('migrating from 7.3.0-xpack which used v1 migrations', () => { { migrations: { skip: false, - enableV2: true, // There are 53 docs in fixtures. Batch size configured to enforce 3 migration steps. batchSize: 20, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts index 755bb5f946e4fc..6956e53ebc7fae 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts @@ -67,7 +67,6 @@ function createRoot({ logFileName, hosts }: RootConfig) { }, migrations: { skip: false, - enableV2: true, batchSize: 100, // fixture contains 5000 docs }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts index 11c5b33c0fd3d9..ef92c823182d8e 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts @@ -67,7 +67,6 @@ async function createRoot({ logFileName }: CreateRootConfig) { }, migrations: { skip: false, - enableV2: true, batchSize: 100, // fixture contains 5000 docs }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts index 19d77637902f81..4718b5d1285fe4 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts @@ -96,7 +96,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts index 78eec5fe94ef37..84c25fcdef8b42 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts @@ -62,7 +62,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index 21468d75523204..338eecf1511747 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -45,7 +45,6 @@ describe('migrationsStateActionMachine', () => { pollInterval: 0, scrollDuration: '0s', skip: false, - enableV2: true, retryAttempts: 5, }, typeRegistry, diff --git a/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts b/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts index a9e1a41f01d916..2b6d64bef4f1af 100644 --- a/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts +++ b/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts @@ -9,18 +9,16 @@ import { IRouter } from '../../../http'; import { catchAndReturnBoomErrors } from '../utils'; import { deleteUnknownTypeObjects } from '../../deprecations'; -import { SavedObjectConfig } from '../../saved_objects_config'; import { KibanaConfigType } from '../../../kibana_config'; interface RouteDependencies { - config: SavedObjectConfig; kibanaConfig: KibanaConfigType; kibanaVersion: string; } export const registerDeleteUnknownTypesRoute = ( router: IRouter, - { config, kibanaConfig, kibanaVersion }: RouteDependencies + { kibanaConfig, kibanaVersion }: RouteDependencies ) => { router.post( { @@ -31,7 +29,6 @@ export const registerDeleteUnknownTypesRoute = ( await deleteUnknownTypeObjects({ esClient: context.core.elasticsearch.client, typeRegistry: context.core.savedObjects.typeRegistry, - savedObjectsConfig: config, kibanaConfig, kibanaVersion, }); diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index e2698c619b10c5..7aa6711a37c339 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -76,5 +76,5 @@ export function registerRoutes({ const internalRouter = http.createRouter('/internal/saved_objects/'); registerMigrateRoute(internalRouter, migratorPromise); - registerDeleteUnknownTypesRoute(internalRouter, { config, kibanaConfig, kibanaVersion }); + registerDeleteUnknownTypesRoute(internalRouter, { kibanaConfig, kibanaVersion }); } diff --git a/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts b/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts index fef2b2d5870e0c..0c7fbdda89fbfe 100644 --- a/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts @@ -13,7 +13,6 @@ import { elasticsearchServiceMock } from '../../../../../core/server/elasticsear import { typeRegistryMock } from '../../saved_objects_type_registry.mock'; import { setupServer } from '../test_utils'; import { KibanaConfigType } from '../../../kibana_config'; -import { SavedObjectConfig } from '../../saved_objects_config'; import { SavedObjectsType } from 'kibana/server'; type SetupServerReturn = UnwrapPromise>; @@ -24,13 +23,6 @@ describe('POST /internal/saved_objects/deprecations/_delete_unknown_types', () = enabled: true, index: '.kibana', }; - const config: SavedObjectConfig = { - maxImportExportSize: 10000, - maxImportPayloadBytes: 24000000, - migration: { - enableV2: true, - } as SavedObjectConfig['migration'], - }; let server: SetupServerReturn['server']; let httpSetup: SetupServerReturn['httpSetup']; @@ -54,7 +46,6 @@ describe('POST /internal/saved_objects/deprecations/_delete_unknown_types', () = registerDeleteUnknownTypesRoute(router, { kibanaVersion, kibanaConfig, - config, }); await server.start(); diff --git a/src/core/server/saved_objects/saved_objects_config.test.ts b/src/core/server/saved_objects/saved_objects_config.test.ts index 720b28403edf20..06b9e9661b746e 100644 --- a/src/core/server/saved_objects/saved_objects_config.test.ts +++ b/src/core/server/saved_objects/saved_objects_config.test.ts @@ -22,7 +22,7 @@ describe('migrations config', function () { const { messages } = applyMigrationsDeprecations({ enableV2: true }); expect(messages).toMatchInlineSnapshot(` Array [ - "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.", + "You no longer need to configure \\"migrations.enableV2\\".", ] `); }); @@ -31,7 +31,7 @@ describe('migrations config', function () { const { messages } = applyMigrationsDeprecations({ enableV2: false }); expect(messages).toMatchInlineSnapshot(` Array [ - "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.", + "You no longer need to configure \\"migrations.enableV2\\".", ] `); }); diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts index c9b4b4499fa808..02fbd974da4ae9 100644 --- a/src/core/server/saved_objects/saved_objects_config.ts +++ b/src/core/server/saved_objects/saved_objects_config.ts @@ -7,8 +7,8 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; +import { ConfigDeprecationProvider } from '../config'; import type { ServiceConfigDescriptor } from '../internal_types'; -import type { ConfigDeprecationProvider } from '../config'; const migrationSchema = schema.object({ batchSize: schema.number({ defaultValue: 1_000 }), @@ -16,29 +16,12 @@ const migrationSchema = schema.object({ scrollDuration: schema.string({ defaultValue: '15m' }), pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), - enableV2: schema.boolean({ defaultValue: true }), retryAttempts: schema.number({ defaultValue: 15 }), }); export type SavedObjectsMigrationConfigType = TypeOf; -const migrationDeprecations: ConfigDeprecationProvider = () => [ - (settings, fromPath, addDeprecation) => { - const migrationsConfig = settings[fromPath]; - if (migrationsConfig?.enableV2 !== undefined) { - addDeprecation({ - configPath: `${fromPath}.enableV2`, - message: - '"migrations.enableV2" is deprecated and will be removed in an upcoming release without any further notice.', - documentationUrl: 'https://ela.st/kbn-so-migration-v2', - correctiveActions: { - manualSteps: [`Remove "migrations.enableV2" from your kibana configs.`], - }, - }); - } - return settings; - }, -]; +const migrationDeprecations: ConfigDeprecationProvider = ({ unused }) => [unused('enableV2')]; export const savedObjectsMigrationConfig: ServiceConfigDescriptor = { diff --git a/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts b/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts index fa065b02b8050a..16e3ba9495f045 100644 --- a/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts +++ b/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts @@ -18,63 +18,27 @@ describe('getIndexForType', () => { typeRegistry = typeRegistryMock.create(); }); - describe('when migV2 is enabled', () => { - const migV2Enabled = true; - - it('returns the correct index for a type specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.foo-index_8.0.0'); - }); - - it('returns the correct index for a type not specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => undefined); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.kibana_8.0.0'); - }); + it('returns the correct index for a type specifying a custom index', () => { + typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); + expect( + getIndexForType({ + type: 'foo', + typeRegistry, + defaultIndex, + kibanaVersion, + }) + ).toEqual('.foo-index_8.0.0'); }); - describe('when migV2 is disabled', () => { - const migV2Enabled = false; - - it('returns the correct index for a type specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.foo-index'); - }); - - it('returns the correct index for a type not specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => undefined); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.kibana'); - }); + it('returns the correct index for a type not specifying a custom index', () => { + typeRegistry.getIndex.mockImplementation((type) => undefined); + expect( + getIndexForType({ + type: 'foo', + typeRegistry, + defaultIndex, + kibanaVersion, + }) + ).toEqual('.kibana_8.0.0'); }); }); diff --git a/src/core/server/saved_objects/service/lib/get_index_for_type.ts b/src/core/server/saved_objects/service/lib/get_index_for_type.ts index cef477e6dd8402..ae34e6063e0a56 100644 --- a/src/core/server/saved_objects/service/lib/get_index_for_type.ts +++ b/src/core/server/saved_objects/service/lib/get_index_for_type.ts @@ -11,7 +11,6 @@ import { ISavedObjectTypeRegistry } from '../../saved_objects_type_registry'; interface GetIndexForTypeOptions { type: string; typeRegistry: ISavedObjectTypeRegistry; - migV2Enabled: boolean; kibanaVersion: string; defaultIndex: string; } @@ -19,18 +18,8 @@ interface GetIndexForTypeOptions { export const getIndexForType = ({ type, typeRegistry, - migV2Enabled, defaultIndex, kibanaVersion, }: GetIndexForTypeOptions): string => { - // TODO migrationsV2: Remove once we remove migrations v1 - // This is a hacky, but it required the least amount of changes to - // existing code to support a migrations v2 index. Long term we would - // want to always use the type registry to resolve a type's index - // (including the default index). - if (migV2Enabled) { - return `${typeRegistry.getIndex(type) || defaultIndex}_${kibanaVersion}`; - } else { - return typeRegistry.getIndex(type) || defaultIndex; - } + return `${typeRegistry.getIndex(type) || defaultIndex}_${kibanaVersion}`; }; diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 82a0dd71700f65..45865a2fb988f9 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -612,12 +612,18 @@ describe('SavedObjectsRepository', () => { it(`should use default index`, async () => { await bulkCreateSuccess([obj1, obj2]); - expectClientCallArgsAction([obj1, obj2], { method: 'create', _index: '.kibana-test' }); + expectClientCallArgsAction([obj1, obj2], { + method: 'create', + _index: '.kibana-test_8.0.0-testing', + }); }); it(`should use custom index`, async () => { await bulkCreateSuccess([obj1, obj2].map((x) => ({ ...x, type: CUSTOM_INDEX_TYPE }))); - expectClientCallArgsAction([obj1, obj2], { method: 'create', _index: 'custom' }); + expectClientCallArgsAction([obj1, obj2], { + method: 'create', + _index: 'custom_8.0.0-testing', + }); }); it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => { @@ -2091,7 +2097,7 @@ describe('SavedObjectsRepository', () => { it(`should use default index`, async () => { await createSuccess(type, attributes, { id }); expect(client.create).toHaveBeenCalledWith( - expect.objectContaining({ index: '.kibana-test' }), + expect.objectContaining({ index: '.kibana-test_8.0.0-testing' }), expect.anything() ); }); @@ -2099,7 +2105,7 @@ describe('SavedObjectsRepository', () => { it(`should use custom index`, async () => { await createSuccess(CUSTOM_INDEX_TYPE, attributes, { id }); expect(client.create).toHaveBeenCalledWith( - expect.objectContaining({ index: 'custom' }), + expect.objectContaining({ index: 'custom_8.0.0-testing' }), expect.anything() ); }); @@ -2679,7 +2685,9 @@ describe('SavedObjectsRepository', () => { it(`should use all indices for types that are not namespace-agnostic`, async () => { await deleteByNamespaceSuccess(namespace); expect(client.updateByQuery).toHaveBeenCalledWith( - expect.objectContaining({ index: ['.kibana-test', 'custom'] }), + expect.objectContaining({ + index: ['.kibana-test_8.0.0-testing', 'custom_8.0.0-testing'], + }), expect.anything() ); }); @@ -2769,7 +2777,7 @@ describe('SavedObjectsRepository', () => { await removeReferencesToSuccess(); expect(client.updateByQuery).toHaveBeenCalledWith( expect.objectContaining({ - index: ['.kibana-test', 'custom'], + index: ['.kibana-test_8.0.0-testing', 'custom_8.0.0-testing'], }), expect.anything() ); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index c74092faad96a8..26ebef4d12ea5c 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -2077,7 +2077,6 @@ export class SavedObjectsRepository { defaultIndex: this._index, typeRegistry: this._registry, kibanaVersion: this._migrator.kibanaVersion, - migV2Enabled: this._migrator.soMigrationsConfig.enableV2, }); } diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 7d486e4717e575..73ed445d272a1e 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -109,7 +109,6 @@ kibana_vars=( map.tilemap.url migrations.batchSize migrations.maxBatchSizeBytes - migrations.enableV2 migrations.pollInterval migrations.retryAttempts migrations.scrollDuration diff --git a/x-pack/plugins/uptime/e2e/config.ts b/x-pack/plugins/uptime/e2e/config.ts index c5d573afccd967..d2c7a691e0a49f 100644 --- a/x-pack/plugins/uptime/e2e/config.ts +++ b/x-pack/plugins/uptime/e2e/config.ts @@ -43,7 +43,6 @@ async function config({ readConfigFile }: FtrConfigProviderContext) { `--uiSettings.overrides.theme:darkMode=true`, `--elasticsearch.username=kibana_system`, `--elasticsearch.password=changeme`, - '--migrations.enableV2=false', '--xpack.reporting.enabled=false', ], }, diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index ad1f897debe329..3bc03eb5b42597 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -11,22 +11,13 @@ import { createTestConfig, CreateTestConfig } from '../common/config'; const apmFtrConfigs = { basic: { license: 'basic' as const, - kibanaConfig: { - // disable v2 migrations to prevent issue where kibana index is deleted - // during a migration - 'migrations.enableV2': 'false', - }, }, trial: { license: 'trial' as const, - kibanaConfig: { - 'migrations.enableV2': 'false', - }, }, rules: { license: 'trial' as const, kibanaConfig: { - 'migrations.enableV2': 'false', 'xpack.ruleRegistry.write.enabled': 'true', }, }, From 058a83cb9c925df46b50c9210fe28f14d9ba9d9a Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Wed, 20 Oct 2021 13:58:47 -0700 Subject: [PATCH 25/35] fix doc build (#115863) --- docs/apm/getting-started.asciidoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/apm/getting-started.asciidoc b/docs/apm/getting-started.asciidoc index 7c7321dc37e8c8..2b4651a9fce971 100644 --- a/docs/apm/getting-started.asciidoc +++ b/docs/apm/getting-started.asciidoc @@ -41,11 +41,7 @@ Notice something awry? Select a service or trace and dive deeper with: * <> TIP: Want to learn more about the Elastic APM ecosystem? -<<<<<<< HEAD -See the {apm-overview-ref-v}/overview.html[APM Overview]. -======= See the {apm-guide-ref}/apm-overview.html[APM Overview]. ->>>>>>> 2daadc0d74d (docs: update links to APM docs (#115664)) include::services.asciidoc[] From 6604daf6be217c346a42def3cbe5c6628fab0e49 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 17:11:05 -0400 Subject: [PATCH 26/35] [Fleet] Add support for "Edit Package Policy" extensions using latest version of a package (#114914) (#115843) * Add support for extensions using latest version of a package and forcing upgrade state for edit policy view * Fix isUpgrade flag on integrations UI version of edit page * Treat non-validation errors as general failures in server and UI * Fix tests + don't call upgrade API when saving * fix i18n * Fix default name always appearing when editing package policies via extension UI * Opt security solution plugin out of new extension option Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kyle Pollich --- x-pack/plugins/apm/public/plugin.ts | 1 + .../components/layout.tsx | 6 +- .../step_define_package_policy.tsx | 11 +- .../create_package_policy_page/types.ts | 4 +- .../edit_package_policy_page/index.tsx | 49 ++++---- .../sections/epm/screens/policy/index.tsx | 16 ++- .../hooks/use_request/package_policy.ts | 7 ++ .../fleet/public/types/ui_extensions.ts | 1 + x-pack/plugins/fleet/server/errors/index.ts | 2 + .../fleet/server/services/package_policy.ts | 49 +++++--- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - x-pack/plugins/uptime/public/apps/plugin.ts | 1 + .../apis/package_policy/upgrade.ts | 113 ++++++++++++++++-- 14 files changed, 194 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index 1e1d9ce5687d10..3a439df245609d 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -367,6 +367,7 @@ export class ApmPlugin implements Plugin { fleet.registerExtension({ package: 'apm', view: 'package-policy-edit', + useLatestPackageVersion: true, Component: getLazyAPMPolicyEditExtension(), }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/layout.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/layout.tsx index 1de37912b05c9f..3daf7fa545f246 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/layout.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/layout.tsx @@ -61,7 +61,11 @@ export const CreatePackagePolicyPageLayout: React.FunctionComponent<{ const isEdit = useMemo(() => ['edit', 'package-edit'].includes(from), [from]); const isUpgrade = useMemo( () => - ['upgrade-from-fleet-policy-list', 'upgrade-from-integrations-policy-list'].includes(from), + [ + 'upgrade-from-fleet-policy-list', + 'upgrade-from-integrations-policy-list', + 'upgrade-from-extension', + ].includes(from), [from] ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx index 7e4896837013c9..29c226ca64f57e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_define_package_policy.tsx @@ -105,11 +105,12 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ agentPolicy?.id || '', packagePolicy.output_id, packagePolicy.namespace, - `${packageInfo.name}-${ - pkgPoliciesWithMatchingNames.length - ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 - : 1 - }`, + packagePolicy.name || + `${packageInfo.name}-${ + pkgPoliciesWithMatchingNames.length + ? pkgPoliciesWithMatchingNames[pkgPoliciesWithMatchingNames.length - 1] + 1 + : 1 + }`, packagePolicy.description, integrationToEnable ) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/types.ts b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/types.ts index d3b1fb12905963..82c18c34977df8 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/types.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/types.ts @@ -11,7 +11,9 @@ export type EditPackagePolicyFrom = | 'policy' | 'edit' | 'upgrade-from-fleet-policy-list' - | 'upgrade-from-integrations-policy-list'; + | 'upgrade-from-integrations-policy-list' + | 'upgrade-from-extension'; + export type PackagePolicyFormState = | 'VALID' | 'INVALID' diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx index 840a5a71a63c7f..71dfb610a91515 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx @@ -41,10 +41,12 @@ import { sendGetOneAgentPolicy, sendGetOnePackagePolicy, sendGetPackageInfoByKey, - sendUpgradePackagePolicy, sendUpgradePackagePolicyDryRun, } from '../../../hooks'; -import { useBreadcrumbs as useIntegrationsBreadcrumbs } from '../../../../integrations/hooks'; +import { + useBreadcrumbs as useIntegrationsBreadcrumbs, + useGetOnePackagePolicy, +} from '../../../../integrations/hooks'; import { Loading, Error, ExtensionWrapper } from '../../../components'; import { ConfirmDeployAgentPolicyModal } from '../components'; import { CreatePackagePolicyPageLayout } from '../create_package_policy_page/components'; @@ -68,7 +70,23 @@ export const EditPackagePolicyPage = memo(() => { params: { packagePolicyId }, } = useRouteMatch<{ policyId: string; packagePolicyId: string }>(); - return ; + const packagePolicy = useGetOnePackagePolicy(packagePolicyId); + + const extensionView = useUIExtension( + packagePolicy.data?.item?.package?.name ?? '', + 'package-policy-edit' + ); + + return ( + + ); }); export const EditPackagePolicyForm = memo<{ @@ -345,29 +363,6 @@ export const EditPackagePolicyForm = memo<{ const { error } = await savePackagePolicy(); if (!error) { - if (isUpgrade) { - const { error: upgradeError } = await sendUpgradePackagePolicy([packagePolicyId]); - - if (upgradeError) { - notifications.toasts.addError(upgradeError, { - title: i18n.translate('xpack.fleet.upgradePackagePolicy.failedNotificationTitle', { - defaultMessage: 'Error upgrading {packagePolicyName}', - values: { - packagePolicyName: packagePolicy.name, - }, - }), - toastMessage: i18n.translate( - 'xpack.fleet.editPackagePolicy.failedConflictNotificationMessage', - { - defaultMessage: `Data is out of date. Refresh the page to get the latest policy.`, - } - ), - }); - - return; - } - } - application.navigateToUrl(successRedirectPath); notifications.toasts.addSuccess({ title: i18n.translate('xpack.fleet.editPackagePolicy.updatedNotificationTitle', { @@ -426,7 +421,7 @@ export const EditPackagePolicyForm = memo<{ const [selectedTab, setSelectedTab] = useState(0); const layoutProps = { - from, + from: extensionView?.useLatestPackageVersion ? 'upgrade-from-extension' : from, cancelUrl, agentPolicy, packageInfo, diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/policy/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/policy/index.tsx index cb95634de4c076..ed8ad166cde9b7 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/policy/index.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/policy/index.tsx @@ -10,11 +10,25 @@ import { useRouteMatch } from 'react-router-dom'; // TODO: Needs to be moved import { EditPackagePolicyForm } from '../../../../../fleet/sections/agent_policy/edit_package_policy_page'; +import { useGetOnePackagePolicy, useUIExtension } from '../../../../hooks'; export const Policy = memo(() => { const { params: { packagePolicyId }, } = useRouteMatch<{ packagePolicyId: string }>(); - return ; + const packagePolicy = useGetOnePackagePolicy(packagePolicyId); + + const extensionView = useUIExtension( + packagePolicy.data?.item?.package?.name ?? '', + 'package-policy-edit' + ); + + return ( + + ); }); diff --git a/x-pack/plugins/fleet/public/hooks/use_request/package_policy.ts b/x-pack/plugins/fleet/public/hooks/use_request/package_policy.ts index f8d14647439b2c..5aa36ba2e8268f 100644 --- a/x-pack/plugins/fleet/public/hooks/use_request/package_policy.ts +++ b/x-pack/plugins/fleet/public/hooks/use_request/package_policy.ts @@ -59,6 +59,13 @@ export function useGetPackagePolicies(query: GetPackagePoliciesRequest['query']) }); } +export const useGetOnePackagePolicy = (packagePolicyId: string) => { + return useRequest({ + path: packagePolicyRouteService.getInfoPath(packagePolicyId), + method: 'get', + }); +}; + export const sendGetOnePackagePolicy = (packagePolicyId: string) => { return sendRequest({ path: packagePolicyRouteService.getInfoPath(packagePolicyId), diff --git a/x-pack/plugins/fleet/public/types/ui_extensions.ts b/x-pack/plugins/fleet/public/types/ui_extensions.ts index d365b798fe83e3..6c959bec1d785e 100644 --- a/x-pack/plugins/fleet/public/types/ui_extensions.ts +++ b/x-pack/plugins/fleet/public/types/ui_extensions.ts @@ -50,6 +50,7 @@ export interface PackagePolicyEditExtensionComponentProps { export interface PackagePolicyEditExtension { package: string; view: 'package-policy-edit'; + useLatestPackageVersion?: boolean; Component: LazyExoticComponent; } diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index f31719d6c43645..22f4b8cd6daaba 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -38,6 +38,8 @@ export class PackageCacheError extends IngestManagerError {} export class PackageOperationNotSupportedError extends IngestManagerError {} export class ConcurrentInstallOperationError extends IngestManagerError {} export class AgentReassignmentError extends IngestManagerError {} +export class PackagePolicyIneligibleForUpgradeError extends IngestManagerError {} +export class PackagePolicyValidationError extends IngestManagerError {} export class HostedAgentPolicyRestrictionRelatedError extends IngestManagerError { constructor(message = 'Cannot perform that action') { super( diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index b0c0b9499c68ec..9928ce3063159d 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -45,6 +45,8 @@ import { HostedAgentPolicyRestrictionRelatedError, IngestManagerError, ingestErrorToResponseOptions, + PackagePolicyIneligibleForUpgradeError, + PackagePolicyValidationError, } from '../errors'; import { NewPackagePolicySchema, UpdatePackagePolicySchema } from '../types'; import type { @@ -528,25 +530,25 @@ class PackagePolicyService { pkgName: packagePolicy.package.name, pkgVersion: installedPackage?.version ?? '', }); + } - const isInstalledVersionLessThanOrEqualToPolicyVersion = semverLte( - installedPackage?.version ?? '', - packagePolicy.package.version - ); + const isInstalledVersionLessThanOrEqualToPolicyVersion = semverLte( + packageInfo?.version ?? '', + packagePolicy.package.version + ); - if (isInstalledVersionLessThanOrEqualToPolicyVersion) { - throw new IngestManagerError( - i18n.translate('xpack.fleet.packagePolicy.ineligibleForUpgradeError', { - defaultMessage: - "Package policy {id}'s package version {version} of package {name} is up to date with the installed package. Please install the latest version of {name}.", - values: { - id: packagePolicy.id, - name: packagePolicy.package.name, - version: packagePolicy.package.version, - }, - }) - ); - } + if (isInstalledVersionLessThanOrEqualToPolicyVersion) { + throw new PackagePolicyIneligibleForUpgradeError( + i18n.translate('xpack.fleet.packagePolicy.ineligibleForUpgradeError', { + defaultMessage: + "Package policy {id}'s package version {version} of package {name} is up to date with the installed package. Please install the latest version of {name}.", + values: { + id: packagePolicy.id, + name: packagePolicy.package.name, + version: packagePolicy.package.version, + }, + }) + ); } return { @@ -600,6 +602,13 @@ class PackagePolicyService { currentVersion: packageInfo.version, }); } catch (error) { + // We only want to specifically handle validation errors for the new package policy. If a more severe or + // general error is thrown elsewhere during the upgrade process, we want to surface that directly in + // order to preserve any status code mappings, etc that might be included w/ the particular error type + if (!(error instanceof PackagePolicyValidationError)) { + throw error; + } + result.push({ id, success: false, @@ -653,6 +662,10 @@ class PackagePolicyService { hasErrors, }; } catch (error) { + if (!(error instanceof PackagePolicyValidationError)) { + throw error; + } + return { hasErrors: true, ...ingestErrorToResponseOptions(error), @@ -1089,7 +1102,7 @@ export function overridePackageInputs( return { ...resultingPackagePolicy, errors: responseFormattedValidationErrors }; } - throw new IngestManagerError( + throw new PackagePolicyValidationError( i18n.translate('xpack.fleet.packagePolicyInvalidError', { defaultMessage: 'Package policy is invalid: {errors}', values: { diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 07b9519f34050a..02102df6bbe910 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -11385,7 +11385,6 @@ "xpack.fleet.upgradeAgents.upgradeMultipleDescription": "このアクションにより、複数のエージェントがバージョン{version}にアップグレードされます。このアクションは元に戻せません。続行していいですか？", "xpack.fleet.upgradeAgents.upgradeSingleDescription": "このアクションにより、「{hostName}」で実行中のエージェントがバージョン{version}にアップグレードされます。このアクションは元に戻せません。続行していいですか？", "xpack.fleet.upgradeAgents.upgradeSingleTitle": "エージェントを最新バージョンにアップグレード", - "xpack.fleet.upgradePackagePolicy.failedNotificationTitle": "{packagePolicyName}のアップグレードエラー", "xpack.fleet.upgradePackagePolicy.pageDescriptionFromUpgrade": "この統合をアップグレードし、選択したエージェントポリシーに変更をデプロイします", "xpack.fleet.upgradePackagePolicy.previousVersionFlyout.title": "'{name}'パッケージポリシー", "xpack.fleet.upgradePackagePolicy.statusCallout.errorContent": "この統合には、バージョン{currentVersion}から{upgradeVersion}で競合するフィールドがあります。構成を確認して保存し、アップグレードを実行してください。{previousConfigurationLink}を参照して比較できます。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9e872393869e4e..9723b92890a21f 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -11509,7 +11509,6 @@ "xpack.fleet.upgradeAgents.upgradeMultipleTitle": "将{count, plural, one {代理} other { {count} 个代理} =true {所有选定代理}}升级到最新版本", "xpack.fleet.upgradeAgents.upgradeSingleDescription": "此操作会将“{hostName}”上运行的代理升级到版本 {version}。此操作无法撤消。是否确定要继续？", "xpack.fleet.upgradeAgents.upgradeSingleTitle": "将代理升级到最新版本", - "xpack.fleet.upgradePackagePolicy.failedNotificationTitle": "升级 {packagePolicyName} 时出错", "xpack.fleet.upgradePackagePolicy.pageDescriptionFromUpgrade": "升级此集成并将更改部署到选定代理策略", "xpack.fleet.upgradePackagePolicy.previousVersionFlyout.title": "“{name}”软件包策略", "xpack.fleet.upgradePackagePolicy.statusCallout.errorContent": "此集成在版本 {currentVersion} 和 {upgradeVersion} 之间有冲突字段。请复查配置并保存，以执行升级。您可以参考您的 {previousConfigurationLink}以进行比较。", diff --git a/x-pack/plugins/uptime/public/apps/plugin.ts b/x-pack/plugins/uptime/public/apps/plugin.ts index 124de9a60110cb..0cd0af6231c9c6 100644 --- a/x-pack/plugins/uptime/public/apps/plugin.ts +++ b/x-pack/plugins/uptime/public/apps/plugin.ts @@ -221,6 +221,7 @@ export class UptimePlugin registerExtension({ package: 'synthetics', view: 'package-policy-edit', + useLatestPackageVersion: true, Component: LazySyntheticsPolicyEditExtension, }); diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts index 0be2d7d0a74689..a61a77fd37f6b2 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts @@ -146,7 +146,7 @@ export default function (providerContext: FtrProviderContext) { describe('upgrade', function () { it('should respond with an error when "dryRun: false" is provided', async function () { - const { body }: { body: UpgradePackagePolicyResponse } = await supertest + await supertest .post(`/api/fleet/package_policies/upgrade`) .set('kbn-xsrf', 'xxxx') .send({ @@ -154,10 +154,7 @@ export default function (providerContext: FtrProviderContext) { dryRun: false, packageVersion: '0.2.0-add-non-required-test-var', }) - .expect(200); - - expect(body.length).to.be(1); - expect(body[0].success).to.be(false); + .expect(400); }); }); }); @@ -1041,31 +1038,121 @@ export default function (providerContext: FtrProviderContext) { describe('when package policy is not found', function () { it('should return an 200 with errors when "dryRun:true" is provided', async function () { - const { body }: { body: UpgradePackagePolicyDryRunResponse } = await supertest + await supertest .post(`/api/fleet/package_policies/upgrade`) .set('kbn-xsrf', 'xxxx') .send({ packagePolicyIds: ['xxxx', 'yyyy'], dryRun: true, }) - .expect(200); - - expect(body[0].hasErrors).to.be(true); - expect(body[1].hasErrors).to.be(true); + .expect(404); }); it('should return a 200 with errors and "success:false" when "dryRun:false" is provided', async function () { - const { body }: { body: UpgradePackagePolicyResponse } = await supertest + await supertest .post(`/api/fleet/package_policies/upgrade`) .set('kbn-xsrf', 'xxxx') .send({ packagePolicyIds: ['xxxx', 'yyyy'], dryRun: false, }) + .expect(404); + }); + }); + + describe("when policy's package version is up to date", function () { + withTestPackageVersion('0.1.0'); + + beforeEach(async function () { + const { body: agentPolicyResponse } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + }) .expect(200); - expect(body[0].success).to.be(false); - expect(body[1].success).to.be(false); + agentPolicyId = agentPolicyResponse.item.id; + + const { body: packagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'package_policy_upgrade_1', + description: '', + namespace: 'default', + policy_id: agentPolicyId, + enabled: true, + output_id: '', + inputs: [ + { + policy_template: 'package_policy_upgrade', + type: 'test_input', + enabled: true, + streams: [ + { + id: 'test-package_policy_upgrade-xxxx', + enabled: true, + data_stream: { + type: 'test_stream', + dataset: 'package_policy_upgrade.test_stream', + }, + }, + ], + }, + ], + package: { + name: 'package_policy_upgrade', + title: 'This is a test package for upgrading package policies', + version: '0.1.0', + }, + }) + .expect(200); + + packagePolicyId = packagePolicyResponse.item.id; + }); + + afterEach(async function () { + await supertest + .post(`/api/fleet/package_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ packagePolicyIds: [packagePolicyId] }) + .expect(200); + + await supertest + .post('/api/fleet/agent_policies/delete') + .set('kbn-xsrf', 'xxxx') + .send({ agentPolicyId }) + .expect(200); + }); + + describe('dry run', function () { + it('should respond with a bad request', async function () { + await supertest + .post(`/api/fleet/package_policies/upgrade`) + .set('kbn-xsrf', 'xxxx') + .send({ + packagePolicyIds: [packagePolicyId], + dryRun: true, + packageVersion: '0.1.0', + }) + .expect(400); + }); + }); + + describe('upgrade', function () { + it('should respond with a bad request', async function () { + await supertest + .post(`/api/fleet/package_policies/upgrade`) + .set('kbn-xsrf', 'xxxx') + .send({ + packagePolicyIds: [packagePolicyId], + dryRun: false, + packageVersion: '0.1.0', + }) + .expect(400); + }); }); }); }); From fbc2e342ffc3d7aa93b213dae94d2d6c783fdd8c Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 17:19:44 -0400 Subject: [PATCH 27/35] Update event view actions column width (#115809) (#115844) Co-authored-by: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> --- .../public/components/t_grid/event_rendered_view/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/timelines/public/components/t_grid/event_rendered_view/index.tsx b/x-pack/plugins/timelines/public/components/t_grid/event_rendered_view/index.tsx index 91bb6ad1dac0b8..523460bfd2d013 100644 --- a/x-pack/plugins/timelines/public/components/t_grid/event_rendered_view/index.tsx +++ b/x-pack/plugins/timelines/public/components/t_grid/event_rendered_view/index.tsx @@ -139,7 +139,7 @@ const EventRenderedViewComponent = ({ ); }, - width: '120px', + width: '152px', }, { field: 'ecs.timestamp', From 1cf5089cc605f41c75c0f4334cc3f4a1e4b0c9fd Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 20 Oct 2021 14:54:47 -0700 Subject: [PATCH 28/35] skip flaky suite (#115859) --- x-pack/test/accessibility/apps/upgrade_assistant.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/accessibility/apps/upgrade_assistant.ts b/x-pack/test/accessibility/apps/upgrade_assistant.ts index 452f6051e0640b..e387eed65d3fdb 100644 --- a/x-pack/test/accessibility/apps/upgrade_assistant.ts +++ b/x-pack/test/accessibility/apps/upgrade_assistant.ts @@ -52,7 +52,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const es = getService('es'); const log = getService('log'); - describe('Upgrade Assistant', () => { + // Failing: See https://github.com/elastic/kibana/issues/115859 + describe.skip('Upgrade Assistant', () => { before(async () => { await PageObjects.upgradeAssistant.navigateToPage(); From 9231d806c9384df4026977ba7435a9302dc2d4ab Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 20 Oct 2021 15:03:10 -0700 Subject: [PATCH 29/35] [Reporting] Add deprecation messages for roles mapped to reporting_user (#115125) * [Reporting] Add deprecation messages for roles mapped to reporting_user This fixes the deprecations for various edge cases: - when security was disabled, users saw a meaningless error message - when users did not have manage_security privilege they saw a meaningless error message Adds support for checking deprecations related to xpack.reporting.roles.allow * updates to content per feedback * updates per feedback * store packageInfo in reportingCore * use branch in the documentation links generated in the server * add tests for scenario where config does not need to be changed --- .../reporting/server/config/index.test.ts | 2 +- .../plugins/reporting/server/config/index.ts | 5 +- x-pack/plugins/reporting/server/core.ts | 9 +- .../__snapshots__/reporting_role.test.ts.snap | 113 +++++++++ .../deprecations/reporting_role.test.ts | 191 +++++++++------ .../server/deprecations/reporting_role.ts | 229 ++++++++++++++---- .../server/lib/deprecations/index.ts | 74 ++++++ .../server/routes/diagnostic/browser.test.ts | 12 +- .../routes/diagnostic/screenshot.test.ts | 12 +- .../server/routes/generate/legacy.ts | 2 +- .../create_mock_reportingplugin.ts | 8 +- 11 files changed, 519 insertions(+), 138 deletions(-) create mode 100644 x-pack/plugins/reporting/server/deprecations/__snapshots__/reporting_role.test.ts.snap diff --git a/x-pack/plugins/reporting/server/config/index.test.ts b/x-pack/plugins/reporting/server/config/index.test.ts index 1a75f6dfec3bdb..f77713551592b2 100644 --- a/x-pack/plugins/reporting/server/config/index.test.ts +++ b/x-pack/plugins/reporting/server/config/index.test.ts @@ -49,7 +49,7 @@ describe('deprecations', () => { const { messages } = applyReportingDeprecations({ roles: { enabled: true } }); expect(messages).toMatchInlineSnapshot(` Array [ - "Use Kibana application privileges to grant reporting privileges. Using \\"xpack.reporting.roles.allow\\" to grant reporting privileges prevents users from using API Keys to create reports. The \\"xpack.reporting.roles.enabled\\" setting will default to false in a future release.", + "Use Kibana application privileges to grant reporting privileges. Using \\"xpack.reporting.roles.allow\\" to grant reporting privileges is deprecated. The \\"xpack.reporting.roles.enabled\\" setting will default to false in a future release.", ] `); }); diff --git a/x-pack/plugins/reporting/server/config/index.ts b/x-pack/plugins/reporting/server/config/index.ts index 1eeafb4e0c5130..244a4577813dab 100644 --- a/x-pack/plugins/reporting/server/config/index.ts +++ b/x-pack/plugins/reporting/server/config/index.ts @@ -64,7 +64,7 @@ export const config: PluginConfigDescriptor = { defaultMessage: `Use Kibana application privileges to grant reporting privileges.` + ` Using "{fromPath}.roles.allow" to grant reporting privileges` + - ` prevents users from using API Keys to create reports.` + + ` is deprecated.` + ` The "{fromPath}.roles.enabled" setting will default to false` + ` in a future release.`, values: { fromPath }, @@ -74,6 +74,9 @@ export const config: PluginConfigDescriptor = { i18n.translate('xpack.reporting.deprecations.reportingRoles.manualStepOne', { defaultMessage: `Set "xpack.reporting.roles.enabled" to "false" in kibana.yml.`, }), + i18n.translate('xpack.reporting.deprecations.reportingRoles.manualStepOnePartOne', { + defaultMessage: `Remove "xpack.reporting.roles.allow" to "false" in kibana.yml, if present.`, + }), i18n.translate('xpack.reporting.deprecations.reportingRoles.manualStepTwo', { defaultMessage: `Create one or more roles that grant the Kibana application` + diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index e09cee8c3c7c2e..6b00e08c586858 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -13,6 +13,7 @@ import { BasePath, IClusterClient, KibanaRequest, + PackageInfo, PluginInitializerContext, SavedObjectsClientContract, SavedObjectsServiceStart, @@ -57,7 +58,7 @@ export interface ReportingInternalStart { } export class ReportingCore { - private kibanaVersion: string; + private packageInfo: PackageInfo; private pluginSetupDeps?: ReportingInternalSetup; private pluginStartDeps?: ReportingInternalStart; private readonly pluginSetup$ = new Rx.ReplaySubject(); // observe async background setupDeps and config each are done @@ -72,7 +73,7 @@ export class ReportingCore { public getContract: () => ReportingSetup; constructor(private logger: LevelLogger, context: PluginInitializerContext) { - this.kibanaVersion = context.env.packageInfo.version; + this.packageInfo = context.env.packageInfo; const syncConfig = context.config.get(); this.deprecatedAllowedRoles = syncConfig.roles.enabled ? syncConfig.roles.allow : false; this.executeTask = new ExecuteReportTask(this, syncConfig, this.logger); @@ -85,8 +86,8 @@ export class ReportingCore { this.executing = new Set(); } - public getKibanaVersion() { - return this.kibanaVersion; + public getKibanaPackageInfo() { + return this.packageInfo; } /* diff --git a/x-pack/plugins/reporting/server/deprecations/__snapshots__/reporting_role.test.ts.snap b/x-pack/plugins/reporting/server/deprecations/__snapshots__/reporting_role.test.ts.snap new file mode 100644 index 00000000000000..00a2a63280c9e9 --- /dev/null +++ b/x-pack/plugins/reporting/server/deprecations/__snapshots__/reporting_role.test.ts.snap @@ -0,0 +1,113 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`roles mapped to a deprecated role includes steps to remove the incompatible config, when applicable 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Set \\"xpack.reporting.roles.enabled: false\\" in kibana.yml.", + "Remove \\"xpack.reporting.roles.allow\\" in kibana.yml, if present.", + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all users and add the custom role. The affected users are: reportron[reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing users have their Reporting privilege granted by a deprecated setting.", + "title": "The \\"reporting_user\\" role is deprecated: check user roles", + }, +] +`; + +exports[`roles mapped to a deprecated role logs a deprecation when a role was found that maps to a deprecated custom role from the roles.allow setting 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all role mappings and add the custom role. The affected role mappings are: dungeon_master[my_test_reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing roles are mapped to a deprecated role for Reporting privileges", + "title": "The \\"reporting_user\\" role is deprecated: check role mappings", + }, +] +`; + +exports[`roles mapped to a deprecated role logs a deprecation when a role was found that maps to the deprecated reporting_user role 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all role mappings and add the custom role. The affected role mappings are: dungeon_master[reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing roles are mapped to a deprecated role for Reporting privileges", + "title": "The \\"reporting_user\\" role is deprecated: check role mappings", + }, +] +`; + +exports[`users assigned to a deprecated role includes steps to remove the incompatible config, when applicable 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Set \\"xpack.reporting.roles.enabled: false\\" in kibana.yml.", + "Remove \\"xpack.reporting.roles.allow\\" in kibana.yml, if present.", + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all users and add the custom role. The affected users are: reportron[reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing users have their Reporting privilege granted by a deprecated setting.", + "title": "The \\"reporting_user\\" role is deprecated: check user roles", + }, +] +`; + +exports[`users assigned to a deprecated role logs a deprecation when a user was found with a deprecated custom role from the roles.allow setting 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all users and add the custom role. The affected users are: reportron[my_test_reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing users have their Reporting privilege granted by a deprecated setting.", + "title": "The \\"reporting_user\\" role is deprecated: check user roles", + }, +] +`; + +exports[`users assigned to a deprecated role logs a deprecation when a user was found with a deprecated reporting_user role 1`] = ` +Array [ + Object { + "correctiveActions": Object { + "manualSteps": Array [ + "Create a custom role with Kibana privileges to grant access to Reporting.", + "Remove the \\"reporting_user\\" role from all users and add the custom role. The affected users are: reportron[reporting_user].", + ], + }, + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/branch/kibana-privileges.html", + "level": "warning", + "message": "Existing users have their Reporting privilege granted by a deprecated setting.", + "title": "The \\"reporting_user\\" role is deprecated: check user roles", + }, +] +`; diff --git a/x-pack/plugins/reporting/server/deprecations/reporting_role.test.ts b/x-pack/plugins/reporting/server/deprecations/reporting_role.test.ts index 5b0719bf6e6b6a..2286a9767f0001 100644 --- a/x-pack/plugins/reporting/server/deprecations/reporting_role.test.ts +++ b/x-pack/plugins/reporting/server/deprecations/reporting_role.test.ts @@ -5,19 +5,25 @@ * 2.0. */ +import { GetDeprecationsContext, IScopedClusterClient } from 'kibana/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { ReportingCore } from '..'; +import { + createMockConfigSchema, + createMockPluginSetup, + createMockReportingCore, +} from '../test_helpers'; import { getDeprecationsInfo } from './reporting_role'; -import { createMockConfigSchema, createMockReportingCore } from '../test_helpers'; -import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import { GetDeprecationsContext, IScopedClusterClient } from 'kibana/server'; let reportingCore: ReportingCore; let context: GetDeprecationsContext; let esClient: jest.Mocked; beforeEach(async () => { - const mockReportingConfig = createMockConfigSchema({ roles: { enabled: false } }); - reportingCore = await createMockReportingCore(mockReportingConfig); + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { enabled: false } }) + ); + esClient = elasticsearchServiceMock.createScopedClusterClient(); esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ body: { xyz: { username: 'normal_user', roles: ['data_analyst'] } }, @@ -26,95 +32,132 @@ beforeEach(async () => { }); test('logs no deprecations when setup has no issues', async () => { - expect( - await getDeprecationsInfo(context, { - reportingCore, - }) - ).toMatchInlineSnapshot(`Array []`); + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchInlineSnapshot(`Array []`); }); -test('logs a plain message when only a reporting_user role issue is found', async () => { - esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ - body: { reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] } }, +describe('users assigned to a deprecated role', () => { + test('logs a deprecation when a user was found with a deprecated reporting_user role', async () => { + esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ + body: { reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] } }, + }); + + reportingCore = await createMockReportingCore(createMockConfigSchema()); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); }); - expect( - await getDeprecationsInfo(context, { - reportingCore, - }) - ).toMatchInlineSnapshot(` - Array [ - Object { - "correctiveActions": Object { - "manualSteps": Array [ - "Create one or more custom roles that provide Kibana application privileges to reporting features in **Management > Security > Roles**.", - "Assign the custom role(s) as desired, and remove the \\"reporting_user\\" role from the user(s).", - ], - }, - "documentationUrl": "https://www.elastic.co/guide/en/kibana/current/secure-reporting.html", - "level": "critical", - "message": "The deprecated \\"reporting_user\\" role has been found for 1 user(s): \\"reportron\\"", - "title": "Found deprecated reporting role", + + test('logs a deprecation when a user was found with a deprecated custom role from the roles.allow setting', async () => { + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { allow: ['my_test_reporting_user'] } }) + ); + esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ + body: { + reportron: { username: 'reportron', roles: ['kibana_admin', 'my_test_reporting_user'] }, }, - ] - `); + }); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); + }); + + test('includes steps to remove the incompatible config, when applicable', async () => { + esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ + body: { reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] } }, + }); + + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { enabled: true } }) + ); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); + }); }); -test('logs multiple entries when multiple reporting_user role issues are found', async () => { - esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ - body: { - reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] }, - supercooluser: { username: 'supercooluser', roles: ['kibana_admin', 'reporting_user'] }, - }, +describe('roles mapped to a deprecated role', () => { + test('logs a deprecation when a role was found that maps to the deprecated reporting_user role', async () => { + esClient.asCurrentUser.security.getRoleMapping = jest.fn().mockResolvedValue({ + body: { dungeon_master: { roles: ['reporting_user'] } }, + }); + + reportingCore = await createMockReportingCore(createMockConfigSchema()); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); + }); + + test('logs a deprecation when a role was found that maps to a deprecated custom role from the roles.allow setting', async () => { + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { allow: ['my_test_reporting_user'] } }) + ); + esClient.asCurrentUser.security.getRoleMapping = jest.fn().mockResolvedValue({ + body: { dungeon_master: { roles: ['my_test_reporting_user'] } }, + }); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); }); - expect( - await getDeprecationsInfo(context, { - reportingCore, - }) - ).toMatchInlineSnapshot(` + test('includes steps to remove the incompatible config, when applicable', async () => { + esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ + body: { reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] } }, + }); + + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { enabled: true } }) + ); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchSnapshot(); + }); +}); + +describe('check deprecations when security is disabled', () => { + test('logs no deprecations: roles not enabled', async () => { + reportingCore = await createMockReportingCore( + createMockConfigSchema({ roles: { enabled: false } }), + createMockPluginSetup({ security: null }) + ); + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchInlineSnapshot(`Array []`); + }); + + test('logs no deprecations: roles enabled', async () => { + const mockReportingConfig = createMockConfigSchema(); // roles.enabled: true is default in 7.x / 8.0 + reportingCore = await createMockReportingCore( + mockReportingConfig, + createMockPluginSetup({ security: null }) + ); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchInlineSnapshot(`Array []`); + }); +}); + +it('insufficient permissions', async () => { + const permissionsError = new Error('you shall not pass'); + (permissionsError as unknown as { statusCode: number }).statusCode = 403; + esClient.asCurrentUser.security.getUser = jest.fn().mockRejectedValue(permissionsError); + esClient.asCurrentUser.security.getRoleMapping = jest.fn().mockRejectedValue(permissionsError); + + expect(await getDeprecationsInfo(context, { reportingCore })).toMatchInlineSnapshot(` Array [ Object { "correctiveActions": Object { "manualSteps": Array [ - "Create one or more custom roles that provide Kibana application privileges to reporting features in **Management > Security > Roles**.", - "Assign the custom role(s) as desired, and remove the \\"reporting_user\\" role from the user(s).", + "Make sure you have a \\"manage_security\\" cluster privilege assigned.", ], }, - "documentationUrl": "https://www.elastic.co/guide/en/kibana/current/secure-reporting.html", - "level": "critical", - "message": "The deprecated \\"reporting_user\\" role has been found for 2 user(s): \\"reportron\\", \\"supercooluser\\"", - "title": "Found deprecated reporting role", + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/current/xpack-security.html#_required_permissions_7", + "level": "fetch_error", + "message": "You do not have enough permissions to fix this deprecation.", + "title": "The \\"reporting_user\\" role is deprecated: check user roles", }, - ] - `); -}); - -test('logs an expanded message when a config issue and a reporting_user role issue is found', async () => { - esClient.asCurrentUser.security.getUser = jest.fn().mockResolvedValue({ - body: { reportron: { username: 'reportron', roles: ['kibana_admin', 'reporting_user'] } }, - }); - - const mockReportingConfig = createMockConfigSchema({ roles: { enabled: true } }); - reportingCore = await createMockReportingCore(mockReportingConfig); - - expect( - await getDeprecationsInfo(context, { - reportingCore, - }) - ).toMatchInlineSnapshot(` - Array [ Object { "correctiveActions": Object { "manualSteps": Array [ - "Set \\"xpack.reporting.roles.enabled: false\\" in kibana.yml", - "Create one or more custom roles that provide Kibana application privileges to reporting features in **Management > Security > Roles**.", - "Assign the custom role(s) as desired, and remove the \\"reporting_user\\" role from the user(s).", + "Make sure you have a \\"manage_security\\" cluster privilege assigned.", ], }, - "documentationUrl": "https://www.elastic.co/guide/en/kibana/current/secure-reporting.html", - "level": "critical", - "message": "The deprecated \\"reporting_user\\" role has been found for 1 user(s): \\"reportron\\"", - "title": "Found deprecated reporting role", + "deprecationType": "feature", + "documentationUrl": "https://www.elastic.co/guide/en/kibana/current/xpack-security.html#_required_permissions_7", + "level": "fetch_error", + "message": "You do not have enough permissions to fix this deprecation.", + "title": "The \\"reporting_user\\" role is deprecated: check role mappings", }, ] `); diff --git a/x-pack/plugins/reporting/server/deprecations/reporting_role.ts b/x-pack/plugins/reporting/server/deprecations/reporting_role.ts index 6e08169727d1bc..a2a7e9c78726da 100644 --- a/x-pack/plugins/reporting/server/deprecations/reporting_role.ts +++ b/x-pack/plugins/reporting/server/deprecations/reporting_role.ts @@ -5,65 +5,206 @@ * 2.0. */ -import type { GetDeprecationsContext, DeprecationsDetails } from 'src/core/server'; +import { + SecurityGetRoleMappingResponse, + SecurityGetUserResponse, +} from '@elastic/elasticsearch/api/types'; import { i18n } from '@kbn/i18n'; -import { ReportingCore } from '..'; +import type { + DeprecationsDetails, + ElasticsearchClient, + GetDeprecationsContext, +} from 'src/core/server'; +import { ReportingCore } from '../'; +import { deprecations } from '../lib/deprecations'; -const deprecatedRole = 'reporting_user'; -const upgradableConfig = 'xpack.reporting.roles.enabled: false'; +const REPORTING_USER_ROLE_NAME = 'reporting_user'; +const getDocumentationUrl = (branch: string) => + `https://www.elastic.co/guide/en/kibana/${branch}/kibana-privileges.html`; interface ExtraDependencies { reportingCore: ReportingCore; } -export const getDeprecationsInfo = async ( +export async function getDeprecationsInfo( { esClient }: GetDeprecationsContext, { reportingCore }: ExtraDependencies -): Promise => { +): Promise { + const client = esClient.asCurrentUser; + const { security } = reportingCore.getPluginSetupDeps(); + + // Nothing to do if security is disabled + if (!security?.license.isEnabled()) { + return []; + } + + const config = reportingCore.getConfig(); + const deprecatedRoles = config.get('roles', 'allow') || ['reporting_user']; + + return [ + ...(await getUsersDeprecations(client, reportingCore, deprecatedRoles)), + ...(await getRoleMappingsDeprecations(client, reportingCore, deprecatedRoles)), + ]; +} + +async function getUsersDeprecations( + client: ElasticsearchClient, + reportingCore: ReportingCore, + deprecatedRoles: string[] +): Promise { const usingDeprecatedConfig = !reportingCore.getContract().usesUiCapabilities(); - const deprecations: DeprecationsDetails[] = []; - const { body: users } = await esClient.asCurrentUser.security.getUser(); + const strings = { + title: i18n.translate('xpack.reporting.deprecations.reportingRoleUsersTitle', { + defaultMessage: 'The "{reportingUserRoleName}" role is deprecated: check user roles', + values: { reportingUserRoleName: REPORTING_USER_ROLE_NAME }, + }), + message: i18n.translate('xpack.reporting.deprecations.reportingRoleUsersMessage', { + defaultMessage: + 'Existing users have their Reporting privilege granted by a deprecated setting.', + }), + manualSteps: (usersRoles: string) => [ + ...(usingDeprecatedConfig + ? [ + i18n.translate('xpack.reporting.deprecations.reportingRoleUsers.manualStepOne', { + defaultMessage: 'Set "xpack.reporting.roles.enabled: false" in kibana.yml.', + }), + i18n.translate('xpack.reporting.deprecations.reportingRoleUsers.manualStepTwo', { + defaultMessage: 'Remove "xpack.reporting.roles.allow" in kibana.yml, if present.', + }), + ] + : []), + + i18n.translate('xpack.reporting.deprecations.reportingRoleUsers.manualStepThree', { + defaultMessage: 'Create a custom role with Kibana privileges to grant access to Reporting.', + }), + i18n.translate('xpack.reporting.deprecations.reportingRoleUsers.manualStepFour', { + defaultMessage: + 'Remove the "reporting_user" role from all users and add the custom role. The affected users are: {usersRoles}.', + values: { usersRoles }, + }), + ], + }; + + let users: SecurityGetUserResponse; + try { + users = (await client.security.getUser()).body; + } catch (err) { + const { logger } = reportingCore.getPluginSetupDeps(); + if (deprecations.getErrorStatusCode(err) === 403) { + logger.warn( + `Failed to retrieve users when checking for deprecations:` + + ` the "manage_security" cluster privilege is required.` + ); + } else { + logger.error( + `Failed to retrieve users when checking for deprecations,` + + ` unexpected error: ${deprecations.getDetailedErrorMessage(err)}.` + ); + } + return deprecations.deprecationError(strings.title, err); + } + + const reportingUsers = Object.entries(users).reduce((userSet, current) => { + const [userName, user] = current; + const foundRole = user.roles.find((role) => deprecatedRoles.includes(role)); + return foundRole ? [...userSet, `${userName}[${foundRole}]`] : userSet; + }, [] as string[]); - const reportingUsers = Object.entries(users) - .filter(([, user]) => user.roles.includes(deprecatedRole)) - .map(([, user]) => user.username); + if (reportingUsers.length === 0) { + return []; + } + + return [ + { + title: strings.title, + message: strings.message, + correctiveActions: { manualSteps: strings.manualSteps(reportingUsers.join(', ')) }, + level: 'warning', + deprecationType: 'feature', + documentationUrl: getDocumentationUrl(reportingCore.getKibanaPackageInfo().branch), + }, + ]; +} - const numReportingUsers = reportingUsers.length; +async function getRoleMappingsDeprecations( + client: ElasticsearchClient, + reportingCore: ReportingCore, + deprecatedRoles: string[] +): Promise { + const usingDeprecatedConfig = !reportingCore.getContract().usesUiCapabilities(); + const strings = { + title: i18n.translate('xpack.reporting.deprecations.reportingRoleMappingsTitle', { + defaultMessage: 'The "{reportingUserRoleName}" role is deprecated: check role mappings', + values: { reportingUserRoleName: REPORTING_USER_ROLE_NAME }, + }), + message: i18n.translate('xpack.reporting.deprecations.reportingRoleMappingsMessage', { + defaultMessage: 'Existing roles are mapped to a deprecated role for Reporting privileges', + }), + manualSteps: (roleMappings: string) => [ + ...(usingDeprecatedConfig + ? [ + i18n.translate('xpack.reporting.deprecations.reportingRoleMappings.manualStepOne', { + defaultMessage: 'Set "xpack.reporting.roles.enabled: false" in kibana.yml.', + }), + i18n.translate('xpack.reporting.deprecations.reportingRoleMappings.manualStepTwo', { + defaultMessage: 'Remove "xpack.reporting.roles.allow" in kibana.yml, if present.', + }), + ] + : []), - if (numReportingUsers > 0) { - deprecations.push({ - title: i18n.translate('xpack.reporting.deprecations.reportingRoleTitle', { - defaultMessage: 'Found deprecated reporting role', + i18n.translate('xpack.reporting.deprecations.reportingRoleMappings.manualStepThree', { + defaultMessage: 'Create a custom role with Kibana privileges to grant access to Reporting.', }), - message: i18n.translate('xpack.reporting.deprecations.reportingRoleMessage', { + i18n.translate('xpack.reporting.deprecations.reportingRoleMappings.manualStepFour', { defaultMessage: - 'The deprecated "{deprecatedRole}" role has been found for {numReportingUsers} user(s): "{usernames}"', - values: { deprecatedRole, numReportingUsers, usernames: reportingUsers.join('", "') }, + 'Remove the "reporting_user" role from all role mappings and add the custom role. The affected role mappings are: {roleMappings}.', + values: { roleMappings }, }), - documentationUrl: 'https://www.elastic.co/guide/en/kibana/current/secure-reporting.html', - level: 'critical', - correctiveActions: { - manualSteps: [ - ...(usingDeprecatedConfig - ? [ - i18n.translate('xpack.reporting.deprecations.reportingRole.manualStepOneMessage', { - defaultMessage: 'Set "{upgradableConfig}" in kibana.yml', - values: { upgradableConfig }, - }), - ] - : []), - i18n.translate('xpack.reporting.deprecations.reportingRole.manualStepTwoMessage', { - defaultMessage: `Create one or more custom roles that provide Kibana application privileges to reporting features in **Management > Security > Roles**.`, - }), - i18n.translate('xpack.reporting.deprecations.reportingRole.manualStepThreeMessage', { - defaultMessage: - 'Assign the custom role(s) as desired, and remove the "{deprecatedRole}" role from the user(s).', - values: { deprecatedRole }, - }), - ], - }, - }); + ], + }; + + let roleMappings: SecurityGetRoleMappingResponse; + try { + roleMappings = (await client.security.getRoleMapping()).body; + } catch (err) { + const { logger } = reportingCore.getPluginSetupDeps(); + if (deprecations.getErrorStatusCode(err) === 403) { + logger.warn( + `Failed to retrieve role mappings when checking for deprecations:` + + ` the "manage_security" cluster privilege is required.` + ); + } else { + logger.error( + `Failed to retrieve role mappings when checking for deprecations,` + + ` unexpected error: ${deprecations.getDetailedErrorMessage(err)}.` + ); + } + return deprecations.deprecationError(strings.title, err); } - return deprecations; -}; + const roleMappingsWithReportingRole: string[] = Object.entries(roleMappings).reduce( + (roleSet, current) => { + const [roleName, role] = current; + const foundMapping = role.roles.find((roll) => deprecatedRoles.includes(roll)); + return foundMapping ? [...roleSet, `${roleName}[${foundMapping}]`] : roleSet; + }, + [] as string[] + ); + + if (roleMappingsWithReportingRole.length === 0) { + return []; + } + + return [ + { + title: strings.title, + message: strings.message, + correctiveActions: { + manualSteps: strings.manualSteps(roleMappingsWithReportingRole.join(', ')), + }, + level: 'warning', + deprecationType: 'feature', + documentationUrl: getDocumentationUrl(reportingCore.getKibanaPackageInfo().branch), + }, + ]; +} diff --git a/x-pack/plugins/reporting/server/lib/deprecations/index.ts b/x-pack/plugins/reporting/server/lib/deprecations/index.ts index 95594940e07e23..2d55c3b4c22d85 100644 --- a/x-pack/plugins/reporting/server/lib/deprecations/index.ts +++ b/x-pack/plugins/reporting/server/lib/deprecations/index.ts @@ -5,8 +5,82 @@ * 2.0. */ +import { errors } from '@elastic/elasticsearch'; +import Boom from '@hapi/boom'; +import { i18n } from '@kbn/i18n'; +import { DeprecationsDetails } from 'kibana/server'; import { checkIlmMigrationStatus } from './check_ilm_migration_status'; +function deprecationError(title: string, error: Error): DeprecationsDetails[] { + if (getErrorStatusCode(error) === 403) { + return [ + { + title, + level: 'fetch_error', // NOTE: is fetch_error not shown in the Upgrade Assistant UI? + deprecationType: 'feature', + message: i18n.translate( + 'xpack.reporting.deprecations.reportingRole.forbiddenErrorMessage', + { defaultMessage: 'You do not have enough permissions to fix this deprecation.' } + ), + documentationUrl: `https://www.elastic.co/guide/en/kibana/current/xpack-security.html#_required_permissions_7`, + correctiveActions: { + manualSteps: [ + i18n.translate( + 'xpack.reporting.deprecations.reportingRole.forbiddenErrorCorrectiveAction', + { + defaultMessage: + 'Make sure you have a "manage_security" cluster privilege assigned.', + } + ), + ], + }, + }, + ]; + } + + return [ + { + title, + level: 'fetch_error', // NOTE: is fetch_error not shown in the Upgrade Assistant UI? + deprecationType: 'feature', + message: i18n.translate('xpack.reporting.deprecations.reportingRole.unknownErrorMessage', { + defaultMessage: 'Failed to perform deprecation check. Check Kibana logs for more details.', + }), + correctiveActions: { + manualSteps: [ + i18n.translate( + 'xpack.reporting.deprecations.reportingRole.unknownErrorCorrectiveAction', + { defaultMessage: 'Check Kibana logs for more details.' } + ), + ], + }, + }, + ]; +} + +function getErrorStatusCode(error: any): number { + if (error instanceof errors.ResponseError) { + return error.statusCode; + } + + return Boom.isBoom(error) ? error.output.statusCode : error.statusCode || error.status; +} + +function getDetailedErrorMessage(error: any): string { + if (error instanceof errors.ResponseError) { + return JSON.stringify(error.body); + } + + if (Boom.isBoom(error)) { + return JSON.stringify(error.output.payload); + } + + return error.message; +} + export const deprecations = { checkIlmMigrationStatus, + deprecationError, + getDetailedErrorMessage, + getErrorStatusCode, }; diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts index f32e1b437bc335..7677f37702f0d8 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts @@ -52,11 +52,13 @@ describe('POST /diagnose/browser', () => { () => ({ usesUiCapabilities: () => false }) ); - const mockSetupDeps = createMockPluginSetup({ - router: httpSetup.createRouter(''), - }); - - core = await createMockReportingCore(config, mockSetupDeps); + core = await createMockReportingCore( + config, + createMockPluginSetup({ + router: httpSetup.createRouter(''), + security: null, + }) + ); mockedSpawn.mockImplementation(() => ({ removeAllListeners: jest.fn(), diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts index 6d844f9637a0bd..dd543707fe66a0 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts @@ -50,11 +50,13 @@ describe('POST /diagnose/screenshot', () => { () => ({ usesUiCapabilities: () => false }) ); - const mockSetupDeps = createMockPluginSetup({ - router: httpSetup.createRouter(''), - }); - - core = await createMockReportingCore(config, mockSetupDeps); + core = await createMockReportingCore( + config, + createMockPluginSetup({ + router: httpSetup.createRouter(''), + security: null, + }) + ); }); afterEach(async () => { diff --git a/x-pack/plugins/reporting/server/routes/generate/legacy.ts b/x-pack/plugins/reporting/server/routes/generate/legacy.ts index 92f1784dc8ecad..f262d186d55317 100644 --- a/x-pack/plugins/reporting/server/routes/generate/legacy.ts +++ b/x-pack/plugins/reporting/server/routes/generate/legacy.ts @@ -53,7 +53,7 @@ export function registerLegacy(reporting: ReportingCore, logger: LevelLogger) { savedObjectId, browserTimezone, queryString, - version: reporting.getKibanaVersion(), + version: reporting.getKibanaPackageInfo().version, }); } catch (err) { throw requestHandler.handleError(err); diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index 9e58d6d4efa41b..d62cc750ccfcc2 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -12,11 +12,13 @@ jest.mock('../browsers'); import _ from 'lodash'; import * as Rx from 'rxjs'; import { coreMock, elasticsearchServiceMock } from 'src/core/server/mocks'; -import { FieldFormatsRegistry } from 'src/plugins/field_formats/common'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { dataPluginMock } from 'src/plugins/data/server/mocks'; +import { FieldFormatsRegistry } from 'src/plugins/field_formats/common'; import { ReportingConfig, ReportingCore } from '../'; import { featuresPluginMock } from '../../../features/server/mocks'; +import { securityMock } from '../../../security/server/mocks'; +import { taskManagerMock } from '../../../task_manager/server/mocks'; import { chromium, HeadlessChromiumDriverFactory, @@ -39,9 +41,9 @@ export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup = features: featuresPluginMock.createSetup(), basePath: { set: jest.fn() }, router: setupMock.router, - security: setupMock.security, + security: securityMock.createSetup(), licensing: { license$: Rx.of({ isAvailable: true, isActive: true, type: 'basic' }) } as any, - taskManager: { registerTaskDefinitions: jest.fn() } as any, + taskManager: taskManagerMock.createSetup(), logger: createMockLevelLogger(), ...setupMock, }; From 252749f0addfaab58e3b7c1d5e3a935b4f2de951 Mon Sep 17 00:00:00 2001 From: Greg Back <1045796+gtback@users.noreply.github.com> Date: Wed, 20 Oct 2021 20:22:35 -0400 Subject: [PATCH 30/35] Hard-code 7.x links to client libraries. (#115879) These four libraries still publish 7.x branches rather than 7.16 --- src/core/public/doc_links/doc_links_service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index 185439172b831b..7068c8dbce5dc5 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -504,12 +504,12 @@ export class DocLinksService { clients: { /** Changes to these URLs must also be synched in src/plugins/custom_integrations/server/language_clients/index.ts */ guide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/index.html`, - goOverview: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/go-api/${DOC_LINK_VERSION}/overview.html`, + goOverview: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/go-api/7.x/overview.html`, javaIndex: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/java-api-client/${DOC_LINK_VERSION}/index.html`, - jsIntro: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/javascript-api/${DOC_LINK_VERSION}/introduction.html`, - netGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/net-api/${DOC_LINK_VERSION}/index.html`, + jsIntro: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/javascript-api/7.x/introduction.html`, + netGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/net-api/7.x/index.html`, perlGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/perl-api/current/index.html`, - phpGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/php-api/${DOC_LINK_VERSION}/index.html`, + phpGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/php-api/$7.x/index.html`, pythonGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/net-api/${DOC_LINK_VERSION}/index.html`, rubyOverview: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/ruby-api/${DOC_LINK_VERSION}/ruby_client.html`, rustGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/rust-api/current/index.html`, From 1d1e94732663bf924755e7e46a557074f3ad2538 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 20:31:33 -0400 Subject: [PATCH 31/35] [buildkite] FTSR / limit concurrency and total jobs (#115857) (#115877) Co-authored-by: Brian Seeders --- .buildkite/pipelines/flaky_tests/pipeline.js | 8 +------ .buildkite/pipelines/flaky_tests/runner.js | 25 ++++++++++++++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipelines/flaky_tests/pipeline.js b/.buildkite/pipelines/flaky_tests/pipeline.js index 1d390ed0d1b63a..5f3633860dfe3a 100644 --- a/.buildkite/pipelines/flaky_tests/pipeline.js +++ b/.buildkite/pipelines/flaky_tests/pipeline.js @@ -17,12 +17,6 @@ const inputs = [ default: 0, required: true, }, - { - key: 'ftsr-concurrency', - text: 'Max concurrency per step', - default: 20, - required: true, - }, ]; for (let i = 1; i <= OSS_CI_GROUPS; i++) { @@ -36,7 +30,7 @@ for (let i = 1; i <= XPACK_CI_GROUPS; i++) { const pipeline = { steps: [ { - input: 'Number of Runs', + input: 'Number of Runs - Click Me', fields: inputs, }, { diff --git a/.buildkite/pipelines/flaky_tests/runner.js b/.buildkite/pipelines/flaky_tests/runner.js index 46c390ce455cab..bdb163504f46c1 100644 --- a/.buildkite/pipelines/flaky_tests/runner.js +++ b/.buildkite/pipelines/flaky_tests/runner.js @@ -9,8 +9,10 @@ const overrideCount = parseInt( execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim() ); -const concurrency = - parseInt(execSync(`buildkite-agent meta-data get 'ftsr-concurrency'`).toString().trim()) || 20; +const concurrency = 25; +const initialJobs = 3; + +let totalJobs = initialJobs; const testSuites = []; for (const key of keys) { @@ -21,12 +23,25 @@ for (const key of keys) { const value = overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim(); + const count = value === '' ? defaultCount : parseInt(value); + totalJobs += count; + testSuites.push({ key: key.replace('ftsr-suite/', ''), - count: value === '' ? defaultCount : parseInt(value), + count: count, }); } +if (totalJobs > 500) { + console.error('+++ Too many tests'); + console.error( + `Buildkite builds can only contain 500 steps in total. Found ${totalJobs} in total. Make sure your test runs are less than ${ + 500 - initialJobs + }` + ); + process.exit(1); +} + const steps = []; const pipeline = { env: { @@ -46,7 +61,7 @@ steps.push({ for (const testSuite of testSuites) { const TEST_SUITE = testSuite.key; const RUN_COUNT = testSuite.count; - const UUID = TEST_SUITE + process.env.UUID; + const UUID = process.env.UUID; const JOB_PARTS = TEST_SUITE.split('/'); const IS_XPACK = JOB_PARTS[0] === 'xpack'; @@ -65,6 +80,7 @@ for (const testSuite of testSuites) { parallelism: RUN_COUNT, concurrency: concurrency, concurrency_group: UUID, + concurrency_method: 'eager', }); } else { steps.push({ @@ -75,6 +91,7 @@ for (const testSuite of testSuites) { parallelism: RUN_COUNT, concurrency: concurrency, concurrency_group: UUID, + concurrency_method: 'eager', }); } } From 7671903013904f5f0e7693f5dcbde366ac48fe5a Mon Sep 17 00:00:00 2001 From: Greg Back <1045796+gtback@users.noreply.github.com> Date: Wed, 20 Oct 2021 21:36:49 -0400 Subject: [PATCH 32/35] Fix PHP and Python doc links (#115884) * Fix PHP doc link I made a mistake in #115879 * Fix Python doc link --- src/core/public/doc_links/doc_links_service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index 7068c8dbce5dc5..5f2da7f4b7e0c8 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -509,8 +509,8 @@ export class DocLinksService { jsIntro: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/javascript-api/7.x/introduction.html`, netGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/net-api/7.x/index.html`, perlGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/perl-api/current/index.html`, - phpGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/php-api/$7.x/index.html`, - pythonGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/net-api/${DOC_LINK_VERSION}/index.html`, + phpGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/php-api/7.x/index.html`, + pythonGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/python-api/${DOC_LINK_VERSION}/index.html`, rubyOverview: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/ruby-api/${DOC_LINK_VERSION}/ruby_client.html`, rustGuide: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/client/rust-api/current/index.html`, }, From cd68f77361e3d7d0152b72ee73d53e8115032af7 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 20 Oct 2021 21:43:24 -0400 Subject: [PATCH 33/35] Features integrations + mobile, copy, design tweaks (#115495) (#115858) * Features integrations + mobile, copy, design tweaks * i18n * ts fix * center button in no data card * tracking ids Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Dave Snider --- src/core/public/rendering/_base.scss | 10 ++- .../empty_index_list_prompt.tsx | 2 +- .../elastic_agent_card.test.tsx.snap | 58 +++++++++----- .../__snapshots__/no_data_card.test.tsx.snap | 66 ++++++++++------ .../no_data_card/elastic_agent_card.tsx | 8 +- .../no_data_card/no_data_card.tsx | 3 +- .../epm/components/integration_preference.tsx | 3 +- .../epm/components/package_list_grid.tsx | 78 ++++++++++--------- .../epm/screens/home/available_packages.tsx | 70 ++++++++++++++++- .../epm/screens/home/category_facets.tsx | 7 ++ .../epm/screens/home/installed_packages.tsx | 10 +-- .../components/app/header/header_menu.tsx | 2 +- .../public/app/home/global_header/index.tsx | 2 +- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 15 files changed, 225 insertions(+), 96 deletions(-) diff --git a/src/core/public/rendering/_base.scss b/src/core/public/rendering/_base.scss index 32a297a4066d9c..c97afbf14a8f65 100644 --- a/src/core/public/rendering/_base.scss +++ b/src/core/public/rendering/_base.scss @@ -43,10 +43,12 @@ top: $headerHeight; } - .kbnStickyMenu { - position: sticky; - max-height: calc(100vh - #{$headerHeight + $euiSize}); - top: $headerHeight + $euiSize; + @include euiBreakpoint('xl', 'l') { + .kbnStickyMenu { + position: sticky; + max-height: calc(100vh - #{$headerHeight + $euiSize}); + top: $headerHeight + $euiSize; + } } } diff --git a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_index_list_prompt/empty_index_list_prompt.tsx b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_index_list_prompt/empty_index_list_prompt.tsx index a5502090958983..3afd7ef1ded6b2 100644 --- a/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_index_list_prompt/empty_index_list_prompt.tsx +++ b/src/plugins/index_pattern_editor/public/components/empty_prompts/empty_index_list_prompt/empty_index_list_prompt.tsx @@ -91,7 +91,7 @@ export const EmptyIndexListPrompt = ({ { - navigateToApp('home', { path: '/app/integrations/browse' }); + navigateToApp('integrations', { path: '/browse' }); closeFlyout(); }} icon={} diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/elastic_agent_card.test.tsx.snap b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/elastic_agent_card.test.tsx.snap index 8e1d0cb92e0065..30703a4a5ebb70 100644 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/elastic_agent_card.test.tsx.snap +++ b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/elastic_agent_card.test.tsx.snap @@ -15,11 +15,16 @@ exports[`ElasticAgentCard props button 1`] = ` Button - + + Button + +
} href="/app/integrations/browse" image="/plugins/kibanaReact/assets/elastic_agent_card.svg" @@ -50,11 +55,15 @@ exports[`ElasticAgentCard props category 1`] = ` - Add Elastic Agent - + + Add Elastic Agent + +
} href="/app/integrations/browse/custom" image="/plugins/kibanaReact/assets/elastic_agent_card.svg" @@ -85,11 +94,16 @@ exports[`ElasticAgentCard props href 1`] = ` Button - + + Button + + } href="#" image="/plugins/kibanaReact/assets/elastic_agent_card.svg" @@ -121,11 +135,15 @@ exports[`ElasticAgentCard props recommended 1`] = ` betaBadgeLabel="Recommended" description="Use Elastic Agent for a simple, unified way to collect data from your machines." footer={ - - Add Elastic Agent - + + Add Elastic Agent + + } href="/app/integrations/browse" image="/plugins/kibanaReact/assets/elastic_agent_card.svg" @@ -156,11 +174,15 @@ exports[`ElasticAgentCard renders 1`] = ` - Add Elastic Agent - + + Add Elastic Agent + + } href="/app/integrations/browse" image="/plugins/kibanaReact/assets/elastic_agent_card.svg" diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/no_data_card.test.tsx.snap b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/no_data_card.test.tsx.snap index fccbbe3a9e8eee..6959e2e29095af 100644 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/no_data_card.test.tsx.snap +++ b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/__snapshots__/no_data_card.test.tsx.snap @@ -2,7 +2,7 @@ exports[`NoDataCard props button 1`] = `
- + +
`; exports[`NoDataCard props href 1`] = `
- + +
`; exports[`NoDataCard props recommended 1`] = `
+
+
+
`; exports[`NoDataCard renders 1`] = `
+
+
+
`; diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/elastic_agent_card.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/elastic_agent_card.tsx index b9d412fe4df89b..d429f9d712081c 100644 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/elastic_agent_card.tsx +++ b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/elastic_agent_card.tsx @@ -44,6 +44,7 @@ export const ElasticAgentCard: FunctionComponent = ({ {i18n.translate('kibana-react.noDataPage.elasticAgentCard.noPermission.title', { @@ -92,7 +93,12 @@ export const ElasticAgentCard: FunctionComponent = ({ defaultMessage: `Use Elastic Agent for a simple, unified way to collect data from your machines.`, })} betaBadgeLabel={recommended ? NO_DATA_RECOMMENDED : undefined} - footer={footer} + footer={ +
+ {button} + {footer} +
+ } layout={layout as 'vertical' | undefined} {...cardRest} /> diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/no_data_card.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/no_data_card.tsx index 9cc38cc5f60389..ad40a4f8f54997 100644 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/no_data_card.tsx +++ b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_card/no_data_card.tsx @@ -27,6 +27,7 @@ export const NoDataCard: FunctionComponent = ({ return ( = ({ defaultMessage: `Proceed without collecting data`, })} betaBadgeLabel={recommended ? NO_DATA_RECOMMENDED : undefined} - footer={footer} + footer={
{footer}
} layout={layout as 'vertical' | undefined} {...cardRest} /> diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/integration_preference.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/integration_preference.tsx index 4634996d6bc730..9c9027fb94ac52 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/integration_preference.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/integration_preference.tsx @@ -46,7 +46,7 @@ const link = ( const title = ( ); @@ -115,6 +115,7 @@ export const IntegrationPreference = ({ initialType, onChange }: Props) => { name="preference" /> + ); }; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.tsx index d2d361cd212a6c..91bd29d14d0b37 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.tsx @@ -18,6 +18,7 @@ import { EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; + import { FormattedMessage } from '@kbn/i18n/react'; import { Loading } from '../../../components'; @@ -32,6 +33,7 @@ export interface Props { controls?: ReactNode | ReactNode[]; title?: string; list: IntegrationCardItem[]; + featuredList?: JSX.Element | null; initialSearch?: string; setSelectedCategory: (category: string) => void; onSearchChange: (search: string) => void; @@ -48,6 +50,7 @@ export const PackageListGrid: FunctionComponent = ({ onSearchChange, setSelectedCategory, showMissingIntegrationMessage = false, + featuredList = null, callout, }) => { const [searchTerm, setSearchTerm] = useState(initialSearch || ''); @@ -106,42 +109,45 @@ export const PackageListGrid: FunctionComponent = ({ } return ( -
- - - {controlsContent} - - - - {callout ? ( - <> - - {callout} - - ) : null} - - {gridContent} - {showMissingIntegrationMessage && ( - <> - - - - )} - - -
+ <> + {featuredList} +
+ + + {controlsContent} + + + + {callout ? ( + <> + + {callout} + + ) : null} + + {gridContent} + {showMissingIntegrationMessage && ( + <> + + + + )} + + +
+ ); }; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx index 73de0e51bea658..4f13a874532f1e 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx @@ -8,7 +8,18 @@ import React, { memo, useMemo, useState } from 'react'; import { useLocation, useHistory, useParams } from 'react-router-dom'; import _ from 'lodash'; -import { EuiHorizontalRule, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { + EuiHorizontalRule, + EuiFlexItem, + EuiFlexGrid, + EuiSpacer, + EuiCard, + EuiIcon, +} from '@elastic/eui'; + +import { useStartServices } from '../../../../hooks'; +import { TrackApplicationView } from '../../../../../../../../../../src/plugins/usage_collection/public'; import { pagePathGetters } from '../../../../constants'; import { @@ -98,6 +109,9 @@ export const AvailablePackages: React.FC = memo(() => { const [preference, setPreference] = useState('recommended'); useBreadcrumbs('integrations_all'); + const { http } = useStartServices(); + const addBasePath = http.basePath.prepend; + const { selectedCategory, searchParam } = getParams( useParams(), useLocation().search @@ -210,8 +224,62 @@ export const AvailablePackages: React.FC = memo(() => { return c.categories.includes(selectedCategory); }); + // TODO: Remove this hard coded list of integrations with a suggestion service + const featuredList = ( + <> + + + + } + href={addBasePath('/app/integrations/detail/endpoint/')} + title={i18n.translate('xpack.fleet.featuredSecurityTitle', { + defaultMessage: 'Endpoint Security', + })} + description={i18n.translate('xpack.fleet.featuredSecurityDesc', { + defaultMessage: + 'Protect your hosts with threat prevention, detection, and deep security data visibility.', + })} + /> + + + + + } + /> + + + + + } + href={addBasePath('/app/enterprise_search/app_search')} + title={i18n.translate('xpack.fleet.featuredSearchTitle', { + defaultMessage: 'Web site crawler', + })} + description={i18n.translate('xpack.fleet.featuredSearchDesc', { + defaultMessage: 'Add search to your website with the App Search web crawler.', + })} + /> + + + + + + ); + return ( { const { docLinks } = useStartServices(); @@ -56,10 +56,6 @@ const Callout = () => ( ); -const title = i18n.translate('xpack.fleet.epmList.installedTitle', { - defaultMessage: 'Installed integrations', -}); - // TODO: clintandrewhall - this component is hard to test due to the hooks, particularly those that use `http` // or `location` to load data. Ideally, we'll split this into "connected" and "pure" components. export const InstalledPackages: React.FC = memo(() => { @@ -115,7 +111,7 @@ export const InstalledPackages: React.FC = memo(() => { const categories: CategoryFacet[] = useMemo( () => [ { - ...ALL_CATEGORY, + ...INSTALLED_CATEGORY, count: allInstalledPackages.length, }, { @@ -153,7 +149,7 @@ export const InstalledPackages: React.FC = memo(() => { return ( Date: Thu, 21 Oct 2021 00:42:58 -0400 Subject: [PATCH 34/35] Adds one time conflict retry and cleans up the exception lists to use the REST API (#115848) (#115878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Improves FTR/e2e conflict retries with exception lists and security rules. Fixes: https://github.com/elastic/kibana/issues/115734 https://github.com/elastic/kibana/issues/115769 https://github.com/elastic/kibana/issues/115715 https://github.com/elastic/kibana/issues/115702 https://github.com/elastic/kibana/issues/115701 This past week we have been seeing increasing flake across tests involving `exception_lists` involving a `409 conflict` on our tests. Looking at each of the tests above and the flake it looks like we were calling Elasticsearch directly within the `.kibana` index to delete the exception list and list items as a shortcut: ``` export const deleteAllExceptions = async (es: KibanaClient): Promise => { return countDownES(async () => { return es.deleteByQuery({ index: '.kibana', q: 'type:exception-list or type:exception-list-agnostic', wait_for_completion: true, refresh: true, body: {}, }); }, 'deleteAllExceptions'); }; ``` Although I think we did everything correctly `wait_for_completion: true` and `refresh: true` within the tests there might be a slight race condition where the delete by query does not immediately happen for us. Since we should prefer to use direct REST API's where we can instead of calling into `.kibana` I changed this to using the exception list API: ``` export const deleteAllExceptions = async ( supertest: SuperTest.SuperTest ): Promise => { await countDownTest( async () => { const { body } = await supertest .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999`) .set('kbn-xsrf', 'true') .send(); const ids: string[] = body.data.map((exception: ExceptionList) => exception.id); for await (const id of ids) { await supertest.delete(`${EXCEPTION_LIST_URL}?id=${id}`).set('kbn-xsrf', 'true').send(); } const { body: finalCheck } = await supertest .get(`${EXCEPTION_LIST_URL}/_find`) .set('kbn-xsrf', 'true') .send(); return finalCheck.data.length === 0; }, 'deleteAllExceptions', 50, 1000 ); }; ``` The additional final check above should ensure it sees that the data has been deleted before returning. Otherwise it will loop around again and keep trying. I also improve both the `createRules` and `createExceptionList` by introducing a one-time, "detect if in conflict" and then "remove if in conflict" within those tests. This should help safe guard against flake if the above does not fix it. I also added more logging statements in case we do encounter this again on the CI system we can further trouble shoot it and add additional retry logic/fix logic. A good side effect is if now you kill your tests half way through and restart them, the additional "detect if conflict" will recover your test for you as a developer. So 👍 that is an added benefit. Example error message you would get (but not test failure) if you remove one of the cleanup sections in the `afterEach` or if you kill a test half way through and then restart it as an engineer: ``` └-: "is" operator └-> "before all" hook for "should find all the text from the data set when no exceptions are set on the rule" └-> should find all the text from the data set when no exceptions are set on the rule └-> "before each" hook: global before each for "should find all the text from the data set when no exceptions are set on the rule" └-> "before each" hook for "should find all the text from the data set when no exceptions are set on the rule" When creating a rule found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: {"message":"rule_id: \"rule-1\" already exists","status_code":409} └- ✓ pass (7.9s) ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Frank Hassanabad --- .../tests/create_endpoint_exceptions.ts | 3 +- .../tests/create_exceptions.ts | 4 +- .../security_and_spaces/tests/create_ml.ts | 5 +- .../exception_operators_data_types/date.ts | 3 +- .../exception_operators_data_types/double.ts | 3 +- .../exception_operators_data_types/float.ts | 3 +- .../exception_operators_data_types/integer.ts | 3 +- .../exception_operators_data_types/ip.ts | 3 +- .../ip_array.ts | 3 +- .../exception_operators_data_types/keyword.ts | 3 +- .../keyword_array.ts | 3 +- .../exception_operators_data_types/long.ts | 3 +- .../exception_operators_data_types/text.ts | 3 +- .../text_array.ts | 3 +- .../detection_engine_api_integration/utils.ts | 149 +++++++++++++++--- .../tests/create_exception_list_items.ts | 3 +- .../tests/create_exception_lists.ts | 3 +- .../tests/delete_exception_list_items.ts | 3 +- .../tests/delete_exception_lists.ts | 3 +- .../security_and_spaces/tests/delete_lists.ts | 3 +- .../tests/export_exception_list.ts | 3 +- .../tests/find_exception_list_items.ts | 3 +- .../tests/find_exception_lists.ts | 3 +- .../tests/read_exception_list_items.ts | 3 +- .../tests/read_exception_lists.ts | 3 +- .../tests/summary_exception_lists.ts | 3 +- .../tests/update_exception_list_items.ts | 3 +- .../tests/update_exception_lists.ts | 3 +- x-pack/test/lists_api_integration/utils.ts | 50 ++++-- 29 files changed, 189 insertions(+), 94 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts index 6d04ffc67c573e..6c6fcc366782af 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts @@ -69,7 +69,6 @@ export const getHostHits = async ( export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for endpoints', () => { before(async () => { @@ -94,7 +93,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts index 0d1e353447e996..1882f488e5a179 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts @@ -79,7 +79,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); describe('elastic admin', () => { @@ -550,7 +550,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should be able to execute against an exception list that does not include valid entries and get back 10 signals', async () => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts index c78ef18635de75..50e0d200acf3b3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts @@ -212,9 +212,10 @@ export default ({ getService }: FtrProviderContext) => { const signalsOpen = await getOpenSignals(supertest, es, createdRule); expect(signalsOpen.hits.hits.length).eql(7); }); + describe('with non-value list exception', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('generates no signals when an exception is added for an ML rule', async () => { const createdRule = await createRuleWithExceptionEntries(supertest, testRule, [ @@ -239,7 +240,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteListsIndex(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('generates no signals when a value list exception is added for an ML rule', async () => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/date.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/date.ts index c2c7313762ae71..742bbf7285e95a 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/date.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/date.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type date', () => { before(async () => { @@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts index 7f659d6795f9ab..a2639ea5224475 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type double', () => { before(async () => { @@ -53,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts index 912596ed7ca00c..288e2353166ee5 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type float', () => { before(async () => { @@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts index da9219e4b52f60..459034cd06569d 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type integer', () => { before(async () => { @@ -53,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts index 3a65a0f0a67e52..2497bf096550a3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type ip', () => { before(async () => { @@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts index 526c6d1c988cea..5df01ff80d67bc 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type ip', () => { before(async () => { @@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts index 4c70dbdf170c70..8a184025b00e29 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type keyword', () => { before(async () => { @@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts index 8571aa8eeaa605..092b81bf446b89 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type keyword', () => { before(async () => { @@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts index 8d5f1515e4ab62..f5bf3b627bc2ff 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type long', () => { before(async () => { @@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts index 367e68f7f9ed1c..ff2f6806540470 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts @@ -31,7 +31,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type text', () => { before(async () => { @@ -52,7 +51,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts index 3eedabd41d6636..3bcf8692d58f94 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts @@ -30,7 +30,6 @@ import { export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); - const es = getService('es'); describe('Rule exception operators for data type text', () => { before(async () => { @@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => { afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); await deleteListsIndex(supertest); }); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index dd1b1ba9661754..edb2db3e9a2610 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -413,17 +413,12 @@ export const getSimpleMlRuleOutput = (ruleId = 'rule-1'): Partial = * @param supertest The supertest agent. */ export const deleteAllAlerts = async ( - supertest: SuperTest.SuperTest, - space?: string + supertest: SuperTest.SuperTest ): Promise => { await countDownTest( async () => { const { body } = await supertest - .get( - space - ? `/s/${space}${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999` - : `${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999` - ) + .get(`${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999`) .set('kbn-xsrf', 'true') .send(); @@ -432,11 +427,7 @@ export const deleteAllAlerts = async ( })); await supertest - .post( - space - ? `/s/${space}${DETECTION_ENGINE_RULES_URL}/_bulk_delete` - : `${DETECTION_ENGINE_RULES_URL}/_bulk_delete` - ) + .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_delete`) .send(ids) .set('kbn-xsrf', 'true'); @@ -899,8 +890,10 @@ export const countDownTest = async ( }; /** - * Helper to cut down on the noise in some of the tests. This checks for - * an expected 200 still and does not try to any retries. + * Helper to cut down on the noise in some of the tests. If this detects + * a conflict it will try to manually remove the rule before re-adding the rule one time and log + * and error about the race condition. + * rule a second attempt. It only re-tries adding the rule if it encounters a conflict once. * @param supertest The supertest deps * @param rule The rule to create */ @@ -908,17 +901,69 @@ export const createRule = async ( supertest: SuperTest.SuperTest, rule: CreateRulesSchema ): Promise => { - const { body } = await supertest + const response = await supertest .post(DETECTION_ENGINE_RULES_URL) .set('kbn-xsrf', 'true') - .send(rule) - .expect(200); - return body; + .send(rule); + if (response.status === 409) { + if (rule.rule_id != null) { + // eslint-disable-next-line no-console + console.log( + `When creating a rule found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: ${JSON.stringify( + response.body + )}` + ); + await deleteRule(supertest, rule.rule_id); + const secondResponseTry = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .send(rule); + if (secondResponseTry.status !== 200) { + throw new Error( + `Unexpected non 200 ok when attempting to create a rule (second try): ${JSON.stringify( + response.body + )}` + ); + } else { + return secondResponseTry.body; + } + } else { + throw new Error('When creating a rule found an unexpected conflict (404)'); + } + } else if (response.status !== 200) { + throw new Error( + `Unexpected non 200 ok when attempting to create a rule: ${JSON.stringify(response.status)}` + ); + } else { + return response.body; + } }; /** - * Helper to cut down on the noise in some of the tests. This checks for - * an expected 200 still and does not try to any retries. + * Helper to cut down on the noise in some of the tests. Does a delete of a rule. + * It does not check for a 200 "ok" on this. + * @param supertest The supertest deps + * @param id The rule id to delete + */ +export const deleteRule = async ( + supertest: SuperTest.SuperTest, + ruleId: string +): Promise => { + const response = await supertest + .delete(`${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleId}`) + .set('kbn-xsrf', 'true'); + if (response.status !== 200) { + // eslint-disable-next-line no-console + console.log( + 'Did not get an expected 200 "ok" when deleting the rule. CI issues could happen. Suspect this line if you are seeing CI issues.' + ); + } + + return response.body; +}; + +/** + * Helper to cut down on the noise in some of the tests. * @param supertest The supertest deps * @param rule The rule to create */ @@ -1017,12 +1062,68 @@ export const createExceptionList = async ( supertest: SuperTest.SuperTest, exceptionList: CreateExceptionListSchema ): Promise => { - const { body } = await supertest + const response = await supertest .post(EXCEPTION_LIST_URL) .set('kbn-xsrf', 'true') - .send(exceptionList) - .expect(200); - return body; + .send(exceptionList); + + if (response.status === 409) { + if (exceptionList.list_id != null) { + // eslint-disable-next-line no-console + console.log( + `When creating an exception list found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: ${JSON.stringify( + response.body + )}` + ); + await deleteExceptionList(supertest, exceptionList.list_id); + const secondResponseTry = await supertest + .post(EXCEPTION_LIST_URL) + .set('kbn-xsrf', 'true') + .send(exceptionList); + if (secondResponseTry.status !== 200) { + throw new Error( + `Unexpected non 200 ok when attempting to create an exception list (second try): ${JSON.stringify( + response.body + )}` + ); + } else { + return secondResponseTry.body; + } + } else { + throw new Error('When creating an exception list found an unexpected conflict (404)'); + } + } else if (response.status !== 200) { + throw new Error( + `Unexpected non 200 ok when attempting to create an exception list: ${JSON.stringify( + response.status + )}` + ); + } else { + return response.body; + } +}; + +/** + * Helper to cut down on the noise in some of the tests. Does a delete of a rule. + * It does not check for a 200 "ok" on this. + * @param supertest The supertest deps + * @param id The rule id to delete + */ +export const deleteExceptionList = async ( + supertest: SuperTest.SuperTest, + listId: string +): Promise => { + const response = await supertest + .delete(`${EXCEPTION_LIST_URL}?list_id=${listId}`) + .set('kbn-xsrf', 'true'); + if (response.status !== 200) { + // eslint-disable-next-line no-console + console.log( + 'Did not get an expected 200 "ok" when deleting an exception list. CI issues could happen. Suspect this line if you are seeing CI issues.' + ); + } + + return response.body; }; /** diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts index 4541a758a02f1b..cb858a4791b718 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts @@ -27,7 +27,6 @@ import { deleteAllExceptions } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('create_exception_list_items', () => { describe('validation errors', () => { @@ -47,7 +46,7 @@ export default ({ getService }: FtrProviderContext) => { describe('creating exception list items', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should create a simple exception list item with a list item id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts index 7e34c80806cc44..e7f5d96bfb76a2 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts @@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('create_exception_lists', () => { describe('creating exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should create a simple exception list', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts index 229d5737a99bb5..acf968c8b78afa 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts @@ -22,12 +22,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('delete_exception_list_items', () => { describe('delete exception list items', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should delete a single exception list item by its item_id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts index d83c4bdc2f1a9a..0f8ca96e5383a0 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts @@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('delete_exception_lists', () => { describe('delete exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should delete a single exception list by its list_id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts index 5d1abf6f74f7ef..3c01d93380736b 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts @@ -33,7 +33,6 @@ import { DETECTION_TYPE, LIST_ID } from '../../../../plugins/lists/common/consta // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('delete_lists', () => { describe('deleting lists', () => { @@ -117,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => { describe('deleting lists referenced in exceptions', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should return an error when deleting a list referenced within an exception list item', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts index c21026d5df3d26..c61f4a2b1d02f8 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts @@ -22,12 +22,11 @@ import { getCreateExceptionListItemMinimalSchemaMock } from '../../../../plugins // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); - const es = getService('es'); describe('export_exception_list_route', () => { describe('exporting exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should set the response content types to be expected', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts index 7b23ab7d6f8668..bdacd674d75193 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts @@ -18,12 +18,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); - const es = getService('es'); describe('find_exception_list_items', () => { describe('find exception list items', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should return an empty find body correctly if no exception list items are loaded', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts index 9972ed6a89171c..158d951b2bd682 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts @@ -17,12 +17,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); - const es = getService('es'); describe('find_exception_lists', () => { describe('find exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should return an empty find body correctly if no exception lists are loaded', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts index 0a7caed8a5e149..9345306e0c3e12 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts @@ -22,12 +22,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties } // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('read_exception_list_items', () => { describe('reading exception list items', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should be able to read a single exception list items using item_id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts index db53a0ed18a0c8..b9f2b89a3e0ee6 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts @@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('read_exception_lists', () => { describe('reading exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should be able to read a single exception list using list_id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts index 13ba7da4c7aaad..b71a7dbe768d21 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts @@ -21,7 +21,6 @@ interface SummaryResponseType { // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('summary_exception_lists', () => { describe('summary exception lists', () => { @@ -30,7 +29,7 @@ export default ({ getService }: FtrProviderContext) => { }); afterEach(async () => { await deleteListsIndex(supertest); - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should give a validation error if the list_id and the id are not supplied', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts index b611d5c31de672..fa0466f14db28f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts @@ -24,12 +24,11 @@ import { getUpdateMinimalExceptionListItemSchemaMock } from '../../../../plugins // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('update_exception_list_items', () => { describe('update exception list items', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should update a single exception list item property of name using an id', async () => { diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts index 75064860da1c2d..2eeaca7f0521b8 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts @@ -23,12 +23,11 @@ import { getUpdateMinimalExceptionListSchemaMock } from '../../../../plugins/lis // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); - const es = getService('es'); describe('update_exception_lists', () => { describe('update exception lists', () => { afterEach(async () => { - await deleteAllExceptions(es); + await deleteAllExceptions(supertest); }); it('should update a single exception list property of name using an id', async () => { diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index b3816ad7563b8f..c8c1acb9f0e876 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -6,7 +6,6 @@ */ import type SuperTest from 'supertest'; -import type { KibanaClient } from '@elastic/elasticsearch/api/kibana'; import type { Type, @@ -14,10 +13,15 @@ import type { ListItemSchema, ExceptionListSchema, ExceptionListItemSchema, + ExceptionList, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_INDEX, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { + EXCEPTION_LIST_URL, + LIST_INDEX, + LIST_ITEM_URL, +} from '@kbn/securitysolution-list-constants'; import { getImportListItemAsBuffer } from '../../plugins/lists/common/schemas/request/import_list_item_schema.mock'; -import { countDownES, countDownTest } from '../detection_engine_api_integration/utils'; +import { countDownTest } from '../detection_engine_api_integration/utils'; /** * Creates the lists and lists items index for use inside of beforeEach blocks of tests @@ -160,20 +164,34 @@ export const binaryToString = (res: any, callback: any): void => { }; /** - * Remove all exceptions from the .kibana index - * This will retry 20 times before giving up and hopefully still not interfere with other tests - * @param es The ElasticSearch handle + * Remove all exceptions + * This will retry 50 times before giving up and hopefully still not interfere with other tests + * @param supertest The supertest handle */ -export const deleteAllExceptions = async (es: KibanaClient): Promise => { - return countDownES(async () => { - return es.deleteByQuery({ - index: '.kibana', - q: 'type:exception-list or type:exception-list-agnostic', - wait_for_completion: true, - refresh: true, - body: {}, - }); - }, 'deleteAllExceptions'); +export const deleteAllExceptions = async ( + supertest: SuperTest.SuperTest +): Promise => { + await countDownTest( + async () => { + const { body } = await supertest + .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999`) + .set('kbn-xsrf', 'true') + .send(); + + const ids: string[] = body.data.map((exception: ExceptionList) => exception.id); + for await (const id of ids) { + await supertest.delete(`${EXCEPTION_LIST_URL}?id=${id}`).set('kbn-xsrf', 'true').send(); + } + const { body: finalCheck } = await supertest + .get(`${EXCEPTION_LIST_URL}/_find`) + .set('kbn-xsrf', 'true') + .send(); + return finalCheck.data.length === 0; + }, + 'deleteAllExceptions', + 50, + 1000 + ); }; /** From 50d97caa5e979c9695a8676024a52d491de343c1 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 20 Oct 2021 23:45:19 -0700 Subject: [PATCH 35/35] [Reporting] Upgrade Puppeteer dependency to 10.2.0 (#115682) (#115864) * [Reporting] Upgrade Puppeteer dependency to 10.2.0 * Update x-pack/plugins/reporting/server/browsers/chromium/paths.ts * self-edit * Apply suggestions from code review Co-authored-by: Michael Dokolin * fix lint Co-authored-by: Michael Dokolin Co-authored-by: Michael Dokolin --- package.json | 2 +- x-pack/build_chromium/README.md | 10 +- .../chromium/driver/chromium_driver.ts | 3 +- .../browsers/chromium/driver_factory/index.ts | 12 --- .../server/browsers/chromium/paths.ts | 30 +++--- .../server/browsers/download/download.ts | 2 + .../download/ensure_downloaded.test.ts | 27 ++++-- .../browsers/download/ensure_downloaded.ts | 50 +++++++--- .../reporting/server/browsers/index.ts | 5 +- .../reporting/server/browsers/install.ts | 23 ++++- yarn.lock | 97 ++++++++++++------- 11 files changed, 165 insertions(+), 96 deletions(-) diff --git a/package.json b/package.json index fe9b8ec09c5c8d..9d30aee0a75151 100644 --- a/package.json +++ b/package.json @@ -315,7 +315,7 @@ "prop-types": "^15.7.2", "proxy-from-env": "1.0.0", "puid": "1.0.7", - "puppeteer": "^8.0.0", + "puppeteer": "^10.2.0", "query-string": "^6.13.2", "random-word-slugs": "^0.0.5", "raw-loader": "^3.1.0", diff --git a/x-pack/build_chromium/README.md b/x-pack/build_chromium/README.md index 267de376c68ffd..c27a654abd671e 100644 --- a/x-pack/build_chromium/README.md +++ b/x-pack/build_chromium/README.md @@ -99,7 +99,15 @@ are created in x64 using cross-compiling. CentOS is not supported for building C ## Artifacts After the build completes, there will be a .zip file and a .md5 file in `~/chromium/chromium/src/out/headless`. These are named like so: `chromium-{first_7_of_SHA}-{platform}-{arch}`, for example: `chromium-4747cc2-linux-x64`. -The zip files and md5 files are copied to a staging bucket in GCP storage. +The zip files and md5 files are copied to a **staging** bucket in GCP storage. + +To publish the built artifacts for bunding in Kibana, copy the files from the `headless_shell_staging` bucket to the `headless_shell` bucket. +``` +gsutil cp gs://headless_shell_staging/chromium-d163fd7-linux_arm64.md5 gs://headless_shell/ +gsutil cp gs://headless_shell_staging/chromium-d163fd7-linux_arm64.zip gs://headless_shell/ +``` + +IMPORTANT: Do not replace builds in the `headless_shell` bucket that are referenced in an active Kibana branch. CI tests on that branch will fail since the archive checksum no longer matches the original version. ## Testing Search the Puppeteer Github repo for known issues that could affect our use case, and make sure to test anywhere that is affected. diff --git a/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index df91b6fe0ba47f..e7c2b68ba27121 100644 --- a/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -265,7 +265,7 @@ export class HeadlessChromiumDriver { } // @ts-ignore - // FIXME: use `await page.target().createCDPSession();` + // FIXME: retrieve the client in open() and pass in the client const client = this.page._client; // We have to reach into the Chrome Devtools Protocol to apply headers as using @@ -372,7 +372,6 @@ export class HeadlessChromiumDriver { await client.send('Debugger.enable'); await client.send('Debugger.pause'); - // @ts-ignore const targetId = target._targetId; const wsEndpoint = this.page.browser().wsEndpoint(); const { port } = parseUrl(wsEndpoint); diff --git a/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts index 688dd425fa8f30..264e673d2bf740 100644 --- a/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts +++ b/x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts @@ -25,18 +25,6 @@ import { HeadlessChromiumDriver } from '../driver'; import { args } from './args'; import { getMetrics, Metrics } from './metrics'; -// Puppeteer type definitions do not match the documentation. -// See https://pptr.dev/#?product=Puppeteer&version=v8.0.0&show=api-puppeteerlaunchoptions -interface ReportingLaunchOptions extends puppeteer.LaunchOptions { - userDataDir?: string; - ignoreHTTPSErrors?: boolean; - args?: string[]; -} - -declare module 'puppeteer' { - function launch(options: ReportingLaunchOptions): Promise; -} - type BrowserConfig = CaptureConfig['browser']['chromium']; export class HeadlessChromiumDriverFactory { diff --git a/x-pack/plugins/reporting/server/browsers/chromium/paths.ts b/x-pack/plugins/reporting/server/browsers/chromium/paths.ts index ed06bba527c437..033ddce8d9ebd5 100644 --- a/x-pack/plugins/reporting/server/browsers/chromium/paths.ts +++ b/x-pack/plugins/reporting/server/browsers/chromium/paths.ts @@ -14,6 +14,7 @@ interface PackageInfo { archiveChecksum: string; binaryChecksum: string; binaryRelativePath: string; + revision: number; } enum BaseUrl { @@ -32,8 +33,6 @@ interface CommonPackageInfo extends PackageInfo { } export class ChromiumArchivePaths { - public readonly revision = '856583'; - public readonly packages: Array = [ { platform: 'darwin', @@ -43,34 +42,38 @@ export class ChromiumArchivePaths { binaryChecksum: 'dfcd6e007214175997663c50c8d871ea', binaryRelativePath: 'headless_shell-darwin_x64/headless_shell', location: 'custom', + revision: 856583, }, { platform: 'linux', architecture: 'x64', - archiveFilename: 'chromium-d163fd7-linux_x64.zip', - archiveChecksum: 'fba0a240d409228a3494aef415c300fc', - binaryChecksum: '99cfab472d516038b94ef86649e52871', + archiveFilename: 'chromium-70f5d88-linux_x64.zip', + archiveChecksum: '7b1c9c2fb613444fbdf004a3b75a58df', + binaryChecksum: '82e80f9727a88ba3836ce230134bd126', binaryRelativePath: 'headless_shell-linux_x64/headless_shell', location: 'custom', + revision: 901912, }, { platform: 'linux', architecture: 'arm64', - archiveFilename: 'chromium-d163fd7-linux_arm64.zip', - archiveChecksum: '29834735bc2f0e0d9134c33bc0580fb6', - binaryChecksum: '13baccf2e5c8385cb9d9588db6a9e2c2', + archiveFilename: 'chromium-70f5d88-linux_arm64.zip', + archiveChecksum: '4a0217cfe7da86ad1e3d0e9e5895ddb5', + binaryChecksum: '29e943fbee6d87a217abd6cb6747058e', binaryRelativePath: 'headless_shell-linux_arm64/headless_shell', location: 'custom', + revision: 901912, }, { platform: 'win32', architecture: 'x64', archiveFilename: 'chrome-win.zip', - archiveChecksum: '64999a384bfb6c96c50c4cb6810dbc05', - binaryChecksum: '13b8bbb4a12f9036b8cc3b57b3a71fec', + archiveChecksum: '861bb8b7b8406a6934a87d3cbbce61d9', + binaryChecksum: 'ffa0949471e1b9a57bc8f8633fca9c7b', binaryRelativePath: 'chrome-win\\chrome.exe', location: 'common', archivePath: 'Win', + revision: 901912, }, ]; @@ -82,7 +85,8 @@ export class ChromiumArchivePaths { } public resolvePath(p: PackageInfo) { - return path.resolve(this.archivesPath, p.archiveFilename); + // adding architecture to the path allows it to download two binaries that have the same name, but are different architecture + return path.resolve(this.archivesPath, p.architecture, p.archiveFilename); } public getAllArchiveFilenames(): string[] { @@ -91,9 +95,9 @@ export class ChromiumArchivePaths { public getDownloadUrl(p: CustomPackageInfo | CommonPackageInfo) { if (p.location === 'common') { - return `${BaseUrl.common}/${p.archivePath}/${this.revision}/${p.archiveFilename}`; + return `${BaseUrl.common}/${p.archivePath}/${p.revision}/${p.archiveFilename}`; } - return BaseUrl.custom + '/' + p.archiveFilename; + return BaseUrl.custom + '/' + p.archiveFilename; // revision is not used for URL if package is a custom build } public getBinaryPath(p: PackageInfo) { diff --git a/x-pack/plugins/reporting/server/browsers/download/download.ts b/x-pack/plugins/reporting/server/browsers/download/download.ts index 77efc75ae1aaa2..528395fe1afb25 100644 --- a/x-pack/plugins/reporting/server/browsers/download/download.ts +++ b/x-pack/plugins/reporting/server/browsers/download/download.ts @@ -49,6 +49,8 @@ export async function download( resolve(); }); }); + } catch (err) { + throw new Error(`Unable to download ${url}: ${err}`); } finally { closeSync(handle); } diff --git a/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.test.ts b/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.test.ts index 344735d6180b62..955e8214af8fa1 100644 --- a/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.test.ts +++ b/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import path from 'path'; import mockFs from 'mock-fs'; import { existsSync, readdirSync } from 'fs'; import { chromium } from '../chromium'; @@ -27,16 +28,16 @@ describe('ensureBrowserDownloaded', () => { } as unknown as typeof logger; (md5 as jest.MockedFunction).mockImplementation( - async (path) => + async (packagePath) => chromium.paths.packages.find( - (packageInfo) => chromium.paths.resolvePath(packageInfo) === path + (packageInfo) => chromium.paths.resolvePath(packageInfo) === packagePath )?.archiveChecksum ?? 'some-md5' ); (download as jest.MockedFunction).mockImplementation( - async (_url, path) => + async (_url, packagePath) => chromium.paths.packages.find( - (packageInfo) => chromium.paths.resolvePath(packageInfo) === path + (packageInfo) => chromium.paths.resolvePath(packageInfo) === packagePath )?.archiveChecksum ?? 'some-md5' ); @@ -93,11 +94,19 @@ describe('ensureBrowserDownloaded', () => { await ensureBrowserDownloaded(logger); expect(download).not.toHaveBeenCalled(); - expect(readdirSync(chromium.paths.archivesPath)).toEqual( - expect.arrayContaining( - chromium.paths.packages.map(({ archiveFilename }) => archiveFilename) - ) - ); + const paths = [ + readdirSync(path.resolve(chromium.paths.archivesPath + '/x64')), + readdirSync(path.resolve(chromium.paths.archivesPath + '/arm64')), + ]; + + expect(paths).toEqual([ + expect.arrayContaining([ + 'chrome-win.zip', + 'chromium-70f5d88-linux_x64.zip', + 'chromium-d163fd7-darwin_x64.zip', + ]), + expect.arrayContaining(['chromium-70f5d88-linux_arm64.zip']), + ]); }); it('should download again if md5 hash different', async () => { diff --git a/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts b/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts index 55d6395b1bd3d2..2766b404f1dd1b 100644 --- a/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts +++ b/x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts @@ -15,7 +15,6 @@ import { download } from './download'; /** * Check for the downloaded archive of each requested browser type and * download them if they are missing or their checksum is invalid - * @return {Promise} */ export async function ensureBrowserDownloaded(logger: GenericLevelLogger) { await ensureDownloaded([chromium], logger); @@ -25,18 +24,19 @@ export async function ensureBrowserDownloaded(logger: GenericLevelLogger) { * Clears the unexpected files in the browsers archivesPath * and ensures that all packages/archives are downloaded and * that their checksums match the declared value - * @param {BrowserSpec} browsers - * @return {Promise} */ async function ensureDownloaded(browsers: BrowserDownload[], logger: GenericLevelLogger) { await Promise.all( browsers.map(async ({ paths: pSet }) => { - ( - await del(`${pSet.archivesPath}/**/*`, { - force: true, - ignore: pSet.getAllArchiveFilenames(), - }) - ).forEach((path) => logger.warning(`Deleting unexpected file ${path}`)); + const removedFiles = await del(`${pSet.archivesPath}/**/*`, { + force: true, + onlyFiles: true, + ignore: pSet.getAllArchiveFilenames(), + }); + + removedFiles.forEach((path) => { + logger.warning(`Deleting unexpected file ${path}`); + }); const invalidChecksums: string[] = []; await Promise.all( @@ -44,22 +44,44 @@ async function ensureDownloaded(browsers: BrowserDownload[], logger: GenericLeve const { archiveFilename, archiveChecksum } = p; if (archiveFilename && archiveChecksum) { const path = pSet.resolvePath(p); + const pathExists = existsSync(path); + + let foundChecksum: string; + try { + foundChecksum = await md5(path).catch(); + } catch { + foundChecksum = 'MISSING'; + } - if (existsSync(path) && (await md5(path)) === archiveChecksum) { - logger.debug(`Browser archive exists in ${path}`); + if (pathExists && foundChecksum === archiveChecksum) { + logger.debug(`Browser archive for ${p.platform}/${p.architecture} found in ${path} `); return; } + if (!pathExists) { + logger.warning( + `Browser archive for ${p.platform}/${p.architecture} not found in ${path}.` + ); + } + if (foundChecksum !== archiveChecksum) { + logger.warning( + `Browser archive checksum for ${p.platform}/${p.architecture} ` + + `is ${foundChecksum} but ${archiveChecksum} was expected.` + ); + } + const url = pSet.getDownloadUrl(p); try { const downloadedChecksum = await download(url, path, logger); if (downloadedChecksum !== archiveChecksum) { + logger.warning( + `Invalid checksum for ${p.platform}/${p.architecture}: ` + + `expected ${archiveChecksum} got ${downloadedChecksum}` + ); invalidChecksums.push(`${url} => ${path}`); } } catch (err) { - const message = new Error(`Failed to download ${url}`); - logger.error(err); - throw message; + throw new Error(`Failed to download ${url}: ${err}`); } } }) diff --git a/x-pack/plugins/reporting/server/browsers/index.ts b/x-pack/plugins/reporting/server/browsers/index.ts index c47514960bb09f..be5c85a6e95812 100644 --- a/x-pack/plugins/reporting/server/browsers/index.ts +++ b/x-pack/plugins/reporting/server/browsers/index.ts @@ -28,7 +28,8 @@ export interface BrowserDownload { } export const initializeBrowserDriverFactory = async (core: ReportingCore, logger: LevelLogger) => { - const { binaryPath$ } = installBrowser(logger); + const chromiumLogger = logger.clone(['chromium']); + const { binaryPath$ } = installBrowser(chromiumLogger); const binaryPath = await binaryPath$.pipe(first()).toPromise(); - return chromium.createDriverFactory(core, binaryPath, logger); + return chromium.createDriverFactory(core, binaryPath, chromiumLogger); }; diff --git a/x-pack/plugins/reporting/server/browsers/install.ts b/x-pack/plugins/reporting/server/browsers/install.ts index 51045c4ef038f5..0441bbcfb53066 100644 --- a/x-pack/plugins/reporting/server/browsers/install.ts +++ b/x-pack/plugins/reporting/server/browsers/install.ts @@ -39,15 +39,28 @@ export function installBrowser( const binaryChecksum = await md5(binaryPath).catch(() => ''); if (binaryChecksum !== pkg.binaryChecksum) { - await ensureBrowserDownloaded(logger); - await del(chromiumPath); + logger.warning( + `Found browser binary checksum for ${pkg.platform}/${pkg.architecture} ` + + `is ${binaryChecksum} but ${pkg.binaryChecksum} was expected. Re-installing...` + ); + try { + await del(chromiumPath); + } catch (err) { + logger.error(err); + } - const archive = path.join(paths.archivesPath, pkg.archiveFilename); - logger.info(`Extracting [${archive}] to [${chromiumPath}]`); - await extract(archive, chromiumPath); + try { + await ensureBrowserDownloaded(logger); + const archive = path.join(paths.archivesPath, pkg.architecture, pkg.archiveFilename); + logger.info(`Extracting [${archive}] to [${chromiumPath}]`); + await extract(archive, chromiumPath); + } catch (err) { + logger.error(err); + } } logger.info(`Browser executable: ${binaryPath}`); + binaryPath$.next(binaryPath); // subscribers wait for download and extract to complete }; diff --git a/yarn.lock b/yarn.lock index ba8c04e1fc122c..c7b641b79b56e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12308,7 +12308,7 @@ debug@3.X, debug@^3.0.0, debug@^3.0.1, debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, dependencies: ms "^2.1.1" -debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1: +debug@4, debug@4.3.1, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== @@ -12758,10 +12758,10 @@ devtools-protocol@0.0.818844: resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.818844.tgz#d1947278ec85b53e4c8ca598f607a28fa785ba9e" integrity sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg== -devtools-protocol@0.0.854822: - version "0.0.854822" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.854822.tgz#eac3a5260a6b3b4e729a09fdc0c77b0d322e777b" - integrity sha512-xd4D8kHQtB0KtWW0c9xBZD5LVtm9chkMOfs/3Yn01RhT/sFIsVtzTtypfKoFfWBaL+7xCYLxjOLkhwPXaX/Kcg== +devtools-protocol@0.0.901419: + version "0.0.901419" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.901419.tgz#79b5459c48fe7e1c5563c02bd72f8fec3e0cebcd" + integrity sha512-4INMPwNm9XRpBukhNbF7OB6fNTTCaI8pzy/fXg0xQzAy5h3zL1P8xT3QazgKqBrb/hAYwIBizqDBZ7GtJE74QQ== dezalgo@^1.0.0: version "1.0.3" @@ -21155,7 +21155,7 @@ node-emoji@^1.10.0: dependencies: lodash.toarray "^4.4.0" -node-fetch@^1.0.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: +node-fetch@2.6.1, node-fetch@^1.0.1, node-fetch@^2.3.0, node-fetch@^2.6.0, node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== @@ -22608,6 +22608,13 @@ pixelmatch@^5.1.0: dependencies: pngjs "^3.4.0" +pkg-dir@4.2.0, pkg-dir@^4.1.0, pkg-dir@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== + dependencies: + find-up "^4.0.0" + pkg-dir@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-2.0.0.tgz#f6d5d1109e19d63edf428e0bd57e12777615334b" @@ -22622,13 +22629,6 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" -pkg-dir@^4.1.0, pkg-dir@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" - integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== - dependencies: - find-up "^4.0.0" - pkg-up@3.1.0, pkg-up@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5" @@ -23353,6 +23353,11 @@ process@~0.5.1: resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf" integrity sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8= +progress@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.1.tgz#c9242169342b1c29d275889c95734621b1952e31" + integrity sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg== + progress@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be" @@ -23497,7 +23502,7 @@ proxy-from-env@1.0.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" integrity sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4= -proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: +proxy-from-env@1.1.0, proxy-from-env@^1.0.0, proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== @@ -23592,6 +23597,24 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" +puppeteer@^10.2.0: + version "10.4.0" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-10.4.0.tgz#a6465ff97fda0576c4ac29601406f67e6fea3dc7" + integrity sha512-2cP8mBoqnu5gzAVpbZ0fRaobBWZM8GEUF4I1F6WbgHrKV/rz7SX8PG2wMymZgD0wo0UBlg2FBPNxlF/xlqW6+w== + dependencies: + debug "4.3.1" + devtools-protocol "0.0.901419" + extract-zip "2.0.1" + https-proxy-agent "5.0.0" + node-fetch "2.6.1" + pkg-dir "4.2.0" + progress "2.0.1" + proxy-from-env "1.1.0" + rimraf "3.0.2" + tar-fs "2.0.0" + unbzip2-stream "1.3.3" + ws "7.4.6" + puppeteer@^5.3.1: version "5.5.0" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-5.5.0.tgz#331a7edd212ca06b4a556156435f58cbae08af00" @@ -23610,24 +23633,6 @@ puppeteer@^5.3.1: unbzip2-stream "^1.3.3" ws "^7.2.3" -puppeteer@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-8.0.0.tgz#a236669118aa795331c2d0ca19877159e7664705" - integrity sha512-D0RzSWlepeWkxPPdK3xhTcefj8rjah1791GE82Pdjsri49sy11ci/JQsAO8K2NRukqvwEtcI+ImP5F4ZiMvtIQ== - dependencies: - debug "^4.1.0" - devtools-protocol "0.0.854822" - extract-zip "^2.0.0" - https-proxy-agent "^5.0.0" - node-fetch "^2.6.1" - pkg-dir "^4.2.0" - progress "^2.0.1" - proxy-from-env "^1.1.0" - rimraf "^3.0.2" - tar-fs "^2.0.0" - unbzip2-stream "^1.3.3" - ws "^7.2.3" - q@^1.1.2, q@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -27680,6 +27685,16 @@ tape@^5.0.1: string.prototype.trim "^1.2.1" through "^2.3.8" +tar-fs@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.0.tgz#677700fc0c8b337a78bee3623fdc235f21d7afad" + integrity sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA== + dependencies: + chownr "^1.1.1" + mkdirp "^0.5.1" + pump "^3.0.0" + tar-stream "^2.0.0" + tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" @@ -28652,6 +28667,14 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" +unbzip2-stream@1.3.3: + version "1.3.3" + resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz#d156d205e670d8d8c393e1c02ebd506422873f6a" + integrity sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg== + dependencies: + buffer "^5.2.1" + through "^2.3.8" + unbzip2-stream@^1.3.3: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" @@ -30495,6 +30518,11 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" +ws@7.4.6, ws@^7.2.3: + version "7.4.6" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" + integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== + ws@>=7.4.6, ws@^7.4.6: version "7.5.5" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" @@ -30507,11 +30535,6 @@ ws@^6.1.2, ws@^6.2.1: dependencies: async-limiter "~1.0.0" -ws@^7.2.3: - version "7.4.6" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c" - integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A== - x-is-function@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/x-is-function/-/x-is-function-1.0.4.tgz#5d294dc3d268cbdd062580e0c5df77a391d1fa1e"