Merge branch 'master' into tsvb-improve-percentiles

docs/developer/advanced/running-elasticsearch.asciidoc

@@ -76,7 +76,6 @@ If many other users will be interacting with your remote cluster, you'll want to
 [source,bash]
 ----
 kibana.index: '.{YourGitHubHandle}-kibana'
-xpack.task_manager.index: '.{YourGitHubHandle}-task-manager-kibana'
 ----
 
 ==== Running remote clusters

docs/settings/task-manager-settings.asciidoc

@@ -22,9 +22,6 @@ Task Manager runs background tasks by polling for work on an interval.  You can
 | `xpack.task_manager.request_capacity`
   | How many requests can Task Manager buffer before it rejects new requests.  Defaults to 1000.
 
-| `xpack.task_manager.index`
-  | The name of the index used to store task information.  Defaults to `.kibana_task_manager`.
-
   | `xpack.task_manager.max_workers`
   | The maximum number of tasks that this Kibana instance will run simultaneously.  Defaults to 10.
     Starting in 8.0, it will not be possible to set the value greater than 100.

docs/user/production-considerations/task-manager-production-considerations.asciidoc

@@ -12,7 +12,7 @@ This has three major benefits:
 
 [IMPORTANT]
 ==============================================
-Task definitions for alerts and actions are stored in the index specified by <<task-manager-settings, `xpack.task_manager.index`>>. The default is `.kibana_task_manager`.
+Task definitions for alerts and actions are stored in the index `.kibana_task_manager`.
  
 You must have at least one replica of this index for production deployments.
 

examples/search_examples/kibana.json

@@ -11,5 +11,10 @@
   "ui": true,
   "requiredPlugins": ["navigation", "data", "developerExamples", "kibanaUtils", "share"],
   "optionalPlugins": [],
-  "requiredBundles": ["kibanaReact"]
+  "requiredBundles": ["kibanaReact"],
+  "owner": {
+    "name": "App Services",
+    "githubTeam": "kibana-app-services"
+  },
+  "description": "Examples for using the data plugin search service. Includes examples for searching using the high level search source, or low-level search services, as well as integrating with search sessions."
 }

packages/kbn-crypto/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["@kbn/babel-preset/node_preset"]
+}

packages/kbn-crypto/BUILD.bazel

@@ -1,6 +1,7 @@
 
 load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
 load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler")
 
 PKG_BASE_NAME = "kbn-crypto"
 PKG_REQUIRE_NAME = "@kbn/crypto"
@@ -26,22 +27,24 @@ NPM_MODULE_EXTRA_FILES = [
   "README.md"
 ]
 
-SRC_DEPS = [
+RUNTIME_DEPS = [
   "//packages/kbn-dev-utils",
   "@npm//node-forge",
 ]
 
 TYPES_DEPS = [
+  "//packages/kbn-dev-utils",
   "@npm//@types/flot",
   "@npm//@types/jest",
   "@npm//@types/node",
   "@npm//@types/node-forge",
-  "@npm//@types/testing-library__jest-dom",
-  "@npm//resize-observer-polyfill",
-  "@npm//@emotion/react",
 ]
 
-DEPS = SRC_DEPS + TYPES_DEPS
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
 
 ts_config(
   name = "tsconfig",
@@ -53,13 +56,14 @@ ts_config(
 )
 
 ts_project(
-  name = "tsc",
+  name = "tsc_types",
   args = ['--pretty'],
   srcs = SRCS,
-  deps = DEPS,
+  deps = TYPES_DEPS,
   declaration = True,
   declaration_map = True,
-  out_dir = "target",
+  emit_declaration_only = True,
+  out_dir = "target_types",
   source_map = True,
   root_dir = "src",
   tsconfig = ":tsconfig",
@@ -68,7 +72,7 @@ ts_project(
 js_library(
   name = PKG_BASE_NAME,
   srcs = NPM_MODULE_EXTRA_FILES,
-  deps = DEPS + [":tsc"],
+  deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
   package_name = PKG_REQUIRE_NAME,
   visibility = ["//visibility:public"],
 )

packages/kbn-crypto/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "license": "SSPL-1.0 OR Elastic License 2.0",
-  "main": "./target/index.js",
-  "types": "./target/index.d.ts"
+  "main": "./target_node/index.js",
+  "types": "./target_types/index.d.ts"
 }

packages/kbn-crypto/tsconfig.json

@@ -1,9 +1,10 @@
 {
   "extends": "../../tsconfig.bazel.json",
   "compilerOptions": {
-    "outDir": "./target/types",
     "declaration": true,
     "declarationMap": true,
+    "emitDeclarationOnly": true,
+    "outDir": "./target_types",
     "rootDir": "src",
     "sourceMap": true,
     "sourceRoot": "../../../../packages/kbn-crypto/src",

packages/kbn-es-archiver/src/lib/docs/index_doc_records_stream.test.ts

@@ -99,10 +99,8 @@ const testRecords = [
   },
 ];
 
-// FLAKY: https://github.com/elastic/kibana/issues/108043
-it.skip('indexes documents using the bulk client helper', async () => {
+it('indexes documents using the bulk client helper', async () => {
   const client = new MockClient();
-  client.helpers.bulk.mockImplementation(async () => {});
 
   const progress = new Progress();
   const stats = createStats('test', log);
@@ -186,11 +184,11 @@ it.skip('indexes documents using the bulk client helper', async () => {
       "results": Array [
         Object {
           "type": "return",
-          "value": Promise {},
+          "value": undefined,
         },
         Object {
           "type": "return",
-          "value": Promise {},
+          "value": undefined,
         },
       ],
     }

packages/kbn-es-query/src/es_query/build_es_query.ts

@@ -12,17 +12,17 @@ import { buildQueryFromFilters } from './from_filters';
 import { buildQueryFromLucene } from './from_lucene';
 import { Filter, Query } from '../filters';
 import { IndexPatternBase } from './types';
+import { KueryQueryOptions } from '../kuery';
 
 /**
  * Configurations to be used while constructing an ES query.
  * @public
  */
-export interface EsQueryConfig {
+export type EsQueryConfig = KueryQueryOptions & {
   allowLeadingWildcards: boolean;
   queryStringOptions: Record<string, any>;
   ignoreFilterIfFieldNotInIndex: boolean;
-  dateFormatTZ?: string;
-}
+};
 
 function removeMatchAll<T>(filters: T[]) {
   return filters.filter(
@@ -59,7 +59,8 @@ export function buildEsQuery(
     indexPattern,
     queriesByLanguage.kuery,
     config.allowLeadingWildcards,
-    config.dateFormatTZ
+    config.dateFormatTZ,
+    config.filtersInMustClause
   );
   const luceneQuery = buildQueryFromLucene(
     queriesByLanguage.lucene,

packages/kbn-es-query/src/es_query/from_kuery.ts

@@ -15,13 +15,14 @@ export function buildQueryFromKuery(
   indexPattern: IndexPatternBase | undefined,
   queries: Query[] = [],
   allowLeadingWildcards: boolean = false,
-  dateFormatTZ?: string
+  dateFormatTZ?: string,
+  filtersInMustClause: boolean = false
 ) {
   const queryASTs = queries.map((query) => {
     return fromKueryExpression(query.query, { allowLeadingWildcards });
   });
 
-  return buildQuery(indexPattern, queryASTs, { dateFormatTZ });
+  return buildQuery(indexPattern, queryASTs, { dateFormatTZ, filtersInMustClause });
 }
 
 function buildQuery(

packages/kbn-es-query/src/kuery/functions/and.test.ts

@@ -55,6 +55,24 @@ describe('kuery functions', () => {
           )
         );
       });
+
+      test("should wrap subqueries in an ES bool query's must clause for scoring if enabled", () => {
+        const node = nodeTypes.function.buildNode('and', [childNode1, childNode2]);
+        const result = and.toElasticsearchQuery(node, indexPattern, {
+          filtersInMustClause: true,
+        });
+
+        expect(result).toHaveProperty('bool');
+        expect(Object.keys(result).length).toBe(1);
+        expect(result.bool).toHaveProperty('must');
+        expect(Object.keys(result.bool).length).toBe(1);
+
+        expect(result.bool.must).toEqual(
+          [childNode1, childNode2].map((childNode) =>
+            ast.toElasticsearchQuery(childNode, indexPattern)
+          )
+        );
+      });
     });
   });
 });

packages/kbn-es-query/src/kuery/functions/and.ts

@@ -7,7 +7,7 @@
  */
 
 import * as ast from '../ast';
-import { IndexPatternBase, KueryNode } from '../..';
+import { IndexPatternBase, KueryNode, KueryQueryOptions } from '../..';
 
 export function buildNodeParams(children: KueryNode[]) {
   return {
@@ -18,14 +18,16 @@ export function buildNodeParams(children: KueryNode[]) {
 export function toElasticsearchQuery(
   node: KueryNode,
   indexPattern?: IndexPatternBase,
-  config: Record<string, any> = {},
+  config: KueryQueryOptions = {},
   context: Record<string, any> = {}
 ) {
+  const { filtersInMustClause } = config;
   const children = node.arguments || [];
+  const key = filtersInMustClause ? 'must' : 'filter';
 
   return {
     bool: {
-      filter: children.map((child: KueryNode) => {
+      [key]: children.map((child: KueryNode) => {
         return ast.toElasticsearchQuery(child, indexPattern, config, context);
       }),
     },

packages/kbn-es-query/src/kuery/index.ts

@@ -9,4 +9,4 @@
 export { KQLSyntaxError } from './kuery_syntax_error';
 export { nodeTypes, nodeBuilder } from './node_types';
 export { fromKueryExpression, toElasticsearchQuery } from './ast';
-export { DslQuery, KueryNode } from './types';
+export { DslQuery, KueryNode, KueryQueryOptions } from './types';

packages/kbn-es-query/src/kuery/types.ts

@@ -32,3 +32,9 @@ export interface KueryParseOptions {
 }
 
 export { nodeTypes } from './node_types';
+
+/** @public */
+export interface KueryQueryOptions {
+  filtersInMustClause?: boolean;
+  dateFormatTZ?: string;
+}

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -395,7 +395,6 @@ kibana_vars=(
     xpack.spaces.enabled
     xpack.spaces.maxSpaces
     xpack.task_manager.enabled
-    xpack.task_manager.index
     xpack.task_manager.max_attempts
     xpack.task_manager.max_poll_inactivity_cycles
     xpack.task_manager.max_workers

src/dev/typescript/root_refs_config.ts

@@ -11,6 +11,7 @@ import Fs from 'fs/promises';
 
 import dedent from 'dedent';
 import { REPO_ROOT, ToolingLog } from '@kbn/dev-utils';
+import normalize from 'normalize-path';
 
 import { PROJECTS } from './projects';
 
@@ -53,7 +54,7 @@ export async function updateRootRefsConfig(log: ToolingLog) {
   }
 
   const refs = PROJECTS.filter((p) => p.isCompositeProject())
-    .map((p) => `./${Path.relative(REPO_ROOT, p.tsConfigPath)}`)
+    .map((p) => `./${normalize(Path.relative(REPO_ROOT, p.tsConfigPath))}`)
     .sort((a, b) => a.localeCompare(b));
 
   log.debug('updating', ROOT_REFS_CONFIG_PATH);

src/dev/typescript/run_type_check_cli.ts

@@ -37,19 +37,34 @@ export async function runTypeCheckCli() {
           : undefined;
 
       const projects = PROJECTS.filter((p) => {
-        return (
-          !p.disableTypeCheck &&
-          (!projectFilter || p.tsConfigPath === projectFilter) &&
-          !p.isCompositeProject()
-        );
+        return !p.disableTypeCheck && (!projectFilter || p.tsConfigPath === projectFilter);
       });
 
       if (!projects.length) {
-        throw createFailError(`Unable to find project at ${flags.project}`);
+        if (projectFilter) {
+          throw createFailError(`Unable to find project at ${flags.project}`);
+        } else {
+          throw createFailError(`Unable to find projects to type-check`);
+        }
+      }
+
+      const nonCompositeProjects = projects.filter((p) => !p.isCompositeProject());
+      if (!nonCompositeProjects.length) {
+        if (projectFilter) {
+          log.success(
+            `${flags.project} is a composite project so its types are validated by scripts/build_ts_refs`
+          );
+        } else {
+          log.success(
+            `All projects are composite so their types are validated by scripts/build_ts_refs`
+          );
+        }
+
+        return;
       }
 
       const concurrency = Math.min(4, Math.round((Os.cpus() || []).length / 2) || 1) || 1;
-      log.info('running type check in', projects.length, 'non-composite projects');
+      log.info('running type check in', nonCompositeProjects.length, 'non-composite projects');
 
       const tscArgs = [
         ...['--emitDeclarationOnly', 'false'],
@@ -61,7 +76,7 @@ export async function runTypeCheckCli() {
       ];
 
       const failureCount = await lastValueFrom(
-        Rx.from(projects).pipe(
+        Rx.from(nonCompositeProjects).pipe(
           mergeMap(async (p) => {
             const relativePath = Path.relative(process.cwd(), p.tsConfigPath);