From fb19aab307fb80740b60fbd4a0861e75380e96f9 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 1 Feb 2021 12:30:58 +0100 Subject: [PATCH 1/4] [ML] Data Frame Analytics: Adds scatterplot matrix to regression/classification results pages. (#88353) - Adds support for scatterplot matrices to regression/classification results pages - Lazy loads the scatterplot matrix including Vega code using Suspense. The approach is taken from the Kibana Vega plugin, creating this separate bundle means you'll load the 600kb+ Vega code only on pages where actually needed and not e.g. already on the analytics job list. Note for reviews: The file scatterplot_matrix_view.tsx did not change besides the default export, it just shows up as a new file because of the refactoring to support lazy loading. - Adds support for analytics configuration that use the excludes instead of includes field list. - Adds the field used for color legends to tooltips. --- .../components/scatterplot_matrix/index.ts | 2 + .../scatterplot_matrix/scatterplot_matrix.tsx | 319 +---------------- .../scatterplot_matrix_loading.tsx | 19 ++ .../scatterplot_matrix_vega_lite_spec.test.ts | 1 + .../scatterplot_matrix_vega_lite_spec.ts | 11 +- ...trix.scss => scatterplot_matrix_view.scss} | 0 .../scatterplot_matrix_view.tsx | 323 ++++++++++++++++++ .../use_scatterplot_field_options.ts | 50 +++ .../get_scatterplot_matrix_legend_type.ts | 22 ++ .../data_frame_analytics/common/index.ts | 1 + .../common/use_results_view_config.ts | 6 + .../configuration_step_form.tsx | 16 +- .../expandable_section_splom.tsx | 13 +- .../exploration_page_wrapper.tsx | 41 ++- .../outlier_exploration.tsx | 15 +- .../translations/translations/ja-JP.json | 4 +- .../translations/translations/zh-CN.json | 4 +- .../classification_creation.ts | 21 ++ .../outlier_detection_creation.ts | 31 ++ .../regression_creation.ts | 20 ++ .../functional/services/canvas_element.ts | 10 +- .../ml/data_frame_analytics_scatterplot.ts | 40 +++ x-pack/test/functional/services/ml/index.ts | 5 + 23 files changed, 629 insertions(+), 345 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx rename x-pack/plugins/ml/public/application/components/scatterplot_matrix/{scatterplot_matrix.scss => scatterplot_matrix_view.scss} (100%) create mode 100644 x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx create mode 100644 x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts create mode 100644 x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts create mode 100644 x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts index 4f564dde8cb43d..903fe5b6ed985d 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/index.ts @@ -4,5 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +export { useScatterplotFieldOptions } from './use_scatterplot_field_options'; export { LEGEND_TYPES } from './scatterplot_matrix_vega_lite_spec'; export { ScatterplotMatrix } from './scatterplot_matrix'; +export type { ScatterplotMatrixViewProps as ScatterplotMatrixProps } from './scatterplot_matrix_view'; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx index b1ee9afb17788a..a90fe924b91ac5 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx @@ -4,316 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useMemo, useEffect, useState, FC } from 'react'; +import React, { FC, Suspense } from 'react'; -// There is still an issue with Vega Lite's typings with the strict mode Kibana is using. -// @ts-ignore -import { compile } from 'vega-lite/build-es5/vega-lite'; -import { parse, View, Warn } from 'vega'; -import { Handler } from 'vega-tooltip'; +import type { ScatterplotMatrixViewProps } from './scatterplot_matrix_view'; +import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading'; -import { - htmlIdGenerator, - EuiComboBox, - EuiComboBoxOptionOption, - EuiFlexGroup, - EuiFlexItem, - EuiFormRow, - EuiLoadingSpinner, - EuiSelect, - EuiSpacer, - EuiSwitch, - EuiText, -} from '@elastic/eui'; +const ScatterplotMatrixLazy = React.lazy(() => import('./scatterplot_matrix_view')); -import { i18n } from '@kbn/i18n'; - -import type { SearchResponse7 } from '../../../../common/types/es_client'; - -import { useMlApiContext } from '../../contexts/kibana'; - -import { getProcessedFields } from '../data_grid'; -import { useCurrentEuiTheme } from '../color_range_legend'; - -import { - getScatterplotMatrixVegaLiteSpec, - LegendType, - OUTLIER_SCORE_FIELD, -} from './scatterplot_matrix_vega_lite_spec'; - -import './scatterplot_matrix.scss'; - -const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1; -const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000; - -const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', { - defaultMessage: 'On', -}); -const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', { - defaultMessage: 'Off', -}); - -const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d })); - -interface ScatterplotMatrixProps { - fields: string[]; - index: string; - resultsField?: string; - color?: string; - legendType?: LegendType; -} - -export const ScatterplotMatrix: FC = ({ - fields: allFields, - index, - resultsField, - color, - legendType, -}) => { - const { esSearch } = useMlApiContext(); - - // dynamicSize is optionally used for outlier charts where the scatterplot marks - // are sized according to outlier_score - const [dynamicSize, setDynamicSize] = useState(false); - - // used to give the use the option to customize the fields used for the matrix axes - const [fields, setFields] = useState([]); - - useEffect(() => { - const defaultFields = - allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS - ? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS) - : allFields; - setFields(defaultFields); - }, [allFields]); - - // the amount of documents to be fetched - const [fetchSize, setFetchSize] = useState(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE); - // flag to add a random score to the ES query to fetch documents - const [randomizeQuery, setRandomizeQuery] = useState(false); - - const [isLoading, setIsLoading] = useState(false); - - // contains the fetched documents and columns to be passed on to the Vega spec. - const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>(); - - // formats the array of field names for EuiComboBox - const fieldOptions = useMemo( - () => - allFields.map((d) => ({ - label: d, - })), - [allFields] - ); - - const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => { - setFields(newFields.map((d) => d.label)); - }; - - const fetchSizeOnChange = (e: React.ChangeEvent) => { - setFetchSize( - Math.min( - Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE), - SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE - ) - ); - }; - - const randomizeQueryOnChange = () => { - setRandomizeQuery(!randomizeQuery); - }; - - const dynamicSizeOnChange = () => { - setDynamicSize(!dynamicSize); - }; - - const { euiTheme } = useCurrentEuiTheme(); - - useEffect(() => { - async function fetchSplom(options: { didCancel: boolean }) { - setIsLoading(true); - try { - const queryFields = [ - ...fields, - ...(color !== undefined ? [color] : []), - ...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]), - ]; - - const query = randomizeQuery - ? { - function_score: { - random_score: { seed: 10, field: '_seq_no' }, - }, - } - : { match_all: {} }; - - const resp: SearchResponse7 = await esSearch({ - index, - body: { - fields: queryFields, - _source: false, - query, - from: 0, - size: fetchSize, - }, - }); - - if (!options.didCancel) { - const items = resp.hits.hits.map((d) => - getProcessedFields(d.fields, (key: string) => - key.startsWith(`${resultsField}.feature_importance`) - ) - ); - - setSplom({ columns: fields, items }); - setIsLoading(false); - } - } catch (e) { - // TODO error handling - setIsLoading(false); - } - } - - const options = { didCancel: false }; - fetchSplom(options); - return () => { - options.didCancel = true; - }; - // stringify the fields array, otherwise the comparator will trigger on new but identical instances. - }, [fetchSize, JSON.stringify(fields), index, randomizeQuery, resultsField]); - - const htmlId = useMemo(() => htmlIdGenerator()(), []); - - useEffect(() => { - if (splom === undefined) { - return; - } - - const { items, columns } = splom; - - const values = - resultsField !== undefined - ? items - : items.map((d) => { - d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0; - return d; - }); - - const vegaSpec = getScatterplotMatrixVegaLiteSpec( - values, - columns, - euiTheme, - resultsField, - color, - legendType, - dynamicSize - ); - - const vgSpec = compile(vegaSpec).spec; - - const view = new View(parse(vgSpec)) - .logLevel(Warn) - .renderer('canvas') - .tooltip(new Handler().call) - .initialize(`#${htmlId}`); - - view.runAsync(); // evaluate and render the view - }, [resultsField, splom, color, legendType, dynamicSize]); - - return ( - <> - {splom === undefined ? ( - - - - - - ) : ( - <> - - - - ({ - label: d, - }))} - onChange={fieldsOnChange} - isClearable={true} - data-test-subj="mlScatterplotMatrixFieldsComboBox" - /> - - - - - - - - - - - - - {resultsField !== undefined && legendType === undefined && ( - - - - - - )} - - -
- - )} - - ); -}; +export const ScatterplotMatrix: FC = (props) => ( + }> + + +); diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx new file mode 100644 index 00000000000000..ccd4153769e9c3 --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_loading.tsx @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { EuiLoadingSpinner, EuiSpacer, EuiText } from '@elastic/eui'; + +export const ScatterplotMatrixLoading = () => { + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts index dd467161ff489e..eada64b7a03cae 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.test.ts @@ -163,6 +163,7 @@ describe('getScatterplotMatrixVegaLiteSpec()', () => { type: 'nominal', }); expect(vegaLiteSpec.spec.encoding.tooltip).toEqual([ + { field: 'the-color-field', type: 'nominal' }, { field: 'x', type: 'quantitative' }, { field: 'y', type: 'quantitative' }, ]); diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts index 9e0834dd8b9221..c943e5d1b06e3b 100644 --- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec.ts @@ -35,6 +35,8 @@ export const getColorSpec = ( color?: string, legendType?: LegendType ) => { + // For outlier detection result pages coloring is done based on a threshold. + // This returns a Vega spec using a conditional to return the color. if (outliers) { return { condition: { @@ -45,6 +47,8 @@ export const getColorSpec = ( }; } + // Based on the type of the color field, + // this returns either a continuous or categorical color spec. if (color !== undefined && legendType !== undefined) { return { field: color, @@ -80,6 +84,8 @@ export const getScatterplotMatrixVegaLiteSpec = ( }); } + const colorSpec = getColorSpec(euiTheme, outliers, color, legendType); + return { $schema: 'https://vega.github.io/schema/vega-lite/v4.17.0.json', background: 'transparent', @@ -115,10 +121,10 @@ export const getScatterplotMatrixVegaLiteSpec = ( : { type: 'circle', opacity: 0.75, size: 8 }), }, encoding: { - color: getColorSpec(euiTheme, outliers, color, legendType), + color: colorSpec, ...(dynamicSize ? { - stroke: getColorSpec(euiTheme, outliers, color, legendType), + stroke: colorSpec, opacity: { condition: { value: 1, @@ -163,6 +169,7 @@ export const getScatterplotMatrixVegaLiteSpec = ( scale: { zero: false }, }, tooltip: [ + ...(color !== undefined ? [{ type: colorSpec.type, field: color }] : []), ...columns.map((d) => ({ type: LEGEND_TYPES.QUANTITATIVE, field: d })), ...(outliers ? [{ type: LEGEND_TYPES.QUANTITATIVE, field: OUTLIER_SCORE_FIELD, format: '.3f' }] diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.scss b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.scss similarity index 100% rename from x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.scss rename to x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.scss diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx new file mode 100644 index 00000000000000..0c065c1154a98d --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix_view.tsx @@ -0,0 +1,323 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useMemo, useEffect, useState, FC } from 'react'; + +// There is still an issue with Vega Lite's typings with the strict mode Kibana is using. +// @ts-ignore +import { compile } from 'vega-lite/build-es5/vega-lite'; +import { parse, View, Warn } from 'vega'; +import { Handler } from 'vega-tooltip'; + +import { + htmlIdGenerator, + EuiComboBox, + EuiComboBoxOptionOption, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSelect, + EuiSwitch, +} from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import type { SearchResponse7 } from '../../../../common/types/es_client'; +import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics'; + +import { useMlApiContext } from '../../contexts/kibana'; + +import { getProcessedFields } from '../data_grid'; +import { useCurrentEuiTheme } from '../color_range_legend'; + +import { ScatterplotMatrixLoading } from './scatterplot_matrix_loading'; + +import { + getScatterplotMatrixVegaLiteSpec, + LegendType, + OUTLIER_SCORE_FIELD, +} from './scatterplot_matrix_vega_lite_spec'; + +import './scatterplot_matrix_view.scss'; + +const SCATTERPLOT_MATRIX_DEFAULT_FIELDS = 4; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE = 1000; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE = 1; +const SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE = 10000; + +const TOGGLE_ON = i18n.translate('xpack.ml.splom.toggleOn', { + defaultMessage: 'On', +}); +const TOGGLE_OFF = i18n.translate('xpack.ml.splom.toggleOff', { + defaultMessage: 'Off', +}); + +const sampleSizeOptions = [100, 1000, 10000].map((d) => ({ value: d, text: '' + d })); + +export interface ScatterplotMatrixViewProps { + fields: string[]; + index: string; + resultsField?: string; + color?: string; + legendType?: LegendType; + searchQuery?: ResultsSearchQuery; +} + +export const ScatterplotMatrixView: FC = ({ + fields: allFields, + index, + resultsField, + color, + legendType, + searchQuery, +}) => { + const { esSearch } = useMlApiContext(); + + // dynamicSize is optionally used for outlier charts where the scatterplot marks + // are sized according to outlier_score + const [dynamicSize, setDynamicSize] = useState(false); + + // used to give the use the option to customize the fields used for the matrix axes + const [fields, setFields] = useState([]); + + useEffect(() => { + const defaultFields = + allFields.length > SCATTERPLOT_MATRIX_DEFAULT_FIELDS + ? allFields.slice(0, SCATTERPLOT_MATRIX_DEFAULT_FIELDS) + : allFields; + setFields(defaultFields); + }, [allFields]); + + // the amount of documents to be fetched + const [fetchSize, setFetchSize] = useState(SCATTERPLOT_MATRIX_DEFAULT_FETCH_SIZE); + // flag to add a random score to the ES query to fetch documents + const [randomizeQuery, setRandomizeQuery] = useState(false); + + const [isLoading, setIsLoading] = useState(false); + + // contains the fetched documents and columns to be passed on to the Vega spec. + const [splom, setSplom] = useState<{ items: any[]; columns: string[] } | undefined>(); + + // formats the array of field names for EuiComboBox + const fieldOptions = useMemo( + () => + allFields.map((d) => ({ + label: d, + })), + [allFields] + ); + + const fieldsOnChange = (newFields: EuiComboBoxOptionOption[]) => { + setFields(newFields.map((d) => d.label)); + }; + + const fetchSizeOnChange = (e: React.ChangeEvent) => { + setFetchSize( + Math.min( + Math.max(parseInt(e.target.value, 10), SCATTERPLOT_MATRIX_DEFAULT_FETCH_MIN_SIZE), + SCATTERPLOT_MATRIX_DEFAULT_FETCH_MAX_SIZE + ) + ); + }; + + const randomizeQueryOnChange = () => { + setRandomizeQuery(!randomizeQuery); + }; + + const dynamicSizeOnChange = () => { + setDynamicSize(!dynamicSize); + }; + + const { euiTheme } = useCurrentEuiTheme(); + + useEffect(() => { + async function fetchSplom(options: { didCancel: boolean }) { + setIsLoading(true); + try { + const queryFields = [ + ...fields, + ...(color !== undefined ? [color] : []), + ...(legendType !== undefined ? [] : [`${resultsField}.${OUTLIER_SCORE_FIELD}`]), + ]; + + const queryFallback = searchQuery !== undefined ? searchQuery : { match_all: {} }; + const query = randomizeQuery + ? { + function_score: { + query: queryFallback, + random_score: { seed: 10, field: '_seq_no' }, + }, + } + : queryFallback; + + const resp: SearchResponse7 = await esSearch({ + index, + body: { + fields: queryFields, + _source: false, + query, + from: 0, + size: fetchSize, + }, + }); + + if (!options.didCancel) { + const items = resp.hits.hits.map((d) => + getProcessedFields(d.fields, (key: string) => + key.startsWith(`${resultsField}.feature_importance`) + ) + ); + + setSplom({ columns: fields, items }); + setIsLoading(false); + } + } catch (e) { + // TODO error handling + setIsLoading(false); + } + } + + const options = { didCancel: false }; + fetchSplom(options); + return () => { + options.didCancel = true; + }; + // stringify the fields array and search, otherwise the comparator will trigger on new but identical instances. + }, [fetchSize, JSON.stringify({ fields, searchQuery }), index, randomizeQuery, resultsField]); + + const htmlId = useMemo(() => htmlIdGenerator()(), []); + + useEffect(() => { + if (splom === undefined) { + return; + } + + const { items, columns } = splom; + + const values = + resultsField !== undefined + ? items + : items.map((d) => { + d[`${resultsField}.${OUTLIER_SCORE_FIELD}`] = 0; + return d; + }); + + const vegaSpec = getScatterplotMatrixVegaLiteSpec( + values, + columns, + euiTheme, + resultsField, + color, + legendType, + dynamicSize + ); + + const vgSpec = compile(vegaSpec).spec; + + const view = new View(parse(vgSpec)) + .logLevel(Warn) + .renderer('canvas') + .tooltip(new Handler().call) + .initialize(`#${htmlId}`); + + view.runAsync(); // evaluate and render the view + }, [resultsField, splom, color, legendType, dynamicSize]); + + return ( + <> + {splom === undefined ? ( + + ) : ( + <> + + + + ({ + label: d, + }))} + onChange={fieldsOnChange} + isClearable={true} + data-test-subj="mlScatterplotMatrixFieldsComboBox" + /> + + + + + + + + + + + + + {resultsField !== undefined && legendType === undefined && ( + + + + + + )} + + +
+ + )} + + ); +}; + +// required for dynamic import using React.lazy() +// eslint-disable-next-line import/no-default-export +export default ScatterplotMatrixView; diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts new file mode 100644 index 00000000000000..f5eedbc03951fb --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/use_scatterplot_field_options.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useMemo } from 'react'; + +import type { IndexPattern } from '../../../../../../../src/plugins/data/public'; + +import { ML__INCREMENTAL_ID } from '../../data_frame_analytics/common/fields'; + +export const useScatterplotFieldOptions = ( + indexPattern?: IndexPattern, + includes?: string[], + excludes?: string[], + resultsField = '' +): string[] => { + return useMemo(() => { + const fields: string[] = []; + + if (indexPattern === undefined || includes === undefined) { + return fields; + } + + if (includes.length > 1) { + fields.push( + ...includes.filter((d) => + indexPattern.fields.some((f) => f.name === d && f.type === 'number') + ) + ); + } else { + fields.push( + ...indexPattern.fields + .filter( + (f) => + f.type === 'number' && + !indexPattern.metaFields.includes(f.name) && + !f.name.startsWith(`${resultsField}.`) && + f.name !== ML__INCREMENTAL_ID + ) + .map((f) => f.name) + ); + } + + return Array.isArray(excludes) && excludes.length > 0 + ? fields.filter((f) => !excludes.includes(f)) + : fields; + }, [indexPattern, includes, excludes]); +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts new file mode 100644 index 00000000000000..8850d42577bd99 --- /dev/null +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_scatterplot_matrix_legend_type.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ANALYSIS_CONFIG_TYPE } from './analytics'; + +import { AnalyticsJobType } from '../pages/analytics_management/hooks/use_create_analytics_form/state'; + +import { LEGEND_TYPES } from '../../components/scatterplot_matrix/scatterplot_matrix_vega_lite_spec'; + +export const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType | 'unknown') => { + switch (jobType) { + case ANALYSIS_CONFIG_TYPE.CLASSIFICATION: + return LEGEND_TYPES.NOMINAL; + case ANALYSIS_CONFIG_TYPE.REGRESSION: + return LEGEND_TYPES.QUANTITATIVE; + default: + return undefined; + } +}; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts index 7ba3e910ddd320..d03f73ad135750 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts @@ -41,6 +41,7 @@ export { export { getIndexData } from './get_index_data'; export { getIndexFields } from './get_index_fields'; +export { getScatterplotMatrixLegendType } from './get_scatterplot_matrix_legend_type'; export { useResultsViewConfig } from './use_results_view_config'; export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts index 185513f75a12c7..361a1262a59f8f 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts @@ -102,6 +102,12 @@ export const useResultsViewConfig = (jobId: string) => { try { indexP = await mlContext.indexPatterns.get(destIndexPatternId); + + // Force refreshing the fields list here because a user directly coming + // from the job creation wizard might land on the page without the + // index pattern being fully initialized because it was created + // before the analytics job populated the destination index. + await mlContext.indexPatterns.refreshFields(indexP); } catch (e) { indexP = undefined; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index a5991f77e88e21..4b86f5ca128963 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -27,10 +27,10 @@ import { TRAINING_PERCENT_MAX, FieldSelectionItem, } from '../../../../common/analytics'; +import { getScatterplotMatrixLegendType } from '../../../../common/get_scatterplot_matrix_legend_type'; import { CreateAnalyticsStepProps } from '../../../analytics_management/hooks/use_create_analytics_form'; import { Messages } from '../shared'; import { - AnalyticsJobType, DEFAULT_MODEL_MEMORY_LIMIT, State, } from '../../../analytics_management/hooks/use_create_analytics_form/state'; @@ -51,18 +51,7 @@ import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/sea import { ExplorationQueryBarProps } from '../../../analytics_exploration/components/exploration_query_bar/exploration_query_bar'; import { Query } from '../../../../../../../../../../src/plugins/data/common/query'; -import { LEGEND_TYPES, ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; - -const getScatterplotMatrixLegendType = (jobType: AnalyticsJobType) => { - switch (jobType) { - case ANALYSIS_CONFIG_TYPE.CLASSIFICATION: - return LEGEND_TYPES.NOMINAL; - case ANALYSIS_CONFIG_TYPE.REGRESSION: - return LEGEND_TYPES.QUANTITATIVE; - default: - return undefined; - } -}; +import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; const requiredFieldsErrorText = i18n.translate( 'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage', @@ -498,6 +487,7 @@ export const ConfigurationStepForm: FC = ({ : undefined } legendType={getScatterplotMatrixLegendType(jobType)} + searchQuery={jobConfigQuery} /> diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx index 5ec8963e0fc25b..8c51c95d7fd639 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_splom.tsx @@ -12,17 +12,14 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { EuiHorizontalRule, EuiSpacer } from '@elastic/eui'; -import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; +import { + ScatterplotMatrix, + ScatterplotMatrixProps, +} from '../../../../../components/scatterplot_matrix'; import { ExpandableSection } from './expandable_section'; -interface ExpandableSectionSplomProps { - fields: string[]; - index: string; - resultsField?: string; -} - -export const ExpandableSectionSplom: FC = (props) => { +export const ExpandableSectionSplom: FC = (props) => { const splomSectionHeaderItems = undefined; const splomSectionContent = ( <> diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx index 1329644322f337..46715af0ef0cbd 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_page_wrapper/exploration_page_wrapper.tsx @@ -9,16 +9,21 @@ import React, { FC, useCallback, useState } from 'react'; import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { getAnalysisType, getDependentVar } from '../../../../../../../common/util/analytics_utils'; + +import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix'; + import { defaultSearchQuery, + getScatterplotMatrixLegendType, useResultsViewConfig, DataFrameAnalyticsConfig, } from '../../../../common'; -import { ResultsSearchQuery } from '../../../../common/analytics'; +import { ResultsSearchQuery, ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics'; import { DataFrameTaskStateType } from '../../../analytics_management/components/analytics_list/common'; -import { ExpandableSectionAnalytics } from '../expandable_section'; +import { ExpandableSectionAnalytics, ExpandableSectionSplom } from '../expandable_section'; import { ExplorationResultsTable } from '../exploration_results_table'; import { ExplorationQueryBar } from '../exploration_query_bar'; import { JobConfigErrorCallout } from '../job_config_error_callout'; @@ -99,6 +104,14 @@ export const ExplorationPageWrapper: FC = ({ language: pageUrlState.queryLanguage, }; + const resultsField = jobConfig?.dest.results_field ?? ''; + const scatterplotFieldOptions = useScatterplotFieldOptions( + indexPattern, + jobConfig?.analyzed_fields.includes, + jobConfig?.analyzed_fields.excludes, + resultsField + ); + if (indexPatternErrorMessage !== undefined) { return ( @@ -125,6 +138,9 @@ export const ExplorationPageWrapper: FC = ({ ); } + const jobType = + jobConfig && jobConfig.analysis ? getAnalysisType(jobConfig?.analysis) : undefined; + return ( <> {typeof jobConfig?.description !== 'undefined' && ( @@ -179,6 +195,27 @@ export const ExplorationPageWrapper: FC = ({ )} + {isLoadingJobConfig === true && jobConfig === undefined && } + {isLoadingJobConfig === false && + jobConfig !== undefined && + isInitialized === true && + typeof jobConfig?.id === 'string' && + scatterplotFieldOptions.length > 1 && + typeof jobConfig?.analysis !== 'undefined' && ( + + )} + {isLoadingJobConfig === true && jobConfig === undefined && } {isLoadingJobConfig === false && jobConfig !== undefined && diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx index 26eee9bc95d730..7e11e0bd970157 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/outlier_exploration.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useState, FC, useCallback } from 'react'; +import React, { useCallback, useState, FC } from 'react'; import { EuiCallOut, EuiPanel, EuiSpacer, EuiText } from '@elastic/eui'; @@ -15,6 +15,7 @@ import { COLOR_RANGE, COLOR_RANGE_SCALE, } from '../../../../../components/color_range_legend'; +import { useScatterplotFieldOptions } from '../../../../../components/scatterplot_matrix'; import { SavedSearchQuery } from '../../../../../contexts/ml'; import { defaultSearchQuery, isOutlierAnalysis, useResultsViewConfig } from '../../../../common'; @@ -90,6 +91,13 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = (d) => d.id === `${resultsField}.${FEATURE_INFLUENCE}.feature_name` ) === -1; + const scatterplotFieldOptions = useScatterplotFieldOptions( + indexPattern, + jobConfig?.analyzed_fields.includes, + jobConfig?.analyzed_fields.excludes, + resultsField + ); + if (indexPatternErrorMessage !== undefined) { return ( @@ -126,11 +134,12 @@ export const OutlierExploration: FC = React.memo(({ jobId }) = )} {typeof jobConfig?.id === 'string' && } - {typeof jobConfig?.id === 'string' && jobConfig?.analyzed_fields.includes.length > 1 && ( + {typeof jobConfig?.id === 'string' && scatterplotFieldOptions.length > 1 && ( )} {showLegacyFeatureInfluenceFormatCallout && ( diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b50a7092e108d3..4369cbf35594dd 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -14209,8 +14209,8 @@ "xpack.ml.splom.dynamicSizeLabel": "動的サイズ", "xpack.ml.splom.fieldSelectionLabel": "フィールド", "xpack.ml.splom.fieldSelectionPlaceholder": "フィールドを選択", - "xpack.ml.splom.RandomScoringLabel": "ランダムスコアリング", - "xpack.ml.splom.SampleSizeLabel": "サンプルサイズ", + "xpack.ml.splom.randomScoringLabel": "ランダムスコアリング", + "xpack.ml.splom.sampleSizeLabel": "サンプルサイズ", "xpack.ml.splom.toggleOff": "オフ", "xpack.ml.splom.toggleOn": "オン", "xpack.ml.splomSpec.outlierScoreThresholdName": "異常スコアしきい値：", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index a93b2c78690c10..d2504c8752c056 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14248,8 +14248,8 @@ "xpack.ml.splom.dynamicSizeLabel": "动态大小", "xpack.ml.splom.fieldSelectionLabel": "字段", "xpack.ml.splom.fieldSelectionPlaceholder": "选择字段", - "xpack.ml.splom.RandomScoringLabel": "随机评分", - "xpack.ml.splom.SampleSizeLabel": "样例大小", + "xpack.ml.splom.randomScoringLabel": "随机评分", + "xpack.ml.splom.sampleSizeLabel": "样例大小", "xpack.ml.splom.toggleOff": "关闭", "xpack.ml.splom.toggleOn": "开启", "xpack.ml.splomSpec.outlierScoreThresholdName": "离群值分数阈值：", diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts index 6b42306c08c92a..b0f1e316e626a0 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts @@ -40,6 +40,17 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '60mb', createIndexPattern: true, expected: { + scatterplotMatrixColorStats: [ + // background + { key: '#000000', value: 94 }, + // tick/grid/axis + { key: '#DDDDDD', value: 1 }, + { key: '#D3DAE6', value: 1 }, + { key: '#F5F7FA', value: 1 }, + // scatterplot circles + { key: '#6A717D', value: 1 }, + { key: '#54B39A', value: 1 }, + ], row: { type: 'classification', status: 'stopped', @@ -89,6 +100,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStats + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -207,6 +224,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStats + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index 53daa0cae2522b..419239d1d15cad 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -49,6 +49,27 @@ export default function ({ getService }: FtrProviderContext) { { chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' }, { chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' }, ], + scatterplotMatrixColorStatsWizard: [ + // background + { key: '#000000', value: 91 }, + // tick/grid/axis + { key: '#6A717D', value: 2 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + // scatterplot circles + { key: '#54B399', value: 1 }, + { key: '#54B39A', value: 1 }, + ], + scatterplotMatrixColorStatsResults: [ + // background + { key: '#000000', value: 91 }, + // tick/grid/axis, grey markers + // the red outlier color is not above the 1% threshold. + { key: '#6A717D', value: 2 }, + { key: '#98A2B3', value: 1 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + ], row: { type: 'outlier_detection', status: 'stopped', @@ -105,6 +126,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStatsWizard + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -221,6 +248,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertOutlierTablePanelExists(); await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStatsResults + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts index fef22fcebc3ed2..f1d19a82caa9bf 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts @@ -39,6 +39,16 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '20mb', createIndexPattern: true, expected: { + scatterplotMatrixColorStats: [ + // background + { key: '#000000', value: 80 }, + // tick/grid/axis + { key: '#6A717D', value: 1 }, + { key: '#F5F7FA', value: 2 }, + { key: '#D3DAE6', value: 1 }, + // because a continuous color scale is used for the scatterplot circles, + // none of the generated colors is above the 1% threshold. + ], row: { type: 'regression', status: 'stopped', @@ -89,6 +99,12 @@ export default function ({ getService }: FtrProviderContext) { await ml.testExecution.logTestStep('displays the include fields selection'); await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + await ml.testExecution.logTestStep('displays the scatterplot matrix'); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow', + testData.expected.scatterplotMatrixColorStats + ); + await ml.testExecution.logTestStep('continues to the additional options step'); await ml.dataFrameAnalyticsCreation.continueToAdditionalOptionsStep(); @@ -207,6 +223,10 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsResults.assertResultsTableExists(); await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist(); await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + await ml.dataFrameAnalyticsScatterplot.assertScatterplotMatrix( + 'mlDFExpandableSection-splom', + testData.expected.scatterplotMatrixColorStats + ); }); it('displays the analytics job in the map view', async () => { diff --git a/x-pack/test/functional/services/canvas_element.ts b/x-pack/test/functional/services/canvas_element.ts index e2a42c5dc43c30..08ac38d970225e 100644 --- a/x-pack/test/functional/services/canvas_element.ts +++ b/x-pack/test/functional/services/canvas_element.ts @@ -43,9 +43,13 @@ export async function CanvasElementProvider({ getService }: FtrProviderContext) public async getImageData(selector: string): Promise { return await driver.executeScript( ` - const el = document.querySelector('${selector}'); - const ctx = el.getContext('2d'); - return ctx.getImageData(0, 0, el.width, el.height).data; + try { + const el = document.querySelector('${selector}'); + const ctx = el.getContext('2d'); + return ctx.getImageData(0, 0, el.width, el.height).data; + } catch(e) { + return []; + } ` ); } diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts b/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts new file mode 100644 index 00000000000000..3472e5079c79a1 --- /dev/null +++ b/x-pack/test/functional/services/ml/data_frame_analytics_scatterplot.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export function MachineLearningDataFrameAnalyticsScatterplotProvider({ + getService, +}: FtrProviderContext) { + const canvasElement = getService('canvasElement'); + const testSubjects = getService('testSubjects'); + + return new (class AnalyticsScatterplot { + public async assertScatterplotMatrix( + dataTestSubj: string, + expectedColorStats: Array<{ + key: string; + value: number; + }> + ) { + await testSubjects.existOrFail(dataTestSubj); + await testSubjects.existOrFail('mlScatterplotMatrix'); + + const actualColorStats = await canvasElement.getColorStats( + `[data-test-subj="mlScatterplotMatrix"] canvas`, + expectedColorStats, + 1 + ); + expect(actualColorStats.every((d) => d.withinTolerance)).to.eql( + true, + `Color stats for scatterplot matrix should be within tolerance. Expected: '${JSON.stringify( + expectedColorStats + )}' (got '${JSON.stringify(actualColorStats)}')` + ); + } + })(); +} diff --git a/x-pack/test/functional/services/ml/index.ts b/x-pack/test/functional/services/ml/index.ts index c1a9ac304dd69f..aa87bc5dc47727 100644 --- a/x-pack/test/functional/services/ml/index.ts +++ b/x-pack/test/functional/services/ml/index.ts @@ -17,6 +17,7 @@ import { MachineLearningDataFrameAnalyticsProvider } from './data_frame_analytic import { MachineLearningDataFrameAnalyticsCreationProvider } from './data_frame_analytics_creation'; import { MachineLearningDataFrameAnalyticsEditProvider } from './data_frame_analytics_edit'; import { MachineLearningDataFrameAnalyticsResultsProvider } from './data_frame_analytics_results'; +import { MachineLearningDataFrameAnalyticsScatterplotProvider } from './data_frame_analytics_scatterplot'; import { MachineLearningDataFrameAnalyticsMapProvider } from './data_frame_analytics_map'; import { MachineLearningDataFrameAnalyticsTableProvider } from './data_frame_analytics_table'; import { MachineLearningDataVisualizerProvider } from './data_visualizer'; @@ -63,6 +64,9 @@ export function MachineLearningProvider(context: FtrProviderContext) { const dataFrameAnalyticsResults = MachineLearningDataFrameAnalyticsResultsProvider(context); const dataFrameAnalyticsMap = MachineLearningDataFrameAnalyticsMapProvider(context); const dataFrameAnalyticsTable = MachineLearningDataFrameAnalyticsTableProvider(context); + const dataFrameAnalyticsScatterplot = MachineLearningDataFrameAnalyticsScatterplotProvider( + context + ); const dataVisualizer = MachineLearningDataVisualizerProvider(context); const dataVisualizerTable = MachineLearningDataVisualizerTableProvider(context, commonUI); @@ -105,6 +109,7 @@ export function MachineLearningProvider(context: FtrProviderContext) { dataFrameAnalyticsResults, dataFrameAnalyticsMap, dataFrameAnalyticsTable, + dataFrameAnalyticsScatterplot, dataVisualizer, dataVisualizerFileBased, dataVisualizerIndexBased, From 53637d01580b4016b110e290f80283eae35b1408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Mon, 1 Feb 2021 14:48:30 +0100 Subject: [PATCH 2/4] [Logs UI] as a kibana embeddable (#88618) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-optimizer/limits.yml | 2 +- x-pack/plugins/infra/kibana.json | 2 +- .../public/components/log_stream/index.ts | 7 ++ .../log_stream/lazy_log_stream_wrapper.tsx | 4 +- .../log_stream/{index.tsx => log_stream.tsx} | 3 +- .../log_stream/log_stream_embeddable.tsx | 91 +++++++++++++++++++ .../log_stream_embeddable_factory.ts | 37 ++++++++ .../containers/logs/log_stream/index.ts | 22 ++++- x-pack/plugins/infra/public/plugin.ts | 9 ++ x-pack/plugins/infra/public/types.ts | 2 + 10 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 x-pack/plugins/infra/public/components/log_stream/index.ts rename x-pack/plugins/infra/public/components/log_stream/{index.tsx => log_stream.tsx} (98%) create mode 100644 x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx create mode 100644 x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index a13976d1487388..794503656ba048 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -34,7 +34,7 @@ pageLoadAssetSize: indexLifecycleManagement: 107090 indexManagement: 140608 indexPatternManagement: 154222 - infra: 197873 + infra: 204800 fleet: 415829 ingestPipelines: 58003 inputControlVis: 172675 diff --git a/x-pack/plugins/infra/kibana.json b/x-pack/plugins/infra/kibana.json index e84767f4931ca8..d1fa83793d1dd8 100644 --- a/x-pack/plugins/infra/kibana.json +++ b/x-pack/plugins/infra/kibana.json @@ -6,7 +6,7 @@ "features", "usageCollection", "spaces", - + "embeddable", "data", "dataEnhanced", "visTypeTimeseries", diff --git a/x-pack/plugins/infra/public/components/log_stream/index.ts b/x-pack/plugins/infra/public/components/log_stream/index.ts new file mode 100644 index 00000000000000..6abb292f919d91 --- /dev/null +++ b/x-pack/plugins/infra/public/components/log_stream/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './log_stream'; diff --git a/x-pack/plugins/infra/public/components/log_stream/lazy_log_stream_wrapper.tsx b/x-pack/plugins/infra/public/components/log_stream/lazy_log_stream_wrapper.tsx index 65433aab15716f..13eb6431f97a37 100644 --- a/x-pack/plugins/infra/public/components/log_stream/lazy_log_stream_wrapper.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/lazy_log_stream_wrapper.tsx @@ -5,9 +5,9 @@ */ import React from 'react'; -import type { LogStreamProps } from './'; +import type { LogStreamProps } from './log_stream'; -const LazyLogStream = React.lazy(() => import('./')); +const LazyLogStream = React.lazy(() => import('./log_stream')); export const LazyLogStreamWrapper: React.FC = (props) => ( }> diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx similarity index 98% rename from x-pack/plugins/infra/public/components/log_stream/index.tsx rename to x-pack/plugins/infra/public/components/log_stream/log_stream.tsx index b485a21221af21..b7410fda6f6fd4 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx @@ -17,6 +17,7 @@ import { useLogStream } from '../../containers/logs/log_stream'; import { ScrollableLogTextStreamView } from '../logging/log_text_stream'; import { LogColumnRenderConfiguration } from '../../utils/log_column_render_configuration'; import { JsonValue } from '../../../../../../src/plugins/kibana_utils/common'; +import { Query } from '../../../../../../src/plugins/data/common'; const PAGE_THRESHOLD = 2; @@ -55,7 +56,7 @@ export interface LogStreamProps { sourceId?: string; startTimestamp: number; endTimestamp: number; - query?: string; + query?: string | Query; center?: LogEntryCursor; highlight?: string; height?: string | number; diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx new file mode 100644 index 00000000000000..0d6dfc50960f97 --- /dev/null +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { CoreStart } from 'kibana/public'; + +import { I18nProvider } from '@kbn/i18n/react'; +import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; +import { EuiThemeProvider } from '../../../../../../src/plugins/kibana_react/common'; +import { Query, TimeRange } from '../../../../../../src/plugins/data/public'; +import { + Embeddable, + EmbeddableInput, + IContainer, +} from '../../../../../../src/plugins/embeddable/public'; +import { datemathToEpochMillis } from '../../utils/datemath'; +import { LazyLogStreamWrapper } from './lazy_log_stream_wrapper'; + +export const LOG_STREAM_EMBEDDABLE = 'LOG_STREAM_EMBEDDABLE'; + +export interface LogStreamEmbeddableInput extends EmbeddableInput { + timeRange: TimeRange; + query: Query; +} + +export class LogStreamEmbeddable extends Embeddable { + public readonly type = LOG_STREAM_EMBEDDABLE; + private node?: HTMLElement; + + constructor( + private services: CoreStart, + initialInput: LogStreamEmbeddableInput, + parent?: IContainer + ) { + super(initialInput, {}, parent); + } + + public render(node: HTMLElement) { + if (this.node) { + ReactDOM.unmountComponentAtNode(this.node); + } + this.node = node; + + this.renderComponent(); + } + + public reload() { + this.renderComponent(); + } + + public destroy() { + super.destroy(); + if (this.node) { + ReactDOM.unmountComponentAtNode(this.node); + } + } + + private renderComponent() { + if (!this.node) { + return; + } + + const startTimestamp = datemathToEpochMillis(this.input.timeRange.from); + const endTimestamp = datemathToEpochMillis(this.input.timeRange.to); + + if (!startTimestamp || !endTimestamp) { + return; + } + + ReactDOM.render( + + + +
+ +
+ + + , + this.node + ); + } +} diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts new file mode 100644 index 00000000000000..f4d1b83a075930 --- /dev/null +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable_factory.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreStart } from 'kibana/public'; +import { + EmbeddableFactoryDefinition, + IContainer, +} from '../../../../../../src/plugins/embeddable/public'; +import { + LogStreamEmbeddable, + LOG_STREAM_EMBEDDABLE, + LogStreamEmbeddableInput, +} from './log_stream_embeddable'; + +export class LogStreamEmbeddableFactoryDefinition + implements EmbeddableFactoryDefinition { + public readonly type = LOG_STREAM_EMBEDDABLE; + + constructor(private getCoreServices: () => Promise) {} + + public async isEditable() { + const { application } = await this.getCoreServices(); + return application.capabilities.logs.save as boolean; + } + + public async create(initialInput: LogStreamEmbeddableInput, parent?: IContainer) { + const services = await this.getCoreServices(); + return new LogStreamEmbeddable(services, initialInput, parent); + } + + public getDisplayName() { + return 'Log stream'; + } +} diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index da7176125dae4c..1d9a7a1b1d7779 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -7,7 +7,7 @@ import { useMemo, useEffect } from 'react'; import useSetState from 'react-use/lib/useSetState'; import usePrevious from 'react-use/lib/usePrevious'; -import { esKuery } from '../../../../../../../src/plugins/data/public'; +import { esKuery, esQuery, Query } from '../../../../../../../src/plugins/data/public'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { LogEntryCursor, LogEntry } from '../../../../common/log_entry'; @@ -18,7 +18,7 @@ interface LogStreamProps { sourceId: string; startTimestamp: number; endTimestamp: number; - query?: string; + query?: string | Query; center?: LogEntryCursor; columns?: LogSourceConfigurationProperties['logColumns']; } @@ -84,9 +84,21 @@ export function useLogStream({ }, [prevEndTimestamp, endTimestamp, setState]); const parsedQuery = useMemo(() => { - return query - ? JSON.stringify(esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query))) - : null; + if (!query) { + return null; + } + + let q; + + if (typeof query === 'string') { + q = esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)); + } else if (query.language === 'kuery') { + q = esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query.query as string)); + } else if (query.language === 'lucene') { + q = esQuery.luceneStringToDsl(query.query as string); + } + + return JSON.stringify(q); }, [query]); // Callbacks diff --git a/x-pack/plugins/infra/public/plugin.ts b/x-pack/plugins/infra/public/plugin.ts index 2bbd0067642c06..809046ee1e17b6 100644 --- a/x-pack/plugins/infra/public/plugin.ts +++ b/x-pack/plugins/infra/public/plugin.ts @@ -19,6 +19,8 @@ import { } from './types'; import { getLogsHasDataFetcher, getLogsOverviewDataFetcher } from './utils/logs_overview_fetchers'; import { createMetricsHasData, createMetricsFetchData } from './metrics_overview_fetchers'; +import { LOG_STREAM_EMBEDDABLE } from './components/log_stream/log_stream_embeddable'; +import { LogStreamEmbeddableFactoryDefinition } from './components/log_stream/log_stream_embeddable_factory'; export class Plugin implements InfraClientPluginClass { constructor(_context: PluginInitializerContext) {} @@ -46,6 +48,13 @@ export class Plugin implements InfraClientPluginClass { }); } + const getCoreServices = async () => (await core.getStartServices())[0]; + + pluginsSetup.embeddable.registerEmbeddableFactory( + LOG_STREAM_EMBEDDABLE, + new LogStreamEmbeddableFactoryDefinition(getCoreServices) + ); + core.application.register({ id: 'logs', title: i18n.translate('xpack.infra.logs.pluginTitle', { diff --git a/x-pack/plugins/infra/public/types.ts b/x-pack/plugins/infra/public/types.ts index f1052672978d5e..037cfa4b7eb2d0 100644 --- a/x-pack/plugins/infra/public/types.ts +++ b/x-pack/plugins/infra/public/types.ts @@ -7,6 +7,7 @@ import type { CoreSetup, CoreStart, Plugin as PluginClass } from 'kibana/public'; import type { DataPublicPluginStart } from '../../../../src/plugins/data/public'; import type { HomePublicPluginSetup } from '../../../../src/plugins/home/public'; +import type { EmbeddableSetup } from '../../../../src/plugins/embeddable/public'; import type { UsageCollectionSetup, UsageCollectionStart, @@ -33,6 +34,7 @@ export interface InfraClientSetupDeps { observability: ObservabilityPluginSetup; triggersActionsUi: TriggersAndActionsUIPublicPluginSetup; usageCollection: UsageCollectionSetup; + embeddable: EmbeddableSetup; } export interface InfraClientStartDeps { From 3255f905c0ac8e8f7a639ea95858fec6bac07cf9 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 1 Feb 2021 15:14:17 +0100 Subject: [PATCH 3/4] [APM] Remove value_count aggregations (#89408) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../get_transaction_error_rate.ts | 4 +- .../index.ts | 31 +++++--------- .../lib/helpers/transaction_error_rate.ts | 41 +++++-------------- .../get_transaction_coordinates.ts | 16 +------- .../get_service_map_service_node_info.test.ts | 4 +- .../get_service_map_service_node_info.ts | 9 +--- .../__snapshots__/queries.test.ts.snap | 27 ------------ .../get_service_instance_transaction_stats.ts | 29 +++++-------- ..._timeseries_data_for_transaction_groups.ts | 6 ++- .../get_transaction_groups_for_page.ts | 9 ++-- .../merge_transaction_group_data.ts | 8 +--- .../get_service_transaction_stats.ts | 14 ++----- .../__snapshots__/queries.test.ts.snap | 10 ----- .../lib/transaction_groups/get_error_rate.ts | 10 ++++- .../get_transaction_group_stats.ts | 9 +--- .../get_throughput_charts/index.ts | 6 --- .../get_throughput_charts/transform.ts | 4 +- 17 files changed, 63 insertions(+), 174 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts index fae43ef148cfa9..f6ddb15cbffa92 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts @@ -44,9 +44,7 @@ export async function getTransactionErrorRateChartPreview({ }, }; - const outcomes = getOutcomeAggregation({ - searchAggregatedTransactions: false, - }); + const outcomes = getOutcomeAggregation(); const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts index 64d9ebb192eb34..9ecf201ede1b7b 100644 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { isEmpty, omit } from 'lodash'; +import { isEmpty, omit, merge } from 'lodash'; import { EventOutcome } from '../../../../common/event_outcome'; import { processSignificantTermAggs, @@ -134,8 +134,7 @@ export async function getErrorRateTimeSeries({ extended_bounds: { min: start, max: end }, }, aggs: { - // TODO: add support for metrics - outcomes: getOutcomeAggregation({ searchAggregatedTransactions: false }), + outcomes: getOutcomeAggregation(), }, }; @@ -147,13 +146,12 @@ export async function getErrorRateTimeSeries({ }; return acc; }, - {} as Record< - string, - { + {} as { + [key: string]: { filter: AggregationOptionsByType['filter']; aggs: { timeseries: typeof timeseriesAgg }; - } - > + }; + } ); const params = { @@ -162,32 +160,25 @@ export async function getErrorRateTimeSeries({ body: { size: 0, query: { bool: { filter: backgroundFilters } }, - aggs: { - // overall aggs - timeseries: timeseriesAgg, - - // per term aggs - ...perTermAggs, - }, + aggs: merge({ timeseries: timeseriesAgg }, perTermAggs), }, }; const response = await apmEventClient.search(params); - type Agg = NonNullable; + const { aggregations } = response; - if (!response.aggregations) { + if (!aggregations) { return {}; } return { overall: { timeseries: getTransactionErrorRateTimeSeries( - response.aggregations.timeseries.buckets + aggregations.timeseries.buckets ), }, significantTerms: topSigTerms.map((topSig, index) => { - // @ts-expect-error - const agg = response.aggregations[`term_${index}`] as Agg; + const agg = aggregations[`term_${index}`]!; return { ...topSig, diff --git a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts index 876fc6b822213c..2d041006e0e272 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts @@ -10,40 +10,21 @@ import { AggregationOptionsByType, AggregationResultOf, } from '../../../../../typings/elasticsearch/aggregations'; -import { getTransactionDurationFieldForAggregatedTransactions } from './aggregated_transactions'; -export function getOutcomeAggregation({ - searchAggregatedTransactions, -}: { - searchAggregatedTransactions: boolean; -}) { - return { - terms: { - field: EVENT_OUTCOME, - include: [EventOutcome.failure, EventOutcome.success], - }, - aggs: { - // simply using the doc count to get the number of requests is not possible for transaction metrics (histograms) - // to work around this we get the number of transactions by counting the number of latency values - count: { - value_count: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, - }, - }, - }; -} +export const getOutcomeAggregation = () => ({ + terms: { + field: EVENT_OUTCOME, + include: [EventOutcome.failure, EventOutcome.success], + }, +}); + +type OutcomeAggregation = ReturnType; export function calculateTransactionErrorPercentage( - outcomeResponse: AggregationResultOf< - ReturnType, - {} - > + outcomeResponse: AggregationResultOf ) { const outcomes = Object.fromEntries( - outcomeResponse.buckets.map(({ key, count }) => [key, count.value]) + outcomeResponse.buckets.map(({ key, doc_count: count }) => [key, count]) ); const failedTransactions = outcomes[EventOutcome.failure] ?? 0; @@ -56,7 +37,7 @@ export function getTransactionErrorRateTimeSeries( buckets: AggregationResultOf< { date_histogram: AggregationOptionsByType['date_histogram']; - aggs: { outcomes: ReturnType }; + aggs: { outcomes: OutcomeAggregation }; }, {} >['buckets'] diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_transaction_coordinates.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_transaction_coordinates.ts index 5531944fc71809..fa4bf6144fb6fb 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/get_transaction_coordinates.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/get_transaction_coordinates.ts @@ -11,10 +11,7 @@ import { rangeFilter } from '../../../common/utils/range_filter'; import { Coordinates } from '../../../../observability/typings/common'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -import { - getProcessorEventForAggregatedTransactions, - getTransactionDurationFieldForAggregatedTransactions, -} from '../helpers/aggregated_transactions'; +import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; export async function getTransactionCoordinates({ setup, @@ -49,15 +46,6 @@ export async function getTransactionCoordinates({ fixed_interval: bucketSize, min_doc_count: 0, }, - aggs: { - count: { - value_count: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, - }, - }, }, }, }, @@ -68,7 +56,7 @@ export async function getTransactionCoordinates({ return ( aggregations?.distribution.buckets.map((bucket) => ({ x: bucket.key, - y: bucket.count.value / deltaAsMinutes, + y: bucket.doc_count / deltaAsMinutes, })) || [] ); } diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.test.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.test.ts index f7ca40ef1052c8..173de796d47e43 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.test.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.test.ts @@ -52,8 +52,10 @@ describe('getServiceMapServiceNodeInfo', () => { apmEventClient: { search: () => Promise.resolve({ + hits: { + total: { value: 1 }, + }, aggregations: { - count: { value: 1 }, duration: { value: null }, avgCpuUsage: { value: null }, avgMemoryUsage: { value: null }, diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts index 82d339686f7ec2..4fe9a1a75d43fa 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts @@ -162,19 +162,12 @@ async function getTransactionStats({ ), }, }, - count: { - value_count: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, - }, }, }, }; const response = await apmEventClient.search(params); - const totalRequests = response.aggregations?.count.value ?? 0; + const totalRequests = response.hits.total.value; return { avgTransactionDuration: response.aggregations?.duration.value ?? null, diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 21402e4c8dac08..239b909e1572c6 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -122,13 +122,6 @@ Array [ }, }, "outcomes": Object { - "aggs": Object { - "count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, - }, "terms": Object { "field": "event.outcome", "include": Array [ @@ -137,11 +130,6 @@ Array [ ], }, }, - "real_document_count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, "timeseries": Object { "aggs": Object { "avg_duration": Object { @@ -150,13 +138,6 @@ Array [ }, }, "outcomes": Object { - "aggs": Object { - "count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, - }, "terms": Object { "field": "event.outcome", "include": Array [ @@ -165,11 +146,6 @@ Array [ ], }, }, - "real_document_count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, }, "date_histogram": Object { "extended_bounds": Object { @@ -184,9 +160,6 @@ Array [ }, "terms": Object { "field": "transaction.type", - "order": Object { - "real_document_count": "desc", - }, }, }, }, diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instance_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instance_transaction_stats.ts index 5880b5cbc9546e..c5e5269c4409e8 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instance_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instance_transaction_stats.ts @@ -30,18 +30,17 @@ export async function getServiceInstanceTransactionStats({ }: ServiceInstanceParams) { const { apmEventClient, start, end, esFilter } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets }); + const { intervalString, bucketSize } = getBucketSize({ + start, + end, + numBuckets, + }); const field = getTransactionDurationFieldForAggregatedTransactions( searchAggregatedTransactions ); const subAggs = { - count: { - value_count: { - field, - }, - }, avg_transaction_duration: { avg: { field, @@ -53,13 +52,6 @@ export async function getServiceInstanceTransactionStats({ [EVENT_OUTCOME]: EventOutcome.failure, }, }, - aggs: { - count: { - value_count: { - field, - }, - }, - }, }, }; @@ -113,12 +105,13 @@ export async function getServiceInstanceTransactionStats({ }); const deltaAsMinutes = (end - start) / 60 / 1000; + const bucketSizeInMinutes = bucketSize / 60; return ( response.aggregations?.[SERVICE_NODE_NAME].buckets.map( (serviceNodeBucket) => { const { - count, + doc_count: count, avg_transaction_duration: avgTransactionDuration, key, failures, @@ -128,17 +121,17 @@ export async function getServiceInstanceTransactionStats({ return { serviceNodeName: String(key), errorRate: { - value: failures.count.value / count.value, + value: failures.doc_count / count, timeseries: timeseries.buckets.map((dateBucket) => ({ x: dateBucket.key, - y: dateBucket.failures.count.value / dateBucket.count.value, + y: dateBucket.failures.doc_count / dateBucket.doc_count, })), }, throughput: { - value: count.value / deltaAsMinutes, + value: count / deltaAsMinutes, timeseries: timeseries.buckets.map((dateBucket) => ({ x: dateBucket.key, - y: dateBucket.count.value / deltaAsMinutes, + y: dateBucket.doc_count / bucketSizeInMinutes, })), }, latency: { diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_timeseries_data_for_transaction_groups.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_timeseries_data_for_transaction_groups.ts index 937155bc316021..745535f2616734 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_timeseries_data_for_transaction_groups.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_timeseries_data_for_transaction_groups.ts @@ -17,6 +17,7 @@ import { import { ESFilter } from '../../../../../../typings/elasticsearch'; import { + getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, getTransactionDurationFieldForAggregatedTransactions, } from '../../helpers/aggregated_transactions'; @@ -76,6 +77,9 @@ export async function getTimeseriesDataForTransactionGroups({ { term: { [SERVICE_NAME]: serviceName } }, { term: { [TRANSACTION_TYPE]: transactionType } }, { range: rangeFilter(start, end) }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), ...esFilter, ], }, @@ -99,10 +103,8 @@ export async function getTimeseriesDataForTransactionGroups({ }, aggs: { ...getLatencyAggregation(latencyAggregationType, field), - transaction_count: { value_count: { field } }, [EVENT_OUTCOME]: { filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - aggs: { transaction_count: { value_count: { field } } }, }, }, }, diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_transaction_groups_for_page.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_transaction_groups_for_page.ts index ccccf946512dd3..400c896e380b4b 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_transaction_groups_for_page.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/get_transaction_groups_for_page.ts @@ -99,10 +99,8 @@ export async function getTransactionGroupsForPage({ }, aggs: { ...getLatencyAggregation(latencyAggregationType, field), - transaction_count: { value_count: { field } }, [EVENT_OUTCOME]: { filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - aggs: { transaction_count: { value_count: { field } } }, }, }, }, @@ -113,9 +111,8 @@ export async function getTransactionGroupsForPage({ const transactionGroups = response.aggregations?.transaction_groups.buckets.map((bucket) => { const errorRate = - bucket.transaction_count.value > 0 - ? (bucket[EVENT_OUTCOME].transaction_count.value ?? 0) / - bucket.transaction_count.value + bucket.doc_count > 0 + ? bucket[EVENT_OUTCOME].doc_count / bucket.doc_count : null; return { @@ -124,7 +121,7 @@ export async function getTransactionGroupsForPage({ latencyAggregationType, aggregation: bucket.latency, }), - throughput: bucket.transaction_count.value / deltaAsMinutes, + throughput: bucket.doc_count / deltaAsMinutes, errorRate, }; }) ?? []; diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/merge_transaction_group_data.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/merge_transaction_group_data.ts index a8794e3c09a40f..b0b1cb09dd7845 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/merge_transaction_group_data.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups/merge_transaction_group_data.ts @@ -52,18 +52,14 @@ export function mergeTransactionGroupData({ ...acc.throughput, timeseries: acc.throughput.timeseries.concat({ x: point.key, - y: point.transaction_count.value / deltaAsMinutes, + y: point.doc_count / deltaAsMinutes, }), }, errorRate: { ...acc.errorRate, timeseries: acc.errorRate.timeseries.concat({ x: point.key, - y: - point.transaction_count.value > 0 - ? (point[EVENT_OUTCOME].transaction_count.value ?? 0) / - point.transaction_count.value - : null, + y: point[EVENT_OUTCOME].doc_count / point.doc_count, }), }, }; diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index 0ee7080dc0834c..efbc30169d178a 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -51,16 +51,9 @@ export async function getServiceTransactionStats({ }: AggregationParams) { const { apmEventClient, start, end, esFilter } = setup; - const outcomes = getOutcomeAggregation({ searchAggregatedTransactions }); + const outcomes = getOutcomeAggregation(); const metrics = { - real_document_count: { - value_count: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, - }, avg_duration: { avg: { field: getTransactionDurationFieldForAggregatedTransactions( @@ -102,7 +95,6 @@ export async function getServiceTransactionStats({ transactionType: { terms: { field: TRANSACTION_TYPE, - order: { real_document_count: 'desc' }, }, aggs: { ...metrics, @@ -180,14 +172,14 @@ export async function getServiceTransactionStats({ }, transactionsPerMinute: { value: calculateAvgDuration({ - value: topTransactionTypeBucket.real_document_count.value, + value: topTransactionTypeBucket.doc_count, deltaAsMinutes, }), timeseries: topTransactionTypeBucket.timeseries.buckets.map( (dateBucket) => ({ x: dateBucket.key, y: calculateAvgDuration({ - value: dateBucket.real_document_count.value, + value: dateBucket.doc_count, deltaAsMinutes, }), }) diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/queries.test.ts.snap index c678e7db711b65..89069d74bacf8b 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/transaction_groups/__snapshots__/queries.test.ts.snap @@ -12,11 +12,6 @@ Array [ "aggs": Object { "transaction_groups": Object { "aggs": Object { - "count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, "transaction_type": Object { "top_hits": Object { "_source": Array [ @@ -226,11 +221,6 @@ Array [ "aggs": Object { "transaction_groups": Object { "aggs": Object { - "count": Object { - "value_count": Object { - "field": "transaction.duration.us", - }, - }, "transaction_type": Object { "top_hits": Object { "_source": Array [ diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts index dfd11203b87f13..a2388dddc7fd4e 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts @@ -14,7 +14,10 @@ import { } from '../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../common/event_outcome'; import { rangeFilter } from '../../../common/utils/range_filter'; -import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; +import { + getDocumentTypeFilterForAggregatedTransactions, + getProcessorEventForAggregatedTransactions, +} from '../helpers/aggregated_transactions'; import { getBucketSize } from '../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { @@ -55,12 +58,15 @@ export async function getErrorRate({ { terms: { [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success] }, }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), ...transactionNamefilter, ...transactionTypefilter, ...esFilter, ]; - const outcomes = getOutcomeAggregation({ searchAggregatedTransactions }); + const outcomes = getOutcomeAggregation(); const params = { apm: { diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts index cfd3540446172b..dba58cecad79be 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts @@ -66,13 +66,6 @@ export async function getCounts({ searchAggregatedTransactions, }: MetricParams) { const params = mergeRequestWithAggs(request, { - count: { - value_count: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, - }, transaction_type: { top_hits: { size: 1, @@ -92,7 +85,7 @@ export async function getCounts({ return { key: bucket.key as BucketKey, - count: bucket.count.value, + count: bucket.doc_count, transactionType: source.transaction.type, }; }); diff --git a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts index be374ccfe3400c..8dfb0a9f65878a 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts @@ -15,7 +15,6 @@ import { rangeFilter } from '../../../../common/utils/range_filter'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, - getTransactionDurationFieldForAggregatedTransactions, } from '../../../lib/helpers/aggregated_transactions'; import { getBucketSize } from '../../../lib/helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request'; @@ -56,10 +55,6 @@ async function searchThroughput({ filter.push({ term: { [TRANSACTION_NAME]: transactionName } }); } - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); - const params = { apm: { events: [ @@ -82,7 +77,6 @@ async function searchThroughput({ min_doc_count: 0, extended_bounds: { min: start, max: end }, }, - aggs: { count: { value_count: { field } } }, }, }, }, diff --git a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/transform.ts b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/transform.ts index a12e36c0e9de45..7e43a0d76f70ae 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/transform.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/transform.ts @@ -25,7 +25,7 @@ export function getThroughputBuckets({ return { x: bucket.key, // divide by minutes - y: bucket.count.value / (bucketSize / 60), + y: bucket.doc_count / (bucketSize / 60), }; }); @@ -34,7 +34,7 @@ export function getThroughputBuckets({ resultKey === '' ? NOT_AVAILABLE_LABEL : (resultKey as string); const docCountTotal = timeseries.buckets - .map((bucket) => bucket.count.value) + .map((bucket) => bucket.doc_count) .reduce((a, b) => a + b, 0); // calculate average throughput From c66124e1703c817a728d6417e5a985be5c10d68e Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Mon, 1 Feb 2021 15:25:16 +0100 Subject: [PATCH 4/4] [Lens] Make Lens intervals default value adapt to histogram:maxBars Advanced Setting changes (#89305) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../indexpattern_datasource/indexpattern.tsx | 2 +- .../definitions/date_histogram.test.tsx | 10 ++-- .../definitions/filters/filters.test.tsx | 7 ++- .../operations/definitions/index.ts | 6 ++- .../definitions/last_value.test.tsx | 7 ++- .../definitions/percentile.test.tsx | 7 ++- .../definitions/ranges/range_editor.tsx | 11 +---- .../definitions/ranges/ranges.test.tsx | 49 +++++++++---------- .../operations/definitions/ranges/ranges.tsx | 8 +-- .../definitions/terms/terms.test.tsx | 13 +++-- .../indexpattern_datasource/to_expression.ts | 15 ++++-- 11 files changed, 75 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index c309212eed164f..7f77a7ce199b65 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -167,7 +167,7 @@ export function getIndexPatternDatasource({ }); }, - toExpression, + toExpression: (state, layerId) => toExpression(state, layerId, uiSettings), renderDataPanel( domElement: Element, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index abd033c0db4cfc..22275533b95542 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -83,9 +83,11 @@ const indexPattern2: IndexPattern = { ]), }; +const uiSettingsMock = {} as IUiSettingsClient; + const defaultOptions = { storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1y', @@ -200,7 +202,8 @@ describe('date_histogram', () => { layer.columns.col1 as DateHistogramIndexPatternColumn, 'col1', indexPattern1, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ @@ -252,7 +255,8 @@ describe('date_histogram', () => { }, ]), }, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index 86767fbc8b4691..3657013fa0bfa1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -16,9 +16,11 @@ import type { IndexPatternLayer } from '../../../types'; import { createMockedIndexPattern } from '../../../mocks'; import { FilterPopover } from './filter_popover'; +const uiSettingsMock = {} as IUiSettingsClient; + const defaultProps = { storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1d', toDate: 'now' }, data: dataPluginMock.createStartContract(), @@ -84,7 +86,8 @@ describe('filters', () => { layer.columns.col1 as FiltersIndexPatternColumn, 'col1', createMockedIndexPattern(), - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 1cdaff53c5458e..0c0aa34bb40b39 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -239,7 +239,8 @@ interface FieldlessOperationDefinition { column: C, columnId: string, indexPattern: IndexPattern, - layer: IndexPatternLayer + layer: IndexPatternLayer, + uiSettings: IUiSettingsClient ) => ExpressionAstFunction; } @@ -283,7 +284,8 @@ interface FieldBasedOperationDefinition { column: C, columnId: string, indexPattern: IndexPattern, - layer: IndexPatternLayer + layer: IndexPatternLayer, + uiSettings: IUiSettingsClient ) => ExpressionAstFunction; /** * Optional function to return the suffix used for ES bucket paths and esaggs column id. diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index 96b12a714e613d..8d5ab50770111f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -15,9 +15,11 @@ import { LastValueIndexPatternColumn } from './last_value'; import { lastValueOperation } from './index'; import type { IndexPattern, IndexPatternLayer } from '../../types'; +const uiSettingsMock = {} as IUiSettingsClient; + const defaultProps = { storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1d', toDate: 'now' }, data: dataPluginMock.createStartContract(), @@ -70,7 +72,8 @@ describe('last_value', () => { { ...lastValueColumn, params: { ...lastValueColumn.params } }, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index c22eec62ea1ab9..a340e17121e901 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -17,9 +17,11 @@ import { EuiFieldNumber } from '@elastic/eui'; import { act } from 'react-dom/test-utils'; import { EuiFormRow } from '@elastic/eui'; +const uiSettingsMock = {} as IUiSettingsClient; + const defaultProps = { storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1d', toDate: 'now' }, data: dataPluginMock.createStartContract(), @@ -72,7 +74,8 @@ describe('percentile', () => { percentileColumn, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/range_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/range_editor.tsx index ad5c146ff66247..d9698252177b07 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/range_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/range_editor.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect, useState } from 'react'; +import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { @@ -190,15 +190,6 @@ export const RangeEditor = ({ }) => { const [isAdvancedEditor, toggleAdvancedEditor] = useState(params.type === MODES.Range); - // if the maxBars in the params is set to auto refresh it with the default value only on bootstrap - useEffect(() => { - if (!isAdvancedEditor) { - if (params.maxBars !== maxBars) { - setParam('maxBars', maxBars); - } - } - }, [maxBars, params.maxBars, setParam, isAdvancedEditor]); - if (isAdvancedEditor) { return ( & React.MouseEvent; +// need this for MAX_HISTOGRAM value +const uiSettingsMock = ({ + get: jest.fn().mockReturnValue(100), +} as unknown) as IUiSettingsClient; + const sourceField = 'MyField'; const defaultOptions = { storage: {} as IStorageWrapper, - // need this for MAX_HISTOGRAM value - uiSettings: ({ - get: () => 100, - } as unknown) as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1y', @@ -143,7 +145,8 @@ describe('ranges', () => { layer.columns.col1 as RangeIndexPatternColumn, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toMatchInlineSnapshot(` Object { @@ -166,6 +169,9 @@ describe('ranges', () => { "interval": Array [ "auto", ], + "maxBars": Array [ + 49.5, + ], "min_doc_count": Array [ false, ], @@ -186,7 +192,8 @@ describe('ranges', () => { layer.columns.col1 as RangeIndexPatternColumn, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( @@ -206,7 +213,8 @@ describe('ranges', () => { layer.columns.col1 as RangeIndexPatternColumn, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( @@ -226,7 +234,8 @@ describe('ranges', () => { layer.columns.col1 as RangeIndexPatternColumn, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect((esAggsFn as { arguments: unknown }).arguments).toEqual( @@ -275,7 +284,7 @@ describe('ranges', () => { it('should start update the state with the default maxBars value', () => { const updateLayerSpy = jest.fn(); - mount( + const instance = mount( { /> ); - expect(updateLayerSpy).toHaveBeenCalledWith({ - ...layer, - columns: { - ...layer.columns, - col1: { - ...layer.columns.col1, - params: { - ...layer.columns.col1.params, - maxBars: GRANULARITY_DEFAULT_VALUE, - }, - }, - }, - }); + expect(instance.find(EuiRange).prop('value')).toEqual(String(GRANULARITY_DEFAULT_VALUE)); }); it('should update state when changing Max bars number', () => { @@ -313,8 +310,6 @@ describe('ranges', () => { /> ); - // There's a useEffect in the component that updates the value on bootstrap - // because there's a debouncer, wait a bit before calling onChange act(() => { jest.advanceTimersByTime(TYPING_DEBOUNCE_TIME * 4); @@ -358,8 +353,6 @@ describe('ranges', () => { /> ); - // There's a useEffect in the component that updates the value on bootstrap - // because there's a debouncer, wait a bit before calling onChange act(() => { jest.advanceTimersByTime(TYPING_DEBOUNCE_TIME * 4); // minus button @@ -368,6 +361,7 @@ describe('ranges', () => { .find('button') .prop('onClick')!({} as ReactMouseEvent); jest.advanceTimersByTime(TYPING_DEBOUNCE_TIME * 4); + instance.update(); }); expect(updateLayerSpy).toHaveBeenCalledWith({ @@ -391,6 +385,7 @@ describe('ranges', () => { .find('button') .prop('onClick')!({} as ReactMouseEvent); jest.advanceTimersByTime(TYPING_DEBOUNCE_TIME * 4); + instance.update(); }); expect(updateLayerSpy).toHaveBeenCalledWith({ @@ -788,7 +783,7 @@ describe('ranges', () => { instance.find(EuiLink).first().prop('onClick')!({} as ReactMouseEvent); }); - expect(updateLayerSpy.mock.calls[1][0].columns.col1.params.format).toEqual({ + expect(updateLayerSpy.mock.calls[0][0].columns.col1.params.format).toEqual({ id: 'custom', params: { decimals: 3 }, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx index aa5cc8255a5841..d8622a5aedf7dd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx @@ -132,7 +132,7 @@ export const rangeOperation: OperationDefinition { + toEsAggsFn: (column, columnId, indexPattern, layer, uiSettings) => { const { sourceField, params } = column; if (params.type === MODES.Range) { return buildExpressionFunction('aggRange', { @@ -158,13 +158,15 @@ export const rangeOperation: OperationDefinition('aggHistogram', { id: columnId, enabled: true, schema: 'segment', field: sourceField, - // fallback to 0 in case of empty string - maxBars: params.maxBars === AUTO_BARS ? undefined : params.maxBars, + maxBars: params.maxBars === AUTO_BARS ? maxBarsDefaultValue : params.maxBars, interval: 'auto', has_extended_bounds: false, min_doc_count: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index d60992bda2e2a7..3e25e127b37f4a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -17,9 +17,11 @@ import type { TermsIndexPatternColumn } from '.'; import { termsOperation } from '../index'; import { IndexPattern, IndexPatternLayer } from '../../../types'; +const uiSettingsMock = {} as IUiSettingsClient; + const defaultProps = { storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, + uiSettings: uiSettingsMock, savedObjectsClient: {} as SavedObjectsClientContract, dateRange: { fromDate: 'now-1d', toDate: 'now' }, data: dataPluginMock.createStartContract(), @@ -66,7 +68,8 @@ describe('terms', () => { { ...termsColumn, params: { ...termsColumn.params, otherBucket: true } }, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ @@ -89,7 +92,8 @@ describe('terms', () => { }, 'col1', {} as IndexPattern, - layer + layer, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ @@ -129,7 +133,8 @@ describe('terms', () => { }, }, }, - } + }, + uiSettingsMock ); expect(esAggsFn).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 38f51f24aae7dd..c9ee77a9f5e157 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import type { IUiSettingsClient } from 'kibana/public'; import { EsaggsExpressionFunctionDefinition, IndexPatternLoadExpressionFunctionDefinition, @@ -24,7 +25,8 @@ import { getEsAggsSuffix } from './operations/definitions/helpers'; function getExpressionForLayer( layer: IndexPatternLayer, - indexPattern: IndexPattern + indexPattern: IndexPattern, + uiSettings: IUiSettingsClient ): ExpressionAstExpression | null { const { columns, columnOrder } = layer; if (columnOrder.length === 0) { @@ -44,7 +46,7 @@ function getExpressionForLayer( aggs.push( buildExpression({ type: 'expression', - chain: [def.toEsAggsFn(col, colId, indexPattern, layer)], + chain: [def.toEsAggsFn(col, colId, indexPattern, layer, uiSettings)], }) ); } @@ -184,11 +186,16 @@ function getExpressionForLayer( return null; } -export function toExpression(state: IndexPatternPrivateState, layerId: string) { +export function toExpression( + state: IndexPatternPrivateState, + layerId: string, + uiSettings: IUiSettingsClient +) { if (state.layers[layerId]) { return getExpressionForLayer( state.layers[layerId], - state.indexPatterns[state.layers[layerId].indexPatternId] + state.indexPatterns[state.layers[layerId].indexPatternId], + uiSettings ); }