From 5941b55e81ff8f74f5bb5a48160f75365e7c1c86 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 21 Jan 2021 18:09:50 +0100 Subject: [PATCH 001/185] Lens formula --- .../functions/common/math.ts | 8 +- .../lens/public/id_generator/id_generator.ts | 2 +- .../dimension_panel/dimension_editor.tsx | 7 +- .../definitions/calculations/derivative.tsx | 2 +- .../operations/definitions/formula.tsx | 265 ++++++++++++++++++ .../operations/definitions/index.ts | 38 ++- .../operations/definitions/math.tsx | 95 +++++++ .../operations/layer_helpers.ts | 48 +++- .../operations/operations.ts | 12 + .../indexpattern_datasource/to_expression.ts | 10 +- 10 files changed, 458 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts index e36644530eae86..dc05833b6a60fe 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts @@ -22,7 +22,7 @@ export function math(): ExpressionFunctionDefinition<'math', Input, Arguments, n return { name: 'math', - type: 'number', + type: undefined, inputTypes: ['number', 'datatable'], help, args: { @@ -54,9 +54,9 @@ export function math(): ExpressionFunctionDefinition<'math', Input, Arguments, n } throw errors.tooManyResults(); } - if (isNaN(result)) { - throw errors.executionFailed(); - } + // if (isNaN(result)) { + // throw errors.executionFailed(); + // } return result; } catch (e) { if (isDatatable(input) && input.rows.length === 0) { diff --git a/x-pack/plugins/lens/public/id_generator/id_generator.ts b/x-pack/plugins/lens/public/id_generator/id_generator.ts index 82579769925eb1..61fcefd8b5a1bd 100644 --- a/x-pack/plugins/lens/public/id_generator/id_generator.ts +++ b/x-pack/plugins/lens/public/id_generator/id_generator.ts @@ -7,5 +7,5 @@ import uuid from 'uuid/v4'; export function generateId() { - return uuid(); + return 'c' + uuid().replaceAll(/-/g, ''); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index dc7b291b7120f3..508674e917b47e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -231,6 +231,7 @@ export function DimensionEditor(props: DimensionEditorProps) { onClick() { if ( operationDefinitionMap[operationType].input === 'none' || + operationDefinitionMap[operationType].input === 'managedReference' || operationDefinitionMap[operationType].input === 'fullReference' ) { // Clear invalid state because we are reseting to a valid column @@ -301,7 +302,8 @@ export function DimensionEditor(props: DimensionEditorProps) { // Need to workout early on the error to decide whether to show this or an help text const fieldErrorMessage = - (selectedOperationDefinition?.input !== 'fullReference' || + ((selectedOperationDefinition?.input !== 'fullReference' && + selectedOperationDefinition?.input !== 'managedReference') || (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field')) && getErrorMessage( selectedColumn, @@ -435,6 +437,7 @@ export function DimensionEditor(props: DimensionEditorProps) { currentColumn={state.layers[layerId].columns[columnId]} dateRange={dateRange} indexPattern={currentIndexPattern} + operationDefinitionMap={operationDefinitionMap} {...services} /> @@ -517,7 +520,7 @@ export function DimensionEditor(props: DimensionEditorProps) { function getErrorMessage( selectedColumn: IndexPatternColumn | undefined, incompleteOperation: boolean, - input: 'none' | 'field' | 'fullReference' | undefined, + input: 'none' | 'field' | 'fullReference' | 'managedReference' | undefined, fieldInvalid: boolean ) { if (selectedColumn && incompleteOperation) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx index 358046ad5bfb9c..02c4e12951027c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx @@ -48,7 +48,7 @@ export const derivativeOperation: OperationDefinition< selectionStyle: 'full', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx new file mode 100644 index 00000000000000..7fcb01427e6756 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -0,0 +1,265 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { useState } from 'react'; +import { parse } from 'tinymath'; +import { EuiButton, EuiTextArea } from '@elastic/eui'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; +import { ReferenceBasedIndexPatternColumn } from './column_types'; +import { IndexPattern, IndexPatternLayer } from '../../types'; +import { getColumnOrder } from '../layer_helpers'; +import { mathOperation } from './math'; + +export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { + operationType: 'formula'; + params: { + ast?: unknown; + // last value on numeric fields can be formatted + format?: { + id: string; + params?: { + decimals: number; + }; + }; + }; +} + +export const formulaOperation: OperationDefinition< + FormulaIndexPatternColumn, + 'managedReference' +> = { + type: 'formula', + displayName: 'Formula', + getDefaultLabel: (column, indexPattern) => 'Formula', + input: 'managedReference', + getDisabledStatus(indexPattern: IndexPattern) { + return undefined; + }, + getErrorMessage(layer, columnId, indexPattern) { + return undefined; + }, + getPossibleOperation() { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + toExpression: (layer, columnId) => { + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + name: [columnId], + exp: [ + { + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [ + `${(layer.columns[columnId] as FormulaIndexPatternColumn).references[0]}`, + ], + }, + }, + ], + }, + ], + }, + }, + ]; + }, + buildColumn() { + return { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }; + }, + isTransferable: (column, newIndexPattern) => { + // TODO has to check all children + return true; + }, + + paramEditor: function ParamEditor({ + layer, + updateLayer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap, + }) { + const [text, setText] = useState(currentColumn.params.ast); + return ( + <> + { + setText(e.target.value); + }} + /> + { + const ast = parse(text); + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns( + columnId, + operationDefinitionMap, + ast, + layer, + indexPattern + ); + + const columns = { + ...layer.columns, + }; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach((extractedColumn, index) => { + columns[`${columnId}X${index}`] = extractedColumn; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + ast: text, + }, + references: [`${columnId}X${extracted.length - 1}`], + }; + + updateLayer({ + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }); + + // TODO + // turn ast into referenced columns + // set state + }} + > + Submit + + + ); + }, +}; + +function extractColumns( + idPrefix: string, + operations: Record, + ast: any, + layer: IndexPatternLayer, + indexPattern: IndexPattern +) { + const columns: IndexPatternColumn[] = []; + // let currentTree: any = cloneDeep(ast); + function parseNode(node: any) { + if (typeof node === 'number' || typeof node === 'string') { + // leaf node + return node; + } + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map((childNode: any) => parseNode(childNode)); + return { + ...node, + args: consumedArgs, + }; + } + // operation node + if (nodeOperation.input === 'field') { + const fieldName = node.args[0]; + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn({ + layer, + indexPattern, + field: indexPattern.getFieldByName(fieldName)!, + }); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push(newCol); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const consumedParam = parseNode(node.args[0]); + const variables = findVariables(consumedParam); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables; + mathColumn.params.tinymathAst = consumedParam; + columns.push(mathColumn); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn({ + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push(newCol); + // replace by new column id + return `${idPrefix}X${columns.length - 1}`; + } + + throw new Error('unexpected node'); + } + const root = parseNode(ast); + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables; + mathColumn.params.tinymathAst = root; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push(mathColumn); + return columns; +} + +// traverse a tree and find all string leaves +function findVariables(node: any): string[] { + if (typeof node === 'string') { + // leaf node + return [node]; + } + if (typeof node === 'number') { + return []; + } + return node.args.flatMap(findVariables); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 7dbc7d3b986a57..13807dd1816ac5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -34,6 +34,8 @@ import { MovingAverageIndexPatternColumn, } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; +import { mathOperation, MathIndexPatternColumn } from './math'; +import { formulaOperation, FormulaIndexPatternColumn } from './formula'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -65,7 +67,9 @@ export type IndexPatternColumn = | CumulativeSumIndexPatternColumn | CounterRateIndexPatternColumn | DerivativeIndexPatternColumn - | MovingAverageIndexPatternColumn; + | MovingAverageIndexPatternColumn + | MathIndexPatternColumn + | FormulaIndexPatternColumn; export type FieldBasedIndexPatternColumn = Extract; @@ -92,6 +96,8 @@ const internalOperationDefinitions = [ counterRateOperation, derivativeOperation, movingAverageOperation, + mathOperation, + formulaOperation, ]; export { termsOperation } from './terms'; @@ -124,6 +130,7 @@ export interface ParamEditorProps { http: HttpSetup; dateRange: DateRange; data: DataPublicPluginStart; + operationDefinitionMap: Record; } export interface HelpProps { @@ -314,6 +321,7 @@ export interface RequiredReference { // operation types. The main use case is Cumulative Sum, where we need to only take the // sum of Count or sum of Sum. specificOperations?: OperationType[]; + multi?: boolean; } // Full reference uses one or more reference operations which are visible to the user @@ -330,8 +338,9 @@ interface FullReferenceOperationDefinition { * The type of UI that is shown in the editor for this function: * - full: List of sub-functions and fields * - field: List of fields, selects first operation per field + * - hidden: Do not allow to use operation directly */ - selectionStyle: 'full' | 'field'; + selectionStyle: 'full' | 'field' | 'hidden'; /** * Builds the column object for the given parameters. Should include default p @@ -357,10 +366,32 @@ interface FullReferenceOperationDefinition { ) => ExpressionAstFunction[]; } +interface ManagedReferenceOperationDefinition { + input: 'managedReference'; + /** + * Builds the column object for the given parameters. Should include default p + */ + buildColumn: (arg: BaseBuildColumnArgs) => ReferenceBasedIndexPatternColumn & C; + /** + * Returns the meta data of the operation if applied. Undefined + * if the operation can't be added with these fields. + */ + getPossibleOperation: () => OperationMetadata | undefined; + /** + * A chain of expression functions which will transform the table + */ + toExpression: ( + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern + ) => ExpressionAstFunction[]; +} + interface OperationDefinitionMap { field: FieldBasedOperationDefinition; none: FieldlessOperationDefinition; fullReference: FullReferenceOperationDefinition; + managedReference: ManagedReferenceOperationDefinition; } /** @@ -386,7 +417,8 @@ export type OperationType = typeof internalOperationDefinitions[number]['type']; export type GenericOperationDefinition = | OperationDefinition | OperationDefinition - | OperationDefinition; + | OperationDefinition + | OperationDefinition; /** * List of all available operation definitions diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx new file mode 100644 index 00000000000000..5354fca20fde80 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { useState } from 'react'; +import { parse } from 'tinymath'; +import { EuiButton, EuiTextArea } from '@elastic/eui'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; +import { ReferenceBasedIndexPatternColumn } from './column_types'; +import { IndexPattern, IndexPatternLayer } from '../../types'; + +export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { + operationType: 'math'; + params: { + tinymathAst: any; + // last value on numeric fields can be formatted + format?: { + id: string; + params?: { + decimals: number; + }; + }; + }; +} + +export const mathOperation: OperationDefinition = { + type: 'math', + displayName: 'Math', + getDefaultLabel: (column, indexPattern) => 'Math', + input: 'managedReference', + getDisabledStatus(indexPattern: IndexPattern) { + return undefined; + }, + getErrorMessage(layer, columnId, indexPattern) { + return undefined; + }, + getPossibleOperation() { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + toExpression: (layer, columnId) => { + const column = layer.columns[columnId] as MathIndexPatternColumn; + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + name: [columnId], + exp: [ + { + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [astToString(column.params.tinymathAst)], + }, + }, + ], + }, + ], + }, + }, + ]; + }, + buildColumn() { + return { + label: 'Math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: {}, + }, + references: [], + }; + }, + isTransferable: (column, newIndexPattern) => { + // TODO has to check all children + return true; + }, +}; + +function astToString(ast: any) { + if (typeof ast === 'number' || typeof ast === 'string') { + return ast; + } + return `${ast.name}(${ast.args.map(astToString).join(',')})`; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index d8244f3902a6e6..d43b93aec9e9e3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -54,12 +54,12 @@ export function insertNewColumn({ const baseOptions = { indexPattern, previousColumn: layer.columns[columnId] }; - if (operationDefinition.input === 'none') { + if (operationDefinition.input === 'none' || operationDefinition.input === 'managedReference') { if (field) { throw new Error(`Can't create operation ${op} with the provided field ${field.name}`); } const possibleOperation = operationDefinition.getPossibleOperation(); - const isBucketed = Boolean(possibleOperation.isBucketed); + const isBucketed = Boolean(possibleOperation?.isBucketed); const addOperationFn = isBucketed ? addBucket : addMetric; return updateDefaultLabels( addOperationFn(layer, operationDefinition.buildColumn({ ...baseOptions, layer }), columnId), @@ -300,7 +300,7 @@ export function replaceColumn({ }); } - if (operationDefinition.input === 'none') { + if (operationDefinition.input === 'none' || operationDefinition.input === 'managedReference') { let newColumn = operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer }); newColumn = copyCustomLabel(newColumn, previousColumn); @@ -798,24 +798,16 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] { const [direct, referenceBased] = _.partition( entries, - ([, col]) => operationDefinitionMap[col.operationType].input !== 'fullReference' + ([, col]) => + operationDefinitionMap[col.operationType].input !== 'fullReference' && + operationDefinitionMap[col.operationType].input !== 'managedReference' ); - // If a reference has another reference as input, put it last in sort order - referenceBased.sort(([idA, a], [idB, b]) => { - if ('references' in a && a.references.includes(idB)) { - return 1; - } - if ('references' in b && b.references.includes(idA)) { - return -1; - } - return 0; - }); const [aggregations, metrics] = _.partition(direct, ([, col]) => col.isBucketed); return aggregations .map(([id]) => id) .concat(metrics.map(([id]) => id)) - .concat(referenceBased.map(([id]) => id)); + .concat(topologicalSort(referenceBased)); } // Splits existing columnOrder into the three categories @@ -965,3 +957,29 @@ export function isColumnValidAsReference({ validation.validateMetadata(column) ); } + +function topologicalSort(columns: Array<[string, IndexPatternColumn]>) { + const allNodes: Record = {}; + columns.forEach(([id, col]) => { + allNodes[id] = 'references' in col ? col.references : []; + }); + // remove real metric references + columns.forEach(([id]) => { + allNodes[id] = allNodes[id].filter((refId) => !!allNodes[refId]); + }); + const ordered: string[] = []; + + while (ordered.length < columns.length) { + Object.keys(allNodes).forEach((id) => { + if (allNodes[id].length === 0) { + ordered.push(id); + delete allNodes[id]; + Object.keys(allNodes).forEach((k) => { + allNodes[k] = allNodes[k].filter((i) => i !== id); + }); + } + }); + } + + return ordered; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts index c111983ea2cd6c..29c40782d4faee 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts @@ -91,6 +91,10 @@ type OperationFieldTuple = | { type: 'fullReference'; operationType: OperationType; + } + | { + type: 'managedReference'; + operationType: OperationType; }; /** @@ -174,6 +178,14 @@ export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { validOperation ); } + } else if (operationDefinition.input === 'managedReference') { + const validOperation = operationDefinition.getPossibleOperation(); + if (validOperation) { + addToMap( + { type: 'managedReference', operationType: operationDefinition.type }, + validOperation + ); + } } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 38f51f24aae7dd..7a4d28a8b61a53 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -38,7 +38,7 @@ function getExpressionForLayer( const expressions: ExpressionAstFunction[] = []; columnEntries.forEach(([colId, col]) => { const def = operationDefinitionMap[col.operationType]; - if (def.input === 'fullReference') { + if (def.input === 'fullReference' || def.input === 'managedReference') { expressions.push(...def.toExpression(layer, colId, indexPattern)); } else { aggs.push( @@ -50,8 +50,12 @@ function getExpressionForLayer( } }); - const idMap = columnEntries.reduce((currentIdMap, [colId, column], index) => { - const esAggsId = `col-${columnEntries.length === 1 ? 0 : index}-${colId}`; + const aggColumnEntries = columnEntries.filter(([, col]) => { + const def = operationDefinitionMap[col.operationType]; + return !(def.input === 'fullReference' || def.input === 'managedReference'); + }); + const idMap = aggColumnEntries.reduce((currentIdMap, [colId, column], index) => { + const esAggsId = `col-${aggColumnEntries.length === 1 ? 0 : index}-${colId}`; const suffix = getEsAggsSuffix(column); return { ...currentIdMap, From 71b08127bbe26af8f0c5324aa6550eca0bad5d2e Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 27 Jan 2021 19:00:53 +0100 Subject: [PATCH 002/185] :alembic: Adding basic error handling + transferable --- .../operations/definitions/formula.tsx | 44 ++++++++++++++++--- .../operations/definitions/helpers.tsx | 35 +++++++++++---- .../operations/definitions/index.ts | 3 +- .../public/indexpattern_datasource/utils.ts | 14 +++++- 4 files changed, 79 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 7fcb01427e6756..9c194680d08c73 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -8,7 +8,7 @@ import { parse } from 'tinymath'; import { EuiButton, EuiTextArea } from '@elastic/eui'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternLayer } from '../../types'; +import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; import { getColumnOrder } from '../layer_helpers'; import { mathOperation } from './math'; @@ -37,8 +37,28 @@ export const formulaOperation: OperationDefinition< getDisabledStatus(indexPattern: IndexPattern) { return undefined; }, - getErrorMessage(layer, columnId, indexPattern) { - return undefined; + getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) { + const column = layer.columns[columnId] as FormulaIndexPatternColumn; + if (!column.params.ast || !operationDefinitionMap) { + return; + } + const ast = parse(column.params.ast); + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + + const errors = extracted + .flatMap(({ operationType, label }) => { + return operationDefinitionMap[operationType]?.getErrorMessage?.( + layer, + label, + indexPattern, + operationDefinitionMap + ); + }) + .filter(Boolean) as string[]; + return errors.length ? errors : undefined; }, getPossibleOperation() { return { @@ -74,7 +94,8 @@ export const formulaOperation: OperationDefinition< }, ]; }, - buildColumn() { + buildColumn({ referenceIds, previousColumn, layer }) { + const metric = layer.columns[referenceIds[0]]; return { label: 'Formula', dataType: 'number', @@ -175,10 +196,18 @@ function extractColumns( const columns: IndexPatternColumn[] = []; // let currentTree: any = cloneDeep(ast); function parseNode(node: any) { - if (typeof node === 'number' || typeof node === 'string') { + if (typeof node === 'number') { // leaf node return node; } + if (typeof node === 'string') { + if (indexPattern.getFieldByName(node)) { + // leaf node + return node; + } + // create a fake node just for making run a validation pass + return parseNode({ name: 'avg', args: [node] }); + } const nodeOperation = operations[node.name]; if (!nodeOperation) { // it's a regular math node @@ -191,13 +220,16 @@ function extractColumns( // operation node if (nodeOperation.input === 'field') { const fieldName = node.args[0]; + + const field = indexPattern.getFieldByName(fieldName); + const newCol = (nodeOperation as OperationDefinition< IndexPatternColumn, 'field' >).buildColumn({ layer, indexPattern, - field: indexPattern.getFieldByName(fieldName)!, + field: field ?? ({ displayName: fieldName, name: fieldName } as IndexPatternField), }); const newColId = `${idPrefix}X${columns.length}`; newCol.customLabel = true; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx index 29148052cee8e3..4e90ef0323ce1d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx @@ -7,7 +7,7 @@ import { useRef } from 'react'; import useDebounce from 'react-use/lib/useDebounce'; import { i18n } from '@kbn/i18n'; -import { IndexPatternColumn, operationDefinitionMap } from '.'; +import { GenericOperationDefinition, IndexPatternColumn, operationDefinitionMap } from '.'; import { FieldBasedIndexPatternColumn } from './column_types'; import { IndexPattern } from '../../types'; @@ -53,14 +53,33 @@ export function getInvalidFieldMessage( operationDefinition.getPossibleOperationForField(field) !== undefined ) ); - return isInvalid - ? [ - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: 'Field {invalidField} was not found', - values: { invalidField: sourceField }, + + const isWrongType = Boolean( + sourceField && + operationDefinition && + field && + !operationDefinition.isTransferable(column as IndexPatternColumn, indexPattern) + ); + if (isInvalid) { + if (isWrongType) { + return [ + i18n.translate('xpack.lens.indexPattern.fieldWrongType', { + defaultMessage: 'Field {invalidField} is of the wrong type', + values: { + invalidField: sourceField, + }, }), - ] - : undefined; + ]; + } + return [ + i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: 'Field {invalidField} was not found', + values: { invalidField: sourceField }, + }), + ]; + } + + return undefined; } export function getEsAggsSuffix(column: IndexPatternColumn) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index dd1944ecb28eef..6005e539e657e2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -204,7 +204,8 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + operationDefinitionMap?: Record ) => string[] | undefined; /* diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index b5a4905a297383..88f1588d3a96fe 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -61,7 +61,12 @@ export function isColumnInvalid( Boolean(getReferencesErrors(layer, column, indexPattern).filter(Boolean).length); return ( - !!operationDefinition.getErrorMessage?.(layer, columnId, indexPattern) || referencesHaveErrors + !!operationDefinition.getErrorMessage?.( + layer, + columnId, + indexPattern, + operationDefinitionMap + ) || referencesHaveErrors ); } @@ -73,7 +78,12 @@ function getReferencesErrors( return column.references?.map((referenceId: string) => { const referencedOperation = layer.columns[referenceId]?.operationType; const referencedDefinition = operationDefinitionMap[referencedOperation]; - return referencedDefinition?.getErrorMessage?.(layer, referenceId, indexPattern); + return referencedDefinition?.getErrorMessage?.( + layer, + referenceId, + indexPattern, + operationDefinitionMap + ); }); } From c89bbf76e46ff544572acbb2ef334cc21c3a8d53 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 28 Jan 2021 14:16:53 +0100 Subject: [PATCH 003/185] :sparkles: Implement new features --- .../operations/definitions/formula.tsx | 173 ++++++++++++------ .../operations/definitions/helpers.tsx | 6 +- .../operations/definitions/index.ts | 12 +- .../operations/layer_helpers.ts | 51 +++++- 4 files changed, 176 insertions(+), 66 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 9c194680d08c73..7e4c95315ae1cd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -43,19 +43,18 @@ export const formulaOperation: OperationDefinition< return; } const ast = parse(column.params.ast); - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); const errors = extracted .flatMap(({ operationType, label }) => { - return operationDefinitionMap[operationType]?.getErrorMessage?.( - layer, - label, - indexPattern, - operationDefinitionMap - ); + if (layer.columns[label]) { + return operationDefinitionMap[operationType]?.getErrorMessage?.( + layer, + label, + indexPattern, + operationDefinitionMap + ); + } }) .filter(Boolean) as string[]; return errors.length ? errors : undefined; @@ -68,6 +67,7 @@ export const formulaOperation: OperationDefinition< }; }, toExpression: (layer, columnId) => { + console.log(layer.columns[columnId]); return [ { type: 'function', @@ -94,21 +94,34 @@ export const formulaOperation: OperationDefinition< }, ]; }, - buildColumn({ referenceIds, previousColumn, layer }) { - const metric = layer.columns[referenceIds[0]]; + buildColumn({ previousColumn, layer }) { + let previousFormula = ''; + if (previousColumn) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + const fieldName = getSafeFieldName(metric?.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; + } else { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )})`; + } + } return { label: 'Formula', dataType: 'number', operationType: 'formula', isBucketed: false, scale: 'ratio', - params: {}, + params: previousFormula ? { ast: previousFormula } : {}, references: [], }; }, - isTransferable: (column, newIndexPattern) => { - // TODO has to check all children - return true; + isTransferable: (column, newIndexPattern, operationDefinitionMap) => { + // Basic idea: if it has any math operation in it, probably it cannot be transferable + const ast = parse(column.params.ast); + return hasMathNode(ast, operationDefinitionMap); }, paramEditor: function ParamEditor({ @@ -130,53 +143,16 @@ export const formulaOperation: OperationDefinition< /> { - const ast = parse(text); - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns( - columnId, - operationDefinitionMap, - ast, - layer, - indexPattern + updateLayer( + regenerateLayerFromAst( + text, + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ) ); - - const columns = { - ...layer.columns, - }; - - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; - } - }); - - extracted.forEach((extractedColumn, index) => { - columns[`${columnId}X${index}`] = extractedColumn; - }); - - columns[columnId] = { - ...currentColumn, - params: { - ...currentColumn.params, - ast: text, - }, - references: [`${columnId}X${extracted.length - 1}`], - }; - - updateLayer({ - ...layer, - columns, - columnOrder: getColumnOrder({ - ...layer, - columns, - }), - }); - - // TODO - // turn ast into referenced columns - // set state }} > Submit @@ -186,6 +162,57 @@ export const formulaOperation: OperationDefinition< }, }; +export function regenerateLayerFromAst( + text: unknown, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const ast = parse(text); + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + + const columns = { + ...layer.columns, + }; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach((extractedColumn, index) => { + columns[`${columnId}X${index}`] = extractedColumn; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + ast: text, + }, + references: [`${columnId}X${extracted.length - 1}`], + }; + + return { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }; + + // TODO + // turn ast into referenced columns + // set state +} + function extractColumns( idPrefix: string, operations: Record, @@ -295,3 +322,27 @@ function findVariables(node: any): string[] { } return node.args.flatMap(findVariables); } + +function hasMathNode(root: any, operations: Record): boolean { + function findMathNodes(node: any): boolean[] { + if (typeof node === 'string') { + return [false]; + } + if (typeof node === 'number') { + return [false]; + } + if (node.name in operations) { + return [false]; + } + return node.args.flatMap(findMathNodes); + } + return Boolean(findMathNodes(root).filter(Boolean).length); +} + +function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx index 4e90ef0323ce1d..d2e2eef7d51285 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx @@ -58,7 +58,11 @@ export function getInvalidFieldMessage( sourceField && operationDefinition && field && - !operationDefinition.isTransferable(column as IndexPatternColumn, indexPattern) + !operationDefinition.isTransferable( + column as IndexPatternColumn, + indexPattern, + operationDefinitionMap + ) ); if (isInvalid) { if (isWrongType) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 6005e539e657e2..65015b07123174 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -182,7 +182,11 @@ interface BaseOperationDefinitionProps { * If this function returns false, the column is removed when switching index pattern * for a layer */ - isTransferable: (column: C, newIndexPattern: IndexPattern) => boolean; + isTransferable: ( + column: C, + newIndexPattern: IndexPattern, + operationDefinitionMap: Record + ) => boolean; /** * Transfering a column to another index pattern. This can be used to * adjust operation specific settings such as reacting to aggregation restrictions @@ -372,7 +376,11 @@ interface ManagedReferenceOperationDefinition /** * Builds the column object for the given parameters. Should include default p */ - buildColumn: (arg: BaseBuildColumnArgs) => ReferenceBasedIndexPatternColumn & C; + buildColumn: ( + arg: BaseBuildColumnArgs & { + previousColumn?: IndexPatternColumn | ReferenceBasedIndexPatternColumn; + } + ) => ReferenceBasedIndexPatternColumn & C; /** * Returns the meta data of the operation if applied. Undefined * if the operation can't be added with these fields. diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index c9e70f3f79fe38..ef00f79b87ad53 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -17,6 +17,7 @@ import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../type import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula'; interface ColumnChange { op: OperationType; @@ -58,6 +59,10 @@ export function insertNewColumn({ if (field) { throw new Error(`Can't create operation ${op} with the provided field ${field.name}`); } + if (operationDefinition.input === 'managedReference') { + // TODO: need to create on the fly the new columns for Formula, + // like we do for fullReferences to show a seamless transition + } const possibleOperation = operationDefinition.getPossibleOperation(); const isBucketed = Boolean(possibleOperation?.isBucketed); const addOperationFn = isBucketed ? addBucket : addMetric; @@ -293,6 +298,44 @@ export function replaceColumn({ } } + // TODO: Refactor all this to be more generic and know less about Formula + // if managed it has to look at the full picture to have a seamless transition + if (operationDefinition.input === 'managedReference') { + const newColumn = copyCustomLabel( + operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer }), + previousColumn + ) as FormulaIndexPatternColumn; + + // now remove the previous references + if (previousDefinition.input === 'fullReference') { + (previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => { + tempLayer = deleteColumn({ layer: tempLayer, columnId: id, indexPattern }); + }); + } + + const basicLayer = { ...tempLayer, columns: { ...tempLayer.columns, [columnId]: newColumn } }; + // rebuild the references again for the specific AST generated + const newLayer = newColumn.params?.ast + ? regenerateLayerFromAst( + newColumn.params.ast, + basicLayer, + columnId, + newColumn, + indexPattern, + operationDefinitionMap + ) + : basicLayer; + + return updateDefaultLabels( + { + ...tempLayer, + columnOrder: getColumnOrder(newLayer), + columns: adjustColumnReferencesForChangedColumn(newLayer, columnId), + }, + indexPattern + ); + } + // This logic comes after the transitions because they need to look at previous columns if (previousDefinition.input === 'fullReference') { (previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => { @@ -300,7 +343,7 @@ export function replaceColumn({ }); } - if (operationDefinition.input === 'none' || operationDefinition.input === 'managedReference') { + if (operationDefinition.input === 'none') { let newColumn = operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer }); newColumn = copyCustomLabel(newColumn, previousColumn); @@ -823,7 +866,11 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st * Returns true if the given column can be applied to the given index pattern */ export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) { - return operationDefinitionMap[column.operationType].isTransferable(column, newIndexPattern); + return operationDefinitionMap[column.operationType].isTransferable( + column, + newIndexPattern, + operationDefinitionMap + ); } export function updateLayerIndexPattern( From cd833219902e9910019915059d6e04cbafa3ea9e Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 29 Jan 2021 11:10:38 +0100 Subject: [PATCH 004/185] :bug: Fix isTransferable from Formula --- .../operations/definitions/formula.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 7e4c95315ae1cd..d6586aa3474d5f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import React, { useState } from 'react'; -import { parse } from 'tinymath'; +import { parse } from '@kbn/tinymath'; import { EuiButton, EuiTextArea } from '@elastic/eui'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; @@ -67,7 +67,6 @@ export const formulaOperation: OperationDefinition< }; }, toExpression: (layer, columnId) => { - console.log(layer.columns[columnId]); return [ { type: 'function', @@ -121,7 +120,7 @@ export const formulaOperation: OperationDefinition< isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable const ast = parse(column.params.ast); - return hasMathNode(ast, operationDefinitionMap); + return !hasMathNode(ast, operationDefinitionMap); }, paramEditor: function ParamEditor({ From 7df6c2dddc824218b041d0cba0abee60acbff83d Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 29 Jan 2021 14:45:45 +0100 Subject: [PATCH 005/185] :sparkles: Add all fullReferences ops + initial params --- .../definitions/calculations/counter_rate.tsx | 2 +- .../calculations/cumulative_sum.tsx | 2 +- .../calculations/moving_average.tsx | 15 +++- .../operations/definitions/formula.tsx | 82 +++++++++++++++---- .../operations/definitions/index.ts | 36 +++++++- 5 files changed, 113 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 4fd045c17740d3..d3599dc3dd6181 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -48,7 +48,7 @@ export const counterRateOperation: OperationDefinition< selectionStyle: 'field', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], specificOperations: ['max'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 7067b6470bec73..4b496c643dfa1f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -46,7 +46,7 @@ export const cumulativeSumOperation: OperationDefinition< selectionStyle: 'field', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], specificOperations: ['count', 'sum'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index d43dbccd92f831..06ad48f96a5044 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -36,6 +36,8 @@ const ofName = buildLabelFunction((name?: string) => { }); }); +const WINDOW_DEFAULT_VALUE = 5; + export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn & ReferenceBasedIndexPatternColumn & { operationType: 'moving_average'; @@ -57,10 +59,11 @@ export const movingAverageOperation: OperationDefinition< selectionStyle: 'full', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], + operationParams: [{ name: 'window', type: 'number', required: true }], getPossibleOperation: (indexPattern) => { if (hasDateField(indexPattern)) { return { @@ -78,8 +81,12 @@ export const movingAverageOperation: OperationDefinition< window: [(layer.columns[columnId] as MovingAverageIndexPatternColumn).params.window], }); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ( + { referenceIds, previousColumn, layer }, + columnParams = { window: WINDOW_DEFAULT_VALUE } + ) => { const metric = layer.columns[referenceIds[0]]; + const { window = WINDOW_DEFAULT_VALUE } = columnParams; return { label: ofName(metric?.label, previousColumn?.timeScale), dataType: 'number', @@ -93,8 +100,8 @@ export const movingAverageOperation: OperationDefinition< previousColumn.params && 'format' in previousColumn.params && previousColumn.params.format - ? { format: previousColumn.params.format, window: 5 } - : { window: 5 }, + ? { format: previousColumn.params.format, window } + : { window }, }; }, paramEditor: MovingAverageParamEditor, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index d6586aa3474d5f..c1a6768cdca275 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -43,7 +43,13 @@ export const formulaOperation: OperationDefinition< return; } const ast = parse(column.params.ast); - const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + const extracted = extractColumns( + columnId, + removeUnderscore(operationDefinitionMap), + ast, + layer, + indexPattern + ); const errors = extracted .flatMap(({ operationType, label }) => { @@ -173,7 +179,13 @@ export function regenerateLayerFromAst( /* { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } */ - const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + const extracted = extractColumns( + columnId, + removeUnderscore(operationDefinitionMap), + ast, + layer, + indexPattern + ); const columns = { ...layer.columns, @@ -245,18 +257,23 @@ function extractColumns( } // operation node if (nodeOperation.input === 'field') { - const fieldName = node.args[0]; + const [fieldName, ...params] = node.args; const field = indexPattern.getFieldByName(fieldName); + const mappedParams = getOperationParams(nodeOperation, params || []); + const newCol = (nodeOperation as OperationDefinition< IndexPatternColumn, 'field' - >).buildColumn({ - layer, - indexPattern, - field: field ?? ({ displayName: fieldName, name: fieldName } as IndexPatternField), - }); + >).buildColumn( + { + layer, + indexPattern, + field: field ?? ({ displayName: fieldName, name: fieldName } as IndexPatternField), + }, + mappedParams + ); const newColId = `${idPrefix}X${columns.length}`; newCol.customLabel = true; newCol.label = newColId; @@ -266,7 +283,9 @@ function extractColumns( } if (nodeOperation.input === 'fullReference') { - const consumedParam = parseNode(node.args[0]); + const [referencedOp, ...params] = node.args; + + const consumedParam = parseNode(referencedOp); const variables = findVariables(consumedParam); const mathColumn = mathOperation.buildColumn({ layer, @@ -277,14 +296,19 @@ function extractColumns( columns.push(mathColumn); mathColumn.customLabel = true; mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, params || []); const newCol = (nodeOperation as OperationDefinition< IndexPatternColumn, 'fullReference' - >).buildColumn({ - layer, - indexPattern, - referenceIds: [`${idPrefix}X${columns.length - 1}`], - }); + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); const newColId = `${idPrefix}X${columns.length}`; newCol.customLabel = true; newCol.label = newColId; @@ -345,3 +369,33 @@ function getSafeFieldName(fieldName: string | undefined) { } return fieldName; } + +function getOperationParams( + operation: + | OperationDefinition + | OperationDefinition, + params: unknown[] +) { + // TODO: to be converted with named params + const formalArgs = operation.operationParams || []; + // At the moment is positional as expressed in operationParams + return params.reduce((args, param, i) => { + if (formalArgs[i]) { + const paramName = formalArgs[i].name; + args[paramName] = param; + } + return args; + }, {}); +} + +function removeUnderscore( + operationDefinitionMap: Record +): Record { + return Object.keys(operationDefinitionMap).reduce((memo, operationTypeSnakeCase) => { + const operationType = operationTypeSnakeCase.replace(/([_][a-z])/, (group) => + group.replace('_', '') + ); + memo[operationType] = operationDefinitionMap[operationTypeSnakeCase]; + return memo; + }, {} as Record); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 65015b07123174..87a15fc0de3f67 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -228,15 +228,28 @@ interface BaseBuildColumnArgs { indexPattern: IndexPattern; } +interface OperationParam { + name: string; + type: string; + required?: boolean; +} + interface FieldlessOperationDefinition { input: 'none'; + + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; /** * Builds the column object for the given parameters. Should include default p */ buildColumn: ( arg: BaseBuildColumnArgs & { previousColumn?: IndexPatternColumn; - } + }, + columnParams?: (IndexPatternColumn & C)['params'] ) => C; /** * Returns the meta data of the operation if applied. Undefined @@ -257,6 +270,12 @@ interface FieldlessOperationDefinition { interface FieldBasedOperationDefinition { input: 'field'; + + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; /** * Returns the meta data of the operation if applied to the given field. Undefined * if the field is not applicable to the operation. @@ -269,7 +288,8 @@ interface FieldBasedOperationDefinition { arg: BaseBuildColumnArgs & { field: IndexPatternField; previousColumn?: IndexPatternColumn; - } + }, + columnParams?: (IndexPatternColumn & C)['params'] ) => C; /** * This method will be called if the user changes the field of an operation. @@ -339,6 +359,12 @@ interface FullReferenceOperationDefinition { */ requiredReferences: RequiredReference[]; + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; + /** * The type of UI that is shown in the editor for this function: * - full: List of sub-functions and fields @@ -354,7 +380,8 @@ interface FullReferenceOperationDefinition { arg: BaseBuildColumnArgs & { referenceIds: string[]; previousColumn?: IndexPatternColumn; - } + }, + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] ) => ReferenceBasedIndexPatternColumn & C; /** * Returns the meta data of the operation if applied. Undefined @@ -379,7 +406,8 @@ interface ManagedReferenceOperationDefinition buildColumn: ( arg: BaseBuildColumnArgs & { previousColumn?: IndexPatternColumn | ReferenceBasedIndexPatternColumn; - } + }, + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] ) => ReferenceBasedIndexPatternColumn & C; /** * Returns the meta data of the operation if applied. Undefined From 0473cb8777863569aa09d6f13c49eca5a21484c9 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 29 Jan 2021 16:18:45 -0500 Subject: [PATCH 006/185] Expand Tinymath grammar, including named arguments --- packages/kbn-tinymath/package.json | 1 + packages/kbn-tinymath/src/grammar.js | 601 +++++++++++------- packages/kbn-tinymath/src/grammar.pegjs | 83 ++- packages/kbn-tinymath/src/index.js | 28 +- packages/kbn-tinymath/test/library.test.js | 182 ++++-- packages/kbn-tinymath/tinymath.d.ts | 45 ++ .../functions/common/math.ts | 1 - ...pe.test.js => get_expression_type.test.ts} | 2 +- .../functions/server/get_field_names.test.ts | 1 - .../functions/server/pointseries/index.ts | 15 +- ...ression_type.js => get_expression_type.ts} | 7 +- .../server/pointseries/lib/get_field_names.ts | 22 +- .../pointseries/lib/is_column_reference.ts | 3 +- 13 files changed, 650 insertions(+), 341 deletions(-) create mode 100644 packages/kbn-tinymath/tinymath.d.ts rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/{get_expression_type.test.js => get_expression_type.test.ts} (96%) rename x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/{get_expression_type.js => get_expression_type.ts} (82%) diff --git a/packages/kbn-tinymath/package.json b/packages/kbn-tinymath/package.json index 34fd593672b5a8..554239ef4e640f 100644 --- a/packages/kbn-tinymath/package.json +++ b/packages/kbn-tinymath/package.json @@ -4,6 +4,7 @@ "license": "SSPL-1.0 OR Elastic License", "private": true, "main": "src/index.js", + "types": "tinymath.d.ts", "scripts": { "kbn:bootstrap": "yarn build", "build": "../../node_modules/.bin/pegjs -o src/grammar.js src/grammar.pegjs" diff --git a/packages/kbn-tinymath/src/grammar.js b/packages/kbn-tinymath/src/grammar.js index 60dfcf4800631b..9b3f3320ff0498 100644 --- a/packages/kbn-tinymath/src/grammar.js +++ b/packages/kbn-tinymath/src/grammar.js @@ -156,11 +156,21 @@ function peg$parse(input, options) { peg$c12 = function(literal) { return literal; }, - peg$c13 = function(first, rest) { // We can open this up later. Strict for now. - return first + rest.join(''); + peg$c13 = function(chars) { + return { + type: 'variable', + value: chars.join(''), + location: simpleLocation(location()), + text: text() + }; }, - peg$c14 = function(first, mid) { - return first + mid.map(m => m[0].join('') + m[1].join('')).join('') + peg$c14 = function(rest) { + return { + type: 'variable', + value: rest.join(''), + location: simpleLocation(location()), + text: text() + }; }, peg$c15 = "+", peg$c16 = peg$literalExpectation("+", false), @@ -168,8 +178,11 @@ function peg$parse(input, options) { peg$c18 = peg$literalExpectation("-", false), peg$c19 = function(left, rest) { return rest.reduce((acc, curr) => ({ + type: 'function', name: curr[0] === '+' ? 'add' : 'subtract', - args: [acc, curr[1]] + args: [acc, curr[1]], + location: simpleLocation(location()), + text: text() }), left) }, peg$c20 = "*", @@ -178,8 +191,11 @@ function peg$parse(input, options) { peg$c23 = peg$literalExpectation("/", false), peg$c24 = function(left, rest) { return rest.reduce((acc, curr) => ({ + type: 'function', name: curr[0] === '*' ? 'multiply' : 'divide', - args: [acc, curr[1]] + args: [acc, curr[1]], + location: simpleLocation(location()), + text: text() }), left) }, peg$c25 = "(", @@ -196,25 +212,49 @@ function peg$parse(input, options) { peg$c34 = function(first, rest) { return [first].concat(rest); }, - peg$c35 = peg$otherExpectation("function"), - peg$c36 = /^[a-z]/, - peg$c37 = peg$classExpectation([["a", "z"]], false, false), - peg$c38 = function(name, args) { - return {name: name.join(''), args: args || []}; + peg$c35 = /^["]/, + peg$c36 = peg$classExpectation(["\""], false, false), + peg$c37 = function(value) { return value.join(''); }, + peg$c38 = /^[']/, + peg$c39 = peg$classExpectation(["'"], false, false), + peg$c40 = /^[a-zA-Z_\-]/, + peg$c41 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), + peg$c42 = "=", + peg$c43 = peg$literalExpectation("=", false), + peg$c44 = function(name, value) { + return { + type: 'namedArgument', + name: name.join(''), + value: value, + location: simpleLocation(location()), + text: text() + }; + }, + peg$c45 = peg$otherExpectation("function"), + peg$c46 = function(name, args) { + return { + type: 'function', + name: name.join(''), + args: args || [], + location: simpleLocation(location()), + text: text() + }; }, - peg$c39 = peg$otherExpectation("number"), - peg$c40 = function() { return parseFloat(text()); }, - peg$c41 = /^[eE]/, - peg$c42 = peg$classExpectation(["e", "E"], false, false), - peg$c43 = peg$otherExpectation("exponent"), - peg$c44 = ".", - peg$c45 = peg$literalExpectation(".", false), - peg$c46 = "0", - peg$c47 = peg$literalExpectation("0", false), - peg$c48 = /^[1-9]/, - peg$c49 = peg$classExpectation([["1", "9"]], false, false), - peg$c50 = /^[0-9]/, - peg$c51 = peg$classExpectation([["0", "9"]], false, false), + peg$c47 = peg$otherExpectation("number"), + peg$c48 = function() { + return parseFloat(text()); + }, + peg$c49 = /^[eE]/, + peg$c50 = peg$classExpectation(["e", "E"], false, false), + peg$c51 = peg$otherExpectation("exponent"), + peg$c52 = ".", + peg$c53 = peg$literalExpectation(".", false), + peg$c54 = "0", + peg$c55 = peg$literalExpectation("0", false), + peg$c56 = /^[1-9]/, + peg$c57 = peg$classExpectation([["1", "9"]], false, false), + peg$c58 = /^[0-9]/, + peg$c59 = peg$classExpectation([["0", "9"]], false, false), peg$currPos = 0, peg$savedPos = 0, @@ -456,10 +496,7 @@ function peg$parse(input, options) { if (s1 !== peg$FAILED) { s2 = peg$parseNumber(); if (s2 === peg$FAILED) { - s2 = peg$parseVariableWithQuote(); - if (s2 === peg$FAILED) { - s2 = peg$parseVariable(); - } + s2 = peg$parseVariable(); } if (s2 !== peg$FAILED) { s3 = peg$parse_(); @@ -489,25 +526,37 @@ function peg$parse(input, options) { } function peg$parseVariable() { - var s0, s1, s2, s3, s4; + var s0, s1, s2, s3, s4, s5; s0 = peg$currPos; s1 = peg$parse_(); if (s1 !== peg$FAILED) { - s2 = peg$parseStartChar(); + s2 = peg$parseQuote(); if (s2 !== peg$FAILED) { s3 = []; s4 = peg$parseValidChar(); + if (s4 === peg$FAILED) { + s4 = peg$parseSpace(); + } while (s4 !== peg$FAILED) { s3.push(s4); s4 = peg$parseValidChar(); + if (s4 === peg$FAILED) { + s4 = peg$parseSpace(); + } } if (s3 !== peg$FAILED) { - s4 = peg$parse_(); + s4 = peg$parseQuote(); if (s4 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c13(s2, s3); - s0 = s1; + s5 = peg$parse_(); + if (s5 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c13(s3); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } } else { peg$currPos = s0; s0 = peg$FAILED; @@ -524,98 +573,26 @@ function peg$parse(input, options) { peg$currPos = s0; s0 = peg$FAILED; } - - return s0; - } - - function peg$parseVariableWithQuote() { - var s0, s1, s2, s3, s4, s5, s6, s7, s8; - - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = peg$parseQuote(); - if (s2 !== peg$FAILED) { - s3 = peg$parseStartChar(); + if (s0 === peg$FAILED) { + s0 = peg$currPos; + s1 = peg$parse_(); + if (s1 !== peg$FAILED) { + s2 = []; + s3 = peg$parseValidChar(); if (s3 !== peg$FAILED) { - s4 = []; - s5 = peg$currPos; - s6 = []; - s7 = peg$parseSpace(); - while (s7 !== peg$FAILED) { - s6.push(s7); - s7 = peg$parseSpace(); + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$parseValidChar(); } - if (s6 !== peg$FAILED) { - s7 = []; - s8 = peg$parseValidChar(); - if (s8 !== peg$FAILED) { - while (s8 !== peg$FAILED) { - s7.push(s8); - s8 = peg$parseValidChar(); - } - } else { - s7 = peg$FAILED; - } - if (s7 !== peg$FAILED) { - s6 = [s6, s7]; - s5 = s6; - } else { - peg$currPos = s5; - s5 = peg$FAILED; - } - } else { - peg$currPos = s5; - s5 = peg$FAILED; - } - while (s5 !== peg$FAILED) { - s4.push(s5); - s5 = peg$currPos; - s6 = []; - s7 = peg$parseSpace(); - while (s7 !== peg$FAILED) { - s6.push(s7); - s7 = peg$parseSpace(); - } - if (s6 !== peg$FAILED) { - s7 = []; - s8 = peg$parseValidChar(); - if (s8 !== peg$FAILED) { - while (s8 !== peg$FAILED) { - s7.push(s8); - s8 = peg$parseValidChar(); - } - } else { - s7 = peg$FAILED; - } - if (s7 !== peg$FAILED) { - s6 = [s6, s7]; - s5 = s6; - } else { - peg$currPos = s5; - s5 = peg$FAILED; - } - } else { - peg$currPos = s5; - s5 = peg$FAILED; - } - } - if (s4 !== peg$FAILED) { - s5 = peg$parseQuote(); - if (s5 !== peg$FAILED) { - s6 = peg$parse_(); - if (s6 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c14(s3, s4); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } + } else { + s2 = peg$FAILED; + } + if (s2 !== peg$FAILED) { + s3 = peg$parse_(); + if (s3 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c14(s2); + s0 = s1; } else { peg$currPos = s0; s0 = peg$FAILED; @@ -628,9 +605,6 @@ function peg$parse(input, options) { peg$currPos = s0; s0 = peg$FAILED; } - } else { - peg$currPos = s0; - s0 = peg$FAILED; } return s0; @@ -911,105 +885,285 @@ function peg$parse(input, options) { return s0; } - function peg$parseArguments() { - var s0, s1, s2, s3, s4, s5, s6, s7, s8; + function peg$parseArgument_List() { + var s0, s1, s2, s3, s4, s5, s6, s7; peg$silentFails++; s0 = peg$currPos; - s1 = peg$parse_(); + s1 = peg$parseArgument(); if (s1 !== peg$FAILED) { - s2 = peg$parseAddSubtract(); - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$currPos; - s5 = peg$parse_(); + s2 = []; + s3 = peg$currPos; + s4 = peg$parse_(); + if (s4 !== peg$FAILED) { + if (input.charCodeAt(peg$currPos) === 44) { + s5 = peg$c31; + peg$currPos++; + } else { + s5 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c32); } + } if (s5 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s6 = peg$c31; - peg$currPos++; - } else { - s6 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } - } + s6 = peg$parse_(); if (s6 !== peg$FAILED) { - s7 = peg$parse_(); + s7 = peg$parseArgument(); if (s7 !== peg$FAILED) { - s8 = peg$parseAddSubtract(); - if (s8 !== peg$FAILED) { - peg$savedPos = s4; - s5 = peg$c33(s2, s8); - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } + peg$savedPos = s3; + s4 = peg$c33(s1, s7); + s3 = s4; } else { - peg$currPos = s4; - s4 = peg$FAILED; + peg$currPos = s3; + s3 = peg$FAILED; } } else { - peg$currPos = s4; - s4 = peg$FAILED; + peg$currPos = s3; + s3 = peg$FAILED; } } else { - peg$currPos = s4; - s4 = peg$FAILED; + peg$currPos = s3; + s3 = peg$FAILED; } - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$currPos; - s5 = peg$parse_(); + } else { + peg$currPos = s3; + s3 = peg$FAILED; + } + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$currPos; + s4 = peg$parse_(); + if (s4 !== peg$FAILED) { + if (input.charCodeAt(peg$currPos) === 44) { + s5 = peg$c31; + peg$currPos++; + } else { + s5 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c32); } + } if (s5 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s6 = peg$c31; - peg$currPos++; - } else { - s6 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } - } + s6 = peg$parse_(); if (s6 !== peg$FAILED) { - s7 = peg$parse_(); + s7 = peg$parseArgument(); if (s7 !== peg$FAILED) { - s8 = peg$parseAddSubtract(); - if (s8 !== peg$FAILED) { - peg$savedPos = s4; - s5 = peg$c33(s2, s8); - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } + peg$savedPos = s3; + s4 = peg$c33(s1, s7); + s3 = s4; } else { - peg$currPos = s4; - s4 = peg$FAILED; + peg$currPos = s3; + s3 = peg$FAILED; } } else { - peg$currPos = s4; - s4 = peg$FAILED; + peg$currPos = s3; + s3 = peg$FAILED; } } else { - peg$currPos = s4; + peg$currPos = s3; + s3 = peg$FAILED; + } + } else { + peg$currPos = s3; + s3 = peg$FAILED; + } + } + if (s2 !== peg$FAILED) { + s3 = peg$parse_(); + if (s3 !== peg$FAILED) { + if (input.charCodeAt(peg$currPos) === 44) { + s4 = peg$c31; + peg$currPos++; + } else { s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c32); } + } + if (s4 === peg$FAILED) { + s4 = null; + } + if (s4 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c34(s1, s2); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + peg$silentFails--; + if (s0 === peg$FAILED) { + s1 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c30); } + } + + return s0; + } + + function peg$parseString() { + var s0, s1, s2, s3; + + s0 = peg$currPos; + if (peg$c35.test(input.charAt(peg$currPos))) { + s1 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s1 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c36); } + } + if (s1 !== peg$FAILED) { + s2 = []; + s3 = peg$parseValidChar(); + if (s3 !== peg$FAILED) { + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$parseValidChar(); + } + } else { + s2 = peg$FAILED; + } + if (s2 !== peg$FAILED) { + if (peg$c35.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c36); } + } + if (s3 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c37(s2); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + if (s0 === peg$FAILED) { + s0 = peg$currPos; + if (peg$c38.test(input.charAt(peg$currPos))) { + s1 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s1 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c39); } + } + if (s1 !== peg$FAILED) { + s2 = []; + s3 = peg$parseValidChar(); + if (s3 !== peg$FAILED) { + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$parseValidChar(); + } + } else { + s2 = peg$FAILED; + } + if (s2 !== peg$FAILED) { + if (peg$c38.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c39); } + } + if (s3 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c37(s2); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + if (s0 === peg$FAILED) { + s0 = peg$currPos; + s1 = []; + s2 = peg$parseValidChar(); + if (s2 !== peg$FAILED) { + while (s2 !== peg$FAILED) { + s1.push(s2); + s2 = peg$parseValidChar(); + } + } else { + s1 = peg$FAILED; + } + if (s1 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c37(s1); + } + s0 = s1; + } + } + + return s0; + } + + function peg$parseArgument() { + var s0, s1, s2, s3, s4, s5, s6; + + s0 = peg$currPos; + s1 = []; + if (peg$c40.test(input.charAt(peg$currPos))) { + s2 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s2 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c41); } + } + if (s2 !== peg$FAILED) { + while (s2 !== peg$FAILED) { + s1.push(s2); + if (peg$c40.test(input.charAt(peg$currPos))) { + s2 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s2 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c41); } + } + } + } else { + s1 = peg$FAILED; + } + if (s1 !== peg$FAILED) { + s2 = peg$parse_(); + if (s2 !== peg$FAILED) { + if (input.charCodeAt(peg$currPos) === 61) { + s3 = peg$c42; + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c43); } } if (s3 !== peg$FAILED) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s5 = peg$c31; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } - } - if (s5 === peg$FAILED) { - s5 = null; - } + s5 = peg$parseString(); if (s5 !== peg$FAILED) { s6 = peg$parse_(); if (s6 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c34(s2, s3); + s1 = peg$c44(s1, s5); s0 = s1; } else { peg$currPos = s0; @@ -1035,10 +1189,8 @@ function peg$parse(input, options) { peg$currPos = s0; s0 = peg$FAILED; } - peg$silentFails--; if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c30); } + s0 = peg$parseAddSubtract(); } return s0; @@ -1052,22 +1204,22 @@ function peg$parse(input, options) { s1 = peg$parse_(); if (s1 !== peg$FAILED) { s2 = []; - if (peg$c36.test(input.charAt(peg$currPos))) { + if (peg$c40.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c37); } + if (peg$silentFails === 0) { peg$fail(peg$c41); } } if (s3 !== peg$FAILED) { while (s3 !== peg$FAILED) { s2.push(s3); - if (peg$c36.test(input.charAt(peg$currPos))) { + if (peg$c40.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c37); } + if (peg$silentFails === 0) { peg$fail(peg$c41); } } } } else { @@ -1084,7 +1236,7 @@ function peg$parse(input, options) { if (s3 !== peg$FAILED) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { - s5 = peg$parseArguments(); + s5 = peg$parseArgument_List(); if (s5 === peg$FAILED) { s5 = null; } @@ -1102,7 +1254,7 @@ function peg$parse(input, options) { s8 = peg$parse_(); if (s8 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c38(s2, s5); + s1 = peg$c46(s2, s5); s0 = s1; } else { peg$currPos = s0; @@ -1139,7 +1291,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c35); } + if (peg$silentFails === 0) { peg$fail(peg$c45); } } return s0; @@ -1174,7 +1326,7 @@ function peg$parse(input, options) { } if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c40(); + s1 = peg$c48(); s0 = s1; } else { peg$currPos = s0; @@ -1195,7 +1347,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c39); } + if (peg$silentFails === 0) { peg$fail(peg$c47); } } return s0; @@ -1204,12 +1356,12 @@ function peg$parse(input, options) { function peg$parseE() { var s0; - if (peg$c41.test(input.charAt(peg$currPos))) { + if (peg$c49.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c42); } + if (peg$silentFails === 0) { peg$fail(peg$c50); } } return s0; @@ -1261,7 +1413,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c43); } + if (peg$silentFails === 0) { peg$fail(peg$c51); } } return s0; @@ -1272,11 +1424,11 @@ function peg$parse(input, options) { s0 = peg$currPos; if (input.charCodeAt(peg$currPos) === 46) { - s1 = peg$c44; + s1 = peg$c52; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } + if (peg$silentFails === 0) { peg$fail(peg$c53); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1308,20 +1460,20 @@ function peg$parse(input, options) { var s0, s1, s2, s3; if (input.charCodeAt(peg$currPos) === 48) { - s0 = peg$c46; + s0 = peg$c54; peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c47); } + if (peg$silentFails === 0) { peg$fail(peg$c55); } } if (s0 === peg$FAILED) { s0 = peg$currPos; - if (peg$c48.test(input.charAt(peg$currPos))) { + if (peg$c56.test(input.charAt(peg$currPos))) { s1 = input.charAt(peg$currPos); peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c49); } + if (peg$silentFails === 0) { peg$fail(peg$c57); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1349,17 +1501,30 @@ function peg$parse(input, options) { function peg$parseDigit() { var s0; - if (peg$c50.test(input.charAt(peg$currPos))) { + if (peg$c58.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c51); } + if (peg$silentFails === 0) { peg$fail(peg$c59); } } return s0; } + + function simpleLocation (location) { + // Returns an object representing the position of the function within the expression, + // demarcated by the position of its first character and last character. We calculate these values + // using the offset because the expression could span multiple lines, and we don't want to deal + // with column and line values. + return { + min: location.start.offset, + max: location.end.offset + } + } + + peg$result = peg$startRuleFunction(); if (peg$result !== peg$FAILED && peg$currPos === input.length) { diff --git a/packages/kbn-tinymath/src/grammar.pegjs b/packages/kbn-tinymath/src/grammar.pegjs index cab8e024e60b33..f0161d3dad38d2 100644 --- a/packages/kbn-tinymath/src/grammar.pegjs +++ b/packages/kbn-tinymath/src/grammar.pegjs @@ -1,5 +1,18 @@ // tinymath parsing grammar +{ + function simpleLocation (location) { + // Returns an object representing the position of the function within the expression, + // demarcated by the position of its first character and last character. We calculate these values + // using the offset because the expression could span multiple lines, and we don't want to deal + // with column and line values. + return { + min: location.start.offset, + max: location.end.offset + } + } +} + start = Expression @@ -23,18 +36,28 @@ ValidChar // literals and variables Literal "literal" - = _ literal:(Number / VariableWithQuote / Variable) _ { + = _ literal:(Number / Variable) _ { return literal; } +// Quoted variables are interpreted as strings +// but unquoted variables are more restrictive Variable - = _ first:StartChar rest:ValidChar* _ { // We can open this up later. Strict for now. - return first + rest.join(''); + = _ Quote chars:(ValidChar / Space)* Quote _ { + return { + type: 'variable', + value: chars.join(''), + location: simpleLocation(location()), + text: text() + }; } - -VariableWithQuote - = _ Quote first:StartChar mid:(Space* ValidChar+)* Quote _ { - return first + mid.map(m => m[0].join('') + m[1].join('')).join('') + / _ rest:ValidChar+ _ { + return { + type: 'variable', + value: rest.join(''), + location: simpleLocation(location()), + text: text() + }; } // expressions @@ -45,16 +68,22 @@ Expression AddSubtract = _ left:MultiplyDivide rest:(('+' / '-') MultiplyDivide)* _ { return rest.reduce((acc, curr) => ({ + type: 'function', name: curr[0] === '+' ? 'add' : 'subtract', - args: [acc, curr[1]] + args: [acc, curr[1]], + location: simpleLocation(location()), + text: text() }), left) } MultiplyDivide = _ left:Factor rest:(('*' / '/') Factor)* _ { return rest.reduce((acc, curr) => ({ + type: 'function', name: curr[0] === '*' ? 'multiply' : 'divide', - args: [acc, curr[1]] + args: [acc, curr[1]], + location: simpleLocation(location()), + text: text() }), left) } @@ -68,20 +97,46 @@ Group return expr } -Arguments "arguments" - = _ first:Expression rest:(_ ',' _ arg:Expression {return arg})* _ ','? _ { +Argument_List "arguments" + = first:Argument rest:(_ ',' _ arg:Argument {return arg})* _ ','? { return [first].concat(rest); } +String + = [\"] value:(ValidChar)+ [\"] { return value.join(''); } + / [\'] value:(ValidChar)+ [\'] { return value.join(''); } + / value:(ValidChar)+ { return value.join(''); } + + +Argument + = name:[a-zA-Z_-]+ _ '=' _ value:String _ { + return { + type: 'namedArgument', + name: name.join(''), + value: value, + location: simpleLocation(location()), + text: text() + }; + } + / arg:Expression + Function "function" - = _ name:[a-z]+ '(' _ args:Arguments? _ ')' _ { - return {name: name.join(''), args: args || []}; + = _ name:[a-zA-Z_-]+ '(' _ args:Argument_List? _ ')' _ { + return { + type: 'function', + name: name.join(''), + args: args || [], + location: simpleLocation(location()), + text: text() + }; } // Numbers. Lol. Number "number" - = '-'? Integer Fraction? Exp? { return parseFloat(text()); } + = '-'? Integer Fraction? Exp? { + return parseFloat(text()); + } E = [eE] diff --git a/packages/kbn-tinymath/src/index.js b/packages/kbn-tinymath/src/index.js index e61956bd63e556..991bd819812784 100644 --- a/packages/kbn-tinymath/src/index.js +++ b/packages/kbn-tinymath/src/index.js @@ -38,17 +38,22 @@ function interpret(node, scope, injectedFunctions) { return exec(node); function exec(node) { - const type = getType(node); + if (typeof node === 'number') { + return node; + } - if (type === 'function') return invoke(node); + if (node.type === 'function') return invoke(node); - if (type === 'string') { - const val = getValue(scope, node); - if (typeof val === 'undefined') throw new Error(`Unknown variable: ${node}`); + if (node.type === 'variable') { + const val = getValue(scope, node.value); + if (typeof val === 'undefined') throw new Error(`Unknown variable: ${node.value}`); return val; } - return node; // Can only be a number at this point + if (node.type === 'namedArgument') { + // We are ignoring named arguments in the interpreter + throw new Error(`Named arguments are not supported in tinymath itself, at ${node.name}`); + } } function invoke(node) { @@ -67,17 +72,6 @@ function getValue(scope, node) { return typeof val !== 'undefined' ? val : scope[node]; } -function getType(x) { - const type = typeof x; - if (type === 'object') { - const keys = Object.keys(x); - if (keys.length !== 2 || !x.name || !x.args) throw new Error('Invalid AST object'); - return 'function'; - } - if (type === 'string' || type === 'number') return type; - throw new Error(`Unknown AST property type: ${type}`); -} - function isOperable(args) { return args.every((arg) => { if (Array.isArray(arg)) return isOperable(arg); diff --git a/packages/kbn-tinymath/test/library.test.js b/packages/kbn-tinymath/test/library.test.js index 7569cf90b2e355..da2dbd42f262cb 100644 --- a/packages/kbn-tinymath/test/library.test.js +++ b/packages/kbn-tinymath/test/library.test.js @@ -11,7 +11,19 @@ Need tests for spacing, etc */ -const { evaluate, parse } = require('..'); +import { evaluate, parse } from '..'; + +function variableEqual(value) { + return expect.objectContaining({ type: 'variable', value }); +} + +function functionEqual(name, args) { + return expect.objectContaining({ type: 'function', name, args }); +} + +function namedArgumentEqual(name, value) { + return expect.objectContaining({ type: 'namedArgument', name, value }); +} describe('Parser', () => { describe('Numbers', () => { @@ -31,96 +43,134 @@ describe('Parser', () => { describe('Variables', () => { it('strings', () => { - expect(parse('f')).toEqual('f'); - expect(parse('foo')).toEqual('foo'); + expect(parse('f')).toEqual(variableEqual('f')); + expect(parse('foo')).toEqual(variableEqual('foo')); + expect(parse('foo1')).toEqual(variableEqual('foo1')); + expect(() => parse('1foo1')).toThrow('but "f" found'); + }); + + it('strings with spaces', () => { + expect(parse(' foo ')).toEqual(variableEqual('foo')); + expect(() => parse(' foo bar ')).toThrow('but "b" found'); }); it('allowed characters', () => { - expect(parse('_foo')).toEqual('_foo'); - expect(parse('@foo')).toEqual('@foo'); - expect(parse('.foo')).toEqual('.foo'); - expect(parse('-foo')).toEqual('-foo'); - expect(parse('_foo0')).toEqual('_foo0'); - expect(parse('@foo0')).toEqual('@foo0'); - expect(parse('.foo0')).toEqual('.foo0'); - expect(parse('-foo0')).toEqual('-foo0'); + expect(parse('_foo')).toEqual(variableEqual('_foo')); + expect(parse('@foo')).toEqual(variableEqual('@foo')); + expect(parse('.foo')).toEqual(variableEqual('.foo')); + expect(parse('-foo')).toEqual(variableEqual('-foo')); + expect(parse('_foo0')).toEqual(variableEqual('_foo0')); + expect(parse('@foo0')).toEqual(variableEqual('@foo0')); + expect(parse('.foo0')).toEqual(variableEqual('.foo0')); + expect(parse('-foo0')).toEqual(variableEqual('-foo0')); }); }); describe('quoted variables', () => { it('strings with double quotes', () => { - expect(parse('"foo"')).toEqual('foo'); - expect(parse('"f b"')).toEqual('f b'); - expect(parse('"foo bar"')).toEqual('foo bar'); - expect(parse('"foo bar fizz buzz"')).toEqual('foo bar fizz buzz'); - expect(parse('"foo bar baby"')).toEqual('foo bar baby'); + expect(parse('"foo"')).toEqual(variableEqual('foo')); + expect(parse('"f b"')).toEqual(variableEqual('f b')); + expect(parse('"foo bar"')).toEqual(variableEqual('foo bar')); + expect(parse('"foo bar fizz buzz"')).toEqual(variableEqual('foo bar fizz buzz')); + expect(parse('"foo bar baby"')).toEqual(variableEqual('foo bar baby')); }); it('strings with single quotes', () => { /* eslint-disable prettier/prettier */ - expect(parse("'foo'")).toEqual('foo'); - expect(parse("'f b'")).toEqual('f b'); - expect(parse("'foo bar'")).toEqual('foo bar'); - expect(parse("'foo bar fizz buzz'")).toEqual('foo bar fizz buzz'); - expect(parse("'foo bar baby'")).toEqual('foo bar baby'); + expect(parse("'foo'")).toEqual(variableEqual('foo')); + expect(parse("'f b'")).toEqual(variableEqual('f b')); + expect(parse("'foo bar'")).toEqual(variableEqual('foo bar')); + expect(parse("'foo bar fizz buzz'")).toEqual(variableEqual('foo bar fizz buzz')); + expect(parse("'foo bar baby'")).toEqual(variableEqual('foo bar baby')); + expect(parse("' foo bar'")).toEqual(variableEqual(" foo bar")); + expect(parse("'foo bar '")).toEqual(variableEqual("foo bar ")); + expect(parse("'0foo'")).toEqual(variableEqual("0foo")); + expect(parse("' foo bar'")).toEqual(variableEqual(" foo bar")); + expect(parse("'foo bar '")).toEqual(variableEqual("foo bar ")); + expect(parse("'0foo'")).toEqual(variableEqual("0foo")); /* eslint-enable prettier/prettier */ }); it('allowed characters', () => { - expect(parse('"_foo bar"')).toEqual('_foo bar'); - expect(parse('"@foo bar"')).toEqual('@foo bar'); - expect(parse('".foo bar"')).toEqual('.foo bar'); - expect(parse('"-foo bar"')).toEqual('-foo bar'); - expect(parse('"_foo0 bar1"')).toEqual('_foo0 bar1'); - expect(parse('"@foo0 bar1"')).toEqual('@foo0 bar1'); - expect(parse('".foo0 bar1"')).toEqual('.foo0 bar1'); - expect(parse('"-foo0 bar1"')).toEqual('-foo0 bar1'); - }); - - it('invalid characters in double quotes', () => { - const check = (str) => () => parse(str); - expect(check('" foo bar"')).toThrow('but "\\"" found'); - expect(check('"foo bar "')).toThrow('but "\\"" found'); - expect(check('"0foo"')).toThrow('but "\\"" found'); - expect(check('" foo bar"')).toThrow('but "\\"" found'); - expect(check('"foo bar "')).toThrow('but "\\"" found'); - expect(check('"0foo"')).toThrow('but "\\"" found'); - }); - - it('invalid characters in single quotes', () => { - const check = (str) => () => parse(str); - /* eslint-disable prettier/prettier */ - expect(check("' foo bar'")).toThrow('but "\'" found'); - expect(check("'foo bar '")).toThrow('but "\'" found'); - expect(check("'0foo'")).toThrow('but "\'" found'); - expect(check("' foo bar'")).toThrow('but "\'" found'); - expect(check("'foo bar '")).toThrow('but "\'" found'); - expect(check("'0foo'")).toThrow('but "\'" found'); - /* eslint-enable prettier/prettier */ + expect(parse('"_foo bar"')).toEqual(variableEqual('_foo bar')); + expect(parse('"@foo bar"')).toEqual(variableEqual('@foo bar')); + expect(parse('".foo bar"')).toEqual(variableEqual('.foo bar')); + expect(parse('"-foo bar"')).toEqual(variableEqual('-foo bar')); + expect(parse('"_foo0 bar1"')).toEqual(variableEqual('_foo0 bar1')); + expect(parse('"@foo0 bar1"')).toEqual(variableEqual('@foo0 bar1')); + expect(parse('".foo0 bar1"')).toEqual(variableEqual('.foo0 bar1')); + expect(parse('"-foo0 bar1"')).toEqual(variableEqual('-foo0 bar1')); + expect(parse('" foo bar"')).toEqual(variableEqual(' foo bar')); + expect(parse('"foo bar "')).toEqual(variableEqual('foo bar ')); + expect(parse('"0foo"')).toEqual(variableEqual('0foo')); + expect(parse('" foo bar"')).toEqual(variableEqual(' foo bar')); + expect(parse('"foo bar "')).toEqual(variableEqual('foo bar ')); + expect(parse('"0foo"')).toEqual(variableEqual('0foo')); }); }); describe('Functions', () => { it('no arguments', () => { - expect(parse('foo()')).toEqual({ name: 'foo', args: [] }); + expect(parse('foo()')).toEqual(functionEqual('foo', [])); }); it('arguments', () => { - expect(parse('foo(5,10)')).toEqual({ name: 'foo', args: [5, 10] }); + expect(parse('foo(5,10)')).toEqual(functionEqual('foo', [5, 10])); }); it('arguments with strings', () => { - expect(parse('foo("string with spaces")')).toEqual({ - name: 'foo', - args: ['string with spaces'], - }); + expect(parse('foo("string with spaces")')).toEqual( + functionEqual('foo', [variableEqual('string with spaces')]) + ); - /* eslint-disable prettier/prettier */ - expect(parse("foo('string with spaces')")).toEqual({ - name: 'foo', - args: ['string with spaces'], - }); - /* eslint-enable prettier/prettier */ + expect(parse("foo('string with spaces')")).toEqual( + functionEqual('foo', [variableEqual('string with spaces')]) + ); + }); + + it('named only', () => { + expect(parse('foo(q=10)')).toEqual(functionEqual('foo', [namedArgumentEqual('q', '10')])); + }); + + it('named and positional', () => { + expect(parse('foo(ref, q="bar")')).toEqual( + functionEqual('foo', [variableEqual('ref'), namedArgumentEqual('q', 'bar')]) + ); + }); + + it('numerically named', () => { + expect(() => parse('foo(1=2)')).toThrow('but "(" found'); + }); + + it('multiple named', () => { + expect(parse('foo(q_param="bar", offset-type="1d")')).toEqual( + functionEqual('foo', [ + namedArgumentEqual('q_param', 'bar'), + namedArgumentEqual('offset-type', '1d'), + ]) + ); + }); + + it('multiple named and positional', () => { + expect(parse('foo(q="bar", ref, offset="1d", 100)')).toEqual( + functionEqual('foo', [ + namedArgumentEqual('q', 'bar'), + variableEqual('ref'), + namedArgumentEqual('offset', '1d'), + 100, + ]) + ); + }); + + it('duplicate named', () => { + expect(parse('foo(q="bar", q="test")')).toEqual( + functionEqual('foo', [namedArgumentEqual('q', 'bar'), namedArgumentEqual('q', 'test')]) + ); + }); + + it('incomplete named', () => { + expect(() => parse('foo(a=)')).toThrow('but "(" found'); + expect(() => parse('foo(=a)')).toThrow('but "(" found'); }); }); @@ -155,7 +205,7 @@ describe('Evaluate', () => { ); }); - it('valiables with dots', () => { + it('variables with dots', () => { expect(evaluate('foo.bar', { 'foo.bar': 20 })).toEqual(20); expect(evaluate('"is.null"', { 'is.null': null })).toEqual(null); expect(evaluate('"is.false"', { 'is.null': null, 'is.false': false })).toEqual(false); @@ -210,6 +260,10 @@ describe('Evaluate', () => { expect(evaluate('sum("space name")', { 'space name': [1, 2, 21] })).toEqual(24); }); + it('throws on named arguments', () => { + expect(() => evaluate('sum(invalid=a)')).toThrow('Named arguments are not supported'); + }); + it('equations with injected functions', () => { expect( evaluate( diff --git a/packages/kbn-tinymath/tinymath.d.ts b/packages/kbn-tinymath/tinymath.d.ts new file mode 100644 index 00000000000000..6af7678a214657 --- /dev/null +++ b/packages/kbn-tinymath/tinymath.d.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * and the Server Side Public License, v 1; you may not use this file except in + * compliance with, at your election, the Elastic License or the Server Side + * Public License, v 1. + */ + +export function parse(expression: string): TinymathAST; +export function evaluate( + expression: string | null, + context: Record +): number | number[]; + +// Named arguments are not top-level parts of the grammar, but can be nested +export type TinymathAST = number | TinymathVariable | TinymathFunction | TinymathNamedArgument; + +// Zero-indexed location +export interface TinymathLocation { + min: number; + max: number; +} + +export interface TinymathFunction { + type: 'function'; + name: string; + text: string; + args: TinymathAST[]; + location: TinymathLocation; +} + +export interface TinymathVariable { + type: 'variable'; + value: string; + text: string; + location: TinymathLocation; +} + +export interface TinymathNamedArgument { + type: 'namedArgument'; + name: string; + value: string; + text: string; + location: TinymathLocation; +} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts index 0f200a92a41f12..80d9c4ff14a1c3 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// @ts-expect-error no @typed def; Elastic library import { evaluate } from '@kbn/tinymath'; import { pivotObjectArray } from '../../../common/lib/pivot_object_array'; import { Datatable, isDatatable, ExpressionFunctionDefinition } from '../../../types'; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.js b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.ts similarity index 96% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.js rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.ts index 0dbc7ca833f059..a901c3ec7b87da 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.js +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_expression_type.test.ts @@ -10,7 +10,7 @@ import { getExpressionType } from './pointseries/lib/get_expression_type'; describe('getExpressionType', () => { it('returns the result type of an evaluated math expression', () => { expect(getExpressionType(testTable.columns, '2')).toBe('number'); - expect(getExpressionType(testTable.colunns, '2 + 3')).toBe('number'); + expect(getExpressionType(testTable.columns, '2 + 3')).toBe('number'); expect(getExpressionType(testTable.columns, 'name')).toBe('string'); expect(getExpressionType(testTable.columns, 'time')).toBe('date'); expect(getExpressionType(testTable.columns, 'price')).toBe('number'); diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_field_names.test.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_field_names.test.ts index a04f39c66bc26b..1c6ab72ed6e8fa 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_field_names.test.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/get_field_names.test.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// @ts-expect-error untyped library import { parse } from '@kbn/tinymath'; import { getFieldNames } from './pointseries/lib/get_field_names'; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/index.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/index.ts index b528eb63ef2b6c..277fbbb15e6db5 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/index.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/index.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// @ts-expect-error Untyped Elastic library import { evaluate } from '@kbn/tinymath'; import { groupBy, zipObject, omit, uniqBy } from 'lodash'; import moment from 'moment'; @@ -19,7 +18,6 @@ import { import { pivotObjectArray } from '../../../../common/lib/pivot_object_array'; import { unquoteString } from '../../../../common/lib/unquote_string'; import { isColumnReference } from './lib/is_column_reference'; -// @ts-expect-error untyped local import { getExpressionType } from './lib/get_expression_type'; import { getFunctionHelp, getFunctionErrors } from '../../../../i18n'; @@ -131,16 +129,17 @@ export function pointseries(): ExpressionFunctionDefinition< [PRIMARY_KEY]: i, })); - function normalizeValue(expression: string, value: string) { + function normalizeValue(expression: string, value: number | number[], index: number) { + const numberValue = Array.isArray(value) ? value[index] : value; switch (getExpressionType(input.columns, expression)) { case 'string': - return String(value); + return String(numberValue); case 'number': - return Number(value); + return Number(numberValue); case 'date': - return moment(value).valueOf(); + return moment(numberValue).valueOf(); default: - return value; + return numberValue; } } @@ -152,7 +151,7 @@ export function pointseries(): ExpressionFunctionDefinition< (acc: Record, { name, value }) => { try { acc[name] = args[name] - ? normalizeValue(value, evaluate(value, mathScope)[i]) + ? normalizeValue(value, evaluate(value, mathScope), i) : '_all'; } catch (e) { // TODO: handle invalid column names... diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.js b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.ts similarity index 82% rename from x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.js rename to x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.ts index ed1f1d5e6c706d..325b391d677b0e 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.js +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_expression_type.ts @@ -5,11 +5,12 @@ */ import { parse } from '@kbn/tinymath'; +import { DatatableColumn } from 'src/plugins/expressions/common'; import { getFieldType } from '../../../../../common/lib/get_field_type'; import { isColumnReference } from './is_column_reference'; import { getFieldNames } from './get_field_names'; -export function getExpressionType(columns, mathExpression) { +export function getExpressionType(columns: DatatableColumn[], mathExpression: string) { // if isColumnReference returns true, then mathExpression is just a string // referencing a column in a datatable if (isColumnReference(mathExpression)) { @@ -18,7 +19,7 @@ export function getExpressionType(columns, mathExpression) { const parsedMath = parse(mathExpression); - if (parsedMath.args) { + if (typeof parsedMath !== 'number' && parsedMath.type === 'function') { const fieldNames = parsedMath.args.reduce(getFieldNames, []); if (fieldNames.length > 0) { @@ -29,7 +30,7 @@ export function getExpressionType(columns, mathExpression) { } return types; - }, []); + }, [] as string[]); return fieldTypes.length === 1 ? fieldTypes[0] : 'string'; } diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_field_names.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_field_names.ts index 78299b8b6f13a5..025ece2c0d1ece 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_field_names.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/get_field_names.ts @@ -4,21 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -type Arg = - | string - | number - | { - name: string; - args: Arg[]; - }; +import { TinymathAST } from '@kbn/tinymath'; -export function getFieldNames(names: string[], arg: Arg): string[] { - if (typeof arg === 'object' && arg.args !== undefined) { - return names.concat(arg.args.reduce(getFieldNames, [])); +export function getFieldNames(names: string[], ast: TinymathAST): string[] { + if (typeof ast === 'number') { + return names; } - if (typeof arg === 'string') { - return names.concat(arg); + if (ast.type === 'function') { + return names.concat(ast.args.reduce(getFieldNames, [])); + } + + if (ast.type === 'variable') { + return names.concat(ast.value); } return names; diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/is_column_reference.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/is_column_reference.ts index 54e1adbeddd785..63e6bbd1e0af1c 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/is_column_reference.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/server/pointseries/lib/is_column_reference.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// @ts-expect-error untyped library import { parse } from '@kbn/tinymath'; export function isColumnReference(mathExpression: string | null): boolean { @@ -12,5 +11,5 @@ export function isColumnReference(mathExpression: string | null): boolean { mathExpression = 'null'; } const parsedMath = parse(mathExpression); - return typeof parsedMath === 'string'; + return typeof parsedMath !== 'number' && parsedMath.type === 'variable'; } From aaf5664db9943fbada39c308eacb86cf168e6741 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 1 Feb 2021 18:50:11 +0100 Subject: [PATCH 007/185] :sparkles: Add enhanced validation for fields, operations and parameters --- .../operations/definitions/formula.tsx | 153 +++++++++--------- .../operations/definitions/math.tsx | 112 ++++++++++++- .../operations/layer_helpers.ts | 2 +- 3 files changed, 189 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index c1a6768cdca275..8498df63361977 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -4,13 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; import { parse } from '@kbn/tinymath'; import { EuiButton, EuiTextArea } from '@elastic/eui'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; import { getColumnOrder } from '../layer_helpers'; -import { mathOperation } from './math'; +import { mathOperation, hasMathNode, findVariables, sanifyOperationNames } from './math'; export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; @@ -43,26 +44,39 @@ export const formulaOperation: OperationDefinition< return; } const ast = parse(column.params.ast); - const extracted = extractColumns( - columnId, - removeUnderscore(operationDefinitionMap), - ast, - layer, - indexPattern - ); - - const errors = extracted - .flatMap(({ operationType, label }) => { - if (layer.columns[label]) { - return operationDefinitionMap[operationType]?.getErrorMessage?.( - layer, - label, - indexPattern, - operationDefinitionMap + const errors = []; + try { + const flattenAstWithParamsValidation = addParamsValidation( + ast, + sanifyOperationNames(operationDefinitionMap) + ); + for (const node of flattenAstWithParamsValidation) { + const missingParams = (node.params as Array<{ name: string; isMissing: boolean }>) + .filter(({ isMissing }) => isMissing) + .map(({ name }) => name); + if (missingParams.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values: { + operation: node.name, + params: missingParams.join(', '), + }, + }) ); } - }) - .filter(Boolean) as string[]; + } + } catch (e) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: 'The Formula {expression} cannot be parsed', + values: { + expression: ast, + }, + }) + ); + } return errors.length ? errors : undefined; }, getPossibleOperation() { @@ -126,7 +140,7 @@ export const formulaOperation: OperationDefinition< isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable const ast = parse(column.params.ast); - return !hasMathNode(ast, operationDefinitionMap); + return !hasMathNode(ast, sanifyOperationNames(operationDefinitionMap)); }, paramEditor: function ParamEditor({ @@ -177,11 +191,11 @@ export function regenerateLayerFromAst( ) { const ast = parse(text); /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ const extracted = extractColumns( columnId, - removeUnderscore(operationDefinitionMap), + sanifyOperationNames(operationDefinitionMap), ast, layer, indexPattern @@ -224,6 +238,33 @@ export function regenerateLayerFromAst( // set state } +function addParamsValidation(ast: any, operations: Record) { + function validateNodeParams(node: any) { + if (typeof node === 'number' || typeof node === 'string') { + return []; + } + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + return []; + } + if (nodeOperation.input === 'field') { + const [_, ...params] = node.args; + // maybe validate params here? + return [{ ...node, params: validateParams(nodeOperation, params) }]; + } + if (nodeOperation.input === 'fullReference') { + const [fieldName, ...params] = node.args; + // maybe validate params here? + return [ + { ...node, params: validateParams(nodeOperation, params) }, + ...validateNodeParams(fieldName), + ]; + } + throw new Error('unexpected node'); + } + return validateNodeParams(ast); +} + function extractColumns( idPrefix: string, operations: Record, @@ -234,18 +275,11 @@ function extractColumns( const columns: IndexPatternColumn[] = []; // let currentTree: any = cloneDeep(ast); function parseNode(node: any) { - if (typeof node === 'number') { + if (typeof node === 'number' || typeof node === 'string') { // leaf node return node; } - if (typeof node === 'string') { - if (indexPattern.getFieldByName(node)) { - // leaf node - return node; - } - // create a fake node just for making run a validation pass - return parseNode({ name: 'avg', args: [node] }); - } + const nodeOperation = operations[node.name]; if (!nodeOperation) { // it's a regular math node @@ -317,7 +351,7 @@ function extractColumns( return `${idPrefix}X${columns.length - 1}`; } - throw new Error('unexpected node'); + return; } const root = parseNode(ast); const variables = findVariables(root); @@ -334,34 +368,6 @@ function extractColumns( return columns; } -// traverse a tree and find all string leaves -function findVariables(node: any): string[] { - if (typeof node === 'string') { - // leaf node - return [node]; - } - if (typeof node === 'number') { - return []; - } - return node.args.flatMap(findVariables); -} - -function hasMathNode(root: any, operations: Record): boolean { - function findMathNodes(node: any): boolean[] { - if (typeof node === 'string') { - return [false]; - } - if (typeof node === 'number') { - return [false]; - } - if (node.name in operations) { - return [false]; - } - return node.args.flatMap(findMathNodes); - } - return Boolean(findMathNodes(root).filter(Boolean).length); -} - function getSafeFieldName(fieldName: string | undefined) { // clean up the "Records" field for now if (!fieldName || fieldName === 'Records') { @@ -370,6 +376,19 @@ function getSafeFieldName(fieldName: string | undefined) { return fieldName; } +function validateParams( + operation: + | OperationDefinition + | OperationDefinition, + params: unknown[] +) { + const paramsObj = getOperationParams(operation, params); + const formalArgs = operation.operationParams || []; + return formalArgs + .filter(({ required }) => required) + .map(({ name }) => ({ name, isMissing: !(name in paramsObj) })); +} + function getOperationParams( operation: | OperationDefinition @@ -387,15 +406,3 @@ function getOperationParams( return args; }, {}); } - -function removeUnderscore( - operationDefinitionMap: Record -): Record { - return Object.keys(operationDefinitionMap).reduce((memo, operationTypeSnakeCase) => { - const operationType = operationTypeSnakeCase.replace(/([_][a-z])/, (group) => - group.replace('_', '') - ); - memo[operationType] = operationDefinitionMap[operationTypeSnakeCase]; - return memo; - }, {} as Record); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx index 5354fca20fde80..17f58080d60d3b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -4,12 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ import React, { useState } from 'react'; -import { parse } from 'tinymath'; -import { EuiButton, EuiTextArea } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, IndexPatternLayer } from '../../types'; +const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); + export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'math'; params: { @@ -32,8 +33,46 @@ export const mathOperation: OperationDefinition !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ); + // need to check the arguments here: check only strings for now + + errors.push( + i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values: { + variablesLength: missingOperations.length, + variablesList: missingVariables.join(', '), + }, + }) + ); + } + } + return errors.length ? errors : undefined; }, getPossibleOperation() { return { @@ -93,3 +132,68 @@ function astToString(ast: any) { } return `${ast.name}(${ast.args.map(astToString).join(',')})`; } + +export function sanifyOperationNames( + operationDefinitionMap: Record +): Record { + return Object.keys(operationDefinitionMap).reduce((memo, operationTypeSnakeCase) => { + const operationType = operationTypeSnakeCase.replace(/([_][a-z])/, (group) => + group.replace('_', '') + ); + memo[operationType] = operationDefinitionMap[operationTypeSnakeCase]; + return memo; + }, {} as Record); +} + +function findMathNodes(root: any, operations: Record) { + function flattenMathNodes(node: any) { + if (typeof node === 'string' || typeof node === 'number' || operations[node.name]) { + return []; + } + return [node, ...node.args.flatMap(findMathNodes)].filter(Boolean); + } + return flattenMathNodes(root); +} + +export function hasMathNode( + root: any, + operations: Record +): boolean { + return Boolean(findMathNodes(root, operations).length); +} + +function hasInvalidOperations( + node: { name: string; args: any[] }, + operations: Record +) { + // avoid duplicates + return Array.from( + new Set( + findMathNodes(node, operations) + .filter(({ name }) => !tinymathValidOperators.has(name)) + .map(({ name }) => name) + ) + ); +} + +// traverse a tree and find all string leaves +export function findVariables(node: any): string[] { + if (typeof node === 'string') { + // leaf node + return [node]; + } + if (typeof node === 'number') { + return []; + } + return node.args.flatMap(findVariables); +} + +function getMathNode(layer: IndexPatternLayer, ast: string | { name: string; args: any[] }) { + if (typeof ast === 'string') { + const refColumn = layer.columns[ast]; + if (refColumn) { + return refColumn.sourceField; + } + } + return ast; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 8a02ea62ba04e6..692ddfe24d576a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -911,7 +911,7 @@ export function getErrorMessages( .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern); + return def.getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap); } }) // remove the undefined values From 66b7dde5926fe3b9823a7b6bd2ba8166a33ff827 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 1 Feb 2021 14:24:46 -0500 Subject: [PATCH 008/185] Add tsconfig project --- .github/CODEOWNERS | 1 + packages/kbn-tinymath/tsconfig.json | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 packages/kbn-tinymath/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3343544d57fad2..4a1ab25f9977f3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -26,6 +26,7 @@ /src/plugins/vis_type_xy/ @elastic/kibana-app /src/plugins/visualize/ @elastic/kibana-app /src/plugins/visualizations/ @elastic/kibana-app +/packages/kbn-tinymath/ @elastic/kibana-app # Application Services /examples/bfetch_explorer/ @elastic/kibana-app-services diff --git a/packages/kbn-tinymath/tsconfig.json b/packages/kbn-tinymath/tsconfig.json new file mode 100644 index 00000000000000..62a7376efdfa6d --- /dev/null +++ b/packages/kbn-tinymath/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "tsBuildInfoFile": "../../build/tsbuildinfo/packages/kbn-tinymath" + }, + "include": ["tinymath.d.ts"] +} From 0e8a7be9b12706f95d0c3a9403766a22012749a5 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 1 Feb 2021 14:55:35 -0500 Subject: [PATCH 009/185] Fix tests --- .../response_processors/series/math.test.js | 4 +--- ...m_object.test.js => get_form_object.test.ts} | 0 .../{get_form_object.js => get_form_object.ts} | 17 +++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) rename x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/{get_form_object.test.js => get_form_object.test.ts} (100%) rename x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/{get_form_object.js => get_form_object.ts} (71%) diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/math.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/math.test.js index 1aba6816a270d7..431242221006c4 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/math.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/math.test.js @@ -134,9 +134,7 @@ describe('math(resp, panel, series)', () => { series )(await mathAgg(resp, panel, series)((results) => results))([]); } catch (e) { - expect(e.message).toEqual( - 'Failed to parse expression. Expected "*", "+", "-", "/", or end of input but "(" found.' - ); + expect(e.message).toEqual('No such function: notExistingFn'); } }); diff --git a/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.test.js b/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.test.ts similarity index 100% rename from x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.test.js rename to x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.test.ts diff --git a/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.js b/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.ts similarity index 71% rename from x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.js rename to x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.ts index fbde9f7f63f418..888df00cd51cf5 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.js +++ b/x-pack/plugins/canvas/canvas_plugin_src/uis/arguments/datacolumn/get_form_object.ts @@ -8,7 +8,7 @@ import { parse } from '@kbn/tinymath'; import { unquoteString } from '../../../../common/lib/unquote_string'; // break out into separate function, write unit tests first -export function getFormObject(argValue) { +export function getFormObject(argValue: string) { if (argValue === '') { return { fn: '', @@ -19,23 +19,28 @@ export function getFormObject(argValue) { // check if the value is a math expression, and set its type if it is const mathObj = parse(argValue); // A symbol node is a plain string, so we guess that they're looking for a column. - if (typeof mathObj === 'string') { + if (typeof mathObj === 'number') { + throw new Error(`Cannot render scalar values or complex math expressions`); + } + + if (mathObj.type === 'variable') { return { fn: '', - column: unquoteString(argValue), + column: unquoteString(mathObj.value), }; } // Check if its a simple function, eg a function wrapping a symbol node // check for only one arg of type string if ( - typeof mathObj === 'object' && + mathObj.type === 'function' && mathObj.args.length === 1 && - typeof mathObj.args[0] === 'string' + typeof mathObj.args[0] !== 'number' && + mathObj.args[0].type === 'variable' ) { return { fn: mathObj.name, - column: unquoteString(mathObj.args[0]), + column: unquoteString(mathObj.args[0].value), }; } From af6da20a67b89cb5ea13005ca4203e0cdf030629 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 2 Feb 2021 13:02:47 +0100 Subject: [PATCH 010/185] :sparkles: Add id and name (optional) decoupling --- .../functions/common/mapColumn.test.js | 10 ++++++++++ .../functions/common/mapColumn.ts | 19 +++++++++++++++---- .../canvas/i18n/functions/dict/map_column.ts | 4 ++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js index cc360a48c7f566..3419e65b23c4eb 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js @@ -53,6 +53,16 @@ describe('mapColumn', () => { }); }); + it('should assign specific id, different from name, when id arg is passed', () => { + return fn(emptyTable, { name: 'name', id: 'myid', expression: pricePlusTwo }).then((result) => { + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(1); + expect(result.columns[0]).toHaveProperty('name', 'name'); + expect(result.columns[0]).toHaveProperty('id', 'myid'); + expect(result.columns[0].meta).toHaveProperty('type', 'null'); + }); + }); + describe('expression', () => { it('maps null values to the new column', () => { return fn(testTable, { name: 'empty' }).then((result) => { diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts index 6d6a432e5553ec..a416b7f364211a 100644 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts +++ b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts @@ -8,6 +8,7 @@ import { Datatable, ExpressionFunctionDefinition, getType } from '../../../types import { getFunctionHelp } from '../../../i18n'; interface Arguments { + id: string | null; name: string; expression: (datatable: Datatable) => Promise; } @@ -27,6 +28,12 @@ export function mapColumn(): ExpressionFunctionDefinition< inputTypes: ['datatable'], help, args: { + id: { + types: ['string', 'null'], + help: argHelp.id, + required: false, + default: null, + }, name: { types: ['string'], aliases: ['_', 'column'], @@ -43,7 +50,7 @@ export function mapColumn(): ExpressionFunctionDefinition< }, fn: (input, args) => { const expression = args.expression || (() => Promise.resolve(null)); - + const columnId = args.id != null ? args.id : args.name; const columns = [...input.columns]; const rowPromises = input.rows.map((row) => { return expression({ @@ -52,14 +59,18 @@ export function mapColumn(): ExpressionFunctionDefinition< rows: [row], }).then((val) => ({ ...row, - [args.name]: val, + [columnId]: val, })); }); return Promise.all(rowPromises).then((rows) => { const existingColumnIndex = columns.findIndex(({ name }) => name === args.name); - const type = rows.length ? getType(rows[0][args.name]) : 'null'; - const newColumn = { id: args.name, name: args.name, meta: { type } }; + const type = rows.length ? getType(rows[0][columnId]) : 'null'; + const newColumn = { + id: columnId, + name: args.name, + meta: { type }, + }; if (existingColumnIndex === -1) { columns.push(newColumn); diff --git a/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts b/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts index 2666a08999fb8c..2c1cf992bdab3f 100644 --- a/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts +++ b/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts @@ -22,6 +22,10 @@ export const help: FunctionHelp> = { }, }), args: { + id: i18n.translate('xpack.canvas.functions.mapColumn.args.idHelpText', { + defaultMessage: + 'An optional id of the resulting column. When `null` the name argument is used as id.', + }), name: i18n.translate('xpack.canvas.functions.mapColumn.args.nameHelpText', { defaultMessage: 'The name of the resulting column.', }), From 7e5b82583c7b322ad2c927310422cb6d8bcbb038 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 2 Feb 2021 13:05:38 +0100 Subject: [PATCH 011/185] :sparkles: Give Formula column custom label --- .../operations/definitions/formula.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 8498df63361977..1ab243fb1bec7c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -92,7 +92,8 @@ export const formulaOperation: OperationDefinition< type: 'function', function: 'mapColumn', arguments: { - name: [columnId], + id: [columnId], + name: [layer.columns[columnId].label], exp: [ { type: 'expression', @@ -217,6 +218,7 @@ export function regenerateLayerFromAst( columns[columnId] = { ...currentColumn, + label: text, params: { ...currentColumn.params, ast: text, @@ -348,7 +350,7 @@ function extractColumns( newCol.label = newColId; columns.push(newCol); // replace by new column id - return `${idPrefix}X${columns.length - 1}`; + return newColId; } return; From bde55d3839af237ded614e31e0766738cacb0b58 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 2 Feb 2021 20:40:17 -0500 Subject: [PATCH 012/185] Allow named arguments with numeric types --- packages/kbn-tinymath/src/grammar.js | 5 ++++- packages/kbn-tinymath/src/grammar.pegjs | 2 +- packages/kbn-tinymath/test/library.test.js | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/kbn-tinymath/src/grammar.js b/packages/kbn-tinymath/src/grammar.js index 9b3f3320ff0498..635e580036cf7e 100644 --- a/packages/kbn-tinymath/src/grammar.js +++ b/packages/kbn-tinymath/src/grammar.js @@ -1158,7 +1158,10 @@ function peg$parse(input, options) { if (s3 !== peg$FAILED) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { - s5 = peg$parseString(); + s5 = peg$parseNumber(); + if (s5 === peg$FAILED) { + s5 = peg$parseString(); + } if (s5 !== peg$FAILED) { s6 = peg$parse_(); if (s6 !== peg$FAILED) { diff --git a/packages/kbn-tinymath/src/grammar.pegjs b/packages/kbn-tinymath/src/grammar.pegjs index f0161d3dad38d2..f5280586cc219e 100644 --- a/packages/kbn-tinymath/src/grammar.pegjs +++ b/packages/kbn-tinymath/src/grammar.pegjs @@ -109,7 +109,7 @@ String Argument - = name:[a-zA-Z_-]+ _ '=' _ value:String _ { + = name:[a-zA-Z_-]+ _ '=' _ value:(Number / String) _ { return { type: 'namedArgument', name: name.join(''), diff --git a/packages/kbn-tinymath/test/library.test.js b/packages/kbn-tinymath/test/library.test.js index da2dbd42f262cb..92ddec915a9c7e 100644 --- a/packages/kbn-tinymath/test/library.test.js +++ b/packages/kbn-tinymath/test/library.test.js @@ -129,7 +129,13 @@ describe('Parser', () => { }); it('named only', () => { - expect(parse('foo(q=10)')).toEqual(functionEqual('foo', [namedArgumentEqual('q', '10')])); + expect(parse('foo(q=10)')).toEqual(functionEqual('foo', [namedArgumentEqual('q', 10)])); + }); + + it('named argument is numeric', () => { + expect(parse('foo(q=10.1234e5)')).toEqual( + functionEqual('foo', [namedArgumentEqual('q', 10.1234e5)]) + ); }); it('named and positional', () => { From bf8f8c46f7b61831b60047b26012245407c7dec3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Feb 2021 15:33:50 +0100 Subject: [PATCH 013/185] :label: Migrate AST to new format + types fix + extended validation --- .../dimension_panel/reference_editor.tsx | 7 +- .../definitions/date_histogram.test.tsx | 1 + .../definitions/filters/filters.test.tsx | 1 + .../operations/definitions/formula.tsx | 315 ++++++++++++------ .../operations/definitions/helpers.tsx | 2 +- .../operations/definitions/index.ts | 3 +- .../definitions/last_value.test.tsx | 1 + .../operations/definitions/math.tsx | 60 ++-- .../definitions/percentile.test.tsx | 1 + .../definitions/ranges/ranges.test.tsx | 1 + .../definitions/terms/terms.test.tsx | 1 + .../operations/layer_helpers.ts | 26 +- 12 files changed, 284 insertions(+), 135 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index 6d1dfe375111fc..961c59068311b4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -39,7 +39,7 @@ const operationPanels = getOperationDisplay(); export interface ReferenceEditorProps { layer: IndexPatternLayer; - selectionStyle: 'full' | 'field'; + selectionStyle: 'full' | 'field' | 'hidden'; validation: RequiredReference; columnId: string; updateLayer: (newLayer: IndexPatternLayer) => void; @@ -191,6 +191,10 @@ export function ReferenceEditor(props: ReferenceEditorProps) { return; } + if (selectionStyle === 'hidden') { + return null; + } + const selectedOption = incompleteOperation ? [functionOptions.find(({ value }) => value === incompleteOperation)!] : column @@ -323,6 +327,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { columnId={columnId} indexPattern={currentIndexPattern} dateRange={dateRange} + operationDefinitionMap={operationDefinitionMap} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index abd033c0db4cfc..f22428deeab238 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -94,6 +94,7 @@ const defaultOptions = { data: dataStart, http: {} as HttpSetup, indexPattern: indexPattern1, + operationDefinitionMap: {}, }; describe('date_histogram', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index 86767fbc8b4691..ff1b05586ce8e0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -24,6 +24,7 @@ const defaultProps = { data: dataPluginMock.createStartContract(), http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), + operationDefinitionMap: {}, }; // mocking random id generator function diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 1ab243fb1bec7c..1c2c92b188cea0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -5,18 +5,33 @@ */ import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { parse } from '@kbn/tinymath'; +import { groupBy, isObject } from 'lodash'; +import { parse, TinymathFunction, TinymathVariable } from '@kbn/tinymath'; +import type { TinymathNamedArgument, TinymathAST } from '@kbn/tinymath'; import { EuiButton, EuiTextArea } from '@elastic/eui'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; +import { IndexPattern, IndexPatternLayer } from '../../types'; import { getColumnOrder } from '../layer_helpers'; -import { mathOperation, hasMathNode, findVariables, sanifyOperationNames } from './math'; +import { mathOperation, hasMathNode, findVariables } from './math'; + +type GroupedNodes = { + [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; +} & + { + [Key in TinymathVariable['type']]: Array; + } & + { + [Key in TinymathFunction['type']]: TinymathFunction[]; + }; + +type TinymathNodeTypes = Exclude; export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; params: { - ast?: unknown; + formula?: string; + isFormulaBroken?: boolean; // last value on numeric fields can be formatted format?: { id: string; @@ -40,43 +55,23 @@ export const formulaOperation: OperationDefinition< }, getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) { const column = layer.columns[columnId] as FormulaIndexPatternColumn; - if (!column.params.ast || !operationDefinitionMap) { + if (!column.params.formula || !operationDefinitionMap) { return; } - const ast = parse(column.params.ast); - const errors = []; + let ast; try { - const flattenAstWithParamsValidation = addParamsValidation( - ast, - sanifyOperationNames(operationDefinitionMap) - ); - for (const node of flattenAstWithParamsValidation) { - const missingParams = (node.params as Array<{ name: string; isMissing: boolean }>) - .filter(({ isMissing }) => isMissing) - .map(({ name }) => name); - if (missingParams.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { - operation: node.name, - params: missingParams.join(', '), - }, - }) - ); - } - } + ast = parse(column.params.formula); } catch (e) { - errors.push( + return [ i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The Formula {expression} cannot be parsed', values: { - expression: ast, + expression: column.params.formula, }, - }) - ); + }), + ]; } + const errors = addASTValidation(ast, indexPattern, operationDefinitionMap); return errors.length ? errors : undefined; }, getPossibleOperation() { @@ -119,13 +114,17 @@ export const formulaOperation: OperationDefinition< if (previousColumn) { if ('references' in previousColumn) { const metric = layer.columns[previousColumn.references[0]]; - const fieldName = getSafeFieldName(metric?.sourceField); - // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; + if (metric && 'sourceField' in metric) { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; + } } else { - previousFormula += `${previousColumn.operationType}(${getSafeFieldName( - previousColumn?.sourceField - )})`; + if (previousColumn && 'sourceField' in previousColumn) { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )})`; + } } } return { @@ -134,14 +133,14 @@ export const formulaOperation: OperationDefinition< operationType: 'formula', isBucketed: false, scale: 'ratio', - params: previousFormula ? { ast: previousFormula } : {}, + params: previousFormula ? { formula: previousFormula, isFormulaBroken: false } : {}, references: [], }; }, isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable - const ast = parse(column.params.ast); - return !hasMathNode(ast, sanifyOperationNames(operationDefinitionMap)); + const ast = parse(column.params.formula || ''); + return !hasMathNode(ast, operationDefinitionMap); }, paramEditor: function ParamEditor({ @@ -152,7 +151,7 @@ export const formulaOperation: OperationDefinition< indexPattern, operationDefinitionMap, }) { - const [text, setText] = useState(currentColumn.params.ast); + const [text, setText] = useState(currentColumn.params.formula); return ( <> { updateLayer( regenerateLayerFromAst( - text, + text || '', layer, columnId, currentColumn, @@ -182,26 +181,40 @@ export const formulaOperation: OperationDefinition< }, }; +function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + try { + const ast = parse(text); + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + return { extracted, hasError: false }; + } catch (e) { + return { extracted: [], hasError: true }; + } +} + export function regenerateLayerFromAst( - text: unknown, + text: string, layer: IndexPatternLayer, columnId: string, currentColumn: FormulaIndexPatternColumn, indexPattern: IndexPattern, operationDefinitionMap: Record ) { - const ast = parse(text); - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns( - columnId, - sanifyOperationNames(operationDefinitionMap), - ast, + const { extracted, hasError } = parseAndExtract( + text, layer, - indexPattern + columnId, + indexPattern, + operationDefinitionMap ); - const columns = { ...layer.columns, }; @@ -218,12 +231,12 @@ export function regenerateLayerFromAst( columns[columnId] = { ...currentColumn, - label: text, params: { ...currentColumn.params, - ast: text, + formula: text, + isFormulaBroken: hasError, }, - references: [`${columnId}X${extracted.length - 1}`], + references: hasError ? [] : [`${columnId}X${extracted.length - 1}`], }; return { @@ -240,44 +253,134 @@ export function regenerateLayerFromAst( // set state } -function addParamsValidation(ast: any, operations: Record) { - function validateNodeParams(node: any) { - if (typeof node === 'number' || typeof node === 'string') { +function addASTValidation( + ast: TinymathAST, + indexPattern: IndexPattern, + operations: Record +) { + function validateNode(node: TinymathAST): string[] { + if (!isObject(node) || node.type !== 'function') { return []; } const nodeOperation = operations[node.name]; if (!nodeOperation) { return []; } + + const errors: string[] = []; + const { namedArguments, variables, functions } = groupArgsByType(node.args); + if (nodeOperation.input === 'field') { - const [_, ...params] = node.args; - // maybe validate params here? - return [{ ...node, params: validateParams(nodeOperation, params) }]; + if (!isFirstArgumentValidType(node.args, 'variable')) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(node.args[0]), + }, + }) + ); + } + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + if (!indexPattern.getFieldByName(fieldName.value)) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { + defaultMessage: 'The field {field} was not found.', + values: { + field: fieldName.value, + }, + }) + ); + } + const missingParameters = validateParams(nodeOperation, namedArguments).filter( + ({ isMissing }) => isMissing + ); + if (missingParameters.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values: { + operation: node.name, + params: missingParameters.join(', '), + }, + }) + ); + } + return errors; } if (nodeOperation.input === 'fullReference') { - const [fieldName, ...params] = node.args; + if (!isFirstArgumentValidType(node.args, 'function')) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(node.args[0]), + }, + }) + ); + } + const missingParameters = validateParams(nodeOperation, namedArguments).filter( + ({ isMissing }) => isMissing + ); + if (missingParameters.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values: { + operation: node.name, + params: missingParameters.join(', '), + }, + }) + ); + } // maybe validate params here? - return [ - { ...node, params: validateParams(nodeOperation, params) }, - ...validateNodeParams(fieldName), - ]; + return errors.concat(validateNode(functions[0])); } - throw new Error('unexpected node'); + return []; + } + return validateNode(ast); +} + +function getValueOrName(node: TinymathAST) { + if (!isObject(node)) { + return node; + } + if (node.type !== 'function') { + return node.value; } - return validateNodeParams(ast); + return node.name; +} + +function groupArgsByType(args: TinymathAST[]) { + const { namedArgument, variable, function: functions } = groupBy( + args, + (arg: TinymathAST) => { + return isObject(arg) ? arg.type : 'variable'; + } + ) as GroupedNodes; + // better naming + return { namedArguments: namedArgument, variables: variable, functions }; } function extractColumns( idPrefix: string, operations: Record, - ast: any, + ast: TinymathAST, layer: IndexPatternLayer, indexPattern: IndexPattern ) { const columns: IndexPatternColumn[] = []; - // let currentTree: any = cloneDeep(ast); - function parseNode(node: any) { - if (typeof node === 'number' || typeof node === 'string') { + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { // leaf node return node; } @@ -285,19 +388,32 @@ function extractColumns( const nodeOperation = operations[node.name]; if (!nodeOperation) { // it's a regular math node - const consumedArgs = node.args.map((childNode: any) => parseNode(childNode)); + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; return { ...node, args: consumedArgs, }; } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + // operation node if (nodeOperation.input === 'field') { - const [fieldName, ...params] = node.args; + if (!isFirstArgumentValidType(node.args, 'variable')) { + throw Error('field as first argument not found'); + } + + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + const field = indexPattern.getFieldByName(fieldName.value); - const field = indexPattern.getFieldByName(fieldName); + if (!field) { + throw Error('field not found'); + } - const mappedParams = getOperationParams(nodeOperation, params || []); + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); const newCol = (nodeOperation as OperationDefinition< IndexPatternColumn, @@ -306,7 +422,7 @@ function extractColumns( { layer, indexPattern, - field: field ?? ({ displayName: fieldName, name: fieldName } as IndexPatternField), + field, }, mappedParams ); @@ -319,21 +435,24 @@ function extractColumns( } if (nodeOperation.input === 'fullReference') { - const [referencedOp, ...params] = node.args; - + if (!isFirstArgumentValidType(node.args, 'function')) { + throw Error('first argument not valid for full reference'); + } + const [referencedOp] = functions; const consumedParam = parseNode(referencedOp); - const variables = findVariables(consumedParam); + + const subNodeVariables = findVariables(consumedParam); const mathColumn = mathOperation.buildColumn({ layer, indexPattern, }); - mathColumn.references = variables; + mathColumn.references = subNodeVariables; mathColumn.params.tinymathAst = consumedParam; columns.push(mathColumn); mathColumn.customLabel = true; mathColumn.label = `${idPrefix}X${columns.length - 1}`; - const mappedParams = getOperationParams(nodeOperation, params || []); + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); const newCol = (nodeOperation as OperationDefinition< IndexPatternColumn, 'fullReference' @@ -353,7 +472,7 @@ function extractColumns( return newColId; } - return; + throw Error('unexpected node'); } const root = parseNode(ast); const variables = findVariables(root); @@ -382,7 +501,7 @@ function validateParams( operation: | OperationDefinition | OperationDefinition, - params: unknown[] + params: TinymathNamedArgument[] ) { const paramsObj = getOperationParams(operation, params); const formalArgs = operation.operationParams || []; @@ -395,16 +514,24 @@ function getOperationParams( operation: | OperationDefinition | OperationDefinition, - params: unknown[] -) { - // TODO: to be converted with named params - const formalArgs = operation.operationParams || []; + params: TinymathNamedArgument[] +): Record { + const formalArgs: Record = (operation.operationParams || []).reduce( + (memo: Record, { name, type }) => { + memo[name] = type; + return memo; + }, + {} + ); // At the moment is positional as expressed in operationParams - return params.reduce((args, param, i) => { - if (formalArgs[i]) { - const paramName = formalArgs[i].name; - args[paramName] = param; + return params.reduce>((args, { name, value }) => { + if (formalArgs[name]) { + args[name] = value; } return args; }, {}); } + +function isFirstArgumentValidType(args: TinymathAST[], type: TinymathNodeTypes['type']) { + return args?.length >= 1 && isObject(args[0]) && args[0].type === type; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx index d2e2eef7d51285..0f201eb4cddffa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx @@ -7,7 +7,7 @@ import { useRef } from 'react'; import useDebounce from 'react-use/lib/useDebounce'; import { i18n } from '@kbn/i18n'; -import { GenericOperationDefinition, IndexPatternColumn, operationDefinitionMap } from '.'; +import { IndexPatternColumn, operationDefinitionMap } from '.'; import { FieldBasedIndexPatternColumn } from './column_types'; import { IndexPattern } from '../../types'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 87a15fc0de3f67..5c0cc6d48c5df8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -333,7 +333,8 @@ interface FieldBasedOperationDefinition { getErrorMessage: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + operationDefinitionMap?: Record ) => string[] | undefined; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index 96b12a714e613d..a865bdd2bdb0f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -26,6 +26,7 @@ const defaultProps = { ...createMockedIndexPattern(), hasRestrictions: false, } as IndexPattern, + operationDefinitionMap: {}, }; describe('last_value', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx index 17f58080d60d3b..d7045b0e21fc5d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -3,9 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; +import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import { isObject } from 'lodash'; +import { OperationDefinition, GenericOperationDefinition } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, IndexPatternLayer } from '../../types'; @@ -14,7 +15,7 @@ const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide'] export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'math'; params: { - tinymathAst: any; + tinymathAst: TinymathAST | string; // last value on numeric fields can be formatted format?: { id: string; @@ -115,7 +116,7 @@ export const mathOperation: OperationDefinition -): Record { - return Object.keys(operationDefinitionMap).reduce((memo, operationTypeSnakeCase) => { - const operationType = operationTypeSnakeCase.replace(/([_][a-z])/, (group) => - group.replace('_', '') - ); - memo[operationType] = operationDefinitionMap[operationTypeSnakeCase]; - return memo; - }, {} as Record); -} - -function findMathNodes(root: any, operations: Record) { - function flattenMathNodes(node: any) { - if (typeof node === 'string' || typeof node === 'number' || operations[node.name]) { +function findMathNodes( + root: TinymathAST | string, + operations: Record +): TinymathFunction[] { + function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function' || operations[node.name]) { return []; } - return [node, ...node.args.flatMap(findMathNodes)].filter(Boolean); + return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean); } return flattenMathNodes(root); } export function hasMathNode( - root: any, + root: TinymathAST, operations: Record ): boolean { return Boolean(findMathNodes(root, operations).length); } function hasInvalidOperations( - node: { name: string; args: any[] }, + node: TinymathAST | string, operations: Record ) { // avoid duplicates @@ -177,21 +175,27 @@ function hasInvalidOperations( } // traverse a tree and find all string leaves -export function findVariables(node: any): string[] { +export function findVariables(node: TinymathAST | string | undefined): string[] { + if (node == null) { + return []; + } if (typeof node === 'string') { - // leaf node return [node]; } - if (typeof node === 'number') { + if (typeof node === 'number' || node.type === 'namedArgument') { return []; } + if (node.type === 'variable') { + // leaf node + return [node.value]; + } return node.args.flatMap(findVariables); } -function getMathNode(layer: IndexPatternLayer, ast: string | { name: string; args: any[] }) { +function getMathNode(layer: IndexPatternLayer, ast: TinymathAST | string) { if (typeof ast === 'string') { const refColumn = layer.columns[ast]; - if (refColumn) { + if (refColumn && 'sourceField' in refColumn) { return refColumn.sourceField; } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index c22eec62ea1ab9..4f46b142687464 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -28,6 +28,7 @@ const defaultProps = { ...createMockedIndexPattern(), hasRestrictions: false, } as IndexPattern, + operationDefinitionMap: {}, }; describe('percentile', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx index 987c8971aa3109..d7a789ed740387 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx @@ -74,6 +74,7 @@ const defaultOptions = { { name: sourceField, type: 'number', displayName: sourceField }, ]), }, + operationDefinitionMap: {}, }; describe('ranges', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index d60992bda2e2a7..efed8f2c57e69c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -25,6 +25,7 @@ const defaultProps = { data: dataPluginMock.createStartContract(), http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), + operationDefinitionMap: {}, }; describe('terms', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 692ddfe24d576a..ea8bc5eebdf6a1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -315,16 +315,22 @@ export function replaceColumn({ const basicLayer = { ...tempLayer, columns: { ...tempLayer.columns, [columnId]: newColumn } }; // rebuild the references again for the specific AST generated - const newLayer = newColumn.params?.ast - ? regenerateLayerFromAst( - newColumn.params.ast, - basicLayer, - columnId, - newColumn, - indexPattern, - operationDefinitionMap - ) - : basicLayer; + let newLayer; + + try { + newLayer = newColumn.params.formula + ? regenerateLayerFromAst( + newColumn.params.formula, + basicLayer, + columnId, + newColumn, + indexPattern, + operationDefinitionMap + ) + : basicLayer; + } catch (e) { + newLayer = basicLayer; + } return updateDefaultLabels( { From d02ee5254d7a012e90662ae2117f34d992a96f20 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Feb 2021 15:45:15 +0100 Subject: [PATCH 014/185] :bug: Improved validation --- .../operations/definitions/formula.tsx | 27 ++++++++++--------- .../operations/definitions/math.tsx | 23 ++++++++-------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 1c2c92b188cea0..f3ef1be2c5b3a8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -283,17 +283,18 @@ function addASTValidation( }, }) ); - } - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - if (!indexPattern.getFieldByName(fieldName.value)) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { - defaultMessage: 'The field {field} was not found.', - values: { - field: fieldName.value, - }, - }) - ); + } else { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + if (!indexPattern.getFieldByName(fieldName.value)) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { + defaultMessage: 'The field {field} was not found.', + values: { + field: fieldName.value, + }, + }) + ); + } } const missingParameters = validateParams(nodeOperation, namedArguments).filter( ({ isMissing }) => isMissing @@ -501,7 +502,7 @@ function validateParams( operation: | OperationDefinition | OperationDefinition, - params: TinymathNamedArgument[] + params: TinymathNamedArgument[] = [] ) { const paramsObj = getOperationParams(operation, params); const formalArgs = operation.operationParams || []; @@ -514,7 +515,7 @@ function getOperationParams( operation: | OperationDefinition | OperationDefinition, - params: TinymathNamedArgument[] + params: TinymathNamedArgument[] = [] ): Record { const formalArgs: Record = (operation.operationParams || []).reduce( (memo: Record, { name, type }) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx index d7045b0e21fc5d..224d267db0c577 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -60,17 +60,18 @@ export const mathOperation: OperationDefinition !indexPattern.getFieldByName(variable) && !layer.columns[variable] ); // need to check the arguments here: check only strings for now - - errors.push( - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: - '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values: { - variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), - }, - }) - ); + if (missingVariables.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values: { + variablesLength: missingOperations.length, + variablesList: missingVariables.join(', '), + }, + }) + ); + } } } return errors.length ? errors : undefined; From 03b1bd4b6899c6b61ab793ed4328569adad9456f Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Feb 2021 18:08:38 +0100 Subject: [PATCH 015/185] :white_check_mark: Add first batch of validation tests --- .../operations/definitions/formula.test.tsx | 286 ++++++++++++++++++ .../operations/definitions/formula.tsx | 87 ++++-- .../operations/definitions/index.ts | 1 + 3 files changed, 347 insertions(+), 27 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx new file mode 100644 index 00000000000000..88d477b06c63d5 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; +import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; +import { createMockedIndexPattern } from '../../mocks'; +import { FormulaIndexPatternColumn } from './formula'; +import { formulaOperation, GenericOperationDefinition } from './index'; +import type { IndexPattern, IndexPatternLayer } from '../../types'; + +const defaultProps = { + storage: {} as IStorageWrapper, + uiSettings: {} as IUiSettingsClient, + savedObjectsClient: {} as SavedObjectsClientContract, + dateRange: { fromDate: 'now-1d', toDate: 'now' }, + data: dataPluginMock.createStartContract(), + http: {} as HttpSetup, + indexPattern: { + ...createMockedIndexPattern(), + hasRestrictions: false, + } as IndexPattern, + operationDefinitionMap: { avg: {} }, +}; + +describe('formula', () => { + let layer: IndexPatternLayer; + const InlineOptions = formulaOperation.paramEditor!; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }; + }); + + describe('getErrorMessage', () => { + let indexPattern: IndexPattern; + let operationDefinitionMap: Record; + + function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer { + return { + columns: { + col1: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula, isFormulaBroken: isBroken }, + references: [], + }, + }, + columnOrder: [], + indexPatternId: '', + }; + } + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + operationDefinitionMap = { + avg: { input: 'field' } as GenericOperationDefinition, + count: { input: 'field' } as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: { + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + } as GenericOperationDefinition, + }; + }); + + it('returns undefined if count is passed without arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('count()'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a field operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('avg(bytes)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + // note that field names can be wrapped in quotes as well + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('avg("bytes")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(avg(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(avg("bytes"))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg(bytes), window=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg("bytes"), "window"=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns an error if field is used with no Lens wrapping operation', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The field bytes cannot be used without operation`]); + }); + + it('returns an error if parsing a syntax invalid formula', () => { + const formulas = [ + '+', + 'avg((', + 'avg((bytes)', + 'avg(bytes) +', + 'avg(""', + 'moving_average(avg(bytes), window=)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The Formula ${formula} cannot be parsed`]); + } + }); + + it('returns an error for scenarios where the field is missing', () => { + // noField, avg(noField), noField + 1, moving_average(noField), moving_average(avg(noField)) + const formulas = ['noField', 'avg(noField)', 'noField + 1', 'moving_average(avg(noField))']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The field noField was not found.']); + } + }); + + it('returns an error if field operation in formula have the wrong first argument', () => { + const formulas = [ + 'avg(7)', + 'avg()', + 'avg(avg(bytes))', + 'avg(1 + 2)', + 'avg(bytes + 5)', + 'derivatives(7)', + 'derivatives(7 + 1)', + 'derivatives(bytes + 7)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + expect.stringMatching( + `The first argument for ${formula.substring(0, formula.indexOf('('))}` + ), + ]); + } + }); + + it('returns an error if an argument is passed to count() operation', () => { + const formulas = ['count(7)', 'count("")', 'count(Records)', 'count(bytes)']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation count does not accept any parameter']); + } + }); + + it('returns an error if an operation with required parameters does not receive them', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg(bytes), myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + }); + + it('returns an error if a parameter is passed to an operation with no parameters', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('avg(bytes, myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation avg in the Formula requires no parameter']); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index f3ef1be2c5b3a8..4e64022dc182ff 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -14,6 +14,7 @@ import { ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, IndexPatternLayer } from '../../types'; import { getColumnOrder } from '../layer_helpers'; import { mathOperation, hasMathNode, findVariables } from './math'; +import { documentField } from '../../document_field'; type GroupedNodes = { [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; @@ -194,9 +195,9 @@ function parseAndExtract( { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } */ const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); - return { extracted, hasError: false }; + return { extracted, isValid: true }; } catch (e) { - return { extracted: [], hasError: true }; + return { extracted: [], isValid: false }; } } @@ -208,13 +209,14 @@ export function regenerateLayerFromAst( indexPattern: IndexPattern, operationDefinitionMap: Record ) { - const { extracted, hasError } = parseAndExtract( + const { extracted, isValid } = parseAndExtract( text, layer, columnId, indexPattern, operationDefinitionMap ); + const columns = { ...layer.columns, }; @@ -234,9 +236,9 @@ export function regenerateLayerFromAst( params: { ...currentColumn.params, formula: text, - isFormulaBroken: hasError, + isFormulaBroken: !isValid, }, - references: hasError ? [] : [`${columnId}X${extracted.length - 1}`], + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], }; return { @@ -271,26 +273,41 @@ function addASTValidation( const { namedArguments, variables, functions } = groupArgsByType(node.args); if (nodeOperation.input === 'field') { - if (!isFirstArgumentValidType(node.args, 'variable')) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { - defaultMessage: - 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { - operation: node.name, - type: 'field', - argument: getValueOrName(node.args[0]), - }, - }) - ); - } else { + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(node.args, 'variable')) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(node.args[0]), + }, + }) + ); + } + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - if (!indexPattern.getFieldByName(fieldName.value)) { + if (fieldName) { + if (!indexPattern.getFieldByName(fieldName.value)) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { + defaultMessage: 'The field {field} was not found.', + values: { + field: fieldName.value, + }, + }) + ); + } + } + } else { + if (node?.args[0]) { errors.push( i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { - defaultMessage: 'The field {field} was not found.', + defaultMessage: 'The operation {operation} does not accept any parameter', values: { - field: fieldName.value, + operation: node.name, }, }) ); @@ -306,7 +323,7 @@ function addASTValidation( 'The operation {operation} in the Formula is missing the following parameters: {params}', values: { operation: node.name, - params: missingParameters.join(', '), + params: missingParameters.map(({ name }) => name).join(', '), }, }) ); @@ -337,7 +354,7 @@ function addASTValidation( 'The operation {operation} in the Formula is missing the following parameters: {params}', values: { operation: node.name, - params: missingParameters.join(', '), + params: missingParameters.map(({ name }) => name).join(', '), }, }) ); @@ -368,7 +385,11 @@ function groupArgsByType(args: TinymathAST[]) { } ) as GroupedNodes; // better naming - return { namedArguments: namedArgument, variables: variable, functions }; + return { + namedArguments: namedArgument || [], + variables: variable || [], + functions: functions || [], + }; } function extractColumns( @@ -403,12 +424,20 @@ function extractColumns( // operation node if (nodeOperation.input === 'field') { - if (!isFirstArgumentValidType(node.args, 'variable')) { - throw Error('field as first argument not found'); + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(node.args, 'variable')) { + throw Error('field as first argument not found'); + } + } else { + if (node?.args[0]) { + throw Error('field as first argument not valid'); + } } const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - const field = indexPattern.getFieldByName(fieldName.value); + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value) + : documentField; if (!field) { throw Error('field not found'); @@ -533,6 +562,10 @@ function getOperationParams( }, {}); } +function shouldHaveFieldArgument(node: TinymathFunction) { + return !['count'].includes(node.name); +} + function isFirstArgumentValidType(args: TinymathAST[], type: TinymathNodeTypes['type']) { return args?.length >= 1 && isObject(args[0]) && args[0].type === type; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 5c0cc6d48c5df8..f071cb2e604498 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -114,6 +114,7 @@ export { derivativeOperation, movingAverageOperation, } from './calculations'; +export { formulaOperation } from './formula'; /** * Properties passed to the operation-specific part of the popover editor From b5ee6abc3d1d4198aff82bacfd2da4bd1369e6b7 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Feb 2021 19:41:56 +0100 Subject: [PATCH 016/185] :white_check_mark: Add more validation tests for formula --- .../operations/definitions/formula.test.tsx | 113 ++++++++---- .../operations/definitions/formula.tsx | 163 +++++++++++++----- .../operations/definitions/math.tsx | 89 +++------- 3 files changed, 219 insertions(+), 146 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx index 88d477b06c63d5..d52f3f36b4fef6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx @@ -4,31 +4,31 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; -import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; -import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; +// import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +// import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; +// import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../mocks'; -import { FormulaIndexPatternColumn } from './formula'; +// import { FormulaIndexPatternColumn } from './formula'; import { formulaOperation, GenericOperationDefinition } from './index'; import type { IndexPattern, IndexPatternLayer } from '../../types'; -const defaultProps = { - storage: {} as IStorageWrapper, - uiSettings: {} as IUiSettingsClient, - savedObjectsClient: {} as SavedObjectsClientContract, - dateRange: { fromDate: 'now-1d', toDate: 'now' }, - data: dataPluginMock.createStartContract(), - http: {} as HttpSetup, - indexPattern: { - ...createMockedIndexPattern(), - hasRestrictions: false, - } as IndexPattern, - operationDefinitionMap: { avg: {} }, -}; +// const defaultProps = { +// storage: {} as IStorageWrapper, +// uiSettings: {} as IUiSettingsClient, +// savedObjectsClient: {} as SavedObjectsClientContract, +// dateRange: { fromDate: 'now-1d', toDate: 'now' }, +// data: dataPluginMock.createStartContract(), +// http: {} as HttpSetup, +// indexPattern: { +// ...createMockedIndexPattern(), +// hasRestrictions: false, +// } as IndexPattern, +// operationDefinitionMap: { avg: {} }, +// }; describe('formula', () => { let layer: IndexPatternLayer; - const InlineOptions = formulaOperation.paramEditor!; + // const InlineOptions = formulaOperation.paramEditor!; beforeEach(() => { layer = { @@ -146,14 +146,15 @@ describe('formula', () => { ) ).toEqual(undefined); - expect( - formulaOperation.getErrorMessage!( - getNewLayerWithFormula('moving_average(avg("bytes"), "window"=7)'), - 'col1', - indexPattern, - operationDefinitionMap - ) - ).toEqual(undefined); + // Not sure it will be supported + // expect( + // formulaOperation.getErrorMessage!( + // getNewLayerWithFormula('moving_average(avg("bytes"), "window"=7)'), + // 'col1', + // indexPattern, + // operationDefinitionMap + // ) + // ).toEqual(undefined); }); it('returns an error if field is used with no Lens wrapping operation', () => { @@ -165,6 +166,16 @@ describe('formula', () => { operationDefinitionMap ) ).toEqual([`The field bytes cannot be used without operation`]); + + // TODO: enable this later + // expect( + // formulaOperation.getErrorMessage!( + // getNewLayerWithFormula('bytes + bytes'), + // 'col1', + // indexPattern, + // operationDefinitionMap + // ) + // ).toEqual([`The field bytes cannot be used without operation`]); }); it('returns an error if parsing a syntax invalid formula', () => { @@ -189,9 +200,23 @@ describe('formula', () => { } }); - it('returns an error for scenarios where the field is missing', () => { - // noField, avg(noField), noField + 1, moving_average(noField), moving_average(avg(noField)) - const formulas = ['noField', 'avg(noField)', 'noField + 1', 'moving_average(avg(noField))']; + it('returns an error if the field is missing', () => { + const formulas = ['noField', 'avg(noField)', 'noField + 1', 'derivative(avg(noField))']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Fields noField not found']); + } + }); + + it('returns an error if an operation is unknown', () => { + const formulas = ['noFn()', 'noFn(bytes)', 'avg(bytes) + noFn()', 'derivative(noFn())']; for (const formula of formulas) { expect( @@ -201,7 +226,20 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual(['The field noField was not found.']); + ).toEqual(['Operation noFn not found']); + } + + const multipleFnFormulas = ['noFn() + noFnTwo()', 'noFn(noFnTwo())']; + + for (const formula of multipleFnFormulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operations noFn, noFnTwo not found']); } }); @@ -212,9 +250,12 @@ describe('formula', () => { 'avg(avg(bytes))', 'avg(1 + 2)', 'avg(bytes + 5)', - 'derivatives(7)', - 'derivatives(7 + 1)', - 'derivatives(bytes + 7)', + 'avg(bytes + bytes)', + 'derivative(7)', + 'derivative(7 + 1)', + 'derivative(bytes + 7)', + 'derivative(bytes + bytes)', + 'derivative(bytes + avg(bytes))', ]; for (const formula of formulas) { @@ -234,7 +275,7 @@ describe('formula', () => { }); it('returns an error if an argument is passed to count() operation', () => { - const formulas = ['count(7)', 'count("")', 'count(Records)', 'count(bytes)']; + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; for (const formula of formulas) { expect( @@ -244,7 +285,7 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual(['The operation count does not accept any parameter']); + ).toEqual(['The operation count does not accept any field as argument']); } }); @@ -280,7 +321,7 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual(['The operation avg in the Formula requires no parameter']); + ).toEqual(['The operation avg does not accept any parameter']); }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 4e64022dc182ff..08f53504aadb22 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -13,7 +13,13 @@ import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } f import { ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, IndexPatternLayer } from '../../types'; import { getColumnOrder } from '../layer_helpers'; -import { mathOperation, hasMathNode, findVariables } from './math'; +import { + mathOperation, + hasMathNode, + findVariables, + isMathNode, + hasInvalidOperations, +} from './math'; import { documentField } from '../../document_field'; type GroupedNodes = { @@ -72,7 +78,54 @@ export const formulaOperation: OperationDefinition< }), ]; } - const errors = addASTValidation(ast, indexPattern, operationDefinitionMap); + const missingErrors: string[] = []; + const missingOperations = hasInvalidOperations(ast, operationDefinitionMap); + + if (missingOperations.length) { + missingErrors.push( + i18n.translate('xpack.lens.indexPattern.operationsNotFound', { + defaultMessage: + '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', + values: { + operationLength: missingOperations.length, + operationsList: missingOperations.join(', '), + }, + }) + ); + } + const missingVariables = findVariables(ast).filter( + // filter empty string as well? + (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ); + + // need to check the arguments here: check only strings for now + if (missingVariables.length) { + missingErrors.push( + i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values: { + variablesLength: missingOperations.length, + variablesList: missingVariables.join(', '), + }, + }) + ); + } + const invalidVariableErrors = []; + // TODO: add check for Math operation of fields as well + if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { + invalidVariableErrors.push( + i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { + defaultMessage: 'The field {field} cannot be used without operation', + values: { + field: ast.value, + }, + }) + ); + } + + const invalidFunctionErrors = addASTValidation(ast, indexPattern, operationDefinitionMap); + const errors = [...missingErrors, ...invalidVariableErrors, ...invalidFunctionErrors]; return errors.length ? errors : undefined; }, getPossibleOperation() { @@ -271,10 +324,11 @@ function addASTValidation( const errors: string[] = []; const { namedArguments, variables, functions } = groupArgsByType(node.args); + const [firstArg] = node?.args || []; if (nodeOperation.input === 'field') { if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(node.args, 'variable')) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { errors.push( i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { defaultMessage: @@ -282,30 +336,16 @@ function addASTValidation( values: { operation: node.name, type: 'field', - argument: getValueOrName(node.args[0]), + argument: getValueOrName(firstArg), }, }) ); } - - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - if (fieldName) { - if (!indexPattern.getFieldByName(fieldName.value)) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { - defaultMessage: 'The field {field} was not found.', - values: { - field: fieldName.value, - }, - }) - ); - } - } } else { - if (node?.args[0]) { + if (firstArg) { errors.push( - i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { - defaultMessage: 'The operation {operation} does not accept any parameter', + i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { + defaultMessage: 'The operation {operation} does not accept any field as argument', values: { operation: node.name, }, @@ -313,57 +353,80 @@ function addASTValidation( ); } } - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { + if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', + i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + defaultMessage: 'The operation {operation} does not accept any parameter', values: { operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), }, }) ); + } else { + const missingParameters = validateParams(nodeOperation, namedArguments).filter( + ({ isMissing }) => isMissing + ); + if (missingParameters.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values: { + operation: node.name, + params: missingParameters.map(({ name }) => name).join(', '), + }, + }) + ); + } } return errors; } if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(node.args, 'function')) { + if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { errors.push( i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { defaultMessage: 'The first argument for {operation} should be a {type} name. Found {argument}', values: { operation: node.name, - type: 'field', + type: 'function', argument: getValueOrName(node.args[0]), }, }) ); } - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { + if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', + i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + defaultMessage: 'The operation {operation} does not accept any parameter', values: { operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), }, }) ); + } else { + const missingParameters = validateParams(nodeOperation, namedArguments).filter( + ({ isMissing }) => isMissing + ); + if (missingParameters.length) { + errors.push( + i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values: { + operation: node.name, + params: missingParameters.map(({ name }) => name).join(', '), + }, + }) + ); + } } // maybe validate params here? return errors.concat(validateNode(functions[0])); } return []; } + return validateNode(ast); } @@ -421,11 +484,13 @@ function extractColumns( // split the args into types for better TS experience const { namedArguments, variables, functions } = groupArgsByType(node.args); + // the first argument is a special one + const [firstArg] = node?.args || []; // operation node if (nodeOperation.input === 'field') { if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(node.args, 'variable')) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { throw Error('field as first argument not found'); } } else { @@ -465,7 +530,7 @@ function extractColumns( } if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(node.args, 'function')) { + if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg, operations)) { throw Error('first argument not valid for full reference'); } const [referencedOp] = functions; @@ -504,6 +569,10 @@ function extractColumns( throw Error('unexpected node'); } + // a special check on the root node + if (isObject(ast) && ast.type === 'variable') { + throw Error('field cannot be used without operation'); + } const root = parseNode(ast); const variables = findVariables(root); const mathColumn = mathOperation.buildColumn({ @@ -527,6 +596,14 @@ function getSafeFieldName(fieldName: string | undefined) { return fieldName; } +function canHaveParams( + operation: + | OperationDefinition + | OperationDefinition +) { + return Boolean((operation.operationParams || []).length); +} + function validateParams( operation: | OperationDefinition @@ -566,6 +643,6 @@ function shouldHaveFieldArgument(node: TinymathFunction) { return !['count'].includes(node.name); } -function isFirstArgumentValidType(args: TinymathAST[], type: TinymathNodeTypes['type']) { - return args?.length >= 1 && isObject(args[0]) && args[0].type === type; +function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { + return isObject(arg) && arg.type === type; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx index 224d267db0c577..625f015ca36575 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -3,12 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { isObject } from 'lodash'; import { OperationDefinition, GenericOperationDefinition } from './index'; import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternLayer } from '../../types'; +import { IndexPattern } from '../../types'; const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); @@ -34,48 +33,6 @@ export const mathOperation: OperationDefinition !indexPattern.getFieldByName(variable) && !layer.columns[variable] - ); - // need to check the arguments here: check only strings for now - if (missingVariables.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: - '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values: { - variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), - }, - }) - ); - } - } - } - return errors.length ? errors : undefined; - }, getPossibleOperation() { return { dataType: 'number', @@ -141,12 +98,13 @@ function astToString(ast: TinymathAST | string): string | number { return `${ast.name}(${ast.args.map(astToString).join(',')})`; } -function findMathNodes( - root: TinymathAST | string, - operations: Record -): TinymathFunction[] { +export function isMathNode(node: TinymathAST) { + return isObject(node) && node.type === 'function' && tinymathValidOperators.has(node.name); +} + +function findMathNodes(root: TinymathAST | string): TinymathFunction[] { function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function' || operations[node.name]) { + if (!isObject(node) || node.type !== 'function' || !isMathNode(node)) { return []; } return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean); @@ -154,22 +112,29 @@ function findMathNodes( return flattenMathNodes(root); } -export function hasMathNode( - root: TinymathAST, - operations: Record -): boolean { - return Boolean(findMathNodes(root, operations).length); +export function hasMathNode(root: TinymathAST): boolean { + return Boolean(findMathNodes(root).length); } -function hasInvalidOperations( +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( node: TinymathAST | string, operations: Record ) { // avoid duplicates return Array.from( new Set( - findMathNodes(node, operations) - .filter(({ name }) => !tinymathValidOperators.has(name)) + findFunctionNodes(node) + .filter((v) => !isMathNode(v) && !operations[v.name]) .map(({ name }) => name) ) ); @@ -192,13 +157,3 @@ export function findVariables(node: TinymathAST | string | undefined): string[] } return node.args.flatMap(findVariables); } - -function getMathNode(layer: IndexPatternLayer, ast: TinymathAST | string) { - if (typeof ast === 'string') { - const refColumn = layer.columns[ast]; - if (refColumn && 'sourceField' in refColumn) { - return refColumn.sourceField; - } - } - return ast; -} From d71c30c87b048b61131c0f098111b6699365c25c Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Feb 2021 15:57:39 +0100 Subject: [PATCH 017/185] :bug: Fix parsing issues + tests --- .../operations/definitions/formula.test.tsx | 146 +++++++++++++++++- .../operations/definitions/formula.tsx | 23 ++- 2 files changed, 158 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx index d52f3f36b4fef6..c8647d76150b71 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx @@ -1,17 +1,25 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ // import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; // import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; // import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../mocks'; -// import { FormulaIndexPatternColumn } from './formula'; -import { formulaOperation, GenericOperationDefinition } from './index'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from './index'; import type { IndexPattern, IndexPatternLayer } from '../../types'; +jest.mock('../layer_helpers', () => { + return { + getColumnOrder: ({ columns }: { columns: Record }) => + Object.keys(columns), + }; +}); + // const defaultProps = { // storage: {} as IStorageWrapper, // uiSettings: {} as IUiSettingsClient, @@ -33,7 +41,7 @@ describe('formula', () => { beforeEach(() => { layer = { indexPatternId: '1', - columnOrder: ['col1', 'col2'], + columnOrder: ['col1'], columns: { col1: { label: 'Top value of category', @@ -51,6 +59,134 @@ describe('formula', () => { }; }); + describe('regenerateLayerFromAst()', () => { + let indexPattern: IndexPattern; + let operationDefinitionMap: Record; + let currentColumn: FormulaIndexPatternColumn; + + function testIsBrokenFormula(formula: string) { + expect( + regenerateLayerFromAst( + formula, + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ) + ).toEqual({ + ...layer, + columns: { + ...layer.columns, + col1: { + ...currentColumn, + params: { + ...currentColumn.params, + formula, + isFormulaBroken: true, + }, + }, + }, + }); + } + + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + operationDefinitionMap = { + avg: { input: 'field' } as GenericOperationDefinition, + count: { input: 'field' } as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: { + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + } as GenericOperationDefinition, + }; + currentColumn = { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula: '', isFormulaBroken: false }, + references: [], + }; + }); + + it('returns no change but error if the formula cannot be parsed', () => { + const formulas = [ + '+', + 'avg((', + 'avg((bytes)', + 'avg(bytes) +', + 'avg(""', + 'moving_average(avg(bytes), window=)', + ]; + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if field is used with no Lens wrapping operation', () => { + testIsBrokenFormula('bytes'); + }); + + it('returns no change but error if at least one field in the formula is missing', () => { + const formulas = ['noField', 'avg(noField)', 'noField + 1', 'derivative(avg(noField))']; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if at least one operation in the formula is missing', () => { + const formulas = [ + 'noFn()', + 'noFn(bytes)', + 'avg(bytes) + noFn()', + 'derivative(noFn())', + 'noFn() + noFnTwo()', + 'noFn(noFnTwo())', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if one operation has the wrong first argument', () => { + const formulas = [ + 'avg(7)', + 'avg()', + 'avg(avg(bytes))', + 'avg(1 + 2)', + 'avg(bytes + 5)', + 'avg(bytes + bytes)', + 'derivative(7)', + 'derivative(7 + 1)', + 'derivative(bytes + 7)', + 'derivative(bytes + bytes)', + 'derivative(bytes + avg(bytes))', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change by error if an argument is passed to count operation', () => { + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if a required parameter is not passed to the operation in formula', () => { + const formula = 'moving_average(avg(bytes))'; + testIsBrokenFormula(formula); + }); + }); + describe('getErrorMessage', () => { let indexPattern: IndexPattern; let operationDefinitionMap: Record; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx index 08f53504aadb22..a80f4cad7a5d43 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx @@ -1,8 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ + import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; import { groupBy, isObject } from 'lodash'; @@ -194,7 +196,7 @@ export const formulaOperation: OperationDefinition< isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable const ast = parse(column.params.formula || ''); - return !hasMathNode(ast, operationDefinitionMap); + return !hasMathNode(ast); }, paramEditor: function ParamEditor({ @@ -323,7 +325,7 @@ function addASTValidation( } const errors: string[] = []; - const { namedArguments, variables, functions } = groupArgsByType(node.args); + const { namedArguments, functions } = groupArgsByType(node.args); const [firstArg] = node?.args || []; if (nodeOperation.input === 'field') { @@ -472,6 +474,9 @@ function extractColumns( const nodeOperation = operations[node.name]; if (!nodeOperation) { + if (!isMathNode(node)) { + throw Error('missing operation'); + } // it's a regular math node const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< number | TinymathVariable @@ -494,7 +499,7 @@ function extractColumns( throw Error('field as first argument not found'); } } else { - if (node?.args[0]) { + if (firstArg) { throw Error('field as first argument not valid'); } } @@ -530,7 +535,7 @@ function extractColumns( } if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg, operations)) { + if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { throw Error('first argument not valid for full reference'); } const [referencedOp] = functions; @@ -575,6 +580,12 @@ function extractColumns( } const root = parseNode(ast); const variables = findVariables(root); + const hasMissingVariables = variables.some( + (variable) => !indexPattern.getFieldByName(variable) || !layer.columns[variable] + ); + if (hasMissingVariables) { + throw Error('missing variable'); + } const mathColumn = mathOperation.buildColumn({ layer, indexPattern, From abcb1a4843afffc5f2960558bbe48728c811dc74 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 4 Feb 2021 12:33:09 -0500 Subject: [PATCH 018/185] Reorganize code to support autocomplete --- .../server/handlers/chain_runner.js | 1 + .../shareable_runtime/webpack.config.js | 2 +- .../config_panel/dimension_container.scss | 3 + .../{ => formula}/formula.test.tsx | 8 +- .../definitions/{ => formula}/formula.tsx | 167 +++++++++-- .../operations/definitions/formula/index.ts | 8 + .../definitions/formula/math_completion.ts | 274 ++++++++++++++++++ .../definitions/formula/math_examples.md | 28 ++ .../definitions/formula/math_tokenization.tsx | 47 +++ .../operations/definitions/math.tsx | 6 +- 10 files changed, 508 insertions(+), 36 deletions(-) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => formula}/formula.test.tsx (98%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => formula}/formula.tsx (81%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx diff --git a/src/plugins/vis_type_timelion/server/handlers/chain_runner.js b/src/plugins/vis_type_timelion/server/handlers/chain_runner.js index b7bdbcdcb57a62..71e4c47c5abd9c 100644 --- a/src/plugins/vis_type_timelion/server/handlers/chain_runner.js +++ b/src/plugins/vis_type_timelion/server/handlers/chain_runner.js @@ -49,6 +49,7 @@ export default function chainRunner(tlConfig) { } case 'reference': { let reference; + console.log(sheet); if (item.series) { reference = sheet[item.plot - 1][item.series - 1]; } else { diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js index 91df495d1552da..8fba6795667012 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js +++ b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js @@ -39,7 +39,7 @@ module.exports = { 'src/plugins/data/public/expressions/interpreter' ), 'kbn/interpreter': path.resolve(KIBANA_ROOT, 'packages/kbn-interpreter/target/common'), - tinymath: path.resolve(KIBANA_ROOT, 'node_modules/tinymath/lib/tinymath.es5.js'), + tinymath: path.resolve(KIBANA_ROOT, 'node_modules/tinymath/lib/tinymath.min.js'), core_app_image_assets: path.resolve(KIBANA_ROOT, 'src/core/public/core_app/images'), }, extensions: ['.js', '.json', '.ts', '.tsx', '.scss'], diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index 5947d62540a0db..5adbdfcfc045e2 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -11,6 +11,9 @@ top: 0; bottom: 0; animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; + clip-path: none; + // z-index: $euiZLevel1; + } .lnsDimensionContainer__footer { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx similarity index 98% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index c8647d76150b71..e2abbe266b44cb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -8,12 +8,12 @@ // import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; // import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; // import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; -import { createMockedIndexPattern } from '../../mocks'; +import { createMockedIndexPattern } from '../../../mocks'; import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; -import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from './index'; -import type { IndexPattern, IndexPatternLayer } from '../../types'; +import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import type { IndexPattern, IndexPatternLayer } from '../../../types'; -jest.mock('../layer_helpers', () => { +jest.mock('../../layer_helpers', () => { return { getColumnOrder: ({ columns }: { columns: Record }) => Object.keys(columns), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx similarity index 81% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index a80f4cad7a5d43..82784361588f4d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,24 +5,38 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useRef, useCallback, useMemo, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { groupBy, isObject } from 'lodash'; -import { parse, TinymathFunction, TinymathVariable } from '@kbn/tinymath'; -import type { TinymathNamedArgument, TinymathAST } from '@kbn/tinymath'; -import { EuiButton, EuiTextArea } from '@elastic/eui'; -import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; -import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternLayer } from '../../types'; -import { getColumnOrder } from '../layer_helpers'; +import { + parse, + TinymathFunction, + TinymathVariable, + TinymathNamedArgument, + TinymathAST, +} from '@kbn/tinymath'; +import { EuiFormRow, EuiFlexItem, EuiFlexGroup, EuiButton } from '@elastic/eui'; +import { monaco } from '@kbn/monaco'; +import { CodeEditor, useKibana } from '../../../../../../../../src/plugins/kibana_react/public'; +import { + OperationDefinition, + GenericOperationDefinition, + IndexPatternColumn, + ParamEditorProps, +} from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { getColumnOrder } from '../../layer_helpers'; import { mathOperation, hasMathNode, findVariables, isMathNode, hasInvalidOperations, -} from './math'; -import { documentField } from '../../document_field'; +} from '../math'; +import { documentField } from '../../../document_field'; +import { suggest, getSuggestion, LensMathSuggestion } from './math_completion'; +import { LANGUAGE_ID } from './math_tokenization'; type GroupedNodes = { [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; @@ -199,23 +213,118 @@ export const formulaOperation: OperationDefinition< return !hasMathNode(ast); }, - paramEditor: function ParamEditor({ - layer, - updateLayer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap, - }) { - const [text, setText] = useState(currentColumn.params.formula); - return ( - <> - { - setText(e.target.value); + paramEditor: FormulaEditor, +}; + +function FormulaEditor({ + layer, + updateLayer, + currentColumn, + columnId, + http, + indexPattern, + operationDefinitionMap, +}: ParamEditorProps) { + const [text, setText] = useState(currentColumn.params.formula); + const functionList = useRef([]); + const kibana = useKibana(); + const argValueSuggestions = useMemo(() => [], []); + + const provideCompletionItems = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + context: monaco.languages.CompletionContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + let wordRange: monaco.Range; + let aSuggestions; + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + + if (context.triggerCharacter === '(') { + const wordUntil = model.getWordAtPosition(position.delta(0, -3)); + if (wordUntil) { + wordRange = new monaco.Range( + position.lineNumber, + position.column, + position.lineNumber, + position.column + ); + + // Retrieve suggestions for subexpressions + // TODO: make this work for expressions nested more than one level deep + aSuggestions = await suggest( + innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + innerText.length - lengthAfterPosition, + context, + undefined + ); + } + } else { + const wordUntil = model.getWordUntilPosition(position); + wordRange = new monaco.Range( + position.lineNumber, + wordUntil.startColumn, + position.lineNumber, + wordUntil.endColumn + ); + aSuggestions = await suggest( + innerText, + innerText.length - lengthAfterPosition, + context, + wordUntil + ); + } + + return { + suggestions: aSuggestions + ? aSuggestions.list.map((s: IMathFunction | MathFunctionArgs) => + getSuggestion(s, aSuggestions.type, wordRange) + ) + : [], + }; + }, + [argValueSuggestions] + ); + + return ( + + + + + { updateLayer( @@ -232,10 +341,10 @@ export const formulaOperation: OperationDefinition< > Submit - - ); - }, -}; + + + ); +} function parseAndExtract( text: string, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts new file mode 100644 index 00000000000000..c797c40a340397 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { formulaOperation, FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts new file mode 100644 index 00000000000000..e465b156640c0d --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, startsWith } from 'lodash'; +import { monaco } from '@kbn/monaco'; + +import { Parser } from 'pegjs'; +import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import type { GenericOperationDefinition } from '..'; +import { operationDefinitionMap } from '..'; + +export enum SUGGESTION_TYPE { + FIELD = 'field', + NAMED_ARGUMENT = 'named_argument', + FUNCTIONS = 'functions', +} + +export type LensMathSuggestion = GenericOperationDefinition | string; +export interface LensMathSuggestions { + list: LensMathSuggestion[]; + type: SUGGESTION_TYPE; +} + +function inLocation(cursorPosition: number, location: TinymathLocation) { + return cursorPosition >= location.min && cursorPosition <= location.max; +} + +function getArgumentsHelp( + functionHelp: ILensFunction | undefined, + functionArgs: FunctionArg[] = [] +) { + if (!functionHelp) { + return []; + } + + // Do not provide 'inputSeries' as argument suggestion for chainable functions + const argsHelp = functionHelp.chainable ? functionHelp.args.slice(1) : functionHelp.args.slice(0); + + // ignore arguments that are already provided in function declaration + const functionArgNames = functionArgs.map((arg) => arg.name); + return argsHelp.filter((arg) => !functionArgNames.includes(arg.name)); +} + +async function extractSuggestionsFromParsedResult( + result: ReturnType, + cursorPosition: number, + functionList: ILensFunction[], + argValueSuggestions: ArgValueSuggestions +) { + const activeFunc = result.functions.find(({ location }: { location: Location }) => + inLocation(cursorPosition, location) + ); + + if (!activeFunc) { + return; + } + + const functionHelp = functionList.find(({ name }) => name === activeFunc.function); + + if (!functionHelp) { + return; + } + + // return function suggestion when cursor is outside of parentheses + // location range includes '.', function name, and '('. + const openParen = activeFunc.location.min + activeFunc.function.length + 2; + if (cursorPosition < openParen) { + return { list: [functionHelp], type: SUGGESTION_TYPE.FUNCTIONS }; + } + + // return argument value suggestions when cursor is inside argument value + const activeArg = activeFunc.arguments.find((argument: FunctionArg) => { + return inLocation(cursorPosition, argument.location); + }); + if ( + activeArg && + activeArg.type === 'namedArg' && + inLocation(cursorPosition, activeArg.value.location) + ) { + const { function: functionName, arguments: functionArgs } = activeFunc; + + const { + name: argName, + value: { text: partialInput }, + } = activeArg; + + let valueSuggestions; + if (argValueSuggestions.hasDynamicSuggestionsForArgument(functionName, argName)) { + valueSuggestions = await argValueSuggestions.getDynamicSuggestionsForArgument( + functionName, + argName, + functionArgs, + partialInput + ); + } else { + const { suggestions: staticSuggestions } = + functionHelp.args.find((arg) => arg.name === activeArg.name) || {}; + valueSuggestions = argValueSuggestions.getStaticSuggestionsForInput( + partialInput, + staticSuggestions + ); + } + return { + list: valueSuggestions, + type: SUGGESTION_TYPE.ARGUMENT_VALUE, + }; + } + + // return argument suggestions + const argsHelp = getArgumentsHelp(functionHelp, activeFunc.arguments); + const argumentSuggestions = argsHelp.filter((arg) => { + if (get(activeArg, 'type') === 'namedArg') { + return startsWith(arg.name, activeArg.name); + } else if (activeArg) { + return startsWith(arg.name, activeArg.text); + } + return true; + }); + return { list: argumentSuggestions, type: SUGGESTION_TYPE.ARGUMENTS }; +} + +const MARKER = 'LENS_MATH_MARKER'; + +function getInfoAtPosition( + ast: TinymathAST, + position: number, + parent?: TinymathFunction +): undefined | { ast: TinymathAST; parent?: TinymathFunction } { + if (typeof ast === 'number') { + return; + } + // const type = getType(ast); + if (!inLocation(position, ast.location)) { + return; + } + if (ast.type === 'function') { + const [match] = ast.args.flatMap((arg) => getInfoAtPosition(arg, position, ast)); + if (match) { + return match.parent ? match : { ...match, parent: ast }; + } + } + return { + ast, + parent, + }; +} + +export async function suggest( + expression: string, + position: number, + context: monaco.languages.CompletionContext, + word: monaco.editor.IWordAtPosition +): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + const tokenInfo = getInfoAtPosition(ast, position); + + console.log(ast, getInfoAtPosition(ast, position)); + + if (context.triggerCharacter === '=' && tokenInfo?.parent) { + return getArgValueSuggestions( + tokenInfo.parent.name, + tokenInfo.parent.args[tokenInfo.parent.args.length - 1] + ); + } else if (tokenInfo?.parent) { + return getArgumentSuggestions(tokenInfo.parent.name, tokenInfo.parent.args.length); + } + if (tokenInfo) { + return getFunctionSuggestions(word); + } + } catch (e) { + // Fail silently + } + return { list: [], type: SUGGESTION_TYPE.FIELD }; +} + +function getFunctionSuggestions(word: monaco.editor.IWordAtPosition) { + const list = Object.keys(operationDefinitionMap) + .filter((func) => startsWith(func, word.word)) + .map((key) => operationDefinitionMap[key]); + return { list, type: SUGGESTION_TYPE.FUNCTIONS }; +} + +function getArgumentSuggestions(name: string, position: number) { + const operation = operationDefinitionMap[name]; + if (!operation) { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + + if (operation.input === 'field') { + return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + } + + if (operation.input === 'fullReference') { + if (operation.selectionStyle === 'field') { + return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + } + return { list: ['count', 'avg'], type: SUGGESTION_TYPE.FUNCTIONS }; + } + + return { list: [], type: SUGGESTION_TYPE.FIELD }; +} + +function getArgValueSuggestions(name: string, position: number) { + const operation = operationDefinitionMap[name]; + if (!operation) { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + + if (operation.input === 'field') { + return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + } + + if (operation.input === 'fullReference') { + if (operation.selectionStyle === 'field') { + return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + } + return { list: ['count', 'avg'], type: SUGGESTION_TYPE.FUNCTIONS }; + } + + return { list: [], type: SUGGESTION_TYPE.FIELD }; +} + +export function getSuggestion( + suggestion: LensMathSuggestion, + type: SUGGESTION_TYPE, + range: monaco.Range +): monaco.languages.CompletionItem { + let kind: monaco.languages.CompletionItemKind = monaco.languages.CompletionItemKind.Method; + let insertText: string = typeof suggestion === 'string' ? suggestion : suggestion.type; + let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; + let detail: string = ''; + let command: monaco.languages.CompletionItem['command']; + + switch (type) { + case SUGGESTION_TYPE.FIELD: + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + kind = monaco.languages.CompletionItemKind.Value; + insertText = `${insertText}`; + + break; + case SUGGESTION_TYPE.FUNCTIONS: + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + kind = monaco.languages.CompletionItemKind.Function; + insertText = `${insertText}($0)`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + detail = typeof suggestion === 'string' ? '' : `(${suggestion.displayName})`; + + break; + } + + return { + detail, + insertText, + insertTextRules, + kind, + label: insertText, + // documentation: suggestion.help, + command, + range, + }; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md new file mode 100644 index 00000000000000..ae244109ed53ee --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md @@ -0,0 +1,28 @@ +Basic numeric functions that we already support in Lens: + +count() +count(normalize_unit='1s') +sum(field name) +avg(field name) +moving_average(sum(field name), window=5) +moving_average(sum(field name), window=5, normalize_unit='1s') +counter_rate(field name, normalize_unit='1s') +differences(count()) +differences(sum(bytes), normalize_unit='1s') +last_value(bytes, sort=timestamp) +percentile(bytes, percent=95) + +Adding features beyond what we already support. New features are: + +* Filtering +* Math across series +* Time offset + +count() * 100 +(count() / count(offset=-7d)) + min(field name) +sum(field name, filter='field.keyword: "KQL autocomplete inside math" AND field.value > 100') + +What about custom formatting using string manipulation? Probably not... + +(avg(bytes) / 1000) + 'kb' + \ No newline at end of file diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx new file mode 100644 index 00000000000000..ae2ef1530ac89b --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { monaco } from '@kbn/monaco'; + +export const LANGUAGE_ID = 'lens_math'; +monaco.languages.register({ id: LANGUAGE_ID }); + +export const languageConfiguration: monaco.languages.LanguageConfiguration = { + brackets: [['(', ')']], + autoClosingPairs: [ + { open: '(', close: ')' }, + { open: `'`, close: `'` }, + { open: '"', close: '"' }, + ], + surroundingPairs: [ + { open: '(', close: ')' }, + { open: `'`, close: `'` }, + { open: '"', close: '"' }, + ], +}; + +export const lexerRules: monaco.languages.IMonarchLanguage = { + defaultToken: 'invalid', + ignoreCase: true, + brackets: [{ open: '(', close: ')', token: 'delimiter.parenthesis' }], + tokenizer: { + root: [ + [/\s+/, 'whitespace'], + [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'identifier'], + [/[,=]/, 'delimiter'], + [/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number'], + [/".+?"/, 'string'], + [/'.+?'/, 'string'], + [/\+|\-|\*|\//, 'keyword.operator'], + [/[\(]/, 'paren.lparen'], + [/[\)]/, 'paren.rparen'], + ], + }, +}; + +monaco.languages.setMonarchTokensProvider(LANGUAGE_ID, lexerRules); +monaco.languages.setLanguageConfiguration(LANGUAGE_ID, languageConfiguration); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx index 625f015ca36575..8e6bfa148690b3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx @@ -1,8 +1,10 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ + import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { isObject } from 'lodash'; import { OperationDefinition, GenericOperationDefinition } from './index'; From 6c4855eda08398dd9ac11136aa29c905c1f86123 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Feb 2021 20:09:54 +0100 Subject: [PATCH 019/185] :sparkles: Refactor validation --- .../operations/definitions/formula.tsx | 659 ------------------ .../{ => formula}/formula.test.tsx | 117 +++- .../definitions/formula/formula.tsx | 361 ++++++++++ .../definitions/{ => formula}/math.tsx | 12 +- .../operations/definitions/formula/types.ts | 25 + .../operations/definitions/formula/util.ts | 66 ++ .../definitions/formula/validation.ts | 419 +++++++++++ .../operations/definitions/index.ts | 6 +- .../operations/layer_helpers.ts | 2 +- 9 files changed, 975 insertions(+), 692 deletions(-) delete mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => formula}/formula.test.tsx (81%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => formula}/math.tsx (94%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx deleted file mode 100644 index a80f4cad7a5d43..00000000000000 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.tsx +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { useState } from 'react'; -import { i18n } from '@kbn/i18n'; -import { groupBy, isObject } from 'lodash'; -import { parse, TinymathFunction, TinymathVariable } from '@kbn/tinymath'; -import type { TinymathNamedArgument, TinymathAST } from '@kbn/tinymath'; -import { EuiButton, EuiTextArea } from '@elastic/eui'; -import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from './index'; -import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternLayer } from '../../types'; -import { getColumnOrder } from '../layer_helpers'; -import { - mathOperation, - hasMathNode, - findVariables, - isMathNode, - hasInvalidOperations, -} from './math'; -import { documentField } from '../../document_field'; - -type GroupedNodes = { - [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; -} & - { - [Key in TinymathVariable['type']]: Array; - } & - { - [Key in TinymathFunction['type']]: TinymathFunction[]; - }; - -type TinymathNodeTypes = Exclude; - -export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { - operationType: 'formula'; - params: { - formula?: string; - isFormulaBroken?: boolean; - // last value on numeric fields can be formatted - format?: { - id: string; - params?: { - decimals: number; - }; - }; - }; -} - -export const formulaOperation: OperationDefinition< - FormulaIndexPatternColumn, - 'managedReference' -> = { - type: 'formula', - displayName: 'Formula', - getDefaultLabel: (column, indexPattern) => 'Formula', - input: 'managedReference', - getDisabledStatus(indexPattern: IndexPattern) { - return undefined; - }, - getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) { - const column = layer.columns[columnId] as FormulaIndexPatternColumn; - if (!column.params.formula || !operationDefinitionMap) { - return; - } - let ast; - try { - ast = parse(column.params.formula); - } catch (e) { - return [ - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: 'The Formula {expression} cannot be parsed', - values: { - expression: column.params.formula, - }, - }), - ]; - } - const missingErrors: string[] = []; - const missingOperations = hasInvalidOperations(ast, operationDefinitionMap); - - if (missingOperations.length) { - missingErrors.push( - i18n.translate('xpack.lens.indexPattern.operationsNotFound', { - defaultMessage: - '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', - values: { - operationLength: missingOperations.length, - operationsList: missingOperations.join(', '), - }, - }) - ); - } - const missingVariables = findVariables(ast).filter( - // filter empty string as well? - (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] - ); - - // need to check the arguments here: check only strings for now - if (missingVariables.length) { - missingErrors.push( - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: - '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values: { - variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), - }, - }) - ); - } - const invalidVariableErrors = []; - // TODO: add check for Math operation of fields as well - if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { - invalidVariableErrors.push( - i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { - defaultMessage: 'The field {field} cannot be used without operation', - values: { - field: ast.value, - }, - }) - ); - } - - const invalidFunctionErrors = addASTValidation(ast, indexPattern, operationDefinitionMap); - const errors = [...missingErrors, ...invalidVariableErrors, ...invalidFunctionErrors]; - return errors.length ? errors : undefined; - }, - getPossibleOperation() { - return { - dataType: 'number', - isBucketed: false, - scale: 'ratio', - }; - }, - toExpression: (layer, columnId) => { - return [ - { - type: 'function', - function: 'mapColumn', - arguments: { - id: [columnId], - name: [layer.columns[columnId].label], - exp: [ - { - type: 'expression', - chain: [ - { - type: 'function', - function: 'math', - arguments: { - expression: [ - `${(layer.columns[columnId] as FormulaIndexPatternColumn).references[0]}`, - ], - }, - }, - ], - }, - ], - }, - }, - ]; - }, - buildColumn({ previousColumn, layer }) { - let previousFormula = ''; - if (previousColumn) { - if ('references' in previousColumn) { - const metric = layer.columns[previousColumn.references[0]]; - if (metric && 'sourceField' in metric) { - const fieldName = getSafeFieldName(metric.sourceField); - // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; - } - } else { - if (previousColumn && 'sourceField' in previousColumn) { - previousFormula += `${previousColumn.operationType}(${getSafeFieldName( - previousColumn?.sourceField - )})`; - } - } - } - return { - label: 'Formula', - dataType: 'number', - operationType: 'formula', - isBucketed: false, - scale: 'ratio', - params: previousFormula ? { formula: previousFormula, isFormulaBroken: false } : {}, - references: [], - }; - }, - isTransferable: (column, newIndexPattern, operationDefinitionMap) => { - // Basic idea: if it has any math operation in it, probably it cannot be transferable - const ast = parse(column.params.formula || ''); - return !hasMathNode(ast); - }, - - paramEditor: function ParamEditor({ - layer, - updateLayer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap, - }) { - const [text, setText] = useState(currentColumn.params.formula); - return ( - <> - { - setText(e.target.value); - }} - /> - { - updateLayer( - regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ) - ); - }} - > - Submit - - - ); - }, -}; - -function parseAndExtract( - text: string, - layer: IndexPatternLayer, - columnId: string, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - try { - const ast = parse(text); - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); - return { extracted, isValid: true }; - } catch (e) { - return { extracted: [], isValid: false }; - } -} - -export function regenerateLayerFromAst( - text: string, - layer: IndexPatternLayer, - columnId: string, - currentColumn: FormulaIndexPatternColumn, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { extracted, isValid } = parseAndExtract( - text, - layer, - columnId, - indexPattern, - operationDefinitionMap - ); - - const columns = { - ...layer.columns, - }; - - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; - } - }); - - extracted.forEach((extractedColumn, index) => { - columns[`${columnId}X${index}`] = extractedColumn; - }); - - columns[columnId] = { - ...currentColumn, - params: { - ...currentColumn.params, - formula: text, - isFormulaBroken: !isValid, - }, - references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], - }; - - return { - ...layer, - columns, - columnOrder: getColumnOrder({ - ...layer, - columns, - }), - }; - - // TODO - // turn ast into referenced columns - // set state -} - -function addASTValidation( - ast: TinymathAST, - indexPattern: IndexPattern, - operations: Record -) { - function validateNode(node: TinymathAST): string[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - return []; - } - - const errors: string[] = []; - const { namedArguments, functions } = groupArgsByType(node.args); - const [firstArg] = node?.args || []; - - if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(firstArg, 'variable')) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { - defaultMessage: - 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { - operation: node.name, - type: 'field', - argument: getValueOrName(firstArg), - }, - }) - ); - } - } else { - if (firstArg) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { - defaultMessage: 'The operation {operation} does not accept any field as argument', - values: { - operation: node.name, - }, - }) - ); - } - } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { - defaultMessage: 'The operation {operation} does not accept any parameter', - values: { - operation: node.name, - }, - }) - ); - } else { - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { - operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), - }, - }) - ); - } - } - return errors; - } - if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { - defaultMessage: - 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { - operation: node.name, - type: 'function', - argument: getValueOrName(node.args[0]), - }, - }) - ); - } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { - defaultMessage: 'The operation {operation} does not accept any parameter', - values: { - operation: node.name, - }, - }) - ); - } else { - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { - operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), - }, - }) - ); - } - } - // maybe validate params here? - return errors.concat(validateNode(functions[0])); - } - return []; - } - - return validateNode(ast); -} - -function getValueOrName(node: TinymathAST) { - if (!isObject(node)) { - return node; - } - if (node.type !== 'function') { - return node.value; - } - return node.name; -} - -function groupArgsByType(args: TinymathAST[]) { - const { namedArgument, variable, function: functions } = groupBy( - args, - (arg: TinymathAST) => { - return isObject(arg) ? arg.type : 'variable'; - } - ) as GroupedNodes; - // better naming - return { - namedArguments: namedArgument || [], - variables: variable || [], - functions: functions || [], - }; -} - -function extractColumns( - idPrefix: string, - operations: Record, - ast: TinymathAST, - layer: IndexPatternLayer, - indexPattern: IndexPattern -) { - const columns: IndexPatternColumn[] = []; - - function parseNode(node: TinymathAST) { - if (typeof node === 'number' || node.type !== 'function') { - // leaf node - return node; - } - - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - if (!isMathNode(node)) { - throw Error('missing operation'); - } - // it's a regular math node - const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< - number | TinymathVariable - >; - return { - ...node, - args: consumedArgs, - }; - } - - // split the args into types for better TS experience - const { namedArguments, variables, functions } = groupArgsByType(node.args); - // the first argument is a special one - const [firstArg] = node?.args || []; - - // operation node - if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(firstArg, 'variable')) { - throw Error('field as first argument not found'); - } - } else { - if (firstArg) { - throw Error('field as first argument not valid'); - } - } - - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value) - : documentField; - - if (!field) { - throw Error('field not found'); - } - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'field' - >).buildColumn( - { - layer, - indexPattern, - field, - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push(newCol); - // replace by new column id - return newColId; - } - - if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { - throw Error('first argument not valid for full reference'); - } - const [referencedOp] = functions; - const consumedParam = parseNode(referencedOp); - - const subNodeVariables = findVariables(consumedParam); - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = subNodeVariables; - mathColumn.params.tinymathAst = consumedParam; - columns.push(mathColumn); - mathColumn.customLabel = true; - mathColumn.label = `${idPrefix}X${columns.length - 1}`; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'fullReference' - >).buildColumn( - { - layer, - indexPattern, - referenceIds: [`${idPrefix}X${columns.length - 1}`], - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push(newCol); - // replace by new column id - return newColId; - } - - throw Error('unexpected node'); - } - // a special check on the root node - if (isObject(ast) && ast.type === 'variable') { - throw Error('field cannot be used without operation'); - } - const root = parseNode(ast); - const variables = findVariables(root); - const hasMissingVariables = variables.some( - (variable) => !indexPattern.getFieldByName(variable) || !layer.columns[variable] - ); - if (hasMissingVariables) { - throw Error('missing variable'); - } - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = variables; - mathColumn.params.tinymathAst = root; - const newColId = `${idPrefix}X${columns.length}`; - mathColumn.customLabel = true; - mathColumn.label = newColId; - columns.push(mathColumn); - return columns; -} - -function getSafeFieldName(fieldName: string | undefined) { - // clean up the "Records" field for now - if (!fieldName || fieldName === 'Records') { - return ''; - } - return fieldName; -} - -function canHaveParams( - operation: - | OperationDefinition - | OperationDefinition -) { - return Boolean((operation.operationParams || []).length); -} - -function validateParams( - operation: - | OperationDefinition - | OperationDefinition, - params: TinymathNamedArgument[] = [] -) { - const paramsObj = getOperationParams(operation, params); - const formalArgs = operation.operationParams || []; - return formalArgs - .filter(({ required }) => required) - .map(({ name }) => ({ name, isMissing: !(name in paramsObj) })); -} - -function getOperationParams( - operation: - | OperationDefinition - | OperationDefinition, - params: TinymathNamedArgument[] = [] -): Record { - const formalArgs: Record = (operation.operationParams || []).reduce( - (memo: Record, { name, type }) => { - memo[name] = type; - return memo; - }, - {} - ); - // At the moment is positional as expressed in operationParams - return params.reduce>((args, { name, value }) => { - if (formalArgs[name]) { - args[name] = value; - } - return args; - }, {}); -} - -function shouldHaveFieldArgument(node: TinymathFunction) { - return !['count'].includes(node.name); -} - -function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { - return isObject(arg) && arg.type === type; -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx similarity index 81% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index c8647d76150b71..bfd341b4363a7e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -8,12 +8,12 @@ // import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; // import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; // import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; -import { createMockedIndexPattern } from '../../mocks'; +import { createMockedIndexPattern } from '../../../mocks'; import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; -import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from './index'; -import type { IndexPattern, IndexPatternLayer } from '../../types'; +import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; -jest.mock('../layer_helpers', () => { +jest.mock('../../layer_helpers', () => { return { getColumnOrder: ({ columns }: { columns: Record }) => Object.keys(columns), @@ -34,6 +34,27 @@ jest.mock('../layer_helpers', () => { // operationDefinitionMap: { avg: {} }, // }; +const operationDefinitionMap: Record = { + avg: ({ + input: 'field', + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'avg', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + count: { input: 'field' } as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: { + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + } as GenericOperationDefinition, +}; + describe('formula', () => { let layer: IndexPatternLayer; // const InlineOptions = formulaOperation.paramEditor!; @@ -61,7 +82,6 @@ describe('formula', () => { describe('regenerateLayerFromAst()', () => { let indexPattern: IndexPattern; - let operationDefinitionMap: Record; let currentColumn: FormulaIndexPatternColumn; function testIsBrokenFormula(formula: string) { @@ -92,15 +112,6 @@ describe('formula', () => { beforeEach(() => { indexPattern = createMockedIndexPattern(); - operationDefinitionMap = { - avg: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, - derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { - input: 'fullReference', - operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, - }; currentColumn = { label: 'Formula', dataType: 'number', @@ -112,6 +123,56 @@ describe('formula', () => { }; }); + it('should mutate the layer with new columns for valid formula expressions', () => { + expect( + regenerateLayerFromAst( + 'avg(bytes)', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ) + ).toEqual({ + ...layer, + columnOrder: ['col1X0', 'col1X1', 'col1'], + columns: { + ...layer.columns, + col1: { + ...currentColumn, + references: ['col1X1'], + params: { + ...currentColumn.params, + formula: 'avg(bytes)', + isFormulaBroken: false, + }, + }, + col1X0: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'avg', + scale: 'ratio', + sourceField: 'bytes', + timeScale: false, + }, + col1X1: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X1', + operationType: 'math', + params: { + tinymathAst: 'col1X0', + }, + references: ['col1X0'], + scale: 'ratio', + }, + }, + }); + }); + it('returns no change but error if the formula cannot be parsed', () => { const formulas = [ '+', @@ -185,11 +246,15 @@ describe('formula', () => { const formula = 'moving_average(avg(bytes))'; testIsBrokenFormula(formula); }); + + it('returns no change but error if a required parameter passed with the wrong type in formula', () => { + const formula = 'moving_average(avg(bytes), window="m")'; + testIsBrokenFormula(formula); + }); }); describe('getErrorMessage', () => { let indexPattern: IndexPattern; - let operationDefinitionMap: Record; function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer { return { @@ -210,15 +275,6 @@ describe('formula', () => { } beforeEach(() => { indexPattern = createMockedIndexPattern(); - operationDefinitionMap = { - avg: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, - derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { - input: 'fullReference', - operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, - }; }); it('returns undefined if count is passed without arguments', () => { @@ -459,5 +515,18 @@ describe('formula', () => { ) ).toEqual(['The operation avg does not accept any parameter']); }); + + it('returns an error if the parameter passed to an operation is of the wrong type', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg(bytes), window="m")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The parameters for the operation moving_average in the Formula are of the wrong type: window', + ]); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx new file mode 100644 index 00000000000000..4c29ceede618c0 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { isObject } from 'lodash'; +import type { TinymathAST, TinymathVariable } from '@kbn/tinymath'; +import { EuiButton, EuiTextArea } from '@elastic/eui'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { getColumnOrder } from '../../layer_helpers'; +import { mathOperation, hasMathNode, findVariables } from './math'; +import { documentField } from '../../../document_field'; +import { + errorsLookup, + isParsingError, + runASTValidation, + shouldHaveFieldArgument, + tryToParse, +} from './validation'; +import { getOperationParams, getSafeFieldName, groupArgsByType } from './util'; + +export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { + operationType: 'formula'; + params: { + formula?: string; + isFormulaBroken?: boolean; + // last value on numeric fields can be formatted + format?: { + id: string; + params?: { + decimals: number; + }; + }; + }; +} + +export const formulaOperation: OperationDefinition< + FormulaIndexPatternColumn, + 'managedReference' +> = { + type: 'formula', + displayName: 'Formula', + getDefaultLabel: (column, indexPattern) => 'Formula', + input: 'managedReference', + getDisabledStatus(indexPattern: IndexPattern) { + return undefined; + }, + getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) { + const column = layer.columns[columnId] as FormulaIndexPatternColumn; + if (!column.params.formula || !operationDefinitionMap) { + return; + } + const { root, error } = tryToParse(column.params.formula); + if (error) { + return [error]; + } + + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + return errors.length ? errors : undefined; + }, + getPossibleOperation() { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + toExpression: (layer, columnId) => { + const currentColumn = layer.columns[columnId] as FormulaIndexPatternColumn; + const params = currentColumn.params; + const label = !params?.isFormulaBroken ? params?.formula : ''; + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + id: [columnId], + name: [label], + exp: [ + { + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [`${currentColumn.references[0]}`], + }, + }, + ], + }, + ], + }, + }, + ]; + }, + buildColumn({ previousColumn, layer }) { + let previousFormula = ''; + if (previousColumn) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + if (metric && 'sourceField' in metric) { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; + } + } else { + if (previousColumn && 'sourceField' in previousColumn) { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )})`; + } + } + } + return { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: previousFormula ? { formula: previousFormula, isFormulaBroken: false } : {}, + references: [], + }; + }, + isTransferable: (column, newIndexPattern, operationDefinitionMap) => { + // Basic idea: if it has any math operation in it, probably it cannot be transferable + const { root, error } = tryToParse(column.params.formula || ''); + return Boolean(!error && !hasMathNode(root)); + }, + + paramEditor: function ParamEditor({ + layer, + updateLayer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap, + }) { + const [text, setText] = useState(currentColumn.params.formula); + return ( + <> + { + setText(e.target.value); + }} + /> + { + updateLayer( + regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ) + ); + }} + > + Submit + + + ); + }, +}; + +function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + try { + const { root } = tryToParse(text, { shouldThrow: true }); + // before extracting the data run the validation task and throw if invalid + runASTValidation(root, layer, indexPattern, operationDefinitionMap, { shouldThrow: true }); + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; + } catch (e) { + const context = e.message as string; + // propagate the error if it's one of those not controlled by the Formula logic + if (!errorsLookup.has(context) && !isParsingError(context)) { + throw e; + } + return { extracted: [], isValid: false }; + } +} + +export function regenerateLayerFromAst( + text: string, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { extracted, isValid } = parseAndExtract( + text, + layer, + columnId, + indexPattern, + operationDefinitionMap + ); + + const columns = { + ...layer.columns, + }; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach((extractedColumn, index) => { + columns[`${columnId}X${index}`] = extractedColumn; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text, + isFormulaBroken: !isValid, + }, + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + }; + + return { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }; + + // TODO + // turn ast into referenced columns + // set state +} + +function extractColumns( + idPrefix: string, + operations: Record, + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern +) { + const columns: IndexPatternColumn[] = []; + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { + // leaf node + return node; + } + + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; + return { + ...node, + args: consumedArgs, + }; + } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + + // operation node + if (nodeOperation.input === 'field') { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value)! + : documentField; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn( + { + layer, + indexPattern, + field, + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push(newCol); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const [referencedOp] = functions; + const consumedParam = parseNode(referencedOp); + + const subNodeVariables = findVariables(consumedParam); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = subNodeVariables; + mathColumn.params.tinymathAst = consumedParam; + columns.push(mathColumn); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push(newCol); + // replace by new column id + return newColId; + } + } + const root = parseNode(ast); + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables; + mathColumn.params.tinymathAst = root; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push(mathColumn); + return columns; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx similarity index 94% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 625f015ca36575..50674a85764cb7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ + import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { isObject } from 'lodash'; -import { OperationDefinition, GenericOperationDefinition } from './index'; -import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern } from '../../types'; +import { OperationDefinition, GenericOperationDefinition } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern } from '../../../types'; const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts new file mode 100644 index 00000000000000..ce853dec1d9513 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + TinymathAST, + TinymathFunction, + TinymathNamedArgument, + TinymathVariable, +} from 'packages/kbn-tinymath'; + +export type GroupedNodes = { + [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; +} & + { + [Key in TinymathVariable['type']]: Array; + } & + { + [Key in TinymathFunction['type']]: TinymathFunction[]; + }; + +export type TinymathNodeTypes = Exclude; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts new file mode 100644 index 00000000000000..01791fb154786e --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { groupBy, isObject } from 'lodash'; +import type { TinymathAST, TinymathNamedArgument } from 'packages/kbn-tinymath'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; +import type { GroupedNodes } from './types'; + +export function groupArgsByType(args: TinymathAST[]) { + const { namedArgument, variable, function: functions } = groupBy( + args, + (arg: TinymathAST) => { + return isObject(arg) ? arg.type : 'variable'; + } + ) as GroupedNodes; + // better naming + return { + namedArguments: namedArgument || [], + variables: variable || [], + functions: functions || [], + }; +} + +export function getValueOrName(node: TinymathAST) { + if (!isObject(node)) { + return node; + } + if (node.type !== 'function') { + return node.value; + } + return node.name; +} + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function getOperationParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +): Record { + const formalArgs: Record = (operation.operationParams || []).reduce( + (memo: Record, { name, type }) => { + memo[name] = type; + return memo; + }, + {} + ); + // At the moment is positional as expressed in operationParams + return params.reduce>((args, { name, value }) => { + if (formalArgs[name]) { + args[name] = value; + } + return args; + }, {}); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts new file mode 100644 index 00000000000000..58453c0f5fcd67 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { parse } from '@kbn/tinymath'; +import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; +import { getOperationParams, getValueOrName, groupArgsByType } from './util'; +import { findVariables, hasInvalidOperations, isMathNode } from './math'; + +import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { IndexPattern, IndexPatternLayer } from '../../../types'; +import type { TinymathNodeTypes } from './types'; + +const validationErrors = { + missingField: 'missing field', + missingOperation: 'missing operation', + missingParameter: 'missing parameter', + wrongTypeParameter: 'wrong type parameter', + wrongFirstArgument: 'wrong first argument', + cannotAcceptParameter: 'cannot accept parameter', + shouldNotHaveField: 'operation should not have field', + unexpectedNode: 'unexpected node', + fieldWithNoOperation: 'unexpected field with no operation', + failedParsing: 'Failed to parse expression.', // note: this string comes from Tinymath, do not change it +}; +export const errorsLookup = new Set(Object.values(validationErrors)); + +type ErrorTypes = keyof typeof validationErrors; + +export function isParsingError(message: string) { + return message.includes(validationErrors.failedParsing); +} + +function getMessageFromId(messageId: ErrorTypes, values: Record) { + switch (messageId) { + case 'wrongFirstArgument': + return i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values, + }); + case 'shouldNotHaveField': + return i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { + defaultMessage: 'The operation {operation} does not accept any field as argument', + values, + }); + case 'cannotAcceptParameter': + return i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + defaultMessage: 'The operation {operation} does not accept any parameter', + values, + }); + case 'missingParameter': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values, + }); + case 'wrongTypeParameter': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', + values, + }); + case 'missingField': + return i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values, + }); + case 'missingOperation': + return i18n.translate('xpack.lens.indexPattern.operationsNotFound', { + defaultMessage: + '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', + values, + }); + case 'fieldWithNoOperation': + return i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { + defaultMessage: 'The field {field} cannot be used without operation', + values, + }); + case 'failedParsing': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: 'The Formula {expression} cannot be parsed', + values, + }); + default: + return 'no Error found'; + } +} + +function addErrorOrThrow({ + messageId, + values, + shouldThrow, +}: { + messageId: ErrorTypes; + values: Record; + shouldThrow?: boolean; +}) { + if (shouldThrow) { + throw Error(validationErrors[messageId]); + } + return getMessageFromId(messageId, values); +} + +export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boolean } = {}) { + let root; + try { + root = parse(formula); + } catch (e) { + if (shouldThrow) { + // propagate the error + throw e; + } + return { + root: null, + error: getMessageFromId('failedParsing', { + expression: formula, + }), + }; + } + return { root, error: null }; +} + +export function runASTValidation( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record, + options: { shouldThrow?: boolean } = {} +) { + return [ + ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations, options), + ...runFullASTValidation(ast, indexPattern, operations, options), + ]; +} + +function checkVariableEdgeCases( + ast: TinymathAST, + missingVariables: string[], + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + const invalidVariableErrors = []; + // TODO: add check for Math operation of fields as well + if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { + invalidVariableErrors.push( + addErrorOrThrow({ + messageId: 'fieldWithNoOperation', + values: { + field: ast.value, + }, + shouldThrow, + }) + ); + } + return invalidVariableErrors; +} + +function checkMissingVariableOrFunctions( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record, + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + const missingErrors: string[] = []; + const missingOperations = hasInvalidOperations(ast, operations); + + if (missingOperations.length) { + missingErrors.push( + addErrorOrThrow({ + messageId: 'missingOperation', + values: { + operationLength: missingOperations.length, + operationsList: missingOperations.join(', '), + }, + shouldThrow, + }) + ); + } + const missingVariables = findVariables(ast).filter( + // filter empty string as well? + (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ); + + // need to check the arguments here: check only strings for now + if (missingVariables.length) { + missingErrors.push( + addErrorOrThrow({ + messageId: 'missingField', + values: { + variablesLength: missingOperations.length, + variablesList: missingVariables.join(', '), + }, + shouldThrow, + }) + ); + } + const invalidVariableErrors = checkVariableEdgeCases(ast, missingErrors, { shouldThrow }); + return [...missingErrors, ...invalidVariableErrors]; +} + +function runFullASTValidation( + ast: TinymathAST, + indexPattern: IndexPattern, + operations: Record, + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + function validateNode(node: TinymathAST): string[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + return []; + } + + const errors: string[] = []; + const { namedArguments, functions } = groupArgsByType(node.args); + const [firstArg] = node?.args || []; + + if (nodeOperation.input === 'field') { + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(firstArg), + }, + shouldThrow, + }) + ); + } + } else { + if (firstArg) { + errors.push( + addErrorOrThrow({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + addErrorOrThrow({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } else { + const missingParams = getMissingParams(nodeOperation, namedArguments); + if (missingParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + } + return errors; + } + if (nodeOperation.input === 'fullReference') { + if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'function', + argument: getValueOrName(firstArg), + }, + shouldThrow, + }) + ); + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + addErrorOrThrow({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } else { + const missingParameters = getMissingParams(nodeOperation, namedArguments); + if (missingParameters.length) { + errors.push( + addErrorOrThrow({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParameters.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + } + + return errors.concat(validateNode(functions[0])); + } + return []; + } + + return validateNode(ast); +} + +export function canHaveParams( + operation: + | OperationDefinition + | OperationDefinition +) { + return Boolean((operation.operationParams || []).length); +} + +export function getInvalidParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isCorrectType, isRequired }) => (isMissing && isRequired) || !isCorrectType + ); +} + +export function getMissingParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isRequired }) => isMissing && isRequired + ); +} + +export function getWrongTypeParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isCorrectType, isMissing }) => !isCorrectType && !isMissing + ); +} + +export function validateParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + const paramsObj = getOperationParams(operation, params); + const formalArgs = operation.operationParams || []; + return formalArgs.map(({ name, type, required }) => ({ + name, + isMissing: !(name in paramsObj), + isCorrectType: typeof paramsObj[name] === type, + isRequired: required, + })); +} + +export function shouldHaveFieldArgument(node: TinymathFunction) { + return !['count'].includes(node.name); +} + +export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { + return isObject(arg) && arg.type === type; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 030b69202d042e..b93097221bed33 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -35,8 +35,8 @@ import { MovingAverageIndexPatternColumn, } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; -import { mathOperation, MathIndexPatternColumn } from './math'; -import { formulaOperation, FormulaIndexPatternColumn } from './formula'; +import { mathOperation, MathIndexPatternColumn } from './formula/math'; +import { formulaOperation, FormulaIndexPatternColumn } from './formula/formula'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -137,7 +137,7 @@ export { derivativeOperation, movingAverageOperation, } from './calculations'; -export { formulaOperation } from './formula'; +export { formulaOperation } from './formula/formula'; /** * Properties passed to the operation-specific part of the popover editor diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 65e4bd0ab141db..ba0fcd70ac4c33 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -18,7 +18,7 @@ import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../type import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula/formula'; interface ColumnChange { op: OperationType; From 1f9ebb925ad06a7e0dac2d381d6617c4c5538f2f Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 4 Feb 2021 14:38:43 -0500 Subject: [PATCH 020/185] Refactoring by dej611 Co-Authored-By: Marco Liberati --- .../definitions/formula/formula.test.tsx | 115 ++++- .../definitions/formula/formula.tsx | 358 ++------------- .../operations/definitions/formula/index.ts | 1 + .../definitions/{ => formula}/math.tsx | 6 +- .../operations/definitions/formula/types.ts | 25 ++ .../operations/definitions/formula/util.ts | 66 +++ .../definitions/formula/validation.ts | 419 ++++++++++++++++++ .../operations/definitions/index.ts | 8 +- 8 files changed, 643 insertions(+), 355 deletions(-) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => formula}/math.tsx (97%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index e2abbe266b44cb..0e2b929808e2ca 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -9,9 +9,9 @@ // import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; // import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../../mocks'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; -import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; -import type { IndexPattern, IndexPatternLayer } from '../../../types'; +import { formulaOperation, FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +import { GenericOperationDefinition, IndexPatternColumn } from '../index'; +import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; jest.mock('../../layer_helpers', () => { return { @@ -34,6 +34,27 @@ jest.mock('../../layer_helpers', () => { // operationDefinitionMap: { avg: {} }, // }; +const operationDefinitionMap: Record = { + avg: ({ + input: 'field', + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'avg', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + count: { input: 'field' } as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: { + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + } as GenericOperationDefinition, +}; + describe('formula', () => { let layer: IndexPatternLayer; // const InlineOptions = formulaOperation.paramEditor!; @@ -61,7 +82,6 @@ describe('formula', () => { describe('regenerateLayerFromAst()', () => { let indexPattern: IndexPattern; - let operationDefinitionMap: Record; let currentColumn: FormulaIndexPatternColumn; function testIsBrokenFormula(formula: string) { @@ -92,15 +112,6 @@ describe('formula', () => { beforeEach(() => { indexPattern = createMockedIndexPattern(); - operationDefinitionMap = { - avg: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, - derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { - input: 'fullReference', - operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, - }; currentColumn = { label: 'Formula', dataType: 'number', @@ -112,6 +123,56 @@ describe('formula', () => { }; }); + it('should mutate the layer with new columns for valid formula expressions', () => { + expect( + regenerateLayerFromAst( + 'avg(bytes)', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ) + ).toEqual({ + ...layer, + columnOrder: ['col1X0', 'col1X1', 'col1'], + columns: { + ...layer.columns, + col1: { + ...currentColumn, + references: ['col1X1'], + params: { + ...currentColumn.params, + formula: 'avg(bytes)', + isFormulaBroken: false, + }, + }, + col1X0: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'avg', + scale: 'ratio', + sourceField: 'bytes', + timeScale: false, + }, + col1X1: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X1', + operationType: 'math', + params: { + tinymathAst: 'col1X0', + }, + references: ['col1X0'], + scale: 'ratio', + }, + }, + }); + }); + it('returns no change but error if the formula cannot be parsed', () => { const formulas = [ '+', @@ -185,11 +246,15 @@ describe('formula', () => { const formula = 'moving_average(avg(bytes))'; testIsBrokenFormula(formula); }); + + it('returns no change but error if a required parameter passed with the wrong type in formula', () => { + const formula = 'moving_average(avg(bytes), window="m")'; + testIsBrokenFormula(formula); + }); }); describe('getErrorMessage', () => { let indexPattern: IndexPattern; - let operationDefinitionMap: Record; function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer { return { @@ -210,15 +275,6 @@ describe('formula', () => { } beforeEach(() => { indexPattern = createMockedIndexPattern(); - operationDefinitionMap = { - avg: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, - derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { - input: 'fullReference', - operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, - }; }); it('returns undefined if count is passed without arguments', () => { @@ -459,5 +515,18 @@ describe('formula', () => { ) ).toEqual(['The operation avg does not accept any parameter']); }); + + it('returns an error if the parameter passed to an operation is of the wrong type', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(avg(bytes), window="m")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The parameters for the operation moving_average in the Formula are of the wrong type: window', + ]); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 82784361588f4d..88cf72689e5ff7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -27,28 +27,18 @@ import { import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern, IndexPatternLayer } from '../../../types'; import { getColumnOrder } from '../../layer_helpers'; -import { - mathOperation, - hasMathNode, - findVariables, - isMathNode, - hasInvalidOperations, -} from '../math'; +import { mathOperation, hasMathNode, findVariables } from './math'; import { documentField } from '../../../document_field'; +import { + errorsLookup, + isParsingError, + runASTValidation, + shouldHaveFieldArgument, + tryToParse, +} from './validation'; import { suggest, getSuggestion, LensMathSuggestion } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; - -type GroupedNodes = { - [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; -} & - { - [Key in TinymathVariable['type']]: Array; - } & - { - [Key in TinymathFunction['type']]: TinymathFunction[]; - }; - -type TinymathNodeTypes = Exclude; +import { getOperationParams, getSafeFieldName, groupArgsByType } from './util'; export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; @@ -81,67 +71,12 @@ export const formulaOperation: OperationDefinition< if (!column.params.formula || !operationDefinitionMap) { return; } - let ast; - try { - ast = parse(column.params.formula); - } catch (e) { - return [ - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: 'The Formula {expression} cannot be parsed', - values: { - expression: column.params.formula, - }, - }), - ]; - } - const missingErrors: string[] = []; - const missingOperations = hasInvalidOperations(ast, operationDefinitionMap); - - if (missingOperations.length) { - missingErrors.push( - i18n.translate('xpack.lens.indexPattern.operationsNotFound', { - defaultMessage: - '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', - values: { - operationLength: missingOperations.length, - operationsList: missingOperations.join(', '), - }, - }) - ); - } - const missingVariables = findVariables(ast).filter( - // filter empty string as well? - (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] - ); - - // need to check the arguments here: check only strings for now - if (missingVariables.length) { - missingErrors.push( - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: - '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values: { - variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), - }, - }) - ); - } - const invalidVariableErrors = []; - // TODO: add check for Math operation of fields as well - if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { - invalidVariableErrors.push( - i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { - defaultMessage: 'The field {field} cannot be used without operation', - values: { - field: ast.value, - }, - }) - ); + const { root, error } = tryToParse(column.params.formula); + if (error) { + return [error]; } - const invalidFunctionErrors = addASTValidation(ast, indexPattern, operationDefinitionMap); - const errors = [...missingErrors, ...invalidVariableErrors, ...invalidFunctionErrors]; + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); return errors.length ? errors : undefined; }, getPossibleOperation() { @@ -152,13 +87,16 @@ export const formulaOperation: OperationDefinition< }; }, toExpression: (layer, columnId) => { + const currentColumn = layer.columns[columnId] as FormulaIndexPatternColumn; + const params = currentColumn.params; + const label = !params?.isFormulaBroken ? params?.formula : ''; return [ { type: 'function', function: 'mapColumn', arguments: { id: [columnId], - name: [layer.columns[columnId].label], + name: [label], exp: [ { type: 'expression', @@ -167,9 +105,7 @@ export const formulaOperation: OperationDefinition< type: 'function', function: 'math', arguments: { - expression: [ - `${(layer.columns[columnId] as FormulaIndexPatternColumn).references[0]}`, - ], + expression: [`${currentColumn.references[0]}`], }, }, ], @@ -209,8 +145,8 @@ export const formulaOperation: OperationDefinition< }, isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable - const ast = parse(column.params.formula || ''); - return !hasMathNode(ast); + const { root, error } = tryToParse(column.params.formula || ''); + return Boolean(!error && !hasMathNode(root)); }, paramEditor: FormulaEditor, @@ -354,13 +290,20 @@ function parseAndExtract( operationDefinitionMap: Record ) { try { - const ast = parse(text); + const { root } = tryToParse(text, { shouldThrow: true }); + // before extracting the data run the validation task and throw if invalid + runASTValidation(root, layer, indexPattern, operationDefinitionMap, { shouldThrow: true }); /* { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } */ - const extracted = extractColumns(columnId, operationDefinitionMap, ast, layer, indexPattern); + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); return { extracted, isValid: true }; } catch (e) { + const context = e.message as string; + // propagate the error if it's one of those not controlled by the Formula logic + if (!errorsLookup.has(context) && !isParsingError(context)) { + throw e; + } return { extracted: [], isValid: false }; } } @@ -419,153 +362,6 @@ export function regenerateLayerFromAst( // set state } -function addASTValidation( - ast: TinymathAST, - indexPattern: IndexPattern, - operations: Record -) { - function validateNode(node: TinymathAST): string[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - return []; - } - - const errors: string[] = []; - const { namedArguments, functions } = groupArgsByType(node.args); - const [firstArg] = node?.args || []; - - if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(firstArg, 'variable')) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { - defaultMessage: - 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { - operation: node.name, - type: 'field', - argument: getValueOrName(firstArg), - }, - }) - ); - } - } else { - if (firstArg) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { - defaultMessage: 'The operation {operation} does not accept any field as argument', - values: { - operation: node.name, - }, - }) - ); - } - } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { - defaultMessage: 'The operation {operation} does not accept any parameter', - values: { - operation: node.name, - }, - }) - ); - } else { - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { - operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), - }, - }) - ); - } - } - return errors; - } - if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { - defaultMessage: - 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { - operation: node.name, - type: 'function', - argument: getValueOrName(node.args[0]), - }, - }) - ); - } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { - defaultMessage: 'The operation {operation} does not accept any parameter', - values: { - operation: node.name, - }, - }) - ); - } else { - const missingParameters = validateParams(nodeOperation, namedArguments).filter( - ({ isMissing }) => isMissing - ); - if (missingParameters.length) { - errors.push( - i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { - defaultMessage: - 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { - operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), - }, - }) - ); - } - } - // maybe validate params here? - return errors.concat(validateNode(functions[0])); - } - return []; - } - - return validateNode(ast); -} - -function getValueOrName(node: TinymathAST) { - if (!isObject(node)) { - return node; - } - if (node.type !== 'function') { - return node.value; - } - return node.name; -} - -function groupArgsByType(args: TinymathAST[]) { - const { namedArgument, variable, function: functions } = groupBy( - args, - (arg: TinymathAST) => { - return isObject(arg) ? arg.type : 'variable'; - } - ) as GroupedNodes; - // better naming - return { - namedArguments: namedArgument || [], - variables: variable || [], - functions: functions || [], - }; -} - function extractColumns( idPrefix: string, operations: Record, @@ -583,9 +379,6 @@ function extractColumns( const nodeOperation = operations[node.name]; if (!nodeOperation) { - if (!isMathNode(node)) { - throw Error('missing operation'); - } // it's a regular math node const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< number | TinymathVariable @@ -598,30 +391,15 @@ function extractColumns( // split the args into types for better TS experience const { namedArguments, variables, functions } = groupArgsByType(node.args); - // the first argument is a special one - const [firstArg] = node?.args || []; // operation node if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(firstArg, 'variable')) { - throw Error('field as first argument not found'); - } - } else { - if (firstArg) { - throw Error('field as first argument not valid'); - } - } - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value) + ? indexPattern.getFieldByName(fieldName.value)! : documentField; - if (!field) { - throw Error('field not found'); - } - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); const newCol = (nodeOperation as OperationDefinition< @@ -644,9 +422,6 @@ function extractColumns( } if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { - throw Error('first argument not valid for full reference'); - } const [referencedOp] = functions; const consumedParam = parseNode(referencedOp); @@ -680,21 +455,9 @@ function extractColumns( // replace by new column id return newColId; } - - throw Error('unexpected node'); - } - // a special check on the root node - if (isObject(ast) && ast.type === 'variable') { - throw Error('field cannot be used without operation'); } const root = parseNode(ast); const variables = findVariables(root); - const hasMissingVariables = variables.some( - (variable) => !indexPattern.getFieldByName(variable) || !layer.columns[variable] - ); - if (hasMissingVariables) { - throw Error('missing variable'); - } const mathColumn = mathOperation.buildColumn({ layer, indexPattern, @@ -707,62 +470,3 @@ function extractColumns( columns.push(mathColumn); return columns; } - -function getSafeFieldName(fieldName: string | undefined) { - // clean up the "Records" field for now - if (!fieldName || fieldName === 'Records') { - return ''; - } - return fieldName; -} - -function canHaveParams( - operation: - | OperationDefinition - | OperationDefinition -) { - return Boolean((operation.operationParams || []).length); -} - -function validateParams( - operation: - | OperationDefinition - | OperationDefinition, - params: TinymathNamedArgument[] = [] -) { - const paramsObj = getOperationParams(operation, params); - const formalArgs = operation.operationParams || []; - return formalArgs - .filter(({ required }) => required) - .map(({ name }) => ({ name, isMissing: !(name in paramsObj) })); -} - -function getOperationParams( - operation: - | OperationDefinition - | OperationDefinition, - params: TinymathNamedArgument[] = [] -): Record { - const formalArgs: Record = (operation.operationParams || []).reduce( - (memo: Record, { name, type }) => { - memo[name] = type; - return memo; - }, - {} - ); - // At the moment is positional as expressed in operationParams - return params.reduce>((args, { name, value }) => { - if (formalArgs[name]) { - args[name] = value; - } - return args; - }, {}); -} - -function shouldHaveFieldArgument(node: TinymathFunction) { - return !['count'].includes(node.name); -} - -function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { - return isObject(arg) && arg.type === type; -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts index c797c40a340397..b5605369d4f9bc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts @@ -6,3 +6,4 @@ */ export { formulaOperation, FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +export { mathOperation, MathIndexPatternColumn } from './math'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx similarity index 97% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 8e6bfa148690b3..50674a85764cb7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -7,9 +7,9 @@ import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { isObject } from 'lodash'; -import { OperationDefinition, GenericOperationDefinition } from './index'; -import { ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern } from '../../types'; +import { OperationDefinition, GenericOperationDefinition } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern } from '../../../types'; const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts new file mode 100644 index 00000000000000..ce853dec1d9513 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + TinymathAST, + TinymathFunction, + TinymathNamedArgument, + TinymathVariable, +} from 'packages/kbn-tinymath'; + +export type GroupedNodes = { + [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; +} & + { + [Key in TinymathVariable['type']]: Array; + } & + { + [Key in TinymathFunction['type']]: TinymathFunction[]; + }; + +export type TinymathNodeTypes = Exclude; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts new file mode 100644 index 00000000000000..01791fb154786e --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { groupBy, isObject } from 'lodash'; +import type { TinymathAST, TinymathNamedArgument } from 'packages/kbn-tinymath'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; +import type { GroupedNodes } from './types'; + +export function groupArgsByType(args: TinymathAST[]) { + const { namedArgument, variable, function: functions } = groupBy( + args, + (arg: TinymathAST) => { + return isObject(arg) ? arg.type : 'variable'; + } + ) as GroupedNodes; + // better naming + return { + namedArguments: namedArgument || [], + variables: variable || [], + functions: functions || [], + }; +} + +export function getValueOrName(node: TinymathAST) { + if (!isObject(node)) { + return node; + } + if (node.type !== 'function') { + return node.value; + } + return node.name; +} + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function getOperationParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +): Record { + const formalArgs: Record = (operation.operationParams || []).reduce( + (memo: Record, { name, type }) => { + memo[name] = type; + return memo; + }, + {} + ); + // At the moment is positional as expressed in operationParams + return params.reduce>((args, { name, value }) => { + if (formalArgs[name]) { + args[name] = value; + } + return args; + }, {}); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts new file mode 100644 index 00000000000000..58453c0f5fcd67 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { parse } from '@kbn/tinymath'; +import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; +import { getOperationParams, getValueOrName, groupArgsByType } from './util'; +import { findVariables, hasInvalidOperations, isMathNode } from './math'; + +import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { IndexPattern, IndexPatternLayer } from '../../../types'; +import type { TinymathNodeTypes } from './types'; + +const validationErrors = { + missingField: 'missing field', + missingOperation: 'missing operation', + missingParameter: 'missing parameter', + wrongTypeParameter: 'wrong type parameter', + wrongFirstArgument: 'wrong first argument', + cannotAcceptParameter: 'cannot accept parameter', + shouldNotHaveField: 'operation should not have field', + unexpectedNode: 'unexpected node', + fieldWithNoOperation: 'unexpected field with no operation', + failedParsing: 'Failed to parse expression.', // note: this string comes from Tinymath, do not change it +}; +export const errorsLookup = new Set(Object.values(validationErrors)); + +type ErrorTypes = keyof typeof validationErrors; + +export function isParsingError(message: string) { + return message.includes(validationErrors.failedParsing); +} + +function getMessageFromId(messageId: ErrorTypes, values: Record) { + switch (messageId) { + case 'wrongFirstArgument': + return i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values, + }); + case 'shouldNotHaveField': + return i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { + defaultMessage: 'The operation {operation} does not accept any field as argument', + values, + }); + case 'cannotAcceptParameter': + return i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + defaultMessage: 'The operation {operation} does not accept any parameter', + values, + }); + case 'missingParameter': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values, + }); + case 'wrongTypeParameter': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', + values, + }); + case 'missingField': + return i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values, + }); + case 'missingOperation': + return i18n.translate('xpack.lens.indexPattern.operationsNotFound', { + defaultMessage: + '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', + values, + }); + case 'fieldWithNoOperation': + return i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { + defaultMessage: 'The field {field} cannot be used without operation', + values, + }); + case 'failedParsing': + return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: 'The Formula {expression} cannot be parsed', + values, + }); + default: + return 'no Error found'; + } +} + +function addErrorOrThrow({ + messageId, + values, + shouldThrow, +}: { + messageId: ErrorTypes; + values: Record; + shouldThrow?: boolean; +}) { + if (shouldThrow) { + throw Error(validationErrors[messageId]); + } + return getMessageFromId(messageId, values); +} + +export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boolean } = {}) { + let root; + try { + root = parse(formula); + } catch (e) { + if (shouldThrow) { + // propagate the error + throw e; + } + return { + root: null, + error: getMessageFromId('failedParsing', { + expression: formula, + }), + }; + } + return { root, error: null }; +} + +export function runASTValidation( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record, + options: { shouldThrow?: boolean } = {} +) { + return [ + ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations, options), + ...runFullASTValidation(ast, indexPattern, operations, options), + ]; +} + +function checkVariableEdgeCases( + ast: TinymathAST, + missingVariables: string[], + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + const invalidVariableErrors = []; + // TODO: add check for Math operation of fields as well + if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { + invalidVariableErrors.push( + addErrorOrThrow({ + messageId: 'fieldWithNoOperation', + values: { + field: ast.value, + }, + shouldThrow, + }) + ); + } + return invalidVariableErrors; +} + +function checkMissingVariableOrFunctions( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record, + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + const missingErrors: string[] = []; + const missingOperations = hasInvalidOperations(ast, operations); + + if (missingOperations.length) { + missingErrors.push( + addErrorOrThrow({ + messageId: 'missingOperation', + values: { + operationLength: missingOperations.length, + operationsList: missingOperations.join(', '), + }, + shouldThrow, + }) + ); + } + const missingVariables = findVariables(ast).filter( + // filter empty string as well? + (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ); + + // need to check the arguments here: check only strings for now + if (missingVariables.length) { + missingErrors.push( + addErrorOrThrow({ + messageId: 'missingField', + values: { + variablesLength: missingOperations.length, + variablesList: missingVariables.join(', '), + }, + shouldThrow, + }) + ); + } + const invalidVariableErrors = checkVariableEdgeCases(ast, missingErrors, { shouldThrow }); + return [...missingErrors, ...invalidVariableErrors]; +} + +function runFullASTValidation( + ast: TinymathAST, + indexPattern: IndexPattern, + operations: Record, + { shouldThrow }: { shouldThrow?: boolean } = {} +) { + function validateNode(node: TinymathAST): string[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + return []; + } + + const errors: string[] = []; + const { namedArguments, functions } = groupArgsByType(node.args); + const [firstArg] = node?.args || []; + + if (nodeOperation.input === 'field') { + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(firstArg), + }, + shouldThrow, + }) + ); + } + } else { + if (firstArg) { + errors.push( + addErrorOrThrow({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + addErrorOrThrow({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } else { + const missingParams = getMissingParams(nodeOperation, namedArguments); + if (missingParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + } + return errors; + } + if (nodeOperation.input === 'fullReference') { + if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'function', + argument: getValueOrName(firstArg), + }, + shouldThrow, + }) + ); + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + addErrorOrThrow({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + shouldThrow, + }) + ); + } else { + const missingParameters = getMissingParams(nodeOperation, namedArguments); + if (missingParameters.length) { + errors.push( + addErrorOrThrow({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParameters.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + addErrorOrThrow({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + shouldThrow, + }) + ); + } + } + + return errors.concat(validateNode(functions[0])); + } + return []; + } + + return validateNode(ast); +} + +export function canHaveParams( + operation: + | OperationDefinition + | OperationDefinition +) { + return Boolean((operation.operationParams || []).length); +} + +export function getInvalidParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isCorrectType, isRequired }) => (isMissing && isRequired) || !isCorrectType + ); +} + +export function getMissingParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isRequired }) => isMissing && isRequired + ); +} + +export function getWrongTypeParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isCorrectType, isMissing }) => !isCorrectType && !isMissing + ); +} + +export function validateParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + const paramsObj = getOperationParams(operation, params); + const formalArgs = operation.operationParams || []; + return formalArgs.map(({ name, type, required }) => ({ + name, + isMissing: !(name in paramsObj), + isCorrectType: typeof paramsObj[name] === type, + isRequired: required, + })); +} + +export function shouldHaveFieldArgument(node: TinymathFunction) { + return !['count'].includes(node.name); +} + +export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { + return isObject(arg) && arg.type === type; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 030b69202d042e..96f543d9f9380b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -35,8 +35,12 @@ import { MovingAverageIndexPatternColumn, } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; -import { mathOperation, MathIndexPatternColumn } from './math'; -import { formulaOperation, FormulaIndexPatternColumn } from './formula'; +import { + mathOperation, + MathIndexPatternColumn, + formulaOperation, + FormulaIndexPatternColumn, +} from './formula'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; From 8ecde1dc997fc82cc57b34297ea2f73e54148c7a Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 4 Feb 2021 19:01:21 -0500 Subject: [PATCH 021/185] Use real fields --- .../definitions/formula/formula.tsx | 8 ++-- .../definitions/formula/math_completion.ts | 38 ++++++++----------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 88cf72689e5ff7..dc8b50dcb23357 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -162,8 +162,6 @@ function FormulaEditor({ operationDefinitionMap, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); - const functionList = useRef([]); - const kibana = useKibana(); const argValueSuggestions = useMemo(() => [], []); const provideCompletionItems = useCallback( @@ -200,7 +198,8 @@ function FormulaEditor({ innerText.substring(0, innerText.length - lengthAfterPosition) + ')', innerText.length - lengthAfterPosition, context, - undefined + undefined, + indexPattern ); } } else { @@ -215,7 +214,8 @@ function FormulaEditor({ innerText, innerText.length - lengthAfterPosition, context, - wordUntil + wordUntil, + indexPattern ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index e465b156640c0d..3584bca270aec0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -10,6 +10,7 @@ import { monaco } from '@kbn/monaco'; import { Parser } from 'pegjs'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import { IndexPattern } from '../../../types'; import type { GenericOperationDefinition } from '..'; import { operationDefinitionMap } from '..'; @@ -153,7 +154,8 @@ export async function suggest( expression: string, position: number, context: monaco.languages.CompletionContext, - word: monaco.editor.IWordAtPosition + word: monaco.editor.IWordAtPosition, + indexPattern: IndexPattern ): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { @@ -169,7 +171,11 @@ export async function suggest( tokenInfo.parent.args[tokenInfo.parent.args.length - 1] ); } else if (tokenInfo?.parent) { - return getArgumentSuggestions(tokenInfo.parent.name, tokenInfo.parent.args.length); + return getArgumentSuggestions( + tokenInfo.parent.name, + tokenInfo.parent.args.length, + indexPattern + ); } if (tokenInfo) { return getFunctionSuggestions(word); @@ -187,43 +193,31 @@ function getFunctionSuggestions(word: monaco.editor.IWordAtPosition) { return { list, type: SUGGESTION_TYPE.FUNCTIONS }; } -function getArgumentSuggestions(name: string, position: number) { +function getArgumentSuggestions(name: string, position: number, indexPattern: IndexPattern) { const operation = operationDefinitionMap[name]; if (!operation) { return { list: [], type: SUGGESTION_TYPE.FIELD }; } + const fields = indexPattern.fields + .filter((field) => field.type === 'number') + .map((field) => field.name); + if (operation.input === 'field') { - return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + return { list: fields, type: SUGGESTION_TYPE.FIELD }; } if (operation.input === 'fullReference') { if (operation.selectionStyle === 'field') { - return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; + return { list: fields, type: SUGGESTION_TYPE.FIELD }; } - return { list: ['count', 'avg'], type: SUGGESTION_TYPE.FUNCTIONS }; + return { list: Object.keys(operationDefinitionMap), type: SUGGESTION_TYPE.FUNCTIONS }; } return { list: [], type: SUGGESTION_TYPE.FIELD }; } function getArgValueSuggestions(name: string, position: number) { - const operation = operationDefinitionMap[name]; - if (!operation) { - return { list: [], type: SUGGESTION_TYPE.FIELD }; - } - - if (operation.input === 'field') { - return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; - } - - if (operation.input === 'fullReference') { - if (operation.selectionStyle === 'field') { - return { list: ['bytes', 'memory'], type: SUGGESTION_TYPE.FIELD }; - } - return { list: ['count', 'avg'], type: SUGGESTION_TYPE.FUNCTIONS }; - } - return { list: [], type: SUGGESTION_TYPE.FIELD }; } From 17f49fa98d90872dda3652a6ebd3180dbb2df37a Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 5 Feb 2021 12:42:26 +0100 Subject: [PATCH 022/185] :bug: Fix most of test issues + hide Math op --- .../dimension_panel/dimension_editor.tsx | 1 + .../definitions/formula/formula.test.tsx | 30 ++-- .../definitions/formula/formula.tsx | 54 +++--- .../operations/definitions/formula/math.tsx | 68 +------- .../operations/definitions/formula/util.ts | 65 ++++++- .../definitions/formula/validation.ts | 165 ++++++++++-------- .../operations/definitions/helpers.test.ts | 2 +- .../operations/definitions/index.ts | 4 + .../operations/operations.test.ts | 8 + 9 files changed, 216 insertions(+), 181 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 0d71971d5d48bb..dbcf3217247857 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -144,6 +144,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) + .filter(({ hidden }) => !hidden) .sort((op1, op2) => { return op1.displayName.localeCompare(op2.displayName); }) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index bfd341b4363a7e..727681526b26de 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -359,15 +359,14 @@ describe('formula', () => { ) ).toEqual([`The field bytes cannot be used without operation`]); - // TODO: enable this later - // expect( - // formulaOperation.getErrorMessage!( - // getNewLayerWithFormula('bytes + bytes'), - // 'col1', - // indexPattern, - // operationDefinitionMap - // ) - // ).toEqual([`The field bytes cannot be used without operation`]); + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes + bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`Math operations are allowed between operations, not fields`]); }); it('returns an error if parsing a syntax invalid formula', () => { @@ -458,11 +457,14 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual([ - expect.stringMatching( - `The first argument for ${formula.substring(0, formula.indexOf('('))}` - ), - ]); + ).toEqual( + // some formulas may contain more errors + expect.arrayContaining([ + expect.stringMatching( + `The first argument for ${formula.substring(0, formula.indexOf('('))}` + ), + ]) + ); } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 4c29ceede618c0..7cfab886cf2ba7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -13,16 +13,16 @@ import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } f import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern, IndexPatternLayer } from '../../../types'; import { getColumnOrder } from '../../layer_helpers'; -import { mathOperation, hasMathNode, findVariables } from './math'; +import { mathOperation } from './math'; import { documentField } from '../../../document_field'; +import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; import { - errorsLookup, - isParsingError, - runASTValidation, - shouldHaveFieldArgument, - tryToParse, -} from './validation'; -import { getOperationParams, getSafeFieldName, groupArgsByType } from './util'; + findVariables, + getOperationParams, + getSafeFieldName, + groupArgsByType, + hasMathNode, +} from './util'; export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; @@ -56,8 +56,8 @@ export const formulaOperation: OperationDefinition< return; } const { root, error } = tryToParse(column.params.formula); - if (error) { - return [error]; + if (root == null) { + return [error as string]; } const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); @@ -80,7 +80,7 @@ export const formulaOperation: OperationDefinition< function: 'mapColumn', arguments: { id: [columnId], - name: [label], + name: [label || ''], exp: [ { type: 'expression', @@ -129,8 +129,8 @@ export const formulaOperation: OperationDefinition< }, isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable - const { root, error } = tryToParse(column.params.formula || ''); - return Boolean(!error && !hasMathNode(root)); + const { root } = tryToParse(column.params.formula || ''); + return Boolean(root != null && !hasMathNode(root)); }, paramEditor: function ParamEditor({ @@ -178,23 +178,20 @@ function parseAndExtract( indexPattern: IndexPattern, operationDefinitionMap: Record ) { - try { - const { root } = tryToParse(text, { shouldThrow: true }); - // before extracting the data run the validation task and throw if invalid - runASTValidation(root, layer, indexPattern, operationDefinitionMap, { shouldThrow: true }); - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); - return { extracted, isValid: true }; - } catch (e) { - const context = e.message as string; - // propagate the error if it's one of those not controlled by the Formula logic - if (!errorsLookup.has(context) && !isParsingError(context)) { - throw e; - } + const { root, error } = tryToParse(text); + if (error || !root) { return { extracted: [], isValid: false }; } + // before extracting the data run the validation task and throw if invalid + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + if (errors.length) { + return { extracted: [], isValid: false }; + } + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; } export function regenerateLayerFromAst( @@ -344,6 +341,7 @@ function extractColumns( // replace by new column id return newColId; } + return node; } const root = parseNode(ast); const variables = findVariables(root); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 50674a85764cb7..e28f582e4fd2c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -6,12 +6,13 @@ */ import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import { i18n } from '@kbn/i18n'; import { isObject } from 'lodash'; -import { OperationDefinition, GenericOperationDefinition } from '../index'; +import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern } from '../../../types'; - -const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); +import { groupArgsByType } from './util'; +import { validateMathNodes } from './validation'; export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'math'; @@ -30,6 +31,7 @@ export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn export const mathOperation: OperationDefinition = { type: 'math', displayName: 'Math', + hidden: true, getDefaultLabel: (column, indexPattern) => 'Math', input: 'managedReference', getDisabledStatus(indexPattern: IndexPattern) { @@ -99,63 +101,3 @@ function astToString(ast: TinymathAST | string): string | number { } return `${ast.name}(${ast.args.map(astToString).join(',')})`; } - -export function isMathNode(node: TinymathAST) { - return isObject(node) && node.type === 'function' && tinymathValidOperators.has(node.name); -} - -function findMathNodes(root: TinymathAST | string): TinymathFunction[] { - function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function' || !isMathNode(node)) { - return []; - } - return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean); - } - return flattenMathNodes(root); -} - -export function hasMathNode(root: TinymathAST): boolean { - return Boolean(findMathNodes(root).length); -} - -function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { - function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); - } - return flattenFunctionNodes(root); -} - -export function hasInvalidOperations( - node: TinymathAST | string, - operations: Record -) { - // avoid duplicates - return Array.from( - new Set( - findFunctionNodes(node) - .filter((v) => !isMathNode(v) && !operations[v.name]) - .map(({ name }) => name) - ) - ); -} - -// traverse a tree and find all string leaves -export function findVariables(node: TinymathAST | string | undefined): string[] { - if (node == null) { - return []; - } - if (typeof node === 'string') { - return [node]; - } - if (typeof node === 'number' || node.type === 'namedArgument') { - return []; - } - if (node.type === 'variable') { - // leaf node - return [node.value]; - } - return node.args.flatMap(findVariables); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 01791fb154786e..c4714e8aeaf871 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -6,8 +6,8 @@ */ import { groupBy, isObject } from 'lodash'; -import type { TinymathAST, TinymathNamedArgument } from 'packages/kbn-tinymath'; -import type { OperationDefinition, IndexPatternColumn } from '../index'; +import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from 'packages/kbn-tinymath'; +import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; import type { GroupedNodes } from './types'; export function groupArgsByType(args: TinymathAST[]) { @@ -64,3 +64,64 @@ export function getOperationParams( return args; }, {}); } +export const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); + +export function isMathNode(node: TinymathAST) { + return isObject(node) && node.type === 'function' && tinymathValidOperators.has(node.name); +} + +export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function' || !isMathNode(node)) { + return []; + } + return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean); + } + return flattenMathNodes(root); +} + +export function hasMathNode(root: TinymathAST): boolean { + return Boolean(findMathNodes(root).length); +} + +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( + node: TinymathAST | string, + operations: Record +) { + // avoid duplicates + return Array.from( + new Set( + findFunctionNodes(node) + .filter((v) => !isMathNode(v) && !operations[v.name]) + .map(({ name }) => name) + ) + ); +} + +// traverse a tree and find all string leaves +export function findVariables(node: TinymathAST | string | undefined): string[] { + if (node == null) { + return []; + } + if (typeof node === 'string') { + return [node]; + } + if (typeof node === 'number' || node.type === 'namedArgument') { + return []; + } + if (node.type === 'variable') { + // leaf node + return [node.value]; + } + return node.args.flatMap(findVariables); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 58453c0f5fcd67..a3e9f519363d2d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -9,8 +9,15 @@ import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse } from '@kbn/tinymath'; import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; -import { getOperationParams, getValueOrName, groupArgsByType } from './util'; -import { findVariables, hasInvalidOperations, isMathNode } from './math'; +import { + findMathNodes, + findVariables, + getOperationParams, + getValueOrName, + groupArgsByType, + hasInvalidOperations, + isMathNode, +} from './util'; import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; import type { IndexPattern, IndexPatternLayer } from '../../../types'; @@ -36,7 +43,13 @@ export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing); } -function getMessageFromId(messageId: ErrorTypes, values: Record) { +function getMessageFromId({ + messageId, + values, +}: { + messageId: ErrorTypes; + values: Record; +}) { switch (messageId) { case 'wrongFirstArgument': return i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { @@ -93,34 +106,18 @@ function getMessageFromId(messageId: ErrorTypes, values: Record; - shouldThrow?: boolean; -}) { - if (shouldThrow) { - throw Error(validationErrors[messageId]); - } - return getMessageFromId(messageId, values); -} - -export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boolean } = {}) { +export function tryToParse(formula: string) { let root; try { root = parse(formula); } catch (e) { - if (shouldThrow) { - // propagate the error - throw e; - } return { root: null, - error: getMessageFromId('failedParsing', { - expression: formula, + error: getMessageFromId({ + messageId: 'failedParsing', + values: { + expression: formula, + }, }), }; } @@ -131,30 +128,23 @@ export function runASTValidation( ast: TinymathAST, layer: IndexPatternLayer, indexPattern: IndexPattern, - operations: Record, - options: { shouldThrow?: boolean } = {} + operations: Record ) { return [ - ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations, options), - ...runFullASTValidation(ast, indexPattern, operations, options), + ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations), + ...runFullASTValidation(ast, indexPattern, layer, operations), ]; } -function checkVariableEdgeCases( - ast: TinymathAST, - missingVariables: string[], - { shouldThrow }: { shouldThrow?: boolean } = {} -) { +function checkVariableEdgeCases(ast: TinymathAST, missingVariables: Set) { const invalidVariableErrors = []; - // TODO: add check for Math operation of fields as well - if (isObject(ast) && ast.type === 'variable' && !missingVariables.includes(ast.value)) { + if (isObject(ast) && ast.type === 'variable' && !missingVariables.has(ast.value)) { invalidVariableErrors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'fieldWithNoOperation', values: { field: ast.value, }, - shouldThrow, }) ); } @@ -165,59 +155,68 @@ function checkMissingVariableOrFunctions( ast: TinymathAST, layer: IndexPatternLayer, indexPattern: IndexPattern, - operations: Record, - { shouldThrow }: { shouldThrow?: boolean } = {} + operations: Record ) { const missingErrors: string[] = []; const missingOperations = hasInvalidOperations(ast, operations); if (missingOperations.length) { missingErrors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'missingOperation', values: { operationLength: missingOperations.length, operationsList: missingOperations.join(', '), }, - shouldThrow, }) ); } - const missingVariables = findVariables(ast).filter( - // filter empty string as well? - (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] - ); + const missingVariables = getMissingVariables(ast, indexPattern, layer); // need to check the arguments here: check only strings for now - if (missingVariables.length) { + if (missingVariables.size) { missingErrors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'missingField', values: { variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), + variablesList: Array.from(missingVariables).join(', '), }, - shouldThrow, }) ); } - const invalidVariableErrors = checkVariableEdgeCases(ast, missingErrors, { shouldThrow }); + const invalidVariableErrors = checkVariableEdgeCases(ast, missingVariables); return [...missingErrors, ...invalidVariableErrors]; } +function getMissingVariables( + root: TinymathAST, + indexPattern: IndexPattern, + layer: IndexPatternLayer +) { + return new Set( + findVariables(root).filter( + // filter empty string as well? + (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ) + ); +} + function runFullASTValidation( ast: TinymathAST, indexPattern: IndexPattern, - operations: Record, - { shouldThrow }: { shouldThrow?: boolean } = {} + layer: IndexPatternLayer, + operations: Record ) { + const missingVariables = getMissingVariables(ast, indexPattern, layer); + function validateNode(node: TinymathAST): string[] { if (!isObject(node) || node.type !== 'function') { return []; } const nodeOperation = operations[node.name]; if (!nodeOperation) { - return []; + return [...validateMathNodes(node, missingVariables)]; } const errors: string[] = []; @@ -228,64 +227,59 @@ function runFullASTValidation( if (shouldHaveFieldArgument(node)) { if (!isFirstArgumentValidType(firstArg, 'variable')) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'wrongFirstArgument', values: { operation: node.name, type: 'field', argument: getValueOrName(firstArg), }, - shouldThrow, }) ); } } else { if (firstArg) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'shouldNotHaveField', values: { operation: node.name, }, - shouldThrow, }) ); } } if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'cannotAcceptParameter', values: { operation: node.name, }, - shouldThrow, }) ); } else { const missingParams = getMissingParams(nodeOperation, namedArguments); if (missingParams.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'missingParameter', values: { operation: node.name, params: missingParams.map(({ name }) => name).join(', '), }, - shouldThrow, }) ); } const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); if (wrongTypeParams.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'wrongTypeParameter', values: { operation: node.name, params: wrongTypeParams.map(({ name }) => name).join(', '), }, - shouldThrow, }) ); } @@ -295,51 +289,47 @@ function runFullASTValidation( if (nodeOperation.input === 'fullReference') { if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'wrongFirstArgument', values: { operation: node.name, - type: 'function', + type: 'operation', argument: getValueOrName(firstArg), }, - shouldThrow, }) ); } if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'cannotAcceptParameter', values: { operation: node.name, }, - shouldThrow, }) ); } else { const missingParameters = getMissingParams(nodeOperation, namedArguments); if (missingParameters.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'missingParameter', values: { operation: node.name, params: missingParameters.map(({ name }) => name).join(', '), }, - shouldThrow, }) ); } const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); if (wrongTypeParams.length) { errors.push( - addErrorOrThrow({ + getMessageFromId({ messageId: 'wrongTypeParameter', values: { operation: node.name, params: wrongTypeParams.map(({ name }) => name).join(', '), }, - shouldThrow, }) ); } @@ -417,3 +407,32 @@ export function shouldHaveFieldArgument(node: TinymathFunction) { export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { return isObject(arg) && arg.type === type; } + +export function validateMathNodes(root: TinymathAST, missingVariableSet: Set) { + const mathNodes = findMathNodes(root); + const errors = []; + const areThereInvalidNodes = mathNodes.some((node: TinymathFunction) => { + // check the following patterns: + const { variables } = groupArgsByType(node.args); + const fieldVariables = variables.filter((v) => isObject(v) && !missingVariableSet.has(v.value)); + // field + field (or string) + const atLeastTwoFields = fieldVariables.length > 1; + // field + number + // when computing the difference, exclude invalid fields + const validVariables = variables.filter( + (v) => !isObject(v) || !missingVariableSet.has(v.value) + ); + // Make sure to have at least one valid field to compare, or skip the check + const mathBetweenFieldAndNumbers = + fieldVariables.length > 0 && validVariables.length - fieldVariables.length > 0; + return atLeastTwoFields || mathBetweenFieldAndNumbers; + }); + if (areThereInvalidNodes) { + errors.push( + i18n.translate('xpack.lens.indexPattern.mathNotAllowedBetweenFields', { + defaultMessage: 'Math operations are allowed between operations, not fields', + }) + ); + } + return errors; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts index 2c0bf895f9dae1..3a168d193c94c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts @@ -37,7 +37,7 @@ describe('helpers', () => { createMockedIndexPattern() ); expect(messages).toHaveLength(1); - expect(messages![0]).toEqual('Field timestamp was not found'); + expect(messages![0]).toEqual('Field timestamp is of the wrong type'); }); it('returns no message if all fields are matching', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index b93097221bed33..373edb0ef06629 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -245,6 +245,10 @@ interface BaseOperationDefinitionProps { timeScalingMode?: TimeScalingMode; getHelpMessage?: (props: HelpProps) => React.ReactNode; + /* + * Operations can be used as middleware for other operations, hence not shown in the panel UI + */ + hidden?: boolean; } interface BaseBuildColumnArgs { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 4249f8397716a1..1c8d815f55de97 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -304,6 +304,14 @@ describe('getOperationTypesForField', () => { "operationType": "last_value", "type": "field", }, + Object { + "operationType": "math", + "type": "managedReference", + }, + Object { + "operationType": "formula", + "type": "managedReference", + }, ], }, Object { From c919670ce2bc1dce5afc2b6bd82824c964b3e85c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 5 Feb 2021 18:00:37 -0500 Subject: [PATCH 023/185] Show errors in formula editor --- .../definitions/formula/formula.tsx | 124 +++++++++++++----- .../operations/definitions/formula/math.tsx | 30 ++--- .../definitions/formula/math_completion.ts | 114 +--------------- .../definitions/formula/validation.ts | 103 ++++++++++----- 4 files changed, 184 insertions(+), 187 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index dc8b50dcb23357..a581fc60baea4b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,19 +5,13 @@ * 2.0. */ -import React, { useRef, useCallback, useMemo, useEffect, useState } from 'react'; -import { i18n } from '@kbn/i18n'; -import { groupBy, isObject } from 'lodash'; -import { - parse, - TinymathFunction, - TinymathVariable, - TinymathNamedArgument, - TinymathAST, -} from '@kbn/tinymath'; -import { EuiFormRow, EuiFlexItem, EuiFlexGroup, EuiButton } from '@elastic/eui'; +import React, { useCallback, useMemo, useState } from 'react'; +import { isObject } from 'lodash'; +import { TinymathVariable, TinymathAST } from '@kbn/tinymath'; +import { EuiFlexItem, EuiFlexGroup, EuiButton } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; -import { CodeEditor, useKibana } from '../../../../../../../../src/plugins/kibana_react/public'; +import { CodeEditor } from '../../../../../../../../src/plugins/kibana_react/public'; +import { useDebounceWithOptions } from '../helpers'; import { OperationDefinition, GenericOperationDefinition, @@ -35,8 +29,9 @@ import { runASTValidation, shouldHaveFieldArgument, tryToParse, + ErrorWrapper, } from './validation'; -import { suggest, getSuggestion, LensMathSuggestion } from './math_completion'; +import { suggest, getSuggestion, LensMathSuggestion, SUGGESTION_TYPE } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; import { getOperationParams, getSafeFieldName, groupArgsByType } from './util'; @@ -72,12 +67,15 @@ export const formulaOperation: OperationDefinition< return; } const { root, error } = tryToParse(column.params.formula); + if (!root) { + return []; + } if (error) { - return [error]; + return [error.message]; } const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); - return errors.length ? errors : undefined; + return errors.length ? errors.map(({ message }) => message) : undefined; }, getPossibleOperation() { return { @@ -96,7 +94,7 @@ export const formulaOperation: OperationDefinition< function: 'mapColumn', arguments: { id: [columnId], - name: [label], + name: [label || ''], exp: [ { type: 'expression', @@ -146,6 +144,7 @@ export const formulaOperation: OperationDefinition< isTransferable: (column, newIndexPattern, operationDefinitionMap) => { // Basic idea: if it has any math operation in it, probably it cannot be transferable const { root, error } = tryToParse(column.params.formula || ''); + if (!root) return true; return Boolean(!error && !hasMathNode(root)); }, @@ -162,8 +161,61 @@ function FormulaEditor({ operationDefinitionMap, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); + const editorModel = React.useRef(null); const argValueSuggestions = useMemo(() => [], []); + useDebounceWithOptions( + () => { + if (!editorModel.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (!root) return; + if (error) { + errors = [error]; + } else { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel.current, + 'LENS', + errors.flatMap((innerError) => + innerError.locations.map((location) => ({ + message: innerError.message, + startColumn: location.min + 1, + endColumn: location.max + 1, + // Fake, assumes single line + startLineNumber: 1, + endLineNumber: 1, + severity: monaco.MarkerSeverity.Error, + })) + ) + ); + } else { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + } + }, + { skipFirstRender: true }, + 256, + [text] + ); + const provideCompletionItems = useCallback( async ( model: monaco.editor.ITextModel, @@ -173,7 +225,10 @@ function FormulaEditor({ const innerText = model.getValue(); const textRange = model.getFullModelRange(); let wordRange: monaco.Range; - let aSuggestions; + let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { + list: [], + type: SUGGESTION_TYPE.FIELD, + }; const lengthAfterPosition = model.getValueLengthInRange({ startLineNumber: position.lineNumber, @@ -198,7 +253,6 @@ function FormulaEditor({ innerText.substring(0, innerText.length - lengthAfterPosition) + ')', innerText.length - lengthAfterPosition, context, - undefined, indexPattern ); } @@ -214,20 +268,16 @@ function FormulaEditor({ innerText, innerText.length - lengthAfterPosition, context, - wordUntil, - indexPattern + indexPattern, + wordUntil ); } return { - suggestions: aSuggestions - ? aSuggestions.list.map((s: IMathFunction | MathFunctionArgs) => - getSuggestion(s, aSuggestions.type, wordRange) - ) - : [], + suggestions: aSuggestions.list.map((s) => getSuggestion(s, aSuggestions.type, wordRange)), }; }, - [argValueSuggestions] + [indexPattern] ); return ( @@ -235,7 +285,6 @@ function FormulaEditor({ { + const model = editor.getModel(); + if (model) { + editorModel.current = model; + } + editor.onDidDispose(() => (editorModel.current = null)); + }} /> @@ -291,6 +346,9 @@ function parseAndExtract( ) { try { const { root } = tryToParse(text, { shouldThrow: true }); + if (!root) { + return { extracted: [], isValid: false }; + } // before extracting the data run the validation task and throw if invalid runASTValidation(root, layer, indexPattern, operationDefinitionMap, { shouldThrow: true }); /* @@ -425,13 +483,13 @@ function extractColumns( const [referencedOp] = functions; const consumedParam = parseNode(referencedOp); - const subNodeVariables = findVariables(consumedParam); + const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; const mathColumn = mathOperation.buildColumn({ layer, indexPattern, }); - mathColumn.references = subNodeVariables; - mathColumn.params.tinymathAst = consumedParam; + mathColumn.references = subNodeVariables.map(({ value }) => value); + mathColumn.params.tinymathAst = consumedParam!; columns.push(mathColumn); mathColumn.customLabel = true; mathColumn.label = `${idPrefix}X${columns.length - 1}`; @@ -462,8 +520,8 @@ function extractColumns( layer, indexPattern, }); - mathColumn.references = variables; - mathColumn.params.tinymathAst = root; + mathColumn.references = variables.map(({ value }) => value); + mathColumn.params.tinymathAst = root!; const newColId = `${idPrefix}X${columns.length}`; mathColumn.customLabel = true; mathColumn.label = newColId; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 50674a85764cb7..caac98b6c76466 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -5,7 +5,12 @@ * 2.0. */ -import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import type { + TinymathAST, + TinymathFunction, + TinymathLocation, + TinymathVariable, +} from '@kbn/tinymath'; import { isObject } from 'lodash'; import { OperationDefinition, GenericOperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; @@ -131,31 +136,26 @@ function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { export function hasInvalidOperations( node: TinymathAST | string, operations: Record -) { - // avoid duplicates - return Array.from( - new Set( - findFunctionNodes(node) - .filter((v) => !isMathNode(v) && !operations[v.name]) - .map(({ name }) => name) - ) - ); +): { names: string[]; locations: TinymathLocation[] } { + const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); + return { + // avoid duplicates + names: Array.from(new Set(nodes.map(({ name }) => name))), + locations: nodes.map(({ location }) => location), + }; } // traverse a tree and find all string leaves -export function findVariables(node: TinymathAST | string | undefined): string[] { +export function findVariables(node: TinymathAST): TinymathVariable[] { if (node == null) { return []; } - if (typeof node === 'string') { - return [node]; - } if (typeof node === 'number' || node.type === 'namedArgument') { return []; } if (node.type === 'variable') { // leaf node - return [node.value]; + return [node]; } return node.args.flatMap(findVariables); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 3584bca270aec0..68707f63fcb59c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -5,10 +5,9 @@ * 2.0. */ -import { get, startsWith } from 'lodash'; +import { startsWith } from 'lodash'; import { monaco } from '@kbn/monaco'; -import { Parser } from 'pegjs'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; import type { GenericOperationDefinition } from '..'; @@ -30,100 +29,6 @@ function inLocation(cursorPosition: number, location: TinymathLocation) { return cursorPosition >= location.min && cursorPosition <= location.max; } -function getArgumentsHelp( - functionHelp: ILensFunction | undefined, - functionArgs: FunctionArg[] = [] -) { - if (!functionHelp) { - return []; - } - - // Do not provide 'inputSeries' as argument suggestion for chainable functions - const argsHelp = functionHelp.chainable ? functionHelp.args.slice(1) : functionHelp.args.slice(0); - - // ignore arguments that are already provided in function declaration - const functionArgNames = functionArgs.map((arg) => arg.name); - return argsHelp.filter((arg) => !functionArgNames.includes(arg.name)); -} - -async function extractSuggestionsFromParsedResult( - result: ReturnType, - cursorPosition: number, - functionList: ILensFunction[], - argValueSuggestions: ArgValueSuggestions -) { - const activeFunc = result.functions.find(({ location }: { location: Location }) => - inLocation(cursorPosition, location) - ); - - if (!activeFunc) { - return; - } - - const functionHelp = functionList.find(({ name }) => name === activeFunc.function); - - if (!functionHelp) { - return; - } - - // return function suggestion when cursor is outside of parentheses - // location range includes '.', function name, and '('. - const openParen = activeFunc.location.min + activeFunc.function.length + 2; - if (cursorPosition < openParen) { - return { list: [functionHelp], type: SUGGESTION_TYPE.FUNCTIONS }; - } - - // return argument value suggestions when cursor is inside argument value - const activeArg = activeFunc.arguments.find((argument: FunctionArg) => { - return inLocation(cursorPosition, argument.location); - }); - if ( - activeArg && - activeArg.type === 'namedArg' && - inLocation(cursorPosition, activeArg.value.location) - ) { - const { function: functionName, arguments: functionArgs } = activeFunc; - - const { - name: argName, - value: { text: partialInput }, - } = activeArg; - - let valueSuggestions; - if (argValueSuggestions.hasDynamicSuggestionsForArgument(functionName, argName)) { - valueSuggestions = await argValueSuggestions.getDynamicSuggestionsForArgument( - functionName, - argName, - functionArgs, - partialInput - ); - } else { - const { suggestions: staticSuggestions } = - functionHelp.args.find((arg) => arg.name === activeArg.name) || {}; - valueSuggestions = argValueSuggestions.getStaticSuggestionsForInput( - partialInput, - staticSuggestions - ); - } - return { - list: valueSuggestions, - type: SUGGESTION_TYPE.ARGUMENT_VALUE, - }; - } - - // return argument suggestions - const argsHelp = getArgumentsHelp(functionHelp, activeFunc.arguments); - const argumentSuggestions = argsHelp.filter((arg) => { - if (get(activeArg, 'type') === 'namedArg') { - return startsWith(arg.name, activeArg.name); - } else if (activeArg) { - return startsWith(arg.name, activeArg.text); - } - return true; - }); - return { list: argumentSuggestions, type: SUGGESTION_TYPE.ARGUMENTS }; -} - const MARKER = 'LENS_MATH_MARKER'; function getInfoAtPosition( @@ -154,8 +59,8 @@ export async function suggest( expression: string, position: number, context: monaco.languages.CompletionContext, - word: monaco.editor.IWordAtPosition, - indexPattern: IndexPattern + indexPattern: IndexPattern, + word?: monaco.editor.IWordAtPosition ): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { @@ -163,13 +68,8 @@ export async function suggest( const tokenInfo = getInfoAtPosition(ast, position); - console.log(ast, getInfoAtPosition(ast, position)); - if (context.triggerCharacter === '=' && tokenInfo?.parent) { - return getArgValueSuggestions( - tokenInfo.parent.name, - tokenInfo.parent.args[tokenInfo.parent.args.length - 1] - ); + // TODO: Look for keys of named arguments before named argument values } else if (tokenInfo?.parent) { return getArgumentSuggestions( tokenInfo.parent.name, @@ -177,7 +77,7 @@ export async function suggest( indexPattern ); } - if (tokenInfo) { + if (tokenInfo && word) { return getFunctionSuggestions(word); } } catch (e) { @@ -217,10 +117,6 @@ function getArgumentSuggestions(name: string, position: number, indexPattern: In return { list: [], type: SUGGESTION_TYPE.FIELD }; } -function getArgValueSuggestions(name: string, position: number) { - return { list: [], type: SUGGESTION_TYPE.FIELD }; -} - export function getSuggestion( suggestion: LensMathSuggestion, type: SUGGESTION_TYPE, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 58453c0f5fcd67..de9e6f60d56326 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -8,7 +8,12 @@ import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse } from '@kbn/tinymath'; -import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; +import type { + TinymathAST, + TinymathFunction, + TinymathNamedArgument, + TinymathLocation, +} from '@kbn/tinymath'; import { getOperationParams, getValueOrName, groupArgsByType } from './util'; import { findVariables, hasInvalidOperations, isMathNode } from './math'; @@ -32,80 +37,104 @@ export const errorsLookup = new Set(Object.values(validationErrors)); type ErrorTypes = keyof typeof validationErrors; +export interface ErrorWrapper { + message: string; + locations: TinymathLocation[]; +} + export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing); } -function getMessageFromId(messageId: ErrorTypes, values: Record) { +function getMessageFromId( + messageId: ErrorTypes, + values: Record, + locations: TinymathLocation[] +): ErrorWrapper { + let message: string; switch (messageId) { case 'wrongFirstArgument': - return i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + message = i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { defaultMessage: 'The first argument for {operation} should be a {type} name. Found {argument}', values, }); + break; case 'shouldNotHaveField': - return i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { + message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { defaultMessage: 'The operation {operation} does not accept any field as argument', values, }); + break; case 'cannotAcceptParameter': - return i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + message = i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { defaultMessage: 'The operation {operation} does not accept any parameter', values, }); + break; case 'missingParameter': - return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The operation {operation} in the Formula is missing the following parameters: {params}', values, }); + break; case 'wrongTypeParameter': - return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', values, }); + break; case 'missingField': - return i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + message = i18n.translate('xpack.lens.indexPattern.fieldNotFound', { defaultMessage: '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', values, }); + break; case 'missingOperation': - return i18n.translate('xpack.lens.indexPattern.operationsNotFound', { + message = i18n.translate('xpack.lens.indexPattern.operationsNotFound', { defaultMessage: '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', values, }); + break; case 'fieldWithNoOperation': - return i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { + message = i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { defaultMessage: 'The field {field} cannot be used without operation', values, }); + break; case 'failedParsing': - return i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The Formula {expression} cannot be parsed', values, }); + break; default: - return 'no Error found'; + message = 'no Error found'; + break; } + + return { message, locations }; } function addErrorOrThrow({ messageId, values, + locations, shouldThrow, }: { messageId: ErrorTypes; values: Record; + locations: TinymathLocation[]; shouldThrow?: boolean; }) { if (shouldThrow) { throw Error(validationErrors[messageId]); } - return getMessageFromId(messageId, values); + return getMessageFromId(messageId, values, locations); } export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boolean } = {}) { @@ -118,10 +147,8 @@ export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boo throw e; } return { - root: null, - error: getMessageFromId('failedParsing', { - expression: formula, - }), + root: undefined, + error: getMessageFromId('failedParsing', { expression: formula }, []), }; } return { root, error: null }; @@ -133,7 +160,7 @@ export function runASTValidation( indexPattern: IndexPattern, operations: Record, options: { shouldThrow?: boolean } = {} -) { +): ErrorWrapper[] { return [ ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations, options), ...runFullASTValidation(ast, indexPattern, operations, options), @@ -155,6 +182,7 @@ function checkVariableEdgeCases( field: ast.value, }, shouldThrow, + locations: [ast.location], }) ); } @@ -167,25 +195,26 @@ function checkMissingVariableOrFunctions( indexPattern: IndexPattern, operations: Record, { shouldThrow }: { shouldThrow?: boolean } = {} -) { - const missingErrors: string[] = []; +): ErrorWrapper[] { + const missingErrors: ErrorWrapper[] = []; const missingOperations = hasInvalidOperations(ast, operations); - if (missingOperations.length) { + if (missingOperations.names.length) { missingErrors.push( addErrorOrThrow({ messageId: 'missingOperation', values: { - operationLength: missingOperations.length, - operationsList: missingOperations.join(', '), + operationLength: missingOperations.names.length, + operationsList: missingOperations.names.join(', '), }, shouldThrow, + locations: missingOperations.locations, }) ); } const missingVariables = findVariables(ast).filter( // filter empty string as well? - (variable) => !indexPattern.getFieldByName(variable) && !layer.columns[variable] + ({ value }) => !indexPattern.getFieldByName(value) && !layer.columns[value] ); // need to check the arguments here: check only strings for now @@ -194,14 +223,19 @@ function checkMissingVariableOrFunctions( addErrorOrThrow({ messageId: 'missingField', values: { - variablesLength: missingOperations.length, - variablesList: missingVariables.join(', '), + variablesLength: missingVariables.length, + variablesList: missingVariables.map(({ value }) => value).join(', '), }, shouldThrow, + locations: missingVariables.map(({ location }) => location), }) ); } - const invalidVariableErrors = checkVariableEdgeCases(ast, missingErrors, { shouldThrow }); + const invalidVariableErrors = checkVariableEdgeCases( + ast, + missingVariables.map(({ value }) => value), + { shouldThrow } + ); return [...missingErrors, ...invalidVariableErrors]; } @@ -210,8 +244,8 @@ function runFullASTValidation( indexPattern: IndexPattern, operations: Record, { shouldThrow }: { shouldThrow?: boolean } = {} -) { - function validateNode(node: TinymathAST): string[] { +): ErrorWrapper[] { + function validateNode(node: TinymathAST): ErrorWrapper[] { if (!isObject(node) || node.type !== 'function') { return []; } @@ -220,7 +254,7 @@ function runFullASTValidation( return []; } - const errors: string[] = []; + const errors: ErrorWrapper[] = []; const { namedArguments, functions } = groupArgsByType(node.args); const [firstArg] = node?.args || []; @@ -236,6 +270,7 @@ function runFullASTValidation( argument: getValueOrName(firstArg), }, shouldThrow, + locations: [node.location], }) ); } @@ -248,6 +283,7 @@ function runFullASTValidation( operation: node.name, }, shouldThrow, + locations: [node.location], }) ); } @@ -260,6 +296,7 @@ function runFullASTValidation( operation: node.name, }, shouldThrow, + locations: [node.location], }) ); } else { @@ -273,6 +310,7 @@ function runFullASTValidation( params: missingParams.map(({ name }) => name).join(', '), }, shouldThrow, + locations: [node.location], }) ); } @@ -286,6 +324,7 @@ function runFullASTValidation( params: wrongTypeParams.map(({ name }) => name).join(', '), }, shouldThrow, + locations: [node.location], }) ); } @@ -303,6 +342,7 @@ function runFullASTValidation( argument: getValueOrName(firstArg), }, shouldThrow, + locations: [node.location], }) ); } @@ -314,6 +354,7 @@ function runFullASTValidation( operation: node.name, }, shouldThrow, + locations: [node.location], }) ); } else { @@ -327,6 +368,7 @@ function runFullASTValidation( params: missingParameters.map(({ name }) => name).join(', '), }, shouldThrow, + locations: [node.location], }) ); } @@ -340,6 +382,7 @@ function runFullASTValidation( params: wrongTypeParams.map(({ name }) => name).join(', '), }, shouldThrow, + locations: [node.location], }) ); } From 155a98687034b460b86fbe480c595f69f8599641 Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 9 Feb 2021 15:20:34 +0100 Subject: [PATCH 024/185] :label: fix types --- .../operations/definitions/formula/math.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index e28f582e4fd2c5..345d407e4d6fc9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -5,14 +5,10 @@ * 2.0. */ -import type { TinymathAST, TinymathFunction } from '@kbn/tinymath'; -import { i18n } from '@kbn/i18n'; -import { isObject } from 'lodash'; +import type { TinymathAST } from '@kbn/tinymath'; import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern } from '../../../types'; -import { groupArgsByType } from './util'; -import { validateMathNodes } from './validation'; export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'math'; From a53d074819c9219d2add2f4774866aa3229d5d76 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 9 Feb 2021 19:28:48 -0500 Subject: [PATCH 025/185] Fix types and parsing --- .../operations/definitions/formula/formula.tsx | 9 ++++++--- .../operations/definitions/formula/math.tsx | 14 ++++++++++++-- .../definitions/formula/math_tokenization.tsx | 8 +++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index a581fc60baea4b..b7ca4d12cb35bb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useState } from 'react'; import { isObject } from 'lodash'; import { TinymathVariable, TinymathAST } from '@kbn/tinymath'; import { EuiFlexItem, EuiFlexGroup, EuiButton } from '@elastic/eui'; @@ -162,7 +162,6 @@ function FormulaEditor({ }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const editorModel = React.useRef(null); - const argValueSuggestions = useMemo(() => [], []); useDebounceWithOptions( () => { @@ -429,7 +428,8 @@ function extractColumns( ) { const columns: IndexPatternColumn[] = []; - function parseNode(node: TinymathAST) { + // String response indicates a new column to add + function parseNode(node: TinymathAST): TinymathAST | string | undefined { if (typeof node === 'number' || node.type !== 'function') { // leaf node return node; @@ -515,6 +515,9 @@ function extractColumns( } } const root = parseNode(ast); + if (root === undefined) { + return []; + } const variables = findVariables(root); const mathColumn = mathOperation.buildColumn({ layer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index caac98b6c76466..4cdc6f1215c092 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -145,8 +145,18 @@ export function hasInvalidOperations( }; } -// traverse a tree and find all string leaves -export function findVariables(node: TinymathAST): TinymathVariable[] { +// traverse a tree and find all string leaves. some string leaves are pre-parsed as column IDs +export function findVariables(node: TinymathAST | string): TinymathVariable[] { + if (typeof node === 'string') { + return [ + { + type: 'variable', + value: node, + text: node, + location: { min: 0, max: 0 }, + }, + ]; + } if (node == null) { return []; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx index ae2ef1530ac89b..918bad35337771 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx @@ -11,6 +11,7 @@ export const LANGUAGE_ID = 'lens_math'; monaco.languages.register({ id: LANGUAGE_ID }); export const languageConfiguration: monaco.languages.LanguageConfiguration = { + wordPattern: /[A-Za-z\.-_@]/g, brackets: [['(', ')']], autoClosingPairs: [ { open: '(', close: ')' }, @@ -26,19 +27,20 @@ export const languageConfiguration: monaco.languages.LanguageConfiguration = { export const lexerRules: monaco.languages.IMonarchLanguage = { defaultToken: 'invalid', + tokenPostfix: '', ignoreCase: true, brackets: [{ open: '(', close: ')', token: 'delimiter.parenthesis' }], tokenizer: { root: [ [/\s+/, 'whitespace'], - [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'identifier'], + [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'keyword'], [/[,=]/, 'delimiter'], [/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number'], [/".+?"/, 'string'], [/'.+?'/, 'string'], [/\+|\-|\*|\//, 'keyword.operator'], - [/[\(]/, 'paren.lparen'], - [/[\)]/, 'paren.rparen'], + [/[\(]/, 'delimiter'], + [/[\)]/, 'delimiter'], ], }, }; From d4c59982e33f53f54a61c100a8704358f24d583b Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 10 Feb 2021 14:43:11 +0100 Subject: [PATCH 026/185] :lipstick: Hack to fix suggestion box --- .../operations/definitions/formula/formula.scss | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss new file mode 100644 index 00000000000000..395a4df025f28c --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss @@ -0,0 +1,4 @@ +.monaco-editor .suggest-widget { + left: auto !important; + width: 100%; +} \ No newline at end of file From 4d5dca27a9b1cf139a3b04671a40a36558936373 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 19 Feb 2021 17:02:24 +0100 Subject: [PATCH 027/185] :bug: Fix validation messages --- .../definitions/formula/formula.tsx | 7 +--- .../definitions/formula/validation.ts | 42 +++++++++++++------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index d87559cbf69c80..449671e6d2fb52 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -68,11 +68,8 @@ export const formulaOperation: OperationDefinition< return; } const { root, error } = tryToParse(column.params.formula); - if (!root) { - return []; - } - if (error) { - return [error.message]; + if (error || !root) { + return [error!.message]; } const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index e5a1260e2e5a7b..bc27a5685d7439 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -126,13 +126,15 @@ function getMessageFromId({ return { message, locations }; } -export function tryToParse(formula: string, { shouldThrow }: { shouldThrow?: boolean } = {}) { +export function tryToParse( + formula: string +): { root: TinymathAST; error: null } | { root: null; error: ErrorWrapper } { let root; try { root = parse(formula); } catch (e) { return { - root: undefined, + root: null, error: getMessageFromId({ messageId: 'failedParsing', values: { @@ -247,17 +249,31 @@ function runFullASTValidation( if (nodeOperation.input === 'field') { if (shouldHaveFieldArgument(node)) { if (!isFirstArgumentValidType(firstArg, 'variable')) { - errors.push( - getMessageFromId({ - messageId: 'wrongFirstArgument', - values: { - operation: node.name, - type: 'field', - argument: getValueOrName(firstArg), - }, - locations: [node.location], - }) - ); + if (isMathNode(firstArg)) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: `math operation`, + }, + locations: [node.location], + }) + ); + } else { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(firstArg), + }, + locations: [node.location], + }) + ); + } } } else { if (firstArg) { From 7c38af37d5c885ebe0fe692a5c379b09b9da0c7f Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 19 Feb 2021 17:03:00 +0100 Subject: [PATCH 028/185] :bug: Relax operations check for managedReferences --- .../operations/layer_helpers.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index ba0fcd70ac4c33..c9cb7949bdfeda 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -996,6 +996,23 @@ export function resetIncomplete(layer: IndexPatternLayer, columnId: string): Ind return { ...layer, incompleteColumns }; } +// managedReferences have a relaxed policy about operation allowed, so let them pass +function maybeValidateOperations({ + column, + validation, +}: { + column: IndexPatternColumn; + validation: RequiredReference; +}) { + if (!validation.specificOperations) { + return true; + } + if (operationDefinitionMap[column.operationType].input === 'managedReference') { + return true; + } + return validation.specificOperations.includes(column.operationType); +} + export function isColumnValidAsReference({ column, validation, @@ -1008,7 +1025,10 @@ export function isColumnValidAsReference({ const operationDefinition = operationDefinitionMap[operationType]; return ( validation.input.includes(operationDefinition.input) && - (!validation.specificOperations || validation.specificOperations.includes(operationType)) && + maybeValidateOperations({ + column, + validation, + }) && validation.validateMetadata(column) ); } From a695329bb26d9734c0a376380a3b9ac48c54f71e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 19 Feb 2021 18:42:25 -0500 Subject: [PATCH 029/185] Change completion params --- .../config_panel/dimension_container.scss | 3 - .../definitions/formula/formula.scss | 5 +- .../definitions/formula/formula.tsx | 1 - .../definitions/formula/math_completion.ts | 64 ++++++++++++++----- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index 5adbdfcfc045e2..5947d62540a0db 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -11,9 +11,6 @@ top: 0; bottom: 0; animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; - clip-path: none; - // z-index: $euiZLevel1; - } .lnsDimensionContainer__footer { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss index 395a4df025f28c..3d30bd4fc1cd6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss @@ -1,4 +1,5 @@ .monaco-editor .suggest-widget { left: auto !important; - width: 100%; -} \ No newline at end of file + // Interior flyout width + width: 280px !important; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 449671e6d2fb52..af6ad89909a2be 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -299,7 +299,6 @@ function FormulaEditor({ minimap: { enabled: false, }, - wordBasedSuggestions: false, wordWrap: 'on', wrappingIndent: 'indent', }} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 68707f63fcb59c..5851073682a2e2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { startsWith } from 'lodash'; +import { uniq, startsWith } from 'lodash'; import { monaco } from '@kbn/monaco'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; +import { getAvailableOperationsByMetadata } from '../../'; import type { GenericOperationDefinition } from '..'; import { operationDefinitionMap } from '..'; @@ -26,7 +27,7 @@ export interface LensMathSuggestions { } function inLocation(cursorPosition: number, location: TinymathLocation) { - return cursorPosition >= location.min && cursorPosition <= location.max; + return cursorPosition >= location.min && cursorPosition < location.max; } const MARKER = 'LENS_MATH_MARKER'; @@ -39,12 +40,11 @@ function getInfoAtPosition( if (typeof ast === 'number') { return; } - // const type = getType(ast); if (!inLocation(position, ast.location)) { return; } if (ast.type === 'function') { - const [match] = ast.args.flatMap((arg) => getInfoAtPosition(arg, position, ast)); + const [match] = ast.args.map((arg) => getInfoAtPosition(arg, position, ast)).filter((a) => a); if (match) { return match.parent ? match : { ...match, parent: ast }; } @@ -69,11 +69,11 @@ export async function suggest( const tokenInfo = getInfoAtPosition(ast, position); if (context.triggerCharacter === '=' && tokenInfo?.parent) { - // TODO: Look for keys of named arguments before named argument values + // TODO } else if (tokenInfo?.parent) { return getArgumentSuggestions( tokenInfo.parent.name, - tokenInfo.parent.args.length, + tokenInfo.parent.args.length - 1, indexPattern ); } @@ -99,19 +99,43 @@ function getArgumentSuggestions(name: string, position: number, indexPattern: In return { list: [], type: SUGGESTION_TYPE.FIELD }; } - const fields = indexPattern.fields - .filter((field) => field.type === 'number') - .map((field) => field.name); + if (position > 0) { + if ('operationParams' in operation) { + const suggestedParam = operation.operationParams!.map((p) => p.name); + return { + list: suggestedParam, + type: SUGGESTION_TYPE.NAMED_ARGUMENT, + }; + } + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } - if (operation.input === 'field') { + if (operation.input === 'field' && position === 0) { + const fields = indexPattern.fields + .filter((field) => field.type === 'number') + .map((field) => field.name); return { list: fields, type: SUGGESTION_TYPE.FIELD }; } if (operation.input === 'fullReference') { - if (operation.selectionStyle === 'field') { - return { list: fields, type: SUGGESTION_TYPE.FIELD }; - } - return { list: Object.keys(operationDefinitionMap), type: SUGGESTION_TYPE.FUNCTIONS }; + const available = getAvailableOperationsByMetadata(indexPattern); + const possibleOperationNames: string[] = []; + available.forEach((a) => { + if ( + operation.requiredReferences.some((requirement) => + requirement.validateMetadata(a.operationMetaData) + ) + ) { + possibleOperationNames.push( + ...a.operations + .filter((o) => + operation.requiredReferences.some((requirement) => requirement.input.includes(o.type)) + ) + .map((o) => o.operationType) + ); + } + }); + return { list: uniq(possibleOperationNames), type: SUGGESTION_TYPE.FUNCTIONS }; } return { list: [], type: SUGGESTION_TYPE.FIELD }; @@ -148,6 +172,17 @@ export function getSuggestion( insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; detail = typeof suggestion === 'string' ? '' : `(${suggestion.displayName})`; + break; + case SUGGESTION_TYPE.NAMED_ARGUMENT: + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + kind = monaco.languages.CompletionItemKind.Field; + insertText = `${insertText}=`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + detail = ''; + break; } @@ -157,7 +192,6 @@ export function getSuggestion( insertTextRules, kind, label: insertText, - // documentation: suggestion.help, command, range, }; From ea574f7038fa2bdcf064517ccac63b158161c6e9 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 22 Feb 2021 17:26:48 +0100 Subject: [PATCH 030/185] :label: Fix missing arg issue --- .../operations/definitions/percentile.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index a4cb84adb32093..238ded090013a0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -203,7 +203,8 @@ describe('percentile', () => { percentile: 95, }, }, - indexPattern + indexPattern, + {} ) ).toBeTruthy(); }); From 417edff935d1db080e20a3cb6dac4aa6f4c468ba Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Mar 2021 15:15:15 +0100 Subject: [PATCH 031/185] :sparkles: Add more tinymath fns --- .../operations/definitions/formula/util.ts | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 4f0e62094bb0d4..33e36acf823150 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -62,7 +62,7 @@ export function getOperationParams( }, {} ); - // At the moment is positional as expressed in operationParams + return params.reduce>((args, { name, value }) => { if (formalArgs[name]) { args[name] = value; @@ -70,7 +70,27 @@ export function getOperationParams( return args; }, {}); } -export const tinymathValidOperators = new Set(['add', 'subtract', 'multiply', 'divide']); +export const tinymathValidOperators = new Set([ + 'add', + 'subtract', + 'multiply', + 'divide', + 'abs', + 'cbrt', + 'ceil', + 'clamp', + 'cube', + 'exp', + 'fix', + 'floor', + 'log', + 'log10', + 'mod', + 'pow', + 'round', + 'sqrt', + 'square', +]); export function isMathNode(node: TinymathAST) { return isObject(node) && node.type === 'function' && tinymathValidOperators.has(node.name); From fcbdc7404c614e0bbc33fb131fda74b9d3856137 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Mar 2021 15:16:38 +0100 Subject: [PATCH 032/185] :bug: Improved validation around math operations + multiple named arguments --- .../definitions/formula/formula.test.tsx | 98 +++++-- .../definitions/formula/validation.ts | 263 +++++++++++------- 2 files changed, 233 insertions(+), 128 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 727681526b26de..7e72267dcb5f54 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -5,12 +5,9 @@ * 2.0. */ -// import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; -// import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; -// import { dataPluginMock } from '../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../../mocks'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; jest.mock('../../layer_helpers', () => { @@ -20,20 +17,6 @@ jest.mock('../../layer_helpers', () => { }; }); -// const defaultProps = { -// storage: {} as IStorageWrapper, -// uiSettings: {} as IUiSettingsClient, -// savedObjectsClient: {} as SavedObjectsClientContract, -// dateRange: { fromDate: 'now-1d', toDate: 'now' }, -// data: dataPluginMock.createStartContract(), -// http: {} as HttpSetup, -// indexPattern: { -// ...createMockedIndexPattern(), -// hasRestrictions: false, -// } as IndexPattern, -// operationDefinitionMap: { avg: {} }, -// }; - const operationDefinitionMap: Record = { avg: ({ input: 'field', @@ -47,12 +30,16 @@ const operationDefinitionMap: Record = { timeScale: false, }), } as unknown) as GenericOperationDefinition, + sum: { input: 'field' } as GenericOperationDefinition, + last_value: { input: 'field' } as GenericOperationDefinition, + max: { input: 'field' } as GenericOperationDefinition, count: { input: 'field' } as GenericOperationDefinition, derivative: { input: 'fullReference' } as GenericOperationDefinition, moving_average: { input: 'fullReference', operationParams: [{ name: 'window', type: 'number', required: true }], } as GenericOperationDefinition, + cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition, }; describe('formula', () => { @@ -181,6 +168,7 @@ describe('formula', () => { 'avg(bytes) +', 'avg(""', 'moving_average(avg(bytes), window=)', + 'avg(bytes) + moving_average(avg(bytes), window=)', ]; for (const formula of formulas) { testIsBrokenFormula(formula); @@ -192,7 +180,13 @@ describe('formula', () => { }); it('returns no change but error if at least one field in the formula is missing', () => { - const formulas = ['noField', 'avg(noField)', 'noField + 1', 'derivative(avg(noField))']; + const formulas = [ + 'noField', + 'avg(noField)', + 'noField + 1', + 'derivative(avg(noField))', + 'avg(bytes) + derivative(avg(noField))', + ]; for (const formula of formulas) { testIsBrokenFormula(formula); @@ -207,6 +201,9 @@ describe('formula', () => { 'derivative(noFn())', 'noFn() + noFnTwo()', 'noFn(noFnTwo())', + 'noFn() + noFnTwo() + 5', + 'avg(bytes) + derivative(noFn())', + 'derivative(avg(bytes) + noFn())', ]; for (const formula of formulas) { @@ -223,10 +220,10 @@ describe('formula', () => { 'avg(bytes + 5)', 'avg(bytes + bytes)', 'derivative(7)', - 'derivative(7 + 1)', 'derivative(bytes + 7)', 'derivative(bytes + bytes)', 'derivative(bytes + avg(bytes))', + 'derivative(bytes + 7 + avg(bytes))', ]; for (const formula of formulas) { @@ -251,6 +248,11 @@ describe('formula', () => { const formula = 'moving_average(avg(bytes), window="m")'; testIsBrokenFormula(formula); }); + + it('returns error if a required parameter is passed multiple time', () => { + const formula = 'moving_average(avg(bytes), window=7, window=3)'; + testIsBrokenFormula(formula); + }); }); describe('getErrorMessage', () => { @@ -402,7 +404,22 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual(['Fields noField not found']); + ).toEqual(['Field noField not found']); + } + }); + + it('returns an error with plural form correctly handled', () => { + const formulas = ['noField + noField2', 'noField + 1 + noField2']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Fields noField, noField2 not found']); } }); @@ -443,10 +460,6 @@ describe('formula', () => { 'avg(bytes + 5)', 'avg(bytes + bytes)', 'derivative(7)', - 'derivative(7 + 1)', - 'derivative(bytes + 7)', - 'derivative(bytes + bytes)', - 'derivative(bytes + avg(bytes))', ]; for (const formula of formulas) { @@ -530,5 +543,40 @@ describe('formula', () => { 'The parameters for the operation moving_average in the Formula are of the wrong type: window', ]); }); + + it('returns no error for the demo formula example', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(` + moving_average( + cumulative_sum( + 7 * clamp(sum(bytes), 0, last_value(memory) + max(memory)) + ), window=10 + ) + `), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns no error if a math operation is passed to fullReference operations', () => { + const formulas = [ + 'derivative(7+1)', + 'derivative(7+avg(bytes))', + 'moving_average(7+avg(bytes), window=7)', + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index bc27a5685d7439..826d844fbdfd30 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -34,6 +34,7 @@ const validationErrors = { unexpectedNode: 'unexpected node', fieldWithNoOperation: 'unexpected field with no operation', failedParsing: 'Failed to parse expression.', // note: this string comes from Tinymath, do not change it + duplicateArgument: 'duplicate argument', }; export const errorsLookup = new Set(Object.values(validationErrors)); @@ -92,6 +93,13 @@ function getMessageFromId({ values, }); break; + case 'duplicateArgument': + message = i18n.translate('xpack.lens.indexPattern.formulaOperationDuplicateParams', { + defaultMessage: + 'The parameters for the operation {operation} have been declared multiple times: {params}', + values, + }); + break; case 'missingField': message = i18n.translate('xpack.lens.indexPattern.fieldNotFound', { defaultMessage: @@ -238,152 +246,186 @@ function runFullASTValidation( return []; } const nodeOperation = operations[node.name]; - if (!nodeOperation) { - return validateMathNodes(node, missingVariablesSet); - } - const errors: ErrorWrapper[] = []; const { namedArguments, functions } = groupArgsByType(node.args); const [firstArg] = node?.args || []; - if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isFirstArgumentValidType(firstArg, 'variable')) { - if (isMathNode(firstArg)) { - errors.push( - getMessageFromId({ - messageId: 'wrongFirstArgument', - values: { - operation: node.name, - type: 'field', - argument: `math operation`, - }, - locations: [node.location], - }) - ); - } else { + if (!nodeOperation) { + errors.push(...validateMathNodes(node, missingVariablesSet)); + // carry on with the validation for all the functions within the math operation + if (functions?.length) { + return errors.concat(functions.flatMap((fn) => validateNode(fn))); + } + } else { + if (nodeOperation.input === 'field') { + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { + if (isMathNode(firstArg)) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: `math operation`, + }, + locations: [node.location], + }) + ); + } else { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(firstArg), + }, + locations: [node.location], + }) + ); + } + } + } else { + if (firstArg) { errors.push( getMessageFromId({ - messageId: 'wrongFirstArgument', + messageId: 'shouldNotHaveField', values: { operation: node.name, - type: 'field', - argument: getValueOrName(firstArg), }, locations: [node.location], }) ); } } - } else { - if (firstArg) { + if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( getMessageFromId({ - messageId: 'shouldNotHaveField', + messageId: 'cannotAcceptParameter', values: { operation: node.name, }, locations: [node.location], }) ); + } else { + const missingParams = getMissingParams(nodeOperation, namedArguments); + if (missingParams.length) { + errors.push( + getMessageFromId({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const duplicateParams = getDuplicateParams(namedArguments); + if (duplicateParams.length) { + errors.push( + getMessageFromId({ + messageId: 'duplicateArgument', + values: { + operation: node.name, + params: duplicateParams.join(', '), + }, + locations: [node.location], + }) + ); + } } + return errors; } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - getMessageFromId({ - messageId: 'cannotAcceptParameter', - values: { - operation: node.name, - }, - locations: [node.location], - }) - ); - } else { - const missingParams = getMissingParams(nodeOperation, namedArguments); - if (missingParams.length) { - errors.push( - getMessageFromId({ - messageId: 'missingParameter', - values: { - operation: node.name, - params: missingParams.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); - if (wrongTypeParams.length) { - errors.push( - getMessageFromId({ - messageId: 'wrongTypeParameter', - values: { - operation: node.name, - params: wrongTypeParams.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - } - return errors; - } - if (nodeOperation.input === 'fullReference') { - if (!isFirstArgumentValidType(firstArg, 'function') || isMathNode(firstArg)) { - errors.push( - getMessageFromId({ - messageId: 'wrongFirstArgument', - values: { - operation: node.name, - type: 'operation', - argument: getValueOrName(firstArg), - }, - locations: [node.location], - }) - ); - } - if (!canHaveParams(nodeOperation) && namedArguments.length) { - errors.push( - getMessageFromId({ - messageId: 'cannotAcceptParameter', - values: { - operation: node.name, - }, - locations: [node.location], - }) - ); - } else { - const missingParameters = getMissingParams(nodeOperation, namedArguments); - if (missingParameters.length) { + if (nodeOperation.input === 'fullReference') { + // What about fn(7 + 1)? We may want to allow that + // In general this should be handled down the Esaggs route rather than here + if ( + !isFirstArgumentValidType(firstArg, 'function') || + (isMathNode(firstArg) && validateMathNodes(firstArg, missingVariablesSet).length) + ) { errors.push( getMessageFromId({ - messageId: 'missingParameter', + messageId: 'wrongFirstArgument', values: { operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), + type: 'operation', + argument: getValueOrName(firstArg), }, locations: [node.location], }) ); } - const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); - if (wrongTypeParams.length) { + if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( getMessageFromId({ - messageId: 'wrongTypeParameter', + messageId: 'cannotAcceptParameter', values: { operation: node.name, - params: wrongTypeParams.map(({ name }) => name).join(', '), }, locations: [node.location], }) ); + } else { + const missingParameters = getMissingParams(nodeOperation, namedArguments); + if (missingParameters.length) { + errors.push( + getMessageFromId({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParameters.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const duplicateParams = getDuplicateParams(namedArguments); + if (duplicateParams.length) { + errors.push( + getMessageFromId({ + messageId: 'duplicateArgument', + values: { + operation: node.name, + params: duplicateParams.join(', '), + }, + locations: [node.location], + }) + ); + } } } - return errors.concat(validateNode(functions[0])); } - return []; + return errors; } return validateNode(ast); @@ -430,6 +472,19 @@ export function getWrongTypeParams( ); } +function getDuplicateParams(params: TinymathNamedArgument[]) { + const uniqueArgs = Object.create(null); + for (const { name } of params) { + const counter = uniqueArgs[name] || 0; + uniqueArgs[name] = counter + 1; + } + const uniqueNames = Object.keys(uniqueArgs); + if (params.length > uniqueNames.length) { + return uniqueNames.filter((name) => uniqueArgs[name] > 1).map(([name]) => name); + } + return []; +} + export function validateParams( operation: | OperationDefinition @@ -459,7 +514,7 @@ export function validateMathNodes(root: TinymathAST, missingVariableSet: Set { // check the following patterns: - const { variables } = groupArgsByType(node.args); + const { variables, functions } = groupArgsByType(node.args); const fieldVariables = variables.filter((v) => isObject(v) && !missingVariableSet.has(v.value)); // field + field (or string) const atLeastTwoFields = fieldVariables.length > 1; @@ -468,10 +523,12 @@ export function validateMathNodes(root: TinymathAST, missingVariableSet: Set !isObject(v) || !missingVariableSet.has(v.value) ); + // field + function + const fieldMathWithFunction = fieldVariables.length > 0 && functions.length > 0; // Make sure to have at least one valid field to compare, or skip the check const mathBetweenFieldAndNumbers = fieldVariables.length > 0 && validVariables.length - fieldVariables.length > 0; - return atLeastTwoFields || mathBetweenFieldAndNumbers; + return atLeastTwoFields || mathBetweenFieldAndNumbers || fieldMathWithFunction; }); if (invalidNodes.length) { errors.push({ From 55d3bc65e8d4e147c41074537e3bdcd0a02c7104 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Mar 2021 17:41:49 +0100 Subject: [PATCH 033/185] :bug: Use new onError feature in math expression --- .../operations/definitions/formula/math.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 345d407e4d6fc9..1060f51e14ff23 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -57,6 +57,7 @@ export const mathOperation: OperationDefinition Date: Wed, 3 Mar 2021 17:42:13 +0100 Subject: [PATCH 034/185] :recycle: Refactor namedArguments validation --- .../definitions/formula/validation.ts | 136 ++++++++---------- 1 file changed, 58 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 826d844fbdfd30..b62178873a38fe 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -229,6 +229,56 @@ function checkMissingVariableOrFunctions( return [...missingErrors, ...invalidVariableErrors]; } +function validateNameArguments( + node: TinymathFunction, + nodeOperation: + | OperationDefinition + | OperationDefinition, + namedArguments: TinymathNamedArgument[] | undefined +) { + const errors = []; + const missingParams = getMissingParams(nodeOperation, namedArguments); + if (missingParams.length) { + errors.push( + getMessageFromId({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const duplicateParams = getDuplicateParams(namedArguments); + if (duplicateParams.length) { + errors.push( + getMessageFromId({ + messageId: 'duplicateArgument', + values: { + operation: node.name, + params: duplicateParams.join(', '), + }, + locations: [node.location], + }) + ); + } + return errors; +} + function runFullASTValidation( ast: TinymathAST, layer: IndexPatternLayer, @@ -310,44 +360,9 @@ function runFullASTValidation( }) ); } else { - const missingParams = getMissingParams(nodeOperation, namedArguments); - if (missingParams.length) { - errors.push( - getMessageFromId({ - messageId: 'missingParameter', - values: { - operation: node.name, - params: missingParams.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); - if (wrongTypeParams.length) { - errors.push( - getMessageFromId({ - messageId: 'wrongTypeParameter', - values: { - operation: node.name, - params: wrongTypeParams.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - const duplicateParams = getDuplicateParams(namedArguments); - if (duplicateParams.length) { - errors.push( - getMessageFromId({ - messageId: 'duplicateArgument', - values: { - operation: node.name, - params: duplicateParams.join(', '), - }, - locations: [node.location], - }) - ); + const argumentsErrors = validateNameArguments(node, nodeOperation, namedArguments); + if (argumentsErrors.length) { + errors.push(...argumentsErrors); } } return errors; @@ -382,44 +397,9 @@ function runFullASTValidation( }) ); } else { - const missingParameters = getMissingParams(nodeOperation, namedArguments); - if (missingParameters.length) { - errors.push( - getMessageFromId({ - messageId: 'missingParameter', - values: { - operation: node.name, - params: missingParameters.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); - if (wrongTypeParams.length) { - errors.push( - getMessageFromId({ - messageId: 'wrongTypeParameter', - values: { - operation: node.name, - params: wrongTypeParams.map(({ name }) => name).join(', '), - }, - locations: [node.location], - }) - ); - } - const duplicateParams = getDuplicateParams(namedArguments); - if (duplicateParams.length) { - errors.push( - getMessageFromId({ - messageId: 'duplicateArgument', - values: { - operation: node.name, - params: duplicateParams.join(', '), - }, - locations: [node.location], - }) - ); + const argumentsErrors = validateNameArguments(node, nodeOperation, namedArguments); + if (argumentsErrors.length) { + errors.push(...argumentsErrors); } } } @@ -472,7 +452,7 @@ export function getWrongTypeParams( ); } -function getDuplicateParams(params: TinymathNamedArgument[]) { +function getDuplicateParams(params: TinymathNamedArgument[] = []) { const uniqueArgs = Object.create(null); for (const { name } of params) { const counter = uniqueArgs[name] || 0; @@ -480,7 +460,7 @@ function getDuplicateParams(params: TinymathNamedArgument[]) { } const uniqueNames = Object.keys(uniqueArgs); if (params.length > uniqueNames.length) { - return uniqueNames.filter((name) => uniqueArgs[name] > 1).map(([name]) => name); + return uniqueNames.filter((name) => uniqueArgs[name] > 1); } return []; } From 08b9b6f982c0761f445b017b402b60047a4c39df Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 3 Mar 2021 17:43:14 +0100 Subject: [PATCH 035/185] :bug: Fix circular dependency issue in tests + minor fixes --- .../definitions/formula/formula.tsx | 10 ++++++--- .../definitions/formula/math_completion.ts | 21 +++++++++++++------ .../operations/layer_helpers.test.ts | 3 ++- .../operations/layer_helpers.ts | 2 +- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index af6ad89909a2be..8e1908ffe58bae 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -208,7 +208,9 @@ function FormulaEditor({ monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); } }, - { skipFirstRender: true }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: false }, 256, [text] ); @@ -250,7 +252,8 @@ function FormulaEditor({ innerText.substring(0, innerText.length - lengthAfterPosition) + ')', innerText.length - lengthAfterPosition, context, - indexPattern + indexPattern, + operationDefinitionMap ); } } else { @@ -266,6 +269,7 @@ function FormulaEditor({ innerText.length - lengthAfterPosition, context, indexPattern, + operationDefinitionMap, wordUntil ); } @@ -274,7 +278,7 @@ function FormulaEditor({ suggestions: aSuggestions.list.map((s) => getSuggestion(s, aSuggestions.type, wordRange)), }; }, - [indexPattern] + [indexPattern, operationDefinitionMap] ); return ( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 5851073682a2e2..4ced303240f4d5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -10,9 +10,8 @@ import { monaco } from '@kbn/monaco'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; -import { getAvailableOperationsByMetadata } from '../../'; +import { getAvailableOperationsByMetadata } from '../../operations'; import type { GenericOperationDefinition } from '..'; -import { operationDefinitionMap } from '..'; export enum SUGGESTION_TYPE { FIELD = 'field', @@ -60,6 +59,7 @@ export async function suggest( position: number, context: monaco.languages.CompletionContext, indexPattern: IndexPattern, + operationDefinitionMap: Record, word?: monaco.editor.IWordAtPosition ): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); @@ -74,11 +74,12 @@ export async function suggest( return getArgumentSuggestions( tokenInfo.parent.name, tokenInfo.parent.args.length - 1, - indexPattern + indexPattern, + operationDefinitionMap ); } if (tokenInfo && word) { - return getFunctionSuggestions(word); + return getFunctionSuggestions(word, operationDefinitionMap); } } catch (e) { // Fail silently @@ -86,14 +87,22 @@ export async function suggest( return { list: [], type: SUGGESTION_TYPE.FIELD }; } -function getFunctionSuggestions(word: monaco.editor.IWordAtPosition) { +function getFunctionSuggestions( + word: monaco.editor.IWordAtPosition, + operationDefinitionMap: Record +) { const list = Object.keys(operationDefinitionMap) .filter((func) => startsWith(func, word.word)) .map((key) => operationDefinitionMap[key]); return { list, type: SUGGESTION_TYPE.FUNCTIONS }; } -function getArgumentSuggestions(name: string, position: number, indexPattern: IndexPattern) { +function getArgumentSuggestions( + name: string, + position: number, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { const operation = operationDefinitionMap[name]; if (!operation) { return { list: [], type: SUGGESTION_TYPE.FIELD }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 1961a4f957d810..69b91219fe2219 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -2238,7 +2238,8 @@ describe('state_helpers', () => { }, }, 'col1', - indexPattern + indexPattern, + operationDefinitionMap ); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index c9cb7949bdfeda..9d8560e5ba4ba9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -7,6 +7,7 @@ import _, { partition } from 'lodash'; import type { OperationMetadata } from '../../types'; +import { getSortScoreByPriority } from './operations'; import { operationDefinitionMap, operationDefinitions, @@ -15,7 +16,6 @@ import { RequiredReference, } from './definitions'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types'; -import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula/formula'; From 7832aeb1e6e33290a8f42770797c384dc4622554 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 3 Mar 2021 19:03:53 -0500 Subject: [PATCH 036/185] Move formula into a tab --- .../dimension_panel/dimension_editor.tsx | 74 ++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 25789eb21afbd1..03f03d9c8546a2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -18,6 +18,7 @@ import { EuiFormLabel, EuiToolTip, EuiText, + EuiTabbedContent, } from '@elastic/eui'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; @@ -318,8 +319,8 @@ export function DimensionEditor(props: DimensionEditorProps) { currentFieldIsInvalid ); - return ( -
+ const quickFunctions = ( + <>
{i18n.translate('xpack.lens.indexPattern.functionsLabel', { @@ -460,6 +461,75 @@ export function DimensionEditor(props: DimensionEditorProps) {
+ + ); + + const tabs = [ + { + id: 'quickFunctions', + name: 'Quick functions', + content: quickFunctions, + disabled: selectedOperationDefinition?.type === 'formula', + }, + { + id: 'formula', + name: 'Formula', + content: ( + <> + {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ParamEditor && ( + <> + + + )} + + ), + }, + ]; + + return ( +
+
+ {operationSupportMatrix.operationWithoutField.has('formula') ? ( + { + if (selectedTab.id === 'quickFunctions') { + // + } else { + // Clear invalid state because we are reseting to a valid column + if (selectedColumn?.operationType === 'formula') { + if (incompleteInfo) { + setStateWrapper(resetIncomplete(state.layers[layerId], columnId)); + } + return; + } + const newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: 'formula', + }); + setStateWrapper(newLayer); + trackUiEvent(`indexpattern_dimension_operation_formula`); + return; + } + }} + size="s" + /> + ) : ( + quickFunctions + )} +
{!currentFieldIsInvalid && (
From f05c647561b008174d40ed19292a2fcfc604da1d Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Mar 2021 13:00:53 +0100 Subject: [PATCH 037/185] :fire: Leftovers from previous merge --- .../functions/common/mapColumn.test.js | 79 ---------------- .../functions/common/mapColumn.ts | 90 ------------------- .../functions/common/math.ts | 70 --------------- .../canvas/i18n/functions/dict/map_column.ts | 42 --------- 4 files changed, 281 deletions(-) delete mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js delete mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts delete mode 100644 x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts delete mode 100644 x-pack/plugins/canvas/i18n/functions/dict/map_column.ts diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js deleted file mode 100644 index 9af2c715cfad2a..00000000000000 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.test.js +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { functionWrapper } from '../../../test_helpers/function_wrapper'; -import { testTable, emptyTable } from './__fixtures__/test_tables'; -import { mapColumn } from './mapColumn'; - -const pricePlusTwo = (datatable) => Promise.resolve(datatable.rows[0].price + 2); - -describe('mapColumn', () => { - const fn = functionWrapper(mapColumn); - - it('returns a datatable with a new column with the values from mapping a function over each row in a datatable', () => { - return fn(testTable, { - id: 'pricePlusTwo', - name: 'pricePlusTwo', - expression: pricePlusTwo, - }).then((result) => { - const arbitraryRowIndex = 2; - - expect(result.type).toBe('datatable'); - expect(result.columns).toEqual([ - ...testTable.columns, - { id: 'pricePlusTwo', name: 'pricePlusTwo', meta: { type: 'number' } }, - ]); - expect(result.columns[result.columns.length - 1]).toHaveProperty('name', 'pricePlusTwo'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); - }); - }); - - it('overwrites existing column with the new column if an existing column name is provided', () => { - return fn(testTable, { name: 'name', expression: pricePlusTwo }).then((result) => { - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(testTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); - }); - }); - - it('adds a column to empty tables', () => { - return fn(emptyTable, { name: 'name', expression: pricePlusTwo }).then((result) => { - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(1); - expect(result.columns[0]).toHaveProperty('name', 'name'); - expect(result.columns[0].meta).toHaveProperty('type', 'null'); - }); - }); - - it('should assign specific id, different from name, when id arg is passed', () => { - return fn(emptyTable, { name: 'name', id: 'myid', expression: pricePlusTwo }).then((result) => { - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(1); - expect(result.columns[0]).toHaveProperty('name', 'name'); - expect(result.columns[0]).toHaveProperty('id', 'myid'); - expect(result.columns[0].meta).toHaveProperty('type', 'null'); - }); - }); - - describe('expression', () => { - it('maps null values to the new column', () => { - return fn(testTable, { name: 'empty' }).then((result) => { - const emptyColumnIndex = result.columns.findIndex(({ name }) => name === 'empty'); - const arbitraryRowIndex = 8; - - expect(result.columns[emptyColumnIndex]).toHaveProperty('name', 'empty'); - expect(result.columns[emptyColumnIndex].meta).toHaveProperty('type', 'null'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('empty', null); - }); - }); - }); -}); diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts deleted file mode 100644 index 2448ce6f9e4224..00000000000000 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/mapColumn.ts +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Datatable, ExpressionFunctionDefinition, getType } from '../../../types'; -import { getFunctionHelp } from '../../../i18n'; - -interface Arguments { - id: string | null; - name: string; - expression: (datatable: Datatable) => Promise; -} - -export function mapColumn(): ExpressionFunctionDefinition< - 'mapColumn', - Datatable, - Arguments, - Promise -> { - const { help, args: argHelp } = getFunctionHelp().mapColumn; - - return { - name: 'mapColumn', - aliases: ['mc'], // midnight commander. So many times I've launched midnight commander instead of moving a file. - type: 'datatable', - inputTypes: ['datatable'], - help, - args: { - id: { - types: ['string', 'null'], - help: argHelp.id, - required: false, - default: null, - }, - name: { - types: ['string'], - aliases: ['_', 'column'], - help: argHelp.name, - required: true, - }, - expression: { - types: ['boolean', 'number', 'string', 'null'], - resolve: false, - aliases: ['exp', 'fn', 'function'], - help: argHelp.expression, - required: true, - }, - }, - fn: (input, args) => { - const expression = args.expression || (() => Promise.resolve(null)); - const columnId = args.id != null ? args.id : args.name; - const columns = [...input.columns]; - const rowPromises = input.rows.map((row) => { - return expression({ - type: 'datatable', - columns, - rows: [row], - }).then((val) => ({ - ...row, - [columnId]: val, - })); - }); - - return Promise.all(rowPromises).then((rows) => { - const existingColumnIndex = columns.findIndex(({ name }) => name === args.name); - const type = rows.length ? getType(rows[0][columnId]) : 'null'; - const newColumn = { - id: columnId, - name: args.name, - meta: { type }, - }; - - if (existingColumnIndex === -1) { - columns.push(newColumn); - } else { - columns[existingColumnIndex] = newColumn; - } - - return { - type: 'datatable', - columns, - rows, - } as Datatable; - }); - }, - }; -} diff --git a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts b/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts deleted file mode 100644 index 2b2eef281a526e..00000000000000 --- a/x-pack/plugins/canvas/canvas_plugin_src/functions/common/math.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { evaluate } from '@kbn/tinymath'; -import { pivotObjectArray } from '../../../common/lib/pivot_object_array'; -import { Datatable, isDatatable, ExpressionFunctionDefinition } from '../../../types'; -import { getFunctionHelp, getFunctionErrors } from '../../../i18n'; - -interface Arguments { - expression: string; -} - -type Input = number | Datatable; - -export function math(): ExpressionFunctionDefinition<'math', Input, Arguments, number> { - const { help, args: argHelp } = getFunctionHelp().math; - const errors = getFunctionErrors().math; - - return { - name: 'math', - type: undefined, - inputTypes: ['number', 'datatable'], - help, - args: { - expression: { - aliases: ['_'], - types: ['string'], - help: argHelp.expression, - }, - }, - fn: (input, args) => { - const { expression } = args; - - if (!expression || expression.trim() === '') { - throw errors.emptyExpression(); - } - - const mathContext = isDatatable(input) - ? pivotObjectArray( - input.rows, - input.columns.map((col) => col.name) - ) - : { value: input }; - - try { - const result = evaluate(expression, mathContext); - if (Array.isArray(result)) { - if (result.length === 1) { - return result[0]; - } - throw errors.tooManyResults(); - } - // if (isNaN(result)) { - // throw errors.executionFailed(); - // } - return result; - } catch (e) { - if (isDatatable(input) && input.rows.length === 0) { - throw errors.emptyDatatable(); - } else { - throw e; - } - } - }, - }; -} diff --git a/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts b/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts deleted file mode 100644 index 980177e49aad9b..00000000000000 --- a/x-pack/plugins/canvas/i18n/functions/dict/map_column.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; -import { mapColumn } from '../../../canvas_plugin_src/functions/common/mapColumn'; -import { FunctionHelp } from '../function_help'; -import { FunctionFactory } from '../../../types'; -import { CANVAS, DATATABLE } from '../../constants'; - -export const help: FunctionHelp> = { - help: i18n.translate('xpack.canvas.functions.mapColumnHelpText', { - defaultMessage: - 'Adds a column calculated as the result of other columns. ' + - 'Changes are made only when you provide arguments.' + - 'See also {alterColumnFn} and {staticColumnFn}.', - values: { - alterColumnFn: '`alterColumn`', - staticColumnFn: '`staticColumn`', - }, - }), - args: { - id: i18n.translate('xpack.canvas.functions.mapColumn.args.idHelpText', { - defaultMessage: - 'An optional id of the resulting column. When `null` the name argument is used as id.', - }), - name: i18n.translate('xpack.canvas.functions.mapColumn.args.nameHelpText', { - defaultMessage: 'The name of the resulting column.', - }), - expression: i18n.translate('xpack.canvas.functions.mapColumn.args.expressionHelpText', { - defaultMessage: - 'A {CANVAS} expression that is passed to each row as a single row {DATATABLE}.', - values: { - CANVAS, - DATATABLE, - }, - }), - }, -}; From d050e6fbae86436c0cbbc91ca810080dd2c4afc5 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Mar 2021 13:01:18 +0100 Subject: [PATCH 038/185] :sparkles: Move over namedArgs from previous function --- .../definitions/formula/formula.tsx | 24 +++++++++++--- .../operations/definitions/formula/util.ts | 32 +++++++++++++++++++ .../operations/definitions/index.ts | 3 +- .../operations/layer_helpers.ts | 6 +++- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 8e1908ffe58bae..0726ede6e2a6bd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -24,6 +24,7 @@ import { mathOperation } from './math'; import { documentField } from '../../../document_field'; import { ErrorWrapper, runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; import { + extractParamsForFormula, findVariables, getOperationParams, getSafeFieldName, @@ -111,7 +112,7 @@ export const formulaOperation: OperationDefinition< }, ]; }, - buildColumn({ previousColumn, layer }) { + buildColumn({ previousColumn, layer }, _, operationDefinitionMap) { let previousFormula = ''; if (previousColumn) { if ('references' in previousColumn) { @@ -119,15 +120,28 @@ export const formulaOperation: OperationDefinition< if (metric && 'sourceField' in metric) { const fieldName = getSafeFieldName(metric.sourceField); // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName}))`; + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; } } else { if (previousColumn && 'sourceField' in previousColumn) { previousFormula += `${previousColumn.operationType}(${getSafeFieldName( previousColumn?.sourceField - )})`; + )}`; } } + const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); + if (formulaNamedArgs.length) { + previousFormula += + ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); + } + // close the formula at the end + previousFormula += ')'; + } + // carry over the format settings from previous operation for seamless transfer + // NOTE: this works only for non-default formatters set in Lens + let prevFormat = {}; + if (previousColumn?.params && 'format' in previousColumn.params) { + prevFormat = { format: previousColumn.params.format }; } return { label: 'Formula', @@ -135,7 +149,9 @@ export const formulaOperation: OperationDefinition< operationType: 'formula', isBucketed: false, scale: 'ratio', - params: previousFormula ? { formula: previousFormula, isFormulaBroken: false } : {}, + params: previousFormula + ? { formula: previousFormula, isFormulaBroken: false, ...prevFormat } + : { ...prevFormat }, references: [], }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 33e36acf823150..abdc9bbf05b393 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -13,6 +13,7 @@ import type { TinymathNamedArgument, TinymathVariable, } from 'packages/kbn-tinymath'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; import type { GroupedNodes } from './types'; @@ -49,6 +50,37 @@ export function getSafeFieldName(fieldName: string | undefined) { return fieldName; } +// Just handle two levels for now +type OeprationParams = Record>; + +export function extractParamsForFormula( + column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, + operationDefinitionMap: Record | undefined +) { + if (!operationDefinitionMap) { + return []; + } + const def = operationDefinitionMap[column.operationType]; + if ('operationParams' in def && column.params) { + return (def.operationParams || []).flatMap(({ name, required }) => { + const value = (column.params as OeprationParams)![name]; + if (isObject(value)) { + return Object.keys(value).map((subName) => ({ + name: `${name}-${subName}`, + value: value[subName] as string | number, + required, + })); + } + return { + name, + value, + required, + }; + }); + } + return []; +} + export function getOperationParams( operation: | OperationDefinition diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index a288b7e68f5343..fa3bdbe85816cc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -442,7 +442,8 @@ interface ManagedReferenceOperationDefinition arg: BaseBuildColumnArgs & { previousColumn?: IndexPatternColumn | ReferenceBasedIndexPatternColumn; }, - columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'], + operationDefinitionMap?: Record ) => ReferenceBasedIndexPatternColumn & C; /** * Returns the meta data of the operation if applied. Undefined diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 9d8560e5ba4ba9..586fe424cb218e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -303,7 +303,11 @@ export function replaceColumn({ // if managed it has to look at the full picture to have a seamless transition if (operationDefinition.input === 'managedReference') { const newColumn = copyCustomLabel( - operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer }), + operationDefinition.buildColumn( + { ...baseOptions, layer: tempLayer }, + previousColumn.params, + operationDefinitionMap + ), previousColumn ) as FormulaIndexPatternColumn; From 37c9c5f887f5637f2bad9a3825c66d37a07303c0 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Mar 2021 13:01:39 +0100 Subject: [PATCH 039/185] :white_check_mark: Add tests for transferable scenarios --- .../definitions/formula/formula.test.tsx | 147 +++++++++++++++++- 1 file changed, 146 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 7e72267dcb5f54..5e357d2647c17b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -30,6 +30,7 @@ const operationDefinitionMap: Record = { timeScale: false, }), } as unknown) as GenericOperationDefinition, + terms: { input: 'field' } as GenericOperationDefinition, sum: { input: 'field' } as GenericOperationDefinition, last_value: { input: 'field' } as GenericOperationDefinition, max: { input: 'field' } as GenericOperationDefinition, @@ -44,7 +45,6 @@ const operationDefinitionMap: Record = { describe('formula', () => { let layer: IndexPatternLayer; - // const InlineOptions = formulaOperation.paramEditor!; beforeEach(() => { layer = { @@ -67,6 +67,151 @@ describe('formula', () => { }; }); + describe('buildColumn', () => { + let indexPattern: IndexPattern; + + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + }); + + it('should start with an empty formula if no previous column is detected', () => { + expect( + formulaOperation.buildColumn({ + layer: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); + + it('should move into Formula previous operation', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: layer.columns.col1, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { isFormulaBroken: false, formula: 'terms(category)' }, + references: [], + }); + }); + + it('it should move over explicit format param if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + params: { + ...layer.columns.col1.params, + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'terms(category)', + params: { + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + }, + references: [], + }); + }); + + it('should move over previous operation parameter if set', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + col2: { + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'avg', + scale: 'ratio', + sourceField: 'bytes', + timeScale: 'd', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'moving_average(avg(bytes), window=3)', + }, + references: [], + }); + }); + }); + describe('regenerateLayerFromAst()', () => { let indexPattern: IndexPattern; let currentColumn: FormulaIndexPatternColumn; From 135987c4469effea912a3bf241e07b5ecb6f58e3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Mar 2021 13:06:58 +0100 Subject: [PATCH 040/185] :white_check_mark: Fixed broken test --- .../operations/definitions/formula/formula.test.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 5e357d2647c17b..c5b9626a2ccc1b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -140,12 +140,10 @@ describe('formula', () => { params: { isFormulaBroken: false, formula: 'terms(category)', - params: { - format: { - id: 'number', - params: { - decimals: 2, - }, + format: { + id: 'number', + params: { + decimals: 2, }, }, }, From 28847f15891fd962789cbaab89efa8abc85dc22a Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 4 Mar 2021 14:43:21 +0100 Subject: [PATCH 041/185] :sparkles: Use custom label for axis --- .../operations/definitions/formula/formula.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 0726ede6e2a6bd..24ba33e30ef673 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -37,6 +37,8 @@ import { LANGUAGE_ID } from './math_tokenization'; import './formula.scss'; +const defaultLabel = 'Formula'; + export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; params: { @@ -57,8 +59,8 @@ export const formulaOperation: OperationDefinition< 'managedReference' > = { type: 'formula', - displayName: 'Formula', - getDefaultLabel: (column, indexPattern) => 'Formula', + displayName: defaultLabel, + getDefaultLabel: (column, indexPattern) => defaultLabel, input: 'managedReference', getDisabledStatus(indexPattern: IndexPattern) { return undefined; @@ -86,7 +88,13 @@ export const formulaOperation: OperationDefinition< toExpression: (layer, columnId) => { const currentColumn = layer.columns[columnId] as FormulaIndexPatternColumn; const params = currentColumn.params; - const label = !params?.isFormulaBroken ? params?.formula : ''; + // TODO: improve this logic + const useDisplayLabel = currentColumn.label !== defaultLabel; + const label = !params?.isFormulaBroken + ? useDisplayLabel + ? currentColumn.label + : params?.formula + : ''; return [ { type: 'function', From 5524cf2e21221f28fe7afc47541a65a6bd4937d4 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 4 Mar 2021 16:56:00 -0500 Subject: [PATCH 042/185] Allow switching back and forth to formula tab --- .../dimension_panel/dimension_editor.tsx | 118 ++++++++++-------- .../definitions/formula/formula.tsx | 6 +- 2 files changed, 69 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 03f03d9c8546a2..f137ac0b89e6c4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -147,6 +147,8 @@ export function DimensionEditor(props: DimensionEditorProps) { const ParamEditor = selectedOperationDefinition?.paramEditor; + const [temporaryQuickFunction, setQuickFunction] = useState(false); + const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) @@ -319,6 +321,12 @@ export function DimensionEditor(props: DimensionEditorProps) { currentFieldIsInvalid ); + const shouldDisplayExtraOptions = + !currentFieldIsInvalid && + !incompleteInfo && + selectedColumn && + selectedColumn.operationType !== 'formula'; + const quickFunctions = ( <>
@@ -377,7 +385,8 @@ export function DimensionEditor(props: DimensionEditorProps) { {!selectedColumn || selectedOperationDefinition?.input === 'field' || - (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field') ? ( + (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field') || + temporaryQuickFunction ? ( ) : null} - {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ParamEditor && ( + {shouldDisplayExtraOptions && ParamEditor && ( <> )} - {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ( + {selectedColumn && shouldDisplayExtraOptions && ( - {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ParamEditor && ( - <> - - - )} + ), }, @@ -497,39 +505,41 @@ export function DimensionEditor(props: DimensionEditorProps) { return (
-
- {operationSupportMatrix.operationWithoutField.has('formula') ? ( - { - if (selectedTab.id === 'quickFunctions') { - // - } else { - // Clear invalid state because we are reseting to a valid column - if (selectedColumn?.operationType === 'formula') { - if (incompleteInfo) { - setStateWrapper(resetIncomplete(state.layers[layerId], columnId)); - } - return; - } - const newLayer = insertOrReplaceColumn({ - layer: props.state.layers[props.layerId], - indexPattern: currentIndexPattern, - columnId, - op: 'formula', - }); - setStateWrapper(newLayer); - trackUiEvent(`indexpattern_dimension_operation_formula`); - return; - } - }} - size="s" - /> - ) : ( - quickFunctions - )} -
+ {operationSupportMatrix.operationWithoutField.has('formula') ? ( + { + if ( + selectedTab.id === 'quickFunctions' && + selectedColumn?.operationType === 'formula' + ) { + // Temporary switch to quick function ui + setQuickFunction(true); + } else if (selectedColumn?.operationType !== 'formula') { + setQuickFunction(false); + const newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: 'formula', + }); + setStateWrapper(newLayer); + trackUiEvent(`indexpattern_dimension_operation_formula`); + return; + } else if (selectedTab.id === 'formula') { + setQuickFunction(false); + } + }} + size="s" + /> + ) : ( + quickFunctions + )} {!currentFieldIsInvalid && (
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 24ba33e30ef673..98753c395cd2e2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -7,6 +7,7 @@ import React, { useCallback, useState } from 'react'; import { isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathVariable } from '@kbn/tinymath'; import { EuiButton, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; @@ -37,7 +38,9 @@ import { LANGUAGE_ID } from './math_tokenization'; import './formula.scss'; -const defaultLabel = 'Formula'; +const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', +}); export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; @@ -62,6 +65,7 @@ export const formulaOperation: OperationDefinition< displayName: defaultLabel, getDefaultLabel: (column, indexPattern) => defaultLabel, input: 'managedReference', + hidden: true, getDisabledStatus(indexPattern: IndexPattern) { return undefined; }, From 6650f67ca46e64de91a8a6b8b768607578205c65 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 4 Mar 2021 18:26:09 -0500 Subject: [PATCH 043/185] Add a section for the function reference --- .../definitions/formula/formula.tsx | 103 +++++++++++++++++- .../definitions/formula/math_completion.ts | 29 +++-- 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 98753c395cd2e2..402bb7a5b1bd95 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -9,7 +9,15 @@ import React, { useCallback, useState } from 'react'; import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathVariable } from '@kbn/tinymath'; -import { EuiButton, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiDescriptionList, + EuiText, + EuiSpacer, + EuiPanel, +} from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import { CodeEditor } from '../../../../../../../../src/plugins/kibana_react/public'; import { @@ -33,7 +41,13 @@ import { hasMathNode, } from './util'; import { useDebounceWithOptions } from '../helpers'; -import { LensMathSuggestion, SUGGESTION_TYPE, suggest, getSuggestion } from './math_completion'; +import { + LensMathSuggestion, + SUGGESTION_TYPE, + suggest, + getSuggestion, + getPossibleFunctions, +} from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; import './formula.scss'; @@ -311,6 +325,49 @@ function FormulaEditor({ return ( + + + + {i18n.translate('xpack.lens.formula.functionReferenceLabel', { + defaultMessage: 'Function reference', + })} + + +
+ +

+ {i18n.translate('xpack.lens.formula.basicFunctions', { + defaultMessage: 'Basic functions', + })} +

+ +

+ +, -, /, * +

+ +

+ pow() +

+ + + + + + {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { + defaultMessage: 'Elasticsearch aggregations', + description: 'Do not translate Elasticsearch', + })} + + ({ + title: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + }))} + /> +
+ + { updateLayer( regenerateLayerFromAst( @@ -357,8 +417,11 @@ function FormulaEditor({ ) ); }} + iconType="play" > - Submit + {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { + defaultMessage: 'Submit', + })} @@ -553,3 +616,37 @@ function extractColumns( columns.push(mathColumn); return columns; } + +// TODO: i18n this whole thing, or move examples into the operation definitions with i18n +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const definition = operationDefinitionMap[type]; + + if (type === 'count') { + return ( + +

Example: count()

+ + ); + } + + return ( + + {definition.input === 'field' ?

Example: {type}(bytes)

: null} + {definition.input === 'fullReference' && !('operationParams' in definition) ? ( +

Example: {type}(sum(bytes))

+ ) : null} + + {'operationParams' in definition && definition.operationParams ? ( +

+

+ Example: {type}(sum(bytes),{' '} + {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) +

+

+ ) : null} + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 4ced303240f4d5..2d7c07f58c4b24 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -79,7 +79,7 @@ export async function suggest( ); } if (tokenInfo && word) { - return getFunctionSuggestions(word, operationDefinitionMap); + return getFunctionSuggestions(word, indexPattern); } } catch (e) { // Fail silently @@ -87,14 +87,25 @@ export async function suggest( return { list: [], type: SUGGESTION_TYPE.FIELD }; } -function getFunctionSuggestions( - word: monaco.editor.IWordAtPosition, - operationDefinitionMap: Record -) { - const list = Object.keys(operationDefinitionMap) - .filter((func) => startsWith(func, word.word)) - .map((key) => operationDefinitionMap[key]); - return { list, type: SUGGESTION_TYPE.FUNCTIONS }; +export function getPossibleFunctions(indexPattern: IndexPattern) { + const available = getAvailableOperationsByMetadata(indexPattern); + const possibleOperationNames: string[] = []; + available.forEach((a) => { + if (a.operationMetaData.dataType === 'number' && !a.operationMetaData.isBucketed) { + possibleOperationNames.push( + ...a.operations.filter((o) => o.type !== 'managedReference').map((o) => o.operationType) + ); + } + }); + + return uniq(possibleOperationNames); +} + +function getFunctionSuggestions(word: monaco.editor.IWordAtPosition, indexPattern: IndexPattern) { + return { + list: uniq(getPossibleFunctions(indexPattern).filter((func) => startsWith(func, word.word))), + type: SUGGESTION_TYPE.FUNCTIONS, + }; } function getArgumentSuggestions( From 1e32e0ad94415a91a2e9341ac784610e46333531 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 5 Mar 2021 16:29:53 -0500 Subject: [PATCH 044/185] Add modal editor and markdown docs --- .../public/code_editor/code_editor.tsx | 6 +- .../kibana_react/public/code_editor/index.tsx | 2 + .../operations/definitions/date_histogram.tsx | 5 +- .../definitions/formula/formula.tsx | 271 ++++++++++++------ .../definitions/formula/math_completion.ts | 15 +- .../operations/definitions/formula/util.ts | 174 +++++++++-- .../definitions/percentile.test.tsx | 17 ++ .../operations/definitions/percentile.tsx | 6 +- 8 files changed, 385 insertions(+), 111 deletions(-) diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index cb96f077b219b9..d1ebbc66210b91 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -35,9 +35,9 @@ export interface Props { /** * Options for the Monaco Code Editor * Documentation of options can be found here: - * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.ieditorconstructionoptions.html + * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.istandaloneeditorconstructionoptions.html */ - options?: monaco.editor.IEditorConstructionOptions; + options?: monaco.editor.IStandaloneEditorConstructionOptions; /** * Suggestion provider for autocompletion @@ -162,7 +162,7 @@ export class CodeEditor extends React.Component { editorDidMount={this._editorDidMount} width={width} height={height} - options={options} + options={{ ...options, wordBasedSuggestions: false }} /> diff --git a/src/plugins/kibana_react/public/code_editor/index.tsx b/src/plugins/kibana_react/public/code_editor/index.tsx index 1607e2b2c11be0..7fd16fea7b415d 100644 --- a/src/plugins/kibana_react/public/code_editor/index.tsx +++ b/src/plugins/kibana_react/public/code_editor/index.tsx @@ -19,6 +19,8 @@ const Fallback = () => ( ); +export type CodeEditorProps = Props; + export const CodeEditor: React.FunctionComponent = (props) => { const darkMode = useUiSetting('theme:darkMode'); return ( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index 4d4556a0ac4ade..cfc5802da159b3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -58,6 +58,7 @@ export const dateHistogramOperation: OperationDefinition< }), input: 'field', priority: 5, // Highest priority level used + operationParams: [{ name: 'interval', type: 'string', required: false }], getErrorMessage: (layer, columnId, indexPattern) => getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), getHelpMessage: (props) => , @@ -75,8 +76,8 @@ export const dateHistogramOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern) => getSafeName(column.sourceField, indexPattern), - buildColumn({ field }) { - let interval = autoInterval; + buildColumn({ field }, columnParams) { + let interval = columnParams?.interval ?? autoInterval; let timeZone: string | undefined; if (field.aggregationRestrictions && field.aggregationRestrictions.date_histogram) { interval = restrictedInterval(field.aggregationRestrictions) as string; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 402bb7a5b1bd95..208122b99b45e8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -16,10 +16,17 @@ import { EuiDescriptionList, EuiText, EuiSpacer, - EuiPanel, + EuiModal, + EuiModalHeader, + EuiModalBody, + EuiModalFooter, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; -import { CodeEditor } from '../../../../../../../../src/plugins/kibana_react/public'; +import { + CodeEditor, + CodeEditorProps, + Markdown, +} from '../../../../../../../../src/plugins/kibana_react/public'; import { OperationDefinition, GenericOperationDefinition, @@ -39,6 +46,7 @@ import { getSafeFieldName, groupArgsByType, hasMathNode, + tinymathFunctions, } from './util'; import { useDebounceWithOptions } from '../helpers'; import { @@ -55,7 +63,6 @@ import './formula.scss'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', }); - export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; params: { @@ -201,6 +208,7 @@ function FormulaEditor({ operationDefinitionMap, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); + const [isOpen, setIsOpen] = useState(false); const editorModel = React.useRef(null); useDebounceWithOptions( @@ -323,84 +331,187 @@ function FormulaEditor({ [indexPattern, operationDefinitionMap] ); - return ( - - - - - {i18n.translate('xpack.lens.formula.functionReferenceLabel', { - defaultMessage: 'Function reference', + const codeEditorOptions: CodeEditorProps = { + height: 280, + width: 300, + languageId: LANGUAGE_ID, + value: text || '', + onChange: setText, + suggestionProvider: { + triggerCharacters: ['.', ',', '(', '='], + provideCompletionItems, + }, + options: { + automaticLayout: true, + fontSize: 14, + folding: false, + lineNumbers: 'off', + scrollBeyondLastLine: false, + minimap: { + enabled: false, + }, + wordWrap: 'on', + // Disable suggestions that appear when we don't provide a default suggestion + wordBasedSuggestions: false, + wrappingIndent: 'indent', + }, + editorDidMount: (editor) => { + const model = editor.getModel(); + if (model) { + editorModel.current = model; + } + editor.onDidDispose(() => (editorModel.current = null)); + }, + }; + + return !isOpen ? ( +
+ + + + + setIsOpen(!isOpen)} iconType="expand" size="s"> + {i18n.translate('xpack.lens.formula.expandEditorLabel', { + defaultMessage: 'Pop out', })} - - -
- -

- {i18n.translate('xpack.lens.formula.basicFunctions', { - defaultMessage: 'Basic functions', + + + + { + updateLayer( + regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ) + ); + }} + iconType="play" + size="s" + > + {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { + defaultMessage: 'Submit', + })} + + + +

+ ) : ( + { + setIsOpen(false); + setText(currentColumn.params.formula); + }} + > + +

+ {i18n.translate('xpack.lens.formula.formulaEditorLabel', { + defaultMessage: 'Formula editor', + })} +

+ + + + +
+ +
+ + +
+ + {i18n.translate('xpack.lens.formula.functionReferenceLabel', { + defaultMessage: 'Function reference', })} -

- -

- +, -, /, * -

- -

- pow() -

- - - - - - {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { - defaultMessage: 'Elasticsearch aggregations', - description: 'Do not translate Elasticsearch', - })} - - ({ - title: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - }))} - /> -
- - - - { - const model = editor.getModel(); - if (model) { - editorModel.current = model; - } - editor.onDidDispose(() => (editorModel.current = null)); + + +
+ + + + key in tinymathFunctions) + .map((key) => ({ + title: `${key}`, + description: , + }))} + /> + + + + + + {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { + defaultMessage: 'Elasticsearch aggregations', + description: 'Do not translate Elasticsearch', + })} + + key in operationDefinitionMap) + .map((key) => ({ + title: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + }))} + /> +
+
+ + + + + { + setIsOpen(false); + setText(currentColumn.params.formula); }} - /> - - + iconType="cross" + > + {i18n.translate('xpack.lens.indexPattern.formulaCancelLabel', { + defaultMessage: 'Cancel', + })} + - - + + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 2d7c07f58c4b24..723efbda2b3be1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -11,6 +11,7 @@ import { monaco } from '@kbn/monaco'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; import { getAvailableOperationsByMetadata } from '../../operations'; +import { tinymathFunctions } from './util'; import type { GenericOperationDefinition } from '..'; export enum SUGGESTION_TYPE { @@ -98,7 +99,7 @@ export function getPossibleFunctions(indexPattern: IndexPattern) { } }); - return uniq(possibleOperationNames); + return [...uniq(possibleOperationNames), ...Object.keys(tinymathFunctions)]; } function getFunctionSuggestions(word: monaco.editor.IWordAtPosition, indexPattern: IndexPattern) { @@ -171,6 +172,7 @@ export function getSuggestion( let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; let detail: string = ''; let command: monaco.languages.CompletionItem['command']; + let documentation: string | monaco.IMarkdownString = ''; switch (type) { case SUGGESTION_TYPE.FIELD: @@ -190,7 +192,15 @@ export function getSuggestion( kind = monaco.languages.CompletionItemKind.Function; insertText = `${insertText}($0)`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; - detail = typeof suggestion === 'string' ? '' : `(${suggestion.displayName})`; + detail = + typeof suggestion === 'string' + ? tinymathFunctions[suggestion] + ? 'TinyMath' + : 'Elasticsearch' + : `(${suggestion.displayName})`; + if (typeof suggestion === 'string' && tinymathFunctions[suggestion]) { + documentation = { value: tinymathFunctions[suggestion].help }; + } break; case SUGGESTION_TYPE.NAMED_ARGUMENT: @@ -212,6 +222,7 @@ export function getSuggestion( insertTextRules, kind, label: insertText, + documentation, command, range, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index abdc9bbf05b393..27648cbca49b88 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -102,30 +102,160 @@ export function getOperationParams( return args; }, {}); } -export const tinymathValidOperators = new Set([ - 'add', - 'subtract', - 'multiply', - 'divide', - 'abs', - 'cbrt', - 'ceil', - 'clamp', - 'cube', - 'exp', - 'fix', - 'floor', - 'log', - 'log10', - 'mod', - 'pow', - 'round', - 'sqrt', - 'square', -]); + +// Todo: i18n everything here +export const tinymathFunctions: Record< + string, + { + positionalArguments: Array<{ + type: 'any' | 'function' | 'number'; + required?: boolean; + }>; + // help: React.ReactElement; + // Help is in Markdown format + help: string; + } +> = { + add: { + positionalArguments: [{ type: 'any' }, { type: 'any', required: true }], + help: ` +Also works with + symbol +Example: ${'`count() + sum(bytes)`'} +Example: ${'`add(count(), 5)`'} + `, + }, + subtract: { + positionalArguments: [{ type: 'any' }, { type: 'any', required: true }], + help: ` +Also works with ${'`-`'} symbol +Example: ${'`subtract(sum(bytes), avg(bytes))`'} + `, + }, + multiply: { + positionalArguments: [{ type: 'function' }], + help: ` +Also works with ${'`*`'} symbol +Example: ${'`multiply(sum(bytes), 2)`'} + `, + }, + divide: { + positionalArguments: [{ type: 'function' }], + help: ` +Also works with ${'`/`'} symbol +Example: ${'`ceil(sum(bytes))`'} + `, + }, + abs: { + positionalArguments: [{ type: 'function' }], + help: ` +Absolute value +Example: ${'`abs(sum(bytes))`'} + `, + }, + cbrt: { + positionalArguments: [{ type: 'function' }], + help: ` +Cube root of value +Example: ${'`cbrt(sum(bytes))`'} + `, + }, + ceil: { + positionalArguments: [{ type: 'function' }], + help: ` +Ceiling of value, rounds up +Example: ${'`ceil(sum(bytes))`'} + `, + }, + clamp: { + positionalArguments: [{ type: 'function' }, { type: 'number' }, { type: 'number' }], + help: ` +Limits the value from a minimum to maximum +Example: ${'`ceil(sum(bytes))`'} + `, + }, + cube: { + positionalArguments: [{ type: 'function' }], + help: ` +Limits the value from a minimum to maximum +Example: ${'`ceil(sum(bytes))`'} + `, + }, + exp: { + positionalArguments: [{ type: 'function' }], + help: ` +Raises e to the nth power. +Example: ${'`exp(sum(bytes))`'} + `, + }, + fix: { + positionalArguments: [{ type: 'function' }], + help: ` +For positive values, takes the floor. For negative values, takes the ceiling. +Example: ${'`fix(sum(bytes))`'} + `, + }, + floor: { + positionalArguments: [{ type: 'function' }], + help: ` +Round down to nearest integer value +Example: ${'`floor(sum(bytes))`'} + `, + }, + log: { + positionalArguments: [{ type: 'function' }, { type: 'number', required: false }], + help: ` +Logarithm with optional base. The natural base e is used as default. +Example: ${'`log(sum(bytes))`'} +Example: ${'`log(sum(bytes), 2)`'} + `, + }, + log10: { + positionalArguments: [{ type: 'function' }], + help: ` +Base 10 logarithm. +Example: ${'`log10(sum(bytes))`'} + `, + }, + mod: { + positionalArguments: [{ type: 'function' }, { type: 'number', required: true }], + help: ` +Remainder after dividing the function by a number +Example: ${'`mod(sum(bytes), 2)`'} + `, + }, + pow: { + positionalArguments: [{ type: 'function' }, { type: 'number', required: true }], + help: ` +Raises the value to a certain power. The second argument is required +Example: ${'`pow(sum(bytes), 3)`'} + `, + }, + round: { + positionalArguments: [{ type: 'function' }, { type: 'number' }], + help: ` +Rounds to a specific number of decimal places, default of 0 +Example: ${'`round(sum(bytes))`'} +Example: ${'`round(sum(bytes), 2)`'} + `, + }, + sqrt: { + positionalArguments: [{ type: 'function' }], + help: ` +Square root of a positive value only +Example: ${'`sqrt(sum(bytes))`'} + `, + }, + square: { + positionalArguments: [{ type: 'function' }], + help: ` +Raise the value to the 2nd power +Example: ${'`square(sum(bytes))`'} + `, + }, +}; export function isMathNode(node: TinymathAST) { - return isObject(node) && node.type === 'function' && tinymathValidOperators.has(node.name); + return isObject(node) && node.type === 'function' && tinymathFunctions[node.name]; } export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index 238ded090013a0..16392ff43466cc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -179,6 +179,23 @@ describe('percentile', () => { expect(percentileColumn.params.percentile).toEqual(95); expect(percentileColumn.label).toEqual('95th percentile of test'); }); + + it('should create a percentile from formula', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileColumn = percentileOperation.buildColumn( + { + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }, + { percentile: 75 } + ); + expect(percentileColumn.dataType).toEqual('number'); + expect(percentileColumn.params.percentile).toEqual(75); + expect(percentileColumn.label).toEqual('95th percentile of test'); + }); }); describe('isTransferable', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index e7654380bd85f1..74b951febf1a47 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -50,6 +50,7 @@ export const percentileOperation: OperationDefinition { if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { return { @@ -71,14 +72,15 @@ export const percentileOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile), - buildColumn: ({ field, previousColumn, indexPattern }) => { + buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { const existingFormat = previousColumn?.params && 'format' in previousColumn?.params ? previousColumn?.params?.format : undefined; const existingPercentileParam = previousColumn?.operationType === 'percentile' && previousColumn?.params.percentile; - const newPercentileParam = existingPercentileParam || DEFAULT_PERCENTILE_VALUE; + const newPercentileParam = + columnParams?.percentile ?? (existingPercentileParam || DEFAULT_PERCENTILE_VALUE); return { label: ofName(getSafeName(field.name, indexPattern), newPercentileParam), dataType: 'number', From 2bc7f6c527d31918b8d0513d636fa583327ec005 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 5 Mar 2021 18:38:30 -0500 Subject: [PATCH 045/185] Change the way math nodes are validated --- .../definitions/formula/math_completion.ts | 16 ++- .../definitions/formula/validation.ts | 111 +++++++++++++----- 2 files changed, 99 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 723efbda2b3be1..9a32bcf3ea7ab7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -116,7 +116,21 @@ function getArgumentSuggestions( operationDefinitionMap: Record ) { const operation = operationDefinitionMap[name]; - if (!operation) { + if (!operation && !tinymathFunctions[name]) { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + + const tinymathFunction = tinymathFunctions[name]; + if (tinymathFunction) { + if ( + tinymathFunction.positionalArguments[position].type === 'function' || + tinymathFunction.positionalArguments[position].type === 'any' + ) { + return { + list: uniq(getPossibleFunctions(indexPattern)), + type: SUGGESTION_TYPE.FUNCTIONS, + }; + } return { list: [], type: SUGGESTION_TYPE.FIELD }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index b62178873a38fe..0b12619577c22f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -17,6 +17,7 @@ import { groupArgsByType, hasInvalidOperations, isMathNode, + tinymathFunctions, } from './util'; import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; @@ -31,7 +32,7 @@ const validationErrors = { wrongFirstArgument: 'wrong first argument', cannotAcceptParameter: 'cannot accept parameter', shouldNotHaveField: 'operation should not have field', - unexpectedNode: 'unexpected node', + tooManyArguments: 'too many arguments', fieldWithNoOperation: 'unexpected field with no operation', failedParsing: 'Failed to parse expression.', // note: this string comes from Tinymath, do not change it duplicateArgument: 'duplicate argument', @@ -126,6 +127,18 @@ function getMessageFromId({ values, }); break; + case 'tooManyArguments': + message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', { + defaultMessage: 'The formula {expression} has too many arguments', + values, + }); + break; + // case 'mathRequiresFunction': + // message = i18n.translate('xpack.lens.indexPattern.formulaMathRequiresFunctionLabel', { + // defaultMessage; 'The function {name} requires an Elasticsearch function', + // values, + // }); + // break; default: message = 'no Error found'; break; @@ -491,32 +504,76 @@ export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTyp export function validateMathNodes(root: TinymathAST, missingVariableSet: Set) { const mathNodes = findMathNodes(root); - const errors = []; - const invalidNodes = mathNodes.filter((node: TinymathFunction) => { - // check the following patterns: - const { variables, functions } = groupArgsByType(node.args); - const fieldVariables = variables.filter((v) => isObject(v) && !missingVariableSet.has(v.value)); - // field + field (or string) - const atLeastTwoFields = fieldVariables.length > 1; - // field + number - // when computing the difference, exclude invalid fields - const validVariables = variables.filter( - (v) => !isObject(v) || !missingVariableSet.has(v.value) - ); - // field + function - const fieldMathWithFunction = fieldVariables.length > 0 && functions.length > 0; - // Make sure to have at least one valid field to compare, or skip the check - const mathBetweenFieldAndNumbers = - fieldVariables.length > 0 && validVariables.length - fieldVariables.length > 0; - return atLeastTwoFields || mathBetweenFieldAndNumbers || fieldMathWithFunction; - }); - if (invalidNodes.length) { - errors.push({ - message: i18n.translate('xpack.lens.indexPattern.mathNotAllowedBetweenFields', { - defaultMessage: 'Math operations are allowed between operations, not fields', - }), - locations: invalidNodes.map(({ location }) => location), + const errors: ErrorWrapper[] = []; + mathNodes.forEach((node: TinymathFunction) => { + const { positionalArguments } = tinymathFunctions[node.name]; + if (!node.args.length) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'operation', + argument: `()`, + }, + locations: [node.location], + }) + ); + } + + if (node.args.length > positionalArguments.length) { + errors.push( + getMessageFromId({ + messageId: 'tooManyArguments', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } + + positionalArguments.forEach((requirements, index) => { + const arg = node.args[index]; + if (requirements.type === 'number' && typeof arg !== 'number') { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: getValueOrName(arg), + }, + locations: [node.location], + }) + ); + } + + if (isObject(arg) && arg.type === 'variable' && !missingVariableSet.has(arg.value)) { + errors.push( + getMessageFromId({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + params: getValueOrName(arg), + }, + locations: [node.location], + }) + ); + } + + if (requirements.type === 'function' && !isObject(arg)) { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: getValueOrName(arg), + }, + locations: [node.location], + }) + ); + } }); - } + }); return errors; } From e0cd7d4561b7cbf783779083cb10dabf2d24a6a6 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 9 Mar 2021 20:05:56 -0500 Subject: [PATCH 046/185] Use custom portal to fix monaco positioning --- .../public/code_editor/code_editor.tsx | 2 +- .../config_panel/dimension_container.tsx | 14 +- .../definitions/formula/formula.scss | 7 +- .../definitions/formula/formula.tsx | 237 +++++++++++++----- .../definitions/formula/math_completion.ts | 73 ++++-- .../operations/definitions/formula/util.ts | 103 ++++++-- 6 files changed, 320 insertions(+), 116 deletions(-) diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index d1ebbc66210b91..5b16d38a8e8a59 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -162,7 +162,7 @@ export class CodeEditor extends React.Component { editorDidMount={this._editorDidMount} width={width} height={height} - options={{ ...options, wordBasedSuggestions: false }} + options={options} /> diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 574cd4029223cc..0dc02da59c28a5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -51,7 +51,19 @@ export function DimensionContainer({ return isOpen ? ( - + { + let current = e.target as HTMLElement; + while (current) { + if (current?.getAttribute('data-test-subj') === 'lnsFormulaWidget') { + return; + } + current = current.parentNode as HTMLElement; + } + closeFlyout(); + }} + isDisabled={!isOpen} + >
(false); const editorModel = React.useRef(null); + const overflowDiv = React.useRef(); + const editorModel2 = React.useRef(null); + const overflowDiv2 = React.useRef(); + useEffect(() => { + const node = (overflowDiv.current = document.createElement('div')); + node.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Add the monaco-editor class because the monaco css depends on it to target + // children + node.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(overflowDiv.current); + return () => { + node.parentNode?.removeChild(node); + }; + }, []); + useEffect(() => { + const node = (overflowDiv2.current = document.createElement('div')); + node.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Add the monaco-editor class because the monaco css depends on it to target + // children + node.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(overflowDiv2.current); + return () => { + node.parentNode?.removeChild(node); + }; + }, []); useDebounceWithOptions( () => { @@ -265,6 +290,60 @@ function FormulaEditor({ [text] ); + useDebounceWithOptions( + () => { + if (!editorModel2.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel2.current, 'LENS', []); + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (!root) return; + if (error) { + errors = [error]; + } else { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel2.current, + 'LENS', + errors.flatMap((innerError) => + innerError.locations.map((location) => ({ + message: innerError.message, + startColumn: location.min + 1, + endColumn: location.max + 1, + // Fake, assumes single line + startLineNumber: 1, + endLineNumber: 1, + severity: monaco.MarkerSeverity.Error, + })) + ) + ); + } else { + monaco.editor.setModelMarkers(editorModel2.current, 'LENS', []); + } + }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: false }, + 256, + [text] + ); + const provideCompletionItems = useCallback( async ( model: monaco.editor.ITextModel, @@ -325,15 +404,17 @@ function FormulaEditor({ } return { - suggestions: aSuggestions.list.map((s) => getSuggestion(s, aSuggestions.type, wordRange)), + suggestions: aSuggestions.list.map((s) => + getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + ), }; }, [indexPattern, operationDefinitionMap] ); + // const provideSignature = useCallback(() => {}, []); + const codeEditorOptions: CodeEditorProps = { - height: 280, - width: 300, languageId: LANGUAGE_ID, value: text || '', onChange: setText, @@ -341,6 +422,7 @@ function FormulaEditor({ triggerCharacters: ['.', ',', '(', '='], provideCompletionItems, }, + // signatureProvider: provideSignature, options: { automaticLayout: true, fontSize: 14, @@ -354,19 +436,32 @@ function FormulaEditor({ // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, wrappingIndent: 'indent', - }, - editorDidMount: (editor) => { - const model = editor.getModel(); - if (model) { - editorModel.current = model; - } - editor.onDidDispose(() => (editorModel.current = null)); + dimension: { width: 300, height: 280 }, + fixedOverflowWidgets: true, }, }; return !isOpen ? (
- + { + const model = editor.getModel(); + if (model) { + editorModel.current = model; + } + editor.onDidDispose(() => { + editorModel.current = null; + // overflowDiv?.current?.parentNode?.removeChild(overflowDiv?.current); + }); + }} + /> @@ -417,26 +512,44 @@ function FormulaEditor({ })} - - - -
- -
- - -
- - {i18n.translate('xpack.lens.formula.functionReferenceLabel', { - defaultMessage: 'Function reference', - })} - - -
- - */} + + +
+ { + const model = editor.getModel(); + if (model) { + editorModel2.current = model; + } + editor.onDidDispose(() => { + editorModel2.current = null; + // overflowDiv2?.current?.parentNode?.removeChild(overflowDiv2?.current); + }); + }} + /> +
+ + +
+ + {i18n.translate('xpack.lens.formula.functionReferenceLabel', { + defaultMessage: 'Function reference', + })} + + +
+ + - - key in tinymathFunctions) - .map((key) => ({ - title: `${key}`, - description: , - }))} - /> - - - - - - {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { - defaultMessage: 'Elasticsearch aggregations', - description: 'Do not translate Elasticsearch', + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', })} - + /> + key in operationDefinitionMap) + .filter((key) => key in tinymathFunctions) .map((key) => ({ - title: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), + title: `${key}`, + description: , }))} /> -
+ + + + + + {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { + defaultMessage: 'Elasticsearch aggregations', + description: 'Do not translate Elasticsearch', + })} + + key in operationDefinitionMap) + .map((key) => ({ + title: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + }))} + />
- - - +
+ + + {/* */} startsWith(func, word.word))), + list: uniq( + getPossibleFunctions(indexPattern).filter((func) => startsWith(func, word.word)) + ).map((func) => ({ label: func, type: 'operation' as const })), type: SUGGESTION_TYPE.FUNCTIONS, }; } @@ -122,12 +130,12 @@ function getArgumentSuggestions( const tinymathFunction = tinymathFunctions[name]; if (tinymathFunction) { - if ( - tinymathFunction.positionalArguments[position].type === 'function' || - tinymathFunction.positionalArguments[position].type === 'any' - ) { + if (tinymathFunction.positionalArguments[position]) { return { - list: uniq(getPossibleFunctions(indexPattern)), + list: uniq(getPossibleFunctions(indexPattern)).map((f) => ({ + type: 'math' as const, + label: f, + })), type: SUGGESTION_TYPE.FUNCTIONS, }; } @@ -170,7 +178,10 @@ function getArgumentSuggestions( ); } }); - return { list: uniq(possibleOperationNames), type: SUGGESTION_TYPE.FUNCTIONS }; + return { + list: uniq(possibleOperationNames).map((n) => ({ label: n, type: 'operation' as const })), + type: SUGGESTION_TYPE.FUNCTIONS, + }; } return { list: [], type: SUGGESTION_TYPE.FIELD }; @@ -179,14 +190,15 @@ function getArgumentSuggestions( export function getSuggestion( suggestion: LensMathSuggestion, type: SUGGESTION_TYPE, - range: monaco.Range + range: monaco.Range, + operationDefinitionMap: Record ): monaco.languages.CompletionItem { let kind: monaco.languages.CompletionItemKind = monaco.languages.CompletionItemKind.Method; - let insertText: string = typeof suggestion === 'string' ? suggestion : suggestion.type; + let label: string = typeof suggestion === 'string' ? suggestion : suggestion.label; + let insertText: string | undefined; let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; let detail: string = ''; let command: monaco.languages.CompletionItem['command']; - let documentation: string | monaco.IMarkdownString = ''; switch (type) { case SUGGESTION_TYPE.FIELD: @@ -195,8 +207,6 @@ export function getSuggestion( id: 'editor.action.triggerSuggest', }; kind = monaco.languages.CompletionItemKind.Value; - insertText = `${insertText}`; - break; case SUGGESTION_TYPE.FUNCTIONS: command = { @@ -204,18 +214,28 @@ export function getSuggestion( id: 'editor.action.triggerSuggest', }; kind = monaco.languages.CompletionItemKind.Function; - insertText = `${insertText}($0)`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; - detail = - typeof suggestion === 'string' - ? tinymathFunctions[suggestion] - ? 'TinyMath' - : 'Elasticsearch' - : `(${suggestion.displayName})`; - if (typeof suggestion === 'string' && tinymathFunctions[suggestion]) { - documentation = { value: tinymathFunctions[suggestion].help }; + if (typeof suggestion !== 'string') { + const tinymathFunction = tinymathFunctions[suggestion.label]; + if (tinymathFunction) { + insertText = `${label}($0)`; + label = `${label}(${tinymathFunction.positionalArguments + .map(({ name }) => name) + .join(', ')})`; + detail = 'TinyMath'; + } else { + const def = operationDefinitionMap[suggestion.label]; + insertText = `${label}($0)`; + if ('operationParams' in def) { + label = `${label}(expression, ${def + .operationParams!.map((p) => `${p.name}=${p.type}`) + .join(', ')}`; + } else { + label = `${label}(expression)`; + } + detail = 'Elasticsearch'; + } } - break; case SUGGESTION_TYPE.NAMED_ARGUMENT: command = { @@ -232,11 +252,10 @@ export function getSuggestion( return { detail, - insertText, - insertTextRules, kind, - label: insertText, - documentation, + label, + insertText: insertText ?? label, + insertTextRules, command, range, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 27648cbca49b88..c527bd4de0cf1e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -6,6 +6,7 @@ */ import { groupBy, isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathFunction, @@ -108,8 +109,8 @@ export const tinymathFunctions: Record< string, { positionalArguments: Array<{ - type: 'any' | 'function' | 'number'; - required?: boolean; + name: string; + optional?: boolean; }>; // help: React.ReactElement; // Help is in Markdown format @@ -117,7 +118,10 @@ export const tinymathFunctions: Record< } > = { add: { - positionalArguments: [{ type: 'any' }, { type: 'any', required: true }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], help: ` Also works with + symbol Example: ${'`count() + sum(bytes)`'} @@ -125,84 +129,117 @@ Example: ${'`add(count(), 5)`'} `, }, subtract: { - positionalArguments: [{ type: 'any' }, { type: 'any', required: true }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], help: ` Also works with ${'`-`'} symbol Example: ${'`subtract(sum(bytes), avg(bytes))`'} `, }, multiply: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], help: ` Also works with ${'`*`'} symbol Example: ${'`multiply(sum(bytes), 2)`'} `, }, divide: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], help: ` Also works with ${'`/`'} symbol Example: ${'`ceil(sum(bytes))`'} `, }, abs: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Absolute value Example: ${'`abs(sum(bytes))`'} `, }, cbrt: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Cube root of value Example: ${'`cbrt(sum(bytes))`'} `, }, ceil: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Ceiling of value, rounds up Example: ${'`ceil(sum(bytes))`'} `, }, clamp: { - positionalArguments: [{ type: 'function' }, { type: 'number' }, { type: 'number' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { name: i18n.translate('xpack.lens.formula.min', { defaultMessage: 'min' }) }, + { name: i18n.translate('xpack.lens.formula.max', { defaultMessage: 'max' }) }, + ], help: ` Limits the value from a minimum to maximum Example: ${'`ceil(sum(bytes))`'} `, }, cube: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Limits the value from a minimum to maximum Example: ${'`ceil(sum(bytes))`'} `, }, exp: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Raises e to the nth power. Example: ${'`exp(sum(bytes))`'} `, }, fix: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` For positive values, takes the floor. For negative values, takes the ceiling. Example: ${'`fix(sum(bytes))`'} `, }, floor: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Round down to nearest integer value Example: ${'`floor(sum(bytes))`'} `, }, log: { - positionalArguments: [{ type: 'function' }, { type: 'number', required: false }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + optional: true, + }, + ], help: ` Logarithm with optional base. The natural base e is used as default. Example: ${'`log(sum(bytes))`'} @@ -210,28 +247,48 @@ Example: ${'`log(sum(bytes), 2)`'} `, }, log10: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Base 10 logarithm. Example: ${'`log10(sum(bytes))`'} `, }, mod: { - positionalArguments: [{ type: 'function' }, { type: 'number', required: true }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + optional: true, + }, + ], help: ` Remainder after dividing the function by a number Example: ${'`mod(sum(bytes), 2)`'} `, }, pow: { - positionalArguments: [{ type: 'function' }, { type: 'number', required: true }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + optional: true, + }, + ], help: ` Raises the value to a certain power. The second argument is required Example: ${'`pow(sum(bytes), 3)`'} `, }, round: { - positionalArguments: [{ type: 'function' }, { type: 'number' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.decimals', { defaultMessage: 'decimals' }), + optional: true, + }, + ], help: ` Rounds to a specific number of decimal places, default of 0 Example: ${'`round(sum(bytes))`'} @@ -239,14 +296,18 @@ Example: ${'`round(sum(bytes), 2)`'} `, }, sqrt: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Square root of a positive value only Example: ${'`sqrt(sum(bytes))`'} `, }, square: { - positionalArguments: [{ type: 'function' }], + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], help: ` Raise the value to the 2nd power Example: ${'`square(sum(bytes))`'} From a1e8412ff45c5c0a5b4da3f6369f167d05dab20c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 10 Mar 2021 15:03:42 -0500 Subject: [PATCH 047/185] Fix model sharing issues --- .../config_panel/dimension_container.tsx | 2 +- .../definitions/formula/formula.tsx | 354 +++++++----------- 2 files changed, 140 insertions(+), 216 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 0dc02da59c28a5..f3962f72dc1f9d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -55,7 +55,7 @@ export function DimensionContainer({ onOutsideClick={(e) => { let current = e.target as HTMLElement; while (current) { - if (current?.getAttribute('data-test-subj') === 'lnsFormulaWidget') { + if (current?.getAttribute?.('data-test-subj') === 'lnsFormulaWidget') { return; } current = current.parentNode as HTMLElement; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 9bfc8e2007d375..e79e98718258d1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -209,10 +209,10 @@ function FormulaEditor({ }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [isOpen, setIsOpen] = useState(false); - const editorModel = React.useRef(null); + const editorModel = React.useRef( + monaco.editor.createModel(text ?? '', LANGUAGE_ID) + ); const overflowDiv = React.useRef(); - const editorModel2 = React.useRef(null); - const overflowDiv2 = React.useRef(); useEffect(() => { const node = (overflowDiv.current = document.createElement('div')); node.setAttribute('data-test-subj', 'lnsFormulaWidget'); @@ -220,18 +220,10 @@ function FormulaEditor({ // children node.classList.add('lnsFormulaOverflow', 'monaco-editor'); document.body.appendChild(overflowDiv.current); + const model = editorModel.current; return () => { - node.parentNode?.removeChild(node); - }; - }, []); - useEffect(() => { - const node = (overflowDiv2.current = document.createElement('div')); - node.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Add the monaco-editor class because the monaco css depends on it to target - // children - node.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(overflowDiv2.current); - return () => { + // Clean up the manually-created model + model.dispose(); node.parentNode?.removeChild(node); }; }, []); @@ -290,60 +282,6 @@ function FormulaEditor({ [text] ); - useDebounceWithOptions( - () => { - if (!editorModel2.current) return; - - if (!text) { - monaco.editor.setModelMarkers(editorModel2.current, 'LENS', []); - return; - } - - let errors: ErrorWrapper[] = []; - - const { root, error } = tryToParse(text); - if (!root) return; - if (error) { - errors = [error]; - } else { - const validationErrors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap - ); - if (validationErrors.length) { - errors = validationErrors; - } - } - - if (errors.length) { - monaco.editor.setModelMarkers( - editorModel2.current, - 'LENS', - errors.flatMap((innerError) => - innerError.locations.map((location) => ({ - message: innerError.message, - startColumn: location.min + 1, - endColumn: location.max + 1, - // Fake, assumes single line - startLineNumber: 1, - endLineNumber: 1, - severity: monaco.MarkerSeverity.Error, - })) - ) - ); - } else { - monaco.editor.setModelMarkers(editorModel2.current, 'LENS', []); - } - }, - // Make it validate on flyout open in case of a broken formula left over - // from a previous edit - { skipFirstRender: false }, - 256, - [text] - ); - const provideCompletionItems = useCallback( async ( model: monaco.editor.ITextModel, @@ -416,22 +354,16 @@ function FormulaEditor({ const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, - value: text || '', + value: text ?? '', onChange: setText, - suggestionProvider: { - triggerCharacters: ['.', ',', '(', '='], - provideCompletionItems, - }, // signatureProvider: provideSignature, options: { - automaticLayout: true, + automaticLayout: false, fontSize: 14, folding: false, lineNumbers: 'off', scrollBeyondLastLine: false, - minimap: { - enabled: false, - }, + minimap: { enabled: false }, wordWrap: 'on', // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, @@ -441,7 +373,15 @@ function FormulaEditor({ }, }; - return !isOpen ? ( + useEffect(() => { + // Because the monaco model is owned by Lens, we need to manually attach handlers + monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + triggerCharacters: ['.', ',', '(', '='], + provideCompletionItems, + }); + }, [provideCompletionItems]); + + return (
{ - const model = editor.getModel(); - if (model) { - editorModel.current = model; - } - editor.onDidDispose(() => { - editorModel.current = null; - // overflowDiv?.current?.parentNode?.removeChild(overflowDiv?.current); - }); + model: editorModel.current, }} /> @@ -497,59 +429,50 @@ function FormulaEditor({ -
- ) : ( - { - setIsOpen(false); - setText(currentColumn.params.formula); - }} - > - -

- {i18n.translate('xpack.lens.formula.formulaEditorLabel', { - defaultMessage: 'Formula editor', - })} -

- - {/* */} - - -
- { - const model = editor.getModel(); - if (model) { - editorModel2.current = model; - } - editor.onDidDispose(() => { - editorModel2.current = null; - // overflowDiv2?.current?.parentNode?.removeChild(overflowDiv2?.current); - }); - }} - /> -
- - -
- - {i18n.translate('xpack.lens.formula.functionReferenceLabel', { - defaultMessage: 'Function reference', + + {isOpen ? ( + { + setIsOpen(false); + setText(currentColumn.params.formula); + }} + > + +

+ {i18n.translate('xpack.lens.formula.formulaEditorLabel', { + defaultMessage: 'Formula editor', })} - - -
- - + + + +
+ +
+ + +
+ + {i18n.translate('xpack.lens.formula.functionReferenceLabel', { + defaultMessage: 'Function reference', + })} + + +
+ + - - key in tinymathFunctions) - .map((key) => ({ - title: `${key}`, - description: , - }))} - /> - - - - - - {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { - defaultMessage: 'Elasticsearch aggregations', - description: 'Do not translate Elasticsearch', - })} - - key in operationDefinitionMap) - .map((key) => ({ - title: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - }))} - /> -
-
- - - {/* */} - - { - setIsOpen(false); - setText(currentColumn.params.formula); - }} - iconType="cross" - > - {i18n.translate('xpack.lens.indexPattern.formulaCancelLabel', { - defaultMessage: 'Cancel', - })} - - { - updateLayer( - regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ) - ); - }} - iconType="play" - > - {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { - defaultMessage: 'Submit', - })} - - - + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + /> + + key in tinymathFunctions) + .map((key) => ({ + title: `${key}`, + description: , + }))} + /> + + + + + + {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { + defaultMessage: 'Elasticsearch aggregations', + description: 'Do not translate Elasticsearch', + })} + + key in operationDefinitionMap) + .map((key) => ({ + title: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + }))} + /> +
+

+ + + + { + setIsOpen(false); + setText(currentColumn.params.formula); + }} + iconType="cross" + > + {i18n.translate('xpack.lens.indexPattern.formulaCancelLabel', { + defaultMessage: 'Cancel', + })} + + { + updateLayer( + regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ) + ); + }} + iconType="play" + > + {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { + defaultMessage: 'Submit', + })} + + + + ) : null} +
); } From 86f36dd10d3ba3341794d98b5299c977032c2736 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 10 Mar 2021 20:00:54 -0500 Subject: [PATCH 048/185] Provide signature help --- .../definitions/formula/formula.tsx | 34 ++++++- .../definitions/formula/math_completion.ts | 93 ++++++++++++++++++- 2 files changed, 122 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index e79e98718258d1..3044f4ecb0b2ec 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -18,7 +18,6 @@ import { EuiSpacer, EuiModal, EuiModalHeader, - EuiModalBody, EuiModalFooter, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; @@ -55,6 +54,7 @@ import { suggest, getSuggestion, getPossibleFunctions, + getSignatureHelp, } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; @@ -350,13 +350,35 @@ function FormulaEditor({ [indexPattern, operationDefinitionMap] ); - // const provideSignature = useCallback(() => {}, []); + const provideSignatureHelp = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken, + context: monaco.languages.SignatureHelpContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getSignatureHelp( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', onChange: setText, - // signatureProvider: provideSignature, options: { automaticLayout: false, fontSize: 14, @@ -379,7 +401,11 @@ function FormulaEditor({ triggerCharacters: ['.', ',', '(', '='], provideCompletionItems, }); - }, [provideCompletionItems]); + monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + signatureHelpTriggerCharacters: ['(', ',', '='], + provideSignatureHelp, + }); + }, [provideCompletionItems, provideSignatureHelp]); return (
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index f7f86ff6c1b067..f7ce656ebff667 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -243,7 +243,7 @@ export function getSuggestion( id: 'editor.action.triggerSuggest', }; kind = monaco.languages.CompletionItemKind.Field; - insertText = `${insertText}=`; + label = `${label}=`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; detail = ''; @@ -260,3 +260,94 @@ export function getSuggestion( range, }; } + +export function getSignatureHelp( + expression: string, + position: number, + operationDefinitionMap: Record +): monaco.languages.SignatureHelpResult { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + const tokenInfo = getInfoAtPosition(ast, position); + + if (tokenInfo?.parent) { + const name = tokenInfo.parent.name; + // reference equality is fine here because of the way the getInfo function works + const index = tokenInfo.parent.args.findIndex((arg) => arg === tokenInfo.ast); + + if (tinymathFunctions[name]) { + const stringify = `${name}(${tinymathFunctions[name].positionalArguments + .map((arg) => arg.name) + .join(', ')})`; + return { + value: { + signatures: [ + { + label: stringify, + parameters: tinymathFunctions[name].positionalArguments.map((arg) => ({ + label: arg.name, + documentation: arg.optional ? 'Optional' : '', + })), + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } else if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + + const firstParam: monaco.languages.ParameterInformation | null = + def.type !== 'count' + ? { + label: + def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; + if ('operationParams' in def) { + return { + value: { + signatures: [ + { + label: `${name}(${ + firstParam ? firstParam.label + ', ' : '' + }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)}`, + parameters: [ + ...(firstParam ? [firstParam] : []), + ...def.operationParams!.map((arg) => ({ + label: `${arg.name}=${arg.type}`, + documentation: arg.required ? 'Required' : '', + })), + ], + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } else { + return { + value: { + signatures: [ + { + label: `${name}(${firstParam ? firstParam.label : ''})`, + parameters: firstParam ? [firstParam] : [], + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } + } + } + } catch (e) { + // do nothing + } + return { value: { signatures: [], activeParameter: 0, activeSignature: 0 }, dispose: () => {} }; +} From dba216c1ca677696d18177aa5b98884e74441e41 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 11 Mar 2021 18:45:49 +0100 Subject: [PATCH 049/185] :bug: Fix small test issue --- .../operations/definitions/percentile.test.tsx | 2 +- .../operations/definitions/percentile.tsx | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index 16392ff43466cc..2640d784d46fe2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -194,7 +194,7 @@ describe('percentile', () => { ); expect(percentileColumn.dataType).toEqual('number'); expect(percentileColumn.params.percentile).toEqual(75); - expect(percentileColumn.label).toEqual('95th percentile of test'); + expect(percentileColumn.label).toEqual('75th percentile of test'); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 125310f1f2d409..89528f21688329 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -74,10 +74,6 @@ export const percentileOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile), buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { - const existingFormat = - previousColumn?.params && 'format' in previousColumn?.params - ? previousColumn?.params?.format - : undefined; const existingPercentileParam = previousColumn?.operationType === 'percentile' && previousColumn?.params.percentile; const newPercentileParam = From 4d5ce24d3bd98c4f867430b61e0fa10ed09ca162 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 11 Mar 2021 18:46:07 +0100 Subject: [PATCH 050/185] :bug: Mark pow arguments as required --- .../operations/definitions/formula/util.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index c527bd4de0cf1e..30658a28c3b30b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -273,7 +273,6 @@ Example: ${'`mod(sum(bytes), 2)`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, { name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), - optional: true, }, ], help: ` From f47c5dba80170d937db5bff204ed0a059dbed704 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 11 Mar 2021 18:46:31 +0100 Subject: [PATCH 051/185] :bug: validate on first render only if a formula is present --- .../operations/definitions/formula/formula.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 3044f4ecb0b2ec..23351465ff67fb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -277,7 +277,7 @@ function FormulaEditor({ }, // Make it validate on flyout open in case of a broken formula left over // from a previous edit - { skipFirstRender: false }, + { skipFirstRender: text == null }, 256, [text] ); From e102ad205b9561e887e47a2acc52453a67ce66c3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 11 Mar 2021 19:15:07 +0100 Subject: [PATCH 052/185] :fire: Remove log10 fn for now --- .../operations/definitions/formula/util.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 30658a28c3b30b..9c8cf303f9c0f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -246,15 +246,16 @@ Example: ${'`log(sum(bytes))`'} Example: ${'`log(sum(bytes), 2)`'} `, }, - log10: { - positionalArguments: [ - { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, - ], - help: ` -Base 10 logarithm. -Example: ${'`log10(sum(bytes))`'} - `, - }, + // TODO: check if this is valid for Tinymath + // log10: { + // positionalArguments: [ + // { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + // ], + // help: ` + // Base 10 logarithm. + // Example: ${'`log10(sum(bytes))`'} + // `, + // }, mod: { positionalArguments: [ { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, From 1483d9dc4b23f5db85eec334d30b7f6a5f963aa7 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 11 Mar 2021 19:15:38 +0100 Subject: [PATCH 053/185] :sparkles: Improved math validation + add tests for math functions --- .../definitions/formula/formula.test.tsx | 64 +++++++- .../definitions/formula/validation.ts | 140 +++++++++++------- 2 files changed, 146 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 85314416ab16e4..98a2a06a83ec93 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -9,6 +9,7 @@ import { createMockedIndexPattern } from '../../../mocks'; import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; +import { tinymathFunctions } from './util'; jest.mock('../../layer_helpers', () => { return { @@ -396,6 +397,16 @@ describe('formula', () => { const formula = 'moving_average(avg(bytes), window=7, window=3)'; testIsBrokenFormula(formula); }); + + it('returns error if a math operation has less arguments than required', () => { + const formula = 'pow(5)'; + testIsBrokenFormula(formula); + }); + + it('returns error if a math operation has the wrong argument type', () => { + const formula = 'pow(bytes)'; + testIsBrokenFormula(formula); + }); }); describe('getErrorMessage', () => { @@ -511,7 +522,7 @@ describe('formula', () => { indexPattern, operationDefinitionMap ) - ).toEqual([`Math operations are allowed between operations, not fields`]); + ).toEqual([`The operation add does not accept any field as argument`]); }); it('returns an error if parsing a syntax invalid formula', () => { @@ -739,5 +750,56 @@ describe('formula', () => { ).toEqual(undefined); } }); + + // there are 4 types of errors for math functions: + // * no argument passed + // * too many arguments passed + // * field passed + // * missing argument + const errors = [ + (operation: string) => + `The first argument for ${operation} should be a operation name. Found ()`, + (operation: string) => `The operation ${operation} has too many arguments`, + (operation: string) => `The operation ${operation} does not accept any field as argument`, + (operation: string) => { + const required = tinymathFunctions[operation].positionalArguments.filter( + ({ optional }) => !optional + ); + return `The operation ${operation} in the Formula is missing ${ + required.length - 1 + } arguments: ${required + .slice(1) + .map(({ name }) => name) + .join(', ')}`; + }, + ]; + // we'll try to map all of these here in this test + for (const fn of Object.keys(tinymathFunctions)) { + it(`returns an error for the math functions available: ${fn}`, () => { + const nArgs = tinymathFunctions[fn].positionalArguments; + // start with the first 3 types + const formulas = [ + `${fn}()`, + `${fn}(1, 2, 3, 4, 5)`, + // to simplify a bit, add the required number of args by the function filled with the field name + `${fn}(${Array(nArgs.length).fill('bytes').join(', ')})`, + ]; + // add the fourth check only for those functions with more than 1 arg required + const enableFourthCheck = nArgs.filter(({ optional }) => !optional).length > 1; + if (enableFourthCheck) { + formulas.push(`${fn}(1)`); + } + formulas.forEach((formula, i) => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([errors[i](fn)]); + }); + }); + } }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 0b12619577c22f..f19f8933aaf2a9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -25,21 +25,43 @@ import type { IndexPattern, IndexPatternLayer } from '../../../types'; import type { TinymathNodeTypes } from './types'; const validationErrors = { - missingField: 'missing field', - missingOperation: 'missing operation', - missingParameter: 'missing parameter', - wrongTypeParameter: 'wrong type parameter', - wrongFirstArgument: 'wrong first argument', - cannotAcceptParameter: 'cannot accept parameter', - shouldNotHaveField: 'operation should not have field', - tooManyArguments: 'too many arguments', - fieldWithNoOperation: 'unexpected field with no operation', - failedParsing: 'Failed to parse expression.', // note: this string comes from Tinymath, do not change it - duplicateArgument: 'duplicate argument', + missingField: { message: 'missing field', type: { variablesLength: 1, variablesList: 'string' } }, + missingOperation: { + message: 'missing operation', + type: { operationLength: 1, operationsList: 'string' }, + }, + missingParameter: { + message: 'missing parameter', + type: { operation: 'string', params: 'string' }, + }, + wrongTypeParameter: { + message: 'wrong type parameter', + type: { operation: 'string', params: 'string' }, + }, + wrongFirstArgument: { + message: 'wrong first argument', + type: { operation: 'string', type: 'string', argument: 'any' as string | number }, + }, + cannotAcceptParameter: { message: 'cannot accept parameter', type: { operation: 'string' } }, + shouldNotHaveField: { message: 'operation should not have field', type: { operation: 'string' } }, + tooManyArguments: { message: 'too many arguments', type: { operation: 'string' } }, + fieldWithNoOperation: { + message: 'unexpected field with no operation', + type: { field: 'string' }, + }, + failedParsing: { message: 'Failed to parse expression', type: { expression: 'string' } }, + duplicateArgument: { + message: 'duplicate argument', + type: { operation: 'string', params: 'string' }, + }, + missingMathArgument: { + message: 'missing math argument', + type: { operation: 'string', count: 1, params: 'string' }, + }, }; -export const errorsLookup = new Set(Object.values(validationErrors)); - +export const errorsLookup = new Set(Object.values(validationErrors).map(({ message }) => message)); type ErrorTypes = keyof typeof validationErrors; +type ErrorValues = typeof validationErrors[K]['type']; export interface ErrorWrapper { message: string; @@ -47,16 +69,16 @@ export interface ErrorWrapper { } export function isParsingError(message: string) { - return message.includes(validationErrors.failedParsing); + return message.includes(validationErrors.failedParsing.message); } -function getMessageFromId({ +function getMessageFromId({ messageId, values, locations, }: { - messageId: ErrorTypes; - values: Record; + messageId: K; + values: ErrorValues; locations: TinymathLocation[]; }): ErrorWrapper { let message: string; @@ -129,7 +151,14 @@ function getMessageFromId({ break; case 'tooManyArguments': message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', { - defaultMessage: 'The formula {expression} has too many arguments', + defaultMessage: 'The operation {operation} has too many arguments', + values, + }); + break; + case 'missingMathArgument': + message = i18n.translate('xpack.lens.indexPattern.formulaMathMissingArgument', { + defaultMessage: + 'The operation {operation} in the Formula is missing {count} arguments: {params}', values, }); break; @@ -508,7 +537,8 @@ export function validateMathNodes(root: TinymathAST, missingVariableSet: Set { const { positionalArguments } = tinymathFunctions[node.name]; if (!node.args.length) { - errors.push( + // we can stop here + return errors.push( getMessageFromId({ messageId: 'wrongFirstArgument', values: { @@ -533,47 +563,43 @@ export function validateMathNodes(root: TinymathAST, missingVariableSet: Set { + // no need to iterate all the arguments, one field is anough to trigger the error + const hasFieldAsArgument = positionalArguments.some((requirements, index) => { const arg = node.args[index]; - if (requirements.type === 'number' && typeof arg !== 'number') { - errors.push( - getMessageFromId({ - messageId: 'wrongTypeParameter', - values: { - operation: node.name, - params: getValueOrName(arg), - }, - locations: [node.location], - }) - ); - } - - if (isObject(arg) && arg.type === 'variable' && !missingVariableSet.has(arg.value)) { - errors.push( - getMessageFromId({ - messageId: 'shouldNotHaveField', - values: { - operation: node.name, - params: getValueOrName(arg), - }, - locations: [node.location], - }) - ); - } - - if (requirements.type === 'function' && !isObject(arg)) { - errors.push( - getMessageFromId({ - messageId: 'wrongTypeParameter', - values: { - operation: node.name, - params: getValueOrName(arg), - }, - locations: [node.location], - }) - ); + if (arg != null && typeof arg !== 'number') { + return arg.type === 'variable' && !missingVariableSet.has(arg.value); } }); + if (hasFieldAsArgument) { + errors.push( + getMessageFromId({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } + + const mandatoryArguments = positionalArguments.filter(({ optional }) => !optional); + // if there is only 1 mandatory arg, this is already handled by the wrongFirstArgument check + if (mandatoryArguments.length > 1 && node.args.length < mandatoryArguments.length) { + const missingArgs = positionalArguments.filter( + ({ name, optional }, i) => !optional && node.args[i] == null + ); + errors.push( + getMessageFromId({ + messageId: 'missingMathArgument', + values: { + operation: node.name, + count: mandatoryArguments.length - node.args.length, + params: missingArgs.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } }); return errors; } From 89bf4d25c142d142d6660048e508a4e7914072dd Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 24 Mar 2021 16:00:02 -0400 Subject: [PATCH 054/185] Fix mount/unmount issues with Monaco --- .../config_panel/dimension_container.tsx | 2 +- .../dimension_panel/dimension_editor.tsx | 22 ++++---- .../definitions/formula/formula.tsx | 52 +++++++++++++------ .../native_renderer/native_renderer.test.tsx | 50 +++++++++++++++++- .../native_renderer/native_renderer.tsx | 26 +++++++++- x-pack/plugins/lens/public/types.ts | 32 +++++++++--- 6 files changed, 145 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index f3962f72dc1f9d..77e35e57afafee 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -30,7 +30,7 @@ export function DimensionContainer({ }: { isOpen: boolean; handleClose: () => void; - panel: React.ReactElement; + panel: React.ReactElement | null; groupLabel: string; }) { const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 29af846c55286d..55ff24e1fc4070 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -457,18 +457,16 @@ export function DimensionEditor(props: DimensionEditorProps) { ) : null} {shouldDisplayExtraOptions && ParamEditor && ( - <> - - + )} {selectedColumn && shouldDisplayExtraOptions && ( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 23351465ff67fb..5e2378953dd9d6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -63,6 +63,7 @@ import './formula.scss'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', }); + export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { operationType: 'formula'; params: { @@ -203,7 +204,6 @@ function FormulaEditor({ updateLayer, currentColumn, columnId, - http, indexPattern, operationDefinitionMap, }: ParamEditorProps) { @@ -212,19 +212,33 @@ function FormulaEditor({ const editorModel = React.useRef( monaco.editor.createModel(text ?? '', LANGUAGE_ID) ); - const overflowDiv = React.useRef(); + const overflowDiv1 = React.useRef(); + const overflowDiv2 = React.useRef(); + + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect + // requires a second render to work, so we are using an if statement to guarantee it happens + // on first render + if (!overflowDiv1?.current) { + const node1 = (overflowDiv1.current = document.createElement('div')); + node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node1); + + const node2 = (overflowDiv2.current = document.createElement('div')); + node2.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node2.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node2); + } + + // Clean up the monaco editor and DOM on unmount useEffect(() => { - const node = (overflowDiv.current = document.createElement('div')); - node.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Add the monaco-editor class because the monaco css depends on it to target - // children - node.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(overflowDiv.current); const model = editorModel.current; return () => { - // Clean up the manually-created model model.dispose(); - node.parentNode?.removeChild(node); + overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); + overflowDiv2.current?.parentNode?.removeChild(overflowDiv2.current); }; }, []); @@ -396,17 +410,23 @@ function FormulaEditor({ }; useEffect(() => { - // Because the monaco model is owned by Lens, we need to manually attach handlers - monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + // Because the monaco model is owned by Lens, we need to manually attach and remove handlers + const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { triggerCharacters: ['.', ',', '(', '='], provideCompletionItems, }); - monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { signatureHelpTriggerCharacters: ['(', ',', '='], provideSignatureHelp, }); + return () => { + dispose1(); + dispose2(); + }; }, [provideCompletionItems, provideSignatureHelp]); + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences + // in the behavior of Monaco when it's first loaded and then reloaded. return (
@@ -476,11 +496,11 @@ function FormulaEditor({ diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx index 34619ae59ae5fe..586c555972cbd4 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { render } from 'react-dom'; import { NativeRenderer } from './native_renderer'; import { act } from 'react-dom/test-utils'; @@ -151,4 +151,52 @@ describe('native_renderer', () => { const containerElement: Element = mountpoint.firstElementChild!; expect(containerElement.nodeName).toBe('SPAN'); }); + + it('should properly unmount a react element that is mounted inside the renderer', () => { + let isUnmounted = false; + + function TestComponent() { + useEffect(() => { + return () => { + isUnmounted = true; + }; + }, []); + return <>Hello; + } + + renderAndTriggerHooks( + { + // This render function mimics the most common usage inside Lens + render(, element); + }} + nativeProps={{}} + />, + mountpoint + ); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(isUnmounted).toBe(true); + }); + + it('should call the unmount function provided for non-react elements', () => { + const unmountCallback = jest.fn(); + + renderAndTriggerHooks( + { + return unmountCallback; + }} + nativeProps={{}} + />, + mountpoint + ); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(unmountCallback).toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx index 68563e01d7f3f4..b5ca37e4d6cf84 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx @@ -5,7 +5,8 @@ * 2.0. */ -import React, { HTMLAttributes } from 'react'; +import React, { HTMLAttributes, useEffect, useRef } from 'react'; +import { unmountComponentAtNode } from 'react-dom'; export interface NativeRendererProps extends HTMLAttributes { render: (domElement: Element, props: T) => void; @@ -19,11 +20,32 @@ export interface NativeRendererProps extends HTMLAttributes { * By default the mountpoint element will be a div, this can be changed with the * `tag` prop. * + * If the rendered component tree was using React, we need to clean it up manually, + * otherwise the unmount event never happens. A future addition is for non-React components + * to get cleaned up, which could be added in the future. + * * @param props */ export function NativeRenderer({ render, nativeProps, tag, ...rest }: NativeRendererProps) { + const elementRef = useRef(); + const cleanupRef = useRef<((cleanupElement: Element) => void) | void>(); + useEffect(() => { + return () => { + if (elementRef.current) { + if (cleanupRef.current) { + cleanupRef.current(elementRef.current); + } + unmountComponentAtNode(elementRef.current); + } + }; + }, []); return React.createElement(tag || 'div', { ...rest, - ref: (el) => el && render(el, nativeProps), + ref: (el) => { + if (el) { + elementRef.current = el; + cleanupRef.current = render(el, nativeProps); + } + }, }); } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 6c88eb20826bb6..66df6a29a3cc3e 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -185,10 +185,22 @@ export interface Datasource { getLayers: (state: T) => string[]; removeColumn: (props: { prevState: T; layerId: string; columnId: string }) => T; - renderDataPanel: (domElement: Element, props: DatasourceDataPanelProps) => void; - renderDimensionTrigger: (domElement: Element, props: DatasourceDimensionTriggerProps) => void; - renderDimensionEditor: (domElement: Element, props: DatasourceDimensionEditorProps) => void; - renderLayerPanel: (domElement: Element, props: DatasourceLayerPanelProps) => void; + renderDataPanel: ( + domElement: Element, + props: DatasourceDataPanelProps + ) => ((cleanupElement: Element) => void) | void; + renderDimensionTrigger: ( + domElement: Element, + props: DatasourceDimensionTriggerProps + ) => ((cleanupElement: Element) => void) | void; + renderDimensionEditor: ( + domElement: Element, + props: DatasourceDimensionEditorProps + ) => ((cleanupElement: Element) => void) | void; + renderLayerPanel: ( + domElement: Element, + props: DatasourceLayerPanelProps + ) => ((cleanupElement: Element) => void) | void; getDropProps: ( props: DatasourceDimensionDropProps & { groupId: string; @@ -585,12 +597,18 @@ export interface Visualization { * Popover contents that open when the user clicks the contextMenuIcon. This can be used * for extra configurability, such as for styling the legend or axis */ - renderLayerContextMenu?: (domElement: Element, props: VisualizationLayerWidgetProps) => void; + renderLayerContextMenu?: ( + domElement: Element, + props: VisualizationLayerWidgetProps + ) => ((cleanupElement: Element) => void) | void; /** * Toolbar rendered above the visualization. This is meant to be used to provide chart-level * settings for the visualization. */ - renderToolbar?: (domElement: Element, props: VisualizationToolbarProps) => void; + renderToolbar?: ( + domElement: Element, + props: VisualizationToolbarProps + ) => ((cleanupElement: Element) => void) | void; /** * Visualizations can provide a custom icon which will open a layer-specific popover * If no icon is provided, gear icon is default @@ -620,7 +638,7 @@ export interface Visualization { renderDimensionEditor?: ( domElement: Element, props: VisualizationDimensionEditorProps - ) => void; + ) => ((cleanupElement: Element) => void) | void; /** * The frame will call this function on all visualizations at different times. The From 2334c1b2988f575ce1b19071b634b0faca877fc2 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 24 Mar 2021 16:14:24 -0400 Subject: [PATCH 055/185] [Lens] Fully unmount React when flyout closes --- .../native_renderer/native_renderer.test.tsx | 50 ++++++++++++++++++- .../native_renderer/native_renderer.tsx | 26 +++++++++- x-pack/plugins/lens/public/types.ts | 32 +++++++++--- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx index 34619ae59ae5fe..586c555972cbd4 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { render } from 'react-dom'; import { NativeRenderer } from './native_renderer'; import { act } from 'react-dom/test-utils'; @@ -151,4 +151,52 @@ describe('native_renderer', () => { const containerElement: Element = mountpoint.firstElementChild!; expect(containerElement.nodeName).toBe('SPAN'); }); + + it('should properly unmount a react element that is mounted inside the renderer', () => { + let isUnmounted = false; + + function TestComponent() { + useEffect(() => { + return () => { + isUnmounted = true; + }; + }, []); + return <>Hello; + } + + renderAndTriggerHooks( + { + // This render function mimics the most common usage inside Lens + render(, element); + }} + nativeProps={{}} + />, + mountpoint + ); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(isUnmounted).toBe(true); + }); + + it('should call the unmount function provided for non-react elements', () => { + const unmountCallback = jest.fn(); + + renderAndTriggerHooks( + { + return unmountCallback; + }} + nativeProps={{}} + />, + mountpoint + ); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(unmountCallback).toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx index 68563e01d7f3f4..b5ca37e4d6cf84 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx @@ -5,7 +5,8 @@ * 2.0. */ -import React, { HTMLAttributes } from 'react'; +import React, { HTMLAttributes, useEffect, useRef } from 'react'; +import { unmountComponentAtNode } from 'react-dom'; export interface NativeRendererProps extends HTMLAttributes { render: (domElement: Element, props: T) => void; @@ -19,11 +20,32 @@ export interface NativeRendererProps extends HTMLAttributes { * By default the mountpoint element will be a div, this can be changed with the * `tag` prop. * + * If the rendered component tree was using React, we need to clean it up manually, + * otherwise the unmount event never happens. A future addition is for non-React components + * to get cleaned up, which could be added in the future. + * * @param props */ export function NativeRenderer({ render, nativeProps, tag, ...rest }: NativeRendererProps) { + const elementRef = useRef(); + const cleanupRef = useRef<((cleanupElement: Element) => void) | void>(); + useEffect(() => { + return () => { + if (elementRef.current) { + if (cleanupRef.current) { + cleanupRef.current(elementRef.current); + } + unmountComponentAtNode(elementRef.current); + } + }; + }, []); return React.createElement(tag || 'div', { ...rest, - ref: (el) => el && render(el, nativeProps), + ref: (el) => { + if (el) { + elementRef.current = el; + cleanupRef.current = render(el, nativeProps); + } + }, }); } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 6c88eb20826bb6..66df6a29a3cc3e 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -185,10 +185,22 @@ export interface Datasource { getLayers: (state: T) => string[]; removeColumn: (props: { prevState: T; layerId: string; columnId: string }) => T; - renderDataPanel: (domElement: Element, props: DatasourceDataPanelProps) => void; - renderDimensionTrigger: (domElement: Element, props: DatasourceDimensionTriggerProps) => void; - renderDimensionEditor: (domElement: Element, props: DatasourceDimensionEditorProps) => void; - renderLayerPanel: (domElement: Element, props: DatasourceLayerPanelProps) => void; + renderDataPanel: ( + domElement: Element, + props: DatasourceDataPanelProps + ) => ((cleanupElement: Element) => void) | void; + renderDimensionTrigger: ( + domElement: Element, + props: DatasourceDimensionTriggerProps + ) => ((cleanupElement: Element) => void) | void; + renderDimensionEditor: ( + domElement: Element, + props: DatasourceDimensionEditorProps + ) => ((cleanupElement: Element) => void) | void; + renderLayerPanel: ( + domElement: Element, + props: DatasourceLayerPanelProps + ) => ((cleanupElement: Element) => void) | void; getDropProps: ( props: DatasourceDimensionDropProps & { groupId: string; @@ -585,12 +597,18 @@ export interface Visualization { * Popover contents that open when the user clicks the contextMenuIcon. This can be used * for extra configurability, such as for styling the legend or axis */ - renderLayerContextMenu?: (domElement: Element, props: VisualizationLayerWidgetProps) => void; + renderLayerContextMenu?: ( + domElement: Element, + props: VisualizationLayerWidgetProps + ) => ((cleanupElement: Element) => void) | void; /** * Toolbar rendered above the visualization. This is meant to be used to provide chart-level * settings for the visualization. */ - renderToolbar?: (domElement: Element, props: VisualizationToolbarProps) => void; + renderToolbar?: ( + domElement: Element, + props: VisualizationToolbarProps + ) => ((cleanupElement: Element) => void) | void; /** * Visualizations can provide a custom icon which will open a layer-specific popover * If no icon is provided, gear icon is default @@ -620,7 +638,7 @@ export interface Visualization { renderDimensionEditor?: ( domElement: Element, props: VisualizationDimensionEditorProps - ) => void; + ) => ((cleanupElement: Element) => void) | void; /** * The frame will call this function on all visualizations at different times. The From 502f74d05ffffbbd42a396125c76fd634d1faeb5 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 25 Mar 2021 14:13:34 -0400 Subject: [PATCH 056/185] Fix bug with editor frame unmounting --- .../native_renderer/native_renderer.test.tsx | 50 +++++++++++++++++++ .../native_renderer/native_renderer.tsx | 21 ++++++-- x-pack/plugins/lens/public/types.ts | 2 +- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx index 586c555972cbd4..8796f619277ff6 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.test.tsx @@ -199,4 +199,54 @@ describe('native_renderer', () => { expect(unmountCallback).toHaveBeenCalled(); }); + + it('should handle when the mount function is asynchronous without a cleanup fn', () => { + let isUnmounted = false; + + function TestComponent() { + useEffect(() => { + return () => { + isUnmounted = true; + }; + }, []); + return <>Hello; + } + + renderAndTriggerHooks( + { + render(, element); + }} + nativeProps={{}} + />, + mountpoint + ); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(isUnmounted).toBe(true); + }); + + it('should handle when the mount function is asynchronous with a cleanup fn', async () => { + const unmountCallback = jest.fn(); + + renderAndTriggerHooks( + { + return unmountCallback; + }} + nativeProps={{}} + />, + mountpoint + ); + + // Schedule a promise cycle to update the DOM + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Replaces the component at the mountpoint with nothing + renderAndTriggerHooks(<>Empty, mountpoint); + + expect(unmountCallback).toHaveBeenCalled(); + }); }); diff --git a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx index b5ca37e4d6cf84..f0659a130b2938 100644 --- a/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx +++ b/x-pack/plugins/lens/public/native_renderer/native_renderer.tsx @@ -8,8 +8,13 @@ import React, { HTMLAttributes, useEffect, useRef } from 'react'; import { unmountComponentAtNode } from 'react-dom'; +type CleanupCallback = (el: Element) => void; + export interface NativeRendererProps extends HTMLAttributes { - render: (domElement: Element, props: T) => void; + render: ( + domElement: Element, + props: T + ) => Promise | CleanupCallback | void; nativeProps: T; tag?: string; } @@ -32,7 +37,7 @@ export function NativeRenderer({ render, nativeProps, tag, ...rest }: NativeR useEffect(() => { return () => { if (elementRef.current) { - if (cleanupRef.current) { + if (cleanupRef.current && typeof cleanupRef.current === 'function') { cleanupRef.current(elementRef.current); } unmountComponentAtNode(elementRef.current); @@ -44,7 +49,17 @@ export function NativeRenderer({ render, nativeProps, tag, ...rest }: NativeR ref: (el) => { if (el) { elementRef.current = el; - cleanupRef.current = render(el, nativeProps); + // Handles the editor frame renderer, which is async + const result = render(el, nativeProps); + if (result instanceof Promise) { + result.then((cleanup) => { + if (typeof cleanup === 'function') { + cleanupRef.current = cleanup; + } + }); + } else if (typeof result === 'function') { + cleanupRef.current = result; + } } }, }); diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 66df6a29a3cc3e..beac4a116091c7 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -64,7 +64,7 @@ export interface EditorFrameProps { showNoDataPopover: () => void; } export interface EditorFrameInstance { - mount: (element: Element, props: EditorFrameProps) => void; + mount: (element: Element, props: EditorFrameProps) => Promise; unmount: () => void; } From 5df313c3f885680fefe26336d41929c0fc857cdc Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 25 Mar 2021 15:13:57 -0400 Subject: [PATCH 057/185] Fix type --- x-pack/plugins/lens/public/app_plugin/app.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/app_plugin/app.test.tsx b/x-pack/plugins/lens/public/app_plugin/app.test.tsx index 38bcf8a377bf2e..20bf349f6b13a1 100644 --- a/x-pack/plugins/lens/public/app_plugin/app.test.tsx +++ b/x-pack/plugins/lens/public/app_plugin/app.test.tsx @@ -72,7 +72,7 @@ const { TopNavMenu } = navigationStartMock.ui; function createMockFrame(): jest.Mocked { return { - mount: jest.fn((el, props) => {}), + mount: jest.fn(async (el, props) => {}), unmount: jest.fn(() => {}), }; } From 2382797958453cf1fdb9d4183ded5cfe52f30f91 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 26 Mar 2021 17:29:48 -0400 Subject: [PATCH 058/185] Add tests for monaco providers, add hover provider --- .../definitions/formula/formula.tsx | 33 +- .../formula/math_completion.test.ts | 296 ++++++++++++++++-- .../definitions/formula/math_completion.ts | 110 ++++++- .../operations/operations.ts | 75 +++-- 4 files changed, 442 insertions(+), 72 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 5e2378953dd9d6..d7b13523cf5330 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -55,6 +55,7 @@ import { getSuggestion, getPossibleFunctions, getSignatureHelp, + getHover, } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; @@ -389,6 +390,30 @@ function FormulaEditor({ [operationDefinitionMap] ); + const provideHover = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getHover( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -412,18 +437,22 @@ function FormulaEditor({ useEffect(() => { // Because the monaco model is owned by Lens, we need to manually attach and remove handlers const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', ',', '(', '='], + triggerCharacters: ['.', ',', '(', '=', ' '], provideCompletionItems, }); const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { signatureHelpTriggerCharacters: ['(', ',', '='], provideSignatureHelp, }); + const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { + provideHover, + }); return () => { dispose1(); dispose2(); + dispose3(); }; - }, [provideCompletionItems, provideSignatureHelp]); + }, [provideCompletionItems, provideSignatureHelp, provideHover]); // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences // in the behavior of Monaco when it's first loaded and then reloaded. diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts index 42374747db255e..b3f7bb904fc6ea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts @@ -6,37 +6,71 @@ */ import { monaco } from '@kbn/monaco'; -import { getSignatureHelp } from './math_completion'; import { createMockedIndexPattern } from '../../../mocks'; -import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; -import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; +import { GenericOperationDefinition } from '../index'; +import type { IndexPatternField } from '../../../types'; +import type { OperationMetadata } from '../../../../types'; import { tinymathFunctions } from './util'; +import { getSignatureHelp, getHover, suggest } from './math_completion'; -const operationDefinitionMap: Record = { - avg: ({ - input: 'field', - buildColumn: ({ field }: { field: IndexPatternField }) => ({ - label: 'avg', +const buildGenericColumn = (type: string) => { + return ({ field }: { field?: IndexPatternField }) => { + return { + label: type, dataType: 'number', - operationType: 'avg', - sourceField: field.name, + operationType: type, + sourceField: field?.name ?? undefined, isBucketed: false, scale: 'ratio', timeScale: false, - }), + }; + }; +}; + +const numericOperation = () => ({ dataType: 'number', isBucketed: false }); +const stringOperation = () => ({ dataType: 'string', isBucketed: true }); + +// Only one of each type is needed +const operationDefinitionMap: Record = { + sum: ({ + type: 'sum', + input: 'field', + buildColumn: buildGenericColumn('sum'), + getPossibleOperationForField: (field: IndexPatternField) => + field.type === 'number' ? numericOperation() : null, } as unknown) as GenericOperationDefinition, - terms: { input: 'field' } as GenericOperationDefinition, - sum: { input: 'field' } as GenericOperationDefinition, - last_value: { input: 'field' } as GenericOperationDefinition, - max: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, - derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { + count: ({ + type: 'count', + input: 'field', + buildColumn: buildGenericColumn('count'), + getPossibleOperationForField: (field: IndexPatternField) => + field.name === 'Records' ? numericOperation() : null, + } as unknown) as GenericOperationDefinition, + moving_average: ({ + type: 'moving_average', input: 'fullReference', + requiredReferences: [ + { + input: ['field', 'managedReference'], + validateMetadata: (meta: OperationMetadata) => + meta.dataType === 'number' && !meta.isBucketed, + }, + ], operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, - cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition, + buildColumn: buildGenericColumn('moving_average'), + getPossibleOperation: numericOperation, + } as unknown) as GenericOperationDefinition, + cumulative_sum: ({ + type: 'cumulative_sum', + input: 'fullReference', + buildColumn: buildGenericColumn('cumulative_sum'), + getPossibleOperation: numericOperation, + } as unknown) as GenericOperationDefinition, + terms: ({ + type: 'terms', + input: 'field', + getPossibleOperationForField: stringOperation, + } as unknown) as GenericOperationDefinition, }; describe('math completion', () => { @@ -46,7 +80,225 @@ describe('math completion', () => { } it('should silently handle parse errors', () => { - expect(getSignatureHelp()); + expect(unwrapSignatures(getSignatureHelp('sum(', 4, operationDefinitionMap))).toBeUndefined(); + }); + + it('should return a signature for a field-based ES function', () => { + expect(unwrapSignatures(getSignatureHelp('sum()', 4, operationDefinitionMap))).toEqual({ + label: 'sum(field)', + parameters: [{ label: 'field' }], + }); + }); + + it('should return a signature for count', () => { + expect(unwrapSignatures(getSignatureHelp('count()', 6, operationDefinitionMap))).toEqual({ + label: 'count()', + parameters: [], + }); + }); + + it('should return a signature for a function with named parameters', () => { + expect( + unwrapSignatures( + getSignatureHelp('2 * moving_average(count(), window=)', 35, operationDefinitionMap) + ) + ).toEqual({ + label: 'moving_average(function, window=number)', + parameters: [ + { label: 'function' }, + { + label: 'window=number', + documentation: 'Required', + }, + ], + }); + }); + + it('should return a signature for an inner function', () => { + expect( + unwrapSignatures( + getSignatureHelp('2 * moving_average(count())', 25, operationDefinitionMap) + ) + ).toEqual({ label: 'count()', parameters: [] }); + }); + + it('should return a signature for a complex tinymath function', () => { + expect( + unwrapSignatures(getSignatureHelp('clamp(count(), 5)', 7, operationDefinitionMap)) + ).toEqual({ + label: 'clamp(expression, min, max)', + parameters: [ + { label: 'expression', documentation: '' }, + { label: 'min', documentation: '' }, + { label: 'max', documentation: '' }, + ], + }); + }); + }); + + describe('hover provider', () => { + it('should silently handle parse errors', () => { + expect(getHover('sum(', 2, operationDefinitionMap)).toEqual({ contents: [] }); + }); + + it('should show signature for a field-based ES function', () => { + expect(getHover('sum()', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'sum(field)' }], + }); + }); + + it('should show signature for count', () => { + expect(getHover('count()', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'count()' }], + }); + }); + + it('should show signature for a function with named parameters', () => { + expect(getHover('2 * moving_average(count())', 10, operationDefinitionMap)).toEqual({ + contents: [{ value: 'moving_average(function, window=number)' }], + }); + }); + + it('should show signature for an inner function', () => { + expect(getHover('2 * moving_average(count())', 22, operationDefinitionMap)).toEqual({ + contents: [{ value: 'count()' }], + }); + }); + + it('should show signature for a complex tinymath function', () => { + expect(getHover('clamp(count(), 5)', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'clamp(expression, min, max)' }], + }); + }); + }); + + describe('autocomplete', () => { + it('should list all valid functions at the top level (fake test)', async () => { + // This test forces an invalid scenario, since the autocomplete actually requires + // some typing + const results = await suggest( + '', + 1, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 1, endColumn: 1 } + ); + expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + }); + + it('should list all valid sub-functions for a fullReference', async () => { + const results = await suggest( + 'moving_average()', + 15, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 15, endColumn: 15 } + ); + expect(results.list).toHaveLength(1); + ['sum'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + }); + + it('should list all valid named arguments for a fullReference', async () => { + const results = await suggest( + 'moving_average(count(),)', + 23, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 23, endColumn: 23 } + ); + expect(results.list).toEqual(['window']); + }); + + it('should list all valid positional arguments for a tinymath function used by name', async () => { + const results = await suggest( + 'divide(count(), )', + 16, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 16, endColumn: 16 } + ); + expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + }); + + it('should list all valid positional arguments for a tinymath function used with alias', async () => { + const results = await suggest( + 'count() / ', + 10, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 10, endColumn: 10 } + ); + expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + }); + + it('should not autocomplete any fields for the count function', async () => { + const results = await suggest( + 'count()', + 6, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 6, endColumn: 6 } + ); + expect(results.list).toHaveLength(0); + }); + + it('should autocomplete and validate the right type of field', async () => { + const results = await suggest( + 'sum()', + 4, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 4, endColumn: 4 } + ); + expect(results.list).toEqual(['bytes', 'memory']); }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index f7ce656ebff667..0460556ba8aaf9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -10,7 +10,7 @@ import { monaco } from '@kbn/monaco'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; -import { getAvailableOperationsByMetadata } from '../../operations'; +import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; import { tinymathFunctions } from './util'; import type { GenericOperationDefinition } from '..'; @@ -86,7 +86,7 @@ export async function suggest( ); } if (tokenInfo && word) { - return getFunctionSuggestions(word, indexPattern); + return getFunctionSuggestions(word, indexPattern, operationDefinitionMap); } } catch (e) { // Fail silently @@ -94,8 +94,11 @@ export async function suggest( return { list: [], type: SUGGESTION_TYPE.FIELD }; } -export function getPossibleFunctions(indexPattern: IndexPattern) { - const available = getAvailableOperationsByMetadata(indexPattern); +export function getPossibleFunctions( + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const available = memoizedGetAvailableOperationsByMetadata(indexPattern, operationDefinitionMap); const possibleOperationNames: string[] = []; available.forEach((a) => { if (a.operationMetaData.dataType === 'number' && !a.operationMetaData.isBucketed) { @@ -108,10 +111,16 @@ export function getPossibleFunctions(indexPattern: IndexPattern) { return [...uniq(possibleOperationNames), ...Object.keys(tinymathFunctions)]; } -function getFunctionSuggestions(word: monaco.editor.IWordAtPosition, indexPattern: IndexPattern) { +function getFunctionSuggestions( + word: monaco.editor.IWordAtPosition, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { return { list: uniq( - getPossibleFunctions(indexPattern).filter((func) => startsWith(func, word.word)) + getPossibleFunctions(indexPattern, operationDefinitionMap).filter((func) => + startsWith(func, word.word) + ) ).map((func) => ({ label: func, type: 'operation' as const })), type: SUGGESTION_TYPE.FUNCTIONS, }; @@ -132,7 +141,7 @@ function getArgumentSuggestions( if (tinymathFunction) { if (tinymathFunction.positionalArguments[position]) { return { - list: uniq(getPossibleFunctions(indexPattern)).map((f) => ({ + list: uniq(getPossibleFunctions(indexPattern, operationDefinitionMap)).map((f) => ({ type: 'math' as const, label: f, })), @@ -153,15 +162,32 @@ function getArgumentSuggestions( return { list: [], type: SUGGESTION_TYPE.FIELD }; } + // TODO: Expand to all valid fields for the function if (operation.input === 'field' && position === 0) { - const fields = indexPattern.fields - .filter((field) => field.type === 'number') - .map((field) => field.name); - return { list: fields, type: SUGGESTION_TYPE.FIELD }; + const available = memoizedGetAvailableOperationsByMetadata( + indexPattern, + operationDefinitionMap + ); + const validOperation = available.find( + ({ operationMetaData }) => + operationMetaData.dataType === 'number' && !operationMetaData.isBucketed + ); + if (validOperation) { + const fields = validOperation.operations + .filter((op) => op.operationType === operation.type) + .map((op) => ('field' in op ? op.field : undefined)) + .filter((field) => field); + return { list: fields, type: SUGGESTION_TYPE.FIELD }; + } else { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } } if (operation.input === 'fullReference') { - const available = getAvailableOperationsByMetadata(indexPattern); + const available = memoizedGetAvailableOperationsByMetadata( + indexPattern, + operationDefinitionMap + ); const possibleOperationNames: string[] = []; available.forEach((a) => { if ( @@ -230,6 +256,8 @@ export function getSuggestion( label = `${label}(expression, ${def .operationParams!.map((p) => `${p.name}=${p.type}`) .join(', ')}`; + } else if (suggestion.label === 'count') { + label = `${label}()`; } else { label = `${label}(expression)`; } @@ -244,7 +272,6 @@ export function getSuggestion( }; kind = monaco.languages.CompletionItemKind.Field; label = `${label}=`; - insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; detail = ''; break; @@ -301,7 +328,7 @@ export function getSignatureHelp( const def = operationDefinitionMap[name]; const firstParam: monaco.languages.ParameterInformation | null = - def.type !== 'count' + name !== 'count' ? { label: def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', @@ -314,7 +341,7 @@ export function getSignatureHelp( { label: `${name}(${ firstParam ? firstParam.label + ', ' : '' - }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)}`, + }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, parameters: [ ...(firstParam ? [firstParam] : []), ...def.operationParams!.map((arg) => ({ @@ -351,3 +378,56 @@ export function getSignatureHelp( } return { value: { signatures: [], activeParameter: 0, activeSignature: 0 }, dispose: () => {} }; } + +export function getHover( + expression: string, + position: number, + operationDefinitionMap: Record +): monaco.languages.Hover { + try { + const ast = parse(expression); + + const tokenInfo = getInfoAtPosition(ast, position); + + if (!tokenInfo || typeof tokenInfo.ast === 'number' || !('name' in tokenInfo.ast)) { + return { contents: [] }; + } + + const name = tokenInfo.ast.name; + + if (tinymathFunctions[name]) { + const stringify = `${name}(${tinymathFunctions[name].positionalArguments + .map((arg) => arg.name) + .join(', ')})`; + return { contents: [{ value: stringify }] }; + } else if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + + const firstParam: monaco.languages.ParameterInformation | null = + name !== 'count' + ? { + label: + def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; + if ('operationParams' in def) { + return { + contents: [ + { + value: `${name}(${ + firstParam ? firstParam.label + ', ' : '' + }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, + }, + ], + }; + } else { + return { + contents: [{ value: `${name}(${firstParam ? firstParam.label : ''})` }], + }; + } + } + } catch (e) { + // do nothing + } + return { contents: [] }; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts index 7d02a0dc83299c..437d2af005961f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts @@ -142,7 +142,11 @@ type OperationFieldTuple = * ] * ``` */ -export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { +export function getAvailableOperationsByMetadata( + indexPattern: IndexPattern, + // For consistency in testing + customOperationDefinitionMap?: Record +) { const operationByMetadata: Record< string, { operationMetaData: OperationMetadata; operations: OperationFieldTuple[] } @@ -165,44 +169,49 @@ export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { } }; - operationDefinitions.sort(getSortScoreByPriority).forEach((operationDefinition) => { - if (operationDefinition.input === 'field') { - indexPattern.fields.forEach((field) => { + (customOperationDefinitionMap + ? Object.values(customOperationDefinitionMap) + : operationDefinitions + ) + .sort(getSortScoreByPriority) + .forEach((operationDefinition) => { + if (operationDefinition.input === 'field') { + indexPattern.fields.forEach((field) => { + addToMap( + { + type: 'field', + operationType: operationDefinition.type, + field: field.name, + }, + operationDefinition.getPossibleOperationForField(field) + ); + }); + } else if (operationDefinition.input === 'none') { addToMap( { - type: 'field', + type: 'none', operationType: operationDefinition.type, - field: field.name, }, - operationDefinition.getPossibleOperationForField(field) - ); - }); - } else if (operationDefinition.input === 'none') { - addToMap( - { - type: 'none', - operationType: operationDefinition.type, - }, - operationDefinition.getPossibleOperation() - ); - } else if (operationDefinition.input === 'fullReference') { - const validOperation = operationDefinition.getPossibleOperation(indexPattern); - if (validOperation) { - addToMap( - { type: 'fullReference', operationType: operationDefinition.type }, - validOperation + operationDefinition.getPossibleOperation() ); + } else if (operationDefinition.input === 'fullReference') { + const validOperation = operationDefinition.getPossibleOperation(indexPattern); + if (validOperation) { + addToMap( + { type: 'fullReference', operationType: operationDefinition.type }, + validOperation + ); + } + } else if (operationDefinition.input === 'managedReference') { + const validOperation = operationDefinition.getPossibleOperation(); + if (validOperation) { + addToMap( + { type: 'managedReference', operationType: operationDefinition.type }, + validOperation + ); + } } - } else if (operationDefinition.input === 'managedReference') { - const validOperation = operationDefinition.getPossibleOperation(); - if (validOperation) { - addToMap( - { type: 'managedReference', operationType: operationDefinition.type }, - validOperation - ); - } - } - }); + }); return Object.values(operationByMetadata); } From f746acfcbc141093103cfb444262ac91abdc986e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 26 Mar 2021 17:41:28 -0400 Subject: [PATCH 059/185] Add test for last_value --- .../formula/math_completion.test.ts | 40 +++++++++++++++---- .../definitions/formula/math_completion.ts | 2 +- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts index b3f7bb904fc6ea..083d4b2267ab80 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts @@ -46,6 +46,15 @@ const operationDefinitionMap: Record = { getPossibleOperationForField: (field: IndexPatternField) => field.name === 'Records' ? numericOperation() : null, } as unknown) as GenericOperationDefinition, + last_value: ({ + type: 'last_value', + input: 'field', + buildColumn: buildGenericColumn('last_value'), + getPossibleOperationForField: (field: IndexPatternField) => ({ + dataType: field.type, + isBucketed: false, + }), + } as unknown) as GenericOperationDefinition, moving_average: ({ type: 'moving_average', input: 'fullReference', @@ -187,8 +196,8 @@ describe('math completion', () => { operationDefinitionMap, { word: '', startColumn: 1, endColumn: 1 } ); - expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); - ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); }); Object.keys(tinymathFunctions).forEach((key) => { @@ -208,8 +217,8 @@ describe('math completion', () => { operationDefinitionMap, { word: '', startColumn: 15, endColumn: 15 } ); - expect(results.list).toHaveLength(1); - ['sum'].forEach((key) => { + expect(results.list).toHaveLength(2); + ['sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); }); }); @@ -241,8 +250,8 @@ describe('math completion', () => { operationDefinitionMap, { word: '', startColumn: 16, endColumn: 16 } ); - expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); - ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); }); Object.keys(tinymathFunctions).forEach((key) => { @@ -262,8 +271,8 @@ describe('math completion', () => { operationDefinitionMap, { word: '', startColumn: 10, endColumn: 10 } ); - expect(results.list).toHaveLength(3 + Object.keys(tinymathFunctions).length); - ['sum', 'moving_average', 'cumulative_sum'].forEach((key) => { + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); }); Object.keys(tinymathFunctions).forEach((key) => { @@ -300,5 +309,20 @@ describe('math completion', () => { ); expect(results.list).toEqual(['bytes', 'memory']); }); + + it('should autocomplete only operations that provide numeric output', async () => { + const results = await suggest( + 'last_value()', + 11, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 11, endColumn: 11 } + ); + expect(results.list).toEqual(['bytes', 'memory']); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 0460556ba8aaf9..931d947b2b7e3e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -162,12 +162,12 @@ function getArgumentSuggestions( return { list: [], type: SUGGESTION_TYPE.FIELD }; } - // TODO: Expand to all valid fields for the function if (operation.input === 'field' && position === 0) { const available = memoizedGetAvailableOperationsByMetadata( indexPattern, operationDefinitionMap ); + // TODO: This only allow numeric functions, will reject last_value(string) for example. const validOperation = available.find( ({ operationMetaData }) => operationMetaData.dataType === 'number' && !operationMetaData.isBucketed From 387059e34890f5bb6dad545b25bb82f6a4e069c9 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 29 Mar 2021 17:34:12 -0400 Subject: [PATCH 060/185] Usability improvements --- .../dimension_panel/dimension_editor.tsx | 1 + .../definitions/formula/formula.tsx | 68 ++++++++++--------- .../formula/math_completion.test.ts | 15 ++++ .../definitions/formula/math_completion.ts | 62 +++++++++++++---- 4 files changed, 103 insertions(+), 43 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 112a440c1ee68c..8f0970bfd1daf7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -594,6 +594,7 @@ export function DimensionEditor(props: DimensionEditorProps) { indexPattern: currentIndexPattern, columnId, op: 'formula', + visualizationGroups: dimensionGroups, }); setStateWrapper(newLayer); trackUiEvent(`indexpattern_dimension_operation_formula`); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index d7b13523cf5330..522e23bc672d1a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -56,6 +56,7 @@ import { getPossibleFunctions, getSignatureHelp, getHover, + offsetToRowColumn, } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; @@ -122,30 +123,33 @@ export const formulaOperation: OperationDefinition< ? currentColumn.label : params?.formula : ''; - return [ - { - type: 'function', - function: 'mapColumn', - arguments: { - id: [columnId], - name: [label || ''], - exp: [ - { - type: 'expression', - chain: [ + + return currentColumn.references.length + ? [ + { + type: 'function', + function: 'mapColumn', + arguments: { + id: [columnId], + name: [label || ''], + exp: [ { - type: 'function', - function: 'math', - arguments: { - expression: [`${currentColumn.references[0]}`], - }, + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [`${currentColumn.references[0]}`], + }, + }, + ], }, ], }, - ], - }, - }, - ]; + }, + ] + : []; }, buildColumn({ previousColumn, layer }, _, operationDefinitionMap) { let previousFormula = ''; @@ -275,15 +279,18 @@ function FormulaEditor({ editorModel.current, 'LENS', errors.flatMap((innerError) => - innerError.locations.map((location) => ({ - message: innerError.message, - startColumn: location.min + 1, - endColumn: location.max + 1, - // Fake, assumes single line - startLineNumber: 1, - endLineNumber: 1, - severity: monaco.MarkerSeverity.Error, - })) + innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Error, + }; + }) ) ); } else { @@ -428,8 +435,7 @@ function FormulaEditor({ wordWrap: 'on', // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, - wrappingIndent: 'indent', - dimension: { width: 300, height: 280 }, + dimension: { width: 290, height: 280 }, fixedOverflowWidgets: true, }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts index 083d4b2267ab80..001bf369ccbe08 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts @@ -238,6 +238,21 @@ describe('math completion', () => { expect(results.list).toEqual(['window']); }); + it('should not list named arguments when they are already in use', async () => { + const results = await suggest( + 'moving_average(count(), window=5, )', + 34, + { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + createMockedIndexPattern(), + operationDefinitionMap, + { word: '', startColumn: 34, endColumn: 34 } + ); + expect(results.list).toEqual([]); + }); + it('should list all valid positional arguments for a tinymath function used by name', async () => { const results = await suggest( 'divide(count(), )', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 931d947b2b7e3e..43c45938ae265f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -11,7 +11,7 @@ import { monaco } from '@kbn/monaco'; import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; import { IndexPattern } from '../../../types'; import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; -import { tinymathFunctions } from './util'; +import { tinymathFunctions, groupArgsByType } from './util'; import type { GenericOperationDefinition } from '..'; export enum SUGGESTION_TYPE { @@ -61,6 +61,32 @@ function getInfoAtPosition( }; } +export function offsetToRowColumn(expression: string, offset: number): monaco.Position { + const lines = expression.split(/\n/); + let remainingChars = offset; + let lineNumber = 1; + for (const line of lines) { + if (line.length >= remainingChars) { + return new monaco.Position(lineNumber, remainingChars); + } + remainingChars -= line.length + 1; + lineNumber++; + } + + throw new Error('Algorithm failure'); +} + +export function monacoPositionToOffset(expression: string, position: monaco.Position): number { + const lines = expression.split(/\n/); + return lines + .slice(0, position.lineNumber - 1) + .reduce( + (prev, current, index) => + prev + index === position.lineNumber - 1 ? position.column - 1 : current.length, + 0 + ); +} + export async function suggest( expression: string, position: number, @@ -79,8 +105,8 @@ export async function suggest( // TODO } else if (tokenInfo?.parent) { return getArgumentSuggestions( - tokenInfo.parent.name, - tokenInfo.parent.args.length - 1, + tokenInfo.parent, + tokenInfo.parent.args.findIndex((a) => a === tokenInfo.ast), indexPattern, operationDefinitionMap ); @@ -96,7 +122,7 @@ export async function suggest( export function getPossibleFunctions( indexPattern: IndexPattern, - operationDefinitionMap: Record + operationDefinitionMap?: Record ) { const available = memoizedGetAvailableOperationsByMetadata(indexPattern, operationDefinitionMap); const possibleOperationNames: string[] = []; @@ -127,11 +153,12 @@ function getFunctionSuggestions( } function getArgumentSuggestions( - name: string, + ast: TinymathFunction, position: number, indexPattern: IndexPattern, operationDefinitionMap: Record ) { + const { name } = ast; const operation = operationDefinitionMap[name]; if (!operation && !tinymathFunctions[name]) { return { list: [], type: SUGGESTION_TYPE.FIELD }; @@ -153,7 +180,15 @@ function getArgumentSuggestions( if (position > 0) { if ('operationParams' in operation) { - const suggestedParam = operation.operationParams!.map((p) => p.name); + // Exclude any previously used named args + const { namedArguments } = groupArgsByType(ast.args); + const suggestedParam = operation + .operationParams!.filter( + (param) => + // Keep the param if it's the first use + !namedArguments.find((arg) => arg.name === param.name) + ) + .map((p) => p.name); return { list: suggestedParam, type: SUGGESTION_TYPE.NAMED_ARGUMENT, @@ -172,7 +207,7 @@ function getArgumentSuggestions( ({ operationMetaData }) => operationMetaData.dataType === 'number' && !operationMetaData.isBucketed ); - if (validOperation) { + if (validOperation && operation.type !== 'count') { const fields = validOperation.operations .filter((op) => op.operationType === operation.type) .map((op) => ('field' in op ? op.field : undefined)) @@ -225,6 +260,7 @@ export function getSuggestion( let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; let detail: string = ''; let command: monaco.languages.CompletionItem['command']; + let sortText: string = ''; switch (type) { case SUGGESTION_TYPE.FIELD: @@ -239,19 +275,19 @@ export function getSuggestion( title: 'Trigger Suggestion Dialog', id: 'editor.action.triggerSuggest', }; - kind = monaco.languages.CompletionItemKind.Function; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; if (typeof suggestion !== 'string') { const tinymathFunction = tinymathFunctions[suggestion.label]; + insertText = `${label}($0)`; if (tinymathFunction) { - insertText = `${label}($0)`; label = `${label}(${tinymathFunction.positionalArguments .map(({ name }) => name) .join(', ')})`; detail = 'TinyMath'; + kind = monaco.languages.CompletionItemKind.Method; } else { const def = operationDefinitionMap[suggestion.label]; - insertText = `${label}($0)`; + kind = monaco.languages.CompletionItemKind.Constant; if ('operationParams' in def) { label = `${label}(expression, ${def .operationParams!.map((p) => `${p.name}=${p.type}`) @@ -262,6 +298,8 @@ export function getSuggestion( label = `${label}(expression)`; } detail = 'Elasticsearch'; + // Always put ES functions first + sortText = `0${label}`; } } break; @@ -270,10 +308,9 @@ export function getSuggestion( title: 'Trigger Suggestion Dialog', id: 'editor.action.triggerSuggest', }; - kind = monaco.languages.CompletionItemKind.Field; + kind = monaco.languages.CompletionItemKind.Keyword; label = `${label}=`; detail = ''; - break; } @@ -285,6 +322,7 @@ export function getSuggestion( insertTextRules, command, range, + sortText, }; } From 46bd80fe1adc61c5d2d9eba06c4c4c919e9c2ba8 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 30 Mar 2021 18:25:05 -0400 Subject: [PATCH 061/185] Add KQL and Lucene named parameters --- packages/kbn-tinymath/src/grammar.js | 451 ++++++++++++------ packages/kbn-tinymath/src/grammar.pegjs | 18 +- packages/kbn-tinymath/test/library.test.js | 16 + .../definitions/calculations/counter_rate.tsx | 12 +- .../calculations/cumulative_sum.tsx | 12 +- .../definitions/calculations/differences.tsx | 12 +- .../calculations/moving_average.tsx | 10 +- .../operations/definitions/cardinality.tsx | 17 +- .../operations/definitions/count.tsx | 12 +- .../definitions/formula/formula.test.tsx | 148 +++--- .../definitions/formula/math_completion.ts | 78 +-- .../definitions/formula/validation.ts | 5 +- .../operations/definitions/index.ts | 12 +- .../operations/definitions/last_value.tsx | 12 +- .../operations/definitions/metrics.tsx | 15 +- 15 files changed, 561 insertions(+), 269 deletions(-) diff --git a/packages/kbn-tinymath/src/grammar.js b/packages/kbn-tinymath/src/grammar.js index 5454143530c398..6e52b5769d4415 100644 --- a/packages/kbn-tinymath/src/grammar.js +++ b/packages/kbn-tinymath/src/grammar.js @@ -156,7 +156,11 @@ function peg$parse(input, options) { peg$c12 = function(literal) { return literal; }, - peg$c13 = function(chars) { + peg$c13 = /^[']/, + peg$c14 = peg$classExpectation(["'"], false, false), + peg$c15 = /^["]/, + peg$c16 = peg$classExpectation(["\""], false, false), + peg$c17 = function(chars) { return { type: 'variable', value: chars.join(''), @@ -164,7 +168,7 @@ function peg$parse(input, options) { text: text() }; }, - peg$c14 = function(rest) { + peg$c18 = function(rest) { return { type: 'variable', value: rest.join(''), @@ -172,11 +176,11 @@ function peg$parse(input, options) { text: text() }; }, - peg$c15 = "+", - peg$c16 = peg$literalExpectation("+", false), - peg$c17 = "-", - peg$c18 = peg$literalExpectation("-", false), - peg$c19 = function(left, rest) { + peg$c19 = "+", + peg$c20 = peg$literalExpectation("+", false), + peg$c21 = "-", + peg$c22 = peg$literalExpectation("-", false), + peg$c23 = function(left, rest) { return rest.reduce((acc, curr) => ({ type: 'function', name: curr[0] === '+' ? 'add' : 'subtract', @@ -185,11 +189,11 @@ function peg$parse(input, options) { text: text() }), left) }, - peg$c20 = "*", - peg$c21 = peg$literalExpectation("*", false), - peg$c22 = "/", - peg$c23 = peg$literalExpectation("/", false), - peg$c24 = function(left, rest) { + peg$c24 = "*", + peg$c25 = peg$literalExpectation("*", false), + peg$c26 = "/", + peg$c27 = peg$literalExpectation("/", false), + peg$c28 = function(left, rest) { return rest.reduce((acc, curr) => ({ type: 'function', name: curr[0] === '*' ? 'multiply' : 'divide', @@ -198,30 +202,30 @@ function peg$parse(input, options) { text: text() }), left) }, - peg$c25 = "(", - peg$c26 = peg$literalExpectation("(", false), - peg$c27 = ")", - peg$c28 = peg$literalExpectation(")", false), - peg$c29 = function(expr) { + peg$c29 = "(", + peg$c30 = peg$literalExpectation("(", false), + peg$c31 = ")", + peg$c32 = peg$literalExpectation(")", false), + peg$c33 = function(expr) { return expr }, - peg$c30 = peg$otherExpectation("arguments"), - peg$c31 = ",", - peg$c32 = peg$literalExpectation(",", false), - peg$c33 = function(first, arg) {return arg}, - peg$c34 = function(first, rest) { + peg$c34 = peg$otherExpectation("arguments"), + peg$c35 = ",", + peg$c36 = peg$literalExpectation(",", false), + peg$c37 = function(first, arg) {return arg}, + peg$c38 = function(first, rest) { return [first].concat(rest); }, - peg$c35 = /^["]/, - peg$c36 = peg$classExpectation(["\""], false, false), - peg$c37 = function(value) { return value.join(''); }, - peg$c38 = /^[']/, - peg$c39 = peg$classExpectation(["'"], false, false), - peg$c40 = /^[a-zA-Z_]/, - peg$c41 = peg$classExpectation([["a", "z"], ["A", "Z"], "_"], false, false), - peg$c42 = "=", - peg$c43 = peg$literalExpectation("=", false), - peg$c44 = function(name, value) { + peg$c39 = /^[^"]/, + peg$c40 = peg$classExpectation(["\""], true, false), + peg$c41 = function(value) { return value.join(''); }, + peg$c42 = /^[^']/, + peg$c43 = peg$classExpectation(["'"], true, false), + peg$c44 = /^[a-zA-Z_]/, + peg$c45 = peg$classExpectation([["a", "z"], ["A", "Z"], "_"], false, false), + peg$c46 = "=", + peg$c47 = peg$literalExpectation("=", false), + peg$c48 = function(name, value) { return { type: 'namedArgument', name: name.join(''), @@ -230,10 +234,10 @@ function peg$parse(input, options) { text: text() }; }, - peg$c45 = peg$otherExpectation("function"), - peg$c46 = /^[a-zA-Z_\-]/, - peg$c47 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), - peg$c48 = function(name, args) { + peg$c49 = peg$otherExpectation("function"), + peg$c50 = /^[a-zA-Z_\-]/, + peg$c51 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), + peg$c52 = function(name, args) { return { type: 'function', name: name.join(''), @@ -242,21 +246,21 @@ function peg$parse(input, options) { text: text() }; }, - peg$c49 = peg$otherExpectation("number"), - peg$c50 = function() { + peg$c53 = peg$otherExpectation("number"), + peg$c54 = function() { return parseFloat(text()); }, - peg$c51 = /^[eE]/, - peg$c52 = peg$classExpectation(["e", "E"], false, false), - peg$c53 = peg$otherExpectation("exponent"), - peg$c54 = ".", - peg$c55 = peg$literalExpectation(".", false), - peg$c56 = "0", - peg$c57 = peg$literalExpectation("0", false), - peg$c58 = /^[1-9]/, - peg$c59 = peg$classExpectation([["1", "9"]], false, false), - peg$c60 = /^[0-9]/, - peg$c61 = peg$classExpectation([["0", "9"]], false, false), + peg$c55 = /^[eE]/, + peg$c56 = peg$classExpectation(["e", "E"], false, false), + peg$c57 = peg$otherExpectation("exponent"), + peg$c58 = ".", + peg$c59 = peg$literalExpectation(".", false), + peg$c60 = "0", + peg$c61 = peg$literalExpectation("0", false), + peg$c62 = /^[1-9]/, + peg$c63 = peg$classExpectation([["1", "9"]], false, false), + peg$c64 = /^[0-9]/, + peg$c65 = peg$classExpectation([["0", "9"]], false, false), peg$currPos = 0, peg$savedPos = 0, @@ -533,27 +537,57 @@ function peg$parse(input, options) { s0 = peg$currPos; s1 = peg$parse_(); if (s1 !== peg$FAILED) { - s2 = peg$parseQuote(); + if (peg$c13.test(input.charAt(peg$currPos))) { + s2 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s2 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c14); } + } if (s2 !== peg$FAILED) { s3 = []; s4 = peg$parseValidChar(); if (s4 === peg$FAILED) { s4 = peg$parseSpace(); + if (s4 === peg$FAILED) { + if (peg$c15.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c16); } + } + } } while (s4 !== peg$FAILED) { s3.push(s4); s4 = peg$parseValidChar(); if (s4 === peg$FAILED) { s4 = peg$parseSpace(); + if (s4 === peg$FAILED) { + if (peg$c15.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c16); } + } + } } } if (s3 !== peg$FAILED) { - s4 = peg$parseQuote(); + if (peg$c13.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c14); } + } if (s4 !== peg$FAILED) { s5 = peg$parse_(); if (s5 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c13(s3); + s1 = peg$c17(s3); s0 = s1; } else { peg$currPos = s0; @@ -579,22 +613,66 @@ function peg$parse(input, options) { s0 = peg$currPos; s1 = peg$parse_(); if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$parseValidChar(); - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseValidChar(); - } + if (peg$c15.test(input.charAt(peg$currPos))) { + s2 = input.charAt(peg$currPos); + peg$currPos++; } else { s2 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c16); } } if (s2 !== peg$FAILED) { - s3 = peg$parse_(); + s3 = []; + s4 = peg$parseValidChar(); + if (s4 === peg$FAILED) { + s4 = peg$parseSpace(); + if (s4 === peg$FAILED) { + if (peg$c13.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c14); } + } + } + } + while (s4 !== peg$FAILED) { + s3.push(s4); + s4 = peg$parseValidChar(); + if (s4 === peg$FAILED) { + s4 = peg$parseSpace(); + if (s4 === peg$FAILED) { + if (peg$c13.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c14); } + } + } + } + } if (s3 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c14(s2); - s0 = s1; + if (peg$c15.test(input.charAt(peg$currPos))) { + s4 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c16); } + } + if (s4 !== peg$FAILED) { + s5 = peg$parse_(); + if (s5 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c17(s3); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } } else { peg$currPos = s0; s0 = peg$FAILED; @@ -607,6 +685,39 @@ function peg$parse(input, options) { peg$currPos = s0; s0 = peg$FAILED; } + if (s0 === peg$FAILED) { + s0 = peg$currPos; + s1 = peg$parse_(); + if (s1 !== peg$FAILED) { + s2 = []; + s3 = peg$parseValidChar(); + if (s3 !== peg$FAILED) { + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$parseValidChar(); + } + } else { + s2 = peg$FAILED; + } + if (s2 !== peg$FAILED) { + s3 = peg$parse_(); + if (s3 !== peg$FAILED) { + peg$savedPos = s0; + s1 = peg$c18(s2); + s0 = s1; + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } else { + peg$currPos = s0; + s0 = peg$FAILED; + } + } } return s0; @@ -623,19 +734,19 @@ function peg$parse(input, options) { s3 = []; s4 = peg$currPos; if (input.charCodeAt(peg$currPos) === 43) { - s5 = peg$c15; + s5 = peg$c19; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } + if (peg$silentFails === 0) { peg$fail(peg$c20); } } if (s5 === peg$FAILED) { if (input.charCodeAt(peg$currPos) === 45) { - s5 = peg$c17; + s5 = peg$c21; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c18); } + if (peg$silentFails === 0) { peg$fail(peg$c22); } } } if (s5 !== peg$FAILED) { @@ -655,19 +766,19 @@ function peg$parse(input, options) { s3.push(s4); s4 = peg$currPos; if (input.charCodeAt(peg$currPos) === 43) { - s5 = peg$c15; + s5 = peg$c19; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } + if (peg$silentFails === 0) { peg$fail(peg$c20); } } if (s5 === peg$FAILED) { if (input.charCodeAt(peg$currPos) === 45) { - s5 = peg$c17; + s5 = peg$c21; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c18); } + if (peg$silentFails === 0) { peg$fail(peg$c22); } } } if (s5 !== peg$FAILED) { @@ -688,7 +799,7 @@ function peg$parse(input, options) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c19(s2, s3); + s1 = peg$c23(s2, s3); s0 = s1; } else { peg$currPos = s0; @@ -721,19 +832,19 @@ function peg$parse(input, options) { s3 = []; s4 = peg$currPos; if (input.charCodeAt(peg$currPos) === 42) { - s5 = peg$c20; + s5 = peg$c24; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c21); } + if (peg$silentFails === 0) { peg$fail(peg$c25); } } if (s5 === peg$FAILED) { if (input.charCodeAt(peg$currPos) === 47) { - s5 = peg$c22; + s5 = peg$c26; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c23); } + if (peg$silentFails === 0) { peg$fail(peg$c27); } } } if (s5 !== peg$FAILED) { @@ -753,19 +864,19 @@ function peg$parse(input, options) { s3.push(s4); s4 = peg$currPos; if (input.charCodeAt(peg$currPos) === 42) { - s5 = peg$c20; + s5 = peg$c24; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c21); } + if (peg$silentFails === 0) { peg$fail(peg$c25); } } if (s5 === peg$FAILED) { if (input.charCodeAt(peg$currPos) === 47) { - s5 = peg$c22; + s5 = peg$c26; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c23); } + if (peg$silentFails === 0) { peg$fail(peg$c27); } } } if (s5 !== peg$FAILED) { @@ -786,7 +897,7 @@ function peg$parse(input, options) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c24(s2, s3); + s1 = peg$c28(s2, s3); s0 = s1; } else { peg$currPos = s0; @@ -829,11 +940,11 @@ function peg$parse(input, options) { s1 = peg$parse_(); if (s1 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 40) { - s2 = peg$c25; + s2 = peg$c29; peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c26); } + if (peg$silentFails === 0) { peg$fail(peg$c30); } } if (s2 !== peg$FAILED) { s3 = peg$parse_(); @@ -843,17 +954,17 @@ function peg$parse(input, options) { s5 = peg$parse_(); if (s5 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 41) { - s6 = peg$c27; + s6 = peg$c31; peg$currPos++; } else { s6 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c28); } + if (peg$silentFails === 0) { peg$fail(peg$c32); } } if (s6 !== peg$FAILED) { s7 = peg$parse_(); if (s7 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c29(s4); + s1 = peg$c33(s4); s0 = s1; } else { peg$currPos = s0; @@ -899,11 +1010,11 @@ function peg$parse(input, options) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 44) { - s5 = peg$c31; + s5 = peg$c35; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } + if (peg$silentFails === 0) { peg$fail(peg$c36); } } if (s5 !== peg$FAILED) { s6 = peg$parse_(); @@ -911,7 +1022,7 @@ function peg$parse(input, options) { s7 = peg$parseArgument(); if (s7 !== peg$FAILED) { peg$savedPos = s3; - s4 = peg$c33(s1, s7); + s4 = peg$c37(s1, s7); s3 = s4; } else { peg$currPos = s3; @@ -935,11 +1046,11 @@ function peg$parse(input, options) { s4 = peg$parse_(); if (s4 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 44) { - s5 = peg$c31; + s5 = peg$c35; peg$currPos++; } else { s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } + if (peg$silentFails === 0) { peg$fail(peg$c36); } } if (s5 !== peg$FAILED) { s6 = peg$parse_(); @@ -947,7 +1058,7 @@ function peg$parse(input, options) { s7 = peg$parseArgument(); if (s7 !== peg$FAILED) { peg$savedPos = s3; - s4 = peg$c33(s1, s7); + s4 = peg$c37(s1, s7); s3 = s4; } else { peg$currPos = s3; @@ -970,18 +1081,18 @@ function peg$parse(input, options) { s3 = peg$parse_(); if (s3 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 44) { - s4 = peg$c31; + s4 = peg$c35; peg$currPos++; } else { s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } + if (peg$silentFails === 0) { peg$fail(peg$c36); } } if (s4 === peg$FAILED) { s4 = null; } if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c34(s1, s2); + s1 = peg$c38(s1, s2); s0 = s1; } else { peg$currPos = s0; @@ -1002,7 +1113,21 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c30); } + if (peg$silentFails === 0) { peg$fail(peg$c34); } + } + + return s0; + } + + function peg$parseStringChar() { + var s0; + + if (peg$c9.test(input.charAt(peg$currPos))) { + s0 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s0 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c10); } } return s0; @@ -1012,35 +1137,43 @@ function peg$parse(input, options) { var s0, s1, s2, s3; s0 = peg$currPos; - if (peg$c35.test(input.charAt(peg$currPos))) { + if (peg$c15.test(input.charAt(peg$currPos))) { s1 = input.charAt(peg$currPos); peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c36); } + if (peg$silentFails === 0) { peg$fail(peg$c16); } } if (s1 !== peg$FAILED) { s2 = []; - s3 = peg$parseValidChar(); - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseValidChar(); - } + if (peg$c39.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; } else { - s2 = peg$FAILED; + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c40); } + } + while (s3 !== peg$FAILED) { + s2.push(s3); + if (peg$c39.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c40); } + } } if (s2 !== peg$FAILED) { - if (peg$c35.test(input.charAt(peg$currPos))) { + if (peg$c15.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c36); } + if (peg$silentFails === 0) { peg$fail(peg$c16); } } if (s3 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c37(s2); + s1 = peg$c41(s2); s0 = s1; } else { peg$currPos = s0; @@ -1056,35 +1189,43 @@ function peg$parse(input, options) { } if (s0 === peg$FAILED) { s0 = peg$currPos; - if (peg$c38.test(input.charAt(peg$currPos))) { + if (peg$c13.test(input.charAt(peg$currPos))) { s1 = input.charAt(peg$currPos); peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c39); } + if (peg$silentFails === 0) { peg$fail(peg$c14); } } if (s1 !== peg$FAILED) { s2 = []; - s3 = peg$parseValidChar(); - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseValidChar(); - } + if (peg$c42.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; } else { - s2 = peg$FAILED; + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c43); } + } + while (s3 !== peg$FAILED) { + s2.push(s3); + if (peg$c42.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c43); } + } } if (s2 !== peg$FAILED) { - if (peg$c38.test(input.charAt(peg$currPos))) { + if (peg$c13.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c39); } + if (peg$silentFails === 0) { peg$fail(peg$c14); } } if (s3 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c37(s2); + s1 = peg$c41(s2); s0 = s1; } else { peg$currPos = s0; @@ -1112,7 +1253,7 @@ function peg$parse(input, options) { } if (s1 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c37(s1); + s1 = peg$c41(s1); } s0 = s1; } @@ -1126,22 +1267,22 @@ function peg$parse(input, options) { s0 = peg$currPos; s1 = []; - if (peg$c40.test(input.charAt(peg$currPos))) { + if (peg$c44.test(input.charAt(peg$currPos))) { s2 = input.charAt(peg$currPos); peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c41); } + if (peg$silentFails === 0) { peg$fail(peg$c45); } } if (s2 !== peg$FAILED) { while (s2 !== peg$FAILED) { s1.push(s2); - if (peg$c40.test(input.charAt(peg$currPos))) { + if (peg$c44.test(input.charAt(peg$currPos))) { s2 = input.charAt(peg$currPos); peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c41); } + if (peg$silentFails === 0) { peg$fail(peg$c45); } } } } else { @@ -1151,11 +1292,11 @@ function peg$parse(input, options) { s2 = peg$parse_(); if (s2 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 61) { - s3 = peg$c42; + s3 = peg$c46; peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c43); } + if (peg$silentFails === 0) { peg$fail(peg$c47); } } if (s3 !== peg$FAILED) { s4 = peg$parse_(); @@ -1168,7 +1309,7 @@ function peg$parse(input, options) { s6 = peg$parse_(); if (s6 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c44(s1, s5); + s1 = peg$c48(s1, s5); s0 = s1; } else { peg$currPos = s0; @@ -1209,22 +1350,22 @@ function peg$parse(input, options) { s1 = peg$parse_(); if (s1 !== peg$FAILED) { s2 = []; - if (peg$c46.test(input.charAt(peg$currPos))) { + if (peg$c50.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c47); } + if (peg$silentFails === 0) { peg$fail(peg$c51); } } if (s3 !== peg$FAILED) { while (s3 !== peg$FAILED) { s2.push(s3); - if (peg$c46.test(input.charAt(peg$currPos))) { + if (peg$c50.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c47); } + if (peg$silentFails === 0) { peg$fail(peg$c51); } } } } else { @@ -1232,11 +1373,11 @@ function peg$parse(input, options) { } if (s2 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 40) { - s3 = peg$c25; + s3 = peg$c29; peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c26); } + if (peg$silentFails === 0) { peg$fail(peg$c30); } } if (s3 !== peg$FAILED) { s4 = peg$parse_(); @@ -1249,17 +1390,17 @@ function peg$parse(input, options) { s6 = peg$parse_(); if (s6 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 41) { - s7 = peg$c27; + s7 = peg$c31; peg$currPos++; } else { s7 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c28); } + if (peg$silentFails === 0) { peg$fail(peg$c32); } } if (s7 !== peg$FAILED) { s8 = peg$parse_(); if (s8 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c48(s2, s5); + s1 = peg$c52(s2, s5); s0 = s1; } else { peg$currPos = s0; @@ -1296,7 +1437,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } + if (peg$silentFails === 0) { peg$fail(peg$c49); } } return s0; @@ -1308,11 +1449,11 @@ function peg$parse(input, options) { peg$silentFails++; s0 = peg$currPos; if (input.charCodeAt(peg$currPos) === 45) { - s1 = peg$c17; + s1 = peg$c21; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c18); } + if (peg$silentFails === 0) { peg$fail(peg$c22); } } if (s1 === peg$FAILED) { s1 = null; @@ -1331,7 +1472,7 @@ function peg$parse(input, options) { } if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c50(); + s1 = peg$c54(); s0 = s1; } else { peg$currPos = s0; @@ -1352,7 +1493,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c49); } + if (peg$silentFails === 0) { peg$fail(peg$c53); } } return s0; @@ -1361,12 +1502,12 @@ function peg$parse(input, options) { function peg$parseE() { var s0; - if (peg$c51.test(input.charAt(peg$currPos))) { + if (peg$c55.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c52); } + if (peg$silentFails === 0) { peg$fail(peg$c56); } } return s0; @@ -1380,11 +1521,11 @@ function peg$parse(input, options) { s1 = peg$parseE(); if (s1 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 45) { - s2 = peg$c17; + s2 = peg$c21; peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c18); } + if (peg$silentFails === 0) { peg$fail(peg$c22); } } if (s2 === peg$FAILED) { s2 = null; @@ -1418,7 +1559,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c53); } + if (peg$silentFails === 0) { peg$fail(peg$c57); } } return s0; @@ -1429,11 +1570,11 @@ function peg$parse(input, options) { s0 = peg$currPos; if (input.charCodeAt(peg$currPos) === 46) { - s1 = peg$c54; + s1 = peg$c58; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c55); } + if (peg$silentFails === 0) { peg$fail(peg$c59); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1465,20 +1606,20 @@ function peg$parse(input, options) { var s0, s1, s2, s3; if (input.charCodeAt(peg$currPos) === 48) { - s0 = peg$c56; + s0 = peg$c60; peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c57); } + if (peg$silentFails === 0) { peg$fail(peg$c61); } } if (s0 === peg$FAILED) { s0 = peg$currPos; - if (peg$c58.test(input.charAt(peg$currPos))) { + if (peg$c62.test(input.charAt(peg$currPos))) { s1 = input.charAt(peg$currPos); peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c59); } + if (peg$silentFails === 0) { peg$fail(peg$c63); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1506,12 +1647,12 @@ function peg$parse(input, options) { function peg$parseDigit() { var s0; - if (peg$c60.test(input.charAt(peg$currPos))) { + if (peg$c64.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c61); } + if (peg$silentFails === 0) { peg$fail(peg$c65); } } return s0; diff --git a/packages/kbn-tinymath/src/grammar.pegjs b/packages/kbn-tinymath/src/grammar.pegjs index 9cb92fa9374a2b..2334049a74ef9d 100644 --- a/packages/kbn-tinymath/src/grammar.pegjs +++ b/packages/kbn-tinymath/src/grammar.pegjs @@ -43,7 +43,15 @@ Literal "literal" // Quoted variables are interpreted as strings // but unquoted variables are more restrictive Variable - = _ Quote chars:(ValidChar / Space)* Quote _ { + = _ [\'] chars:(ValidChar / Space / [\"])* [\'] _ { + return { + type: 'variable', + value: chars.join(''), + location: simpleLocation(location()), + text: text() + }; + } + / _ [\"] chars:(ValidChar / Space / [\'])* [\"] _ { return { type: 'variable', value: chars.join(''), @@ -102,12 +110,14 @@ Argument_List "arguments" return [first].concat(rest); } +StringChar + = [0-9A-Za-z._@\[\]-] + String - = [\"] value:(ValidChar)+ [\"] { return value.join(''); } - / [\'] value:(ValidChar)+ [\'] { return value.join(''); } + = [\"] value:([^"]*) [\"] { return value.join(''); } + / [\'] value:([^']*) [\'] { return value.join(''); } / value:(ValidChar)+ { return value.join(''); } - Argument = name:[a-zA-Z_]+ _ '=' _ value:(Number / String) _ { return { diff --git a/packages/kbn-tinymath/test/library.test.js b/packages/kbn-tinymath/test/library.test.js index d11822625b98f5..1593c5e1dcb681 100644 --- a/packages/kbn-tinymath/test/library.test.js +++ b/packages/kbn-tinymath/test/library.test.js @@ -73,6 +73,7 @@ describe('Parser', () => { expect(parse('"foo bar"')).toEqual(variableEqual('foo bar')); expect(parse('"foo bar fizz buzz"')).toEqual(variableEqual('foo bar fizz buzz')); expect(parse('"foo bar baby"')).toEqual(variableEqual('foo bar baby')); + expect(parse(`"f'oo"`)).toEqual(variableEqual(`f'oo`)); }); it('strings with single quotes', () => { @@ -88,6 +89,7 @@ describe('Parser', () => { expect(parse("' foo bar'")).toEqual(variableEqual(" foo bar")); expect(parse("'foo bar '")).toEqual(variableEqual("foo bar ")); expect(parse("'0foo'")).toEqual(variableEqual("0foo")); + expect(parse(`'f"oo'`)).toEqual(variableEqual(`f"oo`)); /* eslint-enable prettier/prettier */ }); @@ -138,10 +140,18 @@ describe('Parser', () => { ); }); + it('named argument is empty strin', () => { + expect(parse('foo(q="")')).toEqual(functionEqual('foo', [namedArgumentEqual('q', '')])); + expect(parse(`foo(q='')`)).toEqual(functionEqual('foo', [namedArgumentEqual('q', '')])); + }); + it('named and positional', () => { expect(parse('foo(ref, q="bar")')).toEqual( functionEqual('foo', [variableEqual('ref'), namedArgumentEqual('q', 'bar')]) ); + expect(parse(`foo(ref, q='ba"r')`)).toEqual( + functionEqual('foo', [variableEqual('ref'), namedArgumentEqual('q', `ba"r`)]) + ); }); it('numerically named', () => { @@ -182,6 +192,12 @@ describe('Parser', () => { it('invalid named', () => { expect(() => parse('foo(offset-type="1d")')).toThrow('but "(" found'); }); + + it('named with complex strings', () => { + expect(parse(`foo(filter='😀 > "\ttab"')`)).toEqual( + functionEqual('foo', [namedArgumentEqual('filter', `😀 > "\ttab"`)]) + ); + }); }); it('Missing expression', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 7e5eb59de281a1..10d76942274121 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -71,9 +71,17 @@ export const counterRateOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate'); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { const metric = layer.columns[referenceIds[0]]; const timeScale = previousColumn?.timeScale || DEFAULT_TIME_SCALE; + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName(metric && 'sourceField' in metric ? metric.sourceField : undefined, timeScale), dataType: 'number', @@ -82,7 +90,7 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 222127317dfa62..86113a4ca0110f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -69,7 +69,15 @@ export const cumulativeSumOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum'); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } const ref = layer.columns[referenceIds[0]]; return { label: ofName(ref && 'sourceField' in ref ? ref.sourceField : undefined), @@ -77,7 +85,7 @@ export const cumulativeSumOperation: OperationDefinition< operationType: 'cumulative_sum', isBucketed: false, scale: 'ratio', - filter: previousColumn?.filter, + filter, references: referenceIds, params: getFormatFromPreviousColumn(previousColumn), }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index c1c1b719fc557a..31e4ac3c5bb2c0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -72,7 +72,15 @@ export const derivativeOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'derivative'); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } const ref = layer.columns[referenceIds[0]]; return { label: ofName( @@ -85,7 +93,7 @@ export const derivativeOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index afc101b3c48177..88af8e9b6378e0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -88,6 +88,14 @@ export const movingAverageOperation: OperationDefinition< ) => { const metric = layer.columns[referenceIds[0]]; const { window = WINDOW_DEFAULT_VALUE } = columnParams; + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName(metric?.label, previousColumn?.timeScale), dataType: 'number', @@ -96,7 +104,7 @@ export const movingAverageOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter: previousColumn?.filter, + filter, params: { window, ...getFormatFromPreviousColumn(previousColumn), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index fa1691ba9a78e5..df84ecb479de72 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -71,8 +71,21 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), - buildColumn({ field, previousColumn }) { + buildColumn({ field, previousColumn }, columnParams) { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName(field.displayName), dataType: 'number', @@ -80,7 +93,7 @@ export const cardinalityOperation: OperationDefinition adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale), - buildColumn({ field, previousColumn }) { + buildColumn({ field, previousColumn }, columnParams) { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale), dataType: 'number', @@ -61,7 +69,7 @@ export const countOperation: OperationDefinition { }); const operationDefinitionMap: Record = { - avg: ({ + average: ({ input: 'field', buildColumn: ({ field }: { field: IndexPatternField }) => ({ label: 'avg', dataType: 'number', - operationType: 'avg', + operationType: 'average', sourceField: field.name, isBucketed: false, scale: 'ratio', @@ -35,7 +35,19 @@ const operationDefinitionMap: Record = { sum: { input: 'field' } as GenericOperationDefinition, last_value: { input: 'field' } as GenericOperationDefinition, max: { input: 'field' } as GenericOperationDefinition, - count: { input: 'field' } as GenericOperationDefinition, + count: ({ + input: 'field', + operationParams: [{ name: 'kql', type: 'string', required: false }], + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'count', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, derivative: { input: 'fullReference' } as GenericOperationDefinition, moving_average: { input: 'fullReference', @@ -184,7 +196,7 @@ describe('formula', () => { dataType: 'number', isBucketed: false, label: 'col1X0', - operationType: 'avg', + operationType: 'average', scale: 'ratio', sourceField: 'bytes', timeScale: 'd', @@ -204,7 +216,7 @@ describe('formula', () => { scale: 'ratio', params: { isFormulaBroken: false, - formula: 'moving_average(avg(bytes), window=3)', + formula: 'moving_average(average(bytes), window=3)', }, references: [], }); @@ -257,7 +269,7 @@ describe('formula', () => { it('should mutate the layer with new columns for valid formula expressions', () => { expect( regenerateLayerFromAst( - 'avg(bytes)', + 'average(bytes)', layer, 'col1', currentColumn, @@ -274,7 +286,7 @@ describe('formula', () => { references: ['col1X1'], params: { ...currentColumn.params, - formula: 'avg(bytes)', + formula: 'average(bytes)', isFormulaBroken: false, }, }, @@ -283,7 +295,7 @@ describe('formula', () => { dataType: 'number', isBucketed: false, label: 'col1X0', - operationType: 'avg', + operationType: 'average', scale: 'ratio', sourceField: 'bytes', timeScale: false, @@ -307,12 +319,12 @@ describe('formula', () => { it('returns no change but error if the formula cannot be parsed', () => { const formulas = [ '+', - 'avg((', - 'avg((bytes)', - 'avg(bytes) +', - 'avg(""', - 'moving_average(avg(bytes), window=)', - 'avg(bytes) + moving_average(avg(bytes), window=)', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', + 'average(bytes) + moving_average(average(bytes), window=)', ]; for (const formula of formulas) { testIsBrokenFormula(formula); @@ -326,10 +338,10 @@ describe('formula', () => { it('returns no change but error if at least one field in the formula is missing', () => { const formulas = [ 'noField', - 'avg(noField)', + 'average(noField)', 'noField + 1', - 'derivative(avg(noField))', - 'avg(bytes) + derivative(avg(noField))', + 'derivative(average(noField))', + 'average(bytes) + derivative(average(noField))', ]; for (const formula of formulas) { @@ -341,13 +353,13 @@ describe('formula', () => { const formulas = [ 'noFn()', 'noFn(bytes)', - 'avg(bytes) + noFn()', + 'average(bytes) + noFn()', 'derivative(noFn())', 'noFn() + noFnTwo()', 'noFn(noFnTwo())', 'noFn() + noFnTwo() + 5', - 'avg(bytes) + derivative(noFn())', - 'derivative(avg(bytes) + noFn())', + 'average(bytes) + derivative(noFn())', + 'derivative(average(bytes) + noFn())', ]; for (const formula of formulas) { @@ -357,17 +369,17 @@ describe('formula', () => { it('returns no change but error if one operation has the wrong first argument', () => { const formulas = [ - 'avg(7)', - 'avg()', - 'avg(avg(bytes))', - 'avg(1 + 2)', - 'avg(bytes + 5)', - 'avg(bytes + bytes)', + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', 'derivative(7)', 'derivative(bytes + 7)', 'derivative(bytes + bytes)', - 'derivative(bytes + avg(bytes))', - 'derivative(bytes + 7 + avg(bytes))', + 'derivative(bytes + average(bytes))', + 'derivative(bytes + 7 + average(bytes))', ]; for (const formula of formulas) { @@ -375,7 +387,7 @@ describe('formula', () => { } }); - it('returns no change by error if an argument is passed to count operation', () => { + it('returns no change but error if an argument is passed to count operation', () => { const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; for (const formula of formulas) { @@ -384,17 +396,17 @@ describe('formula', () => { }); it('returns no change but error if a required parameter is not passed to the operation in formula', () => { - const formula = 'moving_average(avg(bytes))'; + const formula = 'moving_average(average(bytes))'; testIsBrokenFormula(formula); }); it('returns no change but error if a required parameter passed with the wrong type in formula', () => { - const formula = 'moving_average(avg(bytes), window="m")'; + const formula = 'moving_average(average(bytes), window="m")'; testIsBrokenFormula(formula); }); it('returns error if a required parameter is passed multiple time', () => { - const formula = 'moving_average(avg(bytes), window=7, window=3)'; + const formula = 'moving_average(average(bytes), window=7, window=3)'; testIsBrokenFormula(formula); }); @@ -444,10 +456,21 @@ describe('formula', () => { ).toEqual(undefined); }); + it('returns undefined if count is passed with only a named argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='*')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + it('returns undefined if a field operation is passed with the correct first argument', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('avg(bytes)'), + getNewLayerWithFormula('average(bytes)'), 'col1', indexPattern, operationDefinitionMap @@ -456,7 +479,7 @@ describe('formula', () => { // note that field names can be wrapped in quotes as well expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('avg("bytes")'), + getNewLayerWithFormula('average("bytes")'), 'col1', indexPattern, operationDefinitionMap @@ -467,7 +490,7 @@ describe('formula', () => { it('returns undefined if a fullReference operation is passed with the correct first argument', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('derivative(avg(bytes))'), + getNewLayerWithFormula('derivative(average(bytes))'), 'col1', indexPattern, operationDefinitionMap @@ -476,7 +499,7 @@ describe('formula', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('derivative(avg("bytes"))'), + getNewLayerWithFormula('derivative(average("bytes"))'), 'col1', indexPattern, operationDefinitionMap @@ -487,7 +510,7 @@ describe('formula', () => { it('returns undefined if a fullReference operation is passed with the arguments', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('moving_average(avg(bytes), window=7)'), + getNewLayerWithFormula('moving_average(average(bytes), window=7)'), 'col1', indexPattern, operationDefinitionMap @@ -497,7 +520,7 @@ describe('formula', () => { // Not sure it will be supported // expect( // formulaOperation.getErrorMessage!( - // getNewLayerWithFormula('moving_average(avg("bytes"), "window"=7)'), + // getNewLayerWithFormula('moving_average(average("bytes"), "window"=7)'), // 'col1', // indexPattern, // operationDefinitionMap @@ -528,11 +551,11 @@ describe('formula', () => { it('returns an error if parsing a syntax invalid formula', () => { const formulas = [ '+', - 'avg((', - 'avg((bytes)', - 'avg(bytes) +', - 'avg(""', - 'moving_average(avg(bytes), window=)', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', ]; for (const formula of formulas) { @@ -548,7 +571,12 @@ describe('formula', () => { }); it('returns an error if the field is missing', () => { - const formulas = ['noField', 'avg(noField)', 'noField + 1', 'derivative(avg(noField))']; + const formulas = [ + 'noField', + 'average(noField)', + 'noField + 1', + 'derivative(average(noField))', + ]; for (const formula of formulas) { expect( @@ -578,7 +606,7 @@ describe('formula', () => { }); it('returns an error if an operation is unknown', () => { - const formulas = ['noFn()', 'noFn(bytes)', 'avg(bytes) + noFn()', 'derivative(noFn())']; + const formulas = ['noFn()', 'noFn(bytes)', 'average(bytes) + noFn()', 'derivative(noFn())']; for (const formula of formulas) { expect( @@ -607,12 +635,12 @@ describe('formula', () => { it('returns an error if field operation in formula have the wrong first argument', () => { const formulas = [ - 'avg(7)', - 'avg()', - 'avg(avg(bytes))', - 'avg(1 + 2)', - 'avg(bytes + 5)', - 'avg(bytes + bytes)', + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', 'derivative(7)', ]; @@ -653,7 +681,7 @@ describe('formula', () => { it('returns an error if an operation with required parameters does not receive them', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('moving_average(avg(bytes))'), + getNewLayerWithFormula('moving_average(average(bytes))'), 'col1', indexPattern, operationDefinitionMap @@ -664,7 +692,7 @@ describe('formula', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('moving_average(avg(bytes), myparam=7)'), + getNewLayerWithFormula('moving_average(average(bytes), myparam=7)'), 'col1', indexPattern, operationDefinitionMap @@ -677,18 +705,18 @@ describe('formula', () => { it('returns an error if a parameter is passed to an operation with no parameters', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('avg(bytes, myparam=7)'), + getNewLayerWithFormula('average(bytes, myparam=7)'), 'col1', indexPattern, operationDefinitionMap ) - ).toEqual(['The operation avg does not accept any parameter']); + ).toEqual(['The operation average does not accept any parameter']); }); it('returns an error if the parameter passed to an operation is of the wrong type', () => { expect( formulaOperation.getErrorMessage!( - getNewLayerWithFormula('moving_average(avg(bytes), window="m")'), + getNewLayerWithFormula('moving_average(average(bytes), window="m")'), 'col1', indexPattern, operationDefinitionMap @@ -718,8 +746,8 @@ describe('formula', () => { it('returns no error if a math operation is passed to fullReference operations', () => { const formulas = [ 'derivative(7+1)', - 'derivative(7+avg(bytes))', - 'moving_average(7+avg(bytes), window=7)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', ]; for (const formula of formulas) { expect( @@ -736,8 +764,8 @@ describe('formula', () => { it('returns errors if math operations are used with no arguments', () => { const formulas = [ 'derivative(7+1)', - 'derivative(7+avg(bytes))', - 'moving_average(7+avg(bytes), window=7)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', ]; for (const formula of formulas) { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 43c45938ae265f..28902806e37ec3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -36,6 +36,11 @@ function inLocation(cursorPosition: number, location: TinymathLocation) { return cursorPosition >= location.min && cursorPosition < location.max; } +const filterableOperationParams = [ + { name: 'kql', type: 'string', required: false }, + { name: 'lucene', type: 'string', required: false }, +]; + const MARKER = 'LENS_MATH_MARKER'; function getInfoAtPosition( @@ -76,16 +81,16 @@ export function offsetToRowColumn(expression: string, offset: number): monaco.Po throw new Error('Algorithm failure'); } -export function monacoPositionToOffset(expression: string, position: monaco.Position): number { - const lines = expression.split(/\n/); - return lines - .slice(0, position.lineNumber - 1) - .reduce( - (prev, current, index) => - prev + index === position.lineNumber - 1 ? position.column - 1 : current.length, - 0 - ); -} +// export function monacoPositionToOffset(expression: string, position: monaco.Position): number { +// const lines = expression.split(/\n/); +// return lines +// .slice(0, position.lineNumber - 1) +// .reduce( +// (prev, current, index) => +// prev + index === position.lineNumber - 1 ? position.column - 1 : current.length, +// 0 +// ); +// } export async function suggest( expression: string, @@ -178,23 +183,30 @@ function getArgumentSuggestions( return { list: [], type: SUGGESTION_TYPE.FIELD }; } - if (position > 0) { + if (position > 0 || operation.type === 'count') { + const { namedArguments } = groupArgsByType(ast.args); + const list = []; + if (operation.filterable) { + if (!namedArguments.find((arg) => arg.name === 'kql')) { + list.push('kql'); + } + if (!namedArguments.find((arg) => arg.name === 'lucene')) { + list.push('lucene'); + } + } if ('operationParams' in operation) { // Exclude any previously used named args - const { namedArguments } = groupArgsByType(ast.args); - const suggestedParam = operation - .operationParams!.filter( - (param) => - // Keep the param if it's the first use - !namedArguments.find((arg) => arg.name === param.name) - ) - .map((p) => p.name); - return { - list: suggestedParam, - type: SUGGESTION_TYPE.NAMED_ARGUMENT, - }; + list.push( + ...operation + .operationParams!.filter( + (param) => + // Keep the param if it's the first use + !namedArguments.find((arg) => arg.name === param.name) + ) + .map((p) => p.name) + ); } - return { list: [], type: SUGGESTION_TYPE.FIELD }; + return { list, type: SUGGESTION_TYPE.NAMED_ARGUMENT }; } if (operation.input === 'field' && position === 0) { @@ -207,12 +219,12 @@ function getArgumentSuggestions( ({ operationMetaData }) => operationMetaData.dataType === 'number' && !operationMetaData.isBucketed ); - if (validOperation && operation.type !== 'count') { + if (validOperation) { const fields = validOperation.operations .filter((op) => op.operationType === operation.type) .map((op) => ('field' in op ? op.field : undefined)) .filter((field) => field); - return { list: fields, type: SUGGESTION_TYPE.FIELD }; + return { list: fields as string[], type: SUGGESTION_TYPE.FIELD }; } else { return { list: [], type: SUGGESTION_TYPE.FIELD }; } @@ -288,12 +300,14 @@ export function getSuggestion( } else { const def = operationDefinitionMap[suggestion.label]; kind = monaco.languages.CompletionItemKind.Constant; - if ('operationParams' in def) { + if (suggestion.label === 'count' && 'operationParams' in def) { + label = `${label}(${def + .operationParams!.map((p) => `${p.name}=${p.type}`) + .join(', ')})`; + } else if ('operationParams' in def) { label = `${label}(expression, ${def .operationParams!.map((p) => `${p.name}=${p.type}`) - .join(', ')}`; - } else if (suggestion.label === 'count') { - label = `${label}()`; + .join(', ')})`; } else { label = `${label}(expression)`; } @@ -309,6 +323,10 @@ export function getSuggestion( id: 'editor.action.triggerSuggest', }; kind = monaco.languages.CompletionItemKind.Keyword; + if (label === 'kql' || label === 'lucene') { + insertText = `${label}='$0'`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + } label = `${label}=`; detail = ''; break; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index f19f8933aaf2a9..66c8ace0dd1f52 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -339,7 +339,7 @@ function runFullASTValidation( } const nodeOperation = operations[node.name]; const errors: ErrorWrapper[] = []; - const { namedArguments, functions } = groupArgsByType(node.args); + const { namedArguments, functions, variables } = groupArgsByType(node.args); const [firstArg] = node?.args || []; if (!nodeOperation) { @@ -379,7 +379,8 @@ function runFullASTValidation( } } } else { - if (firstArg) { + // Named arguments only + if (functions?.length || variables?.length) { errors.push( getMessageFromId({ messageId: 'shouldNotHaveField', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index cfe757e8a4c0b9..c66761c46beea2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -247,6 +247,11 @@ interface BaseOperationDefinitionProps { * If set to optional, time scaling won't be enabled by default and can be removed. */ timeScalingMode?: TimeScalingMode; + /** + * Filterable operations can have a KQL or Lucene query added at the dimension level. + * This flag is used by the formula to assign the kql= and lucene= named arguments and set up + * autocomplete. + */ filterable?: boolean; getHelpMessage?: (props: HelpProps) => React.ReactNode; @@ -323,7 +328,7 @@ interface FieldBasedOperationDefinition { field: IndexPatternField; previousColumn?: IndexPatternColumn; }, - columnParams?: (IndexPatternColumn & C)['params'] + columnParams?: (IndexPatternColumn & C)['params'] & { kql?: string; lucene?: string } ) => C; /** * This method will be called if the user changes the field of an operation. @@ -417,7 +422,10 @@ interface FullReferenceOperationDefinition { referenceIds: string[]; previousColumn?: IndexPatternColumn; }, - columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] & { + kql?: string; + lucene?: string; + } ) => ReferenceBasedIndexPatternColumn & C; /** * Returns the meta data of the operation if applied. Undefined diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 4f5c897fb5378b..a61cca89dfecfc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -141,7 +141,7 @@ export const lastValueOperation: OperationDefinition f.type === 'date')?.name; @@ -154,6 +154,14 @@ export const lastValueOperation: OperationDefinition>({ : (layer.columns[thisColumnId] as T), getDefaultLabel: (column, indexPattern, columns) => labelLookup(getSafeName(column.sourceField, indexPattern), column), - buildColumn: ({ field, previousColumn }) => - ({ + buildColumn: ({ field, previousColumn }, columnParams) => { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } + return { label: labelLookup(field.displayName, previousColumn), dataType: 'number', operationType: type, @@ -100,7 +108,8 @@ function buildMetricOperation>({ timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, filter: previousColumn?.filter, params: getFormatFromPreviousColumn(previousColumn), - } as T), + } as T; + }, onFieldChange: (oldColumn, field) => { return { ...oldColumn, From bbf55a23917e727e82460ac0bbb1143593563616 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 31 Mar 2021 18:07:52 -0400 Subject: [PATCH 062/185] Add kql, lucene completion and validation --- .../definitions/formula/formula.test.tsx | 17 ++- .../definitions/formula/formula.tsx | 27 ++-- .../formula/math_completion.test.ts | 136 ++++++++++-------- .../definitions/formula/math_completion.ts | 104 ++++++++++++-- .../definitions/formula/validation.ts | 64 ++++++++- 5 files changed, 255 insertions(+), 93 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 65bcee8de15867..45521a723bfe1c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -37,7 +37,7 @@ const operationDefinitionMap: Record = { max: { input: 'field' } as GenericOperationDefinition, count: ({ input: 'field', - operationParams: [{ name: 'kql', type: 'string', required: false }], + filterable: true, buildColumn: ({ field }: { field: IndexPatternField }) => ({ label: 'avg', dataType: 'number', @@ -467,6 +467,21 @@ describe('formula', () => { ).toEqual(undefined); }); + it('returns a syntax error if the kql argument does not parse', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='invalid: "')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + `Expected "(", "{", value, whitespace but """ found. +invalid: " +---------^`, + ]); + }); + it('returns undefined if a field operation is passed with the correct first argument', () => { expect( formulaOperation.getErrorMessage!( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 522e23bc672d1a..d12fb149294a78 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -211,6 +211,7 @@ function FormulaEditor({ columnId, indexPattern, operationDefinitionMap, + data, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [isOpen, setIsOpen] = useState(false); @@ -337,13 +338,14 @@ function FormulaEditor({ // Retrieve suggestions for subexpressions // TODO: make this work for expressions nested more than one level deep - aSuggestions = await suggest( - innerText.substring(0, innerText.length - lengthAfterPosition) + ')', - innerText.length - lengthAfterPosition, + aSuggestions = await suggest({ + expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + position: innerText.length - lengthAfterPosition, context, indexPattern, - operationDefinitionMap - ); + operationDefinitionMap, + data, + }); } } else { const wordUntil = model.getWordUntilPosition(position); @@ -353,14 +355,15 @@ function FormulaEditor({ position.lineNumber, wordUntil.endColumn ); - aSuggestions = await suggest( - innerText, - innerText.length - lengthAfterPosition, + aSuggestions = await suggest({ + expression: innerText, + position: innerText.length - lengthAfterPosition, context, indexPattern, operationDefinitionMap, - wordUntil - ); + word: wordUntil, + data, + }); } return { @@ -369,7 +372,7 @@ function FormulaEditor({ ), }; }, - [indexPattern, operationDefinitionMap] + [indexPattern, operationDefinitionMap, data] ); const provideSignatureHelp = useCallback( @@ -443,7 +446,7 @@ function FormulaEditor({ useEffect(() => { // Because the monaco model is owned by Lens, we need to manually attach and remove handlers const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', ',', '(', '=', ' '], + triggerCharacters: ['.', ',', '(', '=', ' ', ':'], provideCompletionItems, }); const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts index 001bf369ccbe08..4919ed3e2901ea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts @@ -10,6 +10,7 @@ import { createMockedIndexPattern } from '../../../mocks'; import { GenericOperationDefinition } from '../index'; import type { IndexPatternField } from '../../../types'; import type { OperationMetadata } from '../../../../types'; +import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; import { tinymathFunctions } from './util'; import { getSignatureHelp, getHover, suggest } from './math_completion'; @@ -185,17 +186,18 @@ describe('math completion', () => { it('should list all valid functions at the top level (fake test)', async () => { // This test forces an invalid scenario, since the autocomplete actually requires // some typing - const results = await suggest( - '', - 1, - { + const results = await suggest({ + expression: '', + position: 1, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 1, endColumn: 1 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 1, endColumn: 1 }, + }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); @@ -206,17 +208,18 @@ describe('math completion', () => { }); it('should list all valid sub-functions for a fullReference', async () => { - const results = await suggest( - 'moving_average()', - 15, - { + const results = await suggest({ + expression: 'moving_average()', + position: 15, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 15, endColumn: 15 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 15, endColumn: 15 }, + }); expect(results.list).toHaveLength(2); ['sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); @@ -224,47 +227,50 @@ describe('math completion', () => { }); it('should list all valid named arguments for a fullReference', async () => { - const results = await suggest( - 'moving_average(count(),)', - 23, - { + const results = await suggest({ + expression: 'moving_average(count(),)', + position: 23, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 23, endColumn: 23 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 23, endColumn: 23 }, + }); expect(results.list).toEqual(['window']); }); it('should not list named arguments when they are already in use', async () => { - const results = await suggest( - 'moving_average(count(), window=5, )', - 34, - { + const results = await suggest({ + expression: 'moving_average(count(), window=5, )', + position: 34, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 34, endColumn: 34 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 34, endColumn: 34 }, + }); expect(results.list).toEqual([]); }); it('should list all valid positional arguments for a tinymath function used by name', async () => { - const results = await suggest( - 'divide(count(), )', - 16, - { + const results = await suggest({ + expression: 'divide(count(), )', + position: 16, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 16, endColumn: 16 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 16, endColumn: 16 }, + }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); @@ -275,17 +281,18 @@ describe('math completion', () => { }); it('should list all valid positional arguments for a tinymath function used with alias', async () => { - const results = await suggest( - 'count() / ', - 10, - { + const results = await suggest({ + expression: 'count() / ', + position: 10, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 10, endColumn: 10 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 10, endColumn: 10 }, + }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); @@ -296,47 +303,50 @@ describe('math completion', () => { }); it('should not autocomplete any fields for the count function', async () => { - const results = await suggest( - 'count()', - 6, - { + const results = await suggest({ + expression: 'count()', + position: 6, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 6, endColumn: 6 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 6, endColumn: 6 }, + }); expect(results.list).toHaveLength(0); }); it('should autocomplete and validate the right type of field', async () => { - const results = await suggest( - 'sum()', - 4, - { + const results = await suggest({ + expression: 'sum()', + position: 4, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 4, endColumn: 4 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 4, endColumn: 4 }, + }); expect(results.list).toEqual(['bytes', 'memory']); }); it('should autocomplete only operations that provide numeric output', async () => { - const results = await suggest( - 'last_value()', - 11, - { + const results = await suggest({ + expression: 'last_value()', + position: 11, + context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', }, - createMockedIndexPattern(), + indexPattern: createMockedIndexPattern(), operationDefinitionMap, - { word: '', startColumn: 11, endColumn: 11 } - ); + data: dataPluginMock.createStartContract(), + word: { word: '', startColumn: 11, endColumn: 11 }, + }); expect(results.list).toEqual(['bytes', 'memory']); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 28902806e37ec3..b3408399e48e85 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -7,8 +7,14 @@ import { uniq, startsWith } from 'lodash'; import { monaco } from '@kbn/monaco'; - -import { parse, TinymathLocation, TinymathAST, TinymathFunction } from '@kbn/tinymath'; +import { + parse, + TinymathLocation, + TinymathAST, + TinymathFunction, + TinymathNamedArgument, +} from '@kbn/tinymath'; +import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; import { IndexPattern } from '../../../types'; import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; import { tinymathFunctions, groupArgsByType } from './util'; @@ -18,6 +24,7 @@ export enum SUGGESTION_TYPE { FIELD = 'field', NAMED_ARGUMENT = 'named_argument', FUNCTIONS = 'functions', + KQL = 'kql', } export type LensMathSuggestion = @@ -25,7 +32,8 @@ export type LensMathSuggestion = | { label: string; type: 'operation' | 'math'; - }; + } + | QuerySuggestion; export interface LensMathSuggestions { list: LensMathSuggestion[]; @@ -92,22 +100,43 @@ export function offsetToRowColumn(expression: string, offset: number): monaco.Po // ); // } -export async function suggest( - expression: string, - position: number, - context: monaco.languages.CompletionContext, - indexPattern: IndexPattern, - operationDefinitionMap: Record, - word?: monaco.editor.IWordAtPosition -): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { +export async function suggest({ + expression, + position, + context, + indexPattern, + operationDefinitionMap, + data, + word, +}: { + expression: string; + position: number; + context: monaco.languages.CompletionContext; + indexPattern: IndexPattern; + operationDefinitionMap: Record; + data: DataPublicPluginStart; + word?: monaco.editor.IWordAtPosition; +}): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { const ast = parse(text); const tokenInfo = getInfoAtPosition(ast, position); - if (context.triggerCharacter === '=' && tokenInfo?.parent) { - // TODO + const isNamedArgument = + tokenInfo?.parent && + typeof tokenInfo?.ast !== 'number' && + 'type' in tokenInfo.ast && + tokenInfo.ast.type === 'namedArgument'; + if (tokenInfo?.parent && (context.triggerCharacter === '=' || isNamedArgument)) { + return await getNamedArgumentSuggestions({ + expression: text, + ast: tokenInfo.ast as TinymathNamedArgument, + position, + data, + indexPattern, + operationDefinitionMap, + }); } else if (tokenInfo?.parent) { return getArgumentSuggestions( tokenInfo.parent, @@ -260,6 +289,41 @@ function getArgumentSuggestions( return { list: [], type: SUGGESTION_TYPE.FIELD }; } +export async function getNamedArgumentSuggestions({ + ast, + expression, + position, + data, + indexPattern, + operationDefinitionMap, +}: { + ast: TinymathNamedArgument; + expression: string; + position: number; + indexPattern: IndexPattern; + operationDefinitionMap: Record; + data: DataPublicPluginStart; +}) { + if (ast.name !== 'kql' && ast.name !== 'lucene') { + return { list: [], type: SUGGESTION_TYPE.KQL }; + } + if (!data.autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { + return { list: [], type: SUGGESTION_TYPE.KQL }; + } + + const before = ast.value.split(MARKER)[0]; + // TODO + const suggestions = await data.autocomplete.getQuerySuggestions({ + language: 'kuery', + query: ast.value.split(MARKER)[0], + selectionStart: before.length, + selectionEnd: before.length, + indexPatterns: [indexPattern], + boolFilter: [], + }); + return { list: suggestions ?? [], type: SUGGESTION_TYPE.KQL }; +} + export function getSuggestion( suggestion: LensMathSuggestion, type: SUGGESTION_TYPE, @@ -267,7 +331,12 @@ export function getSuggestion( operationDefinitionMap: Record ): monaco.languages.CompletionItem { let kind: monaco.languages.CompletionItemKind = monaco.languages.CompletionItemKind.Method; - let label: string = typeof suggestion === 'string' ? suggestion : suggestion.label; + let label: string = + typeof suggestion === 'string' + ? suggestion + : 'label' in suggestion + ? suggestion.label + : suggestion.text; let insertText: string | undefined; let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; let detail: string = ''; @@ -330,6 +399,13 @@ export function getSuggestion( label = `${label}=`; detail = ''; break; + case SUGGESTION_TYPE.NAMED_ARGUMENT: + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + kind = monaco.languages.CompletionItemKind.Field; + break; } return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 66c8ace0dd1f52..f5a6b19d2d3489 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -9,6 +9,12 @@ import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse, TinymathLocation } from '@kbn/tinymath'; import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; +import { + AggFunctionsMapping, + Query, + esKuery, + esQuery, +} from '../../../../../../../../src/plugins/data/public'; import { findMathNodes, findVariables, @@ -72,6 +78,23 @@ export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing.message); } +export const getQueryValidationError = ( + query: string, + language: 'kql' | 'lucene', + indexPattern: IndexPattern +): string | undefined => { + try { + if (language === 'kql') { + esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query), indexPattern); + } else { + esQuery.luceneStringToDsl(query); + } + return; + } catch (e) { + return e.message; + } +}; + function getMessageFromId({ messageId, values, @@ -271,12 +294,32 @@ function checkMissingVariableOrFunctions( return [...missingErrors, ...invalidVariableErrors]; } +function getQueryValidationErrors( + namedArguments: TinymathNamedArgument[] | undefined, + indexPattern: IndexPattern +): ErrorWrapper[] { + const errors: ErrorWrapper[] = []; + (namedArguments ?? []).forEach((arg) => { + if (arg.name === 'kql' || arg.name === 'lucene') { + const message = getQueryValidationError(arg.value, arg.name, indexPattern); + if (message) { + errors.push({ + message, + locations: [arg.location], + }); + } + } + }); + return errors; +} + function validateNameArguments( node: TinymathFunction, nodeOperation: | OperationDefinition | OperationDefinition, - namedArguments: TinymathNamedArgument[] | undefined + namedArguments: TinymathNamedArgument[] | undefined, + indexPattern: IndexPattern ) { const errors = []; const missingParams = getMissingParams(nodeOperation, namedArguments); @@ -318,6 +361,10 @@ function validateNameArguments( }) ); } + const queryValidationErrors = getQueryValidationErrors(namedArguments, indexPattern); + if (queryValidationErrors.length) { + errors.push(...queryValidationErrors); + } return errors; } @@ -403,7 +450,12 @@ function runFullASTValidation( }) ); } else { - const argumentsErrors = validateNameArguments(node, nodeOperation, namedArguments); + const argumentsErrors = validateNameArguments( + node, + nodeOperation, + namedArguments, + indexPattern + ); if (argumentsErrors.length) { errors.push(...argumentsErrors); } @@ -459,7 +511,7 @@ export function canHaveParams( | OperationDefinition | OperationDefinition ) { - return Boolean((operation.operationParams || []).length); + return Boolean((operation.operationParams || []).length) || operation.filterable; } export function getInvalidParams( @@ -516,6 +568,12 @@ export function validateParams( ) { const paramsObj = getOperationParams(operation, params); const formalArgs = operation.operationParams || []; + if (operation.filterable) { + formalArgs.push( + { name: 'kql', type: 'string', required: false }, + { name: 'lucene', type: 'string', required: false } + ); + } return formalArgs.map(({ name, type, required }) => ({ name, isMissing: !(name in paramsObj), From c7e415458aa920b82cd1a5fd219bcb43ce643c25 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 1 Apr 2021 19:06:32 -0400 Subject: [PATCH 063/185] Fix autocomplete on weird characters and properly connect KQL --- .../definitions/formula/formula.tsx | 23 +++-- .../definitions/formula/math_completion.ts | 85 ++++++------------- .../operations/definitions/formula/util.ts | 3 + .../definitions/formula/validation.ts | 16 ++-- .../operations/definitions/metrics.tsx | 2 +- 5 files changed, 54 insertions(+), 75 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index d12fb149294a78..6e086193ac5597 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -305,6 +305,19 @@ function FormulaEditor({ [text] ); + /** + * The way that Monaco requests autocompletion is not intuitive, but the way we use it + * we fetch new suggestions in these scenarios: + * + * - If the user types one of the trigger characters, suggestions are always fetched + * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after + * - When the user types the first character into an empty text box, Monaco requests suggestions + * + * Monaco also triggers suggestions automatically when there are no suggestions being displayed + * and the user types a non-whitespace character. + * + * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. + */ const provideCompletionItems = useCallback( async ( model: monaco.editor.ITextModel, @@ -348,20 +361,12 @@ function FormulaEditor({ }); } } else { - const wordUntil = model.getWordUntilPosition(position); - wordRange = new monaco.Range( - position.lineNumber, - wordUntil.startColumn, - position.lineNumber, - wordUntil.endColumn - ); aSuggestions = await suggest({ expression: innerText, position: innerText.length - lengthAfterPosition, context, indexPattern, operationDefinitionMap, - word: wordUntil, data, }); } @@ -446,7 +451,7 @@ function FormulaEditor({ useEffect(() => { // Because the monaco model is owned by Lens, we need to manually attach and remove handlers const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', ',', '(', '=', ' ', ':'], + triggerCharacters: ['.', ',', '(', '=', ' ', ':', `'`], provideCompletionItems, }); const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index b3408399e48e85..b582a9cba5a68c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -44,11 +44,6 @@ function inLocation(cursorPosition: number, location: TinymathLocation) { return cursorPosition >= location.min && cursorPosition < location.max; } -const filterableOperationParams = [ - { name: 'kql', type: 'string', required: false }, - { name: 'lucene', type: 'string', required: false }, -]; - const MARKER = 'LENS_MATH_MARKER'; function getInfoAtPosition( @@ -89,17 +84,6 @@ export function offsetToRowColumn(expression: string, offset: number): monaco.Po throw new Error('Algorithm failure'); } -// export function monacoPositionToOffset(expression: string, position: monaco.Position): number { -// const lines = expression.split(/\n/); -// return lines -// .slice(0, position.lineNumber - 1) -// .reduce( -// (prev, current, index) => -// prev + index === position.lineNumber - 1 ? position.column - 1 : current.length, -// 0 -// ); -// } - export async function suggest({ expression, position, @@ -107,7 +91,6 @@ export async function suggest({ indexPattern, operationDefinitionMap, data, - word, }: { expression: string; position: number; @@ -115,38 +98,44 @@ export async function suggest({ indexPattern: IndexPattern; operationDefinitionMap: Record; data: DataPublicPluginStart; - word?: monaco.editor.IWordAtPosition; }): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { const ast = parse(text); const tokenInfo = getInfoAtPosition(ast, position); + const tokenAst = tokenInfo?.ast; const isNamedArgument = tokenInfo?.parent && - typeof tokenInfo?.ast !== 'number' && - 'type' in tokenInfo.ast && - tokenInfo.ast.type === 'namedArgument'; + typeof tokenAst !== 'number' && + tokenAst && + 'type' in tokenAst && + tokenAst.type === 'namedArgument'; if (tokenInfo?.parent && (context.triggerCharacter === '=' || isNamedArgument)) { return await getNamedArgumentSuggestions({ - expression: text, - ast: tokenInfo.ast as TinymathNamedArgument, - position, + ast: tokenAst as TinymathNamedArgument, data, indexPattern, - operationDefinitionMap, }); } else if (tokenInfo?.parent) { return getArgumentSuggestions( tokenInfo.parent, - tokenInfo.parent.args.findIndex((a) => a === tokenInfo.ast), + tokenInfo.parent.args.findIndex((a) => a === tokenAst), indexPattern, operationDefinitionMap ); } - if (tokenInfo && word) { - return getFunctionSuggestions(word, indexPattern, operationDefinitionMap); + if ( + typeof tokenAst === 'object' && + Boolean(tokenAst.type === 'variable' || tokenAst.type === 'function') + ) { + const nameWithMarker = tokenAst.type === 'function' ? tokenAst.name : tokenAst.value; + return getFunctionSuggestions( + nameWithMarker.split(MARKER)[0], + indexPattern, + operationDefinitionMap + ); } } catch (e) { // Fail silently @@ -172,14 +161,14 @@ export function getPossibleFunctions( } function getFunctionSuggestions( - word: monaco.editor.IWordAtPosition, + prefix: string, indexPattern: IndexPattern, operationDefinitionMap: Record ) { return { list: uniq( getPossibleFunctions(indexPattern, operationDefinitionMap).filter((func) => - startsWith(func, word.word) + startsWith(func, prefix) ) ).map((func) => ({ label: func, type: 'operation' as const })), type: SUGGESTION_TYPE.FUNCTIONS, @@ -291,17 +280,11 @@ function getArgumentSuggestions( export async function getNamedArgumentSuggestions({ ast, - expression, - position, data, indexPattern, - operationDefinitionMap, }: { ast: TinymathNamedArgument; - expression: string; - position: number; indexPattern: IndexPattern; - operationDefinitionMap: Record; data: DataPublicPluginStart; }) { if (ast.name !== 'kql' && ast.name !== 'lucene') { @@ -342,24 +325,17 @@ export function getSuggestion( let detail: string = ''; let command: monaco.languages.CompletionItem['command']; let sortText: string = ''; + const filterText: string = label; switch (type) { case SUGGESTION_TYPE.FIELD: - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; kind = monaco.languages.CompletionItemKind.Value; break; case SUGGESTION_TYPE.FUNCTIONS: - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; - insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; if (typeof suggestion !== 'string') { + if ('text' in suggestion) break; const tinymathFunction = tinymathFunctions[suggestion.label]; - insertText = `${label}($0)`; + insertText = `${label}()`; if (tinymathFunction) { label = `${label}(${tinymathFunction.positionalArguments .map(({ name }) => name) @@ -387,25 +363,18 @@ export function getSuggestion( } break; case SUGGESTION_TYPE.NAMED_ARGUMENT: - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; kind = monaco.languages.CompletionItemKind.Keyword; if (label === 'kql' || label === 'lucene') { + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; insertText = `${label}='$0'`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; } label = `${label}=`; detail = ''; break; - case SUGGESTION_TYPE.NAMED_ARGUMENT: - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; - kind = monaco.languages.CompletionItemKind.Field; - break; } return { @@ -415,8 +384,10 @@ export function getSuggestion( insertText: insertText ?? label, insertTextRules, command, + additionalTextEdits: [], range, sortText, + filterText, }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 9c8cf303f9c0f6..17ca19839a216b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -100,6 +100,9 @@ export function getOperationParams( if (formalArgs[name]) { args[name] = value; } + if (operation.filterable && (name === 'kql' || name === 'lucene')) { + args[name] = value; + } return args; }, {}); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index f5a6b19d2d3489..d2a1d308f66485 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -9,12 +9,7 @@ import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse, TinymathLocation } from '@kbn/tinymath'; import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; -import { - AggFunctionsMapping, - Query, - esKuery, - esQuery, -} from '../../../../../../../../src/plugins/data/public'; +import { esKuery, esQuery } from '../../../../../../../../src/plugins/data/public'; import { findMathNodes, findVariables, @@ -492,7 +487,12 @@ function runFullASTValidation( }) ); } else { - const argumentsErrors = validateNameArguments(node, nodeOperation, namedArguments); + const argumentsErrors = validateNameArguments( + node, + nodeOperation, + namedArguments, + indexPattern + ); if (argumentsErrors.length) { errors.push(...argumentsErrors); } @@ -567,7 +567,7 @@ export function validateParams( params: TinymathNamedArgument[] = [] ) { const paramsObj = getOperationParams(operation, params); - const formalArgs = operation.operationParams || []; + const formalArgs = [...(operation.operationParams ?? [])]; if (operation.filterable) { formalArgs.push( { name: 'kql', type: 'string', required: false }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 5ca15349bf3228..866d232aab5b38 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -106,7 +106,7 @@ function buildMetricOperation>({ isBucketed: false, scale: 'ratio', timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), } as T; }, From 08a985c4452df9bca4922629d1e145620ec4007d Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 2 Apr 2021 18:42:31 -0400 Subject: [PATCH 064/185] Highlight functions that have additional requirements after validating --- .../dimension_panel/dimension_panel.test.tsx | 27 ++---- .../indexpattern.test.ts | 24 +++++ .../definitions/formula/formula.test.tsx | 36 +++++++- .../definitions/formula/formula.tsx | 91 +++++++++++++++---- .../definitions/formula/math_completion.ts | 3 +- .../definitions/formula/validation.ts | 1 + .../operations/layer_helpers.ts | 19 ++++ .../indexpattern_datasource/to_expression.ts | 4 + 8 files changed, 163 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 7d1644d07d2aa5..02b2f7517acb0b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -25,6 +25,7 @@ import { import { mountWithIntl as mount, shallowWithIntl as shallow } from '@kbn/test/jest'; import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreSetup } from 'kibana/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; +import { generateId } from '../../id_generator'; import { IndexPatternPrivateState } from '../types'; import { IndexPatternColumn, replaceColumn } from '../operations'; import { documentField } from '../document_field'; @@ -48,6 +49,7 @@ jest.mock('lodash', () => { debounce: (fn: unknown) => fn, }; }); +jest.mock('../../id_generator'); const fields = [ { @@ -1034,7 +1036,7 @@ describe('IndexPatternDimensionEditorPanel', () => { } it('should not show custom options if time scaling is not available', () => { - wrapper = shallow( + wrapper = mount( { expect( wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-time-scaling-enable"]') ).toHaveLength(0); }); it('should show custom options if time scaling is available', () => { - wrapper = shallow(); + wrapper = mount(); expect( wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-time-scaling-enable"]') ).toHaveLength(1); }); @@ -1076,12 +1074,10 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to set time scaling initially', () => { const props = getProps({}); - wrapper = shallow(); + wrapper = mount(); wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-time-scaling-enable"]') .prop('onClick')!({} as MouseEvent); expect(props.setState).toHaveBeenCalledWith( @@ -1283,7 +1279,7 @@ describe('IndexPatternDimensionEditorPanel', () => { } it('should not show custom options if time scaling is not available', () => { - wrapper = shallow( + wrapper = mount( { expect( wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-filter-by-enable"]') ).toHaveLength(0); }); it('should show custom options if filtering is available', () => { - wrapper = shallow(); + wrapper = mount(); expect( wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-filter-by-enable"]') ).toHaveLength(1); }); @@ -1326,12 +1318,10 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to set filter initially', () => { const props = getProps({}); - wrapper = shallow(); + wrapper = mount(); wrapper .find(DimensionEditor) - .dive() .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-filter-by-enable"]') .prop('onClick')!({} as MouseEvent); expect(props.setState).toHaveBeenCalledWith( @@ -1909,6 +1899,7 @@ describe('IndexPatternDimensionEditorPanel', () => { }); it('should hide the top level field selector when switching from non-reference to reference', () => { + (generateId as jest.Mock).mockReturnValue(`second`); wrapper = mount(); expect(wrapper.find('ReferenceEditor')).toHaveLength(0); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 0ea533e22e4d94..75d8326b7e967d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -287,6 +287,30 @@ describe('IndexPattern Data Source', () => { expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null); }); + it('should generate an empty expression when there is a formula without aggs', async () => { + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Formula', + dataType: 'number', + isBucketed: false, + operationType: 'formula', + references: [], + params: {}, + }, + }, + }, + }, + }; + const state = enrichBaseState(queryBaseState); + expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null); + }); + it('should generate an expression for an aggregated query', async () => { const queryBaseState: IndexPatternBaseState = { currentIndexPatternId: '1', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 45521a723bfe1c..4341ae5f2d452b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -49,10 +49,21 @@ const operationDefinitionMap: Record = { }), } as unknown) as GenericOperationDefinition, derivative: { input: 'fullReference' } as GenericOperationDefinition, - moving_average: { + moving_average: ({ input: 'fullReference', operationParams: [{ name: 'window', type: 'number', required: true }], - } as GenericOperationDefinition, + buildColumn: ({ references }: { references: string[] }) => ({ + label: 'moving_average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + timeScale: false, + params: { window: 5 }, + references, + }), + getErrorMessage: () => ['mock error'], + } as unknown) as GenericOperationDefinition, cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition, }; @@ -236,7 +247,7 @@ describe('formula', () => { currentColumn, indexPattern, operationDefinitionMap - ) + ).newLayer ).toEqual({ ...layer, columns: { @@ -275,7 +286,7 @@ describe('formula', () => { currentColumn, indexPattern, operationDefinitionMap - ) + ).newLayer ).toEqual({ ...layer, columnOrder: ['col1X0', 'col1X1', 'col1'], @@ -419,6 +430,23 @@ describe('formula', () => { const formula = 'pow(bytes)'; testIsBrokenFormula(formula); }); + + it('returns the locations of each function', () => { + expect( + regenerateLayerFromAst( + 'moving_average(average(bytes), window=7) + count()', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).locations + ).toEqual({ + col1X0: { min: 15, max: 29 }, + col1X2: { min: 0, max: 41 }, + col1X3: { min: 43, max: 50 }, + }); + }); }); describe('getErrorMessage', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6e086193ac5597..79f087d36a8415 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -8,7 +8,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { TinymathAST, TinymathVariable } from '@kbn/tinymath'; +import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; import { EuiButton, EuiFlexGroup, @@ -34,7 +34,7 @@ import { } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder } from '../../layer_helpers'; +import { getColumnOrder, getManagedColumnsFrom } from '../../layer_helpers'; import { mathOperation } from './math'; import { documentField } from '../../../document_field'; import { ErrorWrapper, runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; @@ -289,13 +289,60 @@ function FormulaEditor({ startLineNumber: startPosition.lineNumber, endColumn: endPosition.column + 1, endLineNumber: endPosition.lineNumber, - severity: monaco.MarkerSeverity.Error, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, }; }) ) ); } else { monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + + // Only submit if valid + const { newLayer, locations } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + + const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); + const markers: monaco.editor.IMarkerData[] = managedColumns + .flatMap(([id, column]) => { + if (locations[id]) { + const def = operationDefinitionMap[column.operationType]; + if (def.getErrorMessage) { + const messages = def.getErrorMessage( + newLayer, + id, + indexPattern, + operationDefinitionMap + ); + if (messages) { + const startPosition = offsetToRowColumn(text, locations[id].min); + const endPosition = offsetToRowColumn(text, locations[id].max); + return [ + { + message: messages.join(', '), + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Warning, + }, + ]; + } + } + } + return []; + }) + .filter((marker) => marker); + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); } }, // Make it validate on flyout open in case of a broken formula left over @@ -492,7 +539,7 @@ function FormulaEditor({ })} - + {/* - + */} {isOpen ? ( @@ -636,7 +683,7 @@ Use the symbols +, -, /, and * to perform basic math. defaultMessage: 'Cancel', })} - + */} ) : null} @@ -708,14 +755,17 @@ export function regenerateLayerFromAst( ...layer.columns, }; + const locations: Record = {}; + Object.keys(columns).forEach((k) => { if (k.startsWith(columnId)) { delete columns[k]; } }); - extracted.forEach((extractedColumn, index) => { - columns[`${columnId}X${index}`] = extractedColumn; + extracted.forEach(({ column, location }, index) => { + columns[`${columnId}X${index}`] = column; + if (location) locations[`${columnId}X${index}`] = location; }); columns[columnId] = { @@ -729,12 +779,15 @@ export function regenerateLayerFromAst( }; return { - ...layer, - columns, - columnOrder: getColumnOrder({ + newLayer: { ...layer, columns, - }), + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }, + locations, }; // TODO @@ -748,8 +801,8 @@ function extractColumns( ast: TinymathAST, layer: IndexPatternLayer, indexPattern: IndexPattern -) { - const columns: IndexPatternColumn[] = []; +): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { + const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; function parseNode(node: TinymathAST) { if (typeof node === 'number' || node.type !== 'function') { @@ -796,7 +849,7 @@ function extractColumns( const newColId = `${idPrefix}X${columns.length}`; newCol.customLabel = true; newCol.label = newColId; - columns.push(newCol); + columns.push({ column: newCol, location: node.location }); // replace by new column id return newColId; } @@ -812,7 +865,7 @@ function extractColumns( }); mathColumn.references = subNodeVariables.map(({ value }) => value); mathColumn.params.tinymathAst = consumedParam!; - columns.push(mathColumn); + columns.push({ column: mathColumn }); mathColumn.customLabel = true; mathColumn.label = `${idPrefix}X${columns.length - 1}`; @@ -831,7 +884,7 @@ function extractColumns( const newColId = `${idPrefix}X${columns.length}`; newCol.customLabel = true; newCol.label = newColId; - columns.push(newCol); + columns.push({ column: newCol, location: node.location }); // replace by new column id return newColId; } @@ -850,7 +903,7 @@ function extractColumns( const newColId = `${idPrefix}X${columns.length}`; mathColumn.customLabel = true; mathColumn.label = newColId; - columns.push(mathColumn); + columns.push({ column: mathColumn }); return columns; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index b582a9cba5a68c..786379e3bec17a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -332,10 +332,11 @@ export function getSuggestion( kind = monaco.languages.CompletionItemKind.Value; break; case SUGGESTION_TYPE.FUNCTIONS: + insertText = `${label}($0)`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; if (typeof suggestion !== 'string') { if ('text' in suggestion) break; const tinymathFunction = tinymathFunctions[suggestion.label]; - insertText = `${label}()`; if (tinymathFunction) { label = `${label}(${tinymathFunction.positionalArguments .map(({ name }) => name) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index d2a1d308f66485..cb52e22302cbe2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -67,6 +67,7 @@ type ErrorValues = typeof validationErrors[K]['type']; export interface ErrorWrapper { message: string; locations: TinymathLocation[]; + severity?: 'error' | 'warning'; } export function isParsingError(message: string) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index ecfb8928cd4f0b..4f37144d4dd526 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1127,6 +1127,25 @@ export function isColumnValidAsReference({ ); } +export function getManagedColumnsFrom( + columnId: string, + columns: Record +): Array<[string, IndexPatternColumn]> { + const allNodes: Record = {}; + Object.entries(columns).forEach(([id, col]) => { + allNodes[id] = 'references' in col ? col.references : []; + }); + const queue: string[] = allNodes[columnId]; + const store: Array<[string, IndexPatternColumn]> = []; + + while (queue.length > 0) { + const nextId = queue.shift()!; + store.push([nextId, columns[nextId]]); + queue.push(...allNodes[nextId]); + } + return store.filter(([, column]) => column); +} + function topologicalSort(columns: Array<[string, IndexPatternColumn]>) { const allNodes: Record = {}; columns.forEach(([id, col]) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index c7e2bd40fa9120..231c5945daea15 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -105,6 +105,10 @@ function getExpressionForLayer( const def = operationDefinitionMap[col.operationType]; return !(def.input === 'fullReference' || def.input === 'managedReference'); }); + if (aggColumnEntries.length === 0) { + // Return early if there are no aggs, for example if the user has an empty formula + return null; + } const idMap = aggColumnEntries.reduce((currentIdMap, [colId, column], index) => { const esAggsId = `col-${aggColumnEntries.length === 1 ? 0 : index}-${colId}`; return { From daa5685ee58d3d5fbb94b4eaba67d30021d365fa Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 2 Apr 2021 19:14:37 -0400 Subject: [PATCH 065/185] Fix type error and move help text to popover --- src/plugins/kibana_react/public/index.ts | 2 +- .../definitions/formula/formula.tsx | 204 ++++++++---------- .../operations/layer_helpers.ts | 2 +- 3 files changed, 91 insertions(+), 117 deletions(-) diff --git a/src/plugins/kibana_react/public/index.ts b/src/plugins/kibana_react/public/index.ts index f2c2c263da5cd8..612366de59f746 100644 --- a/src/plugins/kibana_react/public/index.ts +++ b/src/plugins/kibana_react/public/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export * from './code_editor'; +export { CodeEditor, CodeEditorProps } from './code_editor'; export * from './url_template_editor'; export * from './exit_full_screen_button'; export * from './context'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 79f087d36a8415..45b95f37483013 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -18,7 +18,7 @@ import { EuiSpacer, EuiModal, EuiModalHeader, - EuiModalFooter, + EuiPopover, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import { @@ -215,6 +215,7 @@ function FormulaEditor({ }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [isOpen, setIsOpen] = useState(false); + const [isHelpOpen, setIsHelpOpen] = useState(false); const editorModel = React.useRef( monaco.editor.createModel(text ?? '', LANGUAGE_ID) ); @@ -515,6 +516,72 @@ function FormulaEditor({ }; }, [provideCompletionItems, provideSignatureHelp, provideHover]); + const helpComponent = ( +
+ + + + key in tinymathFunctions) + .map((key) => ({ + title: `${key}`, + description: , + }))} + /> + + + + + + {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { + defaultMessage: 'Elasticsearch aggregations', + description: 'Do not translate Elasticsearch', + })} + + key in operationDefinitionMap) + .map((key) => ({ + title: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + }))} + /> +
+ ); + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences // in the behavior of Monaco when it's first loaded and then reloaded. return ( @@ -532,38 +599,29 @@ function FormulaEditor({ /> + + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} iconType="help" size="s"> + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + > + {helpComponent} + + + setIsOpen(!isOpen)} iconType="expand" size="s"> - {i18n.translate('xpack.lens.formula.expandEditorLabel', { - defaultMessage: 'Pop out', + {i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Full screen', })} - {/* - { - updateLayer( - regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ) - ); - }} - iconType="play" - size="s" - > - {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { - defaultMessage: 'Submit', - })} - - */} {isOpen ? ( @@ -604,73 +662,11 @@ function FormulaEditor({ })} -
- - - - key in tinymathFunctions) - .map((key) => ({ - title: `${key}`, - description: , - }))} - /> - - - - - - {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { - defaultMessage: 'Elasticsearch aggregations', - description: 'Do not translate Elasticsearch', - })} - - key in operationDefinitionMap) - .map((key) => ({ - title: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - }))} - /> -
+ {helpComponent}
- + {/* { @@ -683,29 +679,7 @@ Use the symbols +, -, /, and * to perform basic math. defaultMessage: 'Cancel', })} - {/* { - updateLayer( - regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ) - ); - }} - iconType="play" - > - {i18n.translate('xpack.lens.indexPattern.formulaSubmitLabel', { - defaultMessage: 'Submit', - })} - */} - + */} ) : null}
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 4f37144d4dd526..e24671d9e8e676 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -353,7 +353,7 @@ export function replaceColumn({ newColumn, indexPattern, operationDefinitionMap - ) + ).newLayer : basicLayer; } catch (e) { newLayer = basicLayer; From 239c72db3dc2e128362791564de9c80ead0293aa Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 5 Apr 2021 16:52:28 -0400 Subject: [PATCH 066/185] Fix escape characters inside KQL --- packages/kbn-tinymath/src/grammar.js | 258 +++++++++++------- packages/kbn-tinymath/src/grammar.pegjs | 9 +- packages/kbn-tinymath/test/library.test.js | 9 + .../definitions/formula/math_completion.ts | 5 + .../definitions/formula/math_tokenization.tsx | 25 +- 5 files changed, 195 insertions(+), 111 deletions(-) diff --git a/packages/kbn-tinymath/src/grammar.js b/packages/kbn-tinymath/src/grammar.js index 6e52b5769d4415..c6aa7fd11003b2 100644 --- a/packages/kbn-tinymath/src/grammar.js +++ b/packages/kbn-tinymath/src/grammar.js @@ -216,16 +216,26 @@ function peg$parse(input, options) { peg$c38 = function(first, rest) { return [first].concat(rest); }, - peg$c39 = /^[^"]/, - peg$c40 = peg$classExpectation(["\""], true, false), - peg$c41 = function(value) { return value.join(''); }, - peg$c42 = /^[^']/, - peg$c43 = peg$classExpectation(["'"], true, false), - peg$c44 = /^[a-zA-Z_]/, - peg$c45 = peg$classExpectation([["a", "z"], ["A", "Z"], "_"], false, false), - peg$c46 = "=", - peg$c47 = peg$literalExpectation("=", false), - peg$c48 = function(name, value) { + peg$c39 = "\"", + peg$c40 = peg$literalExpectation("\"", false), + peg$c41 = "\\\"", + peg$c42 = peg$literalExpectation("\\\"", false), + peg$c43 = function() { return "\""; }, + peg$c44 = /^[^"]/, + peg$c45 = peg$classExpectation(["\""], true, false), + peg$c46 = function(chars) { return chars.join(''); }, + peg$c47 = "'", + peg$c48 = peg$literalExpectation("'", false), + peg$c49 = "\\'", + peg$c50 = peg$literalExpectation("\\'", false), + peg$c51 = function() { return "\'"; }, + peg$c52 = /^[^']/, + peg$c53 = peg$classExpectation(["'"], true, false), + peg$c54 = /^[a-zA-Z_]/, + peg$c55 = peg$classExpectation([["a", "z"], ["A", "Z"], "_"], false, false), + peg$c56 = "=", + peg$c57 = peg$literalExpectation("=", false), + peg$c58 = function(name, value) { return { type: 'namedArgument', name: name.join(''), @@ -234,10 +244,10 @@ function peg$parse(input, options) { text: text() }; }, - peg$c49 = peg$otherExpectation("function"), - peg$c50 = /^[a-zA-Z_\-]/, - peg$c51 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), - peg$c52 = function(name, args) { + peg$c59 = peg$otherExpectation("function"), + peg$c60 = /^[a-zA-Z_\-]/, + peg$c61 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), + peg$c62 = function(name, args) { return { type: 'function', name: name.join(''), @@ -246,21 +256,21 @@ function peg$parse(input, options) { text: text() }; }, - peg$c53 = peg$otherExpectation("number"), - peg$c54 = function() { + peg$c63 = peg$otherExpectation("number"), + peg$c64 = function() { return parseFloat(text()); }, - peg$c55 = /^[eE]/, - peg$c56 = peg$classExpectation(["e", "E"], false, false), - peg$c57 = peg$otherExpectation("exponent"), - peg$c58 = ".", - peg$c59 = peg$literalExpectation(".", false), - peg$c60 = "0", - peg$c61 = peg$literalExpectation("0", false), - peg$c62 = /^[1-9]/, - peg$c63 = peg$classExpectation([["1", "9"]], false, false), - peg$c64 = /^[0-9]/, - peg$c65 = peg$classExpectation([["0", "9"]], false, false), + peg$c65 = /^[eE]/, + peg$c66 = peg$classExpectation(["e", "E"], false, false), + peg$c67 = peg$otherExpectation("exponent"), + peg$c68 = ".", + peg$c69 = peg$literalExpectation(".", false), + peg$c70 = "0", + peg$c71 = peg$literalExpectation("0", false), + peg$c72 = /^[1-9]/, + peg$c73 = peg$classExpectation([["1", "9"]], false, false), + peg$c74 = /^[0-9]/, + peg$c75 = peg$classExpectation([["0", "9"]], false, false), peg$currPos = 0, peg$savedPos = 0, @@ -1119,61 +1129,77 @@ function peg$parse(input, options) { return s0; } - function peg$parseStringChar() { - var s0; - - if (peg$c9.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c10); } - } - - return s0; - } - function peg$parseString() { - var s0, s1, s2, s3; + var s0, s1, s2, s3, s4; s0 = peg$currPos; - if (peg$c15.test(input.charAt(peg$currPos))) { - s1 = input.charAt(peg$currPos); + if (input.charCodeAt(peg$currPos) === 34) { + s1 = peg$c39; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } + if (peg$silentFails === 0) { peg$fail(peg$c40); } } if (s1 !== peg$FAILED) { s2 = []; - if (peg$c39.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; + s3 = peg$currPos; + if (input.substr(peg$currPos, 2) === peg$c41) { + s4 = peg$c41; + peg$currPos += 2; } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c40); } + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c42); } } - while (s3 !== peg$FAILED) { - s2.push(s3); - if (peg$c39.test(input.charAt(peg$currPos))) { + if (s4 !== peg$FAILED) { + peg$savedPos = s3; + s4 = peg$c43(); + } + s3 = s4; + if (s3 === peg$FAILED) { + if (peg$c44.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c40); } + if (peg$silentFails === 0) { peg$fail(peg$c45); } + } + } + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$currPos; + if (input.substr(peg$currPos, 2) === peg$c41) { + s4 = peg$c41; + peg$currPos += 2; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c42); } + } + if (s4 !== peg$FAILED) { + peg$savedPos = s3; + s4 = peg$c43(); + } + s3 = s4; + if (s3 === peg$FAILED) { + if (peg$c44.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c45); } + } } } if (s2 !== peg$FAILED) { - if (peg$c15.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); + if (input.charCodeAt(peg$currPos) === 34) { + s3 = peg$c39; peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } + if (peg$silentFails === 0) { peg$fail(peg$c40); } } if (s3 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c41(s2); + s1 = peg$c46(s2); s0 = s1; } else { peg$currPos = s0; @@ -1189,43 +1215,73 @@ function peg$parse(input, options) { } if (s0 === peg$FAILED) { s0 = peg$currPos; - if (peg$c13.test(input.charAt(peg$currPos))) { - s1 = input.charAt(peg$currPos); + if (input.charCodeAt(peg$currPos) === 39) { + s1 = peg$c47; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } + if (peg$silentFails === 0) { peg$fail(peg$c48); } } if (s1 !== peg$FAILED) { s2 = []; - if (peg$c42.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; + s3 = peg$currPos; + if (input.substr(peg$currPos, 2) === peg$c49) { + s4 = peg$c49; + peg$currPos += 2; } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c43); } + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c50); } } - while (s3 !== peg$FAILED) { - s2.push(s3); - if (peg$c42.test(input.charAt(peg$currPos))) { + if (s4 !== peg$FAILED) { + peg$savedPos = s3; + s4 = peg$c51(); + } + s3 = s4; + if (s3 === peg$FAILED) { + if (peg$c52.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c43); } + if (peg$silentFails === 0) { peg$fail(peg$c53); } + } + } + while (s3 !== peg$FAILED) { + s2.push(s3); + s3 = peg$currPos; + if (input.substr(peg$currPos, 2) === peg$c49) { + s4 = peg$c49; + peg$currPos += 2; + } else { + s4 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c50); } + } + if (s4 !== peg$FAILED) { + peg$savedPos = s3; + s4 = peg$c51(); + } + s3 = s4; + if (s3 === peg$FAILED) { + if (peg$c52.test(input.charAt(peg$currPos))) { + s3 = input.charAt(peg$currPos); + peg$currPos++; + } else { + s3 = peg$FAILED; + if (peg$silentFails === 0) { peg$fail(peg$c53); } + } } } if (s2 !== peg$FAILED) { - if (peg$c13.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); + if (input.charCodeAt(peg$currPos) === 39) { + s3 = peg$c47; peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } + if (peg$silentFails === 0) { peg$fail(peg$c48); } } if (s3 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c41(s2); + s1 = peg$c46(s2); s0 = s1; } else { peg$currPos = s0; @@ -1253,7 +1309,7 @@ function peg$parse(input, options) { } if (s1 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c41(s1); + s1 = peg$c46(s1); } s0 = s1; } @@ -1267,22 +1323,22 @@ function peg$parse(input, options) { s0 = peg$currPos; s1 = []; - if (peg$c44.test(input.charAt(peg$currPos))) { + if (peg$c54.test(input.charAt(peg$currPos))) { s2 = input.charAt(peg$currPos); peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } + if (peg$silentFails === 0) { peg$fail(peg$c55); } } if (s2 !== peg$FAILED) { while (s2 !== peg$FAILED) { s1.push(s2); - if (peg$c44.test(input.charAt(peg$currPos))) { + if (peg$c54.test(input.charAt(peg$currPos))) { s2 = input.charAt(peg$currPos); peg$currPos++; } else { s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } + if (peg$silentFails === 0) { peg$fail(peg$c55); } } } } else { @@ -1292,11 +1348,11 @@ function peg$parse(input, options) { s2 = peg$parse_(); if (s2 !== peg$FAILED) { if (input.charCodeAt(peg$currPos) === 61) { - s3 = peg$c46; + s3 = peg$c56; peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c47); } + if (peg$silentFails === 0) { peg$fail(peg$c57); } } if (s3 !== peg$FAILED) { s4 = peg$parse_(); @@ -1309,7 +1365,7 @@ function peg$parse(input, options) { s6 = peg$parse_(); if (s6 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c48(s1, s5); + s1 = peg$c58(s1, s5); s0 = s1; } else { peg$currPos = s0; @@ -1350,22 +1406,22 @@ function peg$parse(input, options) { s1 = peg$parse_(); if (s1 !== peg$FAILED) { s2 = []; - if (peg$c50.test(input.charAt(peg$currPos))) { + if (peg$c60.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c51); } + if (peg$silentFails === 0) { peg$fail(peg$c61); } } if (s3 !== peg$FAILED) { while (s3 !== peg$FAILED) { s2.push(s3); - if (peg$c50.test(input.charAt(peg$currPos))) { + if (peg$c60.test(input.charAt(peg$currPos))) { s3 = input.charAt(peg$currPos); peg$currPos++; } else { s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c51); } + if (peg$silentFails === 0) { peg$fail(peg$c61); } } } } else { @@ -1400,7 +1456,7 @@ function peg$parse(input, options) { s8 = peg$parse_(); if (s8 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c52(s2, s5); + s1 = peg$c62(s2, s5); s0 = s1; } else { peg$currPos = s0; @@ -1437,7 +1493,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c49); } + if (peg$silentFails === 0) { peg$fail(peg$c59); } } return s0; @@ -1472,7 +1528,7 @@ function peg$parse(input, options) { } if (s4 !== peg$FAILED) { peg$savedPos = s0; - s1 = peg$c54(); + s1 = peg$c64(); s0 = s1; } else { peg$currPos = s0; @@ -1493,7 +1549,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c53); } + if (peg$silentFails === 0) { peg$fail(peg$c63); } } return s0; @@ -1502,12 +1558,12 @@ function peg$parse(input, options) { function peg$parseE() { var s0; - if (peg$c55.test(input.charAt(peg$currPos))) { + if (peg$c65.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c56); } + if (peg$silentFails === 0) { peg$fail(peg$c66); } } return s0; @@ -1559,7 +1615,7 @@ function peg$parse(input, options) { peg$silentFails--; if (s0 === peg$FAILED) { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c57); } + if (peg$silentFails === 0) { peg$fail(peg$c67); } } return s0; @@ -1570,11 +1626,11 @@ function peg$parse(input, options) { s0 = peg$currPos; if (input.charCodeAt(peg$currPos) === 46) { - s1 = peg$c58; + s1 = peg$c68; peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c59); } + if (peg$silentFails === 0) { peg$fail(peg$c69); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1606,20 +1662,20 @@ function peg$parse(input, options) { var s0, s1, s2, s3; if (input.charCodeAt(peg$currPos) === 48) { - s0 = peg$c60; + s0 = peg$c70; peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c61); } + if (peg$silentFails === 0) { peg$fail(peg$c71); } } if (s0 === peg$FAILED) { s0 = peg$currPos; - if (peg$c62.test(input.charAt(peg$currPos))) { + if (peg$c72.test(input.charAt(peg$currPos))) { s1 = input.charAt(peg$currPos); peg$currPos++; } else { s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c63); } + if (peg$silentFails === 0) { peg$fail(peg$c73); } } if (s1 !== peg$FAILED) { s2 = []; @@ -1647,12 +1703,12 @@ function peg$parse(input, options) { function peg$parseDigit() { var s0; - if (peg$c64.test(input.charAt(peg$currPos))) { + if (peg$c74.test(input.charAt(peg$currPos))) { s0 = input.charAt(peg$currPos); peg$currPos++; } else { s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c65); } + if (peg$silentFails === 0) { peg$fail(peg$c75); } } return s0; diff --git a/packages/kbn-tinymath/src/grammar.pegjs b/packages/kbn-tinymath/src/grammar.pegjs index 2334049a74ef9d..d80d7b1c426aaa 100644 --- a/packages/kbn-tinymath/src/grammar.pegjs +++ b/packages/kbn-tinymath/src/grammar.pegjs @@ -110,13 +110,10 @@ Argument_List "arguments" return [first].concat(rest); } -StringChar - = [0-9A-Za-z._@\[\]-] - String - = [\"] value:([^"]*) [\"] { return value.join(''); } - / [\'] value:([^']*) [\'] { return value.join(''); } - / value:(ValidChar)+ { return value.join(''); } + = '"' chars:("\\\"" { return "\""; } / [^"])* '"' { return chars.join(''); } + / "'" chars:("\\\'" { return "\'"; } / [^'])* "'" { return chars.join(''); } + / chars:(ValidChar)+ { return chars.join(''); } Argument = name:[a-zA-Z_]+ _ '=' _ value:(Number / String) _ { diff --git a/packages/kbn-tinymath/test/library.test.js b/packages/kbn-tinymath/test/library.test.js index 1593c5e1dcb681..80df2df43423d4 100644 --- a/packages/kbn-tinymath/test/library.test.js +++ b/packages/kbn-tinymath/test/library.test.js @@ -198,6 +198,15 @@ describe('Parser', () => { functionEqual('foo', [namedArgumentEqual('filter', `😀 > "\ttab"`)]) ); }); + + it('named with escape characters', () => { + expect(parse(`foo(filter='Women\\'s Clothing')`)).toEqual( + functionEqual('foo', [namedArgumentEqual('filter', `Women's Clothing`)]) + ); + expect(parse(`foo(filter="\\"Quoted inner string\\"")`)).toEqual( + functionEqual('foo', [namedArgumentEqual('filter', `"Quoted inner string"`)]) + ); + }); }); it('Missing expression', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 786379e3bec17a..19cbcca3d73e6e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -376,6 +376,11 @@ export function getSuggestion( label = `${label}=`; detail = ''; break; + case SUGGESTION_TYPE.KQL: + if (label.includes(`'`)) { + insertText = label.replaceAll(`'`, "\\'"); + } + break; } return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx index 918bad35337771..51e96bad600439 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx @@ -25,25 +25,42 @@ export const languageConfiguration: monaco.languages.LanguageConfiguration = { ], }; -export const lexerRules: monaco.languages.IMonarchLanguage = { +export const lexerRules = { defaultToken: 'invalid', tokenPostfix: '', ignoreCase: true, brackets: [{ open: '(', close: ')', token: 'delimiter.parenthesis' }], + escapes: /\\(?:[\\"'])/, tokenizer: { root: [ [/\s+/, 'whitespace'], [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'keyword'], [/[,=]/, 'delimiter'], [/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number'], - [/".+?"/, 'string'], - [/'.+?'/, 'string'], + // strings double quoted + [/"([^"\\]|\\.)*$/, 'string.invalid'], // string without termination + [/"/, 'string', '@string_dq'], + // strings single quoted + [/'([^'\\]|\\.)*$/, 'string.invalid'], // string without termination + [/'/, 'string', '@string_sq'], [/\+|\-|\*|\//, 'keyword.operator'], [/[\(]/, 'delimiter'], [/[\)]/, 'delimiter'], ], + string_dq: [ + [/[^\\"]+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/"/, 'string', '@pop'], + ], + string_sq: [ + [/[^\\']+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/'/, 'string', '@pop'], + ], }, -}; +} as monaco.languages.IMonarchLanguage; monaco.languages.setMonarchTokensProvider(LANGUAGE_ID, lexerRules); monaco.languages.setLanguageConfiguration(LANGUAGE_ID, languageConfiguration); From fa579517b4575a38987c2df099d17252d163c09b Mon Sep 17 00:00:00 2001 From: dej611 Date: Tue, 6 Apr 2021 16:58:05 +0200 Subject: [PATCH 067/185] :bug: Fix dataType issue when moving over to Formula --- .../definitions/formula/formula.test.tsx | 71 +++++++++++++++++-- .../definitions/formula/formula.tsx | 17 +++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 4341ae5f2d452b..27c3987effe843 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -95,6 +95,20 @@ describe('formula', () => { let indexPattern: IndexPattern; beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Average', + dataType: 'number', + operationType: 'average', + isBucketed: false, + scale: 'ratio', + sourceField: 'bytes', + }, + }, + }; indexPattern = createMockedIndexPattern(); }); @@ -132,7 +146,7 @@ describe('formula', () => { operationType: 'formula', isBucketed: false, scale: 'ratio', - params: { isFormulaBroken: false, formula: 'terms(category)' }, + params: { isFormulaBroken: false, formula: 'average(bytes)' }, references: [], }); }); @@ -143,7 +157,6 @@ describe('formula', () => { previousColumn: { ...layer.columns.col1, params: { - ...layer.columns.col1.params, format: { id: 'number', params: { @@ -163,7 +176,7 @@ describe('formula', () => { scale: 'ratio', params: { isFormulaBroken: false, - formula: 'terms(category)', + formula: 'average(bytes)', format: { id: 'number', params: { @@ -175,7 +188,7 @@ describe('formula', () => { }); }); - it('should move over previous operation parameter if set', () => { + it('should move over previous operation parameter if set - only numeric', () => { expect( formulaOperation.buildColumn( { @@ -232,6 +245,56 @@ describe('formula', () => { references: [], }); }); + + it('should not move previous column configuration if not numeric', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); }); describe('regenerateLayerFromAst()', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 45b95f37483013..0dbfde922d4891 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -45,6 +45,7 @@ import { getSafeFieldName, groupArgsByType, hasMathNode, + isSupportedFieldType, tinymathFunctions, } from './util'; import { useDebounceWithOptions } from '../helpers'; @@ -151,18 +152,22 @@ export const formulaOperation: OperationDefinition< ] : []; }, - buildColumn({ previousColumn, layer }, _, operationDefinitionMap) { + buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { let previousFormula = ''; if (previousColumn) { if ('references' in previousColumn) { const metric = layer.columns[previousColumn.references[0]]; - if (metric && 'sourceField' in metric) { + if (metric && 'sourceField' in metric && metric.dataType === 'number') { const fieldName = getSafeFieldName(metric.sourceField); // TODO need to check the input type from the definition previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; } } else { - if (previousColumn && 'sourceField' in previousColumn) { + if ( + previousColumn && + 'sourceField' in previousColumn && + previousColumn.dataType === 'number' + ) { previousFormula += `${previousColumn.operationType}(${getSafeFieldName( previousColumn?.sourceField )}`; @@ -173,8 +178,10 @@ export const formulaOperation: OperationDefinition< previousFormula += ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); } - // close the formula at the end - previousFormula += ')'; + if (previousFormula) { + // close the formula at the end + previousFormula += ')'; + } } // carry over the format settings from previous operation for seamless transfer // NOTE: this works only for non-default formatters set in Lens From 1fd06bfcc912af2e64c13d5a8c3c12386f7fc994 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 6 Apr 2021 19:15:01 -0400 Subject: [PATCH 068/185] Automatically insert single quotes on every named param --- .../public/code_editor/code_editor.tsx | 2 +- .../definitions/formula/formula.test.tsx | 28 +++++ .../definitions/formula/formula.tsx | 103 +++++++++++++++--- .../operations/definitions/formula/math.tsx | 3 + 4 files changed, 120 insertions(+), 16 deletions(-) diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index 5b16d38a8e8a59..78861a5d5d27a7 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -30,7 +30,7 @@ export interface Props { value: string; /** Function invoked when text in editor is changed */ - onChange: (value: string) => void; + onChange: (value: string, event: monaco.editor.IModelContentChangedEvent) => void; /** * Options for the Monaco Code Editor diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 27c3987effe843..0e80495c6db0be 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -188,6 +188,34 @@ describe('formula', () => { }); }); + it('it should move over kql arguments if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + filter: { + language: 'kuery', + // Need to test with multiple replaces due to string replace + query: `category.keyword: "Men's Clothing" or category.keyword: "Men's Shoes"`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `terms(category, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, + }, + references: [], + }); + }); + it('should move over previous operation parameter if set - only numeric', () => { expect( formulaOperation.buildColumn( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 0dbfde922d4891..cecb8f12312f68 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -45,7 +45,6 @@ import { getSafeFieldName, groupArgsByType, hasMathNode, - isSupportedFieldType, tinymathFunctions, } from './util'; import { useDebounceWithOptions } from '../helpers'; @@ -178,6 +177,12 @@ export const formulaOperation: OperationDefinition< previousFormula += ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); } + if (previousColumn.filter) { + previousFormula += + ', ' + + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all + } if (previousFormula) { // close the formula at the end previousFormula += ')'; @@ -228,6 +233,9 @@ function FormulaEditor({ ); const overflowDiv1 = React.useRef(); const overflowDiv2 = React.useRef(); + const updateAfterTyping = React.useRef(); + const editor1 = React.useRef(); + const editor2 = React.useRef(); // The Monaco editor needs to have the overflowDiv in the first render. Using an effect // requires a second render to work, so we are using an if statement to guarantee it happens @@ -249,10 +257,16 @@ function FormulaEditor({ // Clean up the monaco editor and DOM on unmount useEffect(() => { const model = editorModel.current; + const disposable1 = updateAfterTyping.current; + const editor1ref = editor1.current; + const editor2ref = editor2.current; return () => { model.dispose(); overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); overflowDiv2.current?.parentNode?.removeChild(overflowDiv2.current); + disposable1?.dispose(); + editor1ref?.dispose(); + editor2ref?.dispose(); }; }, []); @@ -484,6 +498,71 @@ function FormulaEditor({ [operationDefinitionMap] ); + const onTypeHandler = useCallback( + (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { + if (e.isFlush || e.isRedoing || e.isUndoing) { + return; + } + // I think the changes are always length 1 when triggered by a user, but if we + // add more automation it can create change lists. + if (e.changes.length === 1 && e.changes[0].text === '=') { + const currentPosition = e.changes[0].range; + if (currentPosition) { + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.executeEdits( + 'LENS', + [ + { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }, + ], + [ + // After inserting, move the cursor in between the single quotes + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + }, 0); + } + } + }, + [] + ); + + const registerOnTypeHandler = useCallback( + (editor: monaco.editor.IStandaloneCodeEditor) => { + // Toggle between two different callbacks when the editors change + if (updateAfterTyping.current) { + updateAfterTyping.current.dispose(); + } + updateAfterTyping.current = editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }); + }, + [onTypeHandler] + ); + + useEffect(() => { + if (updateAfterTyping.current) { + if (isOpen) { + if (editor2.current) registerOnTypeHandler(editor2.current); + } else { + if (editor1.current) registerOnTypeHandler(editor1.current); + } + } + }, [isOpen, registerOnTypeHandler]); + const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -603,6 +682,10 @@ Use the symbols +, -, /, and * to perform basic math. overflowWidgetsDomNode: overflowDiv1.current, model: editorModel.current, }} + editorDidMount={(editor) => { + editor1.current = editor; + registerOnTypeHandler(editor); + }} /> @@ -658,6 +741,10 @@ Use the symbols +, -, /, and * to perform basic math. overflowWidgetsDomNode: overflowDiv2.current, model: editorModel?.current, }} + editorDidMount={(editor) => { + editor2.current = editor; + registerOnTypeHandler(editor); + }} />
@@ -673,20 +760,6 @@ Use the symbols +, -, /, and * to perform basic math.
- {/* - { - setIsOpen(false); - setText(currentColumn.params.formula); - }} - iconType="cross" - > - {i18n.translate('xpack.lens.indexPattern.formulaCancelLabel', { - defaultMessage: 'Cancel', - })} - - */} ) : null}
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index 1060f51e14ff23..8d0a28e168ba73 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -94,6 +94,9 @@ function astToString(ast: TinymathAST | string): string | number { return ast.value; } if (ast.type === 'namedArgument') { + if (ast.name === 'kql' || ast.name === 'lucene') { + return `${ast.name}='${ast.value}'`; + } return `${ast.name}=${ast.value}`; } return `${ast.name}(${ast.args.map(astToString).join(',')})`; From b62f96ee1e389d97171ab84ae728c564dad5a3b3 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 8 Apr 2021 14:05:38 -0400 Subject: [PATCH 069/185] Only insert single quotes when typing kql= or lucene= --- .../definitions/formula/formula.test.tsx | 28 +++++++++++++++++ .../definitions/formula/formula.tsx | 30 +++++++++++++++++-- .../definitions/formula/math_completion.ts | 22 ++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 0e80495c6db0be..22f332d3e95aac 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -216,6 +216,34 @@ describe('formula', () => { }); }); + it('it should move over lucene arguments without', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + operationType: 'count', + filter: { + language: 'lucene', + query: `*`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `count(lucene='*')`, + }, + references: [], + }); + }); + it('should move over previous operation parameter if set - only numeric', () => { expect( formulaOperation.buildColumn( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index cecb8f12312f68..587442b09e4ab3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -56,7 +56,9 @@ import { getPossibleFunctions, getSignatureHelp, getHover, + getTokenInfo, offsetToRowColumn, + monacoPositionToOffset, } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; @@ -178,8 +180,10 @@ export const formulaOperation: OperationDefinition< ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); } if (previousColumn.filter) { + if (previousColumn.operationType !== 'count') { + previousFormula += ', '; + } previousFormula += - ', ' + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all } @@ -503,11 +507,28 @@ function FormulaEditor({ if (e.isFlush || e.isRedoing || e.isUndoing) { return; } - // I think the changes are always length 1 when triggered by a user, but if we - // add more automation it can create change lists. if (e.changes.length === 1 && e.changes[0].text === '=') { const currentPosition = e.changes[0].range; if (currentPosition) { + const tokenInfo = getTokenInfo( + editor.getValue(), + monacoPositionToOffset( + editor.getValue(), + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + ); + // Make sure that we are only adding kql='' or lucene='', and also + // check that the = sign isn't inside the KQL expression like kql='=' + if ( + !tokenInfo || + typeof tokenInfo.ast === 'number' || + tokenInfo.ast.type !== 'namedArgument' || + (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || + tokenInfo.ast.value !== 'LENS_MATH_MARKER' + ) { + return; + } + // Timeout is required because otherwise the cursor position is not updated. setTimeout(() => { editor.executeEdits( @@ -553,6 +574,9 @@ function FormulaEditor({ [onTypeHandler] ); + // Toggle between the handlers whenever the full screen mode is changed, + // because Monaco only maintains cursor position in the active model + // while it has focus. useEffect(() => { if (updateAfterTyping.current) { if (isOpen) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 19cbcca3d73e6e..24e91126e7c372 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -84,6 +84,17 @@ export function offsetToRowColumn(expression: string, offset: number): monaco.Po throw new Error('Algorithm failure'); } +export function monacoPositionToOffset(expression: string, position: monaco.Position): number { + const lines = expression.split(/\n/); + return lines + .slice(0, position.lineNumber) + .reduce( + (prev, current, index) => + prev + index === position.lineNumber - 1 ? position.column : current.length, + 0 + ); +} + export async function suggest({ expression, position, @@ -540,3 +551,14 @@ export function getHover( } return { contents: [] }; } + +export function getTokenInfo(expression: string, position: number) { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + return getInfoAtPosition(ast, position); + } catch (e) { + return; + } +} From d563cb5ae2c8c4192407ece3a0b4c77dc7e9ac62 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 8 Apr 2021 16:55:48 -0400 Subject: [PATCH 070/185] Reorganize help popover --- .../dimension_panel/dimension_editor.scss | 8 + .../definitions/formula/formula.test.tsx | 3 +- .../definitions/formula/formula.tsx | 421 +++++++++++------- .../operations/layer_helpers.ts | 2 +- 4 files changed, 273 insertions(+), 161 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index bf833c4a369325..29c1ad307fe402 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -10,6 +10,14 @@ background-color: $euiColorLightestShade; } +.lnsIndexPatternDimensionEditor__section--top { + border-bottom: $euiBorderThin; +} + +.lnsIndexPatternDimensionEditor__section--bottom { + border-top: $euiBorderThin; +} + .lnsIndexPatternDimensionEditor__columns { column-count: 2; column-gap: $euiSizeXL; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 22f332d3e95aac..adfce115ef5769 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -210,7 +210,7 @@ describe('formula', () => { scale: 'ratio', params: { isFormulaBroken: false, - formula: `terms(category, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, + formula: `average(bytes, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, }, references: [], }); @@ -222,6 +222,7 @@ describe('formula', () => { previousColumn: { ...layer.columns.col1, operationType: 'count', + sourceField: undefined, filter: { language: 'lucene', query: `*`, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 587442b09e4ab3..a102c14537fd02 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -10,7 +10,7 @@ import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; import { - EuiButton, + EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiDescriptionList, @@ -19,6 +19,8 @@ import { EuiModal, EuiModalHeader, EuiPopover, + EuiSelectable, + EuiSelectableOption, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import { @@ -286,10 +288,9 @@ function FormulaEditor({ let errors: ErrorWrapper[] = []; const { root, error } = tryToParse(text); - if (!root) return; if (error) { errors = [error]; - } else { + } else if (root) { const validationErrors = runASTValidation( root, layer, @@ -305,23 +306,42 @@ function FormulaEditor({ monaco.editor.setModelMarkers( editorModel.current, 'LENS', - errors.flatMap((innerError) => - innerError.locations.map((location) => { - const startPosition = offsetToRowColumn(text, location.min); - const endPosition = offsetToRowColumn(text, location.max); - return { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }; - }) - ) + errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }) ); } else { monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); @@ -626,170 +646,253 @@ function FormulaEditor({ }; }, [provideCompletionItems, provideSignatureHelp, provideHover]); - const helpComponent = ( -
- - +
+ + + + + setIsOpen(!isOpen)} + iconType="fullScreen" + size="s" + color="text" + flush="right" + > + {i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'View full screen', + })} + + + +
+
+ { + editor1.current = editor; + registerOnTypeHandler(editor); + }} + /> + +
+
+ + + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + + + {/* Errors go here */} + + + {isOpen ? ( + { + setIsOpen(false); + setText(currentColumn.params.formula); + }} + > + +

+ {i18n.translate('xpack.lens.formula.formulaEditorLabel', { + defaultMessage: 'Formula editor', + })} +

+ + + +
+ { + editor2.current = editor; + registerOnTypeHandler(editor); + }} + /> +
+ + +
+ + {i18n.translate('xpack.lens.formula.functionReferenceLabel', { + defaultMessage: 'Function reference', + })} + + + +
+ + + + ) : null} +
+ + ); +} + +function FormulaHelp({ + indexPattern, + operationDefinitionMap, +}: { + indexPattern: IndexPattern; + operationDefinitionMap: Record; +}) { + const [selectedFunction, setSelectedFunction] = useState(); + + const helpItems: Array = []; + + helpItems.push({ label: 'Math', isGroupLabel: true }); + + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .map((key) => ({ + label: `${key}`, + description: , + checked: selectedFunction === key ? ('on' as const) : undefined, + })) + ); + + helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + + // Es aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in operationDefinitionMap) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + return ( +
+ + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + - - key in tinymathFunctions) - .map((key) => ({ - title: `${key}`, - description: , - }))} - /> - - - - - - {i18n.translate('xpack.lens.formula.elasticsearchFunctions', { - defaultMessage: 'Elasticsearch aggregations', - description: 'Do not translate Elasticsearch', - })} - - key in operationDefinitionMap) - .map((key) => ({ - title: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - }))} - /> -
- ); - - // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences - // in the behavior of Monaco when it's first loaded and then reloaded. - return ( -
- { - editor1.current = editor; - registerOnTypeHandler(editor); - }} - /> - - - - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} iconType="help" size="s"> - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', })} - - } - > - {helpComponent} - - - - - setIsOpen(!isOpen)} iconType="expand" size="s"> - {i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Full screen', - })} - + /> + )} + - - {isOpen ? ( - { - setIsOpen(false); - setText(currentColumn.params.formula); - }} - > - -

- {i18n.translate('xpack.lens.formula.formulaEditorLabel', { - defaultMessage: 'Formula editor', - })} -

- - - -
- { - editor2.current = editor; - registerOnTypeHandler(editor); - }} - /> -
- - -
- - {i18n.translate('xpack.lens.formula.functionReferenceLabel', { - defaultMessage: 'Function reference', - })} - - - {helpComponent} -
- - - - ) : null}
); } +const MemoizedFormulaHelp = React.memo(FormulaHelp); + function parseAndExtract( text: string, layer: IndexPatternLayer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index e24671d9e8e676..0382a7f84668d6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1133,7 +1133,7 @@ export function getManagedColumnsFrom( ): Array<[string, IndexPatternColumn]> { const allNodes: Record = {}; Object.entries(columns).forEach(([id, col]) => { - allNodes[id] = 'references' in col ? col.references : []; + allNodes[id] = 'references' in col ? [...col.references] : []; }); const queue: string[] = allNodes[columnId]; const store: Array<[string, IndexPatternColumn]> = []; From edb522db5eb121cb725a728338cc9529d1ed6254 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 21 Apr 2021 16:13:24 -0400 Subject: [PATCH 071/185] Fix merge issues --- packages/kbn-tinymath/src/grammar.js | 1752 ----------------- .../kibana_react/public/code_editor/index.tsx | 3 - 2 files changed, 1755 deletions(-) delete mode 100644 packages/kbn-tinymath/src/grammar.js diff --git a/packages/kbn-tinymath/src/grammar.js b/packages/kbn-tinymath/src/grammar.js deleted file mode 100644 index c6aa7fd11003b2..00000000000000 --- a/packages/kbn-tinymath/src/grammar.js +++ /dev/null @@ -1,1752 +0,0 @@ -/* - * Generated by PEG.js 0.10.0. - * - * http://pegjs.org/ - */ - -"use strict"; - -function peg$subclass(child, parent) { - function ctor() { this.constructor = child; } - ctor.prototype = parent.prototype; - child.prototype = new ctor(); -} - -function peg$SyntaxError(message, expected, found, location) { - this.message = message; - this.expected = expected; - this.found = found; - this.location = location; - this.name = "SyntaxError"; - - if (typeof Error.captureStackTrace === "function") { - Error.captureStackTrace(this, peg$SyntaxError); - } -} - -peg$subclass(peg$SyntaxError, Error); - -peg$SyntaxError.buildMessage = function(expected, found) { - var DESCRIBE_EXPECTATION_FNS = { - literal: function(expectation) { - return "\"" + literalEscape(expectation.text) + "\""; - }, - - "class": function(expectation) { - var escapedParts = "", - i; - - for (i = 0; i < expectation.parts.length; i++) { - escapedParts += expectation.parts[i] instanceof Array - ? classEscape(expectation.parts[i][0]) + "-" + classEscape(expectation.parts[i][1]) - : classEscape(expectation.parts[i]); - } - - return "[" + (expectation.inverted ? "^" : "") + escapedParts + "]"; - }, - - any: function(expectation) { - return "any character"; - }, - - end: function(expectation) { - return "end of input"; - }, - - other: function(expectation) { - return expectation.description; - } - }; - - function hex(ch) { - return ch.charCodeAt(0).toString(16).toUpperCase(); - } - - function literalEscape(s) { - return s - .replace(/\\/g, '\\\\') - .replace(/"/g, '\\"') - .replace(/\0/g, '\\0') - .replace(/\t/g, '\\t') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/[\x00-\x0F]/g, function(ch) { return '\\x0' + hex(ch); }) - .replace(/[\x10-\x1F\x7F-\x9F]/g, function(ch) { return '\\x' + hex(ch); }); - } - - function classEscape(s) { - return s - .replace(/\\/g, '\\\\') - .replace(/\]/g, '\\]') - .replace(/\^/g, '\\^') - .replace(/-/g, '\\-') - .replace(/\0/g, '\\0') - .replace(/\t/g, '\\t') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/[\x00-\x0F]/g, function(ch) { return '\\x0' + hex(ch); }) - .replace(/[\x10-\x1F\x7F-\x9F]/g, function(ch) { return '\\x' + hex(ch); }); - } - - function describeExpectation(expectation) { - return DESCRIBE_EXPECTATION_FNS[expectation.type](expectation); - } - - function describeExpected(expected) { - var descriptions = new Array(expected.length), - i, j; - - for (i = 0; i < expected.length; i++) { - descriptions[i] = describeExpectation(expected[i]); - } - - descriptions.sort(); - - if (descriptions.length > 0) { - for (i = 1, j = 1; i < descriptions.length; i++) { - if (descriptions[i - 1] !== descriptions[i]) { - descriptions[j] = descriptions[i]; - j++; - } - } - descriptions.length = j; - } - - switch (descriptions.length) { - case 1: - return descriptions[0]; - - case 2: - return descriptions[0] + " or " + descriptions[1]; - - default: - return descriptions.slice(0, -1).join(", ") - + ", or " - + descriptions[descriptions.length - 1]; - } - } - - function describeFound(found) { - return found ? "\"" + literalEscape(found) + "\"" : "end of input"; - } - - return "Expected " + describeExpected(expected) + " but " + describeFound(found) + " found."; -}; - -function peg$parse(input, options) { - options = options !== void 0 ? options : {}; - - var peg$FAILED = {}, - - peg$startRuleFunctions = { start: peg$parsestart }, - peg$startRuleFunction = peg$parsestart, - - peg$c0 = peg$otherExpectation("whitespace"), - peg$c1 = /^[ \t\n\r]/, - peg$c2 = peg$classExpectation([" ", "\t", "\n", "\r"], false, false), - peg$c3 = /^[ ]/, - peg$c4 = peg$classExpectation([" "], false, false), - peg$c5 = /^["']/, - peg$c6 = peg$classExpectation(["\"", "'"], false, false), - peg$c7 = /^[A-Za-z_@.[\]\-]/, - peg$c8 = peg$classExpectation([["A", "Z"], ["a", "z"], "_", "@", ".", "[", "]", "-"], false, false), - peg$c9 = /^[0-9A-Za-z._@[\]\-]/, - peg$c10 = peg$classExpectation([["0", "9"], ["A", "Z"], ["a", "z"], ".", "_", "@", "[", "]", "-"], false, false), - peg$c11 = peg$otherExpectation("literal"), - peg$c12 = function(literal) { - return literal; - }, - peg$c13 = /^[']/, - peg$c14 = peg$classExpectation(["'"], false, false), - peg$c15 = /^["]/, - peg$c16 = peg$classExpectation(["\""], false, false), - peg$c17 = function(chars) { - return { - type: 'variable', - value: chars.join(''), - location: simpleLocation(location()), - text: text() - }; - }, - peg$c18 = function(rest) { - return { - type: 'variable', - value: rest.join(''), - location: simpleLocation(location()), - text: text() - }; - }, - peg$c19 = "+", - peg$c20 = peg$literalExpectation("+", false), - peg$c21 = "-", - peg$c22 = peg$literalExpectation("-", false), - peg$c23 = function(left, rest) { - return rest.reduce((acc, curr) => ({ - type: 'function', - name: curr[0] === '+' ? 'add' : 'subtract', - args: [acc, curr[1]], - location: simpleLocation(location()), - text: text() - }), left) - }, - peg$c24 = "*", - peg$c25 = peg$literalExpectation("*", false), - peg$c26 = "/", - peg$c27 = peg$literalExpectation("/", false), - peg$c28 = function(left, rest) { - return rest.reduce((acc, curr) => ({ - type: 'function', - name: curr[0] === '*' ? 'multiply' : 'divide', - args: [acc, curr[1]], - location: simpleLocation(location()), - text: text() - }), left) - }, - peg$c29 = "(", - peg$c30 = peg$literalExpectation("(", false), - peg$c31 = ")", - peg$c32 = peg$literalExpectation(")", false), - peg$c33 = function(expr) { - return expr - }, - peg$c34 = peg$otherExpectation("arguments"), - peg$c35 = ",", - peg$c36 = peg$literalExpectation(",", false), - peg$c37 = function(first, arg) {return arg}, - peg$c38 = function(first, rest) { - return [first].concat(rest); - }, - peg$c39 = "\"", - peg$c40 = peg$literalExpectation("\"", false), - peg$c41 = "\\\"", - peg$c42 = peg$literalExpectation("\\\"", false), - peg$c43 = function() { return "\""; }, - peg$c44 = /^[^"]/, - peg$c45 = peg$classExpectation(["\""], true, false), - peg$c46 = function(chars) { return chars.join(''); }, - peg$c47 = "'", - peg$c48 = peg$literalExpectation("'", false), - peg$c49 = "\\'", - peg$c50 = peg$literalExpectation("\\'", false), - peg$c51 = function() { return "\'"; }, - peg$c52 = /^[^']/, - peg$c53 = peg$classExpectation(["'"], true, false), - peg$c54 = /^[a-zA-Z_]/, - peg$c55 = peg$classExpectation([["a", "z"], ["A", "Z"], "_"], false, false), - peg$c56 = "=", - peg$c57 = peg$literalExpectation("=", false), - peg$c58 = function(name, value) { - return { - type: 'namedArgument', - name: name.join(''), - value: value, - location: simpleLocation(location()), - text: text() - }; - }, - peg$c59 = peg$otherExpectation("function"), - peg$c60 = /^[a-zA-Z_\-]/, - peg$c61 = peg$classExpectation([["a", "z"], ["A", "Z"], "_", "-"], false, false), - peg$c62 = function(name, args) { - return { - type: 'function', - name: name.join(''), - args: args || [], - location: simpleLocation(location()), - text: text() - }; - }, - peg$c63 = peg$otherExpectation("number"), - peg$c64 = function() { - return parseFloat(text()); - }, - peg$c65 = /^[eE]/, - peg$c66 = peg$classExpectation(["e", "E"], false, false), - peg$c67 = peg$otherExpectation("exponent"), - peg$c68 = ".", - peg$c69 = peg$literalExpectation(".", false), - peg$c70 = "0", - peg$c71 = peg$literalExpectation("0", false), - peg$c72 = /^[1-9]/, - peg$c73 = peg$classExpectation([["1", "9"]], false, false), - peg$c74 = /^[0-9]/, - peg$c75 = peg$classExpectation([["0", "9"]], false, false), - - peg$currPos = 0, - peg$savedPos = 0, - peg$posDetailsCache = [{ line: 1, column: 1 }], - peg$maxFailPos = 0, - peg$maxFailExpected = [], - peg$silentFails = 0, - - peg$result; - - if ("startRule" in options) { - if (!(options.startRule in peg$startRuleFunctions)) { - throw new Error("Can't start parsing from rule \"" + options.startRule + "\"."); - } - - peg$startRuleFunction = peg$startRuleFunctions[options.startRule]; - } - - function text() { - return input.substring(peg$savedPos, peg$currPos); - } - - function location() { - return peg$computeLocation(peg$savedPos, peg$currPos); - } - - function expected(description, location) { - location = location !== void 0 ? location : peg$computeLocation(peg$savedPos, peg$currPos) - - throw peg$buildStructuredError( - [peg$otherExpectation(description)], - input.substring(peg$savedPos, peg$currPos), - location - ); - } - - function error(message, location) { - location = location !== void 0 ? location : peg$computeLocation(peg$savedPos, peg$currPos) - - throw peg$buildSimpleError(message, location); - } - - function peg$literalExpectation(text, ignoreCase) { - return { type: "literal", text: text, ignoreCase: ignoreCase }; - } - - function peg$classExpectation(parts, inverted, ignoreCase) { - return { type: "class", parts: parts, inverted: inverted, ignoreCase: ignoreCase }; - } - - function peg$anyExpectation() { - return { type: "any" }; - } - - function peg$endExpectation() { - return { type: "end" }; - } - - function peg$otherExpectation(description) { - return { type: "other", description: description }; - } - - function peg$computePosDetails(pos) { - var details = peg$posDetailsCache[pos], p; - - if (details) { - return details; - } else { - p = pos - 1; - while (!peg$posDetailsCache[p]) { - p--; - } - - details = peg$posDetailsCache[p]; - details = { - line: details.line, - column: details.column - }; - - while (p < pos) { - if (input.charCodeAt(p) === 10) { - details.line++; - details.column = 1; - } else { - details.column++; - } - - p++; - } - - peg$posDetailsCache[pos] = details; - return details; - } - } - - function peg$computeLocation(startPos, endPos) { - var startPosDetails = peg$computePosDetails(startPos), - endPosDetails = peg$computePosDetails(endPos); - - return { - start: { - offset: startPos, - line: startPosDetails.line, - column: startPosDetails.column - }, - end: { - offset: endPos, - line: endPosDetails.line, - column: endPosDetails.column - } - }; - } - - function peg$fail(expected) { - if (peg$currPos < peg$maxFailPos) { return; } - - if (peg$currPos > peg$maxFailPos) { - peg$maxFailPos = peg$currPos; - peg$maxFailExpected = []; - } - - peg$maxFailExpected.push(expected); - } - - function peg$buildSimpleError(message, location) { - return new peg$SyntaxError(message, null, null, location); - } - - function peg$buildStructuredError(expected, found, location) { - return new peg$SyntaxError( - peg$SyntaxError.buildMessage(expected, found), - expected, - found, - location - ); - } - - function peg$parsestart() { - var s0; - - s0 = peg$parseAddSubtract(); - - return s0; - } - - function peg$parse_() { - var s0, s1; - - peg$silentFails++; - s0 = []; - if (peg$c1.test(input.charAt(peg$currPos))) { - s1 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c2); } - } - while (s1 !== peg$FAILED) { - s0.push(s1); - if (peg$c1.test(input.charAt(peg$currPos))) { - s1 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c2); } - } - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c0); } - } - - return s0; - } - - function peg$parseSpace() { - var s0; - - if (peg$c3.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c4); } - } - - return s0; - } - - function peg$parseQuote() { - var s0; - - if (peg$c5.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c6); } - } - - return s0; - } - - function peg$parseStartChar() { - var s0; - - if (peg$c7.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c8); } - } - - return s0; - } - - function peg$parseValidChar() { - var s0; - - if (peg$c9.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c10); } - } - - return s0; - } - - function peg$parseLiteral() { - var s0, s1, s2, s3; - - peg$silentFails++; - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = peg$parseNumber(); - if (s2 === peg$FAILED) { - s2 = peg$parseVariable(); - } - if (s2 !== peg$FAILED) { - s3 = peg$parse_(); - if (s3 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c12(s2); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c11); } - } - - return s0; - } - - function peg$parseVariable() { - var s0, s1, s2, s3, s4, s5; - - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - if (peg$c13.test(input.charAt(peg$currPos))) { - s2 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } - } - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$parseValidChar(); - if (s4 === peg$FAILED) { - s4 = peg$parseSpace(); - if (s4 === peg$FAILED) { - if (peg$c15.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } - } - } - } - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$parseValidChar(); - if (s4 === peg$FAILED) { - s4 = peg$parseSpace(); - if (s4 === peg$FAILED) { - if (peg$c15.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } - } - } - } - } - if (s3 !== peg$FAILED) { - if (peg$c13.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } - } - if (s4 !== peg$FAILED) { - s5 = peg$parse_(); - if (s5 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c17(s3); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - if (s0 === peg$FAILED) { - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - if (peg$c15.test(input.charAt(peg$currPos))) { - s2 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } - } - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$parseValidChar(); - if (s4 === peg$FAILED) { - s4 = peg$parseSpace(); - if (s4 === peg$FAILED) { - if (peg$c13.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } - } - } - } - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$parseValidChar(); - if (s4 === peg$FAILED) { - s4 = peg$parseSpace(); - if (s4 === peg$FAILED) { - if (peg$c13.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c14); } - } - } - } - } - if (s3 !== peg$FAILED) { - if (peg$c15.test(input.charAt(peg$currPos))) { - s4 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c16); } - } - if (s4 !== peg$FAILED) { - s5 = peg$parse_(); - if (s5 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c17(s3); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - if (s0 === peg$FAILED) { - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$parseValidChar(); - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseValidChar(); - } - } else { - s2 = peg$FAILED; - } - if (s2 !== peg$FAILED) { - s3 = peg$parse_(); - if (s3 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c18(s2); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } - } - - return s0; - } - - function peg$parseAddSubtract() { - var s0, s1, s2, s3, s4, s5, s6; - - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = peg$parseMultiplyDivide(); - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 43) { - s5 = peg$c19; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c20); } - } - if (s5 === peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 45) { - s5 = peg$c21; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c22); } - } - } - if (s5 !== peg$FAILED) { - s6 = peg$parseMultiplyDivide(); - if (s6 !== peg$FAILED) { - s5 = [s5, s6]; - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 43) { - s5 = peg$c19; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c20); } - } - if (s5 === peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 45) { - s5 = peg$c21; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c22); } - } - } - if (s5 !== peg$FAILED) { - s6 = peg$parseMultiplyDivide(); - if (s6 !== peg$FAILED) { - s5 = [s5, s6]; - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } - if (s3 !== peg$FAILED) { - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c23(s2, s3); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - - return s0; - } - - function peg$parseMultiplyDivide() { - var s0, s1, s2, s3, s4, s5, s6; - - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = peg$parseFactor(); - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 42) { - s5 = peg$c24; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c25); } - } - if (s5 === peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 47) { - s5 = peg$c26; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c27); } - } - } - if (s5 !== peg$FAILED) { - s6 = peg$parseFactor(); - if (s6 !== peg$FAILED) { - s5 = [s5, s6]; - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 42) { - s5 = peg$c24; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c25); } - } - if (s5 === peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 47) { - s5 = peg$c26; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c27); } - } - } - if (s5 !== peg$FAILED) { - s6 = peg$parseFactor(); - if (s6 !== peg$FAILED) { - s5 = [s5, s6]; - s4 = s5; - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } else { - peg$currPos = s4; - s4 = peg$FAILED; - } - } - if (s3 !== peg$FAILED) { - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c28(s2, s3); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - - return s0; - } - - function peg$parseFactor() { - var s0; - - s0 = peg$parseGroup(); - if (s0 === peg$FAILED) { - s0 = peg$parseFunction(); - if (s0 === peg$FAILED) { - s0 = peg$parseLiteral(); - } - } - - return s0; - } - - function peg$parseGroup() { - var s0, s1, s2, s3, s4, s5, s6, s7; - - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 40) { - s2 = peg$c29; - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c30); } - } - if (s2 !== peg$FAILED) { - s3 = peg$parse_(); - if (s3 !== peg$FAILED) { - s4 = peg$parseAddSubtract(); - if (s4 !== peg$FAILED) { - s5 = peg$parse_(); - if (s5 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 41) { - s6 = peg$c31; - peg$currPos++; - } else { - s6 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } - } - if (s6 !== peg$FAILED) { - s7 = peg$parse_(); - if (s7 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c33(s4); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - - return s0; - } - - function peg$parseArgument_List() { - var s0, s1, s2, s3, s4, s5, s6, s7; - - peg$silentFails++; - s0 = peg$currPos; - s1 = peg$parseArgument(); - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$currPos; - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s5 = peg$c35; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c36); } - } - if (s5 !== peg$FAILED) { - s6 = peg$parse_(); - if (s6 !== peg$FAILED) { - s7 = peg$parseArgument(); - if (s7 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c37(s1, s7); - s3 = s4; - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$currPos; - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s5 = peg$c35; - peg$currPos++; - } else { - s5 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c36); } - } - if (s5 !== peg$FAILED) { - s6 = peg$parse_(); - if (s6 !== peg$FAILED) { - s7 = peg$parseArgument(); - if (s7 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c37(s1, s7); - s3 = s4; - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } else { - peg$currPos = s3; - s3 = peg$FAILED; - } - } - if (s2 !== peg$FAILED) { - s3 = peg$parse_(); - if (s3 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 44) { - s4 = peg$c35; - peg$currPos++; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c36); } - } - if (s4 === peg$FAILED) { - s4 = null; - } - if (s4 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c38(s1, s2); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c34); } - } - - return s0; - } - - function peg$parseString() { - var s0, s1, s2, s3, s4; - - s0 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 34) { - s1 = peg$c39; - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c40); } - } - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$currPos; - if (input.substr(peg$currPos, 2) === peg$c41) { - s4 = peg$c41; - peg$currPos += 2; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c42); } - } - if (s4 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c43(); - } - s3 = s4; - if (s3 === peg$FAILED) { - if (peg$c44.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } - } - } - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$currPos; - if (input.substr(peg$currPos, 2) === peg$c41) { - s4 = peg$c41; - peg$currPos += 2; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c42); } - } - if (s4 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c43(); - } - s3 = s4; - if (s3 === peg$FAILED) { - if (peg$c44.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c45); } - } - } - } - if (s2 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 34) { - s3 = peg$c39; - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c40); } - } - if (s3 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c46(s2); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - if (s0 === peg$FAILED) { - s0 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 39) { - s1 = peg$c47; - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c48); } - } - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$currPos; - if (input.substr(peg$currPos, 2) === peg$c49) { - s4 = peg$c49; - peg$currPos += 2; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c50); } - } - if (s4 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c51(); - } - s3 = s4; - if (s3 === peg$FAILED) { - if (peg$c52.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c53); } - } - } - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$currPos; - if (input.substr(peg$currPos, 2) === peg$c49) { - s4 = peg$c49; - peg$currPos += 2; - } else { - s4 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c50); } - } - if (s4 !== peg$FAILED) { - peg$savedPos = s3; - s4 = peg$c51(); - } - s3 = s4; - if (s3 === peg$FAILED) { - if (peg$c52.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c53); } - } - } - } - if (s2 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 39) { - s3 = peg$c47; - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c48); } - } - if (s3 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c46(s2); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - if (s0 === peg$FAILED) { - s0 = peg$currPos; - s1 = []; - s2 = peg$parseValidChar(); - if (s2 !== peg$FAILED) { - while (s2 !== peg$FAILED) { - s1.push(s2); - s2 = peg$parseValidChar(); - } - } else { - s1 = peg$FAILED; - } - if (s1 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c46(s1); - } - s0 = s1; - } - } - - return s0; - } - - function peg$parseArgument() { - var s0, s1, s2, s3, s4, s5, s6; - - s0 = peg$currPos; - s1 = []; - if (peg$c54.test(input.charAt(peg$currPos))) { - s2 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c55); } - } - if (s2 !== peg$FAILED) { - while (s2 !== peg$FAILED) { - s1.push(s2); - if (peg$c54.test(input.charAt(peg$currPos))) { - s2 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c55); } - } - } - } else { - s1 = peg$FAILED; - } - if (s1 !== peg$FAILED) { - s2 = peg$parse_(); - if (s2 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 61) { - s3 = peg$c56; - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c57); } - } - if (s3 !== peg$FAILED) { - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - s5 = peg$parseNumber(); - if (s5 === peg$FAILED) { - s5 = peg$parseString(); - } - if (s5 !== peg$FAILED) { - s6 = peg$parse_(); - if (s6 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c58(s1, s5); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - if (s0 === peg$FAILED) { - s0 = peg$parseAddSubtract(); - } - - return s0; - } - - function peg$parseFunction() { - var s0, s1, s2, s3, s4, s5, s6, s7, s8; - - peg$silentFails++; - s0 = peg$currPos; - s1 = peg$parse_(); - if (s1 !== peg$FAILED) { - s2 = []; - if (peg$c60.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c61); } - } - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - if (peg$c60.test(input.charAt(peg$currPos))) { - s3 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c61); } - } - } - } else { - s2 = peg$FAILED; - } - if (s2 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 40) { - s3 = peg$c29; - peg$currPos++; - } else { - s3 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c30); } - } - if (s3 !== peg$FAILED) { - s4 = peg$parse_(); - if (s4 !== peg$FAILED) { - s5 = peg$parseArgument_List(); - if (s5 === peg$FAILED) { - s5 = null; - } - if (s5 !== peg$FAILED) { - s6 = peg$parse_(); - if (s6 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 41) { - s7 = peg$c31; - peg$currPos++; - } else { - s7 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c32); } - } - if (s7 !== peg$FAILED) { - s8 = peg$parse_(); - if (s8 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c62(s2, s5); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c59); } - } - - return s0; - } - - function peg$parseNumber() { - var s0, s1, s2, s3, s4; - - peg$silentFails++; - s0 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 45) { - s1 = peg$c21; - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c22); } - } - if (s1 === peg$FAILED) { - s1 = null; - } - if (s1 !== peg$FAILED) { - s2 = peg$parseInteger(); - if (s2 !== peg$FAILED) { - s3 = peg$parseFraction(); - if (s3 === peg$FAILED) { - s3 = null; - } - if (s3 !== peg$FAILED) { - s4 = peg$parseExp(); - if (s4 === peg$FAILED) { - s4 = null; - } - if (s4 !== peg$FAILED) { - peg$savedPos = s0; - s1 = peg$c64(); - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c63); } - } - - return s0; - } - - function peg$parseE() { - var s0; - - if (peg$c65.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c66); } - } - - return s0; - } - - function peg$parseExp() { - var s0, s1, s2, s3, s4; - - peg$silentFails++; - s0 = peg$currPos; - s1 = peg$parseE(); - if (s1 !== peg$FAILED) { - if (input.charCodeAt(peg$currPos) === 45) { - s2 = peg$c21; - peg$currPos++; - } else { - s2 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c22); } - } - if (s2 === peg$FAILED) { - s2 = null; - } - if (s2 !== peg$FAILED) { - s3 = []; - s4 = peg$parseDigit(); - if (s4 !== peg$FAILED) { - while (s4 !== peg$FAILED) { - s3.push(s4); - s4 = peg$parseDigit(); - } - } else { - s3 = peg$FAILED; - } - if (s3 !== peg$FAILED) { - s1 = [s1, s2, s3]; - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - peg$silentFails--; - if (s0 === peg$FAILED) { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c67); } - } - - return s0; - } - - function peg$parseFraction() { - var s0, s1, s2, s3; - - s0 = peg$currPos; - if (input.charCodeAt(peg$currPos) === 46) { - s1 = peg$c68; - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c69); } - } - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$parseDigit(); - if (s3 !== peg$FAILED) { - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseDigit(); - } - } else { - s2 = peg$FAILED; - } - if (s2 !== peg$FAILED) { - s1 = [s1, s2]; - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - - return s0; - } - - function peg$parseInteger() { - var s0, s1, s2, s3; - - if (input.charCodeAt(peg$currPos) === 48) { - s0 = peg$c70; - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c71); } - } - if (s0 === peg$FAILED) { - s0 = peg$currPos; - if (peg$c72.test(input.charAt(peg$currPos))) { - s1 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s1 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c73); } - } - if (s1 !== peg$FAILED) { - s2 = []; - s3 = peg$parseDigit(); - while (s3 !== peg$FAILED) { - s2.push(s3); - s3 = peg$parseDigit(); - } - if (s2 !== peg$FAILED) { - s1 = [s1, s2]; - s0 = s1; - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } else { - peg$currPos = s0; - s0 = peg$FAILED; - } - } - - return s0; - } - - function peg$parseDigit() { - var s0; - - if (peg$c74.test(input.charAt(peg$currPos))) { - s0 = input.charAt(peg$currPos); - peg$currPos++; - } else { - s0 = peg$FAILED; - if (peg$silentFails === 0) { peg$fail(peg$c75); } - } - - return s0; - } - - - function simpleLocation (location) { - // Returns an object representing the position of the function within the expression, - // demarcated by the position of its first character and last character. We calculate these values - // using the offset because the expression could span multiple lines, and we don't want to deal - // with column and line values. - return { - min: location.start.offset, - max: location.end.offset - } - } - - - peg$result = peg$startRuleFunction(); - - if (peg$result !== peg$FAILED && peg$currPos === input.length) { - return peg$result; - } else { - if (peg$result !== peg$FAILED && peg$currPos < input.length) { - peg$fail(peg$endExpectation()); - } - - throw peg$buildStructuredError( - peg$maxFailExpected, - peg$maxFailPos < input.length ? input.charAt(peg$maxFailPos) : null, - peg$maxFailPos < input.length - ? peg$computeLocation(peg$maxFailPos, peg$maxFailPos + 1) - : peg$computeLocation(peg$maxFailPos, peg$maxFailPos) - ); - } -} - -module.exports = { - SyntaxError: peg$SyntaxError, - parse: peg$parse -}; diff --git a/src/plugins/kibana_react/public/code_editor/index.tsx b/src/plugins/kibana_react/public/code_editor/index.tsx index 84b9fbbc87ba49..ec0f95b8a60d73 100644 --- a/src/plugins/kibana_react/public/code_editor/index.tsx +++ b/src/plugins/kibana_react/public/code_editor/index.tsx @@ -26,16 +26,13 @@ const Fallback = () => ( ); -<<<<<<< HEAD export type CodeEditorProps = Props; -======= /** * Renders a Monaco code editor with EUI color theme. * * @see CodeEditorField to render a code editor in the same style as other EUI form fields. */ ->>>>>>> origin/master export const CodeEditor: React.FunctionComponent = (props) => { const darkMode = useUiSetting('theme:darkMode'); return ( From 71696553aaf04bd2c6a29b243a613af1a9017e50 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 27 Apr 2021 16:33:54 -0400 Subject: [PATCH 072/185] Update grammar for formulas --- packages/kbn-tinymath/grammar/grammar.peggy | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/kbn-tinymath/grammar/grammar.peggy b/packages/kbn-tinymath/grammar/grammar.peggy index d80d7b1c426aaa..70f275776e45dc 100644 --- a/packages/kbn-tinymath/grammar/grammar.peggy +++ b/packages/kbn-tinymath/grammar/grammar.peggy @@ -43,15 +43,7 @@ Literal "literal" // Quoted variables are interpreted as strings // but unquoted variables are more restrictive Variable - = _ [\'] chars:(ValidChar / Space / [\"])* [\'] _ { - return { - type: 'variable', - value: chars.join(''), - location: simpleLocation(location()), - text: text() - }; - } - / _ [\"] chars:(ValidChar / Space / [\'])* [\"] _ { + = _ Quote chars:(ValidChar / Space)* Quote _ { return { type: 'variable', value: chars.join(''), @@ -111,9 +103,10 @@ Argument_List "arguments" } String - = '"' chars:("\\\"" { return "\""; } / [^"])* '"' { return chars.join(''); } - / "'" chars:("\\\'" { return "\'"; } / [^'])* "'" { return chars.join(''); } - / chars:(ValidChar)+ { return chars.join(''); } + = [\"] value:(ValidChar)+ [\"] { return value.join(''); } + / [\'] value:(ValidChar)+ [\'] { return value.join(''); } + / value:(ValidChar)+ { return value.join(''); } + Argument = name:[a-zA-Z_]+ _ '=' _ value:(Number / String) _ { From e01bca811a046d0a1684de56fb2f4c4b7ea115ea Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 27 Apr 2021 18:53:40 -0400 Subject: [PATCH 073/185] Fix bad merge --- packages/kbn-tinymath/grammar/grammar.peggy | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/kbn-tinymath/grammar/grammar.peggy b/packages/kbn-tinymath/grammar/grammar.peggy index 70f275776e45dc..cbcb0b91bfea90 100644 --- a/packages/kbn-tinymath/grammar/grammar.peggy +++ b/packages/kbn-tinymath/grammar/grammar.peggy @@ -43,7 +43,7 @@ Literal "literal" // Quoted variables are interpreted as strings // but unquoted variables are more restrictive Variable - = _ Quote chars:(ValidChar / Space)* Quote _ { + = _ [\'] chars:(ValidChar / Space / [\"])* [\'] _ { return { type: 'variable', value: chars.join(''), @@ -51,6 +51,14 @@ Variable text: text() }; } + / _ [\"] chars:(ValidChar / Space / [\'])* [\"] _ { + return { + type: 'variable', + value: chars.join(''), + location: simpleLocation(location()), + text: text() + }; + } / _ rest:ValidChar+ _ { return { type: 'variable', @@ -103,10 +111,9 @@ Argument_List "arguments" } String - = [\"] value:(ValidChar)+ [\"] { return value.join(''); } - / [\'] value:(ValidChar)+ [\'] { return value.join(''); } - / value:(ValidChar)+ { return value.join(''); } - + = '"' chars:("\\\"" { return "\""; } / [^"])* '"' { return chars.join(''); } + / "'" chars:("\\\'" { return "\'"; } / [^'])* "'" { return chars.join(''); } + / chars:(ValidChar)+ { return chars.join(''); } Argument = name:[a-zA-Z_]+ _ '=' _ value:(Number / String) _ { From ec493d1cd3939f1e3ffefa67c722a7564a46b446 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 28 Apr 2021 10:55:01 -0400 Subject: [PATCH 074/185] Rough fullscreen mode --- .../config_panel/config_panel.tsx | 9 ++++ .../editor_frame/config_panel/layer_panel.tsx | 3 ++ .../editor_frame/editor_frame.tsx | 1 + .../editor_frame/frame_layout.scss | 4 ++ .../editor_frame/frame_layout.tsx | 31 ++++++----- .../editor_frame/state_management.ts | 6 +++ .../dimension_panel/dimension_editor.tsx | 4 ++ .../dimension_panel/reference_editor.tsx | 3 ++ .../definitions/formula/formula.tsx | 51 +++++++++++-------- .../operations/definitions/index.ts | 1 + x-pack/plugins/lens/public/types.ts | 1 + 11 files changed, 82 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index d52fd29e7233a4..0c26530b6172ca 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -96,6 +96,14 @@ export function LayerPanels( }, [dispatch] ); + const toggleFullscreen = useMemo( + () => () => { + dispatch({ + type: 'TOGGLE_FULLSCREEN', + }); + }, + [dispatch] + ); const datasourcePublicAPIs = props.framePublicAPI.datasourceLayers; @@ -130,6 +138,7 @@ export function LayerPanels( }); removeLayerRef(layerId); }} + toggleFullscreen={toggleFullscreen} /> ) : null )} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index cf3c9099d4b0dd..47055ae70fd7a5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -49,6 +49,7 @@ export function LayerPanel( ) => void; onRemoveLayer: () => void; registerNewLayerRef: (layerId: string, instance: HTMLDivElement | null) => void; + toggleFullscreen: () => void; } ) { const [activeDimension, setActiveDimension] = useState( @@ -65,6 +66,7 @@ export function LayerPanel( activeVisualization, updateVisualization, updateDatasource, + toggleFullscreen, } = props; const datasourcePublicAPI = framePublicAPI.datasourceLayers[layerId]; @@ -433,6 +435,7 @@ export function LayerPanel( hideGrouping: activeGroup.hideGrouping, filterOperations: activeGroup.filterOperations, dimensionGroups: groups, + toggleFullscreen, setState: ( newState: unknown, { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 362787ea91c4fb..d642526893e68a 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -310,6 +310,7 @@ export function EditorFrame(props: EditorFrameProps) { return ( -
- -

- {i18n.translate('xpack.lens.section.dataPanelLabel', { - defaultMessage: 'Data panel', - })} -

- - {props.dataPanel} -
+ {!props.isFullscreen ? ( +
+ +

+ {i18n.translate('xpack.lens.section.dataPanelLabel', { + defaultMessage: 'Data panel', + })} +

+ + {props.dataPanel} +
+ ) : null}

@@ -45,10 +49,13 @@ export function FrameLayout(props: FrameLayoutProps) {

{props.workspacePanel} - {props.suggestionsPanel} + {!props.isFullscreen ? props.suggestionsPanel : null}
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index 53aba0d6f3f6c1..522f1103c927bf 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,6 +24,7 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; + isFullscreenDatasource?: boolean; } export type Action = @@ -96,6 +97,9 @@ export type Action = | { type: 'SWITCH_DATASOURCE'; newDatasourceId: string; + } + | { + type: 'TOGGLE_FULLSCREEN'; }; export function getActiveDatasourceIdFromDoc(doc?: Document) { @@ -290,6 +294,8 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta }, stagedPreview: action.clearStagedPreview ? undefined : state.stagedPreview, }; + case 'TOGGLE_FULLSCREEN': + return { ...state, isFullscreenDatasource: !state.isFullscreenDatasource }; default: return state; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ec53285454ddf5..42e847f719c9de 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -121,6 +121,7 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, + toggleFullscreen, } = props; const services = { data: props.data, @@ -390,6 +391,7 @@ export function DimensionEditor(props: DimensionEditorProps) { currentColumn: state.layers[layerId].columns[columnId], })} dimensionGroups={dimensionGroups} + toggleFullscreen={toggleFullscreen} {...services} /> ); @@ -470,6 +472,7 @@ export function DimensionEditor(props: DimensionEditorProps) { dateRange={dateRange} indexPattern={currentIndexPattern} operationDefinitionMap={operationDefinitionMap} + toggleFullscreen={toggleFullscreen} {...services} /> )} @@ -563,6 +566,7 @@ export function DimensionEditor(props: DimensionEditorProps) { dateRange={dateRange} indexPattern={currentIndexPattern} operationDefinitionMap={operationDefinitionMap} + toggleFullscreen={toggleFullscreen} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index b8a065b088467d..fc949fb5442aa2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -50,6 +50,7 @@ export interface ReferenceEditorProps { dateRange: DateRange; labelAppend?: EuiFormRowProps['labelAppend']; dimensionGroups: VisualizationDimensionGroupConfig[]; + toggleFullscreen: () => void; // Services uiSettings: IUiSettingsClient; @@ -71,6 +72,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { dateRange, labelAppend, dimensionGroups, + toggleFullscreen, ...services } = props; @@ -345,6 +347,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { indexPattern={currentIndexPattern} dateRange={dateRange} operationDefinitionMap={operationDefinitionMap} + toggleFullscreen={toggleFullscreen} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index a102c14537fd02..17c562aac94832 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -230,6 +230,7 @@ function FormulaEditor({ indexPattern, operationDefinitionMap, data, + toggleFullscreen, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [isOpen, setIsOpen] = useState(false); @@ -656,7 +657,10 @@ function FormulaEditor({ setIsOpen(!isOpen)} + onClick={() => { + setIsOpen(!isOpen); + toggleFullscreen(); + }} iconType="fullScreen" size="s" color="text" @@ -690,34 +694,41 @@ function FormulaEditor({
- setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" - > + {isOpen ? ( - + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} {/* Errors go here */} - {isOpen ? ( + {false ? ( { setIsOpen(false); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index da42793fb2eedd..e2798d31024345 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -150,6 +150,7 @@ export interface ParamEditorProps { currentColumn: C; layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; + toggleFullscreen: () => void; columnId: string; indexPattern: IndexPattern; uiSettings: IUiSettingsClient; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 94b4433a825510..cdfc0d9cd7d4e7 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -301,6 +301,7 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro core: Pick; dateRange: DateRange; dimensionGroups: VisualizationDimensionGroupConfig[]; + toggleFullscreen: () => void; }; export type DatasourceDimensionTriggerProps = DatasourceDimensionProps; From 90cff4a64a4589d4886264676b8c55a661e9e452 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 28 Apr 2021 17:13:39 -0400 Subject: [PATCH 075/185] Type updates --- .../operations/definitions/formula/formula.tsx | 1 - .../lens/public/indexpattern_datasource/to_expression.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 17c562aac94832..abdcc55f5001ba 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -13,7 +13,6 @@ import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, - EuiDescriptionList, EuiText, EuiSpacer, EuiModal, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 312d933a5a1e83..81d890f62843d9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -77,7 +77,7 @@ function getExpressionForLayer( esAggEntries.forEach(([colId, col]) => { const def = operationDefinitionMap[col.operationType]; - if (def.input !== 'fullReference') { + if (def.input !== 'fullReference' && def.input !== 'managedReference') { const wrapInFilter = Boolean(def.filterable && col.filter); let aggAst = def.toEsAggsFn( col, From 12391f65b7b17c8cda8e33820ebef1df2395a144 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 28 Apr 2021 17:55:47 -0400 Subject: [PATCH 076/185] Pass through fullscreen state --- .../editor_frame/config_panel/layer_panel.tsx | 7 +++++++ .../editor_frame/config_panel/types.ts | 2 ++ .../editor_frame/editor_frame.tsx | 3 ++- .../editor_frame/frame_layout.scss | 1 + .../dimension_panel/dimension_editor.tsx | 3 +++ .../operations/definitions/formula/formula.tsx | 15 +++++++-------- .../operations/definitions/index.ts | 1 + x-pack/plugins/lens/public/types.ts | 1 + 8 files changed, 24 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 47055ae70fd7a5..ffccc90b6a2083 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -50,6 +50,7 @@ export function LayerPanel( onRemoveLayer: () => void; registerNewLayerRef: (layerId: string, instance: HTMLDivElement | null) => void; toggleFullscreen: () => void; + isFullscreen: boolean; } ) { const [activeDimension, setActiveDimension] = useState( @@ -67,6 +68,7 @@ export function LayerPanel( updateVisualization, updateDatasource, toggleFullscreen, + isFullscreen, } = props; const datasourcePublicAPI = framePublicAPI.datasourceLayers[layerId]; @@ -421,6 +423,9 @@ export function LayerPanel( } } setActiveDimension(initialActiveDimensionState); + if (isFullscreen) { + toggleFullscreen(); + } }} panel={ <> @@ -436,6 +441,7 @@ export function LayerPanel( filterOperations: activeGroup.filterOperations, dimensionGroups: groups, toggleFullscreen, + isFullscreen, setState: ( newState: unknown, { @@ -476,6 +482,7 @@ export function LayerPanel( )} {activeGroup && activeId && + !isFullscreen && !activeDimension.isNew && activeVisualization.renderDimensionEditor && activeGroup?.enableDimensionEditor && ( diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts index 37b2198cfd51f9..1af8c16fa1395f 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts @@ -29,6 +29,7 @@ export interface ConfigPanelWrapperProps { } >; core: DatasourceDimensionEditorProps['core']; + isFullscreen: boolean; } export interface LayerPanelProps { @@ -46,6 +47,7 @@ export interface LayerPanelProps { } >; core: DatasourceDimensionEditorProps['core']; + isFullscreen: boolean; } export interface LayerDatasourceDropProps { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index d642526893e68a..17db00892a51f1 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -310,7 +310,7 @@ export function EditorFrame(props: EditorFrameProps) { return ( ) } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss index aab0e24d9aebcf..98d636b9a56ba4 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss @@ -109,4 +109,5 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ .lnsFrameLayout__sidebar--fullscreen { flex-basis: 50%; + max-width: calc(50%); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 42e847f719c9de..05da746db0e600 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -122,6 +122,7 @@ export function DimensionEditor(props: DimensionEditorProps) { dateRange, dimensionGroups, toggleFullscreen, + isFullscreen, } = props; const services = { data: props.data, @@ -473,6 +474,7 @@ export function DimensionEditor(props: DimensionEditorProps) { indexPattern={currentIndexPattern} operationDefinitionMap={operationDefinitionMap} toggleFullscreen={toggleFullscreen} + isFullscreen={isFullscreen} {...services} /> )} @@ -567,6 +569,7 @@ export function DimensionEditor(props: DimensionEditorProps) { indexPattern={currentIndexPattern} operationDefinitionMap={operationDefinitionMap} toggleFullscreen={toggleFullscreen} + isFullscreen={isFullscreen} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index abdcc55f5001ba..66720e49cb6fe2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -230,9 +230,9 @@ function FormulaEditor({ operationDefinitionMap, data, toggleFullscreen, + isFullscreen, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); - const [isOpen, setIsOpen] = useState(false); const [isHelpOpen, setIsHelpOpen] = useState(false); const editorModel = React.useRef( monaco.editor.createModel(text ?? '', LANGUAGE_ID) @@ -599,13 +599,13 @@ function FormulaEditor({ // while it has focus. useEffect(() => { if (updateAfterTyping.current) { - if (isOpen) { + if (isFullscreen) { if (editor2.current) registerOnTypeHandler(editor2.current); } else { if (editor1.current) registerOnTypeHandler(editor1.current); } } - }, [isOpen, registerOnTypeHandler]); + }, [isFullscreen, registerOnTypeHandler]); const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, @@ -621,7 +621,7 @@ function FormulaEditor({ wordWrap: 'on', // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, - dimension: { width: 290, height: 280 }, + dimension: { width: 290, height: isFullscreen ? 400 : 280 }, fixedOverflowWidgets: true, }, }; @@ -657,7 +657,6 @@ function FormulaEditor({ { - setIsOpen(!isOpen); toggleFullscreen(); }} iconType="fullScreen" @@ -675,7 +674,7 @@ function FormulaEditor({
- {isOpen ? ( + {isFullscreen ? ( { - setIsOpen(false); + // setIsOpen(false); setText(currentColumn.params.formula); }} > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index e2798d31024345..8283e38b544471 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -151,6 +151,7 @@ export interface ParamEditorProps { layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; toggleFullscreen: () => void; + isFullscreen: boolean; columnId: string; indexPattern: IndexPattern; uiSettings: IUiSettingsClient; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index cdfc0d9cd7d4e7..7e8fa69b9650bc 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -302,6 +302,7 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro dateRange: DateRange; dimensionGroups: VisualizationDimensionGroupConfig[]; toggleFullscreen: () => void; + isFullscreen: boolean; }; export type DatasourceDimensionTriggerProps = DatasourceDimensionProps; From 83ae9bf3281c7229764e4f8824c7e8551520d213 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 29 Apr 2021 17:07:15 -0400 Subject: [PATCH 077/185] Remove more chrome from full screen mode --- .../config_panel/dimension_container.tsx | 123 ++++++++++-------- .../editor_frame/config_panel/layer_panel.tsx | 1 + .../editor_frame/editor_frame.tsx | 1 + .../editor_frame/frame_layout.scss | 4 + .../editor_frame/frame_layout.tsx | 8 +- .../workspace_panel/workspace_panel.tsx | 21 ++- .../workspace_panel_wrapper.scss | 4 + .../workspace_panel_wrapper.tsx | 92 +++++++------ .../dimension_panel/dimension_editor.tsx | 12 +- .../definitions/formula/formula.tsx | 90 ++++++------- 10 files changed, 205 insertions(+), 151 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 1bb9eea6ab5857..93a9813be9f9a0 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -29,11 +29,13 @@ export function DimensionContainer({ groupLabel, handleClose, panel, + isFullscreen, }: { isOpen: boolean; handleClose: () => void; panel: React.ReactElement | null; groupLabel: string; + isFullscreen: boolean; }) { const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); @@ -77,6 +79,9 @@ export function DimensionContainer({ { + if (isFullscreen) { + return; + } let current = e.target as HTMLElement; while (current) { if (current?.getAttribute?.('data-test-subj') === 'lnsFormulaWidget') { @@ -88,61 +93,71 @@ export function DimensionContainer({ }} isDisabled={!isOpen} > -
- - - - - - - -

- - {i18n.translate('xpack.lens.configure.configurePanelTitle', { - defaultMessage: '{groupLabel} configuration', - values: { - groupLabel, - }, - })} - -

- - - - - + {isFullscreen ? ( +
{panel} - - - - {i18n.translate('xpack.lens.dimensionContainer.close', { - defaultMessage: 'Close', - })} - - -
+
+ ) : ( +
+ + + + + + + +

+ + {i18n.translate('xpack.lens.configure.configurePanelTitle', { + defaultMessage: '{groupLabel} configuration', + values: { + groupLabel, + }, + })} + +

+ + + + + + {panel} + + + + {i18n.translate('xpack.lens.dimensionContainer.close', { + defaultMessage: 'Close', + })} + + +
+ )} ) : null; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index ffccc90b6a2083..145885d2b552fe 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -410,6 +410,7 @@ export function LayerPanel( { if (layerDatasource.updateStateOnCloseDimension) { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 17db00892a51f1..8886232901aa8d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -363,6 +363,7 @@ export function EditorFrame(props: EditorFrameProps) { visualizationState={state.visualization.state} visualizationMap={props.visualizationMap} dispatch={dispatch} + isFullscreen={Boolean(state.isFullscreenDatasource)} ExpressionRenderer={props.ExpressionRenderer} core={props.core} plugins={props.plugins} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss index 98d636b9a56ba4..56642cff1fbff1 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss @@ -70,6 +70,10 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ &:first-child { padding-left: $euiSize; } + + &.lnsFrameLayout__pageBody--fullscreen { + padding: 0; + } } .lnsFrameLayout__sidebar { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index f83d2f5cc20463..f09915a981af26 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -40,7 +40,13 @@ export function FrameLayout(props: FrameLayoutProps) { {props.dataPanel}
) : null} -
+

{i18n.translate('xpack.lens.section.workspaceLabel', { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index a31146e5004349..e03c74c9af34a9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -76,6 +76,7 @@ export interface WorkspacePanelProps { title?: string; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; + isFullscreen: boolean; } interface WorkspaceState { @@ -127,6 +128,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, + isFullscreen, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -338,6 +340,8 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ ); }; + const element = expression !== null ? renderVisualization() : renderEmptyWorkspace(); + return ( - - {renderVisualization()} - {Boolean(suggestionForDraggedField) && expression !== null && renderEmptyWorkspace()} - + {isFullscreen ? ( + element + ) : ( + + {element} + + )} ); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss index e687e478cd3680..727083097d194b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss @@ -62,6 +62,10 @@ animation: lnsWorkspacePanel__illustrationPulseContinuous 1.5s ease-in-out 0s infinite normal forwards; } } + + &.lnsWorkspacePanel__dragDrop--fullscreen { + border: none; + } } .lnsWorkspacePanel__emptyContent { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx index 85f7601d8fb292..7cf53f26a9ac41 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx @@ -32,6 +32,7 @@ export interface WorkspacePanelWrapperProps { state: unknown; } >; + isFullscreen: boolean; } export function WorkspacePanelWrapper({ @@ -44,6 +45,7 @@ export function WorkspacePanelWrapper({ visualizationMap, datasourceMap, datasourceStates, + isFullscreen, }: WorkspacePanelWrapperProps) { const activeVisualization = visualizationId ? visualizationMap[visualizationId] : null; const setVisualizationState = useCallback( @@ -74,40 +76,42 @@ export function WorkspacePanelWrapper({ wrap={true} justifyContent="spaceBetween" > - - - - - - {activeVisualization && activeVisualization.renderToolbar && ( + {!isFullscreen ? ( + + - - )} - - + {activeVisualization && activeVisualization.renderToolbar && ( + + + + )} + + + ) : null} {warningMessages && warningMessages.length ? ( {warningMessages} @@ -115,17 +119,21 @@ export function WorkspacePanelWrapper({

- - -

- {title || - i18n.translate('xpack.lens.chartTitle.unsaved', { - defaultMessage: 'Unsaved visualization', - })} -

- - {children} - + {isFullscreen ? ( + children + ) : ( + + +

+ {title || + i18n.translate('xpack.lens.chartTitle.unsaved', { + defaultMessage: 'Unsaved visualization', + })} +

+ + {children} + + )} ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 05da746db0e600..c74969161965a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -579,7 +579,9 @@ export function DimensionEditor(props: DimensionEditorProps) { return (
- {operationSupportMatrix.operationWithoutField.has('formula') ? ( + {isFullscreen ? ( + tabs[1].content + ) : operationSupportMatrix.operationWithoutField.has('formula') ? ( {!incompleteInfo && selectedColumn && ( )} - {!incompleteInfo && !hideGrouping && ( + {!isFullscreen && !incompleteInfo && !hideGrouping && ( )} - {selectedColumn && + {!isFullscreen && + selectedColumn && (selectedColumn.dataType === 'number' || selectedColumn.operationType === 'range') ? ( - {i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'View full screen', - })} + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Close full screen', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'View full screen', + })} @@ -822,38 +826,37 @@ function FormulaHelp({ ); return ( -
- - - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); - } else { - setSelectedFunction(chosenType.label); - } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + - )} - - - -
+ description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + /> + )} + + + ); } From f520d2ffdf1dee5beab619a7cb717bb1046eccb4 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 30 Apr 2021 18:58:17 -0400 Subject: [PATCH 078/185] Fix minor bugs in formula typing --- .../editor_frame/config_panel/layer_panel.tsx | 4 +- .../dimension_panel/dimension_editor.scss | 8 ++ .../definitions/formula/formula.tsx | 113 +++++------------- .../definitions/formula/math_completion.ts | 1 + 4 files changed, 43 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 145885d2b552fe..78608ebf995a57 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -429,7 +429,7 @@ export function LayerPanel( } }} panel={ - <> +
{activeGroup && activeId && (
)} - +
} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index 29c1ad307fe402..999371b1dadafe 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -2,6 +2,14 @@ height: 100%; } +.lnsIndexPatternDimensionEditor-fullscreen { + position: absolute; + top: 0; + bottom: 0; + display: flex; + flex-direction: column; +} + .lnsIndexPatternDimensionEditor__section { padding: $euiSizeS; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index ee0513366808f5..715c5e3e8b30be 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -15,13 +15,12 @@ import { EuiFlexItem, EuiText, EuiSpacer, - EuiModal, - EuiModalHeader, EuiPopover, EuiSelectable, EuiSelectableOption, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; +import classNames from 'classnames'; import { CodeEditor, CodeEditorProps, @@ -238,10 +237,8 @@ function FormulaEditor({ monaco.editor.createModel(text ?? '', LANGUAGE_ID) ); const overflowDiv1 = React.useRef(); - const overflowDiv2 = React.useRef(); const updateAfterTyping = React.useRef(); const editor1 = React.useRef(); - const editor2 = React.useRef(); // The Monaco editor needs to have the overflowDiv in the first render. Using an effect // requires a second render to work, so we are using an if statement to guarantee it happens @@ -252,12 +249,6 @@ function FormulaEditor({ // Monaco CSS is targeted on the monaco-editor class node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); document.body.appendChild(node1); - - const node2 = (overflowDiv2.current = document.createElement('div')); - node2.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Monaco CSS is targeted on the monaco-editor class - node2.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(node2); } // Clean up the monaco editor and DOM on unmount @@ -265,14 +256,11 @@ function FormulaEditor({ const model = editorModel.current; const disposable1 = updateAfterTyping.current; const editor1ref = editor1.current; - const editor2ref = editor2.current; return () => { model.dispose(); overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - overflowDiv2.current?.parentNode?.removeChild(overflowDiv2.current); disposable1?.dispose(); editor1ref?.dispose(); - editor2ref?.dispose(); }; }, []); @@ -282,6 +270,19 @@ function FormulaEditor({ if (!text) { monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + if (currentColumn.params.formula) { + // Only submit if valid + const { newLayer } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + } + return; } @@ -599,11 +600,7 @@ function FormulaEditor({ // while it has focus. useEffect(() => { if (updateAfterTyping.current) { - if (isFullscreen) { - if (editor2.current) registerOnTypeHandler(editor2.current); - } else { - if (editor1.current) registerOnTypeHandler(editor1.current); - } + if (editor1.current) registerOnTypeHandler(editor1.current); } }, [isFullscreen, registerOnTypeHandler]); @@ -621,7 +618,9 @@ function FormulaEditor({ wordWrap: 'on', // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, - dimension: { width: 290, height: isFullscreen ? 400 : 280 }, + autoIndent: 'brackets', + wrappingIndent: 'none', + dimension: { width: 290, height: 200 }, fixedOverflowWidgets: true, }, }; @@ -629,11 +628,11 @@ function FormulaEditor({ useEffect(() => { // Because the monaco model is owned by Lens, we need to manually attach and remove handlers const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', ',', '(', '=', ' ', ':', `'`], + triggerCharacters: ['.', '(', '=', ' ', ':', `'`], provideCompletionItems, }); const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { - signatureHelpTriggerCharacters: ['(', ',', '='], + signatureHelpTriggerCharacters: ['(', '='], provideSignatureHelp, }); const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { @@ -649,7 +648,11 @@ function FormulaEditor({ // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences // in the behavior of Monaco when it's first loaded and then reloaded. return ( - <> +
@@ -666,10 +669,10 @@ function FormulaEditor({ > {isFullscreen ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Close full screen', + defaultMessage: 'Collapse formula', }) : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'View full screen', + defaultMessage: 'Expand formula', })} @@ -678,7 +681,7 @@ function FormulaEditor({
{/* Errors go here */} - - {false ? ( - { - // setIsOpen(false); - setText(currentColumn.params.formula); - }} - > - -

- {i18n.translate('xpack.lens.formula.formulaEditorLabel', { - defaultMessage: 'Formula editor', - })} -

- - - -
- { - editor2.current = editor; - registerOnTypeHandler(editor); - }} - /> -
- - -
- - {i18n.translate('xpack.lens.formula.functionReferenceLabel', { - defaultMessage: 'Function reference', - })} - - - -
- - - - ) : null}
- +
); } @@ -826,8 +777,8 @@ function FormulaHelp({ ); return ( - - + + - + {selectedFunction ? ( helpItems.find(({ label }) => label === selectedFunction)?.description diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index 24e91126e7c372..ba469516a338dc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -383,6 +383,7 @@ export function getSuggestion( }; insertText = `${label}='$0'`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + sortText = `zzz${label}`; } label = `${label}=`; detail = ''; From 5a26025c3f11f29fa5d0208eb5d38ab762393f8b Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 4 May 2021 15:28:16 -0400 Subject: [PATCH 079/185] =?UTF-8?q?=F0=9F=90=9B=20Decouple=20column=20orde?= =?UTF-8?q?r=20of=20references=20and=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../droppable/droppable.test.ts | 6 +- .../indexpattern.test.ts | 85 ++++++++++++++++ .../operations/layer_helpers.test.ts | 96 +++++++++++-------- .../operations/layer_helpers.ts | 44 ++------- .../indexpattern_datasource/to_expression.ts | 31 +++++- 5 files changed, 182 insertions(+), 80 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts index 9410843c0811ae..a77a980257c88c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts @@ -904,7 +904,7 @@ describe('IndexPatternDimensionEditorPanel', () => { layers: { first: { ...testState.layers.first, - columnOrder: ['ref1', 'col1', 'ref1Copy', 'col1Copy'], + columnOrder: ['col1', 'ref1', 'ref1Copy', 'col1Copy'], columns: { ref1: testState.layers.first.columns.ref1, col1: testState.layers.first.columns.col1, @@ -974,7 +974,7 @@ describe('IndexPatternDimensionEditorPanel', () => { layers: { first: { ...testState.layers.first, - columnOrder: ['ref1', 'ref2', 'col1', 'ref1Copy', 'ref2Copy', 'col1Copy'], + columnOrder: ['col1', 'ref1', 'ref2', 'ref1Copy', 'col1Copy', 'ref2Copy'], columns: { ref1: testState.layers.first.columns.ref1, ref2: testState.layers.first.columns.ref2, @@ -1061,8 +1061,8 @@ describe('IndexPatternDimensionEditorPanel', () => { 'col1', 'innerRef1Copy', 'ref1Copy', - 'ref2Copy', 'col1Copy', + 'ref2Copy', ], columns: { innerRef1: testState.layers.first.columns.innerRef1, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 7be02e792e955f..29b8ede2d4365b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -922,6 +922,91 @@ describe('IndexPattern Data Source', () => { }), }); }); + + it('should topologically sort references', () => { + // This is a real example of count() + count() + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['date', 'count', 'formula', 'countX0', 'math'], + columns: { + count: { + label: 'count', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + date: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', + isBucketed: true, + scale: 'interval', + params: { + interval: 'auto', + }, + }, + formula: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + formula: 'count() + count()', + isFormulaBroken: false, + }, + references: ['math'], + }, + countX0: { + label: 'countX0', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + math: { + label: 'math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + // @ts-expect-error String args are not valid tinymath, but signals something unique to Lens + args: ['countX0', 'count'], + location: { + min: 0, + max: 17, + }, + text: 'count() + count()', + }, + }, + references: ['countX0', 'count'], + customLabel: true, + }, + }, + }, + }, + }; + + const state = enrichBaseState(queryBaseState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + const chainLength = ast.chain.length; + expect(ast.chain[chainLength - 2].arguments.name).toEqual(['math']); + expect(ast.chain[chainLength - 1].arguments.id).toEqual(['formula']); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 444e38eee3841c..cebbc30e8784c7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -195,7 +195,7 @@ describe('state_helpers', () => { ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2'] })); }); - it('should insert a metric after buckets, but before references', () => { + it('should insert a metric after references', () => { const layer: IndexPatternLayer = { indexPatternId: '1', columnOrder: ['col1'], @@ -231,7 +231,7 @@ describe('state_helpers', () => { field: documentField, visualizationGroups: [], }) - ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2', 'col3'] })); + ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col3', 'col2'] })); }); it('should insert new buckets at the end of previous buckets', () => { @@ -1074,7 +1074,7 @@ describe('state_helpers', () => { referenceIds: ['id1'], }) ); - expect(result.columnOrder).toEqual(['id1', 'col1']); + expect(result.columnOrder).toEqual(['col1', 'id1']); expect(result.columns).toEqual( expect.objectContaining({ id1: expectedColumn, @@ -1196,7 +1196,7 @@ describe('state_helpers', () => { op: 'testReference', }); - expect(result.columnOrder).toEqual(['id1', 'col1']); + expect(result.columnOrder).toEqual(['col1', 'id1']); expect(result.columns).toEqual({ id1: expect.objectContaining({ operationType: 'average', @@ -1459,7 +1459,7 @@ describe('state_helpers', () => { }) ).toEqual( expect.objectContaining({ - columnOrder: ['id1', 'output'], + columnOrder: ['output', 'id1'], columns: { id1: expect.objectContaining({ sourceField: 'timestamp', @@ -2051,58 +2051,78 @@ describe('state_helpers', () => { ).toEqual(['col1', 'col3', 'col2']); }); - it('should correctly sort references to other references', () => { + it('does not topologically sort formulas, but keeps the relative order', () => { expect( getColumnOrder({ - columnOrder: [], indexPatternId: '', + columnOrder: [], columns: { - bucket: { - label: 'Top values of category', - dataType: 'string', + count: { + label: 'count', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + date: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', isBucketed: true, - - // Private - operationType: 'terms', - sourceField: 'category', + scale: 'interval', params: { - size: 5, - orderBy: { - type: 'alphabetical', - }, - orderDirection: 'asc', + interval: 'auto', }, }, - metric: { - label: 'Average of bytes', + formula: { + label: 'Formula', dataType: 'number', + operationType: 'formula', isBucketed: false, - - // Private - operationType: 'average', - sourceField: 'bytes', + scale: 'ratio', + params: { + formula: 'count() + count()', + isFormulaBroken: false, + }, + references: ['math'], }, - ref2: { - label: 'Ref2', + countX0: { + label: 'countX0', dataType: 'number', + operationType: 'count', isBucketed: false, - - // @ts-expect-error only for testing - operationType: 'testReference', - references: ['ref1'], + scale: 'ratio', + sourceField: 'Records', + customLabel: true, }, - ref1: { - label: 'Ref', + math: { + label: 'math', dataType: 'number', + operationType: 'math', isBucketed: false, - - // @ts-expect-error only for testing - operationType: 'testReference', - references: ['bucket'], + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + // @ts-expect-error String args are not valid tinymath, but signals something unique to Lens + args: ['countX0', 'count'], + location: { + min: 0, + max: 17, + }, + text: 'count() + count()', + }, + }, + references: ['countX0', 'count'], + customLabel: true, }, }, }) - ).toEqual(['bucket', 'metric', 'ref1', 'ref2']); + ).toEqual(['date', 'count', 'formula', 'countX0', 'math']); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 2c45505d3f88b4..1bbb7332f815a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1029,8 +1029,12 @@ export function deleteColumn({ ); } -// Derives column order from column object, respects existing columnOrder -// when possible, but also allows new columns to be added to the order +// Column order mostly affects the visual order in the UI. It is derived +// from the columns objects, respecting any existing columnOrder relationships, +// but allowing new columns to be inserted +// +// This does NOT topologically sort references, as this would cause the order in the UI +// to change. Reference order is determined before creating the pipeline in to_expression export function getColumnOrder(layer: IndexPatternLayer): string[] { const entries = Object.entries(layer.columns); entries.sort(([idA], [idB]) => { @@ -1045,16 +1049,6 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] { } }); - // If a reference has another reference as input, put it last in sort order - entries.sort(([idA, a], [idB, b]) => { - if ('references' in a && a.references.includes(idB)) { - return 1; - } - if ('references' in b && b.references.includes(idA)) { - return -1; - } - return 0; - }); const [aggregations, metrics] = _.partition(entries, ([, col]) => col.isBucketed); return aggregations.map(([id]) => id).concat(metrics.map(([id]) => id)); @@ -1259,29 +1253,3 @@ export function getManagedColumnsFrom( } return store.filter(([, column]) => column); } - -function topologicalSort(columns: Array<[string, IndexPatternColumn]>) { - const allNodes: Record = {}; - columns.forEach(([id, col]) => { - allNodes[id] = 'references' in col ? col.references : []; - }); - // remove real metric references - columns.forEach(([id]) => { - allNodes[id] = allNodes[id].filter((refId) => !!allNodes[refId]); - }); - const ordered: string[] = []; - - while (ordered.length < columns.length) { - Object.keys(allNodes).forEach((id) => { - if (allNodes[id].length === 0) { - ordered.push(id); - delete allNodes[id]; - Object.keys(allNodes).forEach((k) => { - allNodes[k] = allNodes[k].filter((i) => i !== id); - }); - } - }); - } - - return ordered; -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 81d890f62843d9..04e5ae9e488b5f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -68,7 +68,9 @@ function getExpressionForLayer( if (referenceEntries.length || esAggEntries.length) { const aggs: ExpressionAstExpressionBuilder[] = []; const expressions: ExpressionAstFunction[] = []; - referenceEntries.forEach(([colId, col]) => { + + sortedReferences(referenceEntries).forEach((colId) => { + const col = columns[colId]; const def = operationDefinitionMap[col.operationType]; if (def.input === 'fullReference' || def.input === 'managedReference') { expressions.push(...def.toExpression(layer, colId, indexPattern)); @@ -255,6 +257,33 @@ function getExpressionForLayer( return null; } +// Topologically sorts references so that we can execute them in sequence +function sortedReferences(columns: Array) { + const allNodes: Record = {}; + columns.forEach(([id, col]) => { + allNodes[id] = 'references' in col ? col.references : []; + }); + // remove real metric references + columns.forEach(([id]) => { + allNodes[id] = allNodes[id].filter((refId) => !!allNodes[refId]); + }); + const ordered: string[] = []; + + while (ordered.length < columns.length) { + Object.keys(allNodes).forEach((id) => { + if (allNodes[id].length === 0) { + ordered.push(id); + delete allNodes[id]; + Object.keys(allNodes).forEach((k) => { + allNodes[k] = allNodes[k].filter((i) => i !== id); + }); + } + }); + } + + return ordered; +} + export function toExpression( state: IndexPatternPrivateState, layerId: string, From e6ff6d7829c7bea345debae5edd79b91352f06d4 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 4 May 2021 19:09:15 -0400 Subject: [PATCH 080/185] =?UTF-8?q?=F0=9F=94=A7=20Fix=20tests=20and=20type?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kibana_react/public/code_editor/index.tsx | 2 +- src/plugins/kibana_react/public/index.ts | 2 +- .../config_panel/config_panel.test.tsx | 2 + .../config_panel/layer_panel.test.tsx | 2 + .../workspace_panel/workspace_panel.test.tsx | 200 ++++-------------- .../workspace_panel_wrapper.test.tsx | 2 + .../lens/public/id_generator/id_generator.ts | 2 +- .../dimension_panel/dimension_editor.tsx | 5 +- .../dimension_panel/dimension_panel.test.tsx | 63 +++--- .../droppable/droppable.test.ts | 2 + .../dimension_panel/reference_editor.test.tsx | 2 + .../dimension_panel/reference_editor.tsx | 3 + .../definitions/date_histogram.test.tsx | 2 + .../definitions/filters/filters.test.tsx | 2 + .../definitions/formula/formula.test.tsx | 2 +- .../definitions/formula/formula.tsx | 7 +- .../formula/math_completion.test.ts | 9 - .../definitions/last_value.test.tsx | 2 + .../definitions/percentile.test.tsx | 2 + .../definitions/ranges/ranges.test.tsx | 2 + .../definitions/terms/terms.test.tsx | 2 + 21 files changed, 118 insertions(+), 199 deletions(-) diff --git a/src/plugins/kibana_react/public/code_editor/index.tsx b/src/plugins/kibana_react/public/code_editor/index.tsx index ec0f95b8a60d73..2440974c3b1d1e 100644 --- a/src/plugins/kibana_react/public/code_editor/index.tsx +++ b/src/plugins/kibana_react/public/code_editor/index.tsx @@ -16,7 +16,7 @@ import { import darkTheme from '@elastic/eui/dist/eui_theme_dark.json'; import lightTheme from '@elastic/eui/dist/eui_theme_light.json'; import { useUiSetting } from '../ui_settings'; -import type { Props } from './code_editor'; +import { Props } from './code_editor'; const LazyBaseEditor = React.lazy(() => import('./code_editor')); diff --git a/src/plugins/kibana_react/public/index.ts b/src/plugins/kibana_react/public/index.ts index 612366de59f746..f2c2c263da5cd8 100644 --- a/src/plugins/kibana_react/public/index.ts +++ b/src/plugins/kibana_react/public/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export { CodeEditor, CodeEditorProps } from './code_editor'; +export * from './code_editor'; export * from './url_template_editor'; export * from './exit_full_screen_button'; export * from './context'; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx index e171c457c541eb..0ef509377c8b53 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx @@ -76,6 +76,8 @@ describe('ConfigPanel', () => { framePublicAPI: frame, dispatch: jest.fn(), core: coreMock.createStart(), + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index 7ee7a27a53c7da..0aa23d8ad947f5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -86,6 +86,8 @@ describe('LayerPanel', () => { core: coreMock.createStart(), layerIndex: 0, registerNewLayerRef: jest.fn(), + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx index e741b9ee243db3..25f354e2a8c755 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx @@ -29,12 +29,7 @@ import { ReactWrapper } from 'enzyme'; import { DragDrop, ChildDragDropProvider } from '../../../drag_drop'; import { fromExpression } from '@kbn/interpreter/common'; import { coreMock } from 'src/core/public/mocks'; -import { - DataPublicPluginStart, - esFilters, - IFieldType, - IIndexPattern, -} from '../../../../../../../src/plugins/data/public'; +import { esFilters, IFieldType, IIndexPattern } from '../../../../../../../src/plugins/data/public'; import { UiActionsStart } from '../../../../../../../src/plugins/ui_actions/public'; import { uiActionsPluginMock } from '../../../../../../../src/plugins/ui_actions/public/mocks'; import { TriggerContract } from '../../../../../../../src/plugins/ui_actions/public/triggers'; @@ -55,6 +50,27 @@ function createCoreStartWithPermissions(newCapabilities = defaultPermissions) { return core; } +function getDefaultProps() { + return { + activeDatasourceId: 'mock', + datasourceStates: {}, + datasourceMap: {}, + framePublicAPI: createMockFramePublicAPI(), + activeVisualizationId: 'vis', + visualizationState: {}, + dispatch: () => {}, + ExpressionRenderer: createExpressionRendererMock(), + core: createCoreStartWithPermissions(), + plugins: { + uiActions: uiActionsPluginMock.createStartContract(), + data: dataPluginMock.createStartContract(), + }, + getSuggestionForField: () => undefined, + isFullscreen: false, + toggleFullscreen: jest.fn(), + }; +} + describe('workspace_panel', () => { let mockVisualization: jest.Mocked; let mockVisualization2: jest.Mocked; @@ -62,21 +78,18 @@ describe('workspace_panel', () => { let expressionRendererMock: jest.Mock; let uiActionsMock: jest.Mocked; - let dataMock: jest.Mocked; let trigger: jest.Mocked; let instance: ReactWrapper; beforeEach(() => { + // These are used in specific tests to assert function calls trigger = ({ exec: jest.fn() } as unknown) as jest.Mocked; uiActionsMock = uiActionsPluginMock.createStartContract(); - dataMock = dataPluginMock.createStartContract(); uiActionsMock.getTrigger.mockReturnValue(trigger); mockVisualization = createMockVisualization(); mockVisualization2 = createMockVisualization(); - mockDatasource = createMockDatasource('a'); - expressionRendererMock = createExpressionRendererMock(); }); @@ -87,23 +100,14 @@ describe('workspace_panel', () => { it('should render an explanatory text if no visualization is active', () => { instance = mount( {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); - expect(instance.find('[data-test-subj="empty-workspace"]')).toHaveLength(2); expect(instance.find(expressionRendererMock)).toHaveLength(0); }); @@ -111,20 +115,10 @@ describe('workspace_panel', () => { it('should render an explanatory text if the visualization does not produce an expression', () => { instance = mount( null }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -135,20 +129,10 @@ describe('workspace_panel', () => { it('should render an explanatory text if the datasource does not produce an expression', () => { instance = mount( 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -166,7 +150,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -204,10 +182,11 @@ describe('workspace_panel', () => { }; mockDatasource.toExpression.mockReturnValue('datasource'); mockDatasource.getLayers.mockReturnValue(['first']); + const props = getDefaultProps(); instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} + plugins={{ ...props.plugins, uiActions: uiActionsMock }} /> ); @@ -251,7 +225,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} dispatch={dispatch} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -298,7 +267,7 @@ describe('workspace_panel', () => { instance = mount( { mock2: mockDatasource2, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -382,7 +345,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -439,7 +396,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -494,7 +445,7 @@ describe('workspace_panel', () => { }; instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -532,7 +476,7 @@ describe('workspace_panel', () => { instance = mount( { visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} // Use cannot navigate to the management page core={createCoreStartWithPermissions({ navLinks: { management: false }, management: { kibana: { indexPatterns: true } }, })} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -575,7 +514,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} // user can go to management, but indexPatterns management is not accessible core={createCoreStartWithPermissions({ navLinks: { management: true }, management: { kibana: { indexPatterns: false } }, })} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -621,7 +554,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -663,7 +589,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: mockVisualization, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -707,7 +626,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: mockVisualization, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -748,7 +660,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -787,7 +692,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -832,7 +731,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -900,7 +793,7 @@ describe('workspace_panel', () => { dropTargetsByOrder={undefined} > { mock: mockDatasource, }} framePublicAPI={frame} - activeVisualizationId={'vis'} visualizationMap={{ vis: mockVisualization, vis2: mockVisualization2, }} - visualizationState={{}} dispatch={mockDispatch} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} getSuggestionForField={mockGetSuggestionForField} /> diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx index 7bb467df9ab0e1..c18b362e2faa4e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx @@ -37,6 +37,7 @@ describe('workspace_panel_wrapper', () => { visualizationMap={{ myVis: mockVisualization }} datasourceMap={{}} datasourceStates={{}} + isFullscreen={false} > @@ -58,6 +59,7 @@ describe('workspace_panel_wrapper', () => { visualizationMap={{ myVis: { ...mockVisualization, renderToolbar: renderToolbarMock } }} datasourceMap={{}} datasourceStates={{}} + isFullscreen={false} /> ); diff --git a/x-pack/plugins/lens/public/id_generator/id_generator.ts b/x-pack/plugins/lens/public/id_generator/id_generator.ts index 988f2c880222a9..363b8035a23f74 100644 --- a/x-pack/plugins/lens/public/id_generator/id_generator.ts +++ b/x-pack/plugins/lens/public/id_generator/id_generator.ts @@ -8,5 +8,5 @@ import uuid from 'uuid/v4'; export function generateId() { - return 'c' + uuid().replaceAll(/-/g, ''); + return uuid(); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index dcc83b31e1b3a8..06256b5a3bf337 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -391,6 +391,7 @@ export function DimensionEditor(props: DimensionEditorProps) { currentColumn: state.layers[layerId].columns[columnId], })} dimensionGroups={dimensionGroups} + isFullscreen={isFullscreen} toggleFullscreen={toggleFullscreen} {...services} /> @@ -548,7 +549,7 @@ export function DimensionEditor(props: DimensionEditorProps) { name: i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', }), - content: ParamEditor && ( + content: ParamEditor ? ( <> + ) : ( + <> ), }, ]; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index e9089fcefdda34..82e32ac84b90d1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -32,8 +32,6 @@ import { documentField } from '../document_field'; import { OperationMetadata } from '../../types'; import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_histogram'; import { getFieldByNameFactory } from '../pure_helpers'; -import { DimensionEditor } from './dimension_editor'; -import { AdvancedOptions } from './advanced_options'; import { Filtering } from './filtering'; jest.mock('../loader'); @@ -205,6 +203,8 @@ describe('IndexPatternDimensionEditorPanel', () => { core: {} as CoreSetup, dimensionGroups: [], groupId: 'a', + isFullscreen: false, + toggleFullscreen: jest.fn(), }; jest.clearAllMocks(); @@ -1082,21 +1082,23 @@ describe('IndexPatternDimensionEditorPanel', () => { })} /> ); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .find(AdvancedOptions) - .find('[data-test-subj="indexPattern-time-scaling-enable"]') + wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes() ).toHaveLength(0); }); it('should show custom options if time scaling is available', () => { wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .find(AdvancedOptions) - .find('[data-test-subj="indexPattern-time-scaling-enable"]') + wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes() ).toHaveLength(1); }); @@ -1114,10 +1116,13 @@ describe('IndexPatternDimensionEditorPanel', () => { const props = getProps({}); wrapper = mount(); wrapper - .find(DimensionEditor) - .find(AdvancedOptions) + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); + wrapper .find('[data-test-subj="indexPattern-time-scaling-enable"]') - .prop('onClick')!({} as MouseEvent); + .hostNodes() + .simulate('click'); expect(props.setState).toHaveBeenCalledWith( { ...props.state, @@ -1201,6 +1206,10 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to change time scaling', () => { const props = getProps({ timeScale: 's', label: 'Count of records per second' }); wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); wrapper .find('[data-test-subj="indexPattern-time-scaling-unit"]') .find(EuiSelect) @@ -1322,24 +1331,27 @@ describe('IndexPatternDimensionEditorPanel', () => { {...getProps({ operationType: 'terms', sourceField: 'bytes', + params: { + orderDirection: 'asc', + orderBy: { type: 'alphabetical' }, + size: 5, + }, })} /> ); expect( - wrapper - .find(DimensionEditor) - .find(AdvancedOptions) - .find('[data-test-subj="indexPattern-filter-by-enable"]') + wrapper.find('[data-test-subj="indexPattern-advanced-popover"]').hostNodes() ).toHaveLength(0); }); it('should show custom options if filtering is available', () => { wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .find(AdvancedOptions) - .find('[data-test-subj="indexPattern-filter-by-enable"]') + wrapper.find('[data-test-subj="indexPattern-filter-by-enable"]').hostNodes() ).toHaveLength(1); }); @@ -1358,10 +1370,13 @@ describe('IndexPatternDimensionEditorPanel', () => { const props = getProps({}); wrapper = mount(); wrapper - .find(DimensionEditor) - .find(AdvancedOptions) + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); + wrapper .find('[data-test-subj="indexPattern-filter-by-enable"]') - .prop('onClick')!({} as MouseEvent); + .hostNodes() + .simulate('click'); expect(props.setState).toHaveBeenCalledWith( { ...props.state, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts index a77a980257c88c..56d255ec02227c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts @@ -284,6 +284,8 @@ describe('IndexPatternDimensionEditorPanel', () => { } as unknown) as DataPublicPluginStart, core: {} as CoreSetup, dimensionGroups: [], + isFullscreen: false, + toggleFullscreen: () => {}, }; jest.clearAllMocks(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx index f17adf9be39f35..d2d14bfba9ec0f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx @@ -51,6 +51,8 @@ describe('reference editor', () => { http: {} as HttpSetup, data: {} as DataPublicPluginStart, dimensionGroups: [], + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index fc949fb5442aa2..488a32d6fb6f55 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -50,6 +50,7 @@ export interface ReferenceEditorProps { dateRange: DateRange; labelAppend?: EuiFormRowProps['labelAppend']; dimensionGroups: VisualizationDimensionGroupConfig[]; + isFullscreen: boolean; toggleFullscreen: () => void; // Services @@ -72,6 +73,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { dateRange, labelAppend, dimensionGroups, + isFullscreen, toggleFullscreen, ...services } = props; @@ -347,6 +349,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { indexPattern={currentIndexPattern} dateRange={dateRange} operationDefinitionMap={operationDefinitionMap} + isFullscreen={isFullscreen} toggleFullscreen={toggleFullscreen} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index eaaf13171124b0..5130e3ce415fda 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -98,6 +98,8 @@ const defaultOptions = { http: {} as HttpSetup, indexPattern: indexPattern1, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; describe('date_histogram', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index 46fddd9b1ffbf4..fc90cf4fc61960 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -28,6 +28,8 @@ const defaultProps = { http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; // mocking random id generator function diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index adfce115ef5769..433e21eb133451 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -222,7 +222,7 @@ describe('formula', () => { previousColumn: { ...layer.columns.col1, operationType: 'count', - sourceField: undefined, + sourceField: 'Records', filter: { language: 'lucene', query: `*`, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 715c5e3e8b30be..72dce6fe07ff88 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -21,11 +21,8 @@ import { } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; -import { - CodeEditor, - CodeEditorProps, - Markdown, -} from '../../../../../../../../src/plugins/kibana_react/public'; +import { CodeEditor, Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; +import type { CodeEditorProps } from '../../../../../../../../src/plugins/kibana_react/public'; import { OperationDefinition, GenericOperationDefinition, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts index 4919ed3e2901ea..9b5e77b7b90dbb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts @@ -196,7 +196,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 1, endColumn: 1 }, }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { @@ -218,7 +217,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 15, endColumn: 15 }, }); expect(results.list).toHaveLength(2); ['sum', 'last_value'].forEach((key) => { @@ -237,7 +235,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 23, endColumn: 23 }, }); expect(results.list).toEqual(['window']); }); @@ -253,7 +250,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 34, endColumn: 34 }, }); expect(results.list).toEqual([]); }); @@ -269,7 +265,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 16, endColumn: 16 }, }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { @@ -291,7 +286,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 10, endColumn: 10 }, }); expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { @@ -313,7 +307,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 6, endColumn: 6 }, }); expect(results.list).toHaveLength(0); }); @@ -329,7 +322,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 4, endColumn: 4 }, }); expect(results.list).toEqual(['bytes', 'memory']); }); @@ -345,7 +337,6 @@ describe('math completion', () => { indexPattern: createMockedIndexPattern(), operationDefinitionMap, data: dataPluginMock.createStartContract(), - word: { word: '', startColumn: 11, endColumn: 11 }, }); expect(results.list).toEqual(['bytes', 'memory']); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index 280cfe9471c9d0..52be5bcf534a93 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -30,6 +30,8 @@ const defaultProps = { hasRestrictions: false, } as IndexPattern, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; describe('last_value', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index a688f95e94c9ed..dd913f9ecfd110 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -32,6 +32,8 @@ const defaultProps = { hasRestrictions: false, } as IndexPattern, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; describe('percentile', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx index 08bcfcb2e93be5..9832aa5edaffe7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx @@ -91,6 +91,8 @@ const defaultOptions = { ]), }, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; describe('ranges', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index b094d3f0ff5cd7..65748768f388fa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -29,6 +29,8 @@ const defaultProps = { http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; describe('terms', () => { From 198ca291b73e69ce4c81828aed83bcb45d2c889f Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 5 May 2021 18:01:12 -0400 Subject: [PATCH 081/185] =?UTF-8?q?=E2=9C=85=20Add=20first=20functional=20?= =?UTF-8?q?test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dimension_panel/dimension_editor.tsx | 2 + .../definitions/formula/formula.tsx | 2 +- .../operations/definitions/formula/math.tsx | 6 +- x-pack/test/functional/apps/lens/formula.ts | 58 +++++++++++++++++++ x-pack/test/functional/apps/lens/index.ts | 1 + .../test/functional/page_objects/lens_page.ts | 29 ++++++++-- 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 x-pack/test/functional/apps/lens/formula.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 06256b5a3bf337..c270e6b50d70b1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -542,6 +542,7 @@ export function DimensionEditor(props: DimensionEditorProps) { name: i18n.translate('xpack.lens.indexPattern.quickFunctionsLabel', { defaultMessage: 'Quick functions', }), + 'data-test-subj': 'lens-dimensionTabs-quickFunctions', content: quickFunctions, }, { @@ -549,6 +550,7 @@ export function DimensionEditor(props: DimensionEditorProps) { name: i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', }), + 'data-test-subj': 'lens-dimensionTabs-formula', content: ParamEditor ? ( <> { + it('should transition from count to formula', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-dimensionTrigger', + operation: 'average', + field: 'bytes', + keepOpen: true, + }); + + await PageObjects.lens.switchToFormula(); + await PageObjects.header.waitUntilLoadingHasFinished(); + // .echLegendItem__title is the only viable way of getting the xy chart's + // legend item(s), so we're using a class selector here. + // 4th item is the other bucket + expect(await find.allByCssSelector('.echLegendItem')).to.have.length(3); + }); + + it('should update and delete a formula', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count(kql=`, + keepOpen: true, + }); + + const input = await find.activeElement(); + await input.type('*'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index bfb0aad7177f4d..f6f162b51e84a3 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -39,6 +39,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./drag_and_drop')); loadTestFile(require.resolve('./lens_reporting')); loadTestFile(require.resolve('./lens_tagging')); + loadTestFile(require.resolve('./formula')); // has to be last one in the suite because it overrides saved objects loadTestFile(require.resolve('./rollup')); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d379..1b84dd77a67c2f 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -107,6 +107,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont isPreviousIncompatible?: boolean; keepOpen?: boolean; palette?: string; + formula?: string; }, layerIndex = 0 ) { @@ -114,10 +115,15 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click(`lns-layerPanel-${layerIndex} > ${opts.dimension}`); await testSubjects.exists(`lns-indexPatternDimension-${opts.operation}`); }); - const operationSelector = opts.isPreviousIncompatible - ? `lns-indexPatternDimension-${opts.operation} incompatible` - : `lns-indexPatternDimension-${opts.operation}`; - await testSubjects.click(operationSelector); + + if (opts.operation === 'formula') { + await this.switchToFormula(); + } else { + const operationSelector = opts.isPreviousIncompatible + ? `lns-indexPatternDimension-${opts.operation} incompatible` + : `lns-indexPatternDimension-${opts.operation}`; + await testSubjects.click(operationSelector); + } if (opts.field) { const target = await testSubjects.find('indexPattern-dimension-field'); @@ -125,6 +131,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await comboBox.setElement(target, opts.field); } + if (opts.formula) { + await this.typeFormula(opts.formula); + } + if (opts.palette) { await testSubjects.click('lns-palettePicker'); await find.clickByCssSelector(`#${opts.palette}`); @@ -907,5 +917,16 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); await PageObjects.header.waitUntilLoadingHasFinished(); }, + + async switchToFormula() { + await testSubjects.click('lens-dimensionTabs-formula'); + }, + + async typeFormula(formula: string) { + await find.byCssSelector('.monaco-editor'); + await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); + const input = await find.activeElement(); + await input.type(formula); + }, }); } From d335aa3b768acf352186b47fc8dab563d8e923f9 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 6 May 2021 18:16:04 -0400 Subject: [PATCH 082/185] Fix copying formulas and empty formula --- .../expression_functions/specs/map_column.ts | 9 +- .../common/expression_functions/specs/math.ts | 3 +- .../droppable/on_drop_handler.ts | 2 +- .../indexpattern.test.ts | 4 +- .../operations/__mocks__/index.ts | 4 +- .../definitions/calculations/utils.test.ts | 4 +- .../definitions/formula/formula.tsx | 67 ++++++---- .../operations/definitions/formula/math.tsx | 4 + .../operations/definitions/index.ts | 11 ++ .../operations/layer_helpers.test.ts | 121 +++++++++++++++++- .../operations/layer_helpers.ts | 47 ++++--- .../operations/mocks.ts | 27 +++- x-pack/test/functional/apps/lens/formula.ts | 28 ++++ .../test/functional/page_objects/lens_page.ts | 2 + 14 files changed, 280 insertions(+), 53 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index c570206670dde5..7293510baa6b5d 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -101,7 +101,14 @@ export const mapColumn: ExpressionFunctionDefinition< }); return Promise.all(rowPromises).then((rows) => { - const existingColumnIndex = columns.findIndex(({ name }) => name === args.name); + const existingColumnIndex = columns.findIndex(({ id, name }) => { + // Columns that have IDs are allowed to have duplicate names, for example esaggs + if (id) { + return id === args.id && name === args.name; + } + // If no ID, name is the unique key. For example, SQL output does not have IDs + return name === args.name; + }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; const newColumn = { id: columnId, diff --git a/src/plugins/expressions/common/expression_functions/specs/math.ts b/src/plugins/expressions/common/expression_functions/specs/math.ts index a70c032769b570..b91600fea8b56e 100644 --- a/src/plugins/expressions/common/expression_functions/specs/math.ts +++ b/src/plugins/expressions/common/expression_functions/specs/math.ts @@ -130,10 +130,11 @@ export const math: ExpressionFunctionDefinition< throw errors.emptyExpression(); } + // Use unique ID if available, otherwise fall back to names const mathContext = isDatatable(input) ? pivotObjectArray( input.rows, - input.columns.map((col) => col.name) + input.columns.map((col) => col.id ?? col.name) ) : { value: input }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts index f65557d4ed6a95..e09c3e904f5358 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts @@ -114,7 +114,7 @@ function onMoveCompatible( const modifiedLayer = copyColumn({ layer, - columnId, + targetId: columnId, sourceColumnId: droppedItem.columnId, sourceColumn, shouldDeleteSource, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 29b8ede2d4365b..0a2bc7fc1beadf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -15,7 +15,7 @@ import { Ast } from '@kbn/interpreter/common'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { getFieldByNameFactory } from './pure_helpers'; import { operationDefinitionMap, getErrorMessages } from './operations'; -import { createMockedReferenceOperation } from './operations/mocks'; +import { createMockedFullReference } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks'; jest.mock('./loader'); @@ -839,7 +839,7 @@ describe('IndexPattern Data Source', () => { describe('references', () => { beforeEach(() => { // @ts-expect-error we are inserting an invalid type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); // @ts-expect-error we are inserting an invalid type operationDefinitionMap.testReference.toExpression.mockReturnValue(['mock']); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts index 6ac208913af2e8..40d7e3ef94ad68 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts @@ -12,6 +12,7 @@ const actualMocks = jest.requireActual('../mocks'); jest.spyOn(actualOperations.operationDefinitionMap.date_histogram, 'paramEditor'); jest.spyOn(actualOperations.operationDefinitionMap.terms, 'onOtherColumnChanged'); +jest.spyOn(actualHelpers, 'copyColumn'); jest.spyOn(actualHelpers, 'insertOrReplaceColumn'); jest.spyOn(actualHelpers, 'insertNewColumn'); jest.spyOn(actualHelpers, 'replaceColumn'); @@ -30,6 +31,7 @@ export const { } = actualOperations; export const { + copyColumn, insertOrReplaceColumn, insertNewColumn, replaceColumn, @@ -50,4 +52,4 @@ export const { export const { adjustTimeScaleLabelSuffix, DEFAULT_TIME_SCALE } = actualTimeScaleUtils; -export const { createMockedReferenceOperation } = actualMocks; +export const { createMockedFullReference } = actualMocks; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts index 4c1101d4c8a791..7a6f96d705b0c0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts @@ -7,7 +7,7 @@ import { checkReferences } from './utils'; import { operationDefinitionMap } from '..'; -import { createMockedReferenceOperation } from '../../mocks'; +import { createMockedFullReference } from '../../mocks'; // Mock prevents issue with circular loading jest.mock('..'); @@ -15,7 +15,7 @@ jest.mock('..'); describe('utils', () => { beforeEach(() => { // @ts-expect-error test-only operation type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); }); describe('checkReferences', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6da482e624f989..f18b2ab84657de 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -123,32 +123,32 @@ export const formulaOperation: OperationDefinition< : params?.formula : ''; - return currentColumn.references.length - ? [ - { - type: 'function', - function: 'mapColumn', - arguments: { - id: [columnId], - name: [label || ''], - exp: [ + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + id: [columnId], + name: [label || ''], + exp: [ + { + type: 'expression', + chain: [ { - type: 'expression', - chain: [ - { - type: 'function', - function: 'math', - arguments: { - expression: [`"${currentColumn.references[0]}"`], - }, - }, - ], + type: 'function', + function: 'math', + arguments: { + expression: [ + currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``, + ], + }, }, ], }, - }, - ] - : []; + ], + }, + }, + ]; }, buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { let previousFormula = ''; @@ -213,6 +213,25 @@ export const formulaOperation: OperationDefinition< if (!root) return true; return Boolean(!error && !hasMathNode(root)); }, + createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) { + const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn; + const tempLayer = { + ...layer, + columns: { + ...layer.columns, + [targetId]: { ...currentColumn }, + }, + }; + const { newLayer } = regenerateLayerFromAst( + currentColumn.params.formula ?? '', + tempLayer, + targetId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + return newLayer; + }, paramEditor: FormulaEditor, }; @@ -891,9 +910,7 @@ export function regenerateLayerFromAst( operationDefinitionMap ); - const columns = { - ...layer.columns, - }; + const columns = { ...layer.columns }; const locations: Record = {}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx index fa757d060a677d..527af324b5b054 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -47,6 +47,7 @@ export const mathOperation: OperationDefinition { + return { ...layer }; + }, }; function astToString(ast: TinymathAST | string): string | number { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 8283e38b544471..ebac3439bdae7a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -462,6 +462,17 @@ interface ManagedReferenceOperationDefinition columnId: string, indexPattern: IndexPattern ) => ExpressionAstFunction[]; + /** + * Managed references control the IDs of their inner columns, so we need to be able to copy from the + * root level + */ + createCopy: ( + layer: IndexPatternLayer, + sourceColumnId: string, + targetColumnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record + ) => IndexPatternLayer; } interface OperationDefinitionMap { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index cebbc30e8784c7..52cc8a07511a47 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -7,6 +7,7 @@ import type { OperationMetadata } from '../../types'; import { + copyColumn, insertNewColumn, replaceColumn, updateColumnParam, @@ -23,7 +24,7 @@ import type { IndexPattern, IndexPatternLayer } from '../types'; import { documentField } from '../document_field'; import { getFieldByNameFactory } from '../pure_helpers'; import { generateId } from '../../id_generator'; -import { createMockedReferenceOperation } from './mocks'; +import { createMockedFullReference, createMockedManagedReference } from './mocks'; jest.mock('../operations'); jest.mock('../../id_generator'); @@ -89,11 +90,127 @@ describe('state_helpers', () => { (generateId as jest.Mock).mockImplementation(() => `id${++count}`); // @ts-expect-error we are inserting an invalid type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.managedReference = createMockedManagedReference(); }); afterEach(() => { delete operationDefinitionMap.testReference; + delete operationDefinitionMap.managedReference; + }); + + describe('copyColumn', () => { + it('should recursively modify a formula and update the math ast', () => { + const source = { + dataType: 'number' as const, + isBucketed: false, + label: 'Formula', + operationType: 'formula' as const, + params: { + formula: 'moving_average(sum(bytes), window=5)', + isFormulaBroken: false, + }, + references: ['formulaX1'], + }; + const math = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'math', + operationType: 'math' as const, + params: { tinymathAst: 'formulaX2' }, + references: ['formulaX2'], + }; + const sum = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX0', + operationType: 'sum' as const, + scale: 'ratio' as const, + sourceField: 'bytes', + }; + const movingAvg = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX2', + operationType: 'moving_average' as const, + params: { window: 5 }, + references: ['formulaX0'], + }; + expect( + copyColumn({ + layer: { + indexPatternId: '', + columnOrder: [], + columns: { + source, + formulaX0: sum, + formulaX1: math, + formulaX2: movingAvg, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + }, + }, + }, + targetId: 'copy', + sourceColumn: source, + shouldDeleteSource: false, + indexPattern, + sourceColumnId: 'source', + }) + ).toEqual({ + indexPatternId: '', + columnOrder: [ + 'source', + 'formulaX0', + 'formulaX1', + 'formulaX2', + 'formulaX3', + 'copyX0', + 'copyX1', + 'copyX2', + 'copyX3', + 'copy', + ], + columns: { + source, + formulaX0: sum, + formulaX1: math, + formulaX2: movingAvg, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + }, + copy: expect.objectContaining({ ...source, references: ['copyX3'] }), + copyX0: expect.objectContaining({ ...sum, label: 'copyX0' }), + copyX1: expect.objectContaining({ + ...math, + label: 'copyX1', + references: ['copyX0'], + params: { tinymathAst: 'copyX0' }, + }), + copyX2: expect.objectContaining({ + ...movingAvg, + label: 'copyX2', + references: ['copyX1'], + }), + copyX3: expect.objectContaining({ + ...math, + label: 'copyX3', + references: ['copyX2'], + params: { tinymathAst: 'copyX2' }, + }), + }, + }); + }); }); describe('insertNewColumn', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 1bbb7332f815a4..49366f2421b7b6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -33,7 +33,7 @@ interface ColumnChange { interface ColumnCopy { layer: IndexPatternLayer; - columnId: string; + targetId: string; sourceColumn: IndexPatternColumn; sourceColumnId: string; indexPattern: IndexPattern; @@ -42,16 +42,19 @@ interface ColumnCopy { export function copyColumn({ layer, - columnId, + targetId, sourceColumn, shouldDeleteSource, indexPattern, sourceColumnId, }: ColumnCopy): IndexPatternLayer { - let modifiedLayer = { - ...layer, - columns: copyReferencesRecursively(layer.columns, sourceColumn, columnId), - }; + let modifiedLayer = copyReferencesRecursively( + layer, + sourceColumn, + sourceColumnId, + targetId, + indexPattern + ); if (shouldDeleteSource) { modifiedLayer = deleteColumn({ @@ -65,16 +68,25 @@ export function copyColumn({ } function copyReferencesRecursively( - columns: Record, + layer: IndexPatternLayer, sourceColumn: IndexPatternColumn, - columnId: string -) { + sourceId: string, + targetId: string, + indexPattern: IndexPattern +): IndexPatternLayer { + let columns = { ...layer.columns }; if ('references' in sourceColumn) { - if (columns[columnId]) { - return columns; + if (columns[targetId]) { + return layer; } + + const def = operationDefinitionMap[sourceColumn.operationType]; + if ('createCopy' in def) { + // Allow managed references to recursively insert new columns + return def.createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap); + } + sourceColumn?.references.forEach((ref, index) => { - // TODO: Add an option to assign IDs without generating the new one const newId = generateId(); const refColumn = { ...columns[ref] }; @@ -83,10 +95,10 @@ function copyReferencesRecursively( // and visible columns shouldn't be copied const refColumnWithInnerRefs = 'references' in refColumn - ? copyReferencesRecursively(columns, refColumn, newId) // if a column has references, copy them too + ? copyReferencesRecursively(layer, refColumn, sourceId, newId, indexPattern).columns // if a column has references, copy them too : { [newId]: refColumn }; - const newColumn = columns[columnId]; + const newColumn = columns[targetId]; let references = [newId]; if (newColumn && 'references' in newColumn) { references = newColumn.references; @@ -96,7 +108,7 @@ function copyReferencesRecursively( columns = { ...columns, ...refColumnWithInnerRefs, - [columnId]: { + [targetId]: { ...sourceColumn, references, }, @@ -105,10 +117,11 @@ function copyReferencesRecursively( } else { columns = { ...columns, - [columnId]: sourceColumn, + [targetId]: sourceColumn, }; } - return columns; + + return { ...layer, columns, columnOrder: getColumnOrder({ ...layer, columns }) }; } export function insertOrReplaceColumn(args: ColumnChange): IndexPatternLayer { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts index 429d881341e791..2d7e70179fb3f7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts @@ -8,7 +8,7 @@ import type { OperationMetadata } from '../../types'; import type { OperationType } from './definitions'; -export const createMockedReferenceOperation = () => { +export const createMockedFullReference = () => { return { input: 'fullReference', displayName: 'Reference test', @@ -40,3 +40,28 @@ export const createMockedReferenceOperation = () => { getErrorMessage: jest.fn(), }; }; + +export const createMockedManagedReference = () => { + return { + input: 'managedReference', + displayName: 'Managed reference test', + type: 'managedReference' as OperationType, + selectionStyle: 'full', + buildColumn: jest.fn((args) => { + return { + label: 'Test reference', + isBucketed: false, + dataType: 'number', + + operationType: 'testReference', + references: args.referenceIds, + }; + }), + filterable: true, + isTransferable: jest.fn(), + toExpression: jest.fn().mockReturnValue([]), + getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), + getDefaultLabel: jest.fn().mockReturnValue('Default label'), + getErrorMessage: jest.fn(), + }; +}; diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index d1d15154eb5cc1..798cb7d3146f55 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -54,5 +54,33 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); }); + + it('should duplicate a moving average formula and be a valid table', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `moving_average(sum(bytes), window=5`, + keepOpen: true, + }); + await PageObjects.lens.closeDimensionEditor(); + + await PageObjects.lens.dragDimensionToDimension( + 'lnsDatatable_metrics > lns-dimensionTrigger', + 'lnsDatatable_metrics > lns-empty-dimension' + ); + expect(await PageObjects.lens.getDatatableCellText(1, 1)).to.eql('222420'); + expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('222420'); + }); }); } diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 1b84dd77a67c2f..080e44da6ffcdf 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -927,6 +927,8 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); const input = await find.activeElement(); await input.type(formula); + // Formula is applied on a 250ms timer, won't be applied if we leave too early + await PageObjects.common.sleep(500); }, }); } From bf264ed716384c6326292cc268a1070970bcfa43 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 6 May 2021 18:59:41 -0400 Subject: [PATCH 083/185] Trigger suggestion prompt when hitting enter on function or typing kql= --- .../operations/definitions/formula/formula.tsx | 1 + .../operations/definitions/formula/math_completion.ts | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index f18b2ab84657de..9d04ac1c803918 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -591,6 +591,7 @@ function FormulaEditor({ ), ] ); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); }, 0); } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts index ba469516a338dc..1ae5da9d6db1d6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts @@ -371,6 +371,10 @@ export function getSuggestion( detail = 'Elasticsearch'; // Always put ES functions first sortText = `0${label}`; + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; } } break; From a317738666e67bb1db55c7a043873d9165e03ad3 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 7 May 2021 15:38:32 -0400 Subject: [PATCH 084/185] =?UTF-8?q?=F0=9F=90=9B=20Prevent=20flyout=20from?= =?UTF-8?q?=20closing=20while=20interacting=20with=20monaco?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config_panel/dimension_container.tsx | 24 +++++----- .../editor_frame/config_panel/layer_panel.tsx | 6 +++ .../dimension_panel/dimension_editor.tsx | 7 +++ .../dimension_panel/reference_editor.test.tsx | 1 + .../dimension_panel/reference_editor.tsx | 3 ++ .../indexpattern_datasource/indexpattern.tsx | 5 ++ .../definitions/date_histogram.test.tsx | 1 + .../definitions/filters/filters.test.tsx | 1 + .../definitions/formula/formula.tsx | 47 +++++++++---------- .../operations/definitions/index.ts | 1 + .../definitions/last_value.test.tsx | 1 + .../definitions/percentile.test.tsx | 1 + .../definitions/ranges/ranges.test.tsx | 1 + .../definitions/terms/terms.test.tsx | 1 + .../public/indexpattern_datasource/types.ts | 2 + x-pack/plugins/lens/public/types.ts | 5 ++ 16 files changed, 68 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index 93a9813be9f9a0..517321218e3444 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -32,7 +32,7 @@ export function DimensionContainer({ isFullscreen, }: { isOpen: boolean; - handleClose: () => void; + handleClose: () => boolean; panel: React.ReactElement | null; groupLabel: string; isFullscreen: boolean; @@ -40,8 +40,11 @@ export function DimensionContainer({ const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); const closeFlyout = useCallback(() => { - handleClose(); - setFocusTrapIsEnabled(false); + const canClose = handleClose(); + if (canClose) { + setFocusTrapIsEnabled(false); + } + return canClose; }, [handleClose]); useEffect(() => { @@ -56,8 +59,10 @@ export function DimensionContainer({ const closeOnEscape = useCallback( (event: KeyboardEvent) => { if (event.key === keys.ESCAPE) { - event.preventDefault(); - closeFlyout(); + const canClose = closeFlyout(); + if (canClose) { + event.preventDefault(); + } } }, [closeFlyout] @@ -78,17 +83,10 @@ export function DimensionContainer({ { + onOutsideClick={() => { if (isFullscreen) { return; } - let current = e.target as HTMLElement; - while (current) { - if (current?.getAttribute?.('data-test-subj') === 'lnsFormulaWidget') { - return; - } - current = current.parentNode as HTMLElement; - } closeFlyout(); }} isDisabled={!isOpen} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 78608ebf995a57..7e06bd2ab110cb 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -413,6 +413,11 @@ export function LayerPanel( isFullscreen={isFullscreen} groupLabel={activeGroup?.groupLabel || ''} handleClose={() => { + if (layerDatasource.canCloseDimensionEditor) { + if (!layerDatasource.canCloseDimensionEditor(layerDatasourceState)) { + return false; + } + } if (layerDatasource.updateStateOnCloseDimension) { const newState = layerDatasource.updateStateOnCloseDimension({ state: layerDatasourceState, @@ -427,6 +432,7 @@ export function LayerPanel( if (isFullscreen) { toggleFullscreen(); } + return true; }} panel={
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index c270e6b50d70b1..28a5438b7af080 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -143,6 +143,10 @@ export function DimensionEditor(props: DimensionEditorProps) { }); }; + const setIsCloseable = (isCloseable: boolean) => { + setState({ ...state, isDimensionClosePrevented: !isCloseable }); + }; + const selectedOperationDefinition = selectedColumn && operationDefinitionMap[selectedColumn.operationType]; @@ -393,6 +397,7 @@ export function DimensionEditor(props: DimensionEditorProps) { dimensionGroups={dimensionGroups} isFullscreen={isFullscreen} toggleFullscreen={toggleFullscreen} + setIsCloseable={setIsCloseable} {...services} /> ); @@ -466,6 +471,7 @@ export function DimensionEditor(props: DimensionEditorProps) { operationDefinitionMap={operationDefinitionMap} toggleFullscreen={toggleFullscreen} isFullscreen={isFullscreen} + setIsCloseable={setIsCloseable} {...services} /> )} @@ -563,6 +569,7 @@ export function DimensionEditor(props: DimensionEditorProps) { operationDefinitionMap={operationDefinitionMap} toggleFullscreen={toggleFullscreen} isFullscreen={isFullscreen} + setIsCloseable={setIsCloseable} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx index d2d14bfba9ec0f..733f39932f276e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx @@ -53,6 +53,7 @@ describe('reference editor', () => { dimensionGroups: [], isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index 488a32d6fb6f55..47259c4834249b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -52,6 +52,7 @@ export interface ReferenceEditorProps { dimensionGroups: VisualizationDimensionGroupConfig[]; isFullscreen: boolean; toggleFullscreen: () => void; + setIsCloseable: (isCloseable: boolean) => void; // Services uiSettings: IUiSettingsClient; @@ -75,6 +76,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { dimensionGroups, isFullscreen, toggleFullscreen, + setIsCloseable, ...services } = props; @@ -351,6 +353,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { operationDefinitionMap={operationDefinitionMap} isFullscreen={isFullscreen} toggleFullscreen={toggleFullscreen} + setIsCloseable={setIsCloseable} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 81eb46e8167155..df749d73c5b887 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -317,6 +317,11 @@ export function getIndexPatternDatasource({ domElement ); }, + + canCloseDimensionEditor: (state) => { + return !state.isDimensionClosePrevented; + }, + getDropProps, onDrop, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index 5130e3ce415fda..7abf274c60e8ba 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -100,6 +100,7 @@ const defaultOptions = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('date_histogram', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index fc90cf4fc61960..75068817c61237 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -30,6 +30,7 @@ const defaultProps = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; // mocking random id generator function diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 9d04ac1c803918..584ea5da38957f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -246,6 +246,7 @@ function FormulaEditor({ data, toggleFullscreen, isFullscreen, + setIsCloseable, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [isHelpOpen, setIsHelpOpen] = useState(false); @@ -253,7 +254,7 @@ function FormulaEditor({ monaco.editor.createModel(text ?? '', LANGUAGE_ID) ); const overflowDiv1 = React.useRef(); - const updateAfterTyping = React.useRef(); + const disposables = React.useRef([]); const editor1 = React.useRef(); // The Monaco editor needs to have the overflowDiv in the first render. Using an effect @@ -270,13 +271,13 @@ function FormulaEditor({ // Clean up the monaco editor and DOM on unmount useEffect(() => { const model = editorModel.current; - const disposable1 = updateAfterTyping.current; + const allDisposables = disposables.current; const editor1ref = editor1.current; return () => { model.dispose(); overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - disposable1?.dispose(); editor1ref?.dispose(); + allDisposables?.forEach((d) => d.dispose()); }; }, []); @@ -599,28 +600,6 @@ function FormulaEditor({ [] ); - const registerOnTypeHandler = useCallback( - (editor: monaco.editor.IStandaloneCodeEditor) => { - // Toggle between two different callbacks when the editors change - if (updateAfterTyping.current) { - updateAfterTyping.current.dispose(); - } - updateAfterTyping.current = editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }); - }, - [onTypeHandler] - ); - - // Toggle between the handlers whenever the full screen mode is changed, - // because Monaco only maintains cursor position in the active model - // while it has focus. - useEffect(() => { - if (updateAfterTyping.current) { - if (editor1.current) registerOnTypeHandler(editor1.current); - } - }, [isFullscreen, registerOnTypeHandler]); - const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -708,7 +687,23 @@ function FormulaEditor({ }} editorDidMount={(editor) => { editor1.current = editor; - registerOnTypeHandler(editor); + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); }} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index ebac3439bdae7a..27982243f8c2b1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -151,6 +151,7 @@ export interface ParamEditorProps { layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; toggleFullscreen: () => void; + setIsCloseable: (isCloseable: boolean) => void; isFullscreen: boolean; columnId: string; indexPattern: IndexPattern; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index 52be5bcf534a93..76562dd9b3d44c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -32,6 +32,7 @@ const defaultProps = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('last_value', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index dd913f9ecfd110..3a2c1aeebf1860 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -34,6 +34,7 @@ const defaultProps = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('percentile', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx index 9832aa5edaffe7..3a9c2ca583cd25 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx @@ -93,6 +93,7 @@ const defaultOptions = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('ranges', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 65748768f388fa..948675bd9ac9eb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -31,6 +31,7 @@ const defaultProps = { operationDefinitionMap: {}, isFullscreen: false, toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('terms', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 98dc767c44c7dd..f24c39f810b214 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -88,6 +88,8 @@ export interface IndexPatternPrivateState { isFirstExistenceFetch: boolean; existenceFetchFailed?: boolean; existenceFetchTimeout?: boolean; + + isDimensionClosePrevented?: boolean; } export interface IndexPatternRef { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 7e8fa69b9650bc..aded0dd478e724 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -213,6 +213,11 @@ export interface Datasource { } ) => { dropTypes: DropType[]; nextLabel?: string } | undefined; onDrop: (props: DatasourceDimensionDropHandlerProps) => false | true | { deleted: string }; + /** + * The datasource is allowed to cancel a close event on the dimension editor, + * mainly used for formulas + */ + canCloseDimensionEditor?: (state: T) => boolean; updateStateOnCloseDimension?: (props: { layerId: string; columnId: string; From bd87d98fbdeabe93ac5a2a020a20cf68b36b938f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:29:44 +0200 Subject: [PATCH 085/185] refactoring --- .../config_panel/config_panel.tsx | 3 +- .../dimension_panel/dimension_editor.tsx | 2 +- .../formula/{ => editor}/formula.scss | 0 .../formula/editor/formula_editor.tsx | 547 +++++++++++ .../formula/editor/formula_help.tsx | 175 ++++ .../definitions/formula/editor/index.ts | 8 + .../{ => editor}/math_completion.test.ts | 0 .../formula/{ => editor}/math_completion.ts | 8 +- .../{ => editor}/math_tokenization.tsx | 0 .../definitions/formula/formula.tsx | 907 +----------------- .../definitions/formula/generate.ts | 90 ++ .../operations/definitions/formula/parse.ts | 150 +++ .../operations/definitions/formula/util.ts | 69 +- .../definitions/formula/validation.ts | 23 +- .../operations/layer_helpers.ts | 30 +- 15 files changed, 1031 insertions(+), 981 deletions(-) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/formula.scss (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.test.ts (100%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.ts (98%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_tokenization.tsx (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 0c26530b6172ca..79c7882a8d56e3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -63,7 +63,8 @@ export function LayerPanels( () => (datasourceId: string, newState: unknown) => { dispatch({ type: 'UPDATE_DATASOURCE_STATE', - updater: () => newState, + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, datasourceId, clearStagedPreview: false, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 28a5438b7af080..c85bc9188f0833 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -144,7 +144,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const setIsCloseable = (isCloseable: boolean) => { - setState({ ...state, isDimensionClosePrevented: !isCloseable }); + setState((prevState) => ({ ...prevState, isDimensionClosePrevented: !isCloseable })); }; const selectedOperationDefinition = diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx new file mode 100644 index 00000000000000..7b96aec4194a4b --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -0,0 +1,547 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiPopover } from '@elastic/eui'; +import { monaco } from '@kbn/monaco'; +import classNames from 'classnames'; +import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; +import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { ParamEditorProps } from '../../index'; +import { getManagedColumnsFrom } from '../../../layer_helpers'; +import { ErrorWrapper, runASTValidation, tryToParse } from '../validation'; +import { useDebounceWithOptions } from '../../helpers'; +import { + LensMathSuggestion, + SUGGESTION_TYPE, + suggest, + getSuggestion, + getSignatureHelp, + getHover, + getTokenInfo, + offsetToRowColumn, + monacoPositionToOffset, +} from './math_completion'; +import { LANGUAGE_ID } from './math_tokenization'; +import { MemoizedFormulaHelp } from './formula_help'; + +import './formula.scss'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from '../formula'; + +export function FormulaEditor({ + layer, + updateLayer, + currentColumn, + columnId, + indexPattern, + operationDefinitionMap, + data, + toggleFullscreen, + isFullscreen, + setIsCloseable, +}: ParamEditorProps) { + const [text, setText] = useState(currentColumn.params.formula); + const [isHelpOpen, setIsHelpOpen] = useState(false); + const editorModel = React.useRef( + monaco.editor.createModel(text ?? '', LANGUAGE_ID) + ); + const overflowDiv1 = React.useRef(); + const disposables = React.useRef([]); + const editor1 = React.useRef(); + + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect + // requires a second render to work, so we are using an if statement to guarantee it happens + // on first render + if (!overflowDiv1?.current) { + const node1 = (overflowDiv1.current = document.createElement('div')); + node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node1); + } + + // Clean up the monaco editor and DOM on unmount + useEffect(() => { + const model = editorModel.current; + const allDisposables = disposables.current; + const editor1ref = editor1.current; + return () => { + model.dispose(); + overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); + editor1ref?.dispose(); + allDisposables?.forEach((d) => d.dispose()); + }; + }, []); + + useDebounceWithOptions( + () => { + if (!editorModel.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + if (currentColumn.params.formula) { + // Only submit if valid + const { newLayer } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + } + + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (error) { + errors = [error]; + } else if (root) { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel.current, + 'LENS', + errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }) + ); + } else { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + + // Only submit if valid + const { newLayer, locations } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + + const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); + const markers: monaco.editor.IMarkerData[] = managedColumns + .flatMap(([id, column]) => { + if (locations[id]) { + const def = operationDefinitionMap[column.operationType]; + if (def.getErrorMessage) { + const messages = def.getErrorMessage( + newLayer, + id, + indexPattern, + operationDefinitionMap + ); + if (messages) { + const startPosition = offsetToRowColumn(text, locations[id].min); + const endPosition = offsetToRowColumn(text, locations[id].max); + return [ + { + message: messages.join(', '), + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Warning, + }, + ]; + } + } + } + return []; + }) + .filter((marker) => marker); + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); + } + }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: text == null }, + 256, + [text] + ); + + /** + * The way that Monaco requests autocompletion is not intuitive, but the way we use it + * we fetch new suggestions in these scenarios: + * + * - If the user types one of the trigger characters, suggestions are always fetched + * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after + * - When the user types the first character into an empty text box, Monaco requests suggestions + * + * Monaco also triggers suggestions automatically when there are no suggestions being displayed + * and the user types a non-whitespace character. + * + * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. + */ + const provideCompletionItems = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + context: monaco.languages.CompletionContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + let wordRange: monaco.Range; + let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { + list: [], + type: SUGGESTION_TYPE.FIELD, + }; + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + + if (context.triggerCharacter === '(') { + const wordUntil = model.getWordAtPosition(position.delta(0, -3)); + if (wordUntil) { + wordRange = new monaco.Range( + position.lineNumber, + position.column, + position.lineNumber, + position.column + ); + + // Retrieve suggestions for subexpressions + // TODO: make this work for expressions nested more than one level deep + aSuggestions = await suggest({ + expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + } else { + aSuggestions = await suggest({ + expression: innerText, + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + + return { + suggestions: aSuggestions.list.map((s) => + getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + ), + }; + }, + [indexPattern, operationDefinitionMap, data] + ); + + const provideSignatureHelp = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken, + context: monaco.languages.SignatureHelpContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getSignatureHelp( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const provideHover = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getHover( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const onTypeHandler = useCallback( + (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { + if (e.isFlush || e.isRedoing || e.isUndoing) { + return; + } + if (e.changes.length === 1 && e.changes[0].text === '=') { + const currentPosition = e.changes[0].range; + if (currentPosition) { + const tokenInfo = getTokenInfo( + editor.getValue(), + monacoPositionToOffset( + editor.getValue(), + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + ); + // Make sure that we are only adding kql='' or lucene='', and also + // check that the = sign isn't inside the KQL expression like kql='=' + if ( + !tokenInfo || + typeof tokenInfo.ast === 'number' || + tokenInfo.ast.type !== 'namedArgument' || + (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || + tokenInfo.ast.value !== 'LENS_MATH_MARKER' + ) { + return; + } + + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.executeEdits( + 'LENS', + [ + { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }, + ], + [ + // After inserting, move the cursor in between the single quotes + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); + }, 0); + } + } + }, + [] + ); + + const codeEditorOptions: CodeEditorProps = { + languageId: LANGUAGE_ID, + value: text ?? '', + onChange: setText, + options: { + automaticLayout: false, + fontSize: 14, + folding: false, + lineNumbers: 'off', + scrollBeyondLastLine: false, + minimap: { enabled: false }, + wordWrap: 'on', + // Disable suggestions that appear when we don't provide a default suggestion + wordBasedSuggestions: false, + autoIndent: 'brackets', + wrappingIndent: 'none', + dimension: { width: 290, height: 200 }, + fixedOverflowWidgets: true, + }, + }; + + useEffect(() => { + // Because the monaco model is owned by Lens, we need to manually attach and remove handlers + const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + triggerCharacters: ['.', '(', '=', ' ', ':', `'`], + provideCompletionItems, + }); + const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + signatureHelpTriggerCharacters: ['(', '='], + provideSignatureHelp, + }); + const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { + provideHover, + }); + return () => { + dispose1(); + dispose2(); + dispose3(); + }; + }, [provideCompletionItems, provideSignatureHelp, provideHover]); + + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences + // in the behavior of Monaco when it's first loaded and then reloaded. + return ( +
+
+ + + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="s" + color="text" + flush="right" + > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse formula', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand formula', + })} + + + +
+
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> + +
+
+ + + {isFullscreen ? ( + + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} + + + {/* Errors go here */} + +
+
+ ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx new file mode 100644 index 00000000000000..1335cfe7e3efaa --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiSelectable, + EuiSelectableOption, +} from '@elastic/eui'; +import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { GenericOperationDefinition, ParamEditorProps } from '../../index'; +import { IndexPattern } from '../../../../types'; +import { tinymathFunctions } from '../util'; +import { getPossibleFunctions } from './math_completion'; + +import { FormulaIndexPatternColumn } from '../formula'; + +function FormulaHelp({ + indexPattern, + operationDefinitionMap, +}: { + indexPattern: IndexPattern; + operationDefinitionMap: Record; +}) { + const [selectedFunction, setSelectedFunction] = useState(); + + const helpItems: Array = []; + + helpItems.push({ label: 'Math', isGroupLabel: true }); + + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .map((key) => ({ + label: `${key}`, + description: , + checked: selectedFunction === key ? ('on' as const) : undefined, + })) + ); + + helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + + // Es aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in operationDefinitionMap) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + return ( + + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + + )} + + + + ); +} + +export const MemoizedFormulaHelp = React.memo(FormulaHelp); + +// TODO: i18n this whole thing, or move examples into the operation definitions with i18n +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const definition = operationDefinitionMap[type]; + + if (type === 'count') { + return ( + +

Example: count()

+ + ); + } + + return ( + + {definition.input === 'field' ?

Example: {type}(bytes)

: null} + {definition.input === 'fullReference' && !('operationParams' in definition) ? ( +

Example: {type}(sum(bytes))

+ ) : null} + + {'operationParams' in definition && definition.operationParams ? ( +

+

+ Example: {type}(sum(bytes),{' '} + {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) +

+

+ ) : null} + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts new file mode 100644 index 00000000000000..4b6acefa6b30ad --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './formula_editor'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts similarity index 98% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 1ae5da9d6db1d6..e8c16fe64651a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -15,10 +15,10 @@ import { TinymathNamedArgument, } from '@kbn/tinymath'; import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; -import { IndexPattern } from '../../../types'; -import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; -import { tinymathFunctions, groupArgsByType } from './util'; -import type { GenericOperationDefinition } from '..'; +import { IndexPattern } from '../../../../types'; +import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; +import { tinymathFunctions, groupArgsByType } from '../util'; +import type { GenericOperationDefinition } from '../..'; export enum SUGGESTION_TYPE { FIELD = 'field', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 584ea5da38957f..6f0abe8f55568d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,61 +5,16 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; -import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; -import { - EuiButtonEmpty, - EuiFlexGroup, - EuiFlexItem, - EuiText, - EuiSpacer, - EuiPopover, - EuiSelectable, - EuiSelectableOption, -} from '@elastic/eui'; -import { monaco } from '@kbn/monaco'; -import classNames from 'classnames'; -import { CodeEditor, Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; -import type { CodeEditorProps } from '../../../../../../../../src/plugins/kibana_react/public'; -import { - OperationDefinition, - GenericOperationDefinition, - IndexPatternColumn, - ParamEditorProps, -} from '../index'; +import type { TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder, getManagedColumnsFrom } from '../../layer_helpers'; -import { mathOperation } from './math'; -import { documentField } from '../../../document_field'; -import { ErrorWrapper, runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; -import { - extractParamsForFormula, - findVariables, - getOperationParams, - getSafeFieldName, - groupArgsByType, - hasMathNode, - tinymathFunctions, -} from './util'; -import { useDebounceWithOptions } from '../helpers'; -import { - LensMathSuggestion, - SUGGESTION_TYPE, - suggest, - getSuggestion, - getPossibleFunctions, - getSignatureHelp, - getHover, - getTokenInfo, - offsetToRowColumn, - monacoPositionToOffset, -} from './math_completion'; -import { LANGUAGE_ID } from './math_tokenization'; - -import './formula.scss'; +import { getColumnOrder } from '../../layer_helpers'; +import { runASTValidation, tryToParse } from './validation'; +import { FormulaEditor } from './editor'; +import { parseAndExtract } from './parse'; +import { generateFormula } from './generate'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', @@ -153,41 +108,12 @@ export const formulaOperation: OperationDefinition< buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { let previousFormula = ''; if (previousColumn) { - if ('references' in previousColumn) { - const metric = layer.columns[previousColumn.references[0]]; - if (metric && 'sourceField' in metric && metric.dataType === 'number') { - const fieldName = getSafeFieldName(metric.sourceField); - // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; - } - } else { - if ( - previousColumn && - 'sourceField' in previousColumn && - previousColumn.dataType === 'number' - ) { - previousFormula += `${previousColumn.operationType}(${getSafeFieldName( - previousColumn?.sourceField - )}`; - } - } - const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); - if (formulaNamedArgs.length) { - previousFormula += - ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); - } - if (previousColumn.filter) { - if (previousColumn.operationType !== 'count') { - previousFormula += ', '; - } - previousFormula += - (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + - `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all - } - if (previousFormula) { - // close the formula at the end - previousFormula += ')'; - } + previousFormula = generateFormula( + previousColumn, + layer, + previousFormula, + operationDefinitionMap + ); } // carry over the format settings from previous operation for seamless transfer // NOTE: this works only for non-default formatters set in Lens @@ -207,11 +133,8 @@ export const formulaOperation: OperationDefinition< references: [], }; }, - isTransferable: (column, newIndexPattern, operationDefinitionMap) => { - // Basic idea: if it has any math operation in it, probably it cannot be transferable - const { root, error } = tryToParse(column.params.formula || ''); - if (!root) return true; - return Boolean(!error && !hasMathNode(root)); + isTransferable: () => { + return true; }, createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) { const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn; @@ -236,660 +159,6 @@ export const formulaOperation: OperationDefinition< paramEditor: FormulaEditor, }; -function FormulaEditor({ - layer, - updateLayer, - currentColumn, - columnId, - indexPattern, - operationDefinitionMap, - data, - toggleFullscreen, - isFullscreen, - setIsCloseable, -}: ParamEditorProps) { - const [text, setText] = useState(currentColumn.params.formula); - const [isHelpOpen, setIsHelpOpen] = useState(false); - const editorModel = React.useRef( - monaco.editor.createModel(text ?? '', LANGUAGE_ID) - ); - const overflowDiv1 = React.useRef(); - const disposables = React.useRef([]); - const editor1 = React.useRef(); - - // The Monaco editor needs to have the overflowDiv in the first render. Using an effect - // requires a second render to work, so we are using an if statement to guarantee it happens - // on first render - if (!overflowDiv1?.current) { - const node1 = (overflowDiv1.current = document.createElement('div')); - node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Monaco CSS is targeted on the monaco-editor class - node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(node1); - } - - // Clean up the monaco editor and DOM on unmount - useEffect(() => { - const model = editorModel.current; - const allDisposables = disposables.current; - const editor1ref = editor1.current; - return () => { - model.dispose(); - overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - editor1ref?.dispose(); - allDisposables?.forEach((d) => d.dispose()); - }; - }, []); - - useDebounceWithOptions( - () => { - if (!editorModel.current) return; - - if (!text) { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - if (currentColumn.params.formula) { - // Only submit if valid - const { newLayer } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - } - - return; - } - - let errors: ErrorWrapper[] = []; - - const { root, error } = tryToParse(text); - if (error) { - errors = [error]; - } else if (root) { - const validationErrors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap - ); - if (validationErrors.length) { - errors = validationErrors; - } - } - - if (errors.length) { - monaco.editor.setModelMarkers( - editorModel.current, - 'LENS', - errors.flatMap((innerError) => { - if (innerError.locations.length) { - return innerError.locations.map((location) => { - const startPosition = offsetToRowColumn(text, location.min); - const endPosition = offsetToRowColumn(text, location.max); - return { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }; - }); - } else { - // Parse errors return no location info - const startPosition = offsetToRowColumn(text, 0); - const endPosition = offsetToRowColumn(text, text.length - 1); - return [ - { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }, - ]; - } - }) - ); - } else { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - - // Only submit if valid - const { newLayer, locations } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - - const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); - const markers: monaco.editor.IMarkerData[] = managedColumns - .flatMap(([id, column]) => { - if (locations[id]) { - const def = operationDefinitionMap[column.operationType]; - if (def.getErrorMessage) { - const messages = def.getErrorMessage( - newLayer, - id, - indexPattern, - operationDefinitionMap - ); - if (messages) { - const startPosition = offsetToRowColumn(text, locations[id].min); - const endPosition = offsetToRowColumn(text, locations[id].max); - return [ - { - message: messages.join(', '), - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: monaco.MarkerSeverity.Warning, - }, - ]; - } - } - } - return []; - }) - .filter((marker) => marker); - monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); - } - }, - // Make it validate on flyout open in case of a broken formula left over - // from a previous edit - { skipFirstRender: text == null }, - 256, - [text] - ); - - /** - * The way that Monaco requests autocompletion is not intuitive, but the way we use it - * we fetch new suggestions in these scenarios: - * - * - If the user types one of the trigger characters, suggestions are always fetched - * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after - * - When the user types the first character into an empty text box, Monaco requests suggestions - * - * Monaco also triggers suggestions automatically when there are no suggestions being displayed - * and the user types a non-whitespace character. - * - * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. - */ - const provideCompletionItems = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - context: monaco.languages.CompletionContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - let wordRange: monaco.Range; - let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { - list: [], - type: SUGGESTION_TYPE.FIELD, - }; - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - - if (context.triggerCharacter === '(') { - const wordUntil = model.getWordAtPosition(position.delta(0, -3)); - if (wordUntil) { - wordRange = new monaco.Range( - position.lineNumber, - position.column, - position.lineNumber, - position.column - ); - - // Retrieve suggestions for subexpressions - // TODO: make this work for expressions nested more than one level deep - aSuggestions = await suggest({ - expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - } else { - aSuggestions = await suggest({ - expression: innerText, - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - - return { - suggestions: aSuggestions.list.map((s) => - getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) - ), - }; - }, - [indexPattern, operationDefinitionMap, data] - ); - - const provideSignatureHelp = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken, - context: monaco.languages.SignatureHelpContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getSignatureHelp( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const provideHover = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getHover( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const onTypeHandler = useCallback( - (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { - if (e.isFlush || e.isRedoing || e.isUndoing) { - return; - } - if (e.changes.length === 1 && e.changes[0].text === '=') { - const currentPosition = e.changes[0].range; - if (currentPosition) { - const tokenInfo = getTokenInfo( - editor.getValue(), - monacoPositionToOffset( - editor.getValue(), - new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) - ) - ); - // Make sure that we are only adding kql='' or lucene='', and also - // check that the = sign isn't inside the KQL expression like kql='=' - if ( - !tokenInfo || - typeof tokenInfo.ast === 'number' || - tokenInfo.ast.type !== 'namedArgument' || - (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || - tokenInfo.ast.value !== 'LENS_MATH_MARKER' - ) { - return; - } - - // Timeout is required because otherwise the cursor position is not updated. - setTimeout(() => { - editor.executeEdits( - 'LENS', - [ - { - range: { - ...currentPosition, - // Insert after the current char - startColumn: currentPosition.startColumn + 1, - endColumn: currentPosition.startColumn + 1, - }, - text: `''`, - }, - ], - [ - // After inserting, move the cursor in between the single quotes - new monaco.Selection( - currentPosition.startLineNumber, - currentPosition.startColumn + 2, - currentPosition.startLineNumber, - currentPosition.startColumn + 2 - ), - ] - ); - editor.trigger('lens', 'editor.action.triggerSuggest', {}); - }, 0); - } - } - }, - [] - ); - - const codeEditorOptions: CodeEditorProps = { - languageId: LANGUAGE_ID, - value: text ?? '', - onChange: setText, - options: { - automaticLayout: false, - fontSize: 14, - folding: false, - lineNumbers: 'off', - scrollBeyondLastLine: false, - minimap: { enabled: false }, - wordWrap: 'on', - // Disable suggestions that appear when we don't provide a default suggestion - wordBasedSuggestions: false, - autoIndent: 'brackets', - wrappingIndent: 'none', - dimension: { width: 290, height: 200 }, - fixedOverflowWidgets: true, - }, - }; - - useEffect(() => { - // Because the monaco model is owned by Lens, we need to manually attach and remove handlers - const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', '(', '=', ' ', ':', `'`], - provideCompletionItems, - }); - const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { - signatureHelpTriggerCharacters: ['(', '='], - provideSignatureHelp, - }); - const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { - provideHover, - }); - return () => { - dispose1(); - dispose2(); - dispose3(); - }; - }, [provideCompletionItems, provideSignatureHelp, provideHover]); - - // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences - // in the behavior of Monaco when it's first loaded and then reloaded. - return ( -
-
- - - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="s" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse formula', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand formula', - })} - - - -
-
- { - editor1.current = editor; - disposables.current.push( - editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); - }) - ); - disposables.current.push( - editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); - }) - ); - // If we ever introduce a second Monaco editor, we need to toggle - // the typing handler to the active editor to maintain the cursor - disposables.current.push( - editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }) - ); - }} - /> - -
-
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" - > - - - )} - - - {/* Errors go here */} - -
-
- ); -} - -function FormulaHelp({ - indexPattern, - operationDefinitionMap, -}: { - indexPattern: IndexPattern; - operationDefinitionMap: Record; -}) { - const [selectedFunction, setSelectedFunction] = useState(); - - const helpItems: Array = []; - - helpItems.push({ label: 'Math', isGroupLabel: true }); - - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in tinymathFunctions) - .map((key) => ({ - label: `${key}`, - description: , - checked: selectedFunction === key ? ('on' as const) : undefined, - })) - ); - - helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); - - // Es aggs - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in operationDefinitionMap) - .map((key) => ({ - label: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - checked: - selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` - ? ('on' as const) - : undefined, - })) - ); - - return ( - - - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); - } else { - setSelectedFunction(chosenType.label); - } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - - )} - - - - ); -} - -const MemoizedFormulaHelp = React.memo(FormulaHelp); - -function parseAndExtract( - text: string, - layer: IndexPatternLayer, - columnId: string, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { root, error } = tryToParse(text); - if (error || !root) { - return { extracted: [], isValid: false }; - } - // before extracting the data run the validation task and throw if invalid - const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); - if (errors.length) { - return { extracted: [], isValid: false }; - } - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); - return { extracted, isValid: true }; -} - export function regenerateLayerFromAst( text: string, layer: IndexPatternLayer, @@ -947,149 +216,3 @@ export function regenerateLayerFromAst( // turn ast into referenced columns // set state } - -function extractColumns( - idPrefix: string, - operations: Record, - ast: TinymathAST, - layer: IndexPatternLayer, - indexPattern: IndexPattern -): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { - const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; - - function parseNode(node: TinymathAST) { - if (typeof node === 'number' || node.type !== 'function') { - // leaf node - return node; - } - - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - // it's a regular math node - const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< - number | TinymathVariable - >; - return { - ...node, - args: consumedArgs, - }; - } - - // split the args into types for better TS experience - const { namedArguments, variables, functions } = groupArgsByType(node.args); - - // operation node - if (nodeOperation.input === 'field') { - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - // a validation task passed before executing this and checked already there's a field - const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value)! - : documentField; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'field' - >).buildColumn( - { - layer, - indexPattern, - field, - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - - if (nodeOperation.input === 'fullReference') { - const [referencedOp] = functions; - const consumedParam = parseNode(referencedOp); - - const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = subNodeVariables.map(({ value }) => value); - mathColumn.params.tinymathAst = consumedParam!; - columns.push({ column: mathColumn }); - mathColumn.customLabel = true; - mathColumn.label = `${idPrefix}X${columns.length - 1}`; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'fullReference' - >).buildColumn( - { - layer, - indexPattern, - referenceIds: [`${idPrefix}X${columns.length - 1}`], - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - } - const root = parseNode(ast); - if (root === undefined) { - return []; - } - const variables = findVariables(root); - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = variables.map(({ value }) => value); - mathColumn.params.tinymathAst = root!; - const newColId = `${idPrefix}X${columns.length}`; - mathColumn.customLabel = true; - mathColumn.label = newColId; - columns.push({ column: mathColumn }); - return columns; -} - -// TODO: i18n this whole thing, or move examples into the operation definitions with i18n -function getHelpText( - type: string, - operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -) { - const definition = operationDefinitionMap[type]; - - if (type === 'count') { - return ( - -

Example: count()

- - ); - } - - return ( - - {definition.input === 'field' ?

Example: {type}(bytes)

: null} - {definition.input === 'fullReference' && !('operationParams' in definition) ? ( -

Example: {type}(sum(bytes))

- ) : null} - - {'operationParams' in definition && definition.operationParams ? ( -

-

- Example: {type}(sum(bytes),{' '} - {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) -

-

- ) : null} - - ); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts new file mode 100644 index 00000000000000..e44cd50ae9c412 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; + +// Just handle two levels for now +type OperationParams = Record>; + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function generateFormula( + previousColumn: ReferenceBasedIndexPatternColumn | IndexPatternColumn, + layer: IndexPatternLayer, + previousFormula: string, + operationDefinitionMap: Record | undefined +) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + if (metric && 'sourceField' in metric && metric.dataType === 'number') { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; + } + } else { + if (previousColumn && 'sourceField' in previousColumn && previousColumn.dataType === 'number') { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )}`; + } + } + const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); + if (formulaNamedArgs.length) { + previousFormula += + ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); + } + if (previousColumn.filter) { + if (previousColumn.operationType !== 'count') { + previousFormula += ', '; + } + previousFormula += + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all + } + if (previousFormula) { + // close the formula at the end + previousFormula += ')'; + } + return previousFormula; +} + +function extractParamsForFormula( + column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, + operationDefinitionMap: Record | undefined +) { + if (!operationDefinitionMap) { + return []; + } + const def = operationDefinitionMap[column.operationType]; + if ('operationParams' in def && column.params) { + return (def.operationParams || []).flatMap(({ name, required }) => { + const value = (column.params as OperationParams)![name]; + if (isObject(value)) { + return Object.keys(value).map((subName) => ({ + name: `${name}-${subName}`, + value: value[subName] as string | number, + required, + })); + } + return { + name, + value, + required, + }; + }); + } + return []; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts new file mode 100644 index 00000000000000..9ddc1973044f8a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { mathOperation } from './math'; +import { documentField } from '../../../document_field'; +import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; +import { findVariables, getOperationParams, groupArgsByType } from './util'; + +export function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { root, error } = tryToParse(text); + if (error || !root) { + return { extracted: [], isValid: false }; + } + // before extracting the data run the validation task and throw if invalid + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + if (errors.length) { + return { extracted: [], isValid: false }; + } + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; +} + +function extractColumns( + idPrefix: string, + operations: Record, + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern +): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { + const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { + // leaf node + return node; + } + + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; + return { + ...node, + args: consumedArgs, + }; + } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + + // operation node + if (nodeOperation.input === 'field') { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value)! + : documentField; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn( + { + layer, + indexPattern, + field, + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const [referencedOp] = functions; + const consumedParam = parseNode(referencedOp); + + const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = subNodeVariables.map(({ value }) => value); + mathColumn.params.tinymathAst = consumedParam!; + columns.push({ column: mathColumn }); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + } + const root = parseNode(ast); + if (root === undefined) { + return []; + } + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables.map(({ value }) => value); + mathColumn.params.tinymathAst = root!; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push({ column: mathColumn }); + return columns; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 17ca19839a216b..5d9a8647eb7ab0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -10,12 +10,10 @@ import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathFunction, - TinymathLocation, TinymathNamedArgument, TinymathVariable, } from 'packages/kbn-tinymath'; -import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; import type { GroupedNodes } from './types'; export function groupArgsByType(args: TinymathAST[]) { @@ -43,45 +41,6 @@ export function getValueOrName(node: TinymathAST) { return node.name; } -export function getSafeFieldName(fieldName: string | undefined) { - // clean up the "Records" field for now - if (!fieldName || fieldName === 'Records') { - return ''; - } - return fieldName; -} - -// Just handle two levels for now -type OeprationParams = Record>; - -export function extractParamsForFormula( - column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, - operationDefinitionMap: Record | undefined -) { - if (!operationDefinitionMap) { - return []; - } - const def = operationDefinitionMap[column.operationType]; - if ('operationParams' in def && column.params) { - return (def.operationParams || []).flatMap(({ name, required }) => { - const value = (column.params as OeprationParams)![name]; - if (isObject(value)) { - return Object.keys(value).map((subName) => ({ - name: `${name}-${subName}`, - value: value[subName] as string | number, - required, - })); - } - return { - name, - value, - required, - }; - }); - } - return []; -} - export function getOperationParams( operation: | OperationDefinition @@ -332,32 +291,6 @@ export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { return flattenMathNodes(root); } -export function hasMathNode(root: TinymathAST): boolean { - return Boolean(findMathNodes(root).length); -} - -function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { - function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); - } - return flattenFunctionNodes(root); -} - -export function hasInvalidOperations( - node: TinymathAST | string, - operations: Record -): { names: string[]; locations: TinymathLocation[] } { - const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); - return { - // avoid duplicates - names: Array.from(new Set(nodes.map(({ name }) => name))), - locations: nodes.map(({ location }) => location), - }; -} - // traverse a tree and find all string leaves export function findVariables(node: TinymathAST | string): TinymathVariable[] { if (typeof node === 'string') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index cb52e22302cbe2..4e5ae21e576e45 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -16,7 +16,6 @@ import { getOperationParams, getValueOrName, groupArgsByType, - hasInvalidOperations, isMathNode, tinymathFunctions, } from './util'; @@ -74,6 +73,28 @@ export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing.message); } +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( + node: TinymathAST | string, + operations: Record +): { names: string[]; locations: TinymathLocation[] } { + const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); + return { + // avoid duplicates + names: Array.from(new Set(nodes.map(({ name }) => name))), + locations: nodes.map(({ location }) => location), + }; +} + export const getQueryValidationError = ( query: string, language: 'kql' | 'lucene', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 49366f2421b7b6..4fd429820379f9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1079,11 +1079,21 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st /** * Returns true if the given column can be applied to the given index pattern */ -export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) { - return operationDefinitionMap[column.operationType].isTransferable( - column, - newIndexPattern, - operationDefinitionMap +export function isColumnTransferable( + column: IndexPatternColumn, + newIndexPattern: IndexPattern, + layer: IndexPatternLayer +): boolean { + return ( + operationDefinitionMap[column.operationType].isTransferable( + column, + newIndexPattern, + operationDefinitionMap + ) && + (!('references' in column) || + column.references.every((columnId) => + isColumnTransferable(layer.columns[columnId], newIndexPattern, layer) + )) ); } @@ -1092,15 +1102,7 @@ export function updateLayerIndexPattern( newIndexPattern: IndexPattern ): IndexPatternLayer { const keptColumns: IndexPatternLayer['columns'] = _.pickBy(layer.columns, (column) => { - if ('references' in column) { - return ( - isColumnTransferable(column, newIndexPattern) && - column.references.every((columnId) => - isColumnTransferable(layer.columns[columnId], newIndexPattern) - ) - ); - } - return isColumnTransferable(column, newIndexPattern); + return isColumnTransferable(column, newIndexPattern, layer); }); const newColumns: IndexPatternLayer['columns'] = _.mapValues(keptColumns, (column) => { const operationDefinition = operationDefinitionMap[column.operationType]; From 571501a26592ff34aab397e4190c4f66a0502d65 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:35:12 +0200 Subject: [PATCH 086/185] move main column generation into parse module --- .../formula/editor/formula_editor.tsx | 3 +- .../definitions/formula/formula.tsx | 66 +------------------ .../operations/definitions/formula/parse.ts | 62 ++++++++++++++++- 3 files changed, 66 insertions(+), 65 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 7b96aec4194a4b..42f4d9cf6ca334 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -31,7 +31,8 @@ import { LANGUAGE_ID } from './math_tokenization'; import { MemoizedFormulaHelp } from './formula_help'; import './formula.scss'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from '../formula'; +import { FormulaIndexPatternColumn } from '../formula'; +import { regenerateLayerFromAst } from '../parse'; export function FormulaEditor({ layer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6f0abe8f55568d..6494c47548f2f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -6,14 +6,12 @@ */ import { i18n } from '@kbn/i18n'; -import type { TinymathLocation } from '@kbn/tinymath'; -import { OperationDefinition, GenericOperationDefinition } from '../index'; +import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder } from '../../layer_helpers'; +import { IndexPattern } from '../../../types'; import { runASTValidation, tryToParse } from './validation'; import { FormulaEditor } from './editor'; -import { parseAndExtract } from './parse'; +import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { @@ -158,61 +156,3 @@ export const formulaOperation: OperationDefinition< paramEditor: FormulaEditor, }; - -export function regenerateLayerFromAst( - text: string, - layer: IndexPatternLayer, - columnId: string, - currentColumn: FormulaIndexPatternColumn, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { extracted, isValid } = parseAndExtract( - text, - layer, - columnId, - indexPattern, - operationDefinitionMap - ); - - const columns = { ...layer.columns }; - - const locations: Record = {}; - - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; - } - }); - - extracted.forEach(({ column, location }, index) => { - columns[`${columnId}X${index}`] = column; - if (location) locations[`${columnId}X${index}`] = location; - }); - - columns[columnId] = { - ...currentColumn, - params: { - ...currentColumn.params, - formula: text, - isFormulaBroken: !isValid, - }, - references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], - }; - - return { - newLayer: { - ...layer, - columns, - columnOrder: getColumnOrder({ - ...layer, - columns, - }), - }, - locations, - }; - - // TODO - // turn ast into referenced columns - // set state -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index 9ddc1973044f8a..70ed2f36dfd1c2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -13,8 +13,10 @@ import { mathOperation } from './math'; import { documentField } from '../../../document_field'; import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; import { findVariables, getOperationParams, groupArgsByType } from './util'; +import { FormulaIndexPatternColumn } from './formula'; +import { getColumnOrder } from '../../layer_helpers'; -export function parseAndExtract( +function parseAndExtract( text: string, layer: IndexPatternLayer, columnId: string, @@ -148,3 +150,61 @@ function extractColumns( columns.push({ column: mathColumn }); return columns; } + +export function regenerateLayerFromAst( + text: string, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { extracted, isValid } = parseAndExtract( + text, + layer, + columnId, + indexPattern, + operationDefinitionMap + ); + + const columns = { ...layer.columns }; + + const locations: Record = {}; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach(({ column, location }, index) => { + columns[`${columnId}X${index}`] = column; + if (location) locations[`${columnId}X${index}`] = location; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text, + isFormulaBroken: !isValid, + }, + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + }; + + return { + newLayer: { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }, + locations, + }; + + // TODO + // turn ast into referenced columns + // set state +} From 70a6b86c86e82f255959d51be70f60803a9678c2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 10:49:08 +0200 Subject: [PATCH 087/185] fix tests --- .../formula/editor/math_completion.test.ts | 12 ++++++------ .../operations/definitions/formula/formula.test.tsx | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 9b5e77b7b90dbb..9e29160b6747b4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -6,12 +6,12 @@ */ import { monaco } from '@kbn/monaco'; -import { createMockedIndexPattern } from '../../../mocks'; -import { GenericOperationDefinition } from '../index'; -import type { IndexPatternField } from '../../../types'; -import type { OperationMetadata } from '../../../../types'; -import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; -import { tinymathFunctions } from './util'; +import { createMockedIndexPattern } from '../../../../mocks'; +import { GenericOperationDefinition } from '../../index'; +import type { IndexPatternField } from '../../../../types'; +import type { OperationMetadata } from '../../../../../types'; +import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; +import { tinymathFunctions } from '../util'; import { getSignatureHelp, getHover, suggest } from './math_completion'; const buildGenericColumn = (type: string) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 433e21eb133451..ce7b48aa1875ea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -7,7 +7,8 @@ import { createMockedIndexPattern } from '../../../mocks'; import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +import { FormulaIndexPatternColumn } from './formula'; +import { regenerateLayerFromAst } from './parse'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; import { tinymathFunctions } from './util'; From 18839645f4193e4798fbdbb4f8b0e82a69ebf14e Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Mon, 10 May 2021 09:38:57 -0400 Subject: [PATCH 088/185] refactor small formula styles and markup --- .../dimension_panel/dimension_editor.scss | 12 +- .../dimension_panel/dimension_editor.tsx | 6 +- .../definitions/formula/formula.scss | 13 ++ .../definitions/formula/formula.tsx | 154 +++++++++--------- 4 files changed, 97 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index 999371b1dadafe..b6b3f55e75f05e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -2,7 +2,7 @@ height: 100%; } -.lnsIndexPatternDimensionEditor-fullscreen { +.lnsIndexPatternDimensionEditor-isFullscreen { position: absolute; top: 0; bottom: 0; @@ -10,7 +10,7 @@ flex-direction: column; } -.lnsIndexPatternDimensionEditor__section { +.lnsIndexPatternDimensionEditor__section--padded { padding: $euiSizeS; } @@ -18,14 +18,6 @@ background-color: $euiColorLightestShade; } -.lnsIndexPatternDimensionEditor__section--top { - border-bottom: $euiBorderThin; -} - -.lnsIndexPatternDimensionEditor__section--bottom { - border-top: $euiBorderThin; -} - .lnsIndexPatternDimensionEditor__columns { column-count: 2; column-gap: $euiSizeXL; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index c270e6b50d70b1..dc4dac64796dea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -344,7 +344,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const quickFunctions = ( <> -
+
{i18n.translate('xpack.lens.indexPattern.functionsLabel', { defaultMessage: 'Select a function', @@ -363,7 +363,7 @@ export function DimensionEditor(props: DimensionEditorProps) { />
-
+
{!incompleteInfo && selectedColumn && 'references' in selectedColumn && @@ -613,7 +613,7 @@ export function DimensionEditor(props: DimensionEditorProps) { )} {!isFullscreen && !currentFieldIsInvalid && ( -
+
{!incompleteInfo && selectedColumn && ( * + * { + border-top: $euiBorderThin; + } +} + +.lnsFormula__header, +.lnsFormula__footer { + padding: $euiSizeS; +} + .lnsFormulaOverflow { // Needs to be higher than the modal and all flyouts z-index: $euiZLevel9 + 1; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6da482e624f989..ce8a8221492063 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -647,88 +647,92 @@ function FormulaEditor({ return (
-
- - - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="s" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse formula', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand formula', - })} - - - -
-
- { - editor1.current = editor; - registerOnTypeHandler(editor); - }} - /> - -
-
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" +
+
+ + {/* TODO: Word wrap button */} + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="xs" + color="text" + flush="right" > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand', + })} + + + +
+ +
+ { + editor1.current = editor; + registerOnTypeHandler(editor); + }} + /> +
+ +
+ + + {isFullscreen ? ( - - )} - - - {/* Errors go here */} - + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} + + + {/* Errors go here */} + +
); From 40c43e412110b2b3f609bc2d8caabb3702079f9b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 15:54:47 +0200 Subject: [PATCH 089/185] documentation --- .../definitions/calculations/counter_rate.tsx | 21 +++ .../calculations/cumulative_sum.tsx | 20 +++ .../definitions/calculations/differences.tsx | 20 +++ .../calculations/moving_average.tsx | 21 +++ .../operations/definitions/cardinality.tsx | 19 +++ .../operations/definitions/count.tsx | 19 +++ .../formula/editor/formula_editor.tsx | 2 + .../formula/editor/formula_help.tsx | 133 +++++++++++------- .../operations/definitions/formula/util.ts | 95 +++++++++---- .../operations/definitions/index.ts | 4 + .../operations/definitions/last_value.tsx | 19 +++ .../operations/definitions/metrics.tsx | 23 +++ .../operations/definitions/percentile.tsx | 17 +++ 13 files changed, 332 insertions(+), 81 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 97582be2f32d66..3fb1367154101f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPatternLayer } from '../../../types'; import { @@ -18,6 +19,7 @@ import { import { DEFAULT_TIME_SCALE } from '../../time_scale_utils'; import { OperationDefinition } from '..'; import { getFormatFromPreviousColumn } from '../helpers'; +import { Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; const ofName = buildLabelFunction((name?: string) => { return i18n.translate('xpack.lens.indexPattern.CounterRateOf', { @@ -126,4 +128,23 @@ export const counterRateOperation: OperationDefinition< }, timeScalingMode: 'mandatory', filterable: true, + documentation: { + section: 'calculation', + description: ( + + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index e6f4f589f6189a..f28f44f0c0dafa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPatternLayer } from '../../../types'; import { @@ -16,6 +17,7 @@ import { } from './utils'; import { OperationDefinition } from '..'; import { getFormatFromPreviousColumn } from '../helpers'; +import { Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { @@ -119,4 +121,22 @@ export const cumulativeSumOperation: OperationDefinition< )?.join(', '); }, filterable: true, + documentation: { + section: 'calculation', + description: ( + + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index b030e604ada061..84222cf1259383 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPatternLayer } from '../../../types'; import { @@ -18,6 +19,7 @@ import { import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils'; import { OperationDefinition } from '..'; import { getFormatFromPreviousColumn } from '../helpers'; +import { Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; const OPERATION_NAME = 'differences'; @@ -116,4 +118,22 @@ export const derivativeOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + documentation: { + section: 'calculation', + description: ( + + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 88af8e9b6378e0..0d2c071ee7ecb5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -23,6 +23,7 @@ import { getFormatFromPreviousColumn, isValidNumber, useDebounceWithOptions } fr import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils'; import { HelpPopover, HelpPopoverButton } from '../../../help_popover'; import type { OperationDefinition, ParamEditorProps } from '..'; +import { Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; const ofName = buildLabelFunction((name?: string) => { return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { @@ -136,6 +137,26 @@ export const movingAverageOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + documentation: { + section: 'calculation', + description: ( + + ), + }, }; function MovingAverageParamEditor({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index df84ecb479de72..63a4b2bd3ad13e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -6,10 +6,12 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { AggFunctionsMapping } from '../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public'; import { OperationDefinition } from './index'; import { FormattedIndexPatternColumn, FieldBasedIndexPatternColumn } from './column_types'; +import { Markdown } from '../../../../../../../src/plugins/kibana_react/public'; import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers'; @@ -112,4 +114,21 @@ export const cardinalityOperation: OperationDefinition + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index d66780a4207e62..d56cb504e9bdf5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { AggFunctionsMapping } from '../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public'; import { OperationDefinition } from './index'; @@ -16,6 +17,7 @@ import { adjustTimeScaleLabelSuffix, adjustTimeScaleOnOtherColumnChange, } from '../time_scale_utils'; +import { Markdown } from '../../../../../../../src/plugins/kibana_react/public'; const countLabel = i18n.translate('xpack.lens.indexPattern.countOf', { defaultMessage: 'Count of records', @@ -97,4 +99,21 @@ export const countOperation: OperationDefinition + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 42f4d9cf6ca334..763e10d1981080 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -512,6 +512,7 @@ export function FormulaEditor({ {isFullscreen ? ( ) : ( @@ -533,6 +534,7 @@ export function FormulaEditor({ anchorPosition="leftDown" > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 1335cfe7e3efaa..7bca50eb9c8463 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, @@ -15,25 +15,37 @@ import { EuiSelectableOption, } from '@elastic/eui'; import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; -import { GenericOperationDefinition, ParamEditorProps } from '../../index'; +import { GenericOperationDefinition } from '../../index'; import { IndexPattern } from '../../../../types'; import { tinymathFunctions } from '../util'; import { getPossibleFunctions } from './math_completion'; -import { FormulaIndexPatternColumn } from '../formula'; - function FormulaHelp({ indexPattern, operationDefinitionMap, + isFullscreen, }: { indexPattern: IndexPattern; operationDefinitionMap: Record; + isFullscreen: boolean; }) { const [selectedFunction, setSelectedFunction] = useState(); + const scrollTargets = useRef>({}); + + useEffect(() => { + if (selectedFunction && scrollTargets.current[selectedFunction]) { + scrollTargets.current[selectedFunction].scrollIntoView(); + } + }, [selectedFunction]); const helpItems: Array = []; - helpItems.push({ label: 'Math', isGroupLabel: true }); + helpItems.push({ + label: i18n.translate('xpack.lens.formulaDocumentation.mathSection', { + defaultMessage: 'Math', + }), + isGroupLabel: true, + }); helpItems.push( ...getPossibleFunctions(indexPattern) @@ -45,15 +57,49 @@ function FormulaHelp({ })) ); - helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + helpItems.push({ + label: i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSection', { + defaultMessage: 'Elasticsearch', + }), + isGroupLabel: true, + }); // Es aggs helpItems.push( ...getPossibleFunctions(indexPattern) - .filter((key) => key in operationDefinitionMap) + .filter( + (key) => + key in operationDefinitionMap && + operationDefinitionMap[key].documentation?.section === 'elasticsearch' + ) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: operationDefinitionMap[key].documentation?.description, + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + helpItems.push({ + label: i18n.translate('xpack.lens.formulaDocumentation.columnCalculationSection', { + defaultMessage: 'Column-wise calculation', + }), + isGroupLabel: true, + }); + + // Calculations aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter( + (key) => + key in operationDefinitionMap && + operationDefinitionMap[key].documentation?.section === 'calculation' + ) .map((key) => ({ label: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), + description: operationDefinitionMap[key].documentation?.description, checked: selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` ? ('on' as const) @@ -62,7 +108,7 @@ function FormulaHelp({ ); return ( - + - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - - )} + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + /> + {helpItems.map((item, index) => { + if (item.isGroupLabel) { + return null; + } else { + return ( +
{ + if (el) { + scrollTargets.current[item.label] = el; + } + }} + > + {item.description} +
+ ); + } + })} @@ -139,37 +198,3 @@ Use the symbols +, -, /, and * to perform basic math. } export const MemoizedFormulaHelp = React.memo(FormulaHelp); - -// TODO: i18n this whole thing, or move examples into the operation definitions with i18n -function getHelpText( - type: string, - operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -) { - const definition = operationDefinitionMap[type]; - - if (type === 'count') { - return ( - -

Example: count()

- - ); - } - - return ( - - {definition.input === 'field' ?

Example: {type}(bytes)

: null} - {definition.input === 'fullReference' && !('operationParams' in definition) ? ( -

Example: {type}(sum(bytes))

- ) : null} - - {'operationParams' in definition && definition.operationParams ? ( -

-

- Example: {type}(sum(bytes),{' '} - {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) -

-

- ) : null} - - ); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 5d9a8647eb7ab0..6b083e59593782 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -85,9 +85,13 @@ export const tinymathFunctions: Record< { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` +# add \`+\` +Adds up two numbers. Also works with + symbol -Example: ${'`count() + sum(bytes)`'} -Example: ${'`add(count(), 5)`'} + +Example: Calculate the sum of two fields \`sum(price) + sum(tax)\` + +Example: Offset count by a static value \`add(count(), 5)\` `, }, subtract: { @@ -96,8 +100,11 @@ Example: ${'`add(count(), 5)`'} { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` +# subtract \`-\` +Subtracts the first number from the second number. Also works with ${'`-`'} symbol -Example: ${'`subtract(sum(bytes), avg(bytes))`'} + +Example: Calculate the range of a field ${'`subtract(max(bytes), min(bytes))`'} `, }, multiply: { @@ -106,8 +113,13 @@ Example: ${'`subtract(sum(bytes), avg(bytes))`'} { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` -Also works with ${'`*`'} symbol -Example: ${'`multiply(sum(bytes), 2)`'} +# multiply \`*\` +Multiplies two numbers. +Also works with ${'`*`'} symbol. + +Example: Calculate price after current tax rate ${'`sum(bytes) * last_value(tax_rate)`'} + +Example: Calculate price after constant tax rate \`multiply(sum(price), 1.2)\` `, }, divide: { @@ -116,8 +128,11 @@ Example: ${'`multiply(sum(bytes), 2)`'} { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` +# divide \`/\` +Divides the first number by the second number. Also works with ${'`/`'} symbol -Example: ${'`ceil(sum(bytes))`'} + +Example: Calculate profit margin \`sum(profit) / sum(revenue)\` `, }, abs: { @@ -125,8 +140,10 @@ Example: ${'`ceil(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -Absolute value -Example: ${'`abs(sum(bytes))`'} +# abs +Calculates absolute value. A negative value is multiplied by -1, a positive value stays the same. + +Example: Calculate average distance to sea level ${'`abs(average(altitude))`'} `, }, cbrt: { @@ -134,8 +151,10 @@ Example: ${'`abs(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -Cube root of value -Example: ${'`cbrt(sum(bytes))`'} +# cbrt +Cube root of value. + +Example: Calculate side length from volume ${'`cbrt(last_value(volume))`'} `, }, ceil: { @@ -143,8 +162,10 @@ Example: ${'`cbrt(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -Ceiling of value, rounds up -Example: ${'`ceil(sum(bytes))`'} +# ceil +Ceiling of value, rounds up. + +Example: Round up price to the next dollar ${'`ceil(sum(price))`'} `, }, clamp: { @@ -154,8 +175,10 @@ Example: ${'`ceil(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.max', { defaultMessage: 'max' }) }, ], help: ` -Limits the value from a minimum to maximum -Example: ${'`ceil(sum(bytes))`'} +# clamp +Limits the value from a minimum to maximum. + +Example: Make sure to catch outliers ${'`clamp(average(bytes), percentile(bytes, percentile=5), percentile(bytes, percentile=95))`'} `, }, cube: { @@ -163,8 +186,10 @@ Example: ${'`ceil(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -Limits the value from a minimum to maximum -Example: ${'`ceil(sum(bytes))`'} +# cube +Calculates the cube of a number. + +Example: Calculate volume from side length ${'`cube(last_value(length))`'} `, }, exp: { @@ -172,8 +197,10 @@ Example: ${'`ceil(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` +# exp Raises e to the nth power. -Example: ${'`exp(sum(bytes))`'} + +Example: Calculate the natural expontential function ${'`exp(last_value(duration))`'} `, }, fix: { @@ -181,8 +208,10 @@ Example: ${'`exp(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` +# fix For positive values, takes the floor. For negative values, takes the ceiling. -Example: ${'`fix(sum(bytes))`'} + +Example: Rounding towards zero ${'`fix(sum(profit))`'} `, }, floor: { @@ -190,8 +219,10 @@ Example: ${'`fix(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` +# floor Round down to nearest integer value -Example: ${'`floor(sum(bytes))`'} + +Example: Round down a price ${'`floor(sum(price))`'} `, }, log: { @@ -203,9 +234,10 @@ Example: ${'`floor(sum(bytes))`'} }, ], help: ` +# log Logarithm with optional base. The natural base e is used as default. -Example: ${'`log(sum(bytes))`'} -Example: ${'`log(sum(bytes), 2)`'} + +Example: Calculate number of bits required to store values ${'`log(max(price), 2)`'} `, }, // TODO: check if this is valid for Tinymath @@ -227,8 +259,10 @@ Example: ${'`log(sum(bytes), 2)`'} }, ], help: ` +# mod Remainder after dividing the function by a number -Example: ${'`mod(sum(bytes), 2)`'} + +Example: Calculate last three digits of a value ${'`mod(sum(price), 1000)`'} `, }, pow: { @@ -239,8 +273,10 @@ Example: ${'`mod(sum(bytes), 2)`'} }, ], help: ` +# pow Raises the value to a certain power. The second argument is required -Example: ${'`pow(sum(bytes), 3)`'} + +Example: Calculate volume based on side length ${'`pow(last_value(length), 3)`'} `, }, round: { @@ -252,9 +288,10 @@ Example: ${'`pow(sum(bytes), 3)`'} }, ], help: ` +# round Rounds to a specific number of decimal places, default of 0 -Example: ${'`round(sum(bytes))`'} -Example: ${'`round(sum(bytes), 2)`'} + +Example: Round to the cent ${'`round(sum(price), 2)`'} `, }, sqrt: { @@ -262,8 +299,10 @@ Example: ${'`round(sum(bytes), 2)`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` +# sqrt Square root of a positive value only -Example: ${'`sqrt(sum(bytes))`'} + +Example: Calculate side length based on area ${'`sqrt(last_value(area))`'} `, }, square: { @@ -271,8 +310,10 @@ Example: ${'`sqrt(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` +# square Raise the value to the 2nd power -Example: ${'`square(sum(bytes))`'} + +Example: Calculate area based on side length ${'`square(last_value(length))`'} `, }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 27982243f8c2b1..510a59b109d102 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -262,6 +262,10 @@ interface BaseOperationDefinitionProps { * Operations can be used as middleware for other operations, hence not shown in the panel UI */ hidden?: boolean; + documentation?: { + description: JSX.Element; + section: 'elasticsearch' | 'calculation'; + }; } interface BaseBuildColumnArgs { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index a61cca89dfecfc..78aace978bfcf0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -16,6 +16,7 @@ import { IndexPatternField, IndexPattern } from '../../types'; import { updateColumnParam } from '../layer_helpers'; import { DataType } from '../../../types'; import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers'; +import { Markdown } from '../../../../../../../src/plugins/kibana_react/public'; function ofName(name: string) { return i18n.translate('xpack.lens.indexPattern.lastValueOf', { @@ -268,4 +269,22 @@ export const lastValueOperation: OperationDefinition ); }, + documentation: { + section: 'elasticsearch', + description: ( + + ), + }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 866d232aab5b38..f1bfb8ccbada69 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import React from 'react'; import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public'; import { OperationDefinition } from './index'; import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers'; @@ -18,6 +19,7 @@ import { adjustTimeScaleLabelSuffix, adjustTimeScaleOnOtherColumnChange, } from '../time_scale_utils'; +import { Markdown } from '../../../../../../../src/plugins/kibana_react/public'; type MetricColumn = FormattedIndexPatternColumn & FieldBasedIndexPatternColumn & { @@ -128,6 +130,27 @@ function buildMetricOperation>({ getErrorMessage: (layer, columnId, indexPattern) => getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), filterable: true, + documentation: { + section: 'elasticsearch', + description: ( + + ), + }, } as OperationDefinition; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 187dc2dc53ffb8..2d621f9baab7da 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -19,6 +19,7 @@ import { useDebounceWithOptions, } from './helpers'; import { FieldBasedIndexPatternColumn } from './column_types'; +import { Markdown } from '../../../../../../../src/plugins/kibana_react/public'; export interface PercentileIndexPatternColumn extends FieldBasedIndexPatternColumn { operationType: 'percentile'; @@ -192,4 +193,20 @@ export const percentileOperation: OperationDefinition ); }, + documentation: { + section: 'elasticsearch', + description: ( + + ), + }, }; From 203b62236f45cd827ffe02a1a9fb76c5ec1e5b3d Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Mon, 10 May 2021 10:55:00 -0400 Subject: [PATCH 090/185] adjustments in formula footer --- .../definitions/formula/formula.tsx | 189 +++++++++--------- 1 file changed, 96 insertions(+), 93 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 2e80875661ba5c..50150048c8882e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -11,10 +11,10 @@ import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; import { EuiButtonEmpty, + EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiText, - EuiSpacer, EuiPopover, EuiSelectable, EuiSelectableOption, @@ -650,103 +650,106 @@ function FormulaEditor({ 'lnsIndexPatternDimensionEditor-isFullscreen': isFullscreen, })} > -
-
- - {/* TODO: Word wrap button */} - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="xs" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand', - })} - - - -
- -
- { - editor1.current = editor; - disposables.current.push( - editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); - }) - ); - disposables.current.push( - editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); - }) - ); - // If we ever introduce a second Monaco editor, we need to toggle - // the typing handler to the active editor to maintain the cursor - disposables.current.push( - editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }) - ); - }} - /> -
- -
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" +
+
+
+ + {/* TODO: Word wrap button */} + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="xs" + color="text" + flush="right" > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand', + })} + + + +
+ +
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> +
+ +
+ + + {isFullscreen ? ( - - )} - - - {/* Errors go here */} - + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + color="text" + aria-label={i18n.translate( + 'xpack.lens.formula.functionReferenceEditorLabel', + { + defaultMessage: 'Function reference', + } + )} + /> + } + anchorPosition="leftDown" + > + + + )} + + + {/* TODO: Errors go here */} + +
From b4a36ade1b551e9ea7f30021b19f02468b46dd54 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 10 May 2021 20:46:22 +0200 Subject: [PATCH 091/185] Formula refactoring (#12) * refactoring * move main column generation into parse module * fix tests --- .../config_panel/config_panel.tsx | 3 +- .../dimension_panel/dimension_editor.tsx | 2 +- .../formula/{ => editor}/formula.scss | 0 .../formula/editor/formula_editor.tsx | 548 ++++++++++ .../formula/editor/formula_help.tsx | 175 ++++ .../definitions/formula/editor/index.ts | 8 + .../{ => editor}/math_completion.test.ts | 12 +- .../formula/{ => editor}/math_completion.ts | 8 +- .../{ => editor}/math_tokenization.tsx | 0 .../definitions/formula/formula.test.tsx | 3 +- .../definitions/formula/formula.tsx | 965 +----------------- .../definitions/formula/generate.ts | 90 ++ .../operations/definitions/formula/parse.ts | 210 ++++ .../operations/definitions/formula/util.ts | 69 +- .../definitions/formula/validation.ts | 23 +- .../operations/layer_helpers.ts | 30 +- 16 files changed, 1099 insertions(+), 1047 deletions(-) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/formula.scss (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.test.ts (96%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_completion.ts (98%) rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/{ => editor}/math_tokenization.tsx (100%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 0c26530b6172ca..79c7882a8d56e3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -63,7 +63,8 @@ export function LayerPanels( () => (datasourceId: string, newState: unknown) => { dispatch({ type: 'UPDATE_DATASOURCE_STATE', - updater: () => newState, + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, datasourceId, clearStagedPreview: false, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 28a5438b7af080..c85bc9188f0833 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -144,7 +144,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const setIsCloseable = (isCloseable: boolean) => { - setState({ ...state, isDimensionClosePrevented: !isCloseable }); + setState((prevState) => ({ ...prevState, isDimensionClosePrevented: !isCloseable })); }; const selectedOperationDefinition = diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx new file mode 100644 index 00000000000000..42f4d9cf6ca334 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -0,0 +1,548 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiPopover } from '@elastic/eui'; +import { monaco } from '@kbn/monaco'; +import classNames from 'classnames'; +import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; +import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { ParamEditorProps } from '../../index'; +import { getManagedColumnsFrom } from '../../../layer_helpers'; +import { ErrorWrapper, runASTValidation, tryToParse } from '../validation'; +import { useDebounceWithOptions } from '../../helpers'; +import { + LensMathSuggestion, + SUGGESTION_TYPE, + suggest, + getSuggestion, + getSignatureHelp, + getHover, + getTokenInfo, + offsetToRowColumn, + monacoPositionToOffset, +} from './math_completion'; +import { LANGUAGE_ID } from './math_tokenization'; +import { MemoizedFormulaHelp } from './formula_help'; + +import './formula.scss'; +import { FormulaIndexPatternColumn } from '../formula'; +import { regenerateLayerFromAst } from '../parse'; + +export function FormulaEditor({ + layer, + updateLayer, + currentColumn, + columnId, + indexPattern, + operationDefinitionMap, + data, + toggleFullscreen, + isFullscreen, + setIsCloseable, +}: ParamEditorProps) { + const [text, setText] = useState(currentColumn.params.formula); + const [isHelpOpen, setIsHelpOpen] = useState(false); + const editorModel = React.useRef( + monaco.editor.createModel(text ?? '', LANGUAGE_ID) + ); + const overflowDiv1 = React.useRef(); + const disposables = React.useRef([]); + const editor1 = React.useRef(); + + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect + // requires a second render to work, so we are using an if statement to guarantee it happens + // on first render + if (!overflowDiv1?.current) { + const node1 = (overflowDiv1.current = document.createElement('div')); + node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node1); + } + + // Clean up the monaco editor and DOM on unmount + useEffect(() => { + const model = editorModel.current; + const allDisposables = disposables.current; + const editor1ref = editor1.current; + return () => { + model.dispose(); + overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); + editor1ref?.dispose(); + allDisposables?.forEach((d) => d.dispose()); + }; + }, []); + + useDebounceWithOptions( + () => { + if (!editorModel.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + if (currentColumn.params.formula) { + // Only submit if valid + const { newLayer } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + } + + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (error) { + errors = [error]; + } else if (root) { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel.current, + 'LENS', + errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }) + ); + } else { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + + // Only submit if valid + const { newLayer, locations } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + + const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); + const markers: monaco.editor.IMarkerData[] = managedColumns + .flatMap(([id, column]) => { + if (locations[id]) { + const def = operationDefinitionMap[column.operationType]; + if (def.getErrorMessage) { + const messages = def.getErrorMessage( + newLayer, + id, + indexPattern, + operationDefinitionMap + ); + if (messages) { + const startPosition = offsetToRowColumn(text, locations[id].min); + const endPosition = offsetToRowColumn(text, locations[id].max); + return [ + { + message: messages.join(', '), + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Warning, + }, + ]; + } + } + } + return []; + }) + .filter((marker) => marker); + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); + } + }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: text == null }, + 256, + [text] + ); + + /** + * The way that Monaco requests autocompletion is not intuitive, but the way we use it + * we fetch new suggestions in these scenarios: + * + * - If the user types one of the trigger characters, suggestions are always fetched + * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after + * - When the user types the first character into an empty text box, Monaco requests suggestions + * + * Monaco also triggers suggestions automatically when there are no suggestions being displayed + * and the user types a non-whitespace character. + * + * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. + */ + const provideCompletionItems = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + context: monaco.languages.CompletionContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + let wordRange: monaco.Range; + let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { + list: [], + type: SUGGESTION_TYPE.FIELD, + }; + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + + if (context.triggerCharacter === '(') { + const wordUntil = model.getWordAtPosition(position.delta(0, -3)); + if (wordUntil) { + wordRange = new monaco.Range( + position.lineNumber, + position.column, + position.lineNumber, + position.column + ); + + // Retrieve suggestions for subexpressions + // TODO: make this work for expressions nested more than one level deep + aSuggestions = await suggest({ + expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + } else { + aSuggestions = await suggest({ + expression: innerText, + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + + return { + suggestions: aSuggestions.list.map((s) => + getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + ), + }; + }, + [indexPattern, operationDefinitionMap, data] + ); + + const provideSignatureHelp = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken, + context: monaco.languages.SignatureHelpContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getSignatureHelp( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const provideHover = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getHover( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const onTypeHandler = useCallback( + (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { + if (e.isFlush || e.isRedoing || e.isUndoing) { + return; + } + if (e.changes.length === 1 && e.changes[0].text === '=') { + const currentPosition = e.changes[0].range; + if (currentPosition) { + const tokenInfo = getTokenInfo( + editor.getValue(), + monacoPositionToOffset( + editor.getValue(), + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + ); + // Make sure that we are only adding kql='' or lucene='', and also + // check that the = sign isn't inside the KQL expression like kql='=' + if ( + !tokenInfo || + typeof tokenInfo.ast === 'number' || + tokenInfo.ast.type !== 'namedArgument' || + (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || + tokenInfo.ast.value !== 'LENS_MATH_MARKER' + ) { + return; + } + + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.executeEdits( + 'LENS', + [ + { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }, + ], + [ + // After inserting, move the cursor in between the single quotes + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); + }, 0); + } + } + }, + [] + ); + + const codeEditorOptions: CodeEditorProps = { + languageId: LANGUAGE_ID, + value: text ?? '', + onChange: setText, + options: { + automaticLayout: false, + fontSize: 14, + folding: false, + lineNumbers: 'off', + scrollBeyondLastLine: false, + minimap: { enabled: false }, + wordWrap: 'on', + // Disable suggestions that appear when we don't provide a default suggestion + wordBasedSuggestions: false, + autoIndent: 'brackets', + wrappingIndent: 'none', + dimension: { width: 290, height: 200 }, + fixedOverflowWidgets: true, + }, + }; + + useEffect(() => { + // Because the monaco model is owned by Lens, we need to manually attach and remove handlers + const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + triggerCharacters: ['.', '(', '=', ' ', ':', `'`], + provideCompletionItems, + }); + const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + signatureHelpTriggerCharacters: ['(', '='], + provideSignatureHelp, + }); + const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { + provideHover, + }); + return () => { + dispose1(); + dispose2(); + dispose3(); + }; + }, [provideCompletionItems, provideSignatureHelp, provideHover]); + + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences + // in the behavior of Monaco when it's first loaded and then reloaded. + return ( +
+
+ + + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="s" + color="text" + flush="right" + > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse formula', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand formula', + })} + + + +
+
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> + +
+
+ + + {isFullscreen ? ( + + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} + + + {/* Errors go here */} + +
+
+ ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx new file mode 100644 index 00000000000000..1335cfe7e3efaa --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiSelectable, + EuiSelectableOption, +} from '@elastic/eui'; +import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { GenericOperationDefinition, ParamEditorProps } from '../../index'; +import { IndexPattern } from '../../../../types'; +import { tinymathFunctions } from '../util'; +import { getPossibleFunctions } from './math_completion'; + +import { FormulaIndexPatternColumn } from '../formula'; + +function FormulaHelp({ + indexPattern, + operationDefinitionMap, +}: { + indexPattern: IndexPattern; + operationDefinitionMap: Record; +}) { + const [selectedFunction, setSelectedFunction] = useState(); + + const helpItems: Array = []; + + helpItems.push({ label: 'Math', isGroupLabel: true }); + + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .map((key) => ({ + label: `${key}`, + description: , + checked: selectedFunction === key ? ('on' as const) : undefined, + })) + ); + + helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + + // Es aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in operationDefinitionMap) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + return ( + + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + + )} + + + + ); +} + +export const MemoizedFormulaHelp = React.memo(FormulaHelp); + +// TODO: i18n this whole thing, or move examples into the operation definitions with i18n +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const definition = operationDefinitionMap[type]; + + if (type === 'count') { + return ( + +

Example: count()

+ + ); + } + + return ( + + {definition.input === 'field' ?

Example: {type}(bytes)

: null} + {definition.input === 'fullReference' && !('operationParams' in definition) ? ( +

Example: {type}(sum(bytes))

+ ) : null} + + {'operationParams' in definition && definition.operationParams ? ( +

+

+ Example: {type}(sum(bytes),{' '} + {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) +

+

+ ) : null} + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts new file mode 100644 index 00000000000000..4b6acefa6b30ad --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './formula_editor'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts similarity index 96% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 9b5e77b7b90dbb..9e29160b6747b4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -6,12 +6,12 @@ */ import { monaco } from '@kbn/monaco'; -import { createMockedIndexPattern } from '../../../mocks'; -import { GenericOperationDefinition } from '../index'; -import type { IndexPatternField } from '../../../types'; -import type { OperationMetadata } from '../../../../types'; -import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; -import { tinymathFunctions } from './util'; +import { createMockedIndexPattern } from '../../../../mocks'; +import { GenericOperationDefinition } from '../../index'; +import type { IndexPatternField } from '../../../../types'; +import type { OperationMetadata } from '../../../../../types'; +import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; +import { tinymathFunctions } from '../util'; import { getSignatureHelp, getHover, suggest } from './math_completion'; const buildGenericColumn = (type: string) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts similarity index 98% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 1ae5da9d6db1d6..e8c16fe64651a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -15,10 +15,10 @@ import { TinymathNamedArgument, } from '@kbn/tinymath'; import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; -import { IndexPattern } from '../../../types'; -import { memoizedGetAvailableOperationsByMetadata } from '../../operations'; -import { tinymathFunctions, groupArgsByType } from './util'; -import type { GenericOperationDefinition } from '..'; +import { IndexPattern } from '../../../../types'; +import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; +import { tinymathFunctions, groupArgsByType } from '../util'; +import type { GenericOperationDefinition } from '../..'; export enum SUGGESTION_TYPE { FIELD = 'field', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx similarity index 100% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_tokenization.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 433e21eb133451..ce7b48aa1875ea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -7,7 +7,8 @@ import { createMockedIndexPattern } from '../../../mocks'; import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +import { FormulaIndexPatternColumn } from './formula'; +import { regenerateLayerFromAst } from './parse'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; import { tinymathFunctions } from './util'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 584ea5da38957f..6494c47548f2f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -5,61 +5,14 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; -import { isObject } from 'lodash'; import { i18n } from '@kbn/i18n'; -import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; -import { - EuiButtonEmpty, - EuiFlexGroup, - EuiFlexItem, - EuiText, - EuiSpacer, - EuiPopover, - EuiSelectable, - EuiSelectableOption, -} from '@elastic/eui'; -import { monaco } from '@kbn/monaco'; -import classNames from 'classnames'; -import { CodeEditor, Markdown } from '../../../../../../../../src/plugins/kibana_react/public'; -import type { CodeEditorProps } from '../../../../../../../../src/plugins/kibana_react/public'; -import { - OperationDefinition, - GenericOperationDefinition, - IndexPatternColumn, - ParamEditorProps, -} from '../index'; +import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import { IndexPattern, IndexPatternLayer } from '../../../types'; -import { getColumnOrder, getManagedColumnsFrom } from '../../layer_helpers'; -import { mathOperation } from './math'; -import { documentField } from '../../../document_field'; -import { ErrorWrapper, runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; -import { - extractParamsForFormula, - findVariables, - getOperationParams, - getSafeFieldName, - groupArgsByType, - hasMathNode, - tinymathFunctions, -} from './util'; -import { useDebounceWithOptions } from '../helpers'; -import { - LensMathSuggestion, - SUGGESTION_TYPE, - suggest, - getSuggestion, - getPossibleFunctions, - getSignatureHelp, - getHover, - getTokenInfo, - offsetToRowColumn, - monacoPositionToOffset, -} from './math_completion'; -import { LANGUAGE_ID } from './math_tokenization'; - -import './formula.scss'; +import { IndexPattern } from '../../../types'; +import { runASTValidation, tryToParse } from './validation'; +import { FormulaEditor } from './editor'; +import { regenerateLayerFromAst } from './parse'; +import { generateFormula } from './generate'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', @@ -153,41 +106,12 @@ export const formulaOperation: OperationDefinition< buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { let previousFormula = ''; if (previousColumn) { - if ('references' in previousColumn) { - const metric = layer.columns[previousColumn.references[0]]; - if (metric && 'sourceField' in metric && metric.dataType === 'number') { - const fieldName = getSafeFieldName(metric.sourceField); - // TODO need to check the input type from the definition - previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; - } - } else { - if ( - previousColumn && - 'sourceField' in previousColumn && - previousColumn.dataType === 'number' - ) { - previousFormula += `${previousColumn.operationType}(${getSafeFieldName( - previousColumn?.sourceField - )}`; - } - } - const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); - if (formulaNamedArgs.length) { - previousFormula += - ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); - } - if (previousColumn.filter) { - if (previousColumn.operationType !== 'count') { - previousFormula += ', '; - } - previousFormula += - (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + - `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all - } - if (previousFormula) { - // close the formula at the end - previousFormula += ')'; - } + previousFormula = generateFormula( + previousColumn, + layer, + previousFormula, + operationDefinitionMap + ); } // carry over the format settings from previous operation for seamless transfer // NOTE: this works only for non-default formatters set in Lens @@ -207,11 +131,8 @@ export const formulaOperation: OperationDefinition< references: [], }; }, - isTransferable: (column, newIndexPattern, operationDefinitionMap) => { - // Basic idea: if it has any math operation in it, probably it cannot be transferable - const { root, error } = tryToParse(column.params.formula || ''); - if (!root) return true; - return Boolean(!error && !hasMathNode(root)); + isTransferable: () => { + return true; }, createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) { const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn; @@ -235,861 +156,3 @@ export const formulaOperation: OperationDefinition< paramEditor: FormulaEditor, }; - -function FormulaEditor({ - layer, - updateLayer, - currentColumn, - columnId, - indexPattern, - operationDefinitionMap, - data, - toggleFullscreen, - isFullscreen, - setIsCloseable, -}: ParamEditorProps) { - const [text, setText] = useState(currentColumn.params.formula); - const [isHelpOpen, setIsHelpOpen] = useState(false); - const editorModel = React.useRef( - monaco.editor.createModel(text ?? '', LANGUAGE_ID) - ); - const overflowDiv1 = React.useRef(); - const disposables = React.useRef([]); - const editor1 = React.useRef(); - - // The Monaco editor needs to have the overflowDiv in the first render. Using an effect - // requires a second render to work, so we are using an if statement to guarantee it happens - // on first render - if (!overflowDiv1?.current) { - const node1 = (overflowDiv1.current = document.createElement('div')); - node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); - // Monaco CSS is targeted on the monaco-editor class - node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); - document.body.appendChild(node1); - } - - // Clean up the monaco editor and DOM on unmount - useEffect(() => { - const model = editorModel.current; - const allDisposables = disposables.current; - const editor1ref = editor1.current; - return () => { - model.dispose(); - overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - editor1ref?.dispose(); - allDisposables?.forEach((d) => d.dispose()); - }; - }, []); - - useDebounceWithOptions( - () => { - if (!editorModel.current) return; - - if (!text) { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - if (currentColumn.params.formula) { - // Only submit if valid - const { newLayer } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - } - - return; - } - - let errors: ErrorWrapper[] = []; - - const { root, error } = tryToParse(text); - if (error) { - errors = [error]; - } else if (root) { - const validationErrors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap - ); - if (validationErrors.length) { - errors = validationErrors; - } - } - - if (errors.length) { - monaco.editor.setModelMarkers( - editorModel.current, - 'LENS', - errors.flatMap((innerError) => { - if (innerError.locations.length) { - return innerError.locations.map((location) => { - const startPosition = offsetToRowColumn(text, location.min); - const endPosition = offsetToRowColumn(text, location.max); - return { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }; - }); - } else { - // Parse errors return no location info - const startPosition = offsetToRowColumn(text, 0); - const endPosition = offsetToRowColumn(text, text.length - 1); - return [ - { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }, - ]; - } - }) - ); - } else { - monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); - - // Only submit if valid - const { newLayer, locations } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ); - updateLayer(newLayer); - - const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); - const markers: monaco.editor.IMarkerData[] = managedColumns - .flatMap(([id, column]) => { - if (locations[id]) { - const def = operationDefinitionMap[column.operationType]; - if (def.getErrorMessage) { - const messages = def.getErrorMessage( - newLayer, - id, - indexPattern, - operationDefinitionMap - ); - if (messages) { - const startPosition = offsetToRowColumn(text, locations[id].min); - const endPosition = offsetToRowColumn(text, locations[id].max); - return [ - { - message: messages.join(', '), - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: monaco.MarkerSeverity.Warning, - }, - ]; - } - } - } - return []; - }) - .filter((marker) => marker); - monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); - } - }, - // Make it validate on flyout open in case of a broken formula left over - // from a previous edit - { skipFirstRender: text == null }, - 256, - [text] - ); - - /** - * The way that Monaco requests autocompletion is not intuitive, but the way we use it - * we fetch new suggestions in these scenarios: - * - * - If the user types one of the trigger characters, suggestions are always fetched - * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after - * - When the user types the first character into an empty text box, Monaco requests suggestions - * - * Monaco also triggers suggestions automatically when there are no suggestions being displayed - * and the user types a non-whitespace character. - * - * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. - */ - const provideCompletionItems = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - context: monaco.languages.CompletionContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - let wordRange: monaco.Range; - let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { - list: [], - type: SUGGESTION_TYPE.FIELD, - }; - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - - if (context.triggerCharacter === '(') { - const wordUntil = model.getWordAtPosition(position.delta(0, -3)); - if (wordUntil) { - wordRange = new monaco.Range( - position.lineNumber, - position.column, - position.lineNumber, - position.column - ); - - // Retrieve suggestions for subexpressions - // TODO: make this work for expressions nested more than one level deep - aSuggestions = await suggest({ - expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - } else { - aSuggestions = await suggest({ - expression: innerText, - position: innerText.length - lengthAfterPosition, - context, - indexPattern, - operationDefinitionMap, - data, - }); - } - - return { - suggestions: aSuggestions.list.map((s) => - getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) - ), - }; - }, - [indexPattern, operationDefinitionMap, data] - ); - - const provideSignatureHelp = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken, - context: monaco.languages.SignatureHelpContext - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getSignatureHelp( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const provideHover = useCallback( - async ( - model: monaco.editor.ITextModel, - position: monaco.Position, - token: monaco.CancellationToken - ) => { - const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); - return getHover( - model.getValue(), - innerText.length - lengthAfterPosition, - operationDefinitionMap - ); - }, - [operationDefinitionMap] - ); - - const onTypeHandler = useCallback( - (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { - if (e.isFlush || e.isRedoing || e.isUndoing) { - return; - } - if (e.changes.length === 1 && e.changes[0].text === '=') { - const currentPosition = e.changes[0].range; - if (currentPosition) { - const tokenInfo = getTokenInfo( - editor.getValue(), - monacoPositionToOffset( - editor.getValue(), - new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) - ) - ); - // Make sure that we are only adding kql='' or lucene='', and also - // check that the = sign isn't inside the KQL expression like kql='=' - if ( - !tokenInfo || - typeof tokenInfo.ast === 'number' || - tokenInfo.ast.type !== 'namedArgument' || - (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || - tokenInfo.ast.value !== 'LENS_MATH_MARKER' - ) { - return; - } - - // Timeout is required because otherwise the cursor position is not updated. - setTimeout(() => { - editor.executeEdits( - 'LENS', - [ - { - range: { - ...currentPosition, - // Insert after the current char - startColumn: currentPosition.startColumn + 1, - endColumn: currentPosition.startColumn + 1, - }, - text: `''`, - }, - ], - [ - // After inserting, move the cursor in between the single quotes - new monaco.Selection( - currentPosition.startLineNumber, - currentPosition.startColumn + 2, - currentPosition.startLineNumber, - currentPosition.startColumn + 2 - ), - ] - ); - editor.trigger('lens', 'editor.action.triggerSuggest', {}); - }, 0); - } - } - }, - [] - ); - - const codeEditorOptions: CodeEditorProps = { - languageId: LANGUAGE_ID, - value: text ?? '', - onChange: setText, - options: { - automaticLayout: false, - fontSize: 14, - folding: false, - lineNumbers: 'off', - scrollBeyondLastLine: false, - minimap: { enabled: false }, - wordWrap: 'on', - // Disable suggestions that appear when we don't provide a default suggestion - wordBasedSuggestions: false, - autoIndent: 'brackets', - wrappingIndent: 'none', - dimension: { width: 290, height: 200 }, - fixedOverflowWidgets: true, - }, - }; - - useEffect(() => { - // Because the monaco model is owned by Lens, we need to manually attach and remove handlers - const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { - triggerCharacters: ['.', '(', '=', ' ', ':', `'`], - provideCompletionItems, - }); - const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { - signatureHelpTriggerCharacters: ['(', '='], - provideSignatureHelp, - }); - const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { - provideHover, - }); - return () => { - dispose1(); - dispose2(); - dispose3(); - }; - }, [provideCompletionItems, provideSignatureHelp, provideHover]); - - // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences - // in the behavior of Monaco when it's first loaded and then reloaded. - return ( -
-
- - - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="s" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse formula', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand formula', - })} - - - -
-
- { - editor1.current = editor; - disposables.current.push( - editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); - }) - ); - disposables.current.push( - editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); - }) - ); - // If we ever introduce a second Monaco editor, we need to toggle - // the typing handler to the active editor to maintain the cursor - disposables.current.push( - editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }) - ); - }} - /> - -
-
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - size="s" - color="text" - > - {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { - defaultMessage: 'Function reference', - })} - - } - anchorPosition="leftDown" - > - - - )} - - - {/* Errors go here */} - -
-
- ); -} - -function FormulaHelp({ - indexPattern, - operationDefinitionMap, -}: { - indexPattern: IndexPattern; - operationDefinitionMap: Record; -}) { - const [selectedFunction, setSelectedFunction] = useState(); - - const helpItems: Array = []; - - helpItems.push({ label: 'Math', isGroupLabel: true }); - - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in tinymathFunctions) - .map((key) => ({ - label: `${key}`, - description: , - checked: selectedFunction === key ? ('on' as const) : undefined, - })) - ); - - helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); - - // Es aggs - helpItems.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in operationDefinitionMap) - .map((key) => ({ - label: `${key}: ${operationDefinitionMap[key].displayName}`, - description: getHelpText(key, operationDefinitionMap), - checked: - selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` - ? ('on' as const) - : undefined, - })) - ); - - return ( - - - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); - } else { - setSelectedFunction(chosenType.label); - } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - - )} - - - - ); -} - -const MemoizedFormulaHelp = React.memo(FormulaHelp); - -function parseAndExtract( - text: string, - layer: IndexPatternLayer, - columnId: string, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { root, error } = tryToParse(text); - if (error || !root) { - return { extracted: [], isValid: false }; - } - // before extracting the data run the validation task and throw if invalid - const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); - if (errors.length) { - return { extracted: [], isValid: false }; - } - /* - { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } - */ - const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); - return { extracted, isValid: true }; -} - -export function regenerateLayerFromAst( - text: string, - layer: IndexPatternLayer, - columnId: string, - currentColumn: FormulaIndexPatternColumn, - indexPattern: IndexPattern, - operationDefinitionMap: Record -) { - const { extracted, isValid } = parseAndExtract( - text, - layer, - columnId, - indexPattern, - operationDefinitionMap - ); - - const columns = { ...layer.columns }; - - const locations: Record = {}; - - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; - } - }); - - extracted.forEach(({ column, location }, index) => { - columns[`${columnId}X${index}`] = column; - if (location) locations[`${columnId}X${index}`] = location; - }); - - columns[columnId] = { - ...currentColumn, - params: { - ...currentColumn.params, - formula: text, - isFormulaBroken: !isValid, - }, - references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], - }; - - return { - newLayer: { - ...layer, - columns, - columnOrder: getColumnOrder({ - ...layer, - columns, - }), - }, - locations, - }; - - // TODO - // turn ast into referenced columns - // set state -} - -function extractColumns( - idPrefix: string, - operations: Record, - ast: TinymathAST, - layer: IndexPatternLayer, - indexPattern: IndexPattern -): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { - const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; - - function parseNode(node: TinymathAST) { - if (typeof node === 'number' || node.type !== 'function') { - // leaf node - return node; - } - - const nodeOperation = operations[node.name]; - if (!nodeOperation) { - // it's a regular math node - const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< - number | TinymathVariable - >; - return { - ...node, - args: consumedArgs, - }; - } - - // split the args into types for better TS experience - const { namedArguments, variables, functions } = groupArgsByType(node.args); - - // operation node - if (nodeOperation.input === 'field') { - const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); - // a validation task passed before executing this and checked already there's a field - const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value)! - : documentField; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'field' - >).buildColumn( - { - layer, - indexPattern, - field, - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - - if (nodeOperation.input === 'fullReference') { - const [referencedOp] = functions; - const consumedParam = parseNode(referencedOp); - - const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = subNodeVariables.map(({ value }) => value); - mathColumn.params.tinymathAst = consumedParam!; - columns.push({ column: mathColumn }); - mathColumn.customLabel = true; - mathColumn.label = `${idPrefix}X${columns.length - 1}`; - - const mappedParams = getOperationParams(nodeOperation, namedArguments || []); - const newCol = (nodeOperation as OperationDefinition< - IndexPatternColumn, - 'fullReference' - >).buildColumn( - { - layer, - indexPattern, - referenceIds: [`${idPrefix}X${columns.length - 1}`], - }, - mappedParams - ); - const newColId = `${idPrefix}X${columns.length}`; - newCol.customLabel = true; - newCol.label = newColId; - columns.push({ column: newCol, location: node.location }); - // replace by new column id - return newColId; - } - } - const root = parseNode(ast); - if (root === undefined) { - return []; - } - const variables = findVariables(root); - const mathColumn = mathOperation.buildColumn({ - layer, - indexPattern, - }); - mathColumn.references = variables.map(({ value }) => value); - mathColumn.params.tinymathAst = root!; - const newColId = `${idPrefix}X${columns.length}`; - mathColumn.customLabel = true; - mathColumn.label = newColId; - columns.push({ column: mathColumn }); - return columns; -} - -// TODO: i18n this whole thing, or move examples into the operation definitions with i18n -function getHelpText( - type: string, - operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -) { - const definition = operationDefinitionMap[type]; - - if (type === 'count') { - return ( - -

Example: count()

- - ); - } - - return ( - - {definition.input === 'field' ?

Example: {type}(bytes)

: null} - {definition.input === 'fullReference' && !('operationParams' in definition) ? ( -

Example: {type}(sum(bytes))

- ) : null} - - {'operationParams' in definition && definition.operationParams ? ( -

-

- Example: {type}(sum(bytes),{' '} - {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) -

-

- ) : null} - - ); -} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts new file mode 100644 index 00000000000000..e44cd50ae9c412 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; + +// Just handle two levels for now +type OperationParams = Record>; + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function generateFormula( + previousColumn: ReferenceBasedIndexPatternColumn | IndexPatternColumn, + layer: IndexPatternLayer, + previousFormula: string, + operationDefinitionMap: Record | undefined +) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + if (metric && 'sourceField' in metric && metric.dataType === 'number') { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; + } + } else { + if (previousColumn && 'sourceField' in previousColumn && previousColumn.dataType === 'number') { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )}`; + } + } + const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); + if (formulaNamedArgs.length) { + previousFormula += + ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); + } + if (previousColumn.filter) { + if (previousColumn.operationType !== 'count') { + previousFormula += ', '; + } + previousFormula += + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all + } + if (previousFormula) { + // close the formula at the end + previousFormula += ')'; + } + return previousFormula; +} + +function extractParamsForFormula( + column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, + operationDefinitionMap: Record | undefined +) { + if (!operationDefinitionMap) { + return []; + } + const def = operationDefinitionMap[column.operationType]; + if ('operationParams' in def && column.params) { + return (def.operationParams || []).flatMap(({ name, required }) => { + const value = (column.params as OperationParams)![name]; + if (isObject(value)) { + return Object.keys(value).map((subName) => ({ + name: `${name}-${subName}`, + value: value[subName] as string | number, + required, + })); + } + return { + name, + value, + required, + }; + }); + } + return []; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts new file mode 100644 index 00000000000000..70ed2f36dfd1c2 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { mathOperation } from './math'; +import { documentField } from '../../../document_field'; +import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; +import { findVariables, getOperationParams, groupArgsByType } from './util'; +import { FormulaIndexPatternColumn } from './formula'; +import { getColumnOrder } from '../../layer_helpers'; + +function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { root, error } = tryToParse(text); + if (error || !root) { + return { extracted: [], isValid: false }; + } + // before extracting the data run the validation task and throw if invalid + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + if (errors.length) { + return { extracted: [], isValid: false }; + } + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; +} + +function extractColumns( + idPrefix: string, + operations: Record, + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern +): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { + const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { + // leaf node + return node; + } + + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; + return { + ...node, + args: consumedArgs, + }; + } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + + // operation node + if (nodeOperation.input === 'field') { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value)! + : documentField; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn( + { + layer, + indexPattern, + field, + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const [referencedOp] = functions; + const consumedParam = parseNode(referencedOp); + + const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = subNodeVariables.map(({ value }) => value); + mathColumn.params.tinymathAst = consumedParam!; + columns.push({ column: mathColumn }); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + } + const root = parseNode(ast); + if (root === undefined) { + return []; + } + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables.map(({ value }) => value); + mathColumn.params.tinymathAst = root!; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push({ column: mathColumn }); + return columns; +} + +export function regenerateLayerFromAst( + text: string, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { extracted, isValid } = parseAndExtract( + text, + layer, + columnId, + indexPattern, + operationDefinitionMap + ); + + const columns = { ...layer.columns }; + + const locations: Record = {}; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach(({ column, location }, index) => { + columns[`${columnId}X${index}`] = column; + if (location) locations[`${columnId}X${index}`] = location; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text, + isFormulaBroken: !isValid, + }, + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + }; + + return { + newLayer: { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }, + locations, + }; + + // TODO + // turn ast into referenced columns + // set state +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 17ca19839a216b..5d9a8647eb7ab0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -10,12 +10,10 @@ import { i18n } from '@kbn/i18n'; import type { TinymathAST, TinymathFunction, - TinymathLocation, TinymathNamedArgument, TinymathVariable, } from 'packages/kbn-tinymath'; -import { ReferenceBasedIndexPatternColumn } from '../column_types'; -import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; import type { GroupedNodes } from './types'; export function groupArgsByType(args: TinymathAST[]) { @@ -43,45 +41,6 @@ export function getValueOrName(node: TinymathAST) { return node.name; } -export function getSafeFieldName(fieldName: string | undefined) { - // clean up the "Records" field for now - if (!fieldName || fieldName === 'Records') { - return ''; - } - return fieldName; -} - -// Just handle two levels for now -type OeprationParams = Record>; - -export function extractParamsForFormula( - column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, - operationDefinitionMap: Record | undefined -) { - if (!operationDefinitionMap) { - return []; - } - const def = operationDefinitionMap[column.operationType]; - if ('operationParams' in def && column.params) { - return (def.operationParams || []).flatMap(({ name, required }) => { - const value = (column.params as OeprationParams)![name]; - if (isObject(value)) { - return Object.keys(value).map((subName) => ({ - name: `${name}-${subName}`, - value: value[subName] as string | number, - required, - })); - } - return { - name, - value, - required, - }; - }); - } - return []; -} - export function getOperationParams( operation: | OperationDefinition @@ -332,32 +291,6 @@ export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { return flattenMathNodes(root); } -export function hasMathNode(root: TinymathAST): boolean { - return Boolean(findMathNodes(root).length); -} - -function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { - function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { - if (!isObject(node) || node.type !== 'function') { - return []; - } - return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); - } - return flattenFunctionNodes(root); -} - -export function hasInvalidOperations( - node: TinymathAST | string, - operations: Record -): { names: string[]; locations: TinymathLocation[] } { - const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); - return { - // avoid duplicates - names: Array.from(new Set(nodes.map(({ name }) => name))), - locations: nodes.map(({ location }) => location), - }; -} - // traverse a tree and find all string leaves export function findVariables(node: TinymathAST | string): TinymathVariable[] { if (typeof node === 'string') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index cb52e22302cbe2..4e5ae21e576e45 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -16,7 +16,6 @@ import { getOperationParams, getValueOrName, groupArgsByType, - hasInvalidOperations, isMathNode, tinymathFunctions, } from './util'; @@ -74,6 +73,28 @@ export function isParsingError(message: string) { return message.includes(validationErrors.failedParsing.message); } +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( + node: TinymathAST | string, + operations: Record +): { names: string[]; locations: TinymathLocation[] } { + const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); + return { + // avoid duplicates + names: Array.from(new Set(nodes.map(({ name }) => name))), + locations: nodes.map(({ location }) => location), + }; +} + export const getQueryValidationError = ( query: string, language: 'kql' | 'lucene', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 49366f2421b7b6..4fd429820379f9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1079,11 +1079,21 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st /** * Returns true if the given column can be applied to the given index pattern */ -export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) { - return operationDefinitionMap[column.operationType].isTransferable( - column, - newIndexPattern, - operationDefinitionMap +export function isColumnTransferable( + column: IndexPatternColumn, + newIndexPattern: IndexPattern, + layer: IndexPatternLayer +): boolean { + return ( + operationDefinitionMap[column.operationType].isTransferable( + column, + newIndexPattern, + operationDefinitionMap + ) && + (!('references' in column) || + column.references.every((columnId) => + isColumnTransferable(layer.columns[columnId], newIndexPattern, layer) + )) ); } @@ -1092,15 +1102,7 @@ export function updateLayerIndexPattern( newIndexPattern: IndexPattern ): IndexPatternLayer { const keptColumns: IndexPatternLayer['columns'] = _.pickBy(layer.columns, (column) => { - if ('references' in column) { - return ( - isColumnTransferable(column, newIndexPattern) && - column.references.every((columnId) => - isColumnTransferable(layer.columns[columnId], newIndexPattern) - ) - ); - } - return isColumnTransferable(column, newIndexPattern); + return isColumnTransferable(column, newIndexPattern, layer); }); const newColumns: IndexPatternLayer['columns'] = _.mapValues(keptColumns, (column) => { const operationDefinition = operationDefinitionMap[column.operationType]; From 9655e7d9fa0a6e343651f94aa4af31bc94b9860c Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Mon, 10 May 2021 15:38:36 -0400 Subject: [PATCH 092/185] more style and markup tweak for custom formula --- .../config_panel/dimension_container.scss | 6 + .../editor_frame/frame_layout.scss | 11 +- .../editor_frame/frame_layout.tsx | 4 +- .../dimension_panel/dimension_editor.scss | 10 +- .../definitions/formula/formula.scss | 34 ++- .../definitions/formula/formula.tsx | 205 +++++++++--------- 6 files changed, 162 insertions(+), 108 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index 91cd706ea77d11..f719cb96aa97c4 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -8,13 +8,19 @@ position: absolute; left: 0; animation: euiFlyout $euiAnimSpeedNormal $euiAnimSlightResistance; + @include euiBreakpoint('l', 'xl') { top: 0 !important; height: 100% !important; } + @include euiBreakpoint('xs', 's', 'm') { @include euiFlyout; } + + .lnsFrameLayout__sidebar-isFullscreen & { + box-shadow: none; + } } .lnsDimensionContainer__footer { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss index 56642cff1fbff1..7bc86b496f6267 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss @@ -67,11 +67,14 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ padding: $euiSize $euiSize 0; position: relative; z-index: $lnsZLevel1; + &:first-child { padding-left: $euiSize; } - &.lnsFrameLayout__pageBody--fullscreen { + &.lnsFrameLayout__pageBody-isFullscreen { + background: $euiColorEmptyShade; + flex: 1; padding: 0; } } @@ -111,7 +114,7 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ } } -.lnsFrameLayout__sidebar--fullscreen { - flex-basis: 50%; - max-width: calc(50%); +.lnsFrameLayout__sidebar-isFullscreen { + flex: 1; + max-width: none; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index f09915a981af26..a2aaf977cf6e6e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -43,7 +43,7 @@ export function FrameLayout(props: FrameLayoutProps) {
@@ -60,7 +60,7 @@ export function FrameLayout(props: FrameLayoutProps) {
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index b6b3f55e75f05e..814b93bcc921b8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -4,10 +4,16 @@ .lnsIndexPatternDimensionEditor-isFullscreen { position: absolute; + left: 0; + right: 0; top: 0; bottom: 0; - display: flex; - flex-direction: column; + // display: flex; + // flex-direction: column; + + .lnsIndexPatternDimensionEditor__section { + height: 100%; + } } .lnsIndexPatternDimensionEditor__section--padded { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss index 6f41fa74c6aa89..a4000718b3b2b2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.scss @@ -1,16 +1,46 @@ .lnsFormula { + display: flex; + flex-direction: column; + + .lnsIndexPatternDimensionEditor-isFullscreen & { + height: 100%; + } +} + +.lnsFormula__editor, +.lnsFormula__docs { + flex: 1; + min-height: 0; +} + +.lnsFormula__editor { border-bottom: $euiBorderThin; + display: flex; + flex-direction: column; & > * + * { border-top: $euiBorderThin; } } -.lnsFormula__header, -.lnsFormula__footer { +.lnsFormula__editorHeader, +.lnsFormula__editorFooter { padding: $euiSizeS; } +.lnsFormula__editorContent { + flex: 1; + max-height: 200px; + + .lnsIndexPatternDimensionEditor-isFullscreen & { + max-height: none; + } +} + +.lnsFormula__docs { + background: $euiColorEmptyShade; +} + .lnsFormulaOverflow { // Needs to be higher than the modal and all flyouts z-index: $euiZLevel9 + 1; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 50150048c8882e..457731ed016fa2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -652,104 +652,112 @@ function FormulaEditor({ >
-
- - {/* TODO: Word wrap button */} - - - { - toggleFullscreen(); - }} - iconType="fullScreen" - size="xs" - color="text" - flush="right" - > - {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { - defaultMessage: 'Collapse', - }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { - defaultMessage: 'Expand', - })} - - - -
-
- { - editor1.current = editor; - disposables.current.push( - editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); - }) - ); - disposables.current.push( - editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); - }) - ); - // If we ever introduce a second Monaco editor, we need to toggle - // the typing handler to the active editor to maintain the cursor - disposables.current.push( - editor.onDidChangeModelContent((e) => { - onTypeHandler(e, editor); - }) - ); - }} - /> -
- -
- - - {isFullscreen ? ( - - ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - color="text" - aria-label={i18n.translate( - 'xpack.lens.formula.functionReferenceEditorLabel', - { - defaultMessage: 'Function reference', - } - )} - /> - } - anchorPosition="leftDown" +
+ + {/* TODO: Word wrap button */} + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="xs" + color="text" + flush="right" > - - - )} - - - {/* TODO: Errors go here */} - + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand', + })} + + + +
+ +
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> +
+ +
+ + + {isFullscreen ? ( +

Accordion button here

+ ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + color="text" + aria-label={i18n.translate( + 'xpack.lens.formula.functionReferenceEditorLabel', + { + defaultMessage: 'Function reference', + } + )} + /> + } + anchorPosition="leftDown" + > + + + )} + + + {/* TODO: Errors go here */} + +
+ + {isFullscreen ? ( +
+ +
+ ) : null}
@@ -796,8 +804,8 @@ function FormulaHelp({ ); return ( - - + + - + + {selectedFunction ? ( helpItems.find(({ label }) => label === selectedFunction)?.description From 09659a2b3ac69cc4117a89a5ec03dd2097b2520e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 10 May 2021 17:19:52 -0400 Subject: [PATCH 093/185] Fix tests --- .../expression_functions/specs/map_column.ts | 2 +- .../specs/tests/map_column.test.ts | 27 ++++++++++--------- .../operations/definitions/formula/index.ts | 3 ++- .../operations/layer_helpers.ts | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index 7293510baa6b5d..dc19c81a99c1f6 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -103,7 +103,7 @@ export const mapColumn: ExpressionFunctionDefinition< return Promise.all(rowPromises).then((rows) => { const existingColumnIndex = columns.findIndex(({ id, name }) => { // Columns that have IDs are allowed to have duplicate names, for example esaggs - if (id) { + if (args.id && id) { return id === args.id && name === args.name; } // If no ID, name is the unique key. For example, SQL output does not have IDs diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index b2966b010b4790..ec8fc87a76c116 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -35,7 +35,7 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); - it('overwrites existing column with the new column if an existing column name is provided', async () => { + it('overwrites existing column with the new column if an existing column name is provided without an id', async () => { const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); const arbitraryRowIndex = 4; @@ -47,6 +47,19 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); }); + it('inserts a new column with a duplicate name if an id and name are provided', async () => { + const result = await runFn(testTable, { id: 'new', name: 'name', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length + 1); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); + }); + it('adds a column to empty tables', async () => { const result = await runFn(emptyTable, { name: 'name', expression: pricePlusTwo }); @@ -66,18 +79,6 @@ describe('mapColumn', () => { expect(result.columns[0].meta).toHaveProperty('type', 'null'); }); - it('should assign specific id, different from name, when id arg is passed for copied column', async () => { - const result = await runFn(testTable, { name: 'name', id: 'myid', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - - expect(result.type).toBe('datatable'); - expect(result.columns[nameColumnIndex]).toEqual({ - id: 'myid', - name: 'name', - meta: { type: 'number' }, - }); - }); - it('should copy over the meta information from the specified column', async () => { const result = await runFn( { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts index b5605369d4f9bc..bafde0d37b3e9f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts @@ -5,5 +5,6 @@ * 2.0. */ -export { formulaOperation, FormulaIndexPatternColumn, regenerateLayerFromAst } from './formula'; +export { formulaOperation, FormulaIndexPatternColumn } from './formula'; +export { regenerateLayerFromAst } from './parse'; export { mathOperation, MathIndexPatternColumn } from './math'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 4fd429820379f9..45345572a8f7a1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -18,7 +18,7 @@ import { import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula/formula'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula'; interface ColumnChange { op: OperationType; From 03563b49b5ecb26e03e42107419ba728c2ae2e99 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 10 May 2021 17:45:59 -0400 Subject: [PATCH 094/185] [Expressions] Use table column ID instead of name when set --- .../expression_functions/specs/map_column.ts | 9 +++- .../common/expression_functions/specs/math.ts | 3 +- .../specs/tests/map_column.test.ts | 37 +++++++++------- .../specs/tests/math.test.ts | 13 +++++- .../expression_functions/specs/tests/utils.ts | 42 +++++++++++++++---- 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index c570206670dde5..7293510baa6b5d 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -101,7 +101,14 @@ export const mapColumn: ExpressionFunctionDefinition< }); return Promise.all(rowPromises).then((rows) => { - const existingColumnIndex = columns.findIndex(({ name }) => name === args.name); + const existingColumnIndex = columns.findIndex(({ id, name }) => { + // Columns that have IDs are allowed to have duplicate names, for example esaggs + if (id) { + return id === args.id && name === args.name; + } + // If no ID, name is the unique key. For example, SQL output does not have IDs + return name === args.name; + }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; const newColumn = { id: columnId, diff --git a/src/plugins/expressions/common/expression_functions/specs/math.ts b/src/plugins/expressions/common/expression_functions/specs/math.ts index a70c032769b570..b91600fea8b56e 100644 --- a/src/plugins/expressions/common/expression_functions/specs/math.ts +++ b/src/plugins/expressions/common/expression_functions/specs/math.ts @@ -130,10 +130,11 @@ export const math: ExpressionFunctionDefinition< throw errors.emptyExpression(); } + // Use unique ID if available, otherwise fall back to names const mathContext = isDatatable(input) ? pivotObjectArray( input.rows, - input.columns.map((col) => col.name) + input.columns.map((col) => col.id ?? col.name) ) : { value: input }; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index b2966b010b4790..19dd60ab9469ea 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -9,7 +9,7 @@ import { of } from 'rxjs'; import { Datatable } from '../../../expression_types'; import { mapColumn, MapColumnArguments } from '../map_column'; -import { emptyTable, functionWrapper, testTable } from './utils'; +import { emptyTable, functionWrapper, testTable, sqlTable } from './utils'; const pricePlusTwo = (datatable: Datatable) => of(datatable.rows[0].price + 2); @@ -35,18 +35,35 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); - it('overwrites existing column with the new column if an existing column name is provided', async () => { - const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); + it('overwrites existing column with the new column if an existing column name is missing an id', async () => { + const result = await runFn(sqlTable, { name: 'name', expression: pricePlusTwo }); const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); const arbitraryRowIndex = 4; expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(testTable.columns.length); + expect(result.columns).toHaveLength(sqlTable.columns.length); expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); }); + it('inserts a new column with a duplicate name if an id and name are provided', async () => { + const result = await runFn(testTable, { + id: 'new', + name: 'name label', + expression: pricePlusTwo, + }); + const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length + 1); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); + }); + it('adds a column to empty tables', async () => { const result = await runFn(emptyTable, { name: 'name', expression: pricePlusTwo }); @@ -66,18 +83,6 @@ describe('mapColumn', () => { expect(result.columns[0].meta).toHaveProperty('type', 'null'); }); - it('should assign specific id, different from name, when id arg is passed for copied column', async () => { - const result = await runFn(testTable, { name: 'name', id: 'myid', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - - expect(result.type).toBe('datatable'); - expect(result.columns[nameColumnIndex]).toEqual({ - id: 'myid', - name: 'name', - meta: { type: 'number' }, - }); - }); - it('should copy over the meta information from the specified column', async () => { const result = await runFn( { diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts index 7541852cdbdaff..5ad23fc24c91ba 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts @@ -7,7 +7,7 @@ */ import { errors, math } from '../math'; -import { emptyTable, functionWrapper, testTable } from './utils'; +import { emptyTable, functionWrapper, testTable, sqlTable } from './utils'; describe('math', () => { const fn = functionWrapper(math); @@ -27,7 +27,7 @@ describe('math', () => { expect(fn(-103, { expression: 'abs(value)' })).toBe(103); }); - it('evaluates math expressions with references to columns in a datatable', () => { + it('evaluates math expressions with references to columns by id in a datatable', () => { expect(fn(testTable, { expression: 'unique(in_stock)' })).toBe(2); expect(fn(testTable, { expression: 'sum(quantity)' })).toBe(2508); expect(fn(testTable, { expression: 'mean(price)' })).toBe(320); @@ -36,6 +36,15 @@ describe('math', () => { expect(fn(testTable, { expression: 'max(price)' })).toBe(605); }); + it('evaluates math expressions with references to columns in a datatable', () => { + expect(fn(sqlTable, { expression: 'unique(in_stock)' })).toBe(2); + expect(fn(sqlTable, { expression: 'sum(quantity)' })).toBe(2508); + expect(fn(sqlTable, { expression: 'mean(price)' })).toBe(320); + expect(fn(sqlTable, { expression: 'min(price)' })).toBe(67); + expect(fn(sqlTable, { expression: 'median(quantity)' })).toBe(256); + expect(fn(sqlTable, { expression: 'max(price)' })).toBe(605); + }); + describe('args', () => { describe('expression', () => { it('sets the math expression to be evaluted', () => { diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts b/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts index 7369570cf2c4bc..a6501cb37b5a67 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts @@ -9,7 +9,7 @@ import { mapValues } from 'lodash'; import { AnyExpressionFunctionDefinition } from '../../types'; import { ExecutionContext } from '../../../execution/types'; -import { Datatable } from '../../../expression_types'; +import { Datatable, DatatableColumn } from '../../../expression_types'; /** * Takes a function spec and passes in default args, @@ -37,27 +37,27 @@ const testTable: Datatable = { columns: [ { id: 'name', - name: 'name', + name: 'name label', meta: { type: 'string' }, }, { id: 'time', - name: 'time', + name: 'time label', meta: { type: 'date' }, }, { id: 'price', - name: 'price', + name: 'price label', meta: { type: 'number' }, }, { id: 'quantity', - name: 'quantity', + name: 'quantity label', meta: { type: 'number' }, }, { id: 'in_stock', - name: 'in_stock', + name: 'in_stock label', meta: { type: 'boolean' }, }, ], @@ -224,4 +224,32 @@ const stringTable: Datatable = { ], }; -export { emptyTable, testTable, stringTable }; +// Emulates a SQL table that doesn't have any IDs +const sqlTable: Datatable = { + type: 'datatable', + columns: [ + ({ + name: 'name', + meta: { type: 'string' }, + } as unknown) as DatatableColumn, + ({ + name: 'time', + meta: { type: 'date' }, + } as unknown) as DatatableColumn, + ({ + name: 'price', + meta: { type: 'number' }, + } as unknown) as DatatableColumn, + ({ + name: 'quantity', + meta: { type: 'number' }, + } as unknown) as DatatableColumn, + ({ + name: 'in_stock', + meta: { type: 'boolean' }, + } as unknown) as DatatableColumn, + ], + rows: [...testTable.rows], +}; + +export { emptyTable, testTable, stringTable, sqlTable }; From f3ba457b313503090f274d8a50c18b25c76a720d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 21 Jan 2021 18:09:50 +0100 Subject: [PATCH 095/185] [Lens] Create managedReference type for formulas --- .../workspace_panel/workspace_panel.test.tsx | 198 ++--- .../lens/public/id_generator/id_generator.ts | 2 +- .../dimension_panel/dimension_editor.tsx | 8 +- .../dimension_panel/dimension_panel.test.tsx | 88 +-- .../droppable/droppable.test.ts | 6 +- .../droppable/on_drop_handler.ts | 2 +- .../dimension_panel/reference_editor.tsx | 7 +- .../indexpattern.test.ts | 113 ++- .../operations/__mocks__/index.ts | 4 +- .../definitions/calculations/counter_rate.tsx | 14 +- .../calculations/cumulative_sum.tsx | 14 +- .../definitions/calculations/differences.tsx | 14 +- .../calculations/moving_average.tsx | 23 +- .../definitions/calculations/utils.test.ts | 4 +- .../operations/definitions/cardinality.tsx | 17 +- .../operations/definitions/count.tsx | 12 +- .../definitions/date_histogram.test.tsx | 1 + .../operations/definitions/date_histogram.tsx | 5 +- .../definitions/filters/filters.test.tsx | 1 + .../definitions/formula/formula.tsx | 155 ++++ .../definitions/formula/generate.ts | 90 +++ .../operations/definitions/formula/index.ts | 10 + .../operations/definitions/formula/math.tsx | 111 +++ .../operations/definitions/formula/parse.ts | 210 ++++++ .../operations/definitions/formula/util.ts | 317 ++++++++ .../definitions/formula/validation.ts | 686 ++++++++++++++++++ .../operations/definitions/helpers.test.ts | 2 +- .../operations/definitions/helpers.tsx | 37 +- .../operations/definitions/index.ts | 115 ++- .../definitions/last_value.test.tsx | 1 + .../operations/definitions/last_value.tsx | 12 +- .../operations/definitions/metrics.tsx | 17 +- .../definitions/percentile.test.tsx | 21 +- .../operations/definitions/percentile.tsx | 6 +- .../definitions/ranges/ranges.test.tsx | 1 + .../definitions/terms/terms.test.tsx | 1 + .../operations/layer_helpers.test.ts | 220 ++++-- .../operations/layer_helpers.ts | 195 +++-- .../operations/mocks.ts | 27 +- .../operations/operations.test.ts | 8 + .../operations/operations.ts | 71 +- .../indexpattern_datasource/to_expression.ts | 47 +- .../public/indexpattern_datasource/utils.ts | 14 +- x-pack/test/functional/apps/lens/formula.ts | 86 +++ .../test/functional/page_objects/lens_page.ts | 31 +- 45 files changed, 2647 insertions(+), 377 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts create mode 100644 x-pack/test/functional/apps/lens/formula.ts diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx index e741b9ee243db3..baa9d45a431eaf 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx @@ -29,12 +29,7 @@ import { ReactWrapper } from 'enzyme'; import { DragDrop, ChildDragDropProvider } from '../../../drag_drop'; import { fromExpression } from '@kbn/interpreter/common'; import { coreMock } from 'src/core/public/mocks'; -import { - DataPublicPluginStart, - esFilters, - IFieldType, - IIndexPattern, -} from '../../../../../../../src/plugins/data/public'; +import { esFilters, IFieldType, IIndexPattern } from '../../../../../../../src/plugins/data/public'; import { UiActionsStart } from '../../../../../../../src/plugins/ui_actions/public'; import { uiActionsPluginMock } from '../../../../../../../src/plugins/ui_actions/public/mocks'; import { TriggerContract } from '../../../../../../../src/plugins/ui_actions/public/triggers'; @@ -55,6 +50,25 @@ function createCoreStartWithPermissions(newCapabilities = defaultPermissions) { return core; } +function getDefaultProps() { + return { + activeDatasourceId: 'mock', + datasourceStates: {}, + datasourceMap: {}, + framePublicAPI: createMockFramePublicAPI(), + activeVisualizationId: 'vis', + visualizationState: {}, + dispatch: () => {}, + ExpressionRenderer: createExpressionRendererMock(), + core: createCoreStartWithPermissions(), + plugins: { + uiActions: uiActionsPluginMock.createStartContract(), + data: dataPluginMock.createStartContract(), + }, + getSuggestionForField: () => undefined, + }; +} + describe('workspace_panel', () => { let mockVisualization: jest.Mocked; let mockVisualization2: jest.Mocked; @@ -62,21 +76,18 @@ describe('workspace_panel', () => { let expressionRendererMock: jest.Mock; let uiActionsMock: jest.Mocked; - let dataMock: jest.Mocked; let trigger: jest.Mocked; let instance: ReactWrapper; beforeEach(() => { + // These are used in specific tests to assert function calls trigger = ({ exec: jest.fn() } as unknown) as jest.Mocked; uiActionsMock = uiActionsPluginMock.createStartContract(); - dataMock = dataPluginMock.createStartContract(); uiActionsMock.getTrigger.mockReturnValue(trigger); mockVisualization = createMockVisualization(); mockVisualization2 = createMockVisualization(); - mockDatasource = createMockDatasource('a'); - expressionRendererMock = createExpressionRendererMock(); }); @@ -87,23 +98,14 @@ describe('workspace_panel', () => { it('should render an explanatory text if no visualization is active', () => { instance = mount( {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); - expect(instance.find('[data-test-subj="empty-workspace"]')).toHaveLength(2); expect(instance.find(expressionRendererMock)).toHaveLength(0); }); @@ -111,20 +113,10 @@ describe('workspace_panel', () => { it('should render an explanatory text if the visualization does not produce an expression', () => { instance = mount( null }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -135,20 +127,10 @@ describe('workspace_panel', () => { it('should render an explanatory text if the datasource does not produce an expression', () => { instance = mount( 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -166,7 +148,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -204,10 +180,11 @@ describe('workspace_panel', () => { }; mockDatasource.toExpression.mockReturnValue('datasource'); mockDatasource.getLayers.mockReturnValue(['first']); + const props = getDefaultProps(); instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} + plugins={{ ...props.plugins, uiActions: uiActionsMock }} /> ); @@ -251,7 +223,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} dispatch={dispatch} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -298,7 +265,7 @@ describe('workspace_panel', () => { instance = mount( { mock2: mockDatasource2, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -382,7 +343,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -439,7 +394,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -494,7 +443,7 @@ describe('workspace_panel', () => { }; instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -532,7 +474,7 @@ describe('workspace_panel', () => { instance = mount( { visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} // Use cannot navigate to the management page core={createCoreStartWithPermissions({ navLinks: { management: false }, management: { kibana: { indexPatterns: true } }, })} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -575,7 +512,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} // user can go to management, but indexPatterns management is not accessible core={createCoreStartWithPermissions({ navLinks: { management: true }, management: { kibana: { indexPatterns: false } }, })} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -621,7 +552,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -663,7 +587,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: mockVisualization, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -707,7 +624,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: mockVisualization, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -748,7 +658,7 @@ describe('workspace_panel', () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); @@ -787,7 +690,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -832,7 +729,7 @@ describe('workspace_panel', () => { await act(async () => { instance = mount( { mock: mockDatasource, }} framePublicAPI={framePublicAPI} - activeVisualizationId="vis" visualizationMap={{ vis: { ...mockVisualization, toExpression: () => 'vis' }, }} - visualizationState={{}} - dispatch={() => {}} ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} - getSuggestionForField={() => undefined} /> ); }); @@ -900,7 +791,7 @@ describe('workspace_panel', () => { dropTargetsByOrder={undefined} > { mock: mockDatasource, }} framePublicAPI={frame} - activeVisualizationId={'vis'} visualizationMap={{ vis: mockVisualization, vis2: mockVisualization2, }} - visualizationState={{}} dispatch={mockDispatch} - ExpressionRenderer={expressionRendererMock} - core={createCoreStartWithPermissions()} - plugins={{ uiActions: uiActionsMock, data: dataMock }} getSuggestionForField={mockGetSuggestionForField} /> diff --git a/x-pack/plugins/lens/public/id_generator/id_generator.ts b/x-pack/plugins/lens/public/id_generator/id_generator.ts index 363b8035a23f74..988f2c880222a9 100644 --- a/x-pack/plugins/lens/public/id_generator/id_generator.ts +++ b/x-pack/plugins/lens/public/id_generator/id_generator.ts @@ -8,5 +8,5 @@ import uuid from 'uuid/v4'; export function generateId() { - return uuid(); + return 'c' + uuid().replaceAll(/-/g, ''); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index b74e97df4a895c..d84d418ff231c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -151,6 +151,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) + .filter(({ hidden }) => !hidden) .sort((op1, op2) => { return op1.displayName.localeCompare(op2.displayName); }) @@ -242,6 +243,7 @@ export function DimensionEditor(props: DimensionEditorProps) { onClick() { if ( operationDefinitionMap[operationType].input === 'none' || + operationDefinitionMap[operationType].input === 'managedReference' || operationDefinitionMap[operationType].input === 'fullReference' ) { // Clear invalid state because we are reseting to a valid column @@ -319,7 +321,8 @@ export function DimensionEditor(props: DimensionEditorProps) { // Need to workout early on the error to decide whether to show this or an help text const fieldErrorMessage = - (selectedOperationDefinition?.input !== 'fullReference' || + ((selectedOperationDefinition?.input !== 'fullReference' && + selectedOperationDefinition?.input !== 'managedReference') || (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field')) && getErrorMessage( selectedColumn, @@ -447,6 +450,7 @@ export function DimensionEditor(props: DimensionEditorProps) { currentColumn={state.layers[layerId].columns[columnId]} dateRange={dateRange} indexPattern={currentIndexPattern} + operationDefinitionMap={operationDefinitionMap} {...services} /> @@ -586,7 +590,7 @@ export function DimensionEditor(props: DimensionEditorProps) { function getErrorMessage( selectedColumn: IndexPatternColumn | undefined, incompleteOperation: boolean, - input: 'none' | 'field' | 'fullReference' | undefined, + input: 'none' | 'field' | 'fullReference' | 'managedReference' | undefined, fieldInvalid: boolean ) { if (selectedColumn && incompleteOperation) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index f80b12aecabdef..5273162034f160 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -25,14 +25,13 @@ import { import { mountWithIntl as mount, shallowWithIntl as shallow } from '@kbn/test/jest'; import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreSetup } from 'kibana/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; +import { generateId } from '../../id_generator'; import { IndexPatternPrivateState } from '../types'; import { IndexPatternColumn, replaceColumn } from '../operations'; import { documentField } from '../document_field'; import { OperationMetadata } from '../../types'; import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_histogram'; import { getFieldByNameFactory } from '../pure_helpers'; -import { DimensionEditor } from './dimension_editor'; -import { AdvancedOptions } from './advanced_options'; import { Filtering } from './filtering'; jest.mock('../loader'); @@ -48,6 +47,7 @@ jest.mock('lodash', () => { debounce: (fn: unknown) => fn, }; }); +jest.mock('../../id_generator'); const fields = [ { @@ -1072,7 +1072,7 @@ describe('IndexPatternDimensionEditorPanel', () => { } it('should not show custom options if time scaling is not available', () => { - wrapper = shallow( + wrapper = mount( { })} /> ); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() - .find('[data-test-subj="indexPattern-time-scaling-enable"]') + wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes() ).toHaveLength(0); }); it('should show custom options if time scaling is available', () => { - wrapper = shallow(); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() - .find('[data-test-subj="indexPattern-time-scaling-enable"]') + wrapper.find('[data-test-subj="indexPattern-time-scaling-enable"]').hostNodes() ).toHaveLength(1); }); @@ -1114,14 +1112,15 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to set time scaling initially', () => { const props = getProps({}); - wrapper = shallow(); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-time-scaling-enable"]') - .prop('onClick')!({} as MouseEvent); + .hostNodes() + .simulate('click'); expect(props.setState).toHaveBeenCalledWith( { ...props.state, @@ -1205,6 +1204,10 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to change time scaling', () => { const props = getProps({ timeScale: 's', label: 'Count of records per second' }); wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); wrapper .find('[data-test-subj="indexPattern-time-scaling-unit"]') .find(EuiSelect) @@ -1321,33 +1324,32 @@ describe('IndexPatternDimensionEditorPanel', () => { } it('should not show custom options if time scaling is not available', () => { - wrapper = shallow( + wrapper = mount( ); expect( - wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() - .find('[data-test-subj="indexPattern-filter-by-enable"]') + wrapper.find('[data-test-subj="indexPattern-advanced-popover"]').hostNodes() ).toHaveLength(0); }); it('should show custom options if filtering is available', () => { - wrapper = shallow(); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); expect( - wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() - .find('[data-test-subj="indexPattern-filter-by-enable"]') + wrapper.find('[data-test-subj="indexPattern-filter-by-enable"]').hostNodes() ).toHaveLength(1); }); @@ -1364,14 +1366,15 @@ describe('IndexPatternDimensionEditorPanel', () => { it('should allow to set filter initially', () => { const props = getProps({}); - wrapper = shallow(); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-advanced-popover"]') + .hostNodes() + .simulate('click'); wrapper - .find(DimensionEditor) - .dive() - .find(AdvancedOptions) - .dive() .find('[data-test-subj="indexPattern-filter-by-enable"]') - .prop('onClick')!({} as MouseEvent); + .hostNodes() + .simulate('click'); expect(props.setState).toHaveBeenCalledWith( { ...props.state, @@ -1934,6 +1937,7 @@ describe('IndexPatternDimensionEditorPanel', () => { }); it('should hide the top level field selector when switching from non-reference to reference', () => { + (generateId as jest.Mock).mockReturnValue(`second`); wrapper = mount(); expect(wrapper.find('ReferenceEditor')).toHaveLength(0); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts index 9410843c0811ae..a77a980257c88c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts @@ -904,7 +904,7 @@ describe('IndexPatternDimensionEditorPanel', () => { layers: { first: { ...testState.layers.first, - columnOrder: ['ref1', 'col1', 'ref1Copy', 'col1Copy'], + columnOrder: ['col1', 'ref1', 'ref1Copy', 'col1Copy'], columns: { ref1: testState.layers.first.columns.ref1, col1: testState.layers.first.columns.col1, @@ -974,7 +974,7 @@ describe('IndexPatternDimensionEditorPanel', () => { layers: { first: { ...testState.layers.first, - columnOrder: ['ref1', 'ref2', 'col1', 'ref1Copy', 'ref2Copy', 'col1Copy'], + columnOrder: ['col1', 'ref1', 'ref2', 'ref1Copy', 'col1Copy', 'ref2Copy'], columns: { ref1: testState.layers.first.columns.ref1, ref2: testState.layers.first.columns.ref2, @@ -1061,8 +1061,8 @@ describe('IndexPatternDimensionEditorPanel', () => { 'col1', 'innerRef1Copy', 'ref1Copy', - 'ref2Copy', 'col1Copy', + 'ref2Copy', ], columns: { innerRef1: testState.layers.first.columns.innerRef1, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts index f65557d4ed6a95..e09c3e904f5358 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/on_drop_handler.ts @@ -114,7 +114,7 @@ function onMoveCompatible( const modifiedLayer = copyColumn({ layer, - columnId, + targetId: columnId, sourceColumnId: droppedItem.columnId, sourceColumn, shouldDeleteSource, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index 353bba9652effe..b8a065b088467d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -41,7 +41,7 @@ const operationPanels = getOperationDisplay(); export interface ReferenceEditorProps { layer: IndexPatternLayer; - selectionStyle: 'full' | 'field'; + selectionStyle: 'full' | 'field' | 'hidden'; validation: RequiredReference; columnId: string; updateLayer: (newLayer: IndexPatternLayer) => void; @@ -197,6 +197,10 @@ export function ReferenceEditor(props: ReferenceEditorProps) { return; } + if (selectionStyle === 'hidden') { + return null; + } + const selectedOption = incompleteOperation ? [functionOptions.find(({ value }) => value === incompleteOperation)!] : column @@ -340,6 +344,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { columnId={columnId} indexPattern={currentIndexPattern} dateRange={dateRange} + operationDefinitionMap={operationDefinitionMap} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index c291c7ab3eac08..0a2bc7fc1beadf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -15,7 +15,7 @@ import { Ast } from '@kbn/interpreter/common'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { getFieldByNameFactory } from './pure_helpers'; import { operationDefinitionMap, getErrorMessages } from './operations'; -import { createMockedReferenceOperation } from './operations/mocks'; +import { createMockedFullReference } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks'; jest.mock('./loader'); @@ -287,6 +287,30 @@ describe('IndexPattern Data Source', () => { expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null); }); + it('should generate an empty expression when there is a formula without aggs', async () => { + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Formula', + dataType: 'number', + isBucketed: false, + operationType: 'formula', + references: [], + params: {}, + }, + }, + }, + }, + }; + const state = enrichBaseState(queryBaseState); + expect(indexPatternDatasource.toExpression(state, 'first')).toEqual(null); + }); + it('should generate an expression for an aggregated query', async () => { const queryBaseState: IndexPatternBaseState = { currentIndexPatternId: '1', @@ -815,7 +839,7 @@ describe('IndexPattern Data Source', () => { describe('references', () => { beforeEach(() => { // @ts-expect-error we are inserting an invalid type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); // @ts-expect-error we are inserting an invalid type operationDefinitionMap.testReference.toExpression.mockReturnValue(['mock']); @@ -898,6 +922,91 @@ describe('IndexPattern Data Source', () => { }), }); }); + + it('should topologically sort references', () => { + // This is a real example of count() + count() + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['date', 'count', 'formula', 'countX0', 'math'], + columns: { + count: { + label: 'count', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + date: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', + isBucketed: true, + scale: 'interval', + params: { + interval: 'auto', + }, + }, + formula: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + formula: 'count() + count()', + isFormulaBroken: false, + }, + references: ['math'], + }, + countX0: { + label: 'countX0', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + math: { + label: 'math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + // @ts-expect-error String args are not valid tinymath, but signals something unique to Lens + args: ['countX0', 'count'], + location: { + min: 0, + max: 17, + }, + text: 'count() + count()', + }, + }, + references: ['countX0', 'count'], + customLabel: true, + }, + }, + }, + }, + }; + + const state = enrichBaseState(queryBaseState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + const chainLength = ast.chain.length; + expect(ast.chain[chainLength - 2].arguments.name).toEqual(['math']); + expect(ast.chain[chainLength - 1].arguments.id).toEqual(['formula']); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts index 6ac208913af2e8..40d7e3ef94ad68 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts @@ -12,6 +12,7 @@ const actualMocks = jest.requireActual('../mocks'); jest.spyOn(actualOperations.operationDefinitionMap.date_histogram, 'paramEditor'); jest.spyOn(actualOperations.operationDefinitionMap.terms, 'onOtherColumnChanged'); +jest.spyOn(actualHelpers, 'copyColumn'); jest.spyOn(actualHelpers, 'insertOrReplaceColumn'); jest.spyOn(actualHelpers, 'insertNewColumn'); jest.spyOn(actualHelpers, 'replaceColumn'); @@ -30,6 +31,7 @@ export const { } = actualOperations; export const { + copyColumn, insertOrReplaceColumn, insertNewColumn, replaceColumn, @@ -50,4 +52,4 @@ export const { export const { adjustTimeScaleLabelSuffix, DEFAULT_TIME_SCALE } = actualTimeScaleUtils; -export const { createMockedReferenceOperation } = actualMocks; +export const { createMockedFullReference } = actualMocks; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index c57f70ba1b58b1..97582be2f32d66 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -50,7 +50,7 @@ export const counterRateOperation: OperationDefinition< selectionStyle: 'field', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], specificOperations: ['max'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, @@ -76,9 +76,17 @@ export const counterRateOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate'); }, - buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }) => { + buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => { const metric = layer.columns[referenceIds[0]]; const timeScale = previousColumn?.timeScale || DEFAULT_TIME_SCALE; + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName( metric && 'sourceField' in metric @@ -92,7 +100,7 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 7cec1fa0d4bbc9..e6f4f589f6189a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -48,7 +48,7 @@ export const cumulativeSumOperation: OperationDefinition< selectionStyle: 'field', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], specificOperations: ['count', 'sum'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, @@ -73,8 +73,16 @@ export const cumulativeSumOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum'); }, - buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }) => { + buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => { const ref = layer.columns[referenceIds[0]]; + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName( ref && 'sourceField' in ref @@ -85,7 +93,7 @@ export const cumulativeSumOperation: OperationDefinition< operationType: 'cumulative_sum', isBucketed: false, scale: 'ratio', - filter: previousColumn?.filter, + filter, references: referenceIds, params: getFormatFromPreviousColumn(previousColumn), }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index bef3fbc2e48aed..b030e604ada061 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -52,7 +52,7 @@ export const derivativeOperation: OperationDefinition< selectionStyle: 'full', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], @@ -71,7 +71,15 @@ export const derivativeOperation: OperationDefinition< toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'derivative'); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } const ref = layer.columns[referenceIds[0]]; return { label: ofName(ref?.label, previousColumn?.timeScale), @@ -81,7 +89,7 @@ export const derivativeOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 46cc64c2bc5184..88af8e9b6378e0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -37,6 +37,8 @@ const ofName = buildLabelFunction((name?: string) => { }); }); +const WINDOW_DEFAULT_VALUE = 5; + export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn & ReferenceBasedIndexPatternColumn & { operationType: 'moving_average'; @@ -58,10 +60,11 @@ export const movingAverageOperation: OperationDefinition< selectionStyle: 'full', requiredReferences: [ { - input: ['field'], + input: ['field', 'managedReference'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], + operationParams: [{ name: 'window', type: 'number', required: true }], getPossibleOperation: (indexPattern) => { if (hasDateField(indexPattern)) { return { @@ -79,8 +82,20 @@ export const movingAverageOperation: OperationDefinition< window: [(layer.columns[columnId] as MovingAverageIndexPatternColumn).params.window], }); }, - buildColumn: ({ referenceIds, previousColumn, layer }) => { + buildColumn: ( + { referenceIds, previousColumn, layer }, + columnParams = { window: WINDOW_DEFAULT_VALUE } + ) => { const metric = layer.columns[referenceIds[0]]; + const { window = WINDOW_DEFAULT_VALUE } = columnParams; + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName(metric?.label, previousColumn?.timeScale), dataType: 'number', @@ -89,9 +104,9 @@ export const movingAverageOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter: previousColumn?.filter, + filter, params: { - window: 5, + window, ...getFormatFromPreviousColumn(previousColumn), }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts index 4c1101d4c8a791..7a6f96d705b0c0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.test.ts @@ -7,7 +7,7 @@ import { checkReferences } from './utils'; import { operationDefinitionMap } from '..'; -import { createMockedReferenceOperation } from '../../mocks'; +import { createMockedFullReference } from '../../mocks'; // Mock prevents issue with circular loading jest.mock('..'); @@ -15,7 +15,7 @@ jest.mock('..'); describe('utils', () => { beforeEach(() => { // @ts-expect-error test-only operation type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); }); describe('checkReferences', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index fa1691ba9a78e5..df84ecb479de72 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -71,8 +71,21 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), - buildColumn({ field, previousColumn }) { + buildColumn({ field, previousColumn }, columnParams) { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: ofName(field.displayName), dataType: 'number', @@ -80,7 +93,7 @@ export const cardinalityOperation: OperationDefinition adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale), - buildColumn({ field, previousColumn }) { + buildColumn({ field, previousColumn }, columnParams) { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } return { label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale), dataType: 'number', @@ -61,7 +69,7 @@ export const countOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index bd7a270fd7ad80..affb84484c820c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -58,6 +58,7 @@ export const dateHistogramOperation: OperationDefinition< }), input: 'field', priority: 5, // Highest priority level used + operationParams: [{ name: 'interval', type: 'string', required: false }], getErrorMessage: (layer, columnId, indexPattern) => getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), getHelpMessage: (props) => , @@ -75,8 +76,8 @@ export const dateHistogramOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern) => getSafeName(column.sourceField, indexPattern), - buildColumn({ field }) { - let interval = autoInterval; + buildColumn({ field }, columnParams) { + let interval = columnParams?.interval ?? autoInterval; let timeZone: string | undefined; if (field.aggregationRestrictions && field.aggregationRestrictions.date_histogram) { interval = restrictedInterval(field.aggregationRestrictions) as string; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index ae097ada0f3b7b..46fddd9b1ffbf4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -27,6 +27,7 @@ const defaultProps = { data: dataPluginMock.createStartContract(), http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), + operationDefinitionMap: {}, }; // mocking random id generator function diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx new file mode 100644 index 00000000000000..de7ecb4bc75da3 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { OperationDefinition } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern } from '../../../types'; +import { runASTValidation, tryToParse } from './validation'; +import { regenerateLayerFromAst } from './parse'; +import { generateFormula } from './generate'; + +const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', +}); + +export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternColumn { + operationType: 'formula'; + params: { + formula?: string; + isFormulaBroken?: boolean; + // last value on numeric fields can be formatted + format?: { + id: string; + params?: { + decimals: number; + }; + }; + }; +} + +export const formulaOperation: OperationDefinition< + FormulaIndexPatternColumn, + 'managedReference' +> = { + type: 'formula', + displayName: defaultLabel, + getDefaultLabel: (column, indexPattern) => defaultLabel, + input: 'managedReference', + hidden: true, + getDisabledStatus(indexPattern: IndexPattern) { + return undefined; + }, + getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap) { + const column = layer.columns[columnId] as FormulaIndexPatternColumn; + if (!column.params.formula || !operationDefinitionMap) { + return; + } + const { root, error } = tryToParse(column.params.formula); + if (error || !root) { + return [error!.message]; + } + + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + return errors.length ? errors.map(({ message }) => message) : undefined; + }, + getPossibleOperation() { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + toExpression: (layer, columnId) => { + const currentColumn = layer.columns[columnId] as FormulaIndexPatternColumn; + const params = currentColumn.params; + // TODO: improve this logic + const useDisplayLabel = currentColumn.label !== defaultLabel; + const label = !params?.isFormulaBroken + ? useDisplayLabel + ? currentColumn.label + : params?.formula + : ''; + + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + id: [columnId], + name: [label || ''], + exp: [ + { + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [ + currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``, + ], + }, + }, + ], + }, + ], + }, + }, + ]; + }, + buildColumn({ previousColumn, layer, indexPattern }, _, operationDefinitionMap) { + let previousFormula = ''; + if (previousColumn) { + previousFormula = generateFormula( + previousColumn, + layer, + previousFormula, + operationDefinitionMap + ); + } + // carry over the format settings from previous operation for seamless transfer + // NOTE: this works only for non-default formatters set in Lens + let prevFormat = {}; + if (previousColumn?.params && 'format' in previousColumn.params) { + prevFormat = { format: previousColumn.params.format }; + } + return { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: previousFormula + ? { formula: previousFormula, isFormulaBroken: false, ...prevFormat } + : { ...prevFormat }, + references: [], + }; + }, + isTransferable: () => { + return true; + }, + createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap) { + const currentColumn = layer.columns[sourceId] as FormulaIndexPatternColumn; + const tempLayer = { + ...layer, + columns: { + ...layer.columns, + [targetId]: { ...currentColumn }, + }, + }; + const { newLayer } = regenerateLayerFromAst( + currentColumn.params.formula ?? '', + tempLayer, + targetId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + return newLayer; + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts new file mode 100644 index 00000000000000..e44cd50ae9c412 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; + +// Just handle two levels for now +type OperationParams = Record>; + +export function getSafeFieldName(fieldName: string | undefined) { + // clean up the "Records" field for now + if (!fieldName || fieldName === 'Records') { + return ''; + } + return fieldName; +} + +export function generateFormula( + previousColumn: ReferenceBasedIndexPatternColumn | IndexPatternColumn, + layer: IndexPatternLayer, + previousFormula: string, + operationDefinitionMap: Record | undefined +) { + if ('references' in previousColumn) { + const metric = layer.columns[previousColumn.references[0]]; + if (metric && 'sourceField' in metric && metric.dataType === 'number') { + const fieldName = getSafeFieldName(metric.sourceField); + // TODO need to check the input type from the definition + previousFormula += `${previousColumn.operationType}(${metric.operationType}(${fieldName})`; + } + } else { + if (previousColumn && 'sourceField' in previousColumn && previousColumn.dataType === 'number') { + previousFormula += `${previousColumn.operationType}(${getSafeFieldName( + previousColumn?.sourceField + )}`; + } + } + const formulaNamedArgs = extractParamsForFormula(previousColumn, operationDefinitionMap); + if (formulaNamedArgs.length) { + previousFormula += + ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); + } + if (previousColumn.filter) { + if (previousColumn.operationType !== 'count') { + previousFormula += ', '; + } + previousFormula += + (previousColumn.filter.language === 'kuery' ? 'kql=' : 'lucene=') + + `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all + } + if (previousFormula) { + // close the formula at the end + previousFormula += ')'; + } + return previousFormula; +} + +function extractParamsForFormula( + column: IndexPatternColumn | ReferenceBasedIndexPatternColumn, + operationDefinitionMap: Record | undefined +) { + if (!operationDefinitionMap) { + return []; + } + const def = operationDefinitionMap[column.operationType]; + if ('operationParams' in def && column.params) { + return (def.operationParams || []).flatMap(({ name, required }) => { + const value = (column.params as OperationParams)![name]; + if (isObject(value)) { + return Object.keys(value).map((subName) => ({ + name: `${name}-${subName}`, + value: value[subName] as string | number, + required, + })); + } + return { + name, + value, + required, + }; + }); + } + return []; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts new file mode 100644 index 00000000000000..bafde0d37b3e9f --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { formulaOperation, FormulaIndexPatternColumn } from './formula'; +export { regenerateLayerFromAst } from './parse'; +export { mathOperation, MathIndexPatternColumn } from './math'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx new file mode 100644 index 00000000000000..527af324b5b054 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TinymathAST } from '@kbn/tinymath'; +import { OperationDefinition } from '../index'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPattern } from '../../../types'; + +export interface MathIndexPatternColumn extends ReferenceBasedIndexPatternColumn { + operationType: 'math'; + params: { + tinymathAst: TinymathAST | string; + // last value on numeric fields can be formatted + format?: { + id: string; + params?: { + decimals: number; + }; + }; + }; +} + +export const mathOperation: OperationDefinition = { + type: 'math', + displayName: 'Math', + hidden: true, + getDefaultLabel: (column, indexPattern) => 'Math', + input: 'managedReference', + getDisabledStatus(indexPattern: IndexPattern) { + return undefined; + }, + getPossibleOperation() { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + toExpression: (layer, columnId) => { + const column = layer.columns[columnId] as MathIndexPatternColumn; + return [ + { + type: 'function', + function: 'mapColumn', + arguments: { + id: [columnId], + name: [columnId], + exp: [ + { + type: 'expression', + chain: [ + { + type: 'function', + function: 'math', + arguments: { + expression: [astToString(column.params.tinymathAst)], + onError: ['null'], + }, + }, + ], + }, + ], + }, + }, + ]; + }, + buildColumn() { + return { + label: 'Math', + dataType: 'number', + operationType: 'math', + isBucketed: false, + scale: 'ratio', + params: { + tinymathAst: '', + }, + references: [], + }; + }, + isTransferable: (column, newIndexPattern) => { + // TODO has to check all children + return true; + }, + createCopy: (layer) => { + return { ...layer }; + }, +}; + +function astToString(ast: TinymathAST | string): string | number { + if (typeof ast === 'number') { + return ast; + } + if (typeof ast === 'string') { + // Double quotes around uuids like 1234-5678X2 to avoid ambiguity + return `"${ast}"`; + } + if (ast.type === 'variable') { + return ast.value; + } + if (ast.type === 'namedArgument') { + if (ast.name === 'kql' || ast.name === 'lucene') { + return `${ast.name}='${ast.value}'`; + } + return `${ast.name}=${ast.value}`; + } + return `${ast.name}(${ast.args.map(astToString).join(',')})`; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts new file mode 100644 index 00000000000000..70ed2f36dfd1c2 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; +import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { mathOperation } from './math'; +import { documentField } from '../../../document_field'; +import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; +import { findVariables, getOperationParams, groupArgsByType } from './util'; +import { FormulaIndexPatternColumn } from './formula'; +import { getColumnOrder } from '../../layer_helpers'; + +function parseAndExtract( + text: string, + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { root, error } = tryToParse(text); + if (error || !root) { + return { extracted: [], isValid: false }; + } + // before extracting the data run the validation task and throw if invalid + const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + if (errors.length) { + return { extracted: [], isValid: false }; + } + /* + { name: 'add', args: [ { name: 'abc', args: [5] }, 5 ] } + */ + const extracted = extractColumns(columnId, operationDefinitionMap, root, layer, indexPattern); + return { extracted, isValid: true }; +} + +function extractColumns( + idPrefix: string, + operations: Record, + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern +): Array<{ column: IndexPatternColumn; location?: TinymathLocation }> { + const columns: Array<{ column: IndexPatternColumn; location?: TinymathLocation }> = []; + + function parseNode(node: TinymathAST) { + if (typeof node === 'number' || node.type !== 'function') { + // leaf node + return node; + } + + const nodeOperation = operations[node.name]; + if (!nodeOperation) { + // it's a regular math node + const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< + number | TinymathVariable + >; + return { + ...node, + args: consumedArgs, + }; + } + + // split the args into types for better TS experience + const { namedArguments, variables, functions } = groupArgsByType(node.args); + + // operation node + if (nodeOperation.input === 'field') { + const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); + // a validation task passed before executing this and checked already there's a field + const field = shouldHaveFieldArgument(node) + ? indexPattern.getFieldByName(fieldName.value)! + : documentField; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'field' + >).buildColumn( + { + layer, + indexPattern, + field, + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + + if (nodeOperation.input === 'fullReference') { + const [referencedOp] = functions; + const consumedParam = parseNode(referencedOp); + + const subNodeVariables = consumedParam ? findVariables(consumedParam) : []; + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = subNodeVariables.map(({ value }) => value); + mathColumn.params.tinymathAst = consumedParam!; + columns.push({ column: mathColumn }); + mathColumn.customLabel = true; + mathColumn.label = `${idPrefix}X${columns.length - 1}`; + + const mappedParams = getOperationParams(nodeOperation, namedArguments || []); + const newCol = (nodeOperation as OperationDefinition< + IndexPatternColumn, + 'fullReference' + >).buildColumn( + { + layer, + indexPattern, + referenceIds: [`${idPrefix}X${columns.length - 1}`], + }, + mappedParams + ); + const newColId = `${idPrefix}X${columns.length}`; + newCol.customLabel = true; + newCol.label = newColId; + columns.push({ column: newCol, location: node.location }); + // replace by new column id + return newColId; + } + } + const root = parseNode(ast); + if (root === undefined) { + return []; + } + const variables = findVariables(root); + const mathColumn = mathOperation.buildColumn({ + layer, + indexPattern, + }); + mathColumn.references = variables.map(({ value }) => value); + mathColumn.params.tinymathAst = root!; + const newColId = `${idPrefix}X${columns.length}`; + mathColumn.customLabel = true; + mathColumn.label = newColId; + columns.push({ column: mathColumn }); + return columns; +} + +export function regenerateLayerFromAst( + text: string, + layer: IndexPatternLayer, + columnId: string, + currentColumn: FormulaIndexPatternColumn, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { extracted, isValid } = parseAndExtract( + text, + layer, + columnId, + indexPattern, + operationDefinitionMap + ); + + const columns = { ...layer.columns }; + + const locations: Record = {}; + + Object.keys(columns).forEach((k) => { + if (k.startsWith(columnId)) { + delete columns[k]; + } + }); + + extracted.forEach(({ column, location }, index) => { + columns[`${columnId}X${index}`] = column; + if (location) locations[`${columnId}X${index}`] = location; + }); + + columns[columnId] = { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text, + isFormulaBroken: !isValid, + }, + references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + }; + + return { + newLayer: { + ...layer, + columns, + columnOrder: getColumnOrder({ + ...layer, + columns, + }), + }, + locations, + }; + + // TODO + // turn ast into referenced columns + // set state +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts new file mode 100644 index 00000000000000..5d9a8647eb7ab0 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -0,0 +1,317 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { groupBy, isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import type { + TinymathAST, + TinymathFunction, + TinymathNamedArgument, + TinymathVariable, +} from 'packages/kbn-tinymath'; +import type { OperationDefinition, IndexPatternColumn } from '../index'; +import type { GroupedNodes } from './types'; + +export function groupArgsByType(args: TinymathAST[]) { + const { namedArgument, variable, function: functions } = groupBy( + args, + (arg: TinymathAST) => { + return isObject(arg) ? arg.type : 'variable'; + } + ) as GroupedNodes; + // better naming + return { + namedArguments: namedArgument || [], + variables: variable || [], + functions: functions || [], + }; +} + +export function getValueOrName(node: TinymathAST) { + if (!isObject(node)) { + return node; + } + if (node.type !== 'function') { + return node.value; + } + return node.name; +} + +export function getOperationParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +): Record { + const formalArgs: Record = (operation.operationParams || []).reduce( + (memo: Record, { name, type }) => { + memo[name] = type; + return memo; + }, + {} + ); + + return params.reduce>((args, { name, value }) => { + if (formalArgs[name]) { + args[name] = value; + } + if (operation.filterable && (name === 'kql' || name === 'lucene')) { + args[name] = value; + } + return args; + }, {}); +} + +// Todo: i18n everything here +export const tinymathFunctions: Record< + string, + { + positionalArguments: Array<{ + name: string; + optional?: boolean; + }>; + // help: React.ReactElement; + // Help is in Markdown format + help: string; + } +> = { + add: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], + help: ` +Also works with + symbol +Example: ${'`count() + sum(bytes)`'} +Example: ${'`add(count(), 5)`'} + `, + }, + subtract: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], + help: ` +Also works with ${'`-`'} symbol +Example: ${'`subtract(sum(bytes), avg(bytes))`'} + `, + }, + multiply: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], + help: ` +Also works with ${'`*`'} symbol +Example: ${'`multiply(sum(bytes), 2)`'} + `, + }, + divide: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.left', { defaultMessage: 'left' }) }, + { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, + ], + help: ` +Also works with ${'`/`'} symbol +Example: ${'`ceil(sum(bytes))`'} + `, + }, + abs: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Absolute value +Example: ${'`abs(sum(bytes))`'} + `, + }, + cbrt: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Cube root of value +Example: ${'`cbrt(sum(bytes))`'} + `, + }, + ceil: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Ceiling of value, rounds up +Example: ${'`ceil(sum(bytes))`'} + `, + }, + clamp: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { name: i18n.translate('xpack.lens.formula.min', { defaultMessage: 'min' }) }, + { name: i18n.translate('xpack.lens.formula.max', { defaultMessage: 'max' }) }, + ], + help: ` +Limits the value from a minimum to maximum +Example: ${'`ceil(sum(bytes))`'} + `, + }, + cube: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Limits the value from a minimum to maximum +Example: ${'`ceil(sum(bytes))`'} + `, + }, + exp: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Raises e to the nth power. +Example: ${'`exp(sum(bytes))`'} + `, + }, + fix: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +For positive values, takes the floor. For negative values, takes the ceiling. +Example: ${'`fix(sum(bytes))`'} + `, + }, + floor: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Round down to nearest integer value +Example: ${'`floor(sum(bytes))`'} + `, + }, + log: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + optional: true, + }, + ], + help: ` +Logarithm with optional base. The natural base e is used as default. +Example: ${'`log(sum(bytes))`'} +Example: ${'`log(sum(bytes), 2)`'} + `, + }, + // TODO: check if this is valid for Tinymath + // log10: { + // positionalArguments: [ + // { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + // ], + // help: ` + // Base 10 logarithm. + // Example: ${'`log10(sum(bytes))`'} + // `, + // }, + mod: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + optional: true, + }, + ], + help: ` +Remainder after dividing the function by a number +Example: ${'`mod(sum(bytes), 2)`'} + `, + }, + pow: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + }, + ], + help: ` +Raises the value to a certain power. The second argument is required +Example: ${'`pow(sum(bytes), 3)`'} + `, + }, + round: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + { + name: i18n.translate('xpack.lens.formula.decimals', { defaultMessage: 'decimals' }), + optional: true, + }, + ], + help: ` +Rounds to a specific number of decimal places, default of 0 +Example: ${'`round(sum(bytes))`'} +Example: ${'`round(sum(bytes), 2)`'} + `, + }, + sqrt: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Square root of a positive value only +Example: ${'`sqrt(sum(bytes))`'} + `, + }, + square: { + positionalArguments: [ + { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, + ], + help: ` +Raise the value to the 2nd power +Example: ${'`square(sum(bytes))`'} + `, + }, +}; + +export function isMathNode(node: TinymathAST) { + return isObject(node) && node.type === 'function' && tinymathFunctions[node.name]; +} + +export function findMathNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenMathNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function' || !isMathNode(node)) { + return []; + } + return [node, ...node.args.flatMap(flattenMathNodes)].filter(Boolean); + } + return flattenMathNodes(root); +} + +// traverse a tree and find all string leaves +export function findVariables(node: TinymathAST | string): TinymathVariable[] { + if (typeof node === 'string') { + return [ + { + type: 'variable', + value: node, + text: node, + location: { min: 0, max: 0 }, + }, + ]; + } + if (node == null) { + return []; + } + if (typeof node === 'number' || node.type === 'namedArgument') { + return []; + } + if (node.type === 'variable') { + // leaf node + return [node]; + } + return node.args.flatMap(findVariables); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts new file mode 100644 index 00000000000000..4e5ae21e576e45 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -0,0 +1,686 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isObject } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { parse, TinymathLocation } from '@kbn/tinymath'; +import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; +import { esKuery, esQuery } from '../../../../../../../../src/plugins/data/public'; +import { + findMathNodes, + findVariables, + getOperationParams, + getValueOrName, + groupArgsByType, + isMathNode, + tinymathFunctions, +} from './util'; + +import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; +import type { IndexPattern, IndexPatternLayer } from '../../../types'; +import type { TinymathNodeTypes } from './types'; + +const validationErrors = { + missingField: { message: 'missing field', type: { variablesLength: 1, variablesList: 'string' } }, + missingOperation: { + message: 'missing operation', + type: { operationLength: 1, operationsList: 'string' }, + }, + missingParameter: { + message: 'missing parameter', + type: { operation: 'string', params: 'string' }, + }, + wrongTypeParameter: { + message: 'wrong type parameter', + type: { operation: 'string', params: 'string' }, + }, + wrongFirstArgument: { + message: 'wrong first argument', + type: { operation: 'string', type: 'string', argument: 'any' as string | number }, + }, + cannotAcceptParameter: { message: 'cannot accept parameter', type: { operation: 'string' } }, + shouldNotHaveField: { message: 'operation should not have field', type: { operation: 'string' } }, + tooManyArguments: { message: 'too many arguments', type: { operation: 'string' } }, + fieldWithNoOperation: { + message: 'unexpected field with no operation', + type: { field: 'string' }, + }, + failedParsing: { message: 'Failed to parse expression', type: { expression: 'string' } }, + duplicateArgument: { + message: 'duplicate argument', + type: { operation: 'string', params: 'string' }, + }, + missingMathArgument: { + message: 'missing math argument', + type: { operation: 'string', count: 1, params: 'string' }, + }, +}; +export const errorsLookup = new Set(Object.values(validationErrors).map(({ message }) => message)); +type ErrorTypes = keyof typeof validationErrors; +type ErrorValues = typeof validationErrors[K]['type']; + +export interface ErrorWrapper { + message: string; + locations: TinymathLocation[]; + severity?: 'error' | 'warning'; +} + +export function isParsingError(message: string) { + return message.includes(validationErrors.failedParsing.message); +} + +function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { + function flattenFunctionNodes(node: TinymathAST | string): TinymathFunction[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + return [node, ...node.args.flatMap(flattenFunctionNodes)].filter(Boolean); + } + return flattenFunctionNodes(root); +} + +export function hasInvalidOperations( + node: TinymathAST | string, + operations: Record +): { names: string[]; locations: TinymathLocation[] } { + const nodes = findFunctionNodes(node).filter((v) => !isMathNode(v) && !operations[v.name]); + return { + // avoid duplicates + names: Array.from(new Set(nodes.map(({ name }) => name))), + locations: nodes.map(({ location }) => location), + }; +} + +export const getQueryValidationError = ( + query: string, + language: 'kql' | 'lucene', + indexPattern: IndexPattern +): string | undefined => { + try { + if (language === 'kql') { + esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query), indexPattern); + } else { + esQuery.luceneStringToDsl(query); + } + return; + } catch (e) { + return e.message; + } +}; + +function getMessageFromId({ + messageId, + values, + locations, +}: { + messageId: K; + values: ErrorValues; + locations: TinymathLocation[]; +}): ErrorWrapper { + let message: string; + switch (messageId) { + case 'wrongFirstArgument': + message = i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: + 'The first argument for {operation} should be a {type} name. Found {argument}', + values, + }); + break; + case 'shouldNotHaveField': + message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { + defaultMessage: 'The operation {operation} does not accept any field as argument', + values, + }); + break; + case 'cannotAcceptParameter': + message = i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { + defaultMessage: 'The operation {operation} does not accept any parameter', + values, + }); + break; + case 'missingParameter': + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The operation {operation} in the Formula is missing the following parameters: {params}', + values, + }); + break; + case 'wrongTypeParameter': + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: + 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', + values, + }); + break; + case 'duplicateArgument': + message = i18n.translate('xpack.lens.indexPattern.formulaOperationDuplicateParams', { + defaultMessage: + 'The parameters for the operation {operation} have been declared multiple times: {params}', + values, + }); + break; + case 'missingField': + message = i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: + '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', + values, + }); + break; + case 'missingOperation': + message = i18n.translate('xpack.lens.indexPattern.operationsNotFound', { + defaultMessage: + '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', + values, + }); + break; + case 'fieldWithNoOperation': + message = i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { + defaultMessage: 'The field {field} cannot be used without operation', + values, + }); + break; + case 'failedParsing': + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + defaultMessage: 'The Formula {expression} cannot be parsed', + values, + }); + break; + case 'tooManyArguments': + message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', { + defaultMessage: 'The operation {operation} has too many arguments', + values, + }); + break; + case 'missingMathArgument': + message = i18n.translate('xpack.lens.indexPattern.formulaMathMissingArgument', { + defaultMessage: + 'The operation {operation} in the Formula is missing {count} arguments: {params}', + values, + }); + break; + // case 'mathRequiresFunction': + // message = i18n.translate('xpack.lens.indexPattern.formulaMathRequiresFunctionLabel', { + // defaultMessage; 'The function {name} requires an Elasticsearch function', + // values, + // }); + // break; + default: + message = 'no Error found'; + break; + } + + return { message, locations }; +} + +export function tryToParse( + formula: string +): { root: TinymathAST; error: null } | { root: null; error: ErrorWrapper } { + let root; + try { + root = parse(formula); + } catch (e) { + return { + root: null, + error: getMessageFromId({ + messageId: 'failedParsing', + values: { + expression: formula, + }, + locations: [], + }), + }; + } + return { root, error: null }; +} + +export function runASTValidation( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record +) { + return [ + ...checkMissingVariableOrFunctions(ast, layer, indexPattern, operations), + ...runFullASTValidation(ast, layer, indexPattern, operations), + ]; +} + +function checkVariableEdgeCases(ast: TinymathAST, missingVariables: Set) { + const invalidVariableErrors = []; + if (isObject(ast) && ast.type === 'variable' && !missingVariables.has(ast.value)) { + invalidVariableErrors.push( + getMessageFromId({ + messageId: 'fieldWithNoOperation', + values: { + field: ast.value, + }, + locations: [ast.location], + }) + ); + } + return invalidVariableErrors; +} + +function checkMissingVariableOrFunctions( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record +): ErrorWrapper[] { + const missingErrors: ErrorWrapper[] = []; + const missingOperations = hasInvalidOperations(ast, operations); + + if (missingOperations.names.length) { + missingErrors.push( + getMessageFromId({ + messageId: 'missingOperation', + values: { + operationLength: missingOperations.names.length, + operationsList: missingOperations.names.join(', '), + }, + locations: missingOperations.locations, + }) + ); + } + const missingVariables = findVariables(ast).filter( + // filter empty string as well? + ({ value }) => !indexPattern.getFieldByName(value) && !layer.columns[value] + ); + + // need to check the arguments here: check only strings for now + if (missingVariables.length) { + missingErrors.push( + getMessageFromId({ + messageId: 'missingField', + values: { + variablesLength: missingVariables.length, + variablesList: missingVariables.map(({ value }) => value).join(', '), + }, + locations: missingVariables.map(({ location }) => location), + }) + ); + } + const invalidVariableErrors = checkVariableEdgeCases( + ast, + new Set(missingVariables.map(({ value }) => value)) + ); + return [...missingErrors, ...invalidVariableErrors]; +} + +function getQueryValidationErrors( + namedArguments: TinymathNamedArgument[] | undefined, + indexPattern: IndexPattern +): ErrorWrapper[] { + const errors: ErrorWrapper[] = []; + (namedArguments ?? []).forEach((arg) => { + if (arg.name === 'kql' || arg.name === 'lucene') { + const message = getQueryValidationError(arg.value, arg.name, indexPattern); + if (message) { + errors.push({ + message, + locations: [arg.location], + }); + } + } + }); + return errors; +} + +function validateNameArguments( + node: TinymathFunction, + nodeOperation: + | OperationDefinition + | OperationDefinition, + namedArguments: TinymathNamedArgument[] | undefined, + indexPattern: IndexPattern +) { + const errors = []; + const missingParams = getMissingParams(nodeOperation, namedArguments); + if (missingParams.length) { + errors.push( + getMessageFromId({ + messageId: 'missingParameter', + values: { + operation: node.name, + params: missingParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const wrongTypeParams = getWrongTypeParams(nodeOperation, namedArguments); + if (wrongTypeParams.length) { + errors.push( + getMessageFromId({ + messageId: 'wrongTypeParameter', + values: { + operation: node.name, + params: wrongTypeParams.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + const duplicateParams = getDuplicateParams(namedArguments); + if (duplicateParams.length) { + errors.push( + getMessageFromId({ + messageId: 'duplicateArgument', + values: { + operation: node.name, + params: duplicateParams.join(', '), + }, + locations: [node.location], + }) + ); + } + const queryValidationErrors = getQueryValidationErrors(namedArguments, indexPattern); + if (queryValidationErrors.length) { + errors.push(...queryValidationErrors); + } + return errors; +} + +function runFullASTValidation( + ast: TinymathAST, + layer: IndexPatternLayer, + indexPattern: IndexPattern, + operations: Record +): ErrorWrapper[] { + const missingVariables = findVariables(ast).filter( + // filter empty string as well? + ({ value }) => !indexPattern.getFieldByName(value) && !layer.columns[value] + ); + const missingVariablesSet = new Set(missingVariables.map(({ value }) => value)); + + function validateNode(node: TinymathAST): ErrorWrapper[] { + if (!isObject(node) || node.type !== 'function') { + return []; + } + const nodeOperation = operations[node.name]; + const errors: ErrorWrapper[] = []; + const { namedArguments, functions, variables } = groupArgsByType(node.args); + const [firstArg] = node?.args || []; + + if (!nodeOperation) { + errors.push(...validateMathNodes(node, missingVariablesSet)); + // carry on with the validation for all the functions within the math operation + if (functions?.length) { + return errors.concat(functions.flatMap((fn) => validateNode(fn))); + } + } else { + if (nodeOperation.input === 'field') { + if (shouldHaveFieldArgument(node)) { + if (!isFirstArgumentValidType(firstArg, 'variable')) { + if (isMathNode(firstArg)) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: `math operation`, + }, + locations: [node.location], + }) + ); + } else { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'field', + argument: getValueOrName(firstArg), + }, + locations: [node.location], + }) + ); + } + } + } else { + // Named arguments only + if (functions?.length || variables?.length) { + errors.push( + getMessageFromId({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + getMessageFromId({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } else { + const argumentsErrors = validateNameArguments( + node, + nodeOperation, + namedArguments, + indexPattern + ); + if (argumentsErrors.length) { + errors.push(...argumentsErrors); + } + } + return errors; + } + if (nodeOperation.input === 'fullReference') { + // What about fn(7 + 1)? We may want to allow that + // In general this should be handled down the Esaggs route rather than here + if ( + !isFirstArgumentValidType(firstArg, 'function') || + (isMathNode(firstArg) && validateMathNodes(firstArg, missingVariablesSet).length) + ) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'operation', + argument: getValueOrName(firstArg), + }, + locations: [node.location], + }) + ); + } + if (!canHaveParams(nodeOperation) && namedArguments.length) { + errors.push( + getMessageFromId({ + messageId: 'cannotAcceptParameter', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } else { + const argumentsErrors = validateNameArguments( + node, + nodeOperation, + namedArguments, + indexPattern + ); + if (argumentsErrors.length) { + errors.push(...argumentsErrors); + } + } + } + return errors.concat(validateNode(functions[0])); + } + return errors; + } + + return validateNode(ast); +} + +export function canHaveParams( + operation: + | OperationDefinition + | OperationDefinition +) { + return Boolean((operation.operationParams || []).length) || operation.filterable; +} + +export function getInvalidParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isCorrectType, isRequired }) => (isMissing && isRequired) || !isCorrectType + ); +} + +export function getMissingParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isMissing, isRequired }) => isMissing && isRequired + ); +} + +export function getWrongTypeParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + return validateParams(operation, params).filter( + ({ isCorrectType, isMissing }) => !isCorrectType && !isMissing + ); +} + +function getDuplicateParams(params: TinymathNamedArgument[] = []) { + const uniqueArgs = Object.create(null); + for (const { name } of params) { + const counter = uniqueArgs[name] || 0; + uniqueArgs[name] = counter + 1; + } + const uniqueNames = Object.keys(uniqueArgs); + if (params.length > uniqueNames.length) { + return uniqueNames.filter((name) => uniqueArgs[name] > 1); + } + return []; +} + +export function validateParams( + operation: + | OperationDefinition + | OperationDefinition, + params: TinymathNamedArgument[] = [] +) { + const paramsObj = getOperationParams(operation, params); + const formalArgs = [...(operation.operationParams ?? [])]; + if (operation.filterable) { + formalArgs.push( + { name: 'kql', type: 'string', required: false }, + { name: 'lucene', type: 'string', required: false } + ); + } + return formalArgs.map(({ name, type, required }) => ({ + name, + isMissing: !(name in paramsObj), + isCorrectType: typeof paramsObj[name] === type, + isRequired: required, + })); +} + +export function shouldHaveFieldArgument(node: TinymathFunction) { + return !['count'].includes(node.name); +} + +export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { + return isObject(arg) && arg.type === type; +} + +export function validateMathNodes(root: TinymathAST, missingVariableSet: Set) { + const mathNodes = findMathNodes(root); + const errors: ErrorWrapper[] = []; + mathNodes.forEach((node: TinymathFunction) => { + const { positionalArguments } = tinymathFunctions[node.name]; + if (!node.args.length) { + // we can stop here + return errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: 'operation', + argument: `()`, + }, + locations: [node.location], + }) + ); + } + + if (node.args.length > positionalArguments.length) { + errors.push( + getMessageFromId({ + messageId: 'tooManyArguments', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } + + // no need to iterate all the arguments, one field is anough to trigger the error + const hasFieldAsArgument = positionalArguments.some((requirements, index) => { + const arg = node.args[index]; + if (arg != null && typeof arg !== 'number') { + return arg.type === 'variable' && !missingVariableSet.has(arg.value); + } + }); + if (hasFieldAsArgument) { + errors.push( + getMessageFromId({ + messageId: 'shouldNotHaveField', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } + + const mandatoryArguments = positionalArguments.filter(({ optional }) => !optional); + // if there is only 1 mandatory arg, this is already handled by the wrongFirstArgument check + if (mandatoryArguments.length > 1 && node.args.length < mandatoryArguments.length) { + const missingArgs = positionalArguments.filter( + ({ name, optional }, i) => !optional && node.args[i] == null + ); + errors.push( + getMessageFromId({ + messageId: 'missingMathArgument', + values: { + operation: node.name, + count: mandatoryArguments.length - node.args.length, + params: missingArgs.map(({ name }) => name).join(', '), + }, + locations: [node.location], + }) + ); + } + }); + return errors; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts index bff997c8a81e85..bf24e31ad4f59b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.test.ts @@ -37,7 +37,7 @@ describe('helpers', () => { createMockedIndexPattern() ); expect(messages).toHaveLength(1); - expect(messages![0]).toEqual('Field timestamp was not found'); + expect(messages![0]).toEqual('Field timestamp is of the wrong type'); }); it('returns no message if all fields are matching', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx index b7e92a0b549527..675a418c7cdc95 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx @@ -54,14 +54,37 @@ export function getInvalidFieldMessage( operationDefinition.getPossibleOperationForField(field) !== undefined ) ); - return isInvalid - ? [ - i18n.translate('xpack.lens.indexPattern.fieldNotFound', { - defaultMessage: 'Field {invalidField} was not found', - values: { invalidField: sourceField }, + + const isWrongType = Boolean( + sourceField && + operationDefinition && + field && + !operationDefinition.isTransferable( + column as IndexPatternColumn, + indexPattern, + operationDefinitionMap + ) + ); + if (isInvalid) { + if (isWrongType) { + return [ + i18n.translate('xpack.lens.indexPattern.fieldWrongType', { + defaultMessage: 'Field {invalidField} is of the wrong type', + values: { + invalidField: sourceField, + }, }), - ] - : undefined; + ]; + } + return [ + i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + defaultMessage: 'Field {invalidField} was not found', + values: { invalidField: sourceField }, + }), + ]; + } + + return undefined; } export function getSafeName(name: string, indexPattern: IndexPattern): string { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 37bd64251ed814..a7402bc13c0a88 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -35,6 +35,12 @@ import { MovingAverageIndexPatternColumn, } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; +import { + mathOperation, + MathIndexPatternColumn, + formulaOperation, + FormulaIndexPatternColumn, +} from './formula'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -66,7 +72,9 @@ export type IndexPatternColumn = | CumulativeSumIndexPatternColumn | CounterRateIndexPatternColumn | DerivativeIndexPatternColumn - | MovingAverageIndexPatternColumn; + | MovingAverageIndexPatternColumn + | MathIndexPatternColumn + | FormulaIndexPatternColumn; export type FieldBasedIndexPatternColumn = Extract; @@ -115,6 +123,8 @@ const internalOperationDefinitions = [ counterRateOperation, derivativeOperation, movingAverageOperation, + mathOperation, + formulaOperation, ]; export { termsOperation } from './terms'; @@ -131,6 +141,7 @@ export { derivativeOperation, movingAverageOperation, } from './calculations'; +export { formulaOperation } from './formula/formula'; /** * Properties passed to the operation-specific part of the popover editor @@ -147,6 +158,7 @@ export interface ParamEditorProps { http: HttpSetup; dateRange: DateRange; data: DataPublicPluginStart; + operationDefinitionMap: Record; } export interface HelpProps { @@ -198,7 +210,11 @@ interface BaseOperationDefinitionProps { * If this function returns false, the column is removed when switching index pattern * for a layer */ - isTransferable: (column: C, newIndexPattern: IndexPattern) => boolean; + isTransferable: ( + column: C, + newIndexPattern: IndexPattern, + operationDefinitionMap: Record + ) => boolean; /** * Transfering a column to another index pattern. This can be used to * adjust operation specific settings such as reacting to aggregation restrictions @@ -220,7 +236,8 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + operationDefinitionMap?: Record ) => string[] | undefined; /* @@ -230,9 +247,18 @@ interface BaseOperationDefinitionProps { * If set to optional, time scaling won't be enabled by default and can be removed. */ timeScalingMode?: TimeScalingMode; + /** + * Filterable operations can have a KQL or Lucene query added at the dimension level. + * This flag is used by the formula to assign the kql= and lucene= named arguments and set up + * autocomplete. + */ filterable?: boolean; getHelpMessage?: (props: HelpProps) => React.ReactNode; + /* + * Operations can be used as middleware for other operations, hence not shown in the panel UI + */ + hidden?: boolean; } interface BaseBuildColumnArgs { @@ -240,15 +266,28 @@ interface BaseBuildColumnArgs { indexPattern: IndexPattern; } +interface OperationParam { + name: string; + type: string; + required?: boolean; +} + interface FieldlessOperationDefinition { input: 'none'; + + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; /** * Builds the column object for the given parameters. Should include default p */ buildColumn: ( arg: BaseBuildColumnArgs & { previousColumn?: IndexPatternColumn; - } + }, + columnParams?: (IndexPatternColumn & C)['params'] ) => C; /** * Returns the meta data of the operation if applied. Undefined @@ -270,6 +309,12 @@ interface FieldlessOperationDefinition { interface FieldBasedOperationDefinition { input: 'field'; + + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; /** * Returns the meta data of the operation if applied to the given field. Undefined * if the field is not applicable to the operation. @@ -282,7 +327,8 @@ interface FieldBasedOperationDefinition { arg: BaseBuildColumnArgs & { field: IndexPatternField; previousColumn?: IndexPatternColumn; - } + }, + columnParams?: (IndexPatternColumn & C)['params'] & { kql?: string; lucene?: string } ) => C; /** * This method will be called if the user changes the field of an operation. @@ -320,7 +366,8 @@ interface FieldBasedOperationDefinition { getErrorMessage: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + operationDefinitionMap?: Record ) => string[] | undefined; } @@ -333,6 +380,7 @@ export interface RequiredReference { // operation types. The main use case is Cumulative Sum, where we need to only take the // sum of Count or sum of Sum. specificOperations?: OperationType[]; + multi?: boolean; } // Full reference uses one or more reference operations which are visible to the user @@ -345,12 +393,19 @@ interface FullReferenceOperationDefinition { */ requiredReferences: RequiredReference[]; + /** + * The specification of the arguments used by the operations used for both validation, + * and use from external managed operations + */ + operationParams?: OperationParam[]; + /** * The type of UI that is shown in the editor for this function: * - full: List of sub-functions and fields * - field: List of fields, selects first operation per field + * - hidden: Do not allow to use operation directly */ - selectionStyle: 'full' | 'field'; + selectionStyle: 'full' | 'field' | 'hidden'; /** * Builds the column object for the given parameters. Should include default p @@ -359,6 +414,10 @@ interface FullReferenceOperationDefinition { arg: BaseBuildColumnArgs & { referenceIds: string[]; previousColumn?: IndexPatternColumn; + }, + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'] & { + kql?: string; + lucene?: string; } ) => ReferenceBasedIndexPatternColumn & C; /** @@ -376,10 +435,49 @@ interface FullReferenceOperationDefinition { ) => ExpressionAstFunction[]; } +interface ManagedReferenceOperationDefinition { + input: 'managedReference'; + /** + * Builds the column object for the given parameters. Should include default p + */ + buildColumn: ( + arg: BaseBuildColumnArgs & { + previousColumn?: IndexPatternColumn | ReferenceBasedIndexPatternColumn; + }, + columnParams?: (ReferenceBasedIndexPatternColumn & C)['params'], + operationDefinitionMap?: Record + ) => ReferenceBasedIndexPatternColumn & C; + /** + * Returns the meta data of the operation if applied. Undefined + * if the operation can't be added with these fields. + */ + getPossibleOperation: () => OperationMetadata | undefined; + /** + * A chain of expression functions which will transform the table + */ + toExpression: ( + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern + ) => ExpressionAstFunction[]; + /** + * Managed references control the IDs of their inner columns, so we need to be able to copy from the + * root level + */ + createCopy: ( + layer: IndexPatternLayer, + sourceColumnId: string, + targetColumnId: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record + ) => IndexPatternLayer; +} + interface OperationDefinitionMap { field: FieldBasedOperationDefinition; none: FieldlessOperationDefinition; fullReference: FullReferenceOperationDefinition; + managedReference: ManagedReferenceOperationDefinition; } /** @@ -405,7 +503,8 @@ export type OperationType = typeof internalOperationDefinitions[number]['type']; export type GenericOperationDefinition = | OperationDefinition | OperationDefinition - | OperationDefinition; + | OperationDefinition + | OperationDefinition; /** * List of all available operation definitions diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index b2244e0cc769ff..280cfe9471c9d0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -29,6 +29,7 @@ const defaultProps = { ...createMockedIndexPattern(), hasRestrictions: false, } as IndexPattern, + operationDefinitionMap: {}, }; describe('last_value', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 4f5c897fb5378b..a61cca89dfecfc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -141,7 +141,7 @@ export const lastValueOperation: OperationDefinition f.type === 'date')?.name; @@ -154,6 +154,14 @@ export const lastValueOperation: OperationDefinition>({ : (layer.columns[thisColumnId] as T), getDefaultLabel: (column, indexPattern, columns) => labelLookup(getSafeName(column.sourceField, indexPattern), column), - buildColumn: ({ field, previousColumn }) => - ({ + buildColumn: ({ field, previousColumn }, columnParams) => { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } + return { label: labelLookup(field.displayName, previousColumn), dataType: 'number', operationType: type, @@ -98,9 +106,10 @@ function buildMetricOperation>({ isBucketed: false, scale: 'ratio', timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, - filter: previousColumn?.filter, + filter, params: getFormatFromPreviousColumn(previousColumn), - } as T), + } as T; + }, onFieldChange: (oldColumn, field) => { return { ...oldColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index c14ff9f86f6027..a688f95e94c9ed 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -31,6 +31,7 @@ const defaultProps = { ...createMockedIndexPattern(), hasRestrictions: false, } as IndexPattern, + operationDefinitionMap: {}, }; describe('percentile', () => { @@ -178,6 +179,23 @@ describe('percentile', () => { expect(percentileColumn.params.percentile).toEqual(95); expect(percentileColumn.label).toEqual('95th percentile of test'); }); + + it('should create a percentile from formula', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileColumn = percentileOperation.buildColumn( + { + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }, + { percentile: 75 } + ); + expect(percentileColumn.dataType).toEqual('number'); + expect(percentileColumn.params.percentile).toEqual(75); + expect(percentileColumn.label).toEqual('75th percentile of test'); + }); }); describe('isTransferable', () => { @@ -202,7 +220,8 @@ describe('percentile', () => { percentile: 95, }, }, - indexPattern + indexPattern, + {} ) ).toBeTruthy(); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index dd0f3b978da5fc..187dc2dc53ffb8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -51,6 +51,7 @@ export const percentileOperation: OperationDefinition { if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { @@ -73,13 +74,14 @@ export const percentileOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile), - buildColumn: ({ field, previousColumn, indexPattern }) => { + buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { const existingPercentileParam = previousColumn?.operationType === 'percentile' && previousColumn.params && 'percentile' in previousColumn.params && previousColumn.params.percentile; - const newPercentileParam = existingPercentileParam || DEFAULT_PERCENTILE_VALUE; + const newPercentileParam = + columnParams?.percentile ?? (existingPercentileParam || DEFAULT_PERCENTILE_VALUE); return { label: ofName(getSafeName(field.name, indexPattern), newPercentileParam), dataType: 'number', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx index 62c729aa2b3f19..08bcfcb2e93be5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx @@ -90,6 +90,7 @@ const defaultOptions = { { name: sourceField, type: 'number', displayName: sourceField }, ]), }, + operationDefinitionMap: {}, }; describe('ranges', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 2e7307f6a2ec4a..b094d3f0ff5cd7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -28,6 +28,7 @@ const defaultProps = { data: dataPluginMock.createStartContract(), http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), + operationDefinitionMap: {}, }; describe('terms', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index c506e800d6d01c..52cc8a07511a47 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -7,6 +7,7 @@ import type { OperationMetadata } from '../../types'; import { + copyColumn, insertNewColumn, replaceColumn, updateColumnParam, @@ -23,7 +24,7 @@ import type { IndexPattern, IndexPatternLayer } from '../types'; import { documentField } from '../document_field'; import { getFieldByNameFactory } from '../pure_helpers'; import { generateId } from '../../id_generator'; -import { createMockedReferenceOperation } from './mocks'; +import { createMockedFullReference, createMockedManagedReference } from './mocks'; jest.mock('../operations'); jest.mock('../../id_generator'); @@ -89,11 +90,127 @@ describe('state_helpers', () => { (generateId as jest.Mock).mockImplementation(() => `id${++count}`); // @ts-expect-error we are inserting an invalid type - operationDefinitionMap.testReference = createMockedReferenceOperation(); + operationDefinitionMap.testReference = createMockedFullReference(); + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.managedReference = createMockedManagedReference(); }); afterEach(() => { delete operationDefinitionMap.testReference; + delete operationDefinitionMap.managedReference; + }); + + describe('copyColumn', () => { + it('should recursively modify a formula and update the math ast', () => { + const source = { + dataType: 'number' as const, + isBucketed: false, + label: 'Formula', + operationType: 'formula' as const, + params: { + formula: 'moving_average(sum(bytes), window=5)', + isFormulaBroken: false, + }, + references: ['formulaX1'], + }; + const math = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'math', + operationType: 'math' as const, + params: { tinymathAst: 'formulaX2' }, + references: ['formulaX2'], + }; + const sum = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX0', + operationType: 'sum' as const, + scale: 'ratio' as const, + sourceField: 'bytes', + }; + const movingAvg = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX2', + operationType: 'moving_average' as const, + params: { window: 5 }, + references: ['formulaX0'], + }; + expect( + copyColumn({ + layer: { + indexPatternId: '', + columnOrder: [], + columns: { + source, + formulaX0: sum, + formulaX1: math, + formulaX2: movingAvg, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + }, + }, + }, + targetId: 'copy', + sourceColumn: source, + shouldDeleteSource: false, + indexPattern, + sourceColumnId: 'source', + }) + ).toEqual({ + indexPatternId: '', + columnOrder: [ + 'source', + 'formulaX0', + 'formulaX1', + 'formulaX2', + 'formulaX3', + 'copyX0', + 'copyX1', + 'copyX2', + 'copyX3', + 'copy', + ], + columns: { + source, + formulaX0: sum, + formulaX1: math, + formulaX2: movingAvg, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + }, + copy: expect.objectContaining({ ...source, references: ['copyX3'] }), + copyX0: expect.objectContaining({ ...sum, label: 'copyX0' }), + copyX1: expect.objectContaining({ + ...math, + label: 'copyX1', + references: ['copyX0'], + params: { tinymathAst: 'copyX0' }, + }), + copyX2: expect.objectContaining({ + ...movingAvg, + label: 'copyX2', + references: ['copyX1'], + }), + copyX3: expect.objectContaining({ + ...math, + label: 'copyX3', + references: ['copyX2'], + params: { tinymathAst: 'copyX2' }, + }), + }, + }); + }); }); describe('insertNewColumn', () => { @@ -195,7 +312,7 @@ describe('state_helpers', () => { ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2'] })); }); - it('should insert a metric after buckets, but before references', () => { + it('should insert a metric after references', () => { const layer: IndexPatternLayer = { indexPatternId: '1', columnOrder: ['col1'], @@ -231,7 +348,7 @@ describe('state_helpers', () => { field: documentField, visualizationGroups: [], }) - ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2', 'col3'] })); + ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col3', 'col2'] })); }); it('should insert new buckets at the end of previous buckets', () => { @@ -1074,7 +1191,7 @@ describe('state_helpers', () => { referenceIds: ['id1'], }) ); - expect(result.columnOrder).toEqual(['id1', 'col1']); + expect(result.columnOrder).toEqual(['col1', 'id1']); expect(result.columns).toEqual( expect.objectContaining({ id1: expectedColumn, @@ -1196,7 +1313,7 @@ describe('state_helpers', () => { op: 'testReference', }); - expect(result.columnOrder).toEqual(['id1', 'col1']); + expect(result.columnOrder).toEqual(['col1', 'id1']); expect(result.columns).toEqual({ id1: expect.objectContaining({ operationType: 'average', @@ -1459,7 +1576,7 @@ describe('state_helpers', () => { }) ).toEqual( expect.objectContaining({ - columnOrder: ['id1', 'output'], + columnOrder: ['output', 'id1'], columns: { id1: expect.objectContaining({ sourceField: 'timestamp', @@ -2051,58 +2168,78 @@ describe('state_helpers', () => { ).toEqual(['col1', 'col3', 'col2']); }); - it('should correctly sort references to other references', () => { + it('does not topologically sort formulas, but keeps the relative order', () => { expect( getColumnOrder({ - columnOrder: [], indexPatternId: '', + columnOrder: [], columns: { - bucket: { - label: 'Top values of category', - dataType: 'string', + count: { + label: 'count', + dataType: 'number', + operationType: 'count', + isBucketed: false, + scale: 'ratio', + sourceField: 'Records', + customLabel: true, + }, + date: { + label: 'timestamp', + dataType: 'date', + operationType: 'date_histogram', + sourceField: 'timestamp', isBucketed: true, - - // Private - operationType: 'terms', - sourceField: 'category', + scale: 'interval', params: { - size: 5, - orderBy: { - type: 'alphabetical', - }, - orderDirection: 'asc', + interval: 'auto', }, }, - metric: { - label: 'Average of bytes', + formula: { + label: 'Formula', dataType: 'number', + operationType: 'formula', isBucketed: false, - - // Private - operationType: 'average', - sourceField: 'bytes', + scale: 'ratio', + params: { + formula: 'count() + count()', + isFormulaBroken: false, + }, + references: ['math'], }, - ref2: { - label: 'Ref2', + countX0: { + label: 'countX0', dataType: 'number', + operationType: 'count', isBucketed: false, - - // @ts-expect-error only for testing - operationType: 'testReference', - references: ['ref1'], + scale: 'ratio', + sourceField: 'Records', + customLabel: true, }, - ref1: { - label: 'Ref', + math: { + label: 'math', dataType: 'number', + operationType: 'math', isBucketed: false, - - // @ts-expect-error only for testing - operationType: 'testReference', - references: ['bucket'], + scale: 'ratio', + params: { + tinymathAst: { + type: 'function', + name: 'add', + // @ts-expect-error String args are not valid tinymath, but signals something unique to Lens + args: ['countX0', 'count'], + location: { + min: 0, + max: 17, + }, + text: 'count() + count()', + }, + }, + references: ['countX0', 'count'], + customLabel: true, }, }, }) - ).toEqual(['bucket', 'metric', 'ref1', 'ref2']); + ).toEqual(['date', 'count', 'formula', 'countX0', 'math']); }); }); @@ -2459,7 +2596,8 @@ describe('state_helpers', () => { }, }, 'col1', - indexPattern + indexPattern, + operationDefinitionMap ); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index beebb72fff6763..45345572a8f7a1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -6,6 +6,7 @@ */ import _, { partition } from 'lodash'; +import { getSortScoreByPriority } from './operations'; import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types'; import { operationDefinitionMap, @@ -15,9 +16,9 @@ import { RequiredReference, } from './definitions'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types'; -import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; +import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula'; interface ColumnChange { op: OperationType; @@ -32,7 +33,7 @@ interface ColumnChange { interface ColumnCopy { layer: IndexPatternLayer; - columnId: string; + targetId: string; sourceColumn: IndexPatternColumn; sourceColumnId: string; indexPattern: IndexPattern; @@ -41,16 +42,19 @@ interface ColumnCopy { export function copyColumn({ layer, - columnId, + targetId, sourceColumn, shouldDeleteSource, indexPattern, sourceColumnId, }: ColumnCopy): IndexPatternLayer { - let modifiedLayer = { - ...layer, - columns: copyReferencesRecursively(layer.columns, sourceColumn, columnId), - }; + let modifiedLayer = copyReferencesRecursively( + layer, + sourceColumn, + sourceColumnId, + targetId, + indexPattern + ); if (shouldDeleteSource) { modifiedLayer = deleteColumn({ @@ -64,16 +68,25 @@ export function copyColumn({ } function copyReferencesRecursively( - columns: Record, + layer: IndexPatternLayer, sourceColumn: IndexPatternColumn, - columnId: string -) { + sourceId: string, + targetId: string, + indexPattern: IndexPattern +): IndexPatternLayer { + let columns = { ...layer.columns }; if ('references' in sourceColumn) { - if (columns[columnId]) { - return columns; + if (columns[targetId]) { + return layer; + } + + const def = operationDefinitionMap[sourceColumn.operationType]; + if ('createCopy' in def) { + // Allow managed references to recursively insert new columns + return def.createCopy(layer, sourceId, targetId, indexPattern, operationDefinitionMap); } + sourceColumn?.references.forEach((ref, index) => { - // TODO: Add an option to assign IDs without generating the new one const newId = generateId(); const refColumn = { ...columns[ref] }; @@ -82,10 +95,10 @@ function copyReferencesRecursively( // and visible columns shouldn't be copied const refColumnWithInnerRefs = 'references' in refColumn - ? copyReferencesRecursively(columns, refColumn, newId) // if a column has references, copy them too + ? copyReferencesRecursively(layer, refColumn, sourceId, newId, indexPattern).columns // if a column has references, copy them too : { [newId]: refColumn }; - const newColumn = columns[columnId]; + const newColumn = columns[targetId]; let references = [newId]; if (newColumn && 'references' in newColumn) { references = newColumn.references; @@ -95,7 +108,7 @@ function copyReferencesRecursively( columns = { ...columns, ...refColumnWithInnerRefs, - [columnId]: { + [targetId]: { ...sourceColumn, references, }, @@ -104,10 +117,11 @@ function copyReferencesRecursively( } else { columns = { ...columns, - [columnId]: sourceColumn, + [targetId]: sourceColumn, }; } - return columns; + + return { ...layer, columns, columnOrder: getColumnOrder({ ...layer, columns }) }; } export function insertOrReplaceColumn(args: ColumnChange): IndexPatternLayer { @@ -141,12 +155,16 @@ export function insertNewColumn({ const baseOptions = { indexPattern, previousColumn: layer.columns[columnId] }; - if (operationDefinition.input === 'none') { + if (operationDefinition.input === 'none' || operationDefinition.input === 'managedReference') { if (field) { throw new Error(`Can't create operation ${op} with the provided field ${field.name}`); } + if (operationDefinition.input === 'managedReference') { + // TODO: need to create on the fly the new columns for Formula, + // like we do for fullReferences to show a seamless transition + } const possibleOperation = operationDefinition.getPossibleOperation(); - const isBucketed = Boolean(possibleOperation.isBucketed); + const isBucketed = Boolean(possibleOperation?.isBucketed); const addOperationFn = isBucketed ? addBucket : addMetric; return updateDefaultLabels( addOperationFn( @@ -395,6 +413,54 @@ export function replaceColumn({ } } + // TODO: Refactor all this to be more generic and know less about Formula + // if managed it has to look at the full picture to have a seamless transition + if (operationDefinition.input === 'managedReference') { + const newColumn = copyCustomLabel( + operationDefinition.buildColumn( + { ...baseOptions, layer: tempLayer }, + previousColumn.params, + operationDefinitionMap + ), + previousColumn + ) as FormulaIndexPatternColumn; + + // now remove the previous references + if (previousDefinition.input === 'fullReference') { + (previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => { + tempLayer = deleteColumn({ layer: tempLayer, columnId: id, indexPattern }); + }); + } + + const basicLayer = { ...tempLayer, columns: { ...tempLayer.columns, [columnId]: newColumn } }; + // rebuild the references again for the specific AST generated + let newLayer; + + try { + newLayer = newColumn.params.formula + ? regenerateLayerFromAst( + newColumn.params.formula, + basicLayer, + columnId, + newColumn, + indexPattern, + operationDefinitionMap + ).newLayer + : basicLayer; + } catch (e) { + newLayer = basicLayer; + } + + return updateDefaultLabels( + { + ...tempLayer, + columnOrder: getColumnOrder(newLayer), + columns: adjustColumnReferencesForChangedColumn(newLayer, columnId), + }, + indexPattern + ); + } + // This logic comes after the transitions because they need to look at previous columns if (previousDefinition.input === 'fullReference') { (previousColumn as ReferenceBasedIndexPatternColumn).references.forEach((id: string) => { @@ -976,8 +1042,12 @@ export function deleteColumn({ ); } -// Derives column order from column object, respects existing columnOrder -// when possible, but also allows new columns to be added to the order +// Column order mostly affects the visual order in the UI. It is derived +// from the columns objects, respecting any existing columnOrder relationships, +// but allowing new columns to be inserted +// +// This does NOT topologically sort references, as this would cause the order in the UI +// to change. Reference order is determined before creating the pipeline in to_expression export function getColumnOrder(layer: IndexPatternLayer): string[] { const entries = Object.entries(layer.columns); entries.sort(([idA], [idB]) => { @@ -992,16 +1062,6 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] { } }); - // If a reference has another reference as input, put it last in sort order - entries.sort(([idA, a], [idB, b]) => { - if ('references' in a && a.references.includes(idB)) { - return 1; - } - if ('references' in b && b.references.includes(idA)) { - return -1; - } - return 0; - }); const [aggregations, metrics] = _.partition(entries, ([, col]) => col.isBucketed); return aggregations.map(([id]) => id).concat(metrics.map(([id]) => id)); @@ -1019,8 +1079,22 @@ export function getExistingColumnGroups(layer: IndexPatternLayer): [string[], st /** * Returns true if the given column can be applied to the given index pattern */ -export function isColumnTransferable(column: IndexPatternColumn, newIndexPattern: IndexPattern) { - return operationDefinitionMap[column.operationType].isTransferable(column, newIndexPattern); +export function isColumnTransferable( + column: IndexPatternColumn, + newIndexPattern: IndexPattern, + layer: IndexPatternLayer +): boolean { + return ( + operationDefinitionMap[column.operationType].isTransferable( + column, + newIndexPattern, + operationDefinitionMap + ) && + (!('references' in column) || + column.references.every((columnId) => + isColumnTransferable(layer.columns[columnId], newIndexPattern, layer) + )) + ); } export function updateLayerIndexPattern( @@ -1028,15 +1102,7 @@ export function updateLayerIndexPattern( newIndexPattern: IndexPattern ): IndexPatternLayer { const keptColumns: IndexPatternLayer['columns'] = _.pickBy(layer.columns, (column) => { - if ('references' in column) { - return ( - isColumnTransferable(column, newIndexPattern) && - column.references.every((columnId) => - isColumnTransferable(layer.columns[columnId], newIndexPattern) - ) - ); - } - return isColumnTransferable(column, newIndexPattern); + return isColumnTransferable(column, newIndexPattern, layer); }); const newColumns: IndexPatternLayer['columns'] = _.mapValues(keptColumns, (column) => { const operationDefinition = operationDefinitionMap[column.operationType]; @@ -1069,7 +1135,7 @@ export function getErrorMessages( .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern); + return def.getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap); } }) // remove the undefined values @@ -1147,6 +1213,23 @@ export function resetIncomplete(layer: IndexPatternLayer, columnId: string): Ind return { ...layer, incompleteColumns }; } +// managedReferences have a relaxed policy about operation allowed, so let them pass +function maybeValidateOperations({ + column, + validation, +}: { + column: IndexPatternColumn; + validation: RequiredReference; +}) { + if (!validation.specificOperations) { + return true; + } + if (operationDefinitionMap[column.operationType].input === 'managedReference') { + return true; + } + return validation.specificOperations.includes(column.operationType); +} + export function isColumnValidAsReference({ column, validation, @@ -1159,7 +1242,29 @@ export function isColumnValidAsReference({ const operationDefinition = operationDefinitionMap[operationType]; return ( validation.input.includes(operationDefinition.input) && - (!validation.specificOperations || validation.specificOperations.includes(operationType)) && + maybeValidateOperations({ + column, + validation, + }) && validation.validateMetadata(column) ); } + +export function getManagedColumnsFrom( + columnId: string, + columns: Record +): Array<[string, IndexPatternColumn]> { + const allNodes: Record = {}; + Object.entries(columns).forEach(([id, col]) => { + allNodes[id] = 'references' in col ? [...col.references] : []; + }); + const queue: string[] = allNodes[columnId]; + const store: Array<[string, IndexPatternColumn]> = []; + + while (queue.length > 0) { + const nextId = queue.shift()!; + store.push([nextId, columns[nextId]]); + queue.push(...allNodes[nextId]); + } + return store.filter(([, column]) => column); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts index 429d881341e791..2d7e70179fb3f7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts @@ -8,7 +8,7 @@ import type { OperationMetadata } from '../../types'; import type { OperationType } from './definitions'; -export const createMockedReferenceOperation = () => { +export const createMockedFullReference = () => { return { input: 'fullReference', displayName: 'Reference test', @@ -40,3 +40,28 @@ export const createMockedReferenceOperation = () => { getErrorMessage: jest.fn(), }; }; + +export const createMockedManagedReference = () => { + return { + input: 'managedReference', + displayName: 'Managed reference test', + type: 'managedReference' as OperationType, + selectionStyle: 'full', + buildColumn: jest.fn((args) => { + return { + label: 'Test reference', + isBucketed: false, + dataType: 'number', + + operationType: 'testReference', + references: args.referenceIds, + }; + }), + filterable: true, + isTransferable: jest.fn(), + toExpression: jest.fn().mockReturnValue([]), + getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), + getDefaultLabel: jest.fn().mockReturnValue('Default label'), + getErrorMessage: jest.fn(), + }; +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 4c54b777b66f3b..7df096c27d9a0d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -354,6 +354,14 @@ describe('getOperationTypesForField', () => { "operationType": "last_value", "type": "field", }, + Object { + "operationType": "math", + "type": "managedReference", + }, + Object { + "operationType": "formula", + "type": "managedReference", + }, ], }, Object { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts index a45650f9323f99..437d2af005961f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts @@ -106,6 +106,10 @@ type OperationFieldTuple = | { type: 'fullReference'; operationType: OperationType; + } + | { + type: 'managedReference'; + operationType: OperationType; }; /** @@ -138,7 +142,11 @@ type OperationFieldTuple = * ] * ``` */ -export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { +export function getAvailableOperationsByMetadata( + indexPattern: IndexPattern, + // For consistency in testing + customOperationDefinitionMap?: Record +) { const operationByMetadata: Record< string, { operationMetaData: OperationMetadata; operations: OperationFieldTuple[] } @@ -161,36 +169,49 @@ export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { } }; - operationDefinitions.sort(getSortScoreByPriority).forEach((operationDefinition) => { - if (operationDefinition.input === 'field') { - indexPattern.fields.forEach((field) => { + (customOperationDefinitionMap + ? Object.values(customOperationDefinitionMap) + : operationDefinitions + ) + .sort(getSortScoreByPriority) + .forEach((operationDefinition) => { + if (operationDefinition.input === 'field') { + indexPattern.fields.forEach((field) => { + addToMap( + { + type: 'field', + operationType: operationDefinition.type, + field: field.name, + }, + operationDefinition.getPossibleOperationForField(field) + ); + }); + } else if (operationDefinition.input === 'none') { addToMap( { - type: 'field', + type: 'none', operationType: operationDefinition.type, - field: field.name, }, - operationDefinition.getPossibleOperationForField(field) - ); - }); - } else if (operationDefinition.input === 'none') { - addToMap( - { - type: 'none', - operationType: operationDefinition.type, - }, - operationDefinition.getPossibleOperation() - ); - } else if (operationDefinition.input === 'fullReference') { - const validOperation = operationDefinition.getPossibleOperation(indexPattern); - if (validOperation) { - addToMap( - { type: 'fullReference', operationType: operationDefinition.type }, - validOperation + operationDefinition.getPossibleOperation() ); + } else if (operationDefinition.input === 'fullReference') { + const validOperation = operationDefinition.getPossibleOperation(indexPattern); + if (validOperation) { + addToMap( + { type: 'fullReference', operationType: operationDefinition.type }, + validOperation + ); + } + } else if (operationDefinition.input === 'managedReference') { + const validOperation = operationDefinition.getPossibleOperation(); + if (validOperation) { + addToMap( + { type: 'managedReference', operationType: operationDefinition.type }, + validOperation + ); + } } - } - }); + }); return Object.values(operationByMetadata); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 4f596aa2825104..04e5ae9e488b5f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -60,22 +60,26 @@ function getExpressionForLayer( const [referenceEntries, esAggEntries] = partition( columnEntries, - ([, col]) => operationDefinitionMap[col.operationType]?.input === 'fullReference' + ([, col]) => + operationDefinitionMap[col.operationType]?.input === 'fullReference' || + operationDefinitionMap[col.operationType]?.input === 'managedReference' ); if (referenceEntries.length || esAggEntries.length) { const aggs: ExpressionAstExpressionBuilder[] = []; const expressions: ExpressionAstFunction[] = []; - referenceEntries.forEach(([colId, col]) => { + + sortedReferences(referenceEntries).forEach((colId) => { + const col = columns[colId]; const def = operationDefinitionMap[col.operationType]; - if (def.input === 'fullReference') { + if (def.input === 'fullReference' || def.input === 'managedReference') { expressions.push(...def.toExpression(layer, colId, indexPattern)); } }); esAggEntries.forEach(([colId, col]) => { const def = operationDefinitionMap[col.operationType]; - if (def.input !== 'fullReference') { + if (def.input !== 'fullReference' && def.input !== 'managedReference') { const wrapInFilter = Boolean(def.filterable && col.filter); let aggAst = def.toEsAggsFn( col, @@ -112,6 +116,14 @@ function getExpressionForLayer( } }); + const aggColumnEntries = columnEntries.filter(([, col]) => { + const def = operationDefinitionMap[col.operationType]; + return !(def.input === 'fullReference' || def.input === 'managedReference'); + }); + if (aggColumnEntries.length === 0) { + // Return early if there are no aggs, for example if the user has an empty formula + return null; + } const idMap = esAggEntries.reduce((currentIdMap, [colId, column], index) => { const esAggsId = `col-${index}-${colId}`; return { @@ -245,6 +257,33 @@ function getExpressionForLayer( return null; } +// Topologically sorts references so that we can execute them in sequence +function sortedReferences(columns: Array) { + const allNodes: Record = {}; + columns.forEach(([id, col]) => { + allNodes[id] = 'references' in col ? col.references : []; + }); + // remove real metric references + columns.forEach(([id]) => { + allNodes[id] = allNodes[id].filter((refId) => !!allNodes[refId]); + }); + const ordered: string[] = []; + + while (ordered.length < columns.length) { + Object.keys(allNodes).forEach((id) => { + if (allNodes[id].length === 0) { + ordered.push(id); + delete allNodes[id]; + Object.keys(allNodes).forEach((k) => { + allNodes[k] = allNodes[k].filter((i) => i !== id); + }); + } + }); + } + + return ordered; +} + export function toExpression( state: IndexPatternPrivateState, layerId: string, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 19c37da5bf2a93..23c7adb86d34fa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -62,7 +62,12 @@ export function isColumnInvalid( Boolean(getReferencesErrors(layer, column, indexPattern).filter(Boolean).length); return ( - !!operationDefinition.getErrorMessage?.(layer, columnId, indexPattern) || referencesHaveErrors + !!operationDefinition.getErrorMessage?.( + layer, + columnId, + indexPattern, + operationDefinitionMap + ) || referencesHaveErrors ); } @@ -74,7 +79,12 @@ function getReferencesErrors( return column.references?.map((referenceId: string) => { const referencedOperation = layer.columns[referenceId]?.operationType; const referencedDefinition = operationDefinitionMap[referencedOperation]; - return referencedDefinition?.getErrorMessage?.(layer, referenceId, indexPattern); + return referencedDefinition?.getErrorMessage?.( + layer, + referenceId, + indexPattern, + operationDefinitionMap + ); }); } diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts new file mode 100644 index 00000000000000..798cb7d3146f55 --- /dev/null +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const find = getService('find'); + const listingTable = getService('listingTable'); + + describe('lens formula', () => { + it('should transition from count to formula', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-dimensionTrigger', + operation: 'average', + field: 'bytes', + keepOpen: true, + }); + + await PageObjects.lens.switchToFormula(); + await PageObjects.header.waitUntilLoadingHasFinished(); + // .echLegendItem__title is the only viable way of getting the xy chart's + // legend item(s), so we're using a class selector here. + // 4th item is the other bucket + expect(await find.allByCssSelector('.echLegendItem')).to.have.length(3); + }); + + it('should update and delete a formula', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count(kql=`, + keepOpen: true, + }); + + const input = await find.activeElement(); + await input.type('*'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); + }); + + it('should duplicate a moving average formula and be a valid table', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `moving_average(sum(bytes), window=5`, + keepOpen: true, + }); + await PageObjects.lens.closeDimensionEditor(); + + await PageObjects.lens.dragDimensionToDimension( + 'lnsDatatable_metrics > lns-dimensionTrigger', + 'lnsDatatable_metrics > lns-empty-dimension' + ); + expect(await PageObjects.lens.getDatatableCellText(1, 1)).to.eql('222420'); + expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('222420'); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d379..080e44da6ffcdf 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -107,6 +107,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont isPreviousIncompatible?: boolean; keepOpen?: boolean; palette?: string; + formula?: string; }, layerIndex = 0 ) { @@ -114,10 +115,15 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click(`lns-layerPanel-${layerIndex} > ${opts.dimension}`); await testSubjects.exists(`lns-indexPatternDimension-${opts.operation}`); }); - const operationSelector = opts.isPreviousIncompatible - ? `lns-indexPatternDimension-${opts.operation} incompatible` - : `lns-indexPatternDimension-${opts.operation}`; - await testSubjects.click(operationSelector); + + if (opts.operation === 'formula') { + await this.switchToFormula(); + } else { + const operationSelector = opts.isPreviousIncompatible + ? `lns-indexPatternDimension-${opts.operation} incompatible` + : `lns-indexPatternDimension-${opts.operation}`; + await testSubjects.click(operationSelector); + } if (opts.field) { const target = await testSubjects.find('indexPattern-dimension-field'); @@ -125,6 +131,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await comboBox.setElement(target, opts.field); } + if (opts.formula) { + await this.typeFormula(opts.formula); + } + if (opts.palette) { await testSubjects.click('lns-palettePicker'); await find.clickByCssSelector(`#${opts.palette}`); @@ -907,5 +917,18 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); await PageObjects.header.waitUntilLoadingHasFinished(); }, + + async switchToFormula() { + await testSubjects.click('lens-dimensionTabs-formula'); + }, + + async typeFormula(formula: string) { + await find.byCssSelector('.monaco-editor'); + await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); + const input = await find.activeElement(); + await input.type(formula); + // Formula is applied on a 250ms timer, won't be applied if we leave too early + await PageObjects.common.sleep(500); + }, }); } From c210c4b44fd915e3b9695cb102604efe023be3a5 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 11 May 2021 11:45:31 -0400 Subject: [PATCH 096/185] Fix test failures --- .../lens/public/id_generator/id_generator.ts | 2 +- .../dimension_panel/reference_editor.tsx | 1 + .../operations/definitions/formula/types.ts | 25 ++++++ .../definitions/formula/validation.ts | 34 ++++---- .../operations/layer_helpers.test.ts | 5 +- .../operations/mocks.ts | 25 ------ x-pack/test/functional/apps/lens/formula.ts | 86 ------------------- .../test/functional/page_objects/lens_page.ts | 31 +------ 8 files changed, 49 insertions(+), 160 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts delete mode 100644 x-pack/test/functional/apps/lens/formula.ts diff --git a/x-pack/plugins/lens/public/id_generator/id_generator.ts b/x-pack/plugins/lens/public/id_generator/id_generator.ts index 988f2c880222a9..363b8035a23f74 100644 --- a/x-pack/plugins/lens/public/id_generator/id_generator.ts +++ b/x-pack/plugins/lens/public/id_generator/id_generator.ts @@ -8,5 +8,5 @@ import uuid from 'uuid/v4'; export function generateId() { - return 'c' + uuid().replaceAll(/-/g, ''); + return uuid(); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index b8a065b088467d..e02a014935458c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -92,6 +92,7 @@ export function ReferenceEditor(props: ReferenceEditorProps) { const operationByField: Partial>> = {}; const fieldByOperation: Partial>> = {}; Object.values(operationDefinitionMap) + .filter(({ hidden }) => !hidden) .sort((op1, op2) => { return op1.displayName.localeCompare(op2.displayName); }) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts new file mode 100644 index 00000000000000..ce853dec1d9513 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + TinymathAST, + TinymathFunction, + TinymathNamedArgument, + TinymathVariable, +} from 'packages/kbn-tinymath'; + +export type GroupedNodes = { + [Key in TinymathNamedArgument['type']]: TinymathNamedArgument[]; +} & + { + [Key in TinymathVariable['type']]: Array; + } & + { + [Key in TinymathFunction['type']]: TinymathFunction[]; + }; + +export type TinymathNodeTypes = Exclude; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 4e5ae21e576e45..f36b1f71f05ceb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -114,7 +114,7 @@ export const getQueryValidationError = ( function getMessageFromId({ messageId, - values, + values: { ...values }, locations, }: { messageId: K; @@ -127,85 +127,85 @@ function getMessageFromId({ message = i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { defaultMessage: 'The first argument for {operation} should be a {type} name. Found {argument}', - values, + values: { operation: values.operation, type: values.type, argument: values.argument }, }); break; case 'shouldNotHaveField': message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { defaultMessage: 'The operation {operation} does not accept any field as argument', - values, + values: { operation: values.operation }, }); break; case 'cannotAcceptParameter': message = i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { defaultMessage: 'The operation {operation} does not accept any parameter', - values, + values: { operation: values.operation }, }); break; case 'missingParameter': message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The operation {operation} in the Formula is missing the following parameters: {params}', - values, + values: { operation: values.operation, params: values.params }, }); break; case 'wrongTypeParameter': - message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionWrongType', { defaultMessage: 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', - values, + values: { operation: values.operation, params: values.params }, }); break; case 'duplicateArgument': message = i18n.translate('xpack.lens.indexPattern.formulaOperationDuplicateParams', { defaultMessage: 'The parameters for the operation {operation} have been declared multiple times: {params}', - values, + values: { operation: values.operation, params: values.params }, }); break; case 'missingField': - message = i18n.translate('xpack.lens.indexPattern.fieldNotFound', { + message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { defaultMessage: '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values, + values: { variablesLength: values.variablesLength, variablesList: values.variablesList }, }); break; case 'missingOperation': message = i18n.translate('xpack.lens.indexPattern.operationsNotFound', { defaultMessage: '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', - values, + values: { operationLength: values.operationLength, operationsList: values.operationsList }, }); break; case 'fieldWithNoOperation': message = i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { defaultMessage: 'The field {field} cannot be used without operation', - values, + values: { field: values.field }, }); break; case 'failedParsing': - message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { + message = i18n.translate('xpack.lens.indexPattern.formulaExpressionParseError', { defaultMessage: 'The Formula {expression} cannot be parsed', - values, + values: { expression: values.expression }, }); break; case 'tooManyArguments': message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', { defaultMessage: 'The operation {operation} has too many arguments', - values, + values: { operation: values.operation }, }); break; case 'missingMathArgument': message = i18n.translate('xpack.lens.indexPattern.formulaMathMissingArgument', { defaultMessage: 'The operation {operation} in the Formula is missing {count} arguments: {params}', - values, + values: { operation: values.operation, count: params.count, params: values.params }, }); break; // case 'mathRequiresFunction': // message = i18n.translate('xpack.lens.indexPattern.formulaMathRequiresFunctionLabel', { // defaultMessage; 'The function {name} requires an Elasticsearch function', - // values, + // values: { ...values }, // }); // break; default: diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 52cc8a07511a47..80717684686881 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -24,7 +24,7 @@ import type { IndexPattern, IndexPatternLayer } from '../types'; import { documentField } from '../document_field'; import { getFieldByNameFactory } from '../pure_helpers'; import { generateId } from '../../id_generator'; -import { createMockedFullReference, createMockedManagedReference } from './mocks'; +import { createMockedFullReference } from './mocks'; jest.mock('../operations'); jest.mock('../../id_generator'); @@ -91,13 +91,10 @@ describe('state_helpers', () => { // @ts-expect-error we are inserting an invalid type operationDefinitionMap.testReference = createMockedFullReference(); - // @ts-expect-error we are inserting an invalid type - operationDefinitionMap.managedReference = createMockedManagedReference(); }); afterEach(() => { delete operationDefinitionMap.testReference; - delete operationDefinitionMap.managedReference; }); describe('copyColumn', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts index 2d7e70179fb3f7..4a2e065269063a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts @@ -40,28 +40,3 @@ export const createMockedFullReference = () => { getErrorMessage: jest.fn(), }; }; - -export const createMockedManagedReference = () => { - return { - input: 'managedReference', - displayName: 'Managed reference test', - type: 'managedReference' as OperationType, - selectionStyle: 'full', - buildColumn: jest.fn((args) => { - return { - label: 'Test reference', - isBucketed: false, - dataType: 'number', - - operationType: 'testReference', - references: args.referenceIds, - }; - }), - filterable: true, - isTransferable: jest.fn(), - toExpression: jest.fn().mockReturnValue([]), - getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), - getDefaultLabel: jest.fn().mockReturnValue('Default label'), - getErrorMessage: jest.fn(), - }; -}; diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts deleted file mode 100644 index 798cb7d3146f55..00000000000000 --- a/x-pack/test/functional/apps/lens/formula.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../ftr_provider_context'; - -export default function ({ getService, getPageObjects }: FtrProviderContext) { - const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); - const find = getService('find'); - const listingTable = getService('listingTable'); - - describe('lens formula', () => { - it('should transition from count to formula', async () => { - await PageObjects.visualize.gotoVisualizationLandingPage(); - await listingTable.searchForItemWithName('lnsXYvis'); - await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); - await PageObjects.lens.goToTimeRange(); - - await PageObjects.lens.configureDimension({ - dimension: 'lnsXY_yDimensionPanel > lns-dimensionTrigger', - operation: 'average', - field: 'bytes', - keepOpen: true, - }); - - await PageObjects.lens.switchToFormula(); - await PageObjects.header.waitUntilLoadingHasFinished(); - // .echLegendItem__title is the only viable way of getting the xy chart's - // legend item(s), so we're using a class selector here. - // 4th item is the other bucket - expect(await find.allByCssSelector('.echLegendItem')).to.have.length(3); - }); - - it('should update and delete a formula', async () => { - await PageObjects.visualize.navigateToNewVisualization(); - await PageObjects.visualize.clickVisType('lens'); - await PageObjects.lens.goToTimeRange(); - await PageObjects.lens.switchToVisualization('lnsDatatable'); - - await PageObjects.lens.configureDimension({ - dimension: 'lnsDatatable_metrics > lns-empty-dimension', - operation: 'formula', - formula: `count(kql=`, - keepOpen: true, - }); - - const input = await find.activeElement(); - await input.type('*'); - - await PageObjects.header.waitUntilLoadingHasFinished(); - expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); - }); - - it('should duplicate a moving average formula and be a valid table', async () => { - await PageObjects.visualize.navigateToNewVisualization(); - await PageObjects.visualize.clickVisType('lens'); - await PageObjects.lens.goToTimeRange(); - await PageObjects.lens.switchToVisualization('lnsDatatable'); - - await PageObjects.lens.configureDimension({ - dimension: 'lnsDatatable_rows > lns-empty-dimension', - operation: 'date_histogram', - field: '@timestamp', - }); - - await PageObjects.lens.configureDimension({ - dimension: 'lnsDatatable_metrics > lns-empty-dimension', - operation: 'formula', - formula: `moving_average(sum(bytes), window=5`, - keepOpen: true, - }); - await PageObjects.lens.closeDimensionEditor(); - - await PageObjects.lens.dragDimensionToDimension( - 'lnsDatatable_metrics > lns-dimensionTrigger', - 'lnsDatatable_metrics > lns-empty-dimension' - ); - expect(await PageObjects.lens.getDatatableCellText(1, 1)).to.eql('222420'); - expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('222420'); - }); - }); -} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 080e44da6ffcdf..100ed8e079d379 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -107,7 +107,6 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont isPreviousIncompatible?: boolean; keepOpen?: boolean; palette?: string; - formula?: string; }, layerIndex = 0 ) { @@ -115,15 +114,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click(`lns-layerPanel-${layerIndex} > ${opts.dimension}`); await testSubjects.exists(`lns-indexPatternDimension-${opts.operation}`); }); - - if (opts.operation === 'formula') { - await this.switchToFormula(); - } else { - const operationSelector = opts.isPreviousIncompatible - ? `lns-indexPatternDimension-${opts.operation} incompatible` - : `lns-indexPatternDimension-${opts.operation}`; - await testSubjects.click(operationSelector); - } + const operationSelector = opts.isPreviousIncompatible + ? `lns-indexPatternDimension-${opts.operation} incompatible` + : `lns-indexPatternDimension-${opts.operation}`; + await testSubjects.click(operationSelector); if (opts.field) { const target = await testSubjects.find('indexPattern-dimension-field'); @@ -131,10 +125,6 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await comboBox.setElement(target, opts.field); } - if (opts.formula) { - await this.typeFormula(opts.formula); - } - if (opts.palette) { await testSubjects.click('lns-palettePicker'); await find.clickByCssSelector(`#${opts.palette}`); @@ -917,18 +907,5 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); await PageObjects.header.waitUntilLoadingHasFinished(); }, - - async switchToFormula() { - await testSubjects.click('lens-dimensionTabs-formula'); - }, - - async typeFormula(formula: string) { - await find.byCssSelector('.monaco-editor'); - await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); - const input = await find.activeElement(); - await input.type(formula); - // Formula is applied on a 250ms timer, won't be applied if we leave too early - await PageObjects.common.sleep(500); - }, }); } From e8ef7e8a18254ee9806fd1dd481bd03ab4ec6370 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 11 May 2021 13:21:43 -0400 Subject: [PATCH 097/185] Fix i18n types --- .../definitions/formula/validation.ts | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index f36b1f71f05ceb..579b2822f49112 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -122,84 +122,86 @@ function getMessageFromId({ locations: TinymathLocation[]; }): ErrorWrapper { let message: string; + // Use a less strict type instead of doing a typecast on each message type + const out = (values as unknown) as Record; switch (messageId) { case 'wrongFirstArgument': message = i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { defaultMessage: 'The first argument for {operation} should be a {type} name. Found {argument}', - values: { operation: values.operation, type: values.type, argument: values.argument }, + values: { operation: out.operation, type: out.type, argument: out.argument }, }); break; case 'shouldNotHaveField': message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotRequired', { defaultMessage: 'The operation {operation} does not accept any field as argument', - values: { operation: values.operation }, + values: { operation: out.operation }, }); break; case 'cannotAcceptParameter': message = i18n.translate('xpack.lens.indexPattern.formulaParameterNotRequired', { defaultMessage: 'The operation {operation} does not accept any parameter', - values: { operation: values.operation }, + values: { operation: out.operation }, }); break; case 'missingParameter': message = i18n.translate('xpack.lens.indexPattern.formulaExpressionNotHandled', { defaultMessage: 'The operation {operation} in the Formula is missing the following parameters: {params}', - values: { operation: values.operation, params: values.params }, + values: { operation: out.operation, params: out.params }, }); break; case 'wrongTypeParameter': message = i18n.translate('xpack.lens.indexPattern.formulaExpressionWrongType', { defaultMessage: 'The parameters for the operation {operation} in the Formula are of the wrong type: {params}', - values: { operation: values.operation, params: values.params }, + values: { operation: out.operation, params: out.params }, }); break; case 'duplicateArgument': message = i18n.translate('xpack.lens.indexPattern.formulaOperationDuplicateParams', { defaultMessage: 'The parameters for the operation {operation} have been declared multiple times: {params}', - values: { operation: values.operation, params: values.params }, + values: { operation: out.operation, params: out.params }, }); break; case 'missingField': message = i18n.translate('xpack.lens.indexPattern.formulaFieldNotFound', { defaultMessage: '{variablesLength, plural, one {Field} other {Fields}} {variablesList} not found', - values: { variablesLength: values.variablesLength, variablesList: values.variablesList }, + values: { variablesLength: out.variablesLength, variablesList: out.variablesList }, }); break; case 'missingOperation': message = i18n.translate('xpack.lens.indexPattern.operationsNotFound', { defaultMessage: '{operationLength, plural, one {Operation} other {Operations}} {operationsList} not found', - values: { operationLength: values.operationLength, operationsList: values.operationsList }, + values: { operationLength: out.operationLength, operationsList: out.operationsList }, }); break; case 'fieldWithNoOperation': message = i18n.translate('xpack.lens.indexPattern.fieldNoOperation', { defaultMessage: 'The field {field} cannot be used without operation', - values: { field: values.field }, + values: { field: out.field }, }); break; case 'failedParsing': message = i18n.translate('xpack.lens.indexPattern.formulaExpressionParseError', { defaultMessage: 'The Formula {expression} cannot be parsed', - values: { expression: values.expression }, + values: { expression: out.expression }, }); break; case 'tooManyArguments': message = i18n.translate('xpack.lens.indexPattern.formulaWithTooManyArguments', { defaultMessage: 'The operation {operation} has too many arguments', - values: { operation: values.operation }, + values: { operation: out.operation }, }); break; case 'missingMathArgument': message = i18n.translate('xpack.lens.indexPattern.formulaMathMissingArgument', { defaultMessage: 'The operation {operation} in the Formula is missing {count} arguments: {params}', - values: { operation: values.operation, count: params.count, params: values.params }, + values: { operation: out.operation, count: out.count, params: out.params }, }); break; // case 'mathRequiresFunction': From 16387e88cc9b9fc5ddda55260140410b460154c5 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Tue, 11 May 2021 13:51:33 -0400 Subject: [PATCH 098/185] fix fullscreen flex issues --- .../definitions/formula/editor/formula.scss | 32 +++++++ .../formula/editor/formula_editor.tsx | 14 +-- .../formula/editor/formula_help.tsx | 85 ++++++++++--------- 3 files changed, 85 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index a4000718b3b2b2..c602b24e0b355f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -31,6 +31,7 @@ .lnsFormula__editorContent { flex: 1; max-height: 200px; + min-height: 0; .lnsIndexPatternDimensionEditor-isFullscreen & { max-height: none; @@ -41,6 +42,37 @@ background: $euiColorEmptyShade; } +.lnsFormula__docs--inline { + display: flex; + flex-direction: column; +} + +.lnsFormula__docsContent { + max-height: 40vh; + max-width: 60vh; + + .lnsFormula__docs--inline & { + flex: 1; + max-height: none; + max-width: none; + min-height: 0; + } + + & > * + * { + border-left: $euiBorderThin; + } +} + +.lnsFormula__docsNav { + background: $euiColorLightestShade; + padding: $euiSizeS; +} + +.lnsFormula__docsText { + @include euiYScroll; + padding: $euiSize; +} + .lnsFormulaOverflow { // Needs to be higher than the modal and all flyouts z-index: $euiZLevel9 + 1; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 344092eebaec3c..f3e4ee03000a67 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -453,7 +453,7 @@ export function FormulaEditor({
- {/* TODO: Word wrap button */} + Word wrap button {isFullscreen ? ( -

Accordion button here

+ <>Docs toggle ) : ( setIsHelpOpen(false)} button={ @@ -533,7 +534,6 @@ export function FormulaEditor({ )} /> } - anchorPosition="leftDown" > - {/* TODO: Errors go here */} + Error count
{isFullscreen ? ( -
+
- - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); - } else { - setSelectedFunction(chosenType.label); - } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - {selectedFunction ? ( - helpItems.find(({ label }) => label === selectedFunction)?.description - ) : ( - + Formula reference + + + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + - )} - - - + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + /> + )} + + + + ); } From 911425630276fbf4b8565f3ffdfc72a1c713dcdc Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 11 May 2021 14:17:15 -0400 Subject: [PATCH 099/185] Delete managedReference when replacing --- .../dimension_panel/dimension_panel.test.tsx | 9 +++ .../dimension_panel/reference_editor.test.tsx | 25 ++++++ .../operations/layer_helpers.test.ts | 81 ++++++++++++++++++- .../operations/layer_helpers.ts | 13 +++ .../indexpattern_datasource/to_expression.ts | 6 +- 5 files changed, 127 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 5273162034f160..333caf259fe2f7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -388,6 +388,15 @@ describe('IndexPatternDimensionEditorPanel', () => { ); }); + it('should not display hidden operation types', () => { + wrapper = mount(); + + const items: EuiListGroupItemProps[] = wrapper.find(EuiListGroup).prop('listItems') || []; + + expect(items.find(({ id }) => id === 'math')).toBeUndefined(); + expect(items.find(({ id }) => id === 'formula')).toBeUndefined(); + }); + it('should indicate that reference-based operations are not compatible when they are incomplete', () => { wrapper = mount( { ); }); + it('should not display hidden sub-function types', () => { + // This may happen for saved objects after changing the type of a field + wrapper = mount( + true, + }} + /> + ); + + const subFunctionSelect = wrapper + .find('[data-test-subj="indexPattern-reference-function"]') + .first(); + + expect(subFunctionSelect.prop('isInvalid')).toEqual(true); + expect(subFunctionSelect.prop('selectedOptions')).not.toEqual( + expect.arrayContaining([expect.objectContaining({ value: 'math' })]) + ); + expect(subFunctionSelect.prop('selectedOptions')).not.toEqual( + expect.arrayContaining([expect.objectContaining({ value: 'formula' })]) + ); + }); + it('should hide the function selector when using a field-only selection style', () => { wrapper = mount( { formula: 'moving_average(sum(bytes), window=5)', isFormulaBroken: false, }, - references: ['formulaX1'], + references: ['formulaX3'], }; const math = { customLabel: true, @@ -135,7 +135,7 @@ describe('state_helpers', () => { label: 'formulaX2', operationType: 'moving_average' as const, params: { window: 5 }, - references: ['formulaX0'], + references: ['formulaX1'], }; expect( copyColumn({ @@ -1540,6 +1540,83 @@ describe('state_helpers', () => { ); }); + it('should transition from managedReference to fullReference by deleting the managedReference', () => { + const math = { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'math', + operationType: 'math' as const, + }; + const layer: IndexPatternLayer = { + indexPatternId: '', + columnOrder: [], + columns: { + source: { + dataType: 'number' as const, + isBucketed: false, + label: 'Formula', + operationType: 'formula' as const, + params: { + formula: 'moving_average(sum(bytes), window=5)', + isFormulaBroken: false, + }, + references: ['formulaX3'], + }, + formulaX0: { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX0', + operationType: 'sum' as const, + scale: 'ratio' as const, + sourceField: 'bytes', + }, + formulaX1: { + ...math, + label: 'formulaX1', + references: ['formulaX0'], + params: { tinymathAst: 'formulaX0' }, + }, + formulaX2: { + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + label: 'formulaX2', + operationType: 'moving_average' as const, + params: { window: 5 }, + references: ['formulaX1'], + }, + formulaX3: { + ...math, + label: 'formulaX3', + references: ['formulaX2'], + params: { tinymathAst: 'formulaX2' }, + }, + }, + }; + + expect( + replaceColumn({ + layer, + indexPattern, + columnId: 'source', + // @ts-expect-error not statically available + op: 'secondTest', + }) + ).toEqual( + expect.objectContaining({ + columnOrder: ['source'], + columns: { + source: expect.objectContaining({ + operationType: 'secondTest', + references: ['id1'], + }), + }, + }) + ); + }); + it('should transition by using the field from the previous reference if nothing else works (case new5)', () => { const layer: IndexPatternLayer = { indexPatternId: '1', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 45345572a8f7a1..1eecbac1b4cbce 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -351,6 +351,19 @@ export function replaceColumn({ tempLayer = resetIncomplete(tempLayer, columnId); + if (previousDefinition.input === 'managedReference') { + // Every transition away from a managedReference resets it, we don't have a way to keep the state + tempLayer = deleteColumn({ layer: tempLayer, columnId, indexPattern }); + return insertNewColumn({ + layer: tempLayer, + columnId, + indexPattern, + op, + field, + visualizationGroups, + }); + } + if (operationDefinition.input === 'fullReference') { return applyReferenceTransition({ layer: tempLayer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 04e5ae9e488b5f..4905bd75d64987 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -116,11 +116,7 @@ function getExpressionForLayer( } }); - const aggColumnEntries = columnEntries.filter(([, col]) => { - const def = operationDefinitionMap[col.operationType]; - return !(def.input === 'fullReference' || def.input === 'managedReference'); - }); - if (aggColumnEntries.length === 0) { + if (esAggEntries.length === 0) { // Return early if there are no aggs, for example if the user has an empty formula return null; } From 4ea324d3fdc3166109802b97ab5cdf3dcb961bc3 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Tue, 11 May 2021 15:49:59 -0400 Subject: [PATCH 100/185] refactor css and markup; add button placeholders --- .../config_panel/dimension_container.scss | 1 + .../definitions/formula/editor/formula.scss | 42 +++++++++----- .../formula/editor/formula_editor.tsx | 58 ++++++++++++++----- .../formula/editor/formula_help.tsx | 4 +- 4 files changed, 74 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss index f719cb96aa97c4..e6eee76832e211 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.scss @@ -19,6 +19,7 @@ } .lnsFrameLayout__sidebar-isFullscreen & { + border-left: $euiBorderThin; // Force border regardless of theme in fullscreen box-shadow: none; } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index c602b24e0b355f..4e22cf2fd380e4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -5,18 +5,20 @@ .lnsIndexPatternDimensionEditor-isFullscreen & { height: 100%; } -} -.lnsFormula__editor, -.lnsFormula__docs { - flex: 1; - min-height: 0; + & > * { + flex: 1; + min-height: 0; + } } .lnsFormula__editor { border-bottom: $euiBorderThin; - display: flex; - flex-direction: column; + + .lnsIndexPatternDimensionEditor-isFullscreen & { + display: flex; + flex-direction: column; + } & > * + * { border-top: $euiBorderThin; @@ -28,13 +30,23 @@ padding: $euiSizeS; } -.lnsFormula__editorContent { +.lnsFormula__editorHeaderGroup, +.lnsFormula__editorFooterGroup { + display: block; // Overrides EUI's styling of `display: flex` on `EuiFlexItem` components +} + +.lnsIndexPatternDimensionEditor-isFullscreen .lnsFormula__editorContent { flex: 1; - max-height: 200px; min-height: 0; +} - .lnsIndexPatternDimensionEditor-isFullscreen & { - max-height: none; +.lnsFormula__editorHelp--inline { + align-items: center; + display: flex; + padding: $euiSizeXS; + + & > * + * { + margin-left: $euiSizeXS; } } @@ -48,13 +60,13 @@ } .lnsFormula__docsContent { - max-height: 40vh; - max-width: 60vh; + .lnsFormula__docs--overlay & { + height: 40vh; + width: 65vh; + } .lnsFormula__docs--inline & { flex: 1; - max-height: none; - max-width: none; min-height: 0; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index f3e4ee03000a67..e8d212a32f7445 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -7,7 +7,15 @@ import React, { useCallback, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiButtonIcon, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiPopover } from '@elastic/eui'; +import { + EuiButtonIcon, + EuiButtonEmpty, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiLink, + EuiPopover, +} from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; @@ -453,23 +461,33 @@ export function FormulaEditor({
- Word wrap button + + {/* TODO: Replace `bolt` with `wordWrap` icon (after latest EUI is deployed) and hook up button to enable/disable word wrapping. */} + + - + + {/* TODO: Replace `bolt` with `fullScreenExit` icon (after latest EUI is deployed). */} { toggleFullscreen(); }} - iconType="fullScreen" + iconType={isFullscreen ? 'bolt' : 'fullScreen'} size="xs" color="text" flush="right" > {isFullscreen - ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + ? i18n.translate('xpack.lens.formula.fullScreenExitLabel', { defaultMessage: 'Collapse', }) - : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + : i18n.translate('xpack.lens.formula.fullScreenEnterLabel', { defaultMessage: 'Expand', })} @@ -511,9 +529,19 @@ export function FormulaEditor({
- + {isFullscreen ? ( - <>Docs toggle + // TODO: Hook up the below `EuiLink` button so that it toggles the presence of the `.lnsFormula__docs--inline` element in fullscreen mode. Note that when docs are hidden, the `arrowDown` button should change to `arrowUp`. + + + + ) : ( setIsHelpOpen(false)} button={ setIsHelpOpen(!isHelpOpen)} iconType="help" color="text" - aria-label={i18n.translate( - 'xpack.lens.formula.functionReferenceEditorLabel', - { - defaultMessage: 'Function reference', - } - )} + aria-label={i18n.translate('xpack.lens.formula.editorHelpOverlayLabel', { + defaultMessage: 'Function reference', + })} /> } > @@ -543,7 +569,9 @@ export function FormulaEditor({ )} - Error count + + Error count +
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 502671663a3611..ef08b6d03a7b79 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -64,7 +64,9 @@ function FormulaHelp({ return ( <> - Formula reference + + Formula reference + From 394555c5728bb93ef231fa50fbd0961ee91ef666 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 11 May 2021 16:33:23 -0400 Subject: [PATCH 101/185] [Lens] Formulas --- .../expression_functions/specs/map_column.ts | 9 +- .../common/expression_functions/specs/math.ts | 3 +- .../specs/tests/map_column.test.ts | 27 +- .../public/code_editor/code_editor.tsx | 6 +- .../kibana_react/public/code_editor/index.tsx | 4 +- .../config_panel/config_panel.test.tsx | 2 + .../config_panel/config_panel.tsx | 12 +- .../config_panel/dimension_container.tsx | 147 +-- .../config_panel/layer_panel.test.tsx | 2 + .../editor_frame/config_panel/layer_panel.tsx | 21 +- .../editor_frame/config_panel/types.ts | 2 + .../editor_frame/editor_frame.tsx | 3 + .../editor_frame/frame_layout.scss | 9 + .../editor_frame/frame_layout.tsx | 39 +- .../editor_frame/state_management.ts | 6 + .../workspace_panel/workspace_panel.test.tsx | 2 + .../workspace_panel/workspace_panel.tsx | 21 +- .../workspace_panel_wrapper.scss | 4 + .../workspace_panel_wrapper.test.tsx | 2 + .../workspace_panel_wrapper.tsx | 92 +- .../dimension_panel/dimension_editor.scss | 16 + .../dimension_panel/dimension_editor.tsx | 138 ++- .../dimension_panel/dimension_panel.test.tsx | 2 + .../droppable/droppable.test.ts | 2 + .../dimension_panel/reference_editor.test.tsx | 3 + .../dimension_panel/reference_editor.tsx | 9 + .../indexpattern_datasource/indexpattern.tsx | 5 + .../definitions/date_histogram.test.tsx | 3 + .../definitions/filters/filters.test.tsx | 3 + .../definitions/formula/editor/formula.scss | 4 + .../formula/editor/formula_editor.tsx | 548 ++++++++++ .../formula/editor/formula_help.tsx | 175 +++ .../definitions/formula/editor/index.ts | 8 + .../formula/editor/math_completion.test.ts | 344 ++++++ .../formula/editor/math_completion.ts | 569 ++++++++++ .../formula/editor/math_tokenization.tsx | 66 ++ .../definitions/formula/formula.test.tsx | 997 ++++++++++++++++++ .../definitions/formula/formula.tsx | 3 + .../definitions/formula/math_examples.md | 28 + .../definitions/formula/validation.ts | 2 +- .../operations/definitions/index.ts | 3 + .../definitions/last_value.test.tsx | 3 + .../definitions/percentile.test.tsx | 3 + .../definitions/ranges/ranges.test.tsx | 3 + .../definitions/terms/terms.test.tsx | 3 + .../operations/layer_helpers.test.ts | 9 +- .../operations/mocks.ts | 25 + .../public/indexpattern_datasource/types.ts | 2 + x-pack/plugins/lens/public/types.ts | 7 + x-pack/test/functional/apps/lens/formula.ts | 86 ++ x-pack/test/functional/apps/lens/index.ts | 1 + .../test/functional/page_objects/lens_page.ts | 31 +- 52 files changed, 3344 insertions(+), 170 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md create mode 100644 x-pack/test/functional/apps/lens/formula.ts diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index c570206670dde5..dc19c81a99c1f6 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -101,7 +101,14 @@ export const mapColumn: ExpressionFunctionDefinition< }); return Promise.all(rowPromises).then((rows) => { - const existingColumnIndex = columns.findIndex(({ name }) => name === args.name); + const existingColumnIndex = columns.findIndex(({ id, name }) => { + // Columns that have IDs are allowed to have duplicate names, for example esaggs + if (args.id && id) { + return id === args.id && name === args.name; + } + // If no ID, name is the unique key. For example, SQL output does not have IDs + return name === args.name; + }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; const newColumn = { id: columnId, diff --git a/src/plugins/expressions/common/expression_functions/specs/math.ts b/src/plugins/expressions/common/expression_functions/specs/math.ts index a70c032769b570..b91600fea8b56e 100644 --- a/src/plugins/expressions/common/expression_functions/specs/math.ts +++ b/src/plugins/expressions/common/expression_functions/specs/math.ts @@ -130,10 +130,11 @@ export const math: ExpressionFunctionDefinition< throw errors.emptyExpression(); } + // Use unique ID if available, otherwise fall back to names const mathContext = isDatatable(input) ? pivotObjectArray( input.rows, - input.columns.map((col) => col.name) + input.columns.map((col) => col.id ?? col.name) ) : { value: input }; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index b2966b010b4790..ec8fc87a76c116 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -35,7 +35,7 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); - it('overwrites existing column with the new column if an existing column name is provided', async () => { + it('overwrites existing column with the new column if an existing column name is provided without an id', async () => { const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); const arbitraryRowIndex = 4; @@ -47,6 +47,19 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); }); + it('inserts a new column with a duplicate name if an id and name are provided', async () => { + const result = await runFn(testTable, { id: 'new', name: 'name', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length + 1); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); + }); + it('adds a column to empty tables', async () => { const result = await runFn(emptyTable, { name: 'name', expression: pricePlusTwo }); @@ -66,18 +79,6 @@ describe('mapColumn', () => { expect(result.columns[0].meta).toHaveProperty('type', 'null'); }); - it('should assign specific id, different from name, when id arg is passed for copied column', async () => { - const result = await runFn(testTable, { name: 'name', id: 'myid', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - - expect(result.type).toBe('datatable'); - expect(result.columns[nameColumnIndex]).toEqual({ - id: 'myid', - name: 'name', - meta: { type: 'number' }, - }); - }); - it('should copy over the meta information from the specified column', async () => { const result = await runFn( { diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index 51344e2d28ab67..55e10e7861e518 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -34,14 +34,14 @@ export interface Props { value: string; /** Function invoked when text in editor is changed */ - onChange: (value: string) => void; + onChange: (value: string, event: monaco.editor.IModelContentChangedEvent) => void; /** * Options for the Monaco Code Editor * Documentation of options can be found here: - * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.ieditorconstructionoptions.html + * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.istandaloneeditorconstructionoptions.html */ - options?: monaco.editor.IEditorConstructionOptions; + options?: monaco.editor.IStandaloneEditorConstructionOptions; /** * Suggestion provider for autocompletion diff --git a/src/plugins/kibana_react/public/code_editor/index.tsx b/src/plugins/kibana_react/public/code_editor/index.tsx index 635e84b1d8c202..2440974c3b1d1e 100644 --- a/src/plugins/kibana_react/public/code_editor/index.tsx +++ b/src/plugins/kibana_react/public/code_editor/index.tsx @@ -16,7 +16,7 @@ import { import darkTheme from '@elastic/eui/dist/eui_theme_dark.json'; import lightTheme from '@elastic/eui/dist/eui_theme_light.json'; import { useUiSetting } from '../ui_settings'; -import type { Props } from './code_editor'; +import { Props } from './code_editor'; const LazyBaseEditor = React.lazy(() => import('./code_editor')); @@ -26,6 +26,8 @@ const Fallback = () => ( ); +export type CodeEditorProps = Props; + /** * Renders a Monaco code editor with EUI color theme. * diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx index e171c457c541eb..0ef509377c8b53 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx @@ -76,6 +76,8 @@ describe('ConfigPanel', () => { framePublicAPI: frame, dispatch: jest.fn(), core: coreMock.createStart(), + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index d52fd29e7233a4..79c7882a8d56e3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -63,7 +63,8 @@ export function LayerPanels( () => (datasourceId: string, newState: unknown) => { dispatch({ type: 'UPDATE_DATASOURCE_STATE', - updater: () => newState, + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, datasourceId, clearStagedPreview: false, }); @@ -96,6 +97,14 @@ export function LayerPanels( }, [dispatch] ); + const toggleFullscreen = useMemo( + () => () => { + dispatch({ + type: 'TOGGLE_FULLSCREEN', + }); + }, + [dispatch] + ); const datasourcePublicAPIs = props.framePublicAPI.datasourceLayers; @@ -130,6 +139,7 @@ export function LayerPanels( }); removeLayerRef(layerId); }} + toggleFullscreen={toggleFullscreen} /> ) : null )} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx index b8d3170b3e1650..517321218e3444 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/dimension_container.tsx @@ -29,17 +29,22 @@ export function DimensionContainer({ groupLabel, handleClose, panel, + isFullscreen, }: { isOpen: boolean; - handleClose: () => void; - panel: React.ReactElement; + handleClose: () => boolean; + panel: React.ReactElement | null; groupLabel: string; + isFullscreen: boolean; }) { const [focusTrapIsEnabled, setFocusTrapIsEnabled] = useState(false); const closeFlyout = useCallback(() => { - handleClose(); - setFocusTrapIsEnabled(false); + const canClose = handleClose(); + if (canClose) { + setFocusTrapIsEnabled(false); + } + return canClose; }, [handleClose]); useEffect(() => { @@ -54,8 +59,10 @@ export function DimensionContainer({ const closeOnEscape = useCallback( (event: KeyboardEvent) => { if (event.key === keys.ESCAPE) { - event.preventDefault(); - closeFlyout(); + const canClose = closeFlyout(); + if (canClose) { + event.preventDefault(); + } } }, [closeFlyout] @@ -75,62 +82,80 @@ export function DimensionContainer({ return isOpen ? ( - -
- - - - - - - -

- - {i18n.translate('xpack.lens.configure.configurePanelTitle', { - defaultMessage: '{groupLabel} configuration', - values: { - groupLabel, - }, - })} - -

- - - - - + { + if (isFullscreen) { + return; + } + closeFlyout(); + }} + isDisabled={!isOpen} + > + {isFullscreen ? ( +
{panel} - - - - {i18n.translate('xpack.lens.dimensionContainer.close', { - defaultMessage: 'Close', - })} - - -
+
+ ) : ( +
+ + + + + + + +

+ + {i18n.translate('xpack.lens.configure.configurePanelTitle', { + defaultMessage: '{groupLabel} configuration', + values: { + groupLabel, + }, + })} + +

+ + + + + + {panel} + + + + {i18n.translate('xpack.lens.dimensionContainer.close', { + defaultMessage: 'Close', + })} + + +
+ )} ) : null; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index 7ee7a27a53c7da..0aa23d8ad947f5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -86,6 +86,8 @@ describe('LayerPanel', () => { core: coreMock.createStart(), layerIndex: 0, registerNewLayerRef: jest.fn(), + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index cf3c9099d4b0dd..7e06bd2ab110cb 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -49,6 +49,8 @@ export function LayerPanel( ) => void; onRemoveLayer: () => void; registerNewLayerRef: (layerId: string, instance: HTMLDivElement | null) => void; + toggleFullscreen: () => void; + isFullscreen: boolean; } ) { const [activeDimension, setActiveDimension] = useState( @@ -65,6 +67,8 @@ export function LayerPanel( activeVisualization, updateVisualization, updateDatasource, + toggleFullscreen, + isFullscreen, } = props; const datasourcePublicAPI = framePublicAPI.datasourceLayers[layerId]; @@ -406,8 +410,14 @@ export function LayerPanel( { + if (layerDatasource.canCloseDimensionEditor) { + if (!layerDatasource.canCloseDimensionEditor(layerDatasourceState)) { + return false; + } + } if (layerDatasource.updateStateOnCloseDimension) { const newState = layerDatasource.updateStateOnCloseDimension({ state: layerDatasourceState, @@ -419,9 +429,13 @@ export function LayerPanel( } } setActiveDimension(initialActiveDimensionState); + if (isFullscreen) { + toggleFullscreen(); + } + return true; }} panel={ - <> +
{activeGroup && activeId && (
)} - +
} /> diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts index 37b2198cfd51f9..1af8c16fa1395f 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/types.ts @@ -29,6 +29,7 @@ export interface ConfigPanelWrapperProps { } >; core: DatasourceDimensionEditorProps['core']; + isFullscreen: boolean; } export interface LayerPanelProps { @@ -46,6 +47,7 @@ export interface LayerPanelProps { } >; core: DatasourceDimensionEditorProps['core']; + isFullscreen: boolean; } export interface LayerDatasourceDropProps { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 362787ea91c4fb..8886232901aa8d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -310,6 +310,7 @@ export function EditorFrame(props: EditorFrameProps) { return ( ) } @@ -361,6 +363,7 @@ export function EditorFrame(props: EditorFrameProps) { visualizationState={state.visualization.state} visualizationMap={props.visualizationMap} dispatch={dispatch} + isFullscreen={Boolean(state.isFullscreenDatasource)} ExpressionRenderer={props.ExpressionRenderer} core={props.core} plugins={props.plugins} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss index 0756c13f6999bf..56642cff1fbff1 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss @@ -70,6 +70,10 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ &:first-child { padding-left: $euiSize; } + + &.lnsFrameLayout__pageBody--fullscreen { + padding: 0; + } } .lnsFrameLayout__sidebar { @@ -106,3 +110,8 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ } } } + +.lnsFrameLayout__sidebar--fullscreen { + flex-basis: 50%; + max-width: calc(50%); +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index a54901a2a2fe1d..f09915a981af26 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -10,12 +10,14 @@ import './frame_layout.scss'; import React from 'react'; import { EuiPage, EuiPageBody, EuiScreenReaderOnly } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import classNames from 'classnames'; export interface FrameLayoutProps { dataPanel: React.ReactNode; configPanel?: React.ReactNode; suggestionsPanel?: React.ReactNode; workspacePanel?: React.ReactNode; + isFullscreen?: boolean; } export function FrameLayout(props: FrameLayoutProps) { @@ -26,17 +28,25 @@ export function FrameLayout(props: FrameLayoutProps) { className="lnsFrameLayout__pageContent" aria-labelledby="lns_ChartTitle" > -
- -

- {i18n.translate('xpack.lens.section.dataPanelLabel', { - defaultMessage: 'Data panel', - })} -

- - {props.dataPanel} -
-
+ {!props.isFullscreen ? ( +
+ +

+ {i18n.translate('xpack.lens.section.dataPanelLabel', { + defaultMessage: 'Data panel', + })} +

+ + {props.dataPanel} +
+ ) : null} +

{i18n.translate('xpack.lens.section.workspaceLabel', { @@ -45,10 +55,13 @@ export function FrameLayout(props: FrameLayoutProps) {

{props.workspacePanel} - {props.suggestionsPanel} + {!props.isFullscreen ? props.suggestionsPanel : null}
diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index 53aba0d6f3f6c1..522f1103c927bf 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,6 +24,7 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; + isFullscreenDatasource?: boolean; } export type Action = @@ -96,6 +97,9 @@ export type Action = | { type: 'SWITCH_DATASOURCE'; newDatasourceId: string; + } + | { + type: 'TOGGLE_FULLSCREEN'; }; export function getActiveDatasourceIdFromDoc(doc?: Document) { @@ -290,6 +294,8 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta }, stagedPreview: action.clearStagedPreview ? undefined : state.stagedPreview, }; + case 'TOGGLE_FULLSCREEN': + return { ...state, isFullscreenDatasource: !state.isFullscreenDatasource }; default: return state; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx index baa9d45a431eaf..25f354e2a8c755 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.test.tsx @@ -66,6 +66,8 @@ function getDefaultProps() { data: dataPluginMock.createStartContract(), }, getSuggestionForField: () => undefined, + isFullscreen: false, + toggleFullscreen: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index a31146e5004349..e03c74c9af34a9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -76,6 +76,7 @@ export interface WorkspacePanelProps { title?: string; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; + isFullscreen: boolean; } interface WorkspaceState { @@ -127,6 +128,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, + isFullscreen, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -338,6 +340,8 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ ); }; + const element = expression !== null ? renderVisualization() : renderEmptyWorkspace(); + return ( - - {renderVisualization()} - {Boolean(suggestionForDraggedField) && expression !== null && renderEmptyWorkspace()} - + {isFullscreen ? ( + element + ) : ( + + {element} + + )} ); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss index e687e478cd3680..727083097d194b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.scss @@ -62,6 +62,10 @@ animation: lnsWorkspacePanel__illustrationPulseContinuous 1.5s ease-in-out 0s infinite normal forwards; } } + + &.lnsWorkspacePanel__dragDrop--fullscreen { + border: none; + } } .lnsWorkspacePanel__emptyContent { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx index 7bb467df9ab0e1..c18b362e2faa4e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.test.tsx @@ -37,6 +37,7 @@ describe('workspace_panel_wrapper', () => { visualizationMap={{ myVis: mockVisualization }} datasourceMap={{}} datasourceStates={{}} + isFullscreen={false} > @@ -58,6 +59,7 @@ describe('workspace_panel_wrapper', () => { visualizationMap={{ myVis: { ...mockVisualization, renderToolbar: renderToolbarMock } }} datasourceMap={{}} datasourceStates={{}} + isFullscreen={false} /> ); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx index 85f7601d8fb292..7cf53f26a9ac41 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx @@ -32,6 +32,7 @@ export interface WorkspacePanelWrapperProps { state: unknown; } >; + isFullscreen: boolean; } export function WorkspacePanelWrapper({ @@ -44,6 +45,7 @@ export function WorkspacePanelWrapper({ visualizationMap, datasourceMap, datasourceStates, + isFullscreen, }: WorkspacePanelWrapperProps) { const activeVisualization = visualizationId ? visualizationMap[visualizationId] : null; const setVisualizationState = useCallback( @@ -74,40 +76,42 @@ export function WorkspacePanelWrapper({ wrap={true} justifyContent="spaceBetween" > - - - - - - {activeVisualization && activeVisualization.renderToolbar && ( + {!isFullscreen ? ( + + - - )} - - + {activeVisualization && activeVisualization.renderToolbar && ( + + + + )} + + + ) : null} {warningMessages && warningMessages.length ? ( {warningMessages} @@ -115,17 +119,21 @@ export function WorkspacePanelWrapper({
- - -

- {title || - i18n.translate('xpack.lens.chartTitle.unsaved', { - defaultMessage: 'Unsaved visualization', - })} -

- - {children} - + {isFullscreen ? ( + children + ) : ( + + +

+ {title || + i18n.translate('xpack.lens.chartTitle.unsaved', { + defaultMessage: 'Unsaved visualization', + })} +

+ + {children} + + )} ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index bf833c4a369325..999371b1dadafe 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -2,6 +2,14 @@ height: 100%; } +.lnsIndexPatternDimensionEditor-fullscreen { + position: absolute; + top: 0; + bottom: 0; + display: flex; + flex-direction: column; +} + .lnsIndexPatternDimensionEditor__section { padding: $euiSizeS; } @@ -10,6 +18,14 @@ background-color: $euiColorLightestShade; } +.lnsIndexPatternDimensionEditor__section--top { + border-bottom: $euiBorderThin; +} + +.lnsIndexPatternDimensionEditor__section--bottom { + border-top: $euiBorderThin; +} + .lnsIndexPatternDimensionEditor__columns { column-count: 2; column-gap: $euiSizeXL; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index d84d418ff231c5..c85bc9188f0833 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -18,6 +18,7 @@ import { EuiFormLabel, EuiToolTip, EuiText, + EuiTabbedContent, } from '@elastic/eui'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; @@ -119,6 +120,8 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, + toggleFullscreen, + isFullscreen, } = props; const services = { data: props.data, @@ -140,6 +143,10 @@ export function DimensionEditor(props: DimensionEditorProps) { }); }; + const setIsCloseable = (isCloseable: boolean) => { + setState((prevState) => ({ ...prevState, isDimensionClosePrevented: !isCloseable })); + }; + const selectedOperationDefinition = selectedColumn && operationDefinitionMap[selectedColumn.operationType]; @@ -149,6 +156,8 @@ export function DimensionEditor(props: DimensionEditorProps) { const ParamEditor = selectedOperationDefinition?.paramEditor; + const [temporaryQuickFunction, setQuickFunction] = useState(false); + const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) @@ -331,8 +340,14 @@ export function DimensionEditor(props: DimensionEditorProps) { currentFieldIsInvalid ); - return ( -
+ const shouldDisplayExtraOptions = + !currentFieldIsInvalid && + !incompleteInfo && + selectedColumn && + selectedColumn.operationType !== 'formula'; + + const quickFunctions = ( + <>
{i18n.translate('xpack.lens.indexPattern.functionsLabel', { @@ -380,6 +395,9 @@ export function DimensionEditor(props: DimensionEditorProps) { currentColumn: state.layers[layerId].columns[columnId], })} dimensionGroups={dimensionGroups} + isFullscreen={isFullscreen} + toggleFullscreen={toggleFullscreen} + setIsCloseable={setIsCloseable} {...services} /> ); @@ -390,7 +408,8 @@ export function DimensionEditor(props: DimensionEditorProps) { {!selectedColumn || selectedOperationDefinition?.input === 'field' || - (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field') ? ( + (incompleteOperation && operationDefinitionMap[incompleteOperation].input === 'field') || + temporaryQuickFunction ? ( ) : null} - {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ParamEditor && ( - <> - - + {shouldDisplayExtraOptions && ParamEditor && ( + )} {!currentFieldIsInvalid && !incompleteInfo && selectedColumn && ( @@ -519,8 +539,87 @@ export function DimensionEditor(props: DimensionEditorProps) {
+ + ); + + const tabs = [ + { + id: 'quickFunctions', + name: i18n.translate('xpack.lens.indexPattern.quickFunctionsLabel', { + defaultMessage: 'Quick functions', + }), + 'data-test-subj': 'lens-dimensionTabs-quickFunctions', + content: quickFunctions, + }, + { + id: 'formula', + name: i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', + }), + 'data-test-subj': 'lens-dimensionTabs-formula', + content: ParamEditor ? ( + <> + + + ) : ( + <> + ), + }, + ]; + + return ( +
+ {isFullscreen ? ( + tabs[1].content + ) : operationSupportMatrix.operationWithoutField.has('formula') ? ( + { + if ( + selectedTab.id === 'quickFunctions' && + selectedColumn?.operationType === 'formula' + ) { + setQuickFunction(true); + } else if (selectedColumn?.operationType !== 'formula') { + setQuickFunction(false); + const newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: 'formula', + visualizationGroups: dimensionGroups, + }); + setStateWrapper(newLayer); + trackUiEvent(`indexpattern_dimension_operation_formula`); + return; + } else if (selectedTab.id === 'formula') { + setQuickFunction(false); + } + }} + size="s" + /> + ) : ( + quickFunctions + )} - {!currentFieldIsInvalid && ( + {!isFullscreen && !currentFieldIsInvalid && (
{!incompleteInfo && selectedColumn && ( )} - {!incompleteInfo && !hideGrouping && ( + {!isFullscreen && !incompleteInfo && !hideGrouping && ( )} - {selectedColumn && + {!isFullscreen && + selectedColumn && (selectedColumn.dataType === 'number' || selectedColumn.operationType === 'range') ? ( { core: {} as CoreSetup, dimensionGroups: [], groupId: 'a', + isFullscreen: false, + toggleFullscreen: jest.fn(), }; jest.clearAllMocks(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts index a77a980257c88c..56d255ec02227c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/droppable.test.ts @@ -284,6 +284,8 @@ describe('IndexPatternDimensionEditorPanel', () => { } as unknown) as DataPublicPluginStart, core: {} as CoreSetup, dimensionGroups: [], + isFullscreen: false, + toggleFullscreen: () => {}, }; jest.clearAllMocks(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx index 645b6bfe70a97c..fd3ad9a4e5dd54 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.test.tsx @@ -51,6 +51,9 @@ describe('reference editor', () => { http: {} as HttpSetup, data: {} as DataPublicPluginStart, dimensionGroups: [], + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index e02a014935458c..2a82f4885f3657 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -50,6 +50,9 @@ export interface ReferenceEditorProps { dateRange: DateRange; labelAppend?: EuiFormRowProps['labelAppend']; dimensionGroups: VisualizationDimensionGroupConfig[]; + isFullscreen: boolean; + toggleFullscreen: () => void; + setIsCloseable: (isCloseable: boolean) => void; // Services uiSettings: IUiSettingsClient; @@ -71,6 +74,9 @@ export function ReferenceEditor(props: ReferenceEditorProps) { dateRange, labelAppend, dimensionGroups, + isFullscreen, + toggleFullscreen, + setIsCloseable, ...services } = props; @@ -346,6 +352,9 @@ export function ReferenceEditor(props: ReferenceEditorProps) { indexPattern={currentIndexPattern} dateRange={dateRange} operationDefinitionMap={operationDefinitionMap} + isFullscreen={isFullscreen} + toggleFullscreen={toggleFullscreen} + setIsCloseable={setIsCloseable} {...services} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 81eb46e8167155..df749d73c5b887 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -317,6 +317,11 @@ export function getIndexPatternDatasource({ domElement ); }, + + canCloseDimensionEditor: (state) => { + return !state.isDimensionClosePrevented; + }, + getDropProps, onDrop, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index eaaf13171124b0..7abf274c60e8ba 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -98,6 +98,9 @@ const defaultOptions = { http: {} as HttpSetup, indexPattern: indexPattern1, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('date_histogram', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx index 46fddd9b1ffbf4..75068817c61237 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.test.tsx @@ -28,6 +28,9 @@ const defaultProps = { http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; // mocking random id generator function diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss new file mode 100644 index 00000000000000..783a6809c25f2a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -0,0 +1,4 @@ +.lnsFormulaOverflow { + // Needs to be higher than the modal and all flyouts + z-index: $euiZLevel9 + 1; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx new file mode 100644 index 00000000000000..42f4d9cf6ca334 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -0,0 +1,548 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiPopover } from '@elastic/eui'; +import { monaco } from '@kbn/monaco'; +import classNames from 'classnames'; +import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; +import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { ParamEditorProps } from '../../index'; +import { getManagedColumnsFrom } from '../../../layer_helpers'; +import { ErrorWrapper, runASTValidation, tryToParse } from '../validation'; +import { useDebounceWithOptions } from '../../helpers'; +import { + LensMathSuggestion, + SUGGESTION_TYPE, + suggest, + getSuggestion, + getSignatureHelp, + getHover, + getTokenInfo, + offsetToRowColumn, + monacoPositionToOffset, +} from './math_completion'; +import { LANGUAGE_ID } from './math_tokenization'; +import { MemoizedFormulaHelp } from './formula_help'; + +import './formula.scss'; +import { FormulaIndexPatternColumn } from '../formula'; +import { regenerateLayerFromAst } from '../parse'; + +export function FormulaEditor({ + layer, + updateLayer, + currentColumn, + columnId, + indexPattern, + operationDefinitionMap, + data, + toggleFullscreen, + isFullscreen, + setIsCloseable, +}: ParamEditorProps) { + const [text, setText] = useState(currentColumn.params.formula); + const [isHelpOpen, setIsHelpOpen] = useState(false); + const editorModel = React.useRef( + monaco.editor.createModel(text ?? '', LANGUAGE_ID) + ); + const overflowDiv1 = React.useRef(); + const disposables = React.useRef([]); + const editor1 = React.useRef(); + + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect + // requires a second render to work, so we are using an if statement to guarantee it happens + // on first render + if (!overflowDiv1?.current) { + const node1 = (overflowDiv1.current = document.createElement('div')); + node1.setAttribute('data-test-subj', 'lnsFormulaWidget'); + // Monaco CSS is targeted on the monaco-editor class + node1.classList.add('lnsFormulaOverflow', 'monaco-editor'); + document.body.appendChild(node1); + } + + // Clean up the monaco editor and DOM on unmount + useEffect(() => { + const model = editorModel.current; + const allDisposables = disposables.current; + const editor1ref = editor1.current; + return () => { + model.dispose(); + overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); + editor1ref?.dispose(); + allDisposables?.forEach((d) => d.dispose()); + }; + }, []); + + useDebounceWithOptions( + () => { + if (!editorModel.current) return; + + if (!text) { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + if (currentColumn.params.formula) { + // Only submit if valid + const { newLayer } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + } + + return; + } + + let errors: ErrorWrapper[] = []; + + const { root, error } = tryToParse(text); + if (error) { + errors = [error]; + } else if (root) { + const validationErrors = runASTValidation( + root, + layer, + indexPattern, + operationDefinitionMap + ); + if (validationErrors.length) { + errors = validationErrors; + } + } + + if (errors.length) { + monaco.editor.setModelMarkers( + editorModel.current, + 'LENS', + errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }) + ); + } else { + monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); + + // Only submit if valid + const { newLayer, locations } = regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ); + updateLayer(newLayer); + + const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); + const markers: monaco.editor.IMarkerData[] = managedColumns + .flatMap(([id, column]) => { + if (locations[id]) { + const def = operationDefinitionMap[column.operationType]; + if (def.getErrorMessage) { + const messages = def.getErrorMessage( + newLayer, + id, + indexPattern, + operationDefinitionMap + ); + if (messages) { + const startPosition = offsetToRowColumn(text, locations[id].min); + const endPosition = offsetToRowColumn(text, locations[id].max); + return [ + { + message: messages.join(', '), + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: monaco.MarkerSeverity.Warning, + }, + ]; + } + } + } + return []; + }) + .filter((marker) => marker); + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); + } + }, + // Make it validate on flyout open in case of a broken formula left over + // from a previous edit + { skipFirstRender: text == null }, + 256, + [text] + ); + + /** + * The way that Monaco requests autocompletion is not intuitive, but the way we use it + * we fetch new suggestions in these scenarios: + * + * - If the user types one of the trigger characters, suggestions are always fetched + * - When the user selects the kql= suggestion, we tell Monaco to trigger new suggestions after + * - When the user types the first character into an empty text box, Monaco requests suggestions + * + * Monaco also triggers suggestions automatically when there are no suggestions being displayed + * and the user types a non-whitespace character. + * + * While suggestions are being displayed, Monaco uses an in-memory cache of the last known suggestions. + */ + const provideCompletionItems = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + context: monaco.languages.CompletionContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + let wordRange: monaco.Range; + let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { + list: [], + type: SUGGESTION_TYPE.FIELD, + }; + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + + if (context.triggerCharacter === '(') { + const wordUntil = model.getWordAtPosition(position.delta(0, -3)); + if (wordUntil) { + wordRange = new monaco.Range( + position.lineNumber, + position.column, + position.lineNumber, + position.column + ); + + // Retrieve suggestions for subexpressions + // TODO: make this work for expressions nested more than one level deep + aSuggestions = await suggest({ + expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + } else { + aSuggestions = await suggest({ + expression: innerText, + position: innerText.length - lengthAfterPosition, + context, + indexPattern, + operationDefinitionMap, + data, + }); + } + + return { + suggestions: aSuggestions.list.map((s) => + getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + ), + }; + }, + [indexPattern, operationDefinitionMap, data] + ); + + const provideSignatureHelp = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken, + context: monaco.languages.SignatureHelpContext + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getSignatureHelp( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const provideHover = useCallback( + async ( + model: monaco.editor.ITextModel, + position: monaco.Position, + token: monaco.CancellationToken + ) => { + const innerText = model.getValue(); + const textRange = model.getFullModelRange(); + + const lengthAfterPosition = model.getValueLengthInRange({ + startLineNumber: position.lineNumber, + startColumn: position.column, + endLineNumber: textRange.endLineNumber, + endColumn: textRange.endColumn, + }); + return getHover( + model.getValue(), + innerText.length - lengthAfterPosition, + operationDefinitionMap + ); + }, + [operationDefinitionMap] + ); + + const onTypeHandler = useCallback( + (e: monaco.editor.IModelContentChangedEvent, editor: monaco.editor.IStandaloneCodeEditor) => { + if (e.isFlush || e.isRedoing || e.isUndoing) { + return; + } + if (e.changes.length === 1 && e.changes[0].text === '=') { + const currentPosition = e.changes[0].range; + if (currentPosition) { + const tokenInfo = getTokenInfo( + editor.getValue(), + monacoPositionToOffset( + editor.getValue(), + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + ); + // Make sure that we are only adding kql='' or lucene='', and also + // check that the = sign isn't inside the KQL expression like kql='=' + if ( + !tokenInfo || + typeof tokenInfo.ast === 'number' || + tokenInfo.ast.type !== 'namedArgument' || + (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || + tokenInfo.ast.value !== 'LENS_MATH_MARKER' + ) { + return; + } + + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.executeEdits( + 'LENS', + [ + { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }, + ], + [ + // After inserting, move the cursor in between the single quotes + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); + }, 0); + } + } + }, + [] + ); + + const codeEditorOptions: CodeEditorProps = { + languageId: LANGUAGE_ID, + value: text ?? '', + onChange: setText, + options: { + automaticLayout: false, + fontSize: 14, + folding: false, + lineNumbers: 'off', + scrollBeyondLastLine: false, + minimap: { enabled: false }, + wordWrap: 'on', + // Disable suggestions that appear when we don't provide a default suggestion + wordBasedSuggestions: false, + autoIndent: 'brackets', + wrappingIndent: 'none', + dimension: { width: 290, height: 200 }, + fixedOverflowWidgets: true, + }, + }; + + useEffect(() => { + // Because the monaco model is owned by Lens, we need to manually attach and remove handlers + const { dispose: dispose1 } = monaco.languages.registerCompletionItemProvider(LANGUAGE_ID, { + triggerCharacters: ['.', '(', '=', ' ', ':', `'`], + provideCompletionItems, + }); + const { dispose: dispose2 } = monaco.languages.registerSignatureHelpProvider(LANGUAGE_ID, { + signatureHelpTriggerCharacters: ['(', '='], + provideSignatureHelp, + }); + const { dispose: dispose3 } = monaco.languages.registerHoverProvider(LANGUAGE_ID, { + provideHover, + }); + return () => { + dispose1(); + dispose2(); + dispose3(); + }; + }, [provideCompletionItems, provideSignatureHelp, provideHover]); + + // The Monaco editor will lazily load Monaco, which takes a render cycle to trigger. This can cause differences + // in the behavior of Monaco when it's first loaded and then reloaded. + return ( +
+
+ + + + + { + toggleFullscreen(); + }} + iconType="fullScreen" + size="s" + color="text" + flush="right" + > + {isFullscreen + ? i18n.translate('xpack.lens.formula.fullScreenCloseLabel', { + defaultMessage: 'Collapse formula', + }) + : i18n.translate('xpack.lens.formula.fullScreenEditorLabel', { + defaultMessage: 'Expand formula', + })} + + + +
+
+ { + editor1.current = editor; + disposables.current.push( + editor.onDidFocusEditorWidget(() => { + setIsCloseable(false); + }) + ); + disposables.current.push( + editor.onDidBlurEditorWidget(() => { + setIsCloseable(true); + }) + ); + // If we ever introduce a second Monaco editor, we need to toggle + // the typing handler to the active editor to maintain the cursor + disposables.current.push( + editor.onDidChangeModelContent((e) => { + onTypeHandler(e, editor); + }) + ); + }} + /> + +
+
+ + + {isFullscreen ? ( + + ) : ( + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + size="s" + color="text" + > + {i18n.translate('xpack.lens.formula.functionReferenceEditorLabel', { + defaultMessage: 'Function reference', + })} + + } + anchorPosition="leftDown" + > + + + )} + + + {/* Errors go here */} + +
+
+ ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx new file mode 100644 index 00000000000000..1335cfe7e3efaa --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiText, + EuiSelectable, + EuiSelectableOption, +} from '@elastic/eui'; +import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { GenericOperationDefinition, ParamEditorProps } from '../../index'; +import { IndexPattern } from '../../../../types'; +import { tinymathFunctions } from '../util'; +import { getPossibleFunctions } from './math_completion'; + +import { FormulaIndexPatternColumn } from '../formula'; + +function FormulaHelp({ + indexPattern, + operationDefinitionMap, +}: { + indexPattern: IndexPattern; + operationDefinitionMap: Record; +}) { + const [selectedFunction, setSelectedFunction] = useState(); + + const helpItems: Array = []; + + helpItems.push({ label: 'Math', isGroupLabel: true }); + + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .map((key) => ({ + label: `${key}`, + description: , + checked: selectedFunction === key ? ('on' as const) : undefined, + })) + ); + + helpItems.push({ label: 'Elasticsearch', isGroupLabel: true }); + + // Es aggs + helpItems.push( + ...getPossibleFunctions(indexPattern) + .filter((key) => key in operationDefinitionMap) + .map((key) => ({ + label: `${key}: ${operationDefinitionMap[key].displayName}`, + description: getHelpText(key, operationDefinitionMap), + checked: + selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` + ? ('on' as const) + : undefined, + })) + ); + + return ( + + + { + const chosenType = newOptions.find(({ checked }) => checked === 'on')!; + if (!chosenType) { + setSelectedFunction(undefined); + } else { + setSelectedFunction(chosenType.label); + } + }} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + {selectedFunction ? ( + helpItems.find(({ label }) => label === selectedFunction)?.description + ) : ( + + )} + + + + ); +} + +export const MemoizedFormulaHelp = React.memo(FormulaHelp); + +// TODO: i18n this whole thing, or move examples into the operation definitions with i18n +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const definition = operationDefinitionMap[type]; + + if (type === 'count') { + return ( + +

Example: count()

+ + ); + } + + return ( + + {definition.input === 'field' ?

Example: {type}(bytes)

: null} + {definition.input === 'fullReference' && !('operationParams' in definition) ? ( +

Example: {type}(sum(bytes))

+ ) : null} + + {'operationParams' in definition && definition.operationParams ? ( +

+

+ Example: {type}(sum(bytes),{' '} + {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) +

+

+ ) : null} + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts new file mode 100644 index 00000000000000..4b6acefa6b30ad --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './formula_editor'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts new file mode 100644 index 00000000000000..9e29160b6747b4 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -0,0 +1,344 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { monaco } from '@kbn/monaco'; +import { createMockedIndexPattern } from '../../../../mocks'; +import { GenericOperationDefinition } from '../../index'; +import type { IndexPatternField } from '../../../../types'; +import type { OperationMetadata } from '../../../../../types'; +import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; +import { tinymathFunctions } from '../util'; +import { getSignatureHelp, getHover, suggest } from './math_completion'; + +const buildGenericColumn = (type: string) => { + return ({ field }: { field?: IndexPatternField }) => { + return { + label: type, + dataType: 'number', + operationType: type, + sourceField: field?.name ?? undefined, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }; + }; +}; + +const numericOperation = () => ({ dataType: 'number', isBucketed: false }); +const stringOperation = () => ({ dataType: 'string', isBucketed: true }); + +// Only one of each type is needed +const operationDefinitionMap: Record = { + sum: ({ + type: 'sum', + input: 'field', + buildColumn: buildGenericColumn('sum'), + getPossibleOperationForField: (field: IndexPatternField) => + field.type === 'number' ? numericOperation() : null, + } as unknown) as GenericOperationDefinition, + count: ({ + type: 'count', + input: 'field', + buildColumn: buildGenericColumn('count'), + getPossibleOperationForField: (field: IndexPatternField) => + field.name === 'Records' ? numericOperation() : null, + } as unknown) as GenericOperationDefinition, + last_value: ({ + type: 'last_value', + input: 'field', + buildColumn: buildGenericColumn('last_value'), + getPossibleOperationForField: (field: IndexPatternField) => ({ + dataType: field.type, + isBucketed: false, + }), + } as unknown) as GenericOperationDefinition, + moving_average: ({ + type: 'moving_average', + input: 'fullReference', + requiredReferences: [ + { + input: ['field', 'managedReference'], + validateMetadata: (meta: OperationMetadata) => + meta.dataType === 'number' && !meta.isBucketed, + }, + ], + operationParams: [{ name: 'window', type: 'number', required: true }], + buildColumn: buildGenericColumn('moving_average'), + getPossibleOperation: numericOperation, + } as unknown) as GenericOperationDefinition, + cumulative_sum: ({ + type: 'cumulative_sum', + input: 'fullReference', + buildColumn: buildGenericColumn('cumulative_sum'), + getPossibleOperation: numericOperation, + } as unknown) as GenericOperationDefinition, + terms: ({ + type: 'terms', + input: 'field', + getPossibleOperationForField: stringOperation, + } as unknown) as GenericOperationDefinition, +}; + +describe('math completion', () => { + describe('signature help', () => { + function unwrapSignatures(signatureResult: monaco.languages.SignatureHelpResult) { + return signatureResult.value.signatures[0]; + } + + it('should silently handle parse errors', () => { + expect(unwrapSignatures(getSignatureHelp('sum(', 4, operationDefinitionMap))).toBeUndefined(); + }); + + it('should return a signature for a field-based ES function', () => { + expect(unwrapSignatures(getSignatureHelp('sum()', 4, operationDefinitionMap))).toEqual({ + label: 'sum(field)', + parameters: [{ label: 'field' }], + }); + }); + + it('should return a signature for count', () => { + expect(unwrapSignatures(getSignatureHelp('count()', 6, operationDefinitionMap))).toEqual({ + label: 'count()', + parameters: [], + }); + }); + + it('should return a signature for a function with named parameters', () => { + expect( + unwrapSignatures( + getSignatureHelp('2 * moving_average(count(), window=)', 35, operationDefinitionMap) + ) + ).toEqual({ + label: 'moving_average(function, window=number)', + parameters: [ + { label: 'function' }, + { + label: 'window=number', + documentation: 'Required', + }, + ], + }); + }); + + it('should return a signature for an inner function', () => { + expect( + unwrapSignatures( + getSignatureHelp('2 * moving_average(count())', 25, operationDefinitionMap) + ) + ).toEqual({ label: 'count()', parameters: [] }); + }); + + it('should return a signature for a complex tinymath function', () => { + expect( + unwrapSignatures(getSignatureHelp('clamp(count(), 5)', 7, operationDefinitionMap)) + ).toEqual({ + label: 'clamp(expression, min, max)', + parameters: [ + { label: 'expression', documentation: '' }, + { label: 'min', documentation: '' }, + { label: 'max', documentation: '' }, + ], + }); + }); + }); + + describe('hover provider', () => { + it('should silently handle parse errors', () => { + expect(getHover('sum(', 2, operationDefinitionMap)).toEqual({ contents: [] }); + }); + + it('should show signature for a field-based ES function', () => { + expect(getHover('sum()', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'sum(field)' }], + }); + }); + + it('should show signature for count', () => { + expect(getHover('count()', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'count()' }], + }); + }); + + it('should show signature for a function with named parameters', () => { + expect(getHover('2 * moving_average(count())', 10, operationDefinitionMap)).toEqual({ + contents: [{ value: 'moving_average(function, window=number)' }], + }); + }); + + it('should show signature for an inner function', () => { + expect(getHover('2 * moving_average(count())', 22, operationDefinitionMap)).toEqual({ + contents: [{ value: 'count()' }], + }); + }); + + it('should show signature for a complex tinymath function', () => { + expect(getHover('clamp(count(), 5)', 2, operationDefinitionMap)).toEqual({ + contents: [{ value: 'clamp(expression, min, max)' }], + }); + }); + }); + + describe('autocomplete', () => { + it('should list all valid functions at the top level (fake test)', async () => { + // This test forces an invalid scenario, since the autocomplete actually requires + // some typing + const results = await suggest({ + expression: '', + position: 1, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + }); + + it('should list all valid sub-functions for a fullReference', async () => { + const results = await suggest({ + expression: 'moving_average()', + position: 15, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toHaveLength(2); + ['sum', 'last_value'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'operation' }])); + }); + }); + + it('should list all valid named arguments for a fullReference', async () => { + const results = await suggest({ + expression: 'moving_average(count(),)', + position: 23, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toEqual(['window']); + }); + + it('should not list named arguments when they are already in use', async () => { + const results = await suggest({ + expression: 'moving_average(count(), window=5, )', + position: 34, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toEqual([]); + }); + + it('should list all valid positional arguments for a tinymath function used by name', async () => { + const results = await suggest({ + expression: 'divide(count(), )', + position: 16, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + }); + + it('should list all valid positional arguments for a tinymath function used with alias', async () => { + const results = await suggest({ + expression: 'count() / ', + position: 10, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: ',', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toHaveLength(4 + Object.keys(tinymathFunctions).length); + ['sum', 'moving_average', 'cumulative_sum', 'last_value'].forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + Object.keys(tinymathFunctions).forEach((key) => { + expect(results.list).toEqual(expect.arrayContaining([{ label: key, type: 'math' }])); + }); + }); + + it('should not autocomplete any fields for the count function', async () => { + const results = await suggest({ + expression: 'count()', + position: 6, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toHaveLength(0); + }); + + it('should autocomplete and validate the right type of field', async () => { + const results = await suggest({ + expression: 'sum()', + position: 4, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toEqual(['bytes', 'memory']); + }); + + it('should autocomplete only operations that provide numeric output', async () => { + const results = await suggest({ + expression: 'last_value()', + position: 11, + context: { + triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: '(', + }, + indexPattern: createMockedIndexPattern(), + operationDefinitionMap, + data: dataPluginMock.createStartContract(), + }); + expect(results.list).toEqual(['bytes', 'memory']); + }); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts new file mode 100644 index 00000000000000..e8c16fe64651a4 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -0,0 +1,569 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniq, startsWith } from 'lodash'; +import { monaco } from '@kbn/monaco'; +import { + parse, + TinymathLocation, + TinymathAST, + TinymathFunction, + TinymathNamedArgument, +} from '@kbn/tinymath'; +import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; +import { IndexPattern } from '../../../../types'; +import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; +import { tinymathFunctions, groupArgsByType } from '../util'; +import type { GenericOperationDefinition } from '../..'; + +export enum SUGGESTION_TYPE { + FIELD = 'field', + NAMED_ARGUMENT = 'named_argument', + FUNCTIONS = 'functions', + KQL = 'kql', +} + +export type LensMathSuggestion = + | string + | { + label: string; + type: 'operation' | 'math'; + } + | QuerySuggestion; + +export interface LensMathSuggestions { + list: LensMathSuggestion[]; + type: SUGGESTION_TYPE; +} + +function inLocation(cursorPosition: number, location: TinymathLocation) { + return cursorPosition >= location.min && cursorPosition < location.max; +} + +const MARKER = 'LENS_MATH_MARKER'; + +function getInfoAtPosition( + ast: TinymathAST, + position: number, + parent?: TinymathFunction +): undefined | { ast: TinymathAST; parent?: TinymathFunction } { + if (typeof ast === 'number') { + return; + } + if (!inLocation(position, ast.location)) { + return; + } + if (ast.type === 'function') { + const [match] = ast.args.map((arg) => getInfoAtPosition(arg, position, ast)).filter((a) => a); + if (match) { + return match.parent ? match : { ...match, parent: ast }; + } + } + return { + ast, + parent, + }; +} + +export function offsetToRowColumn(expression: string, offset: number): monaco.Position { + const lines = expression.split(/\n/); + let remainingChars = offset; + let lineNumber = 1; + for (const line of lines) { + if (line.length >= remainingChars) { + return new monaco.Position(lineNumber, remainingChars); + } + remainingChars -= line.length + 1; + lineNumber++; + } + + throw new Error('Algorithm failure'); +} + +export function monacoPositionToOffset(expression: string, position: monaco.Position): number { + const lines = expression.split(/\n/); + return lines + .slice(0, position.lineNumber) + .reduce( + (prev, current, index) => + prev + index === position.lineNumber - 1 ? position.column : current.length, + 0 + ); +} + +export async function suggest({ + expression, + position, + context, + indexPattern, + operationDefinitionMap, + data, +}: { + expression: string; + position: number; + context: monaco.languages.CompletionContext; + indexPattern: IndexPattern; + operationDefinitionMap: Record; + data: DataPublicPluginStart; +}): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + const tokenInfo = getInfoAtPosition(ast, position); + const tokenAst = tokenInfo?.ast; + + const isNamedArgument = + tokenInfo?.parent && + typeof tokenAst !== 'number' && + tokenAst && + 'type' in tokenAst && + tokenAst.type === 'namedArgument'; + if (tokenInfo?.parent && (context.triggerCharacter === '=' || isNamedArgument)) { + return await getNamedArgumentSuggestions({ + ast: tokenAst as TinymathNamedArgument, + data, + indexPattern, + }); + } else if (tokenInfo?.parent) { + return getArgumentSuggestions( + tokenInfo.parent, + tokenInfo.parent.args.findIndex((a) => a === tokenAst), + indexPattern, + operationDefinitionMap + ); + } + if ( + typeof tokenAst === 'object' && + Boolean(tokenAst.type === 'variable' || tokenAst.type === 'function') + ) { + const nameWithMarker = tokenAst.type === 'function' ? tokenAst.name : tokenAst.value; + return getFunctionSuggestions( + nameWithMarker.split(MARKER)[0], + indexPattern, + operationDefinitionMap + ); + } + } catch (e) { + // Fail silently + } + return { list: [], type: SUGGESTION_TYPE.FIELD }; +} + +export function getPossibleFunctions( + indexPattern: IndexPattern, + operationDefinitionMap?: Record +) { + const available = memoizedGetAvailableOperationsByMetadata(indexPattern, operationDefinitionMap); + const possibleOperationNames: string[] = []; + available.forEach((a) => { + if (a.operationMetaData.dataType === 'number' && !a.operationMetaData.isBucketed) { + possibleOperationNames.push( + ...a.operations.filter((o) => o.type !== 'managedReference').map((o) => o.operationType) + ); + } + }); + + return [...uniq(possibleOperationNames), ...Object.keys(tinymathFunctions)]; +} + +function getFunctionSuggestions( + prefix: string, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + return { + list: uniq( + getPossibleFunctions(indexPattern, operationDefinitionMap).filter((func) => + startsWith(func, prefix) + ) + ).map((func) => ({ label: func, type: 'operation' as const })), + type: SUGGESTION_TYPE.FUNCTIONS, + }; +} + +function getArgumentSuggestions( + ast: TinymathFunction, + position: number, + indexPattern: IndexPattern, + operationDefinitionMap: Record +) { + const { name } = ast; + const operation = operationDefinitionMap[name]; + if (!operation && !tinymathFunctions[name]) { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + + const tinymathFunction = tinymathFunctions[name]; + if (tinymathFunction) { + if (tinymathFunction.positionalArguments[position]) { + return { + list: uniq(getPossibleFunctions(indexPattern, operationDefinitionMap)).map((f) => ({ + type: 'math' as const, + label: f, + })), + type: SUGGESTION_TYPE.FUNCTIONS, + }; + } + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + + if (position > 0 || operation.type === 'count') { + const { namedArguments } = groupArgsByType(ast.args); + const list = []; + if (operation.filterable) { + if (!namedArguments.find((arg) => arg.name === 'kql')) { + list.push('kql'); + } + if (!namedArguments.find((arg) => arg.name === 'lucene')) { + list.push('lucene'); + } + } + if ('operationParams' in operation) { + // Exclude any previously used named args + list.push( + ...operation + .operationParams!.filter( + (param) => + // Keep the param if it's the first use + !namedArguments.find((arg) => arg.name === param.name) + ) + .map((p) => p.name) + ); + } + return { list, type: SUGGESTION_TYPE.NAMED_ARGUMENT }; + } + + if (operation.input === 'field' && position === 0) { + const available = memoizedGetAvailableOperationsByMetadata( + indexPattern, + operationDefinitionMap + ); + // TODO: This only allow numeric functions, will reject last_value(string) for example. + const validOperation = available.find( + ({ operationMetaData }) => + operationMetaData.dataType === 'number' && !operationMetaData.isBucketed + ); + if (validOperation) { + const fields = validOperation.operations + .filter((op) => op.operationType === operation.type) + .map((op) => ('field' in op ? op.field : undefined)) + .filter((field) => field); + return { list: fields as string[], type: SUGGESTION_TYPE.FIELD }; + } else { + return { list: [], type: SUGGESTION_TYPE.FIELD }; + } + } + + if (operation.input === 'fullReference') { + const available = memoizedGetAvailableOperationsByMetadata( + indexPattern, + operationDefinitionMap + ); + const possibleOperationNames: string[] = []; + available.forEach((a) => { + if ( + operation.requiredReferences.some((requirement) => + requirement.validateMetadata(a.operationMetaData) + ) + ) { + possibleOperationNames.push( + ...a.operations + .filter((o) => + operation.requiredReferences.some((requirement) => requirement.input.includes(o.type)) + ) + .map((o) => o.operationType) + ); + } + }); + return { + list: uniq(possibleOperationNames).map((n) => ({ label: n, type: 'operation' as const })), + type: SUGGESTION_TYPE.FUNCTIONS, + }; + } + + return { list: [], type: SUGGESTION_TYPE.FIELD }; +} + +export async function getNamedArgumentSuggestions({ + ast, + data, + indexPattern, +}: { + ast: TinymathNamedArgument; + indexPattern: IndexPattern; + data: DataPublicPluginStart; +}) { + if (ast.name !== 'kql' && ast.name !== 'lucene') { + return { list: [], type: SUGGESTION_TYPE.KQL }; + } + if (!data.autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { + return { list: [], type: SUGGESTION_TYPE.KQL }; + } + + const before = ast.value.split(MARKER)[0]; + // TODO + const suggestions = await data.autocomplete.getQuerySuggestions({ + language: 'kuery', + query: ast.value.split(MARKER)[0], + selectionStart: before.length, + selectionEnd: before.length, + indexPatterns: [indexPattern], + boolFilter: [], + }); + return { list: suggestions ?? [], type: SUGGESTION_TYPE.KQL }; +} + +export function getSuggestion( + suggestion: LensMathSuggestion, + type: SUGGESTION_TYPE, + range: monaco.Range, + operationDefinitionMap: Record +): monaco.languages.CompletionItem { + let kind: monaco.languages.CompletionItemKind = monaco.languages.CompletionItemKind.Method; + let label: string = + typeof suggestion === 'string' + ? suggestion + : 'label' in suggestion + ? suggestion.label + : suggestion.text; + let insertText: string | undefined; + let insertTextRules: monaco.languages.CompletionItem['insertTextRules']; + let detail: string = ''; + let command: monaco.languages.CompletionItem['command']; + let sortText: string = ''; + const filterText: string = label; + + switch (type) { + case SUGGESTION_TYPE.FIELD: + kind = monaco.languages.CompletionItemKind.Value; + break; + case SUGGESTION_TYPE.FUNCTIONS: + insertText = `${label}($0)`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + if (typeof suggestion !== 'string') { + if ('text' in suggestion) break; + const tinymathFunction = tinymathFunctions[suggestion.label]; + if (tinymathFunction) { + label = `${label}(${tinymathFunction.positionalArguments + .map(({ name }) => name) + .join(', ')})`; + detail = 'TinyMath'; + kind = monaco.languages.CompletionItemKind.Method; + } else { + const def = operationDefinitionMap[suggestion.label]; + kind = monaco.languages.CompletionItemKind.Constant; + if (suggestion.label === 'count' && 'operationParams' in def) { + label = `${label}(${def + .operationParams!.map((p) => `${p.name}=${p.type}`) + .join(', ')})`; + } else if ('operationParams' in def) { + label = `${label}(expression, ${def + .operationParams!.map((p) => `${p.name}=${p.type}`) + .join(', ')})`; + } else { + label = `${label}(expression)`; + } + detail = 'Elasticsearch'; + // Always put ES functions first + sortText = `0${label}`; + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + } + } + break; + case SUGGESTION_TYPE.NAMED_ARGUMENT: + kind = monaco.languages.CompletionItemKind.Keyword; + if (label === 'kql' || label === 'lucene') { + command = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', + }; + insertText = `${label}='$0'`; + insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; + sortText = `zzz${label}`; + } + label = `${label}=`; + detail = ''; + break; + case SUGGESTION_TYPE.KQL: + if (label.includes(`'`)) { + insertText = label.replaceAll(`'`, "\\'"); + } + break; + } + + return { + detail, + kind, + label, + insertText: insertText ?? label, + insertTextRules, + command, + additionalTextEdits: [], + range, + sortText, + filterText, + }; +} + +export function getSignatureHelp( + expression: string, + position: number, + operationDefinitionMap: Record +): monaco.languages.SignatureHelpResult { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + const tokenInfo = getInfoAtPosition(ast, position); + + if (tokenInfo?.parent) { + const name = tokenInfo.parent.name; + // reference equality is fine here because of the way the getInfo function works + const index = tokenInfo.parent.args.findIndex((arg) => arg === tokenInfo.ast); + + if (tinymathFunctions[name]) { + const stringify = `${name}(${tinymathFunctions[name].positionalArguments + .map((arg) => arg.name) + .join(', ')})`; + return { + value: { + signatures: [ + { + label: stringify, + parameters: tinymathFunctions[name].positionalArguments.map((arg) => ({ + label: arg.name, + documentation: arg.optional ? 'Optional' : '', + })), + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } else if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + + const firstParam: monaco.languages.ParameterInformation | null = + name !== 'count' + ? { + label: + def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; + if ('operationParams' in def) { + return { + value: { + signatures: [ + { + label: `${name}(${ + firstParam ? firstParam.label + ', ' : '' + }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, + parameters: [ + ...(firstParam ? [firstParam] : []), + ...def.operationParams!.map((arg) => ({ + label: `${arg.name}=${arg.type}`, + documentation: arg.required ? 'Required' : '', + })), + ], + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } else { + return { + value: { + signatures: [ + { + label: `${name}(${firstParam ? firstParam.label : ''})`, + parameters: firstParam ? [firstParam] : [], + }, + ], + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; + } + } + } + } catch (e) { + // do nothing + } + return { value: { signatures: [], activeParameter: 0, activeSignature: 0 }, dispose: () => {} }; +} + +export function getHover( + expression: string, + position: number, + operationDefinitionMap: Record +): monaco.languages.Hover { + try { + const ast = parse(expression); + + const tokenInfo = getInfoAtPosition(ast, position); + + if (!tokenInfo || typeof tokenInfo.ast === 'number' || !('name' in tokenInfo.ast)) { + return { contents: [] }; + } + + const name = tokenInfo.ast.name; + + if (tinymathFunctions[name]) { + const stringify = `${name}(${tinymathFunctions[name].positionalArguments + .map((arg) => arg.name) + .join(', ')})`; + return { contents: [{ value: stringify }] }; + } else if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + + const firstParam: monaco.languages.ParameterInformation | null = + name !== 'count' + ? { + label: + def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; + if ('operationParams' in def) { + return { + contents: [ + { + value: `${name}(${ + firstParam ? firstParam.label + ', ' : '' + }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, + }, + ], + }; + } else { + return { + contents: [{ value: `${name}(${firstParam ? firstParam.label : ''})` }], + }; + } + } + } catch (e) { + // do nothing + } + return { contents: [] }; +} + +export function getTokenInfo(expression: string, position: number) { + const text = expression.substr(0, position) + MARKER + expression.substr(position); + try { + const ast = parse(text); + + return getInfoAtPosition(ast, position); + } catch (e) { + return; + } +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx new file mode 100644 index 00000000000000..51e96bad600439 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { monaco } from '@kbn/monaco'; + +export const LANGUAGE_ID = 'lens_math'; +monaco.languages.register({ id: LANGUAGE_ID }); + +export const languageConfiguration: monaco.languages.LanguageConfiguration = { + wordPattern: /[A-Za-z\.-_@]/g, + brackets: [['(', ')']], + autoClosingPairs: [ + { open: '(', close: ')' }, + { open: `'`, close: `'` }, + { open: '"', close: '"' }, + ], + surroundingPairs: [ + { open: '(', close: ')' }, + { open: `'`, close: `'` }, + { open: '"', close: '"' }, + ], +}; + +export const lexerRules = { + defaultToken: 'invalid', + tokenPostfix: '', + ignoreCase: true, + brackets: [{ open: '(', close: ')', token: 'delimiter.parenthesis' }], + escapes: /\\(?:[\\"'])/, + tokenizer: { + root: [ + [/\s+/, 'whitespace'], + [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'keyword'], + [/[,=]/, 'delimiter'], + [/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number'], + // strings double quoted + [/"([^"\\]|\\.)*$/, 'string.invalid'], // string without termination + [/"/, 'string', '@string_dq'], + // strings single quoted + [/'([^'\\]|\\.)*$/, 'string.invalid'], // string without termination + [/'/, 'string', '@string_sq'], + [/\+|\-|\*|\//, 'keyword.operator'], + [/[\(]/, 'delimiter'], + [/[\)]/, 'delimiter'], + ], + string_dq: [ + [/[^\\"]+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/"/, 'string', '@pop'], + ], + string_sq: [ + [/[^\\']+/, 'string'], + [/@escapes/, 'string.escape'], + [/\\./, 'string.escape.invalid'], + [/'/, 'string', '@pop'], + ], + }, +} as monaco.languages.IMonarchLanguage; + +monaco.languages.setMonarchTokensProvider(LANGUAGE_ID, lexerRules); +monaco.languages.setLanguageConfiguration(LANGUAGE_ID, languageConfiguration); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx new file mode 100644 index 00000000000000..ce7b48aa1875ea --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -0,0 +1,997 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockedIndexPattern } from '../../../mocks'; +import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { FormulaIndexPatternColumn } from './formula'; +import { regenerateLayerFromAst } from './parse'; +import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; +import { tinymathFunctions } from './util'; + +jest.mock('../../layer_helpers', () => { + return { + getColumnOrder: ({ columns }: { columns: Record }) => + Object.keys(columns), + }; +}); + +const operationDefinitionMap: Record = { + average: ({ + input: 'field', + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'average', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + terms: { input: 'field' } as GenericOperationDefinition, + sum: { input: 'field' } as GenericOperationDefinition, + last_value: { input: 'field' } as GenericOperationDefinition, + max: { input: 'field' } as GenericOperationDefinition, + count: ({ + input: 'field', + filterable: true, + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'count', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: ({ + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + buildColumn: ({ references }: { references: string[] }) => ({ + label: 'moving_average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + timeScale: false, + params: { window: 5 }, + references, + }), + getErrorMessage: () => ['mock error'], + } as unknown) as GenericOperationDefinition, + cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition, +}; + +describe('formula', () => { + let layer: IndexPatternLayer; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }; + }); + + describe('buildColumn', () => { + let indexPattern: IndexPattern; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Average', + dataType: 'number', + operationType: 'average', + isBucketed: false, + scale: 'ratio', + sourceField: 'bytes', + }, + }, + }; + indexPattern = createMockedIndexPattern(); + }); + + it('should start with an empty formula if no previous column is detected', () => { + expect( + formulaOperation.buildColumn({ + layer: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); + + it('should move into Formula previous operation', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: layer.columns.col1, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { isFormulaBroken: false, formula: 'average(bytes)' }, + references: [], + }); + }); + + it('it should move over explicit format param if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + params: { + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'average(bytes)', + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + references: [], + }); + }); + + it('it should move over kql arguments if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + filter: { + language: 'kuery', + // Need to test with multiple replaces due to string replace + query: `category.keyword: "Men's Clothing" or category.keyword: "Men's Shoes"`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `average(bytes, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, + }, + references: [], + }); + }); + + it('it should move over lucene arguments without', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + operationType: 'count', + sourceField: 'Records', + filter: { + language: 'lucene', + query: `*`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `count(lucene='*')`, + }, + references: [], + }); + }); + + it('should move over previous operation parameter if set - only numeric', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + col2: { + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'average', + scale: 'ratio', + sourceField: 'bytes', + timeScale: 'd', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'moving_average(average(bytes), window=3)', + }, + references: [], + }); + }); + + it('should not move previous column configuration if not numeric', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); + }); + + describe('regenerateLayerFromAst()', () => { + let indexPattern: IndexPattern; + let currentColumn: FormulaIndexPatternColumn; + + function testIsBrokenFormula(formula: string) { + expect( + regenerateLayerFromAst( + formula, + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).newLayer + ).toEqual({ + ...layer, + columns: { + ...layer.columns, + col1: { + ...currentColumn, + params: { + ...currentColumn.params, + formula, + isFormulaBroken: true, + }, + }, + }, + }); + } + + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + currentColumn = { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula: '', isFormulaBroken: false }, + references: [], + }; + }); + + it('should mutate the layer with new columns for valid formula expressions', () => { + expect( + regenerateLayerFromAst( + 'average(bytes)', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).newLayer + ).toEqual({ + ...layer, + columnOrder: ['col1X0', 'col1X1', 'col1'], + columns: { + ...layer.columns, + col1: { + ...currentColumn, + references: ['col1X1'], + params: { + ...currentColumn.params, + formula: 'average(bytes)', + isFormulaBroken: false, + }, + }, + col1X0: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'average', + scale: 'ratio', + sourceField: 'bytes', + timeScale: false, + }, + col1X1: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X1', + operationType: 'math', + params: { + tinymathAst: 'col1X0', + }, + references: ['col1X0'], + scale: 'ratio', + }, + }, + }); + }); + + it('returns no change but error if the formula cannot be parsed', () => { + const formulas = [ + '+', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', + 'average(bytes) + moving_average(average(bytes), window=)', + ]; + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if field is used with no Lens wrapping operation', () => { + testIsBrokenFormula('bytes'); + }); + + it('returns no change but error if at least one field in the formula is missing', () => { + const formulas = [ + 'noField', + 'average(noField)', + 'noField + 1', + 'derivative(average(noField))', + 'average(bytes) + derivative(average(noField))', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if at least one operation in the formula is missing', () => { + const formulas = [ + 'noFn()', + 'noFn(bytes)', + 'average(bytes) + noFn()', + 'derivative(noFn())', + 'noFn() + noFnTwo()', + 'noFn(noFnTwo())', + 'noFn() + noFnTwo() + 5', + 'average(bytes) + derivative(noFn())', + 'derivative(average(bytes) + noFn())', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if one operation has the wrong first argument', () => { + const formulas = [ + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', + 'derivative(7)', + 'derivative(bytes + 7)', + 'derivative(bytes + bytes)', + 'derivative(bytes + average(bytes))', + 'derivative(bytes + 7 + average(bytes))', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if an argument is passed to count operation', () => { + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if a required parameter is not passed to the operation in formula', () => { + const formula = 'moving_average(average(bytes))'; + testIsBrokenFormula(formula); + }); + + it('returns no change but error if a required parameter passed with the wrong type in formula', () => { + const formula = 'moving_average(average(bytes), window="m")'; + testIsBrokenFormula(formula); + }); + + it('returns error if a required parameter is passed multiple time', () => { + const formula = 'moving_average(average(bytes), window=7, window=3)'; + testIsBrokenFormula(formula); + }); + + it('returns error if a math operation has less arguments than required', () => { + const formula = 'pow(5)'; + testIsBrokenFormula(formula); + }); + + it('returns error if a math operation has the wrong argument type', () => { + const formula = 'pow(bytes)'; + testIsBrokenFormula(formula); + }); + + it('returns the locations of each function', () => { + expect( + regenerateLayerFromAst( + 'moving_average(average(bytes), window=7) + count()', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).locations + ).toEqual({ + col1X0: { min: 15, max: 29 }, + col1X2: { min: 0, max: 41 }, + col1X3: { min: 43, max: 50 }, + }); + }); + }); + + describe('getErrorMessage', () => { + let indexPattern: IndexPattern; + + function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer { + return { + columns: { + col1: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula, isFormulaBroken: isBroken }, + references: [], + }, + }, + columnOrder: [], + indexPatternId: '', + }; + } + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + }); + + it('returns undefined if count is passed without arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('count()'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if count is passed with only a named argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='*')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns a syntax error if the kql argument does not parse', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='invalid: "')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + `Expected "(", "{", value, whitespace but """ found. +invalid: " +---------^`, + ]); + }); + + it('returns undefined if a field operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average(bytes)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + // note that field names can be wrapped in quotes as well + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average("bytes")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(average(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(average("bytes"))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), window=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + + // Not sure it will be supported + // expect( + // formulaOperation.getErrorMessage!( + // getNewLayerWithFormula('moving_average(average("bytes"), "window"=7)'), + // 'col1', + // indexPattern, + // operationDefinitionMap + // ) + // ).toEqual(undefined); + }); + + it('returns an error if field is used with no Lens wrapping operation', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The field bytes cannot be used without operation`]); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes + bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The operation add does not accept any field as argument`]); + }); + + it('returns an error if parsing a syntax invalid formula', () => { + const formulas = [ + '+', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The Formula ${formula} cannot be parsed`]); + } + }); + + it('returns an error if the field is missing', () => { + const formulas = [ + 'noField', + 'average(noField)', + 'noField + 1', + 'derivative(average(noField))', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Field noField not found']); + } + }); + + it('returns an error with plural form correctly handled', () => { + const formulas = ['noField + noField2', 'noField + 1 + noField2']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Fields noField, noField2 not found']); + } + }); + + it('returns an error if an operation is unknown', () => { + const formulas = ['noFn()', 'noFn(bytes)', 'average(bytes) + noFn()', 'derivative(noFn())']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operation noFn not found']); + } + + const multipleFnFormulas = ['noFn() + noFnTwo()', 'noFn(noFnTwo())']; + + for (const formula of multipleFnFormulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operations noFn, noFnTwo not found']); + } + }); + + it('returns an error if field operation in formula have the wrong first argument', () => { + const formulas = [ + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', + 'derivative(7)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual( + // some formulas may contain more errors + expect.arrayContaining([ + expect.stringMatching( + `The first argument for ${formula.substring(0, formula.indexOf('('))}` + ), + ]) + ); + } + }); + + it('returns an error if an argument is passed to count() operation', () => { + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation count does not accept any field as argument']); + } + }); + + it('returns an error if an operation with required parameters does not receive them', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + }); + + it('returns an error if a parameter is passed to an operation with no parameters', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average(bytes, myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation average does not accept any parameter']); + }); + + it('returns an error if the parameter passed to an operation is of the wrong type', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), window="m")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The parameters for the operation moving_average in the Formula are of the wrong type: window', + ]); + }); + + it('returns no error for the demo formula example', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(` + moving_average( + cumulative_sum( + 7 * clamp(sum(bytes), 0, last_value(memory) + max(memory)) + ), window=10 + ) + `), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns no error if a math operation is passed to fullReference operations', () => { + const formulas = [ + 'derivative(7+1)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + + it('returns errors if math operations are used with no arguments', () => { + const formulas = [ + 'derivative(7+1)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + + // there are 4 types of errors for math functions: + // * no argument passed + // * too many arguments passed + // * field passed + // * missing argument + const errors = [ + (operation: string) => + `The first argument for ${operation} should be a operation name. Found ()`, + (operation: string) => `The operation ${operation} has too many arguments`, + (operation: string) => `The operation ${operation} does not accept any field as argument`, + (operation: string) => { + const required = tinymathFunctions[operation].positionalArguments.filter( + ({ optional }) => !optional + ); + return `The operation ${operation} in the Formula is missing ${ + required.length - 1 + } arguments: ${required + .slice(1) + .map(({ name }) => name) + .join(', ')}`; + }, + ]; + // we'll try to map all of these here in this test + for (const fn of Object.keys(tinymathFunctions)) { + it(`returns an error for the math functions available: ${fn}`, () => { + const nArgs = tinymathFunctions[fn].positionalArguments; + // start with the first 3 types + const formulas = [ + `${fn}()`, + `${fn}(1, 2, 3, 4, 5)`, + // to simplify a bit, add the required number of args by the function filled with the field name + `${fn}(${Array(nArgs.length).fill('bytes').join(', ')})`, + ]; + // add the fourth check only for those functions with more than 1 arg required + const enableFourthCheck = nArgs.filter(({ optional }) => !optional).length > 1; + if (enableFourthCheck) { + formulas.push(`${fn}(1)`); + } + formulas.forEach((formula, i) => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([errors[i](fn)]); + }); + }); + } + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index de7ecb4bc75da3..6494c47548f2f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -10,6 +10,7 @@ import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern } from '../../../types'; import { runASTValidation, tryToParse } from './validation'; +import { FormulaEditor } from './editor'; import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; @@ -152,4 +153,6 @@ export const formulaOperation: OperationDefinition< ); return newLayer; }, + + paramEditor: FormulaEditor, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md new file mode 100644 index 00000000000000..ae244109ed53ee --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math_examples.md @@ -0,0 +1,28 @@ +Basic numeric functions that we already support in Lens: + +count() +count(normalize_unit='1s') +sum(field name) +avg(field name) +moving_average(sum(field name), window=5) +moving_average(sum(field name), window=5, normalize_unit='1s') +counter_rate(field name, normalize_unit='1s') +differences(count()) +differences(sum(bytes), normalize_unit='1s') +last_value(bytes, sort=timestamp) +percentile(bytes, percent=95) + +Adding features beyond what we already support. New features are: + +* Filtering +* Math across series +* Time offset + +count() * 100 +(count() / count(offset=-7d)) + min(field name) +sum(field name, filter='field.keyword: "KQL autocomplete inside math" AND field.value > 100') + +What about custom formatting using string manipulation? Probably not... + +(avg(bytes) / 1000) + 'kb' + \ No newline at end of file diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 579b2822f49112..cf715fbfc66158 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -114,7 +114,7 @@ export const getQueryValidationError = ( function getMessageFromId({ messageId, - values: { ...values }, + values, locations, }: { messageId: K; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index a7402bc13c0a88..27982243f8c2b1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -150,6 +150,9 @@ export interface ParamEditorProps { currentColumn: C; layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; + toggleFullscreen: () => void; + setIsCloseable: (isCloseable: boolean) => void; + isFullscreen: boolean; columnId: string; indexPattern: IndexPattern; uiSettings: IUiSettingsClient; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx index 280cfe9471c9d0..76562dd9b3d44c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.test.tsx @@ -30,6 +30,9 @@ const defaultProps = { hasRestrictions: false, } as IndexPattern, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('last_value', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index a688f95e94c9ed..3a2c1aeebf1860 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -32,6 +32,9 @@ const defaultProps = { hasRestrictions: false, } as IndexPattern, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('percentile', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx index 08bcfcb2e93be5..3a9c2ca583cd25 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.test.tsx @@ -91,6 +91,9 @@ const defaultOptions = { ]), }, operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('ranges', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index b094d3f0ff5cd7..948675bd9ac9eb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -29,6 +29,9 @@ const defaultProps = { http: {} as HttpSetup, indexPattern: createMockedIndexPattern(), operationDefinitionMap: {}, + isFullscreen: false, + toggleFullscreen: jest.fn(), + setIsCloseable: jest.fn(), }; describe('terms', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 4dd56d2de1144a..cc69d66079da08 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -24,7 +24,7 @@ import type { IndexPattern, IndexPatternLayer } from '../types'; import { documentField } from '../document_field'; import { getFieldByNameFactory } from '../pure_helpers'; import { generateId } from '../../id_generator'; -import { createMockedFullReference } from './mocks'; +import { createMockedFullReference, createMockedManagedReference } from './mocks'; jest.mock('../operations'); jest.mock('../../id_generator'); @@ -91,10 +91,13 @@ describe('state_helpers', () => { // @ts-expect-error we are inserting an invalid type operationDefinitionMap.testReference = createMockedFullReference(); + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.managedReference = createMockedManagedReference(); }); afterEach(() => { delete operationDefinitionMap.testReference; + delete operationDefinitionMap.managedReference; }); describe('copyColumn', () => { @@ -108,7 +111,7 @@ describe('state_helpers', () => { formula: 'moving_average(sum(bytes), window=5)', isFormulaBroken: false, }, - references: ['formulaX3'], + references: ['formulaX1'], }; const math = { customLabel: true, @@ -135,7 +138,7 @@ describe('state_helpers', () => { label: 'formulaX2', operationType: 'moving_average' as const, params: { window: 5 }, - references: ['formulaX1'], + references: ['formulaX0'], }; expect( copyColumn({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts index 4a2e065269063a..2d7e70179fb3f7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts @@ -40,3 +40,28 @@ export const createMockedFullReference = () => { getErrorMessage: jest.fn(), }; }; + +export const createMockedManagedReference = () => { + return { + input: 'managedReference', + displayName: 'Managed reference test', + type: 'managedReference' as OperationType, + selectionStyle: 'full', + buildColumn: jest.fn((args) => { + return { + label: 'Test reference', + isBucketed: false, + dataType: 'number', + + operationType: 'testReference', + references: args.referenceIds, + }; + }), + filterable: true, + isTransferable: jest.fn(), + toExpression: jest.fn().mockReturnValue([]), + getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), + getDefaultLabel: jest.fn().mockReturnValue('Default label'), + getErrorMessage: jest.fn(), + }; +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 98dc767c44c7dd..f24c39f810b214 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -88,6 +88,8 @@ export interface IndexPatternPrivateState { isFirstExistenceFetch: boolean; existenceFetchFailed?: boolean; existenceFetchTimeout?: boolean; + + isDimensionClosePrevented?: boolean; } export interface IndexPatternRef { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 94b4433a825510..aded0dd478e724 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -213,6 +213,11 @@ export interface Datasource { } ) => { dropTypes: DropType[]; nextLabel?: string } | undefined; onDrop: (props: DatasourceDimensionDropHandlerProps) => false | true | { deleted: string }; + /** + * The datasource is allowed to cancel a close event on the dimension editor, + * mainly used for formulas + */ + canCloseDimensionEditor?: (state: T) => boolean; updateStateOnCloseDimension?: (props: { layerId: string; columnId: string; @@ -301,6 +306,8 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro core: Pick; dateRange: DateRange; dimensionGroups: VisualizationDimensionGroupConfig[]; + toggleFullscreen: () => void; + isFullscreen: boolean; }; export type DatasourceDimensionTriggerProps = DatasourceDimensionProps; diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts new file mode 100644 index 00000000000000..798cb7d3146f55 --- /dev/null +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + const find = getService('find'); + const listingTable = getService('listingTable'); + + describe('lens formula', () => { + it('should transition from count to formula', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('lnsXYvis'); + await PageObjects.lens.clickVisualizeListItemTitle('lnsXYvis'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-dimensionTrigger', + operation: 'average', + field: 'bytes', + keepOpen: true, + }); + + await PageObjects.lens.switchToFormula(); + await PageObjects.header.waitUntilLoadingHasFinished(); + // .echLegendItem__title is the only viable way of getting the xy chart's + // legend item(s), so we're using a class selector here. + // 4th item is the other bucket + expect(await find.allByCssSelector('.echLegendItem')).to.have.length(3); + }); + + it('should update and delete a formula', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count(kql=`, + keepOpen: true, + }); + + const input = await find.activeElement(); + await input.type('*'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); + }); + + it('should duplicate a moving average formula and be a valid table', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `moving_average(sum(bytes), window=5`, + keepOpen: true, + }); + await PageObjects.lens.closeDimensionEditor(); + + await PageObjects.lens.dragDimensionToDimension( + 'lnsDatatable_metrics > lns-dimensionTrigger', + 'lnsDatatable_metrics > lns-empty-dimension' + ); + expect(await PageObjects.lens.getDatatableCellText(1, 1)).to.eql('222420'); + expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('222420'); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index bfb0aad7177f4d..f6f162b51e84a3 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -39,6 +39,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./drag_and_drop')); loadTestFile(require.resolve('./lens_reporting')); loadTestFile(require.resolve('./lens_tagging')); + loadTestFile(require.resolve('./formula')); // has to be last one in the suite because it overrides saved objects loadTestFile(require.resolve('./rollup')); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d379..080e44da6ffcdf 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -107,6 +107,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont isPreviousIncompatible?: boolean; keepOpen?: boolean; palette?: string; + formula?: string; }, layerIndex = 0 ) { @@ -114,10 +115,15 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click(`lns-layerPanel-${layerIndex} > ${opts.dimension}`); await testSubjects.exists(`lns-indexPatternDimension-${opts.operation}`); }); - const operationSelector = opts.isPreviousIncompatible - ? `lns-indexPatternDimension-${opts.operation} incompatible` - : `lns-indexPatternDimension-${opts.operation}`; - await testSubjects.click(operationSelector); + + if (opts.operation === 'formula') { + await this.switchToFormula(); + } else { + const operationSelector = opts.isPreviousIncompatible + ? `lns-indexPatternDimension-${opts.operation} incompatible` + : `lns-indexPatternDimension-${opts.operation}`; + await testSubjects.click(operationSelector); + } if (opts.field) { const target = await testSubjects.find('indexPattern-dimension-field'); @@ -125,6 +131,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await comboBox.setElement(target, opts.field); } + if (opts.formula) { + await this.typeFormula(opts.formula); + } + if (opts.palette) { await testSubjects.click('lns-palettePicker'); await find.clickByCssSelector(`#${opts.palette}`); @@ -907,5 +917,18 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ); await PageObjects.header.waitUntilLoadingHasFinished(); }, + + async switchToFormula() { + await testSubjects.click('lens-dimensionTabs-formula'); + }, + + async typeFormula(formula: string) { + await find.byCssSelector('.monaco-editor'); + await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); + const input = await find.activeElement(); + await input.type(formula); + // Formula is applied on a 250ms timer, won't be applied if we leave too early + await PageObjects.common.sleep(500); + }, }); } From a33545e008655891f04eba6dde4ae64ec25645bd Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 11 May 2021 16:49:47 -0400 Subject: [PATCH 102/185] Tests for formula Co-authored-by: Marco Liberati --- .../definitions/formula/formula.test.tsx | 987 ++++++++++++++++++ 1 file changed, 987 insertions(+) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx new file mode 100644 index 00000000000000..4a511e14d59e00 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -0,0 +1,987 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockedIndexPattern } from '../../../mocks'; +import { formulaOperation, GenericOperationDefinition, IndexPatternColumn } from '../index'; +import { FormulaIndexPatternColumn } from './formula'; +import { regenerateLayerFromAst } from './parse'; +import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; +import { tinymathFunctions } from './util'; + +jest.mock('../../layer_helpers', () => { + return { + getColumnOrder: ({ columns }: { columns: Record }) => + Object.keys(columns), + }; +}); + +const operationDefinitionMap: Record = { + average: ({ + input: 'field', + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'average', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + terms: { input: 'field' } as GenericOperationDefinition, + sum: { input: 'field' } as GenericOperationDefinition, + last_value: { input: 'field' } as GenericOperationDefinition, + max: { input: 'field' } as GenericOperationDefinition, + count: ({ + input: 'field', + filterable: true, + buildColumn: ({ field }: { field: IndexPatternField }) => ({ + label: 'avg', + dataType: 'number', + operationType: 'count', + sourceField: field.name, + isBucketed: false, + scale: 'ratio', + timeScale: false, + }), + } as unknown) as GenericOperationDefinition, + derivative: { input: 'fullReference' } as GenericOperationDefinition, + moving_average: ({ + input: 'fullReference', + operationParams: [{ name: 'window', type: 'number', required: true }], + buildColumn: ({ references }: { references: string[] }) => ({ + label: 'moving_average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + timeScale: false, + params: { window: 5 }, + references, + }), + getErrorMessage: () => ['mock error'], + } as unknown) as GenericOperationDefinition, + cumulative_sum: { input: 'fullReference' } as GenericOperationDefinition, +}; + +describe('formula', () => { + let layer: IndexPatternLayer; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }; + }); + + describe('buildColumn', () => { + let indexPattern: IndexPattern; + + beforeEach(() => { + layer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Average', + dataType: 'number', + operationType: 'average', + isBucketed: false, + scale: 'ratio', + sourceField: 'bytes', + }, + }, + }; + indexPattern = createMockedIndexPattern(); + }); + + it('should start with an empty formula if no previous column is detected', () => { + expect( + formulaOperation.buildColumn({ + layer: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); + + it('should move into Formula previous operation', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: layer.columns.col1, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { isFormulaBroken: false, formula: 'average(bytes)' }, + references: [], + }); + }); + + it('it should move over explicit format param if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + params: { + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'average(bytes)', + format: { + id: 'number', + params: { + decimals: 2, + }, + }, + }, + references: [], + }); + }); + + it('it should move over kql arguments if set', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + filter: { + language: 'kuery', + // Need to test with multiple replaces due to string replace + query: `category.keyword: "Men's Clothing" or category.keyword: "Men's Shoes"`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `average(bytes, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, + }, + references: [], + }); + }); + + it('it should move over lucene arguments without', () => { + expect( + formulaOperation.buildColumn({ + previousColumn: { + ...layer.columns.col1, + operationType: 'count', + sourceField: 'Records', + filter: { + language: 'lucene', + query: `*`, + }, + } as IndexPatternColumn, + layer, + indexPattern, + }) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: `count(lucene='*')`, + }, + references: [], + }); + }); + + it('should move over previous operation parameter if set - only numeric', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Moving Average', + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: ['col2'], + timeScale: 'd', + params: { window: 3 }, + }, + col2: { + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'average', + scale: 'ratio', + sourceField: 'bytes', + timeScale: 'd', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { + isFormulaBroken: false, + formula: 'moving_average(average(bytes), window=3)', + }, + references: [], + }); + }); + + it('should not move previous column configuration if not numeric', () => { + expect( + formulaOperation.buildColumn( + { + previousColumn: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + layer: { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: { + label: 'Top value of category', + dataType: 'string', + isBucketed: true, + operationType: 'terms', + params: { + orderBy: { type: 'alphabetical' }, + size: 3, + orderDirection: 'asc', + }, + sourceField: 'category', + }, + }, + }, + indexPattern, + }, + {}, + operationDefinitionMap + ) + ).toEqual({ + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: {}, + references: [], + }); + }); + }); + + describe('regenerateLayerFromAst()', () => { + let indexPattern: IndexPattern; + let currentColumn: FormulaIndexPatternColumn; + + function testIsBrokenFormula(formula: string) { + expect( + regenerateLayerFromAst( + formula, + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).newLayer + ).toEqual({ + ...layer, + columns: { + ...layer.columns, + col1: { + ...currentColumn, + params: { + ...currentColumn.params, + formula, + isFormulaBroken: true, + }, + }, + }, + }); + } + + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + currentColumn = { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula: '', isFormulaBroken: false }, + references: [], + }; + }); + + it('should mutate the layer with new columns for valid formula expressions', () => { + expect( + regenerateLayerFromAst( + 'average(bytes)', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).newLayer + ).toEqual({ + ...layer, + columnOrder: ['col1X0', 'col1X1', 'col1'], + columns: { + ...layer.columns, + col1: { + ...currentColumn, + references: ['col1X1'], + params: { + ...currentColumn.params, + formula: 'average(bytes)', + isFormulaBroken: false, + }, + }, + col1X0: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X0', + operationType: 'average', + scale: 'ratio', + sourceField: 'bytes', + timeScale: false, + }, + col1X1: { + customLabel: true, + dataType: 'number', + isBucketed: false, + label: 'col1X1', + operationType: 'math', + params: { + tinymathAst: 'col1X0', + }, + references: ['col1X0'], + scale: 'ratio', + }, + }, + }); + }); + + it('returns no change but error if the formula cannot be parsed', () => { + const formulas = [ + '+', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', + 'average(bytes) + moving_average(average(bytes), window=)', + ]; + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if field is used with no Lens wrapping operation', () => { + testIsBrokenFormula('bytes'); + }); + + it('returns no change but error if at least one field in the formula is missing', () => { + const formulas = [ + 'noField', + 'average(noField)', + 'noField + 1', + 'derivative(average(noField))', + 'average(bytes) + derivative(average(noField))', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if at least one operation in the formula is missing', () => { + const formulas = [ + 'noFn()', + 'noFn(bytes)', + 'average(bytes) + noFn()', + 'derivative(noFn())', + 'noFn() + noFnTwo()', + 'noFn(noFnTwo())', + 'noFn() + noFnTwo() + 5', + 'average(bytes) + derivative(noFn())', + 'derivative(average(bytes) + noFn())', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if one operation has the wrong first argument', () => { + const formulas = [ + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', + 'derivative(7)', + 'derivative(bytes + 7)', + 'derivative(bytes + bytes)', + 'derivative(bytes + average(bytes))', + 'derivative(bytes + 7 + average(bytes))', + ]; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if an argument is passed to count operation', () => { + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; + + for (const formula of formulas) { + testIsBrokenFormula(formula); + } + }); + + it('returns no change but error if a required parameter is not passed to the operation in formula', () => { + const formula = 'moving_average(average(bytes))'; + testIsBrokenFormula(formula); + }); + + it('returns no change but error if a required parameter passed with the wrong type in formula', () => { + const formula = 'moving_average(average(bytes), window="m")'; + testIsBrokenFormula(formula); + }); + + it('returns error if a required parameter is passed multiple time', () => { + const formula = 'moving_average(average(bytes), window=7, window=3)'; + testIsBrokenFormula(formula); + }); + + it('returns error if a math operation has less arguments than required', () => { + const formula = 'pow(5)'; + testIsBrokenFormula(formula); + }); + + it('returns error if a math operation has the wrong argument type', () => { + const formula = 'pow(bytes)'; + testIsBrokenFormula(formula); + }); + + it('returns the locations of each function', () => { + expect( + regenerateLayerFromAst( + 'moving_average(average(bytes), window=7) + count()', + layer, + 'col1', + currentColumn, + indexPattern, + operationDefinitionMap + ).locations + ).toEqual({ + col1X0: { min: 15, max: 29 }, + col1X2: { min: 0, max: 41 }, + col1X3: { min: 43, max: 50 }, + }); + }); + }); + + describe('getErrorMessage', () => { + let indexPattern: IndexPattern; + + function getNewLayerWithFormula(formula: string, isBroken = true): IndexPatternLayer { + return { + columns: { + col1: { + label: 'Formula', + dataType: 'number', + operationType: 'formula', + isBucketed: false, + scale: 'ratio', + params: { formula, isFormulaBroken: isBroken }, + references: [], + }, + }, + columnOrder: [], + indexPatternId: '', + }; + } + beforeEach(() => { + indexPattern = createMockedIndexPattern(); + }); + + it('returns undefined if count is passed without arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('count()'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if count is passed with only a named argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='*')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns a syntax error if the kql argument does not parse', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='invalid: "')`, false), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + `Expected "(", "{", value, whitespace but """ found. +invalid: " +---------^`, + ]); + }); + + it('returns undefined if a field operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average(bytes)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + // note that field names can be wrapped in quotes as well + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average("bytes")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the correct first argument', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(average(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('derivative(average("bytes"))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns undefined if a fullReference operation is passed with the arguments', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), window=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns an error if field is used with no Lens wrapping operation', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The field bytes cannot be used without operation`]); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('bytes + bytes'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The operation add does not accept any field as argument`]); + }); + + it('returns an error if parsing a syntax invalid formula', () => { + const formulas = [ + '+', + 'average((', + 'average((bytes)', + 'average(bytes) +', + 'average(""', + 'moving_average(average(bytes), window=)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The Formula ${formula} cannot be parsed`]); + } + }); + + it('returns an error if the field is missing', () => { + const formulas = [ + 'noField', + 'average(noField)', + 'noField + 1', + 'derivative(average(noField))', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Field noField not found']); + } + }); + + it('returns an error with plural form correctly handled', () => { + const formulas = ['noField + noField2', 'noField + 1 + noField2']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Fields noField, noField2 not found']); + } + }); + + it('returns an error if an operation is unknown', () => { + const formulas = ['noFn()', 'noFn(bytes)', 'average(bytes) + noFn()', 'derivative(noFn())']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operation noFn not found']); + } + + const multipleFnFormulas = ['noFn() + noFnTwo()', 'noFn(noFnTwo())']; + + for (const formula of multipleFnFormulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operations noFn, noFnTwo not found']); + } + }); + + it('returns an error if field operation in formula have the wrong first argument', () => { + const formulas = [ + 'average(7)', + 'average()', + 'average(average(bytes))', + 'average(1 + 2)', + 'average(bytes + 5)', + 'average(bytes + bytes)', + 'derivative(7)', + ]; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual( + // some formulas may contain more errors + expect.arrayContaining([ + expect.stringMatching( + `The first argument for ${formula.substring(0, formula.indexOf('('))}` + ), + ]) + ); + } + }); + + it('returns an error if an argument is passed to count() operation', () => { + const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; + + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation count does not accept any field as argument']); + } + }); + + it('returns an error if an operation with required parameters does not receive them', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes))'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The operation moving_average in the Formula is missing the following parameters: window', + ]); + }); + + it('returns an error if a parameter is passed to an operation with no parameters', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('average(bytes, myparam=7)'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation average does not accept any parameter']); + }); + + it('returns an error if the parameter passed to an operation is of the wrong type', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('moving_average(average(bytes), window="m")'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([ + 'The parameters for the operation moving_average in the Formula are of the wrong type: window', + ]); + }); + + it('returns no error for the demo formula example', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(` + moving_average( + cumulative_sum( + 7 * clamp(sum(bytes), 0, last_value(memory) + max(memory)) + ), window=10 + ) + `), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + }); + + it('returns no error if a math operation is passed to fullReference operations', () => { + const formulas = [ + 'derivative(7+1)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + + it('returns errors if math operations are used with no arguments', () => { + const formulas = [ + 'derivative(7+1)', + 'derivative(7+average(bytes))', + 'moving_average(7+average(bytes), window=7)', + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + + // there are 4 types of errors for math functions: + // * no argument passed + // * too many arguments passed + // * field passed + // * missing argument + const errors = [ + (operation: string) => + `The first argument for ${operation} should be a operation name. Found ()`, + (operation: string) => `The operation ${operation} has too many arguments`, + (operation: string) => `The operation ${operation} does not accept any field as argument`, + (operation: string) => { + const required = tinymathFunctions[operation].positionalArguments.filter( + ({ optional }) => !optional + ); + return `The operation ${operation} in the Formula is missing ${ + required.length - 1 + } arguments: ${required + .slice(1) + .map(({ name }) => name) + .join(', ')}`; + }, + ]; + // we'll try to map all of these here in this test + for (const fn of Object.keys(tinymathFunctions)) { + it(`returns an error for the math functions available: ${fn}`, () => { + const nArgs = tinymathFunctions[fn].positionalArguments; + // start with the first 3 types + const formulas = [ + `${fn}()`, + `${fn}(1, 2, 3, 4, 5)`, + // to simplify a bit, add the required number of args by the function filled with the field name + `${fn}(${Array(nArgs.length).fill('bytes').join(', ')})`, + ]; + // add the fourth check only for those functions with more than 1 arg required + const enableFourthCheck = nArgs.filter(({ optional }) => !optional).length > 1; + if (enableFourthCheck) { + formulas.push(`${fn}(1)`); + } + formulas.forEach((formula, i) => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([errors[i](fn)]); + }); + }); + } + }); +}); From 1a74b65ef93ec11105e4eef6e944f043ff0c1576 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Wed, 12 May 2021 09:24:39 -0400 Subject: [PATCH 103/185] added error count placeholder --- .../definitions/formula/editor/formula.scss | 4 ++++ .../definitions/formula/editor/formula_editor.tsx | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 4e22cf2fd380e4..60102ac2311f23 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -50,6 +50,10 @@ } } +.lnsFormula__editorError { + white-space: nowrap; +} + .lnsFormula__docs { background: $euiColorEmptyShade; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index e8d212a32f7445..4b905b203e2384 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -15,6 +15,7 @@ import { EuiIcon, EuiLink, EuiPopover, + EuiText, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; @@ -531,10 +532,10 @@ export function FormulaEditor({ {isFullscreen ? ( - // TODO: Hook up the below `EuiLink` button so that it toggles the presence of the `.lnsFormula__docs--inline` element in fullscreen mode. Note that when docs are hidden, the `arrowDown` button should change to `arrowUp`. + // TODO: Hook up the below `EuiLink` button so that it toggles the presence of the `.lnsFormula__docs--inline` element in fullscreen mode. Note that when docs are hidden, the `arrowDown` button should change to `arrowUp` and the label should change to `Show function reference`. + {/* TODO: Hook up the below so that the error count conditionally appears to notify users of the number of errors in their formula. */} - Error count + + 1 error +
From c0577f0d29722f87f4daaeee619d5184a187ec48 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Wed, 12 May 2021 09:34:46 -0400 Subject: [PATCH 104/185] Add tooltips --- .../formula/editor/formula_editor.tsx | 96 ++++++++++++------- 1 file changed, 62 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 4b905b203e2384..1dec09824876d0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -16,6 +16,7 @@ import { EuiLink, EuiPopover, EuiText, + EuiToolTip, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; @@ -464,13 +465,21 @@ export function FormulaEditor({ {/* TODO: Replace `bolt` with `wordWrap` icon (after latest EUI is deployed) and hook up button to enable/disable word wrapping. */} - + delay="long" + position="top" + > + + @@ -533,40 +542,59 @@ export function FormulaEditor({ {isFullscreen ? ( // TODO: Hook up the below `EuiLink` button so that it toggles the presence of the `.lnsFormula__docs--inline` element in fullscreen mode. Note that when docs are hidden, the `arrowDown` button should change to `arrowUp` and the label should change to `Show function reference`. - - - - + + + + + ) : ( - setIsHelpOpen(false)} - button={ - setIsHelpOpen(!isHelpOpen)} - iconType="help" - color="text" - aria-label={i18n.translate('xpack.lens.formula.editorHelpOverlayLabel', { - defaultMessage: 'Function reference', - })} - /> - } + - - + setIsHelpOpen(false)} + button={ + setIsHelpOpen(!isHelpOpen)} + iconType="help" + color="text" + aria-label={i18n.translate( + 'xpack.lens.formula.editorHelpOverlayLabel', + { + defaultMessage: 'Function reference', + } + )} + /> + } + > + + + )} From 0b347ffca1aa00928a83a23d761af1c6e49af69c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 12 May 2021 14:43:24 -0400 Subject: [PATCH 105/185] Refactoring from code review --- .../definitions/calculations/counter_rate.tsx | 12 +--- .../calculations/cumulative_sum.tsx | 12 +--- .../definitions/calculations/differences.tsx | 12 +--- .../calculations/moving_average.tsx | 17 +++-- .../operations/definitions/cardinality.tsx | 17 +++-- .../operations/definitions/count.tsx | 12 +--- .../operations/definitions/formula/parse.ts | 24 +++---- .../definitions/formula/validation.ts | 63 +++++++++---------- .../operations/definitions/helpers.tsx | 15 +++++ .../operations/definitions/last_value.tsx | 17 +++-- .../operations/definitions/metrics.tsx | 17 +++-- .../definitions/percentile.test.tsx | 18 ++++++ .../operations/definitions/percentile.tsx | 3 +- .../operations/layer_helpers.ts | 4 -- 14 files changed, 114 insertions(+), 129 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 97582be2f32d66..fc9504f003198c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -17,7 +17,7 @@ import { } from './utils'; import { DEFAULT_TIME_SCALE } from '../../time_scale_utils'; import { OperationDefinition } from '..'; -import { getFormatFromPreviousColumn } from '../helpers'; +import { getFormatFromPreviousColumn, getFilter } from '../helpers'; const ofName = buildLabelFunction((name?: string) => { return i18n.translate('xpack.lens.indexPattern.CounterRateOf', { @@ -79,14 +79,6 @@ export const counterRateOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => { const metric = layer.columns[referenceIds[0]]; const timeScale = previousColumn?.timeScale || DEFAULT_TIME_SCALE; - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: ofName( metric && 'sourceField' in metric @@ -100,7 +92,7 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale, - filter, + filter: getFilter(previousColumn, columnParams), params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index e6f4f589f6189a..2adb9a1376f606 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -15,7 +15,7 @@ import { hasDateField, } from './utils'; import { OperationDefinition } from '..'; -import { getFormatFromPreviousColumn } from '../helpers'; +import { getFormatFromPreviousColumn, getFilter } from '../helpers'; const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { @@ -75,14 +75,6 @@ export const cumulativeSumOperation: OperationDefinition< }, buildColumn: ({ referenceIds, previousColumn, layer, indexPattern }, columnParams) => { const ref = layer.columns[referenceIds[0]]; - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: ofName( ref && 'sourceField' in ref @@ -93,7 +85,7 @@ export const cumulativeSumOperation: OperationDefinition< operationType: 'cumulative_sum', isBucketed: false, scale: 'ratio', - filter, + filter: getFilter(previousColumn, columnParams), references: referenceIds, params: getFormatFromPreviousColumn(previousColumn), }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index b030e604ada061..06555a9b41c2ff 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -17,7 +17,7 @@ import { } from './utils'; import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils'; import { OperationDefinition } from '..'; -import { getFormatFromPreviousColumn } from '../helpers'; +import { getFormatFromPreviousColumn, getFilter } from '../helpers'; const OPERATION_NAME = 'differences'; @@ -72,14 +72,6 @@ export const derivativeOperation: OperationDefinition< return dateBasedOperationToExpression(layer, columnId, 'derivative'); }, buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } const ref = layer.columns[referenceIds[0]]; return { label: ofName(ref?.label, previousColumn?.timeScale), @@ -89,7 +81,7 @@ export const derivativeOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter, + filter: getFilter(previousColumn, columnParams), params: getFormatFromPreviousColumn(previousColumn), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 88af8e9b6378e0..8d18a2752fd7e3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -19,7 +19,12 @@ import { hasDateField, } from './utils'; import { updateColumnParam } from '../../layer_helpers'; -import { getFormatFromPreviousColumn, isValidNumber, useDebounceWithOptions } from '../helpers'; +import { + getFormatFromPreviousColumn, + isValidNumber, + useDebounceWithOptions, + getFilter, +} from '../helpers'; import { adjustTimeScaleOnOtherColumnChange } from '../../time_scale_utils'; import { HelpPopover, HelpPopoverButton } from '../../../help_popover'; import type { OperationDefinition, ParamEditorProps } from '..'; @@ -88,14 +93,6 @@ export const movingAverageOperation: OperationDefinition< ) => { const metric = layer.columns[referenceIds[0]]; const { window = WINDOW_DEFAULT_VALUE } = columnParams; - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: ofName(metric?.label, previousColumn?.timeScale), dataType: 'number', @@ -104,7 +101,7 @@ export const movingAverageOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale: previousColumn?.timeScale, - filter, + filter: getFilter(previousColumn, columnParams), params: { window, ...getFormatFromPreviousColumn(previousColumn), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index df84ecb479de72..e77357a6f441a7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -11,7 +11,12 @@ import { buildExpressionFunction } from '../../../../../../../src/plugins/expres import { OperationDefinition } from './index'; import { FormattedIndexPatternColumn, FieldBasedIndexPatternColumn } from './column_types'; -import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers'; +import { + getFormatFromPreviousColumn, + getInvalidFieldMessage, + getSafeName, + getFilter, +} from './helpers'; const supportedTypes = new Set([ 'string', @@ -78,14 +83,6 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), buildColumn({ field, previousColumn }, columnParams) { - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: ofName(field.displayName), dataType: 'number', @@ -93,7 +90,7 @@ export const cardinalityOperation: OperationDefinition adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale), buildColumn({ field, previousColumn }, columnParams) { - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale), dataType: 'number', @@ -69,7 +61,7 @@ export const countOperation: OperationDefinition value); mathColumn.params.tinymathAst = root!; - const newColId = `${idPrefix}X${columns.length}`; + const newColId = getManagedId(idPrefix, columns.length); mathColumn.customLabel = true; mathColumn.label = newColId; columns.push({ column: mathColumn }); @@ -178,8 +182,8 @@ export function regenerateLayerFromAst( }); extracted.forEach(({ column, location }, index) => { - columns[`${columnId}X${index}`] = column; - if (location) locations[`${columnId}X${index}`] = location; + columns[getManagedId(columnId, index)] = column; + if (location) locations[getManagedId(columnId, index)] = location; }); columns[columnId] = { @@ -189,7 +193,7 @@ export function regenerateLayerFromAst( formula: text, isFormulaBroken: !isValid, }, - references: !isValid ? [] : [`${columnId}X${extracted.length - 1}`], + references: !isValid ? [] : [getManagedId(columnId, extracted.length - 1)], }; return { @@ -203,8 +207,4 @@ export function regenerateLayerFromAst( }, locations, }; - - // TODO - // turn ast into referenced columns - // set state } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 579b2822f49112..5145c7959f1bb5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -24,44 +24,43 @@ import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinitio import type { IndexPattern, IndexPatternLayer } from '../../../types'; import type { TinymathNodeTypes } from './types'; -const validationErrors = { - missingField: { message: 'missing field', type: { variablesLength: 1, variablesList: 'string' } }, +interface ValidationErrors { + missingField: { message: string; type: { variablesLength: number; variablesList: string } }; missingOperation: { - message: 'missing operation', - type: { operationLength: 1, operationsList: 'string' }, - }, + message: string; + type: { operationLength: number; operationsList: string }; + }; missingParameter: { - message: 'missing parameter', - type: { operation: 'string', params: 'string' }, - }, + message: string; + type: { operation: string; params: string }; + }; wrongTypeParameter: { - message: 'wrong type parameter', - type: { operation: 'string', params: 'string' }, - }, + message: string; + type: { operation: string; params: string }; + }; wrongFirstArgument: { - message: 'wrong first argument', - type: { operation: 'string', type: 'string', argument: 'any' as string | number }, - }, - cannotAcceptParameter: { message: 'cannot accept parameter', type: { operation: 'string' } }, - shouldNotHaveField: { message: 'operation should not have field', type: { operation: 'string' } }, - tooManyArguments: { message: 'too many arguments', type: { operation: 'string' } }, + message: string; + type: { operation: string; type: string; argument: string | number }; + }; + cannotAcceptParameter: { message: string; type: { operation: string } }; + shouldNotHaveField: { message: string; type: { operation: string } }; + tooManyArguments: { message: string; type: { operation: string } }; fieldWithNoOperation: { - message: 'unexpected field with no operation', - type: { field: 'string' }, - }, - failedParsing: { message: 'Failed to parse expression', type: { expression: 'string' } }, + message: string; + type: { field: string }; + }; + failedParsing: { message: string; type: { expression: string } }; duplicateArgument: { - message: 'duplicate argument', - type: { operation: 'string', params: 'string' }, - }, + message: string; + type: { operation: string; params: string }; + }; missingMathArgument: { - message: 'missing math argument', - type: { operation: 'string', count: 1, params: 'string' }, - }, -}; -export const errorsLookup = new Set(Object.values(validationErrors).map(({ message }) => message)); -type ErrorTypes = keyof typeof validationErrors; -type ErrorValues = typeof validationErrors[K]['type']; + message: string; + type: { operation: string; count: number; params: string }; + }; +} +type ErrorTypes = keyof ValidationErrors; +type ErrorValues = ValidationErrors[K]['type']; export interface ErrorWrapper { message: string; @@ -70,7 +69,7 @@ export interface ErrorWrapper { } export function isParsingError(message: string) { - return message.includes(validationErrors.failedParsing.message); + return message.includes('Failed to parse expression'); } function findFunctionNodes(root: TinymathAST | string): TinymathFunction[] { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx index 675a418c7cdc95..f719ac4250912e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/helpers.tsx @@ -123,3 +123,18 @@ export function getFormatFromPreviousColumn(previousColumn: IndexPatternColumn | ? { format: previousColumn.params.format } : undefined; } + +export function getFilter( + previousColumn: IndexPatternColumn | undefined, + columnParams: { kql?: string | undefined; lucene?: string | undefined } | undefined +) { + let filter = previousColumn?.filter; + if (columnParams) { + if ('kql' in columnParams) { + filter = { query: columnParams.kql ?? '', language: 'kuery' }; + } else if ('lucene' in columnParams) { + filter = { query: columnParams.lucene ?? '', language: 'lucene' }; + } + } + return filter; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index a61cca89dfecfc..4632d262c441d4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -15,7 +15,12 @@ import { FieldBasedIndexPatternColumn } from './column_types'; import { IndexPatternField, IndexPattern } from '../../types'; import { updateColumnParam } from '../layer_helpers'; import { DataType } from '../../../types'; -import { getFormatFromPreviousColumn, getInvalidFieldMessage, getSafeName } from './helpers'; +import { + getFormatFromPreviousColumn, + getInvalidFieldMessage, + getSafeName, + getFilter, +} from './helpers'; function ofName(name: string) { return i18n.translate('xpack.lens.indexPattern.lastValueOf', { @@ -154,14 +159,6 @@ export const lastValueOperation: OperationDefinition>({ getDefaultLabel: (column, indexPattern, columns) => labelLookup(getSafeName(column.sourceField, indexPattern), column), buildColumn: ({ field, previousColumn }, columnParams) => { - let filter = previousColumn?.filter; - if (columnParams) { - if ('kql' in columnParams) { - filter = { query: columnParams.kql ?? '', language: 'kuery' }; - } else if ('lucene' in columnParams) { - filter = { query: columnParams.lucene ?? '', language: 'lucene' }; - } - } return { label: labelLookup(field.displayName, previousColumn), dataType: 'number', @@ -106,7 +103,7 @@ function buildMetricOperation>({ isBucketed: false, scale: 'ratio', timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, - filter, + filter: getFilter(previousColumn, columnParams), params: getFormatFromPreviousColumn(previousColumn), } as T; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index a688f95e94c9ed..59da0f6f7bcdea 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -196,6 +196,24 @@ describe('percentile', () => { expect(percentileColumn.params.percentile).toEqual(75); expect(percentileColumn.label).toEqual('75th percentile of test'); }); + + it('should create a percentile from formula with filter', () => { + const indexPattern = createMockedIndexPattern(); + const bytesField = indexPattern.fields.find(({ name }) => name === 'bytes')!; + bytesField.displayName = 'test'; + const percentileColumn = percentileOperation.buildColumn( + { + indexPattern, + field: bytesField, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, + }, + { percentile: 75, kql: 'bytes > 100' } + ); + expect(percentileColumn.dataType).toEqual('number'); + expect(percentileColumn.params.percentile).toEqual(75); + expect(percentileColumn.filter).toEqual({ language: 'kuery', query: 'bytes > 100' }); + expect(percentileColumn.label).toEqual('75th percentile of test'); + }); }); describe('isTransferable', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 187dc2dc53ffb8..705a1f7172fff8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -17,6 +17,7 @@ import { getSafeName, isValidNumber, useDebounceWithOptions, + getFilter, } from './helpers'; import { FieldBasedIndexPatternColumn } from './column_types'; @@ -89,7 +90,7 @@ export const percentileOperation: OperationDefinition Date: Wed, 12 May 2021 18:31:40 -0400 Subject: [PATCH 106/185] Fix some editor issues --- .../dimension_panel/dimension_editor.tsx | 6 +- .../formula/editor/formula_editor.tsx | 57 +++++++++++++++---- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ec2bf270080919..c52c5b7a68b9de 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -250,6 +250,10 @@ export function DimensionEditor(props: DimensionEditorProps) { }`, [`aria-pressed`]: isActive, onClick() { + if (temporaryQuickFunction) { + setQuickFunction(false); + } + if ( operationDefinitionMap[operationType].input === 'none' || operationDefinitionMap[operationType].input === 'managedReference' || @@ -619,7 +623,7 @@ export function DimensionEditor(props: DimensionEditorProps) { quickFunctions )} - {!isFullscreen && !currentFieldIsInvalid && ( + {!isFullscreen && !currentFieldIsInvalid && !temporaryQuickFunction && (
{!incompleteInfo && selectedColumn && ( ) { const [text, setText] = useState(currentColumn.params.formula); + const [warnings, setWarnings] = useState>([]); const [isHelpOpen, setIsHelpOpen] = useState(false); const editorModel = React.useRef( monaco.editor.createModel(text ?? '', LANGUAGE_ID) @@ -94,6 +95,7 @@ export function FormulaEditor({ if (!editorModel.current) return; if (!text) { + setWarnings([]); monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); if (currentColumn.params.formula) { // Only submit if valid @@ -129,10 +131,8 @@ export function FormulaEditor({ } if (errors.length) { - monaco.editor.setModelMarkers( - editorModel.current, - 'LENS', - errors.flatMap((innerError) => { + const markers = errors + .flatMap((innerError) => { if (innerError.locations.length) { return innerError.locations.map((location) => { const startPosition = offsetToRowColumn(text, location.min); @@ -168,7 +168,10 @@ export function FormulaEditor({ ]; } }) - ); + .filter((marker) => marker); + + monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); + setWarnings(markers.map(({ severity }) => ({ severity }))); } else { monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); @@ -214,6 +217,7 @@ export function FormulaEditor({ return []; }) .filter((marker) => marker); + setWarnings(markers.map(({ severity }) => ({ severity }))); monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); } }, @@ -224,6 +228,12 @@ export function FormulaEditor({ [text] ); + const errorCount = warnings.filter((marker) => marker.severity === monaco.MarkerSeverity.Error) + .length; + const warningCount = warnings.filter( + (marker) => marker.severity === monaco.MarkerSeverity.Warning + ).length; + /** * The way that Monaco requests autocompletion is not intuitive, but the way we use it * we fetch new suggestions in these scenarios: @@ -478,6 +488,15 @@ export function FormulaEditor({ aria-label={i18n.translate('xpack.lens.formula.disableWordWrapLabel', { defaultMessage: 'Disable word wrap', })} + onClick={() => { + editor1.current?.updateOptions({ + wordWrap: + editor1.current?.getOption(monaco.editor.EditorOption.wordWrap) === + 'off' + ? 'on' + : 'off', + }); + }} /> @@ -598,12 +617,28 @@ export function FormulaEditor({ )} - {/* TODO: Hook up the below so that the error count conditionally appears to notify users of the number of errors in their formula. */} - - - 1 error - - + {errorCount || warningCount ? ( + + {errorCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaErrorCount', { + defaultMessage: '{count} {count, plural, one {error} other {errors}}', + values: { count: errorCount }, + })} + + ) : null} + {warningCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaWarningCount', { + defaultMessage: '{count} {count, plural, one {warning} other {warnings}}', + values: { count: warningCount }, + })} + + ) : null} + + ) : null}
From 628cf8d37323a0fc731ed3f69c1adb87f8c7f75f Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 13 May 2021 12:13:17 -0400 Subject: [PATCH 107/185] Update ID matching to match by name sometimes --- .../common/expression_functions/specs/map_column.ts | 8 ++++++-- .../specs/tests/map_column.test.ts | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index 7293510baa6b5d..cc10d9c980678f 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -103,10 +103,14 @@ export const mapColumn: ExpressionFunctionDefinition< return Promise.all(rowPromises).then((rows) => { const existingColumnIndex = columns.findIndex(({ id, name }) => { // Columns that have IDs are allowed to have duplicate names, for example esaggs + if (args.id) { + return id === args.id; + } + // If the column has an ID, but there is no ID argument to mapColumn if (id) { - return id === args.id && name === args.name; + return id === args.name; } - // If no ID, name is the unique key. For example, SQL output does not have IDs + // Columns without ID use name as the unique key. For example, SQL output does not have IDs return name === args.name; }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index 19dd60ab9469ea..235d67af99bc4d 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -35,6 +35,18 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); + it('matches name to id when mapColumn is called without an id', async () => { + const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(sqlTable.columns.length); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + }); + it('overwrites existing column with the new column if an existing column name is missing an id', async () => { const result = await runFn(sqlTable, { name: 'name', expression: pricePlusTwo }); const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); From 54217754fbd12dfdcdc1bf652ac86fba1e500b79 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 13 May 2021 17:48:37 -0400 Subject: [PATCH 108/185] Improve performance of Monaco, fix formulas with 0, update labels --- .../common/expression_functions/specs/map_column.ts | 6 +++--- .../kibana_react/public/code_editor/code_editor.tsx | 7 ++++++- .../definitions/formula/editor/formula.scss | 6 +++++- .../definitions/formula/editor/formula_editor.tsx | 12 +++++++++--- .../formula/editor/math_tokenization.tsx | 6 +++--- .../operations/definitions/formula/formula.tsx | 10 +++++----- .../operations/definitions/formula/parse.ts | 13 ++++++++++--- 7 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index dc19c81a99c1f6..3211d89b51f53e 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -10,7 +10,7 @@ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition } from '../types'; -import { Datatable, getType } from '../../expression_types'; +import { Datatable, DatatableColumn, getType } from '../../expression_types'; export interface MapColumnArguments { id?: string | null; @@ -110,10 +110,10 @@ export const mapColumn: ExpressionFunctionDefinition< return name === args.name; }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; - const newColumn = { + const newColumn: DatatableColumn = { id: columnId, name: args.name, - meta: { type }, + meta: { type, params: { id: type } }, }; if (args.copyMetaFrom) { const metaSourceFrom = columns.find(({ id }) => id === args.copyMetaFrom); diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index 55e10e7861e518..af00980c85284e 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -190,7 +190,12 @@ export class CodeEditor extends React.Component { ...options, }} /> - + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 60102ac2311f23..40cdbd58c3acf9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -35,9 +35,13 @@ display: block; // Overrides EUI's styling of `display: flex` on `EuiFlexItem` components } +.lnsFormula__editorContent { + height: 201px; +} + .lnsIndexPatternDimensionEditor-isFullscreen .lnsFormula__editorContent { flex: 1; - min-height: 0; + min-height: 201px; } .lnsFormula__editorHelp--inline { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 8ec75fd44394b7..615ca6623d57e1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -44,6 +44,8 @@ import './formula.scss'; import { FormulaIndexPatternColumn } from '../formula'; import { regenerateLayerFromAst } from '../parse'; +export const MemoizedFormulaEditor = React.memo(FormulaEditor); + export function FormulaEditor({ layer, updateLayer, @@ -434,7 +436,7 @@ export function FormulaEditor({ wordBasedSuggestions: false, autoIndent: 'brackets', wrappingIndent: 'none', - dimension: { width: 290, height: 200 }, + dimension: { width: 320, height: 200 }, fixedOverflowWidgets: true, }, }; @@ -537,12 +539,16 @@ export function FormulaEditor({ editor1.current = editor; disposables.current.push( editor.onDidFocusEditorWidget(() => { - setIsCloseable(false); + setTimeout(() => { + setIsCloseable(false); + }); }) ); disposables.current.push( editor.onDidBlurEditorWidget(() => { - setIsCloseable(true); + setTimeout(() => { + setIsCloseable(true); + }); }) ); // If we ever introduce a second Monaco editor, we need to toggle diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx index 51e96bad600439..17394560f8031c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_tokenization.tsx @@ -11,7 +11,7 @@ export const LANGUAGE_ID = 'lens_math'; monaco.languages.register({ id: LANGUAGE_ID }); export const languageConfiguration: monaco.languages.LanguageConfiguration = { - wordPattern: /[A-Za-z\.-_@]/g, + wordPattern: /[^()'"\s]+/g, brackets: [['(', ')']], autoClosingPairs: [ { open: '(', close: ')' }, @@ -34,9 +34,9 @@ export const lexerRules = { tokenizer: { root: [ [/\s+/, 'whitespace'], - [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'keyword'], - [/[,=]/, 'delimiter'], [/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number'], + [/[a-zA-Z0-9][a-zA-Z0-9_\-\.]*/, 'keyword'], + [/[,=:]/, 'delimiter'], // strings double quoted [/"([^"\\]|\\.)*$/, 'string.invalid'], // string without termination [/"/, 'string', '@string_dq'], diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6494c47548f2f6..24dab3fa47a006 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -10,7 +10,7 @@ import { OperationDefinition } from '../index'; import { ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPattern } from '../../../types'; import { runASTValidation, tryToParse } from './validation'; -import { FormulaEditor } from './editor'; +import { MemoizedFormulaEditor } from './editor'; import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; @@ -39,7 +39,7 @@ export const formulaOperation: OperationDefinition< > = { type: 'formula', displayName: defaultLabel, - getDefaultLabel: (column, indexPattern) => defaultLabel, + getDefaultLabel: (column, indexPattern) => column.params.formula ?? defaultLabel, input: 'managedReference', hidden: true, getDisabledStatus(indexPattern: IndexPattern) { @@ -115,12 +115,12 @@ export const formulaOperation: OperationDefinition< } // carry over the format settings from previous operation for seamless transfer // NOTE: this works only for non-default formatters set in Lens - let prevFormat = {}; + let prevFormat: FormulaIndexPatternColumn['params'] = {}; if (previousColumn?.params && 'format' in previousColumn.params) { prevFormat = { format: previousColumn.params.format }; } return { - label: 'Formula', + label: previousFormula ?? defaultLabel, dataType: 'number', operationType: 'formula', isBucketed: false, @@ -154,5 +154,5 @@ export const formulaOperation: OperationDefinition< return newLayer; }, - paramEditor: FormulaEditor, + paramEditor: MemoizedFormulaEditor, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index 3bfc6fcbfc011e..efdd2c9dfc8556 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { i18n } from '@kbn/i18n'; import { isObject } from 'lodash'; import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; import { OperationDefinition, GenericOperationDefinition, IndexPatternColumn } from '../index'; @@ -61,9 +62,9 @@ function extractColumns( const nodeOperation = operations[node.name]; if (!nodeOperation) { // it's a regular math node - const consumedArgs = node.args.map(parseNode).filter(Boolean) as Array< - number | TinymathVariable - >; + const consumedArgs = node.args + .map(parseNode) + .filter((n) => typeof n !== 'undefined' && n !== null) as Array; return { ...node, args: consumedArgs, @@ -188,6 +189,12 @@ export function regenerateLayerFromAst( columns[columnId] = { ...currentColumn, + label: !currentColumn.customLabel + ? text ?? + i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', + }) + : currentColumn.label, params: { ...currentColumn.params, formula: text, From 8f6a6a0f1a8a6b759047e036346927185cf52beb Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 14 May 2021 17:20:03 -0400 Subject: [PATCH 109/185] Improve performance of full screen toggle --- .../editor_frame/frame_layout.scss | 7 ++++ .../editor_frame/frame_layout.tsx | 33 +++++++++++-------- .../workspace_panel/workspace_panel.tsx | 10 ++---- .../dimension_panel/dimension_editor.scss | 2 -- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss index 7bc86b496f6267..282e69cd7636c7 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.scss @@ -88,6 +88,13 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ position: relative; } +.lnsFrameLayout-isFullscreen .lnsFrameLayout__sidebar--left, +.lnsFrameLayout-isFullscreen .lnsFrameLayout__suggestionPanel { + // Hide the datapanel and suggestions in fullscreen mode. Using display: none does trigger + // a rerender when the container becomes visible again, maybe pushing offscreen is better + display: none; +} + .lnsFrameLayout__sidebar--right { flex-basis: 25%; background-color: lightOrDarkTheme($euiColorLightestShade, $euiColorInk); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index a2aaf977cf6e6e..f27e0f9c24d7bd 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -22,24 +22,29 @@ export interface FrameLayoutProps { export function FrameLayout(props: FrameLayoutProps) { return ( - + - {!props.isFullscreen ? ( -
- -

- {i18n.translate('xpack.lens.section.dataPanelLabel', { - defaultMessage: 'Data panel', - })} -

- - {props.dataPanel} -
- ) : null} +
+ +

+ {i18n.translate('xpack.lens.section.dataPanelLabel', { + defaultMessage: 'Data panel', + })} +

+ + {props.dataPanel} +
{props.workspacePanel} - {!props.isFullscreen ? props.suggestionsPanel : null} +
{props.suggestionsPanel}
- {isFullscreen ? ( - element - ) : ( - - {element} - - )} + + {element} + ); }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index 44958451c294f3..3fb5e3c39498a6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -8,8 +8,6 @@ right: 0; top: 0; bottom: 0; - // display: flex; - // flex-direction: column; .lnsIndexPatternDimensionEditor__section { height: 100%; From a6a6fae1ef96f9423f423ca25f50c4d9bf877d86 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 14 May 2021 17:35:38 -0400 Subject: [PATCH 110/185] Fix formula tests --- .../operations/definitions/formula/formula.test.tsx | 12 +++++++----- .../operations/definitions/formula/formula.tsx | 2 +- .../operations/layer_helpers.test.ts | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 4a511e14d59e00..9862099c3b1ba5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -142,7 +142,7 @@ describe('formula', () => { indexPattern, }) ).toEqual({ - label: 'Formula', + label: 'average(bytes)', dataType: 'number', operationType: 'formula', isBucketed: false, @@ -170,7 +170,7 @@ describe('formula', () => { indexPattern, }) ).toEqual({ - label: 'Formula', + label: 'average(bytes)', dataType: 'number', operationType: 'formula', isBucketed: false, @@ -204,7 +204,7 @@ describe('formula', () => { indexPattern, }) ).toEqual({ - label: 'Formula', + label: `average(bytes, kql='category.keyword: "Men\\'s Clothing" or category.keyword: "Men\\'s Shoes"')`, dataType: 'number', operationType: 'formula', isBucketed: false, @@ -233,7 +233,7 @@ describe('formula', () => { indexPattern, }) ).toEqual({ - label: 'Formula', + label: `count(lucene='*')`, dataType: 'number', operationType: 'formula', isBucketed: false, @@ -291,7 +291,7 @@ describe('formula', () => { operationDefinitionMap ) ).toEqual({ - label: 'Formula', + label: 'moving_average(average(bytes), window=3)', dataType: 'number', operationType: 'formula', isBucketed: false, @@ -375,6 +375,7 @@ describe('formula', () => { ...layer.columns, col1: { ...currentColumn, + label: formula, params: { ...currentColumn.params, formula, @@ -415,6 +416,7 @@ describe('formula', () => { ...layer.columns, col1: { ...currentColumn, + label: 'average(bytes)', references: ['col1X1'], params: { ...currentColumn.params, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index ceade6092b0048..796a8ae97deda1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -120,7 +120,7 @@ export const formulaOperation: OperationDefinition< prevFormat = { format: previousColumn.params.format }; } return { - label: previousFormula ?? defaultLabel, + label: previousFormula || defaultLabel, dataType: 'number', operationType: 'formula', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index cc69d66079da08..25e46e5f77076c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -105,7 +105,7 @@ describe('state_helpers', () => { const source = { dataType: 'number' as const, isBucketed: false, - label: 'Formula', + label: 'moving_average(sum(bytes), window=5)', operationType: 'formula' as const, params: { formula: 'moving_average(sum(bytes), window=5)', @@ -117,7 +117,7 @@ describe('state_helpers', () => { customLabel: true, dataType: 'number' as const, isBucketed: false, - label: 'math', + label: 'formulaX2', operationType: 'math' as const, params: { tinymathAst: 'formulaX2' }, references: ['formulaX2'], From 507dd537881ec139fa7cfcf5e01e978a3ff99f85 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 14:02:00 +0200 Subject: [PATCH 111/185] fix stuff --- .../definitions/calculations/counter_rate.tsx | 7 +- .../calculations/cumulative_sum.tsx | 7 +- .../definitions/calculations/differences.tsx | 7 +- .../calculations/moving_average.tsx | 7 +- .../operations/definitions/cardinality.tsx | 13 +- .../operations/definitions/count.tsx | 13 +- .../definitions/formula/editor/formula.scss | 9 ++ .../formula/editor/formula_editor.tsx | 20 ++- .../formula/editor/formula_help.tsx | 132 +++++++++++------ .../operations/definitions/formula/util.ts | 140 +++++++++++++----- .../operations/definitions/last_value.tsx | 7 +- .../operations/definitions/metrics.tsx | 12 +- .../operations/definitions/percentile.tsx | 7 +- 13 files changed, 269 insertions(+), 112 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 8f88de2de6cd71..5efb4dbb44767f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -126,14 +126,17 @@ export const counterRateOperation: OperationDefinition< diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 06807f61f73169..5c26f61bd63906 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -119,13 +119,16 @@ export const cumulativeSumOperation: OperationDefinition< diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index dd4f0c8c76bba4..63e1b4bff648e3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -116,13 +116,16 @@ export const derivativeOperation: OperationDefinition< diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 2554a2bdd2c6cf..afef871ee733ae 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -140,7 +140,7 @@ export const movingAverageOperation: OperationDefinition< diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 16945bc1859882..8d990c7740ec54 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -117,12 +117,19 @@ export const cardinalityOperation: OperationDefinition diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 0544bc4f9f460b..ca1feed4c3af20 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -97,12 +97,19 @@ export const countOperation: OperationDefinition diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 40cdbd58c3acf9..5e97f592a04744 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -30,6 +30,12 @@ padding: $euiSizeS; } +.lnsFormula__editorFooter { + // make sure docs are rendered in front of monaco + z-index: 1; + background-color: $euiColorLightestShade; +} + .lnsFormula__editorHeaderGroup, .lnsFormula__editorFooterGroup { display: block; // Overrides EUI's styling of `display: flex` on `EuiFlexItem` components @@ -65,6 +71,8 @@ .lnsFormula__docs--inline { display: flex; flex-direction: column; + // make sure docs are rendered in front of monaco + z-index: 1; } .lnsFormula__docsContent { @@ -84,6 +92,7 @@ } .lnsFormula__docsNav { + @include euiYScroll; background: $euiColorLightestShade; padding: $euiSizeS; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 095e56e8460051..0366aeab2043e8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -60,7 +60,7 @@ export function FormulaEditor({ }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); const [warnings, setWarnings] = useState>([]); - const [isHelpOpen, setIsHelpOpen] = useState(false); + const [isHelpOpen, setIsHelpOpen] = useState(isFullscreen); const editorModel = React.useRef( monaco.editor.createModel(text ?? '', LANGUAGE_ID) ); @@ -566,11 +566,16 @@ export function FormulaEditor({ {isFullscreen ? ( - // TODO: Hook up the below `EuiLink` button so that it toggles the presence of the `.lnsFormula__docs--inline` element in fullscreen mode. Note that when docs are hidden, the `arrowDown` button should change to `arrowUp` and the label should change to `Show function reference`. @@ -580,9 +585,10 @@ export function FormulaEditor({ })} className="lnsFormula__editorHelp lnsFormula__editorHelp--inline" color="text" + onClick={() => setIsHelpOpen(!isHelpOpen)} > - + ) : ( @@ -650,7 +656,7 @@ export function FormulaEditor({
- {isFullscreen ? ( + {isFullscreen && isHelpOpen ? (
+ {i18n.translate('xpack.lens.formulaDocumentation.mathSectionDescription', { + defaultMessage: + 'These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.', + })} + + ), }); helpItems.push( ...getPossibleFunctions(indexPattern) .filter((key) => key in tinymathFunctions) + .sort() .map((key) => ({ label: `${key}`, description: , - checked: selectedFunction === key ? ('on' as const) : undefined, })) ); @@ -63,6 +74,14 @@ function FormulaHelp({ defaultMessage: 'Elasticsearch', }), isGroupLabel: true, + description: ( + + {i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSectionDescription', { + defaultMessage: + 'These functions will be executed on the raw documents for each row of the resulting table, aggregating all documents matching the break down dimensions into a single value.', + })} + + ), }); // Es aggs @@ -73,13 +92,10 @@ function FormulaHelp({ key in operationDefinitionMap && operationDefinitionMap[key].documentation?.section === 'elasticsearch' ) + .sort() .map((key) => ({ - label: `${key}: ${operationDefinitionMap[key].displayName}`, + label: key, description: operationDefinitionMap[key].documentation?.description, - checked: - selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` - ? ('on' as const) - : undefined, })) ); @@ -88,6 +104,14 @@ function FormulaHelp({ defaultMessage: 'Column-wise calculation', }), isGroupLabel: true, + description: ( + + {i18n.translate('xpack.lens.formulaDocumentation.columnCalculationSectionDescription', { + defaultMessage: + 'These functions will be executed for reach row of the resulting table, using data from cells from other rows as well as the current value.', + })} + + ), }); // Calculations aggs @@ -98,8 +122,9 @@ function FormulaHelp({ key in operationDefinitionMap && operationDefinitionMap[key].documentation?.section === 'calculation' ) + .sort() .map((key) => ({ - label: `${key}: ${operationDefinitionMap[key].displayName}`, + label: key, description: operationDefinitionMap[key].documentation?.description, checked: selectedFunction === `${key}: ${operationDefinitionMap[key].displayName}` @@ -111,32 +136,42 @@ function FormulaHelp({ return ( <> - Formula reference + {i18n.translate('xpack.lens.formulaDocumentation.header', { + defaultMessage: 'Formula reference', + })} - { - const chosenType = newOptions.find(({ checked }) => checked === 'on')!; - if (!chosenType) { - setSelectedFunction(undefined); + + {helpItems.map((helpItem) => { + if (helpItem.isGroupLabel) { + return ( + { + setSelectedFunction(helpItem.label); + }} + /> + ); } else { - setSelectedFunction(chosenType.label); + return ( + { + setSelectedFunction(helpItem.label); + }} + /> + ); } - }} - > - {(list, search) => ( - <> - {search} - {list} - - )} - + })} + @@ -174,30 +209,37 @@ queries. If your search has a single quote in it, use a backslash to escape, lik Math functions can take positional arguments, like pow(count(), 3) is the same as count() * count() * count() -### Basic math - Use the symbols +, -, /, and * to perform basic math. `, description: 'Text is in markdown. Do not translate function names or field names like sum(bytes)', })} /> + {helpItems.map((item, index) => { - if (item.isGroupLabel) { - return null; - } else { - return ( -
{ - if (el) { - scrollTargets.current[item.label] = el; - } - }} - > - {item.description} -
- ); - } + return ( +
{ + if (el) { + scrollTargets.current[item.label] = el; + } + }} + > + {item.isGroupLabel ? ( + +

{item.label}

+ {item.description} + + + ) : ( + + {item.description} + {helpItems.length - 1 !== index && } + + )} +
+ ); })} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 6b083e59593782..2f68522b1ef511 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -85,13 +85,20 @@ export const tinymathFunctions: Record< { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` -# add \`+\` +### add(summand1: number, summand2: number) \`+\` Adds up two numbers. Also works with + symbol -Example: Calculate the sum of two fields \`sum(price) + sum(tax)\` +Example: Calculate the sum of two fields +\`\`\` +sum(price) + sum(tax) +\`\`\` -Example: Offset count by a static value \`add(count(), 5)\` +Example: Offset count by a static value + +\`\`\` +add(count(), 5) +\`\`\` `, }, subtract: { @@ -100,11 +107,14 @@ Example: Offset count by a static value \`add(count(), 5)\` { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` -# subtract \`-\` +### subtract(minuend: number, subtrahend: number) \`-\` Subtracts the first number from the second number. Also works with ${'`-`'} symbol -Example: Calculate the range of a field ${'`subtract(max(bytes), min(bytes))`'} +Example: Calculate the range of a field +\`\`\` +subtract(max(bytes), min(bytes)) +\`\`\` `, }, multiply: { @@ -113,13 +123,19 @@ Example: Calculate the range of a field ${'`subtract(max(bytes), min(bytes))`'} { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` -# multiply \`*\` +### multiply(factor1: number, factor2: number) \`*\` Multiplies two numbers. Also works with ${'`*`'} symbol. -Example: Calculate price after current tax rate ${'`sum(bytes) * last_value(tax_rate)`'} +Example: Calculate price after current tax rate +\`\`\` +sum(bytes) * last_value(tax_rate) +\`\`\` -Example: Calculate price after constant tax rate \`multiply(sum(price), 1.2)\` +Example: Calculate price after constant tax rate +\`\`\` +multiply(sum(price), 1.2) +\`\`\` `, }, divide: { @@ -128,11 +144,14 @@ Example: Calculate price after constant tax rate \`multiply(sum(price), 1.2)\` { name: i18n.translate('xpack.lens.formula.right', { defaultMessage: 'right' }) }, ], help: ` -# divide \`/\` +### divide(dividend: number, divisor: number) \`/\` Divides the first number by the second number. Also works with ${'`/`'} symbol -Example: Calculate profit margin \`sum(profit) / sum(revenue)\` +Example: Calculate profit margin +\`\`\` +sum(profit) / sum(revenue) +\`\`\` `, }, abs: { @@ -140,7 +159,7 @@ Example: Calculate profit margin \`sum(profit) / sum(revenue)\` { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# abs +### abs(value: number) Calculates absolute value. A negative value is multiplied by -1, a positive value stays the same. Example: Calculate average distance to sea level ${'`abs(average(altitude))`'} @@ -151,10 +170,13 @@ Example: Calculate average distance to sea level ${'`abs(average(altitude))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# cbrt +### cbrt(value: number) Cube root of value. -Example: Calculate side length from volume ${'`cbrt(last_value(volume))`'} +Example: Calculate side length from volume +\`\`\` +cbrt(last_value(volume)) +\`\`\` `, }, ceil: { @@ -162,10 +184,13 @@ Example: Calculate side length from volume ${'`cbrt(last_value(volume))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# ceil +### ceil(value: number) Ceiling of value, rounds up. -Example: Round up price to the next dollar ${'`ceil(sum(price))`'} +Example: Round up price to the next dollar +\`\`\` +ceil(sum(price)) +\`\`\` `, }, clamp: { @@ -175,21 +200,31 @@ Example: Round up price to the next dollar ${'`ceil(sum(price))`'} { name: i18n.translate('xpack.lens.formula.max', { defaultMessage: 'max' }) }, ], help: ` -# clamp +### clamp(value: number, minimum: number, maximum: number) Limits the value from a minimum to maximum. -Example: Make sure to catch outliers ${'`clamp(average(bytes), percentile(bytes, percentile=5), percentile(bytes, percentile=95))`'} - `, +Example: Make sure to catch outliers +\`\`\` +clamp( + average(bytes), + percentile(bytes, percentile=5), + percentile(bytes, percentile=95) +) +\`\`\` +`, }, cube: { positionalArguments: [ { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# cube +### cube(value: number) Calculates the cube of a number. -Example: Calculate volume from side length ${'`cube(last_value(length))`'} +Example: Calculate volume from side length +\`\`\` +cube(last_value(length)) +\`\`\` `, }, exp: { @@ -197,10 +232,13 @@ Example: Calculate volume from side length ${'`cube(last_value(length))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# exp +### exp(value: number) Raises e to the nth power. -Example: Calculate the natural expontential function ${'`exp(last_value(duration))`'} +Example: Calculate the natural expontential function +\`\`\` +exp(last_value(duration)) +\`\`\` `, }, fix: { @@ -208,10 +246,13 @@ Example: Calculate the natural expontential function ${'`exp(last_value(duration { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# fix +### fix(value: number) For positive values, takes the floor. For negative values, takes the ceiling. -Example: Rounding towards zero ${'`fix(sum(profit))`'} +Example: Rounding towards zero +\`\`\` +fix(sum(profit)) +\`\`\` `, }, floor: { @@ -219,10 +260,13 @@ Example: Rounding towards zero ${'`fix(sum(profit))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# floor +### floor(value: number) Round down to nearest integer value -Example: Round down a price ${'`floor(sum(price))`'} +Example: Round down a price +\`\`\` +floor(sum(price)) +\`\`\` `, }, log: { @@ -234,10 +278,13 @@ Example: Round down a price ${'`floor(sum(price))`'} }, ], help: ` -# log +### log(value: number, base?: number) Logarithm with optional base. The natural base e is used as default. -Example: Calculate number of bits required to store values ${'`log(max(price), 2)`'} +Example: Calculate number of bits required to store values +\`\`\` +log(max(price), 2) +\`\`\` `, }, // TODO: check if this is valid for Tinymath @@ -259,24 +306,30 @@ Example: Calculate number of bits required to store values ${'`log(max(price), 2 }, ], help: ` -# mod +### mod(value: number) Remainder after dividing the function by a number -Example: Calculate last three digits of a value ${'`mod(sum(price), 1000)`'} +Example: Calculate last three digits of a value +\`\`\` +mod(sum(price), 1000) +\`\`\` `, }, pow: { positionalArguments: [ { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, { - name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), + name: i18n.translate('xpack.lens.\\formula.base', { defaultMessage: 'base' }), }, ], help: ` -# pow +### pow(value: number, power: number) Raises the value to a certain power. The second argument is required -Example: Calculate volume based on side length ${'`pow(last_value(length), 3)`'} +Example: Calculate volume based on side length +\`\`\` +pow(last_value(length), 3) +\`\`\` `, }, round: { @@ -288,10 +341,13 @@ Example: Calculate volume based on side length ${'`pow(last_value(length), 3)`'} }, ], help: ` -# round +### round(value: number, digits: number = 0) Rounds to a specific number of decimal places, default of 0 -Example: Round to the cent ${'`round(sum(price), 2)`'} +Example: Round to the cent +\`\`\` +round(sum(price), 2) +\`\`\` `, }, sqrt: { @@ -299,10 +355,13 @@ Example: Round to the cent ${'`round(sum(price), 2)`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# sqrt +### sqrt(value: number) Square root of a positive value only -Example: Calculate side length based on area ${'`sqrt(last_value(area))`'} +Example: Calculate side length based on area +\`\`\` +sqrt(last_value(area)) +\`\`\` `, }, square: { @@ -310,10 +369,13 @@ Example: Calculate side length based on area ${'`sqrt(last_value(area))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -# square +### square(value: number) Raise the value to the 2nd power -Example: Calculate area based on side length ${'`square(last_value(length))`'} +Example: Calculate area based on side length +\`\`\` +square(last_value(length)) +\`\`\` `, }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 8df92254b505f4..5ec17b6ac3e6f5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -272,13 +272,16 @@ export const lastValueOperation: OperationDefinition diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 6a3856feb0d2b3..2c5bda1d2870d8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -133,13 +133,19 @@ function buildMetricOperation>({ From 468cfcf0049d4883ba289fab44e103ffb3eda9fd Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 18 May 2021 13:40:12 -0400 Subject: [PATCH 112/185] Add an extra case to prevent insertion of duplicate column --- .../expression_functions/specs/map_column.ts | 7 +- .../specs/tests/map_column.test.ts | 89 ++++++++++++------- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index cc10d9c980678f..685e2d54404a72 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -53,7 +53,7 @@ export const mapColumn: ExpressionFunctionDefinition< types: ['string'], aliases: ['_', 'column'], help: i18n.translate('expressions.functions.mapColumn.args.nameHelpText', { - defaultMessage: 'The name of the resulting column.', + defaultMessage: 'The name of the resulting column. Names are not required to be unique.', }), required: true, }, @@ -102,8 +102,11 @@ export const mapColumn: ExpressionFunctionDefinition< return Promise.all(rowPromises).then((rows) => { const existingColumnIndex = columns.findIndex(({ id, name }) => { - // Columns that have IDs are allowed to have duplicate names, for example esaggs if (args.id) { + if (!id) { + return name === args.id; + } + // Columns that have IDs are allowed to have duplicate names, for example esaggs return id === args.id; } // If the column has an ID, but there is no ID argument to mapColumn diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index 235d67af99bc4d..4efc5701951ed1 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -35,45 +35,66 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); - it('matches name to id when mapColumn is called without an id', async () => { - const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(sqlTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); - }); - - it('overwrites existing column with the new column if an existing column name is missing an id', async () => { - const result = await runFn(sqlTable, { name: 'name', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - const arbitraryRowIndex = 4; + describe('when the table columns have id', () => { + it('does not require the id arg by using the name arg as column id', async () => { + const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(sqlTable.columns.length); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + }); - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(sqlTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + it('allows a duplicate name when the ids are different', async () => { + const result = await runFn(testTable, { + id: 'new', + name: 'name label', + expression: pricePlusTwo, + }); + const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length + 1); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); + }); }); - it('inserts a new column with a duplicate name if an id and name are provided', async () => { - const result = await runFn(testTable, { - id: 'new', - name: 'name label', - expression: pricePlusTwo, + describe('when the table columns do not have id', () => { + it('uses name as unique key when id arg is also missing', async () => { + const result = await runFn(sqlTable, { name: 'name', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(sqlTable.columns.length); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); }); - const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); - const arbitraryRowIndex = 4; - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(testTable.columns.length + 1); - expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); + it('overwrites columns matching id === name when the column is missing an id', async () => { + const result = await runFn(sqlTable, { + id: 'name', + name: 'name is ignored', + expression: pricePlusTwo, + }); + const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name is ignored'); + const arbitraryRowIndex = 4; + + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(sqlTable.columns.length); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'name'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name is ignored'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + }); }); it('adds a column to empty tables', async () => { diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ae61f24201ce59..4e11339f44c414 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -2022,7 +2022,6 @@ "expressions.functions.mapColumn.args.copyMetaFromHelpText": "設定されている場合、指定した列IDのメタオブジェクトが指定したターゲット列にコピーされます。列が存在しない場合は失敗し、エラーは表示されません。", "expressions.functions.mapColumn.args.expressionHelpText": "すべての行で実行される式。単一行の{DATATABLE}と一緒に指定され、セル値を返します。", "expressions.functions.mapColumn.args.idHelpText": "結果列の任意のID。「null」の場合、name/column引数がIDとして使用されます。", - "expressions.functions.mapColumn.args.nameHelpText": "結果の列の名前です。", "expressions.functions.mapColumnHelpText": "他の列の結果として計算された列を追加します。引数が指定された場合のみ変更が加えられます。{alterColumnFn}と{staticColumnFn}もご参照ください。", "expressions.functions.math.args.expressionHelpText": "評価された {TINYMATH} 表現です。{TINYMATH_URL} をご覧ください。", "expressions.functions.math.args.onErrorHelpText": "{TINYMATH}評価が失敗するか、NaNが返される場合、戻り値はonErrorで指定されます。「'throw'」の場合、例外が発生し、式の実行が終了します (デフォルト) 。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index ecd6c0d68a94fb..2c886f2f1ce9f2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -2034,7 +2034,6 @@ "expressions.functions.mapColumn.args.copyMetaFromHelpText": "如果设置，指定列 ID 的元对象将复制到指定目标列。如果列不存在，复制将无提示失败。", "expressions.functions.mapColumn.args.expressionHelpText": "在每行上执行的表达式，为其提供了单行 {DATATABLE} 上下文，其将返回单元格值。", "expressions.functions.mapColumn.args.idHelpText": "结果列的可选 ID。如果为 `null`，名称/列参数将用作 ID。", - "expressions.functions.mapColumn.args.nameHelpText": "结果列的名称。", "expressions.functions.mapColumnHelpText": "添加计算为其他列的结果的列。只有提供参数时，才会执行更改。另请参见 {alterColumnFn} 和 {staticColumnFn}。", "expressions.functions.math.args.expressionHelpText": "已计算的 {TINYMATH} 表达式。请参阅 {TINYMATH_URL}。", "expressions.functions.math.args.onErrorHelpText": "如果 {TINYMATH} 评估失败或返回 NaN，返回值将由 onError 指定。为 `'throw'` 时，其将引发异常，从而终止表达式执行 (默认) 。", From 9f4d6474708ba8f11375fc81893cc153bb87ba7c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 18 May 2021 16:55:41 -0400 Subject: [PATCH 113/185] Simplify logic and add test for output ID --- .../expression_functions/specs/map_column.ts | 30 +++---- .../specs/tests/map_column.test.ts | 81 ++++++------------- .../specs/tests/math.test.ts | 11 +-- .../expression_functions/specs/tests/utils.ts | 32 +------- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 6 files changed, 40 insertions(+), 116 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index 685e2d54404a72..e6ce5ab04386ec 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -44,7 +44,7 @@ export const mapColumn: ExpressionFunctionDefinition< types: ['string', 'null'], help: i18n.translate('expressions.functions.mapColumn.args.idHelpText', { defaultMessage: - 'An optional id of the resulting column. When `null` the name/column argument is used as id.', + 'An optional id of the resulting column. When no id is provided, the id will be looked up from the existing column and default to the name.', }), required: false, default: null, @@ -86,9 +86,18 @@ export const mapColumn: ExpressionFunctionDefinition< .expression?.(...params) .pipe(take(1)) .toPromise() ?? Promise.resolve(null); - const columnId = args.id != null ? args.id : args.name; const columns = [...input.columns]; + const existingColumnIndex = columns.findIndex(({ id, name }) => { + if (args.id) { + return id === args.id; + } + return name === args.name; + }); + const columnLength = columns.length; + const columnId = + existingColumnIndex === -1 ? args.id ?? args.name : columns[existingColumnIndex].id; + const rowPromises = input.rows.map((row) => { return expression({ type: 'datatable', @@ -101,21 +110,6 @@ export const mapColumn: ExpressionFunctionDefinition< }); return Promise.all(rowPromises).then((rows) => { - const existingColumnIndex = columns.findIndex(({ id, name }) => { - if (args.id) { - if (!id) { - return name === args.id; - } - // Columns that have IDs are allowed to have duplicate names, for example esaggs - return id === args.id; - } - // If the column has an ID, but there is no ID argument to mapColumn - if (id) { - return id === args.name; - } - // Columns without ID use name as the unique key. For example, SQL output does not have IDs - return name === args.name; - }); const type = rows.length ? getType(rows[0][columnId]) : 'null'; const newColumn = { id: columnId, @@ -128,7 +122,7 @@ export const mapColumn: ExpressionFunctionDefinition< } if (existingColumnIndex === -1) { - columns.push(newColumn); + columns[columnLength] = newColumn; } else { columns[existingColumnIndex] = newColumn; } diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts index 4efc5701951ed1..f5c1f3838f66c7 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/map_column.test.ts @@ -9,7 +9,7 @@ import { of } from 'rxjs'; import { Datatable } from '../../../expression_types'; import { mapColumn, MapColumnArguments } from '../map_column'; -import { emptyTable, functionWrapper, testTable, sqlTable } from './utils'; +import { emptyTable, functionWrapper, testTable } from './utils'; const pricePlusTwo = (datatable: Datatable) => of(datatable.rows[0].price + 2); @@ -35,66 +35,35 @@ describe('mapColumn', () => { expect(result.rows[arbitraryRowIndex]).toHaveProperty('pricePlusTwo'); }); - describe('when the table columns have id', () => { - it('does not require the id arg by using the name arg as column id', async () => { - const result = await runFn(testTable, { name: 'name', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(sqlTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); - }); + it('allows the id arg to be optional, looking up by name instead', async () => { + const result = await runFn(testTable, { name: 'name label', expression: pricePlusTwo }); + const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name label'); + const arbitraryRowIndex = 4; - it('allows a duplicate name when the ids are different', async () => { - const result = await runFn(testTable, { - id: 'new', - name: 'name label', - expression: pricePlusTwo, - }); - const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(testTable.columns.length + 1); - expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); - }); + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'name'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + expect(result.rows[arbitraryRowIndex]).not.toHaveProperty('name label'); }); - describe('when the table columns do not have id', () => { - it('uses name as unique key when id arg is also missing', async () => { - const result = await runFn(sqlTable, { name: 'name', expression: pricePlusTwo }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(sqlTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); + it('allows a duplicate name when the ids are different', async () => { + const result = await runFn(testTable, { + id: 'new', + name: 'name label', + expression: pricePlusTwo, }); + const nameColumnIndex = result.columns.findIndex(({ id }) => id === 'new'); + const arbitraryRowIndex = 4; - it('overwrites columns matching id === name when the column is missing an id', async () => { - const result = await runFn(sqlTable, { - id: 'name', - name: 'name is ignored', - expression: pricePlusTwo, - }); - const nameColumnIndex = result.columns.findIndex(({ name }) => name === 'name is ignored'); - const arbitraryRowIndex = 4; - - expect(result.type).toBe('datatable'); - expect(result.columns).toHaveLength(sqlTable.columns.length); - expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'name'); - expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name is ignored'); - expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); - expect(result.rows[arbitraryRowIndex]).toHaveProperty('name', 202); - }); + expect(result.type).toBe('datatable'); + expect(result.columns).toHaveLength(testTable.columns.length + 1); + expect(result.columns[nameColumnIndex]).toHaveProperty('id', 'new'); + expect(result.columns[nameColumnIndex]).toHaveProperty('name', 'name label'); + expect(result.columns[nameColumnIndex].meta).toHaveProperty('type', 'number'); + expect(result.rows[arbitraryRowIndex]).toHaveProperty('new', 202); }); it('adds a column to empty tables', async () => { diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts index 5ad23fc24c91ba..9eac0911afafd0 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts @@ -7,7 +7,7 @@ */ import { errors, math } from '../math'; -import { emptyTable, functionWrapper, testTable, sqlTable } from './utils'; +import { emptyTable, functionWrapper, testTable } from './utils'; describe('math', () => { const fn = functionWrapper(math); @@ -36,15 +36,6 @@ describe('math', () => { expect(fn(testTable, { expression: 'max(price)' })).toBe(605); }); - it('evaluates math expressions with references to columns in a datatable', () => { - expect(fn(sqlTable, { expression: 'unique(in_stock)' })).toBe(2); - expect(fn(sqlTable, { expression: 'sum(quantity)' })).toBe(2508); - expect(fn(sqlTable, { expression: 'mean(price)' })).toBe(320); - expect(fn(sqlTable, { expression: 'min(price)' })).toBe(67); - expect(fn(sqlTable, { expression: 'median(quantity)' })).toBe(256); - expect(fn(sqlTable, { expression: 'max(price)' })).toBe(605); - }); - describe('args', () => { describe('expression', () => { it('sets the math expression to be evaluted', () => { diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts b/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts index a6501cb37b5a67..60d22d2b8575cb 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/utils.ts @@ -9,7 +9,7 @@ import { mapValues } from 'lodash'; import { AnyExpressionFunctionDefinition } from '../../types'; import { ExecutionContext } from '../../../execution/types'; -import { Datatable, DatatableColumn } from '../../../expression_types'; +import { Datatable } from '../../../expression_types'; /** * Takes a function spec and passes in default args, @@ -224,32 +224,4 @@ const stringTable: Datatable = { ], }; -// Emulates a SQL table that doesn't have any IDs -const sqlTable: Datatable = { - type: 'datatable', - columns: [ - ({ - name: 'name', - meta: { type: 'string' }, - } as unknown) as DatatableColumn, - ({ - name: 'time', - meta: { type: 'date' }, - } as unknown) as DatatableColumn, - ({ - name: 'price', - meta: { type: 'number' }, - } as unknown) as DatatableColumn, - ({ - name: 'quantity', - meta: { type: 'number' }, - } as unknown) as DatatableColumn, - ({ - name: 'in_stock', - meta: { type: 'boolean' }, - } as unknown) as DatatableColumn, - ], - rows: [...testTable.rows], -}; - -export { emptyTable, testTable, stringTable, sqlTable }; +export { emptyTable, testTable, stringTable }; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 0ba3d2eb954802..7ba8c9eaab4f58 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -2021,7 +2021,6 @@ "expressions.functions.fontHelpText": "フォントスタイルを作成します。", "expressions.functions.mapColumn.args.copyMetaFromHelpText": "設定されている場合、指定した列IDのメタオブジェクトが指定したターゲット列にコピーされます。列が存在しない場合は失敗し、エラーは表示されません。", "expressions.functions.mapColumn.args.expressionHelpText": "すべての行で実行される式。単一行の{DATATABLE}と一緒に指定され、セル値を返します。", - "expressions.functions.mapColumn.args.idHelpText": "結果列の任意のID。「null」の場合、name/column引数がIDとして使用されます。", "expressions.functions.mapColumnHelpText": "他の列の結果として計算された列を追加します。引数が指定された場合のみ変更が加えられます。{alterColumnFn}と{staticColumnFn}もご参照ください。", "expressions.functions.math.args.expressionHelpText": "評価された {TINYMATH} 表現です。{TINYMATH_URL} をご覧ください。", "expressions.functions.math.args.onErrorHelpText": "{TINYMATH}評価が失敗するか、NaNが返される場合、戻り値はonErrorで指定されます。「'throw'」の場合、例外が発生し、式の実行が終了します (デフォルト) 。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index e068cec32a80bf..1ee38ab13ea173 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -2033,7 +2033,6 @@ "expressions.functions.fontHelpText": "创建字体样式。", "expressions.functions.mapColumn.args.copyMetaFromHelpText": "如果设置，指定列 ID 的元对象将复制到指定目标列。如果列不存在，复制将无提示失败。", "expressions.functions.mapColumn.args.expressionHelpText": "在每行上执行的表达式，为其提供了单行 {DATATABLE} 上下文，其将返回单元格值。", - "expressions.functions.mapColumn.args.idHelpText": "结果列的可选 ID。如果为 `null`，名称/列参数将用作 ID。", "expressions.functions.mapColumnHelpText": "添加计算为其他列的结果的列。只有提供参数时，才会执行更改。另请参见 {alterColumnFn} 和 {staticColumnFn}。", "expressions.functions.math.args.expressionHelpText": "已计算的 {TINYMATH} 表达式。请参阅 {TINYMATH_URL}。", "expressions.functions.math.args.onErrorHelpText": "如果 {TINYMATH} 评估失败或返回 NaN，返回值将由 onError 指定。为 `'throw'` 时，其将引发异常，从而终止表达式执行 (默认) 。", From c0d28a9c485084734fea724f8b2d3b8cf91df940 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 18 May 2021 23:10:50 +0200 Subject: [PATCH 114/185] add telemetry for Lens formula (#15) --- .../server/usage/dashboard_telemetry.test.ts | 17 +++++++++ .../server/usage/dashboard_telemetry.ts | 23 ++++++++++++ .../formula/editor/formula_editor.tsx | 2 ++ x-pack/plugins/lens/server/usage/schema.ts | 11 ++++++ .../lens/server/usage/visualization_counts.ts | 36 ++++++++++++++++--- .../schema/xpack_plugins.json | 33 +++++++++++++++++ 6 files changed, 118 insertions(+), 4 deletions(-) diff --git a/src/plugins/dashboard/server/usage/dashboard_telemetry.test.ts b/src/plugins/dashboard/server/usage/dashboard_telemetry.test.ts index 1cfa9d862e6b95..60f1f7eb0955c0 100644 --- a/src/plugins/dashboard/server/usage/dashboard_telemetry.test.ts +++ b/src/plugins/dashboard/server/usage/dashboard_telemetry.test.ts @@ -72,6 +72,22 @@ const lensXYSeriesB = ({ visualization: { preferredSeriesType: 'seriesB', }, + datasourceStates: { + indexpattern: { + layers: { + first: { + columns: { + first: { + operationType: 'terms', + }, + second: { + operationType: 'formula', + }, + }, + }, + }, + }, + }, }, }, }, @@ -144,6 +160,7 @@ describe('dashboard telemetry', () => { expect(collectorData.lensByValue.a).toBe(3); expect(collectorData.lensByValue.seriesA).toBe(2); expect(collectorData.lensByValue.seriesB).toBe(1); + expect(collectorData.lensByValue.formula).toBe(1); }); it('handles misshapen lens panels', () => { diff --git a/src/plugins/dashboard/server/usage/dashboard_telemetry.ts b/src/plugins/dashboard/server/usage/dashboard_telemetry.ts index 02d492de4fe666..714a4092d78b85 100644 --- a/src/plugins/dashboard/server/usage/dashboard_telemetry.ts +++ b/src/plugins/dashboard/server/usage/dashboard_telemetry.ts @@ -27,6 +27,16 @@ interface LensPanel extends SavedDashboardPanel730ToLatest { visualization?: { preferredSeriesType?: string; }; + datasourceStates?: { + indexpattern?: { + layers: Record< + string, + { + columns: Record; + } + >; + }; + }; }; }; }; @@ -105,6 +115,19 @@ export const collectByValueLensInfo: DashboardCollectorFunction = (panels, colle } collectorData.lensByValue[type] = collectorData.lensByValue[type] + 1; + + const hasFormula = Object.values( + lensPanel.embeddableConfig.attributes.state?.datasourceStates?.indexpattern?.layers || {} + ).some((layer) => + Object.values(layer.columns).some((column) => column.operationType === 'formula') + ); + + if (hasFormula && !collectorData.lensByValue.formula) { + collectorData.lensByValue.formula = 0; + } + if (hasFormula) { + collectorData.lensByValue.formula++; + } } } }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 615ca6623d57e1..e4d255f50196ff 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -39,6 +39,7 @@ import { } from './math_completion'; import { LANGUAGE_ID } from './math_tokenization'; import { MemoizedFormulaHelp } from './formula_help'; +import { trackUiEvent } from '../../../../../lens_ui_telemetry'; import './formula.scss'; import { FormulaIndexPatternColumn } from '../formula'; @@ -508,6 +509,7 @@ export function FormulaEditor({ { toggleFullscreen(); + trackUiEvent('toggle_formula_fullscreen'); }} iconType={isFullscreen ? 'bolt' : 'fullScreen'} size="xs" diff --git a/x-pack/plugins/lens/server/usage/schema.ts b/x-pack/plugins/lens/server/usage/schema.ts index ab3945a0162a68..62b39ee6793ee2 100644 --- a/x-pack/plugins/lens/server/usage/schema.ts +++ b/x-pack/plugins/lens/server/usage/schema.ts @@ -14,6 +14,12 @@ const eventsSchema: MakeSchemaFrom = { type: 'long', _meta: { description: 'Number of times the user opened one of the in-product help popovers.' }, }, + toggle_fullscreen_formula: { + type: 'long', + _meta: { + description: 'Number of times the user toggled fullscreen mode on formula.', + }, + }, indexpattern_field_info_click: { type: 'long' }, loaded: { type: 'long' }, app_filters_updated: { type: 'long' }, @@ -162,6 +168,10 @@ const eventsSchema: MakeSchemaFrom = { type: 'long', _meta: { description: 'Number of times the moving average function was selected' }, }, + indexpattern_dimension_operation_formula: { + type: 'long', + _meta: { description: 'Number of times the formula function was selected' }, + }, }; const suggestionEventsSchema: MakeSchemaFrom = { @@ -183,6 +193,7 @@ const savedSchema: MakeSchemaFrom = { lnsDatatable: { type: 'long' }, lnsPie: { type: 'long' }, lnsMetric: { type: 'long' }, + formula: { type: 'long' }, }; export const lensUsageSchema: MakeSchemaFrom = { diff --git a/x-pack/plugins/lens/server/usage/visualization_counts.ts b/x-pack/plugins/lens/server/usage/visualization_counts.ts index 3b9bb99caf5b81..f0c48fb1152e81 100644 --- a/x-pack/plugins/lens/server/usage/visualization_counts.ts +++ b/x-pack/plugins/lens/server/usage/visualization_counts.ts @@ -43,6 +43,31 @@ export async function getVisualizationCounts( size: 100, }, }, + usesFormula: { + filter: { + match: { + operation_type: 'formula', + }, + }, + }, + }, + }, + }, + runtime_mappings: { + operation_type: { + type: 'keyword', + script: { + lang: 'painless', + source: `try { + if(doc['lens.state'].size() == 0) return; + HashMap layers = params['_source'].get('lens').get('state').get('datasourceStates').get('indexpattern').get('layers'); + for(layerId in layers.keySet()) { + HashMap columns = layers.get(layerId).get('columns'); + for(columnId in columns.keySet()) { + emit(columns.get(columnId).get('operationType')) + } + } + } catch(Exception e) {}`, }, }, }, @@ -56,16 +81,19 @@ export async function getVisualizationCounts( // eslint-disable-next-line @typescript-eslint/no-explicit-any function bucketsToObject(arg: any) { const obj: Record = {}; - arg.buckets.forEach((bucket: { key: string; doc_count: number }) => { + arg.byType.buckets.forEach((bucket: { key: string; doc_count: number }) => { obj[bucket.key] = bucket.doc_count + (obj[bucket.key] ?? 0); }); + if (arg.usesFormula.doc_count > 0) { + obj.formula = arg.usesFormula.doc_count; + } return obj; } return { - saved_overall: bucketsToObject(buckets.overall.byType), - saved_30_days: bucketsToObject(buckets.last30.byType), - saved_90_days: bucketsToObject(buckets.last90.byType), + saved_overall: bucketsToObject(buckets.overall), + saved_30_days: bucketsToObject(buckets.last30), + saved_90_days: bucketsToObject(buckets.last90), saved_overall_total: buckets.overall.doc_count, saved_30_days_total: buckets.last30.doc_count, saved_90_days_total: buckets.last90.doc_count, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 5495ef10f223af..998bf7fa8b6c4c 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -2110,6 +2110,12 @@ "description": "Number of times the user opened one of the in-product help popovers." } }, + "toggle_fullscreen_formula": { + "type": "long", + "_meta": { + "description": "Number of times the user toggled fullscreen mode on formula." + } + }, "indexpattern_field_info_click": { "type": "long" }, @@ -2319,6 +2325,12 @@ "_meta": { "description": "Number of times the moving average function was selected" } + }, + "indexpattern_dimension_operation_formula": { + "type": "long", + "_meta": { + "description": "Number of times the formula function was selected" + } } } }, @@ -2333,6 +2345,12 @@ "description": "Number of times the user opened one of the in-product help popovers." } }, + "toggle_fullscreen_formula": { + "type": "long", + "_meta": { + "description": "Number of times the user toggled fullscreen mode on formula." + } + }, "indexpattern_field_info_click": { "type": "long" }, @@ -2542,6 +2560,12 @@ "_meta": { "description": "Number of times the moving average function was selected" } + }, + "indexpattern_dimension_operation_formula": { + "type": "long", + "_meta": { + "description": "Number of times the formula function was selected" + } } } }, @@ -2614,6 +2638,9 @@ }, "lnsMetric": { "type": "long" + }, + "formula": { + "type": "long" } } }, @@ -2657,6 +2684,9 @@ }, "lnsMetric": { "type": "long" + }, + "formula": { + "type": "long" } } }, @@ -2700,6 +2730,9 @@ }, "lnsMetric": { "type": "long" + }, + "formula": { + "type": "long" } } } From 7f1df87339a8db511f6f7aa8aa6258c09c063c09 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 19 May 2021 10:45:25 -0400 Subject: [PATCH 115/185] Respond to review comments --- .../expression_functions/specs/map_column.ts | 5 ++--- .../common/expression_functions/specs/math.ts | 2 +- .../expression_functions/specs/tests/math.test.ts | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/plugins/expressions/common/expression_functions/specs/map_column.ts b/src/plugins/expressions/common/expression_functions/specs/map_column.ts index e6ce5ab04386ec..7939441ff0d602 100644 --- a/src/plugins/expressions/common/expression_functions/specs/map_column.ts +++ b/src/plugins/expressions/common/expression_functions/specs/map_column.ts @@ -44,7 +44,7 @@ export const mapColumn: ExpressionFunctionDefinition< types: ['string', 'null'], help: i18n.translate('expressions.functions.mapColumn.args.idHelpText', { defaultMessage: - 'An optional id of the resulting column. When no id is provided, the id will be looked up from the existing column and default to the name.', + 'An optional id of the resulting column. When no id is provided, the id will be looked up from the existing column by the provided name argument. If no column with this name exists yet, a new column with this name and an identical id will be added to the table.', }), required: false, default: null, @@ -94,7 +94,6 @@ export const mapColumn: ExpressionFunctionDefinition< } return name === args.name; }); - const columnLength = columns.length; const columnId = existingColumnIndex === -1 ? args.id ?? args.name : columns[existingColumnIndex].id; @@ -122,7 +121,7 @@ export const mapColumn: ExpressionFunctionDefinition< } if (existingColumnIndex === -1) { - columns[columnLength] = newColumn; + columns.push(newColumn); } else { columns[existingColumnIndex] = newColumn; } diff --git a/src/plugins/expressions/common/expression_functions/specs/math.ts b/src/plugins/expressions/common/expression_functions/specs/math.ts index b91600fea8b56e..92a10976428a36 100644 --- a/src/plugins/expressions/common/expression_functions/specs/math.ts +++ b/src/plugins/expressions/common/expression_functions/specs/math.ts @@ -134,7 +134,7 @@ export const math: ExpressionFunctionDefinition< const mathContext = isDatatable(input) ? pivotObjectArray( input.rows, - input.columns.map((col) => col.id ?? col.name) + input.columns.map((col) => col.id) ) : { value: input }; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts index 9eac0911afafd0..6da00061244da8 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/math.test.ts @@ -36,6 +36,21 @@ describe('math', () => { expect(fn(testTable, { expression: 'max(price)' })).toBe(605); }); + it('does not use the name for math', () => { + expect(() => fn(testTable, { expression: 'unique("in_stock label")' })).toThrow( + 'Unknown variable' + ); + expect(() => fn(testTable, { expression: 'sum("quantity label")' })).toThrow( + 'Unknown variable' + ); + expect(() => fn(testTable, { expression: 'mean("price label")' })).toThrow('Unknown variable'); + expect(() => fn(testTable, { expression: 'min("price label")' })).toThrow('Unknown variable'); + expect(() => fn(testTable, { expression: 'median("quantity label")' })).toThrow( + 'Unknown variable' + ); + expect(() => fn(testTable, { expression: 'max("price label")' })).toThrow('Unknown variable'); + }); + describe('args', () => { describe('expression', () => { it('sets the math expression to be evaluted', () => { From 65e61a1e38d51e63aee022f97b89323c8be2a4e3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Wed, 19 May 2021 17:39:21 +0200 Subject: [PATCH 116/185] :sparkles: Improve the signatures with better documentation and examples --- .../definitions/calculations/counter_rate.tsx | 4 + .../calculations/cumulative_sum.tsx | 4 + .../definitions/calculations/differences.tsx | 4 + .../calculations/moving_average.tsx | 11 +- .../operations/definitions/cardinality.tsx | 4 + .../operations/definitions/count.tsx | 3 + .../formula/editor/formula_help.tsx | 170 +++++++++++++--- .../formula/editor/math_completion.ts | 187 +++++++++--------- .../operations/definitions/formula/util.ts | 38 ++-- .../definitions/formula/validation.ts | 6 +- .../operations/definitions/index.ts | 5 + .../operations/definitions/last_value.tsx | 4 + .../operations/definitions/metrics.tsx | 23 +++ .../operations/definitions/percentile.tsx | 11 +- 14 files changed, 337 insertions(+), 137 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index fc9504f003198c..cffd3a610987bc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -47,6 +47,10 @@ export const counterRateOperation: OperationDefinition< defaultMessage: 'Counter rate', }), input: 'fullReference', + description: i18n.translate('xpack.lens.indexPattern.counterRate.description', { + defaultMessage: + 'An aggregation that calculates a rate of documents or a field in each date_histogram bucket.', + }), selectionStyle: 'field', requiredReferences: [ { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 2adb9a1376f606..e2b9141accd5df 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -45,6 +45,10 @@ export const cumulativeSumOperation: OperationDefinition< defaultMessage: 'Cumulative sum', }), input: 'fullReference', + description: i18n.translate('xpack.lens.indexPattern.cumulativeSum.description', { + defaultMessage: + 'An aggregation that calculates the cumulative sum of a specified field in each date_histogram bucket. Cumulative sums always start with 0.', + }), selectionStyle: 'field', requiredReferences: [ { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index 06555a9b41c2ff..4e0b59cda6a0e4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -49,6 +49,10 @@ export const derivativeOperation: OperationDefinition< defaultMessage: 'Differences', }), input: 'fullReference', + description: i18n.translate('xpack.lens.indexPattern.derivative.description', { + defaultMessage: + 'An aggregation that calculates the difference over numeric values of a specified field between each pair of date_histogram buckets. Derivative always start with an undefined value for the first bucket, and it requires a minimum of two buckets.', + }), selectionStyle: 'full', requiredReferences: [ { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 8d18a2752fd7e3..19ad66630dc63c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -63,13 +63,22 @@ export const movingAverageOperation: OperationDefinition< }), input: 'fullReference', selectionStyle: 'full', + description: i18n.translate('xpack.lens.indexPattern.movingAverage.description', { + defaultMessage: + 'Given an ordered series of data, the aggregation will slide a window across the data and emit the average value of that window. The default window value is {defaultValue}', + values: { + defaultValue: WINDOW_DEFAULT_VALUE, + }, + }), requiredReferences: [ { input: ['field', 'managedReference'], validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], - operationParams: [{ name: 'window', type: 'number', required: true }], + operationParams: [ + { name: 'window', type: 'number', required: false, defaultValue: WINDOW_DEFAULT_VALUE }, + ], getPossibleOperation: (indexPattern) => { if (hasDateField(indexPattern)) { return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index e77357a6f441a7..d4da7b350349f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -54,6 +54,10 @@ export const cardinalityOperation: OperationDefinition { if ( supportedTypes.has(type) && diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index fd474ea04a165b..f79b04a6471e6a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -33,6 +33,9 @@ export const countOperation: OperationDefinition getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), onFieldChange: (oldColumn, field) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index ef08b6d03a7b79..525abb78682a96 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -14,14 +14,24 @@ import { EuiText, EuiSelectable, EuiSelectableOption, + EuiCode, + EuiSpacer, + EuiMarkdownFormat, + EuiTitle, } from '@elastic/eui'; import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; -import { GenericOperationDefinition, ParamEditorProps } from '../../index'; -import { IndexPattern } from '../../../../types'; import { tinymathFunctions } from '../util'; import { getPossibleFunctions } from './math_completion'; +import { hasFunctionFieldArgument } from '../validation'; -import { FormulaIndexPatternColumn } from '../formula'; +import type { + GenericOperationDefinition, + IndexPatternColumn, + OperationDefinition, + ParamEditorProps, +} from '../../index'; +import type { IndexPattern } from '../../../../types'; +import type { FormulaIndexPatternColumn } from '../formula'; function FormulaHelp({ indexPattern, @@ -41,7 +51,16 @@ function FormulaHelp({ .filter((key) => key in tinymathFunctions) .map((key) => ({ label: `${key}`, - description: , + description: ( + <> + +

{getFunctionSignatureLabel(key, operationDefinitionMap)}

+ + + {tinymathFunctions[key].help.replace(/\n/g, '\n\n')} + + + ), checked: selectedFunction === key ? ('on' as const) : undefined, })) ); @@ -65,7 +84,9 @@ function FormulaHelp({ return ( <> - Formula reference + {i18n.translate('xpack.lens.formulaReference', { + defaultMessage: 'Formula reference', + })} @@ -149,36 +170,127 @@ Use the symbols +, -, /, and * to perform basic math. export const MemoizedFormulaHelp = React.memo(FormulaHelp); -// TODO: i18n this whole thing, or move examples into the operation definitions with i18n -function getHelpText( +export function getFunctionSignatureLabel( + name: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'], + firstParam?: { label: string | [number, number] } | null +): string { + if (tinymathFunctions[name]) { + return `${name}(${tinymathFunctions[name].positionalArguments + .map(({ name: argName, optional }) => `${argName}${optional ? '?' : ''}`) + .join(', ')})`; + } + if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + if ('operationParams' in def && def.operationParams) { + return `${name}(${firstParam ? firstParam.label + ', ' : ''}${def.operationParams.map( + ({ name: argName, type, required }) => `${argName}${required ? '' : '?'}=${type}` + )})`; + } + return `${name}(${firstParam ? firstParam.label : ''})`; + } + return ''; +} + +function getFunctionArgumentsStringified( + params: Required< + OperationDefinition + >['operationParams'] +) { + return params + .map( + ({ name, type: argType, defaultValue = 5 }) => + `${name}=${argType === 'string' ? `"${defaultValue}"` : defaultValue}` + ) + .join(', '); +} + +/** + * Get an array of strings containing all possible information about a specific + * operation type: examples and infos. + */ +export function getHelpTextContent( type: string, operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -) { +): { description: string; examples: string[] } { const definition = operationDefinitionMap[type]; + const description: string = definition.description ?? ''; + // as for the time being just add examples text. + // Later will enrich with more information taken from the operation definitions. + const examples: string[] = []; - if (type === 'count') { - return ( - -

Example: count()

- - ); + if (!hasFunctionFieldArgument(type)) { + // ideally this should have the same example automation as the operations below + examples.push(`${type}()`); + return { description, examples }; } + if (definition.input === 'field') { + const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; + if (mandatoryArgs.length === 0) { + examples.push(`${type}(bytes)`); + } + if (mandatoryArgs.length) { + const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); + examples.push(`${type}(bytes, ${additionalArgs})`); + } + if (definition.operationParams && mandatoryArgs.length !== definition.operationParams.length) { + const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); + examples.push(`${type}(bytes, ${additionalArgs})`); + } + } + if (definition.input === 'fullReference') { + const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; + if (mandatoryArgs.length === 0) { + examples.push(`${type}(sum(bytes))`); + } + if (mandatoryArgs.length) { + const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); + examples.push(`${type}(sum(bytes), ${additionalArgs})`); + } + if (definition.operationParams && mandatoryArgs.length !== definition.operationParams.length) { + const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); + examples.push(`${type}(sum(bytes), ${additionalArgs})`); + } + } + return { description, examples }; +} +function getHelpText( + type: string, + operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] +) { + const { description, examples } = getHelpTextContent(type, operationDefinitionMap); + const def = operationDefinitionMap[type]; + const firstParam = hasFunctionFieldArgument(type) + ? { + label: def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; return ( - - {definition.input === 'field' ?

Example: {type}(bytes)

: null} - {definition.input === 'fullReference' && !('operationParams' in definition) ? ( -

Example: {type}(sum(bytes))

- ) : null} - - {'operationParams' in definition && definition.operationParams ? ( -

-

- Example: {type}(sum(bytes),{' '} - {definition.operationParams.map((p) => `${p.name}=5`).join(', ')}) -

-

- ) : null} - + <> + +

{getFunctionSignatureLabel(type, operationDefinitionMap, firstParam)}

+ + + {description} + {examples.length ? ( + <> + +

+ + {i18n.translate('xpack.lens.formulaExamples', { + defaultMessage: 'Examples', + })} + +

+ {examples.map((example) => ( +

+ {example} +

+ ))} + + ) : null} + + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index e8c16fe64651a4..f73600faf4418e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -6,6 +6,7 @@ */ import { uniq, startsWith } from 'lodash'; +import { i18n } from '@kbn/i18n'; import { monaco } from '@kbn/monaco'; import { parse, @@ -19,6 +20,8 @@ import { IndexPattern } from '../../../../types'; import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; import { tinymathFunctions, groupArgsByType } from '../util'; import type { GenericOperationDefinition } from '../..'; +import { getFunctionSignatureLabel, getHelpTextContent } from './formula_help'; +import { hasFunctionFieldArgument } from '../validation'; export enum SUGGESTION_TYPE { FIELD = 'field', @@ -212,7 +215,7 @@ function getArgumentSuggestions( return { list: [], type: SUGGESTION_TYPE.FIELD }; } - if (position > 0 || operation.type === 'count') { + if (position > 0 || !hasFunctionFieldArgument(operation.type)) { const { namedArguments } = groupArgsByType(ast.args); const list = []; if (operation.filterable) { @@ -357,7 +360,7 @@ export function getSuggestion( } else { const def = operationDefinitionMap[suggestion.label]; kind = monaco.languages.CompletionItemKind.Constant; - if (suggestion.label === 'count' && 'operationParams' in def) { + if (!hasFunctionFieldArgument(suggestion.label) && 'operationParams' in def) { label = `${label}(${def .operationParams!.map((p) => `${p.name}=${p.type}`) .join(', ')})`; @@ -413,6 +416,88 @@ export function getSuggestion( }; } +function getOperationTypeHelp( + name: string, + operationDefinitionMap: Record +) { + const { description, examples } = getHelpTextContent(name, operationDefinitionMap); + const descriptionInMarkdown = description.replace(/\n/g, '\n\n'); + const examplesInMarkdown = `**${i18n.translate('xpack.lens.formulaExampleMarkdown', { + defaultMessage: 'Examples', + })}** + + ${examples.map((example) => `\`${example}\``).join('\n\n')}`; + return { + value: `${descriptionInMarkdown}\n\n${examplesInMarkdown}`, + }; +} + +function getSignaturesForFunction( + name: string, + operationDefinitionMap: Record +) { + if (tinymathFunctions[name]) { + const stringify = getFunctionSignatureLabel(name, operationDefinitionMap); + const documentation = tinymathFunctions[name].help.replace(/\n/g, '\n\n'); + return [ + { + label: stringify, + documentation: { value: documentation }, + parameters: tinymathFunctions[name].positionalArguments.map((arg) => ({ + label: arg.name, + documentation: arg.optional + ? i18n.translate('xpack.lens.formula.optionalArgument', { + defaultMessage: 'Optional. Default value is {defaultValue}', + values: { + defaultValue: arg.defaultValue, + }, + }) + : '', + })), + }, + ]; + } + if (operationDefinitionMap[name]) { + const def = operationDefinitionMap[name]; + + const firstParam: monaco.languages.ParameterInformation | null = hasFunctionFieldArgument(name) + ? { + label: def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', + } + : null; + + const functionLabel = getFunctionSignatureLabel(name, operationDefinitionMap, firstParam); + const documentation = getOperationTypeHelp(name, operationDefinitionMap); + if ('operationParams' in def && def.operationParams) { + return [ + { + label: functionLabel, + parameters: [ + ...(firstParam ? [firstParam] : []), + ...def.operationParams.map((arg) => ({ + label: `${arg.name}=${arg.type}`, + documentation: arg.required + ? i18n.translate('xpack.lens.formula.requiredArgument', { + defaultMessage: 'Required', + }) + : '', + })), + ], + documentation, + }, + ]; + } + return [ + { + label: functionLabel, + parameters: firstParam ? [firstParam] : [], + documentation, + }, + ]; + } + return []; +} + export function getSignatureHelp( expression: string, position: number, @@ -429,73 +514,16 @@ export function getSignatureHelp( // reference equality is fine here because of the way the getInfo function works const index = tokenInfo.parent.args.findIndex((arg) => arg === tokenInfo.ast); - if (tinymathFunctions[name]) { - const stringify = `${name}(${tinymathFunctions[name].positionalArguments - .map((arg) => arg.name) - .join(', ')})`; + const signatures = getSignaturesForFunction(name, operationDefinitionMap); + if (signatures.length) { return { value: { - signatures: [ - { - label: stringify, - parameters: tinymathFunctions[name].positionalArguments.map((arg) => ({ - label: arg.name, - documentation: arg.optional ? 'Optional' : '', - })), - }, - ], + signatures, activeParameter: index, activeSignature: 0, }, dispose: () => {}, }; - } else if (operationDefinitionMap[name]) { - const def = operationDefinitionMap[name]; - - const firstParam: monaco.languages.ParameterInformation | null = - name !== 'count' - ? { - label: - def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', - } - : null; - if ('operationParams' in def) { - return { - value: { - signatures: [ - { - label: `${name}(${ - firstParam ? firstParam.label + ', ' : '' - }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, - parameters: [ - ...(firstParam ? [firstParam] : []), - ...def.operationParams!.map((arg) => ({ - label: `${arg.name}=${arg.type}`, - documentation: arg.required ? 'Required' : '', - })), - ], - }, - ], - activeParameter: index, - activeSignature: 0, - }, - dispose: () => {}, - }; - } else { - return { - value: { - signatures: [ - { - label: `${name}(${firstParam ? firstParam.label : ''})`, - parameters: firstParam ? [firstParam] : [], - }, - ], - activeParameter: index, - activeSignature: 0, - }, - dispose: () => {}, - }; - } } } } catch (e) { @@ -519,37 +547,10 @@ export function getHover( } const name = tokenInfo.ast.name; - - if (tinymathFunctions[name]) { - const stringify = `${name}(${tinymathFunctions[name].positionalArguments - .map((arg) => arg.name) - .join(', ')})`; - return { contents: [{ value: stringify }] }; - } else if (operationDefinitionMap[name]) { - const def = operationDefinitionMap[name]; - - const firstParam: monaco.languages.ParameterInformation | null = - name !== 'count' - ? { - label: - def.input === 'field' ? 'field' : def.input === 'fullReference' ? 'function' : '', - } - : null; - if ('operationParams' in def) { - return { - contents: [ - { - value: `${name}(${ - firstParam ? firstParam.label + ', ' : '' - }${def.operationParams!.map((arg) => `${arg.name}=${arg.type}`)})`, - }, - ], - }; - } else { - return { - contents: [{ value: `${name}(${firstParam ? firstParam.label : ''})` }], - }; - } + const signatures = getSignaturesForFunction(name, operationDefinitionMap); + if (signatures.length) { + const { label, documentation } = signatures[0]; + return { contents: [{ value: label }, documentation] }; } } catch (e) { // do nothing diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 5d9a8647eb7ab0..076afb02a45f87 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -73,6 +73,7 @@ export const tinymathFunctions: Record< positionalArguments: Array<{ name: string; optional?: boolean; + defaultValue?: string | number; }>; // help: React.ReactElement; // Help is in Markdown format @@ -86,8 +87,12 @@ export const tinymathFunctions: Record< ], help: ` Also works with + symbol -Example: ${'`count() + sum(bytes)`'} -Example: ${'`add(count(), 5)`'} + +**Examples**: +\`\`\` +count() + sum(bytes) +add(count(), 5) +\`\`\` `, }, subtract: { @@ -117,7 +122,7 @@ Example: ${'`multiply(sum(bytes), 2)`'} ], help: ` Also works with ${'`/`'} symbol -Example: ${'`ceil(sum(bytes))`'} +Example: ${'`divide(sum(bytes), 2)`'} `, }, abs: { @@ -155,7 +160,7 @@ Example: ${'`ceil(sum(bytes))`'} ], help: ` Limits the value from a minimum to maximum -Example: ${'`ceil(sum(bytes))`'} +Example: ${'`clamp(sum(bytes), 1, 100)`'} `, }, cube: { @@ -164,7 +169,7 @@ Example: ${'`ceil(sum(bytes))`'} ], help: ` Limits the value from a minimum to maximum -Example: ${'`ceil(sum(bytes))`'} +Example: ${'`cube(sum(bytes))`'} `, }, exp: { @@ -172,7 +177,7 @@ Example: ${'`ceil(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, ], help: ` -Raises e to the nth power. +Raises *e* to the nth power. Example: ${'`exp(sum(bytes))`'} `, }, @@ -200,12 +205,17 @@ Example: ${'`floor(sum(bytes))`'} { name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), optional: true, + defaultValue: 'e', }, ], help: ` -Logarithm with optional base. The natural base e is used as default. -Example: ${'`log(sum(bytes))`'} -Example: ${'`log(sum(bytes), 2)`'} +Logarithm with optional base. The natural base *e* is used as default. + +**Examples**: +\`\`\` +log(sum(bytes)) +log(sum(bytes), 2) +\`\`\` `, }, // TODO: check if this is valid for Tinymath @@ -223,7 +233,6 @@ Example: ${'`log(sum(bytes), 2)`'} { name: i18n.translate('xpack.lens.formula.expression', { defaultMessage: 'expression' }) }, { name: i18n.translate('xpack.lens.formula.base', { defaultMessage: 'base' }), - optional: true, }, ], help: ` @@ -249,12 +258,17 @@ Example: ${'`pow(sum(bytes), 3)`'} { name: i18n.translate('xpack.lens.formula.decimals', { defaultMessage: 'decimals' }), optional: true, + defaultValue: 0, }, ], help: ` Rounds to a specific number of decimal places, default of 0 -Example: ${'`round(sum(bytes))`'} -Example: ${'`round(sum(bytes), 2)`'} + +**Examples**: +\`\`\` +round(sum(bytes)) +round(sum(bytes), 2) +\`\`\` `, }, sqrt: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 1dfdbb2910e2a0..fa73cb69b20f04 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -606,7 +606,11 @@ export function validateParams( } export function shouldHaveFieldArgument(node: TinymathFunction) { - return !['count'].includes(node.name); + return hasFunctionFieldArgument(node.name); +} + +export function hasFunctionFieldArgument(type: string) { + return !['count'].includes(type); } export function isFirstArgumentValidType(arg: TinymathAST, type: TinymathNodeTypes['type']) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 27982243f8c2b1..04a600d7e63761 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -185,6 +185,10 @@ interface BaseOperationDefinitionProps { * Should be i18n-ified. */ displayName: string; + /** + * A short description of the operation, useful for help or documentation + */ + description?: string; /** * The default label is assigned by the editor */ @@ -273,6 +277,7 @@ interface OperationParam { name: string; type: string; required?: boolean; + defaultValue?: string | number; } interface FieldlessOperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 4632d262c441d4..417420a08fe778 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -98,6 +98,10 @@ export const lastValueOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), input: 'field', + description: i18n.translate('xpack.lens.indexPattern.lastValue.description', { + defaultMessage: + 'A single-value metric aggregation that returns the last value (based on time) of the provided field extracted from the aggregated documents.', + }), onFieldChange: (oldColumn, field) => { const newParams = { ...oldColumn.params }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 725ef93203a438..e4e60a1ba8738b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -42,6 +42,7 @@ const supportedTypes = ['number', 'histogram']; function buildMetricOperation>({ type, displayName, + description, ofName, priority, optionalTimeScaling, @@ -51,6 +52,7 @@ function buildMetricOperation>({ ofName: (name: string) => string; priority?: number; optionalTimeScaling?: boolean; + description?: string; }) { const labelLookup = (name: string, column?: BaseIndexPatternColumn) => { const label = ofName(name); @@ -64,6 +66,7 @@ function buildMetricOperation>({ type, priority, displayName, + description, input: 'field', timeScalingMode: optionalTimeScaling ? 'optional' : undefined, getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { @@ -144,6 +147,10 @@ export const minOperation = buildMetricOperation({ defaultMessage: 'Minimum of {name}', values: { name }, }), + description: i18n.translate('xpack.lens.indexPattern.min.description', { + defaultMessage: + 'A single-value metrics aggregation that returns the minimum value among the numeric values extracted from the aggregated documents.', + }), }); export const maxOperation = buildMetricOperation({ @@ -156,6 +163,10 @@ export const maxOperation = buildMetricOperation({ defaultMessage: 'Maximum of {name}', values: { name }, }), + description: i18n.translate('xpack.lens.indexPattern.max.description', { + defaultMessage: + 'A single-value metrics aggregation that returns the maximum value among the numeric values extracted from the aggregated documents.', + }), }); export const averageOperation = buildMetricOperation({ @@ -169,6 +180,10 @@ export const averageOperation = buildMetricOperation({ defaultMessage: 'Average of {name}', values: { name }, }), + description: i18n.translate('xpack.lens.indexPattern.avg.description', { + defaultMessage: + 'A single-value metric aggregation that computes the average of numeric values that are extracted from the aggregated documents', + }), }); export const sumOperation = buildMetricOperation({ @@ -183,6 +198,10 @@ export const sumOperation = buildMetricOperation({ values: { name }, }), optionalTimeScaling: true, + description: i18n.translate('xpack.lens.indexPattern.sum.description', { + defaultMessage: + 'A single-value metrics aggregation that sums up numeric values that are extracted from the aggregated documents.', + }), }); export const medianOperation = buildMetricOperation({ @@ -196,4 +215,8 @@ export const medianOperation = buildMetricOperation({ defaultMessage: 'Median of {name}', values: { name }, }), + description: i18n.translate('xpack.lens.indexPattern.median.description', { + defaultMessage: + 'A single-value metrics aggregation that computes the median value that are extracted from the aggregated documents.', + }), }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 705a1f7172fff8..52c9b2d76bab43 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -52,7 +52,16 @@ export const percentileOperation: OperationDefinition { if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { From 371703c7720917765da70a460c8fbbc27b7298a0 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Wed, 19 May 2021 12:33:16 -0400 Subject: [PATCH 117/185] adjust border styles to account for docs collapse --- .../operations/definitions/formula/editor/formula.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 5e97f592a04744..5a686efe946a6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -10,12 +10,17 @@ flex: 1; min-height: 0; } + + & > * + * { + border-top: $euiBorderThin; + } } .lnsFormula__editor { border-bottom: $euiBorderThin; .lnsIndexPatternDimensionEditor-isFullscreen & { + border-bottom: none; display: flex; flex-direction: column; } From eb658febdbdb8cbbc421d48762061f8281ed0c77 Mon Sep 17 00:00:00 2001 From: Michael Marcialis Date: Wed, 19 May 2021 16:32:48 -0400 Subject: [PATCH 118/185] refactor docs markup; restructure docs obj; styles --- .../definitions/formula/editor/formula.scss | 21 +++ .../formula/editor/formula_help.tsx | 144 ++++++++++-------- 2 files changed, 101 insertions(+), 64 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 5a686efe946a6f..56d0960862e7c0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -99,7 +99,18 @@ .lnsFormula__docsNav { @include euiYScroll; background: $euiColorLightestShade; +} + +.lnsFormula__docsNavGroup { padding: $euiSizeS; + + & + & { + border-top: $euiBorderThin; + } +} + +.lnsFormula__docsNavGroupLink { + font-weight: inherit; } .lnsFormula__docsText { @@ -107,6 +118,16 @@ padding: $euiSize; } +.lnsFormula__docsTextGroup, +.lnsFormula__docsTextItem { + margin-top: $euiSizeXXL; +} + +.lnsFormula__docsTextGroup { + border-top: $euiBorderThin; + padding-top: $euiSizeXXL; +} + .lnsFormulaOverflow { // Needs to be higher than the modal and all flyouts z-index: $euiZLevel9 + 1; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 0ff120ea110dbc..c5b18ea36957e0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -10,13 +10,12 @@ import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, EuiFlexItem, + EuiLink, EuiPopoverTitle, EuiText, EuiListGroupItem, - EuiSelectableOption, EuiListGroup, - EuiHorizontalRule, - EuiSpacer, + EuiTitle, } from '@elastic/eui'; import { Markdown } from '../../../../../../../../../src/plugins/kibana_react/public'; import { GenericOperationDefinition } from '../../index'; @@ -34,7 +33,7 @@ function FormulaHelp({ isFullscreen: boolean; }) { const [selectedFunction, setSelectedFunction] = useState(); - const scrollTargets = useRef>({}); + const scrollTargets = useRef>({}); useEffect(() => { if (selectedFunction && scrollTargets.current[selectedFunction]) { @@ -42,24 +41,28 @@ function FormulaHelp({ } }, [selectedFunction]); - const helpItems: Array = []; + const helpGroups: Array<{ + label: string; + description?: JSX.Element; + items: Array<{ label: string; description?: JSX.Element }>; + }> = []; - helpItems.push({ + helpGroups.push({ label: i18n.translate('xpack.lens.formulaDocumentation.mathSection', { defaultMessage: 'Math', }), - isGroupLabel: true, description: ( - + {i18n.translate('xpack.lens.formulaDocumentation.mathSectionDescription', { defaultMessage: 'These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.', })} ), + items: [], }); - helpItems.push( + helpGroups[0].items.push( ...getPossibleFunctions(indexPattern) .filter((key) => key in tinymathFunctions) .sort() @@ -69,23 +72,23 @@ function FormulaHelp({ })) ); - helpItems.push({ + helpGroups.push({ label: i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSection', { defaultMessage: 'Elasticsearch', }), - isGroupLabel: true, description: ( - + {i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSectionDescription', { defaultMessage: 'These functions will be executed on the raw documents for each row of the resulting table, aggregating all documents matching the break down dimensions into a single value.', })} ), + items: [], }); // Es aggs - helpItems.push( + helpGroups[1].items.push( ...getPossibleFunctions(indexPattern) .filter( (key) => @@ -99,23 +102,23 @@ function FormulaHelp({ })) ); - helpItems.push({ + helpGroups.push({ label: i18n.translate('xpack.lens.formulaDocumentation.columnCalculationSection', { defaultMessage: 'Column-wise calculation', }), - isGroupLabel: true, description: ( - + {i18n.translate('xpack.lens.formulaDocumentation.columnCalculationSectionDescription', { defaultMessage: 'These functions will be executed for reach row of the resulting table, using data from cells from other rows as well as the current value.', })} ), + items: [], }); // Calculations aggs - helpItems.push( + helpGroups[2].items.push( ...getPossibleFunctions(indexPattern) .filter( (key) => @@ -143,35 +146,40 @@ function FormulaHelp({ - - {helpItems.map((helpItem) => { - if (helpItem.isGroupLabel) { - return ( - { - setSelectedFunction(helpItem.label); - }} - /> - ); - } else { - return ( - { - setSelectedFunction(helpItem.label); - }} - /> - ); - } - })} - + {helpGroups.map((helpGroup, index) => { + return ( + + ); + })} @@ -215,30 +223,38 @@ Use the symbols +, -, /, and * to perform basic math. 'Text is in markdown. Do not translate function names or field names like sum(bytes)', })} /> - - {helpItems.map((item, index) => { + + {helpGroups.map((helpGroup, index) => { return ( -
{ if (el) { - scrollTargets.current[item.label] = el; + scrollTargets.current[helpGroup.label] = el; } }} > - {item.isGroupLabel ? ( - -

{item.label}

- {item.description} - - - ) : ( - - {item.description} - {helpItems.length - 1 !== index && } - - )} -
+

{helpGroup.label}

+ + {helpGroup.description} + + {helpGroups[index].items.map((helpItem) => { + return ( +
{ + if (el) { + scrollTargets.current[helpItem.label] = el; + } + }} + > + {helpItem.description} +
+ ); + })} + ); })} From 4d7d2ba91c8de9341e94d9dcf27b6daf5e821e82 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 20 May 2021 15:27:51 +0200 Subject: [PATCH 119/185] Fix formula auto reordering (#18) * fix formula auto reordering * add unit test --- .../operations/layer_helpers.test.ts | 36 +++++++++++++++++++ .../operations/layer_helpers.ts | 8 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index 25e46e5f77076c..f3d9810a1ffd57 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -390,6 +390,42 @@ describe('state_helpers', () => { ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2', 'col3'] })); }); + it('should not change order of metrics and references on inserting new buckets', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Cumulative sum of count of records', + dataType: 'number', + isBucketed: false, + + // Private + operationType: 'cumulative_sum', + references: ['col2'], + }, + col2: { + label: 'Count of records', + dataType: 'document', + isBucketed: false, + + // Private + operationType: 'count', + sourceField: 'Records', + }, + }, + }; + expect( + insertNewColumn({ + layer, + indexPattern, + columnId: 'col3', + op: 'filters', + visualizationGroups: [], + }) + ).toEqual(expect.objectContaining({ columnOrder: ['col3', 'col1', 'col2'] })); + }); + it('should insert both incomplete states if the aggregation does not support the field', () => { expect( insertNewColumn({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 1eecbac1b4cbce..8e38223ec4d21c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -850,7 +850,10 @@ function addBucket( visualizationGroups: VisualizationDimensionGroupConfig[], targetGroup?: string ): IndexPatternLayer { - const [buckets, metrics, references] = getExistingColumnGroups(layer); + const [buckets, metrics] = _.partition( + layer.columnOrder, + (colId) => layer.columns[colId].isBucketed + ); const oldDateHistogramIndex = layer.columnOrder.findIndex( (columnId) => layer.columns[columnId].operationType === 'date_histogram' @@ -864,12 +867,11 @@ function addBucket( addedColumnId, ...buckets.slice(oldDateHistogramIndex, buckets.length), ...metrics, - ...references, ]; } else { // Insert the new bucket after existing buckets. Users will see the same data // they already had, with an extra level of detail. - updatedColumnOrder = [...buckets, addedColumnId, ...metrics, ...references]; + updatedColumnOrder = [...buckets, addedColumnId, ...metrics]; } updatedColumnOrder = reorderByGroups( visualizationGroups, From c97f6f6b03ffb1424dd14314cb51910f87f78a3f Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Mon, 24 May 2021 19:08:21 +0200 Subject: [PATCH 120/185] Fix and improve suggestion experience in Formula (#19) * :sparkles: Revisit documentation and suggestions * :ok_hand: Integrated feedback --- .../definitions/calculations/counter_rate.tsx | 5 +- .../calculations/cumulative_sum.tsx | 4 +- .../definitions/calculations/differences.tsx | 4 +- .../calculations/moving_average.tsx | 4 +- .../operations/definitions/cardinality.tsx | 10 +- .../operations/definitions/count.tsx | 12 +- .../formula/editor/formula_help.tsx | 213 ++++++++++-------- .../formula/editor/math_completion.ts | 51 ++--- .../operations/definitions/formula/util.ts | 52 ++--- .../operations/definitions/last_value.tsx | 6 +- .../operations/definitions/metrics.tsx | 10 +- .../operations/definitions/percentile.tsx | 6 +- 12 files changed, 179 insertions(+), 198 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 4de7060d181911..8dec6c17caa3d1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -129,11 +129,10 @@ Calculates the rate of an ever increasing counter. This function will only yield If the value does get smaller, it will interpret this as a counter reset. To get most precise results, \`counter_rate\` should be calculated on the \`max\` of a field. This calculation will be done separately for separate series defined by filters or top values dimensions. +It uses the current interval when used in Formula. Example: Visualize the rate of bytes received over time by a memcached server: -\`\`\` -counter_rate(max(memcached.stats.read.bytes)) -\`\`\` +${'`counter_rate(max(memcached.stats.read.bytes))`'} `, }), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 06e613b62716ee..230899143e1547 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -123,9 +123,7 @@ Calculates the cumulative sum of a metric over time, adding all previous values This calculation will be done separately for separate series defined by filters or top values dimensions. Example: Visualize the received bytes accumulated over time: -\`\`\` -cumulative_sum(sum(bytes)) -\`\`\` +${'`cumulative_sum(sum(bytes))`'} `, }), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index 7511489bfd10a0..5c757bd22b36b0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -121,9 +121,7 @@ Differences requires the data to be sequential. If your data is empty when using This calculation will be done separately for separate series defined by filters or top values dimensions. Example: Visualize the change in bytes received over time: -\`\`\` -differences(sum(bytes)) -\`\`\` +${'`differences(sum(bytes))`'} `, }), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 0457286545aa5b..a5948b6a854be7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -150,9 +150,7 @@ This calculation will be done separately for separate series defined by filters Takes a named parameter \`window\` which specifies how many last values to include in the average calculation for the current value. Example: Smooth a line of measurements: -\`\`\` -moving_average(sum(bytes), window=5) -\`\`\` +${'`moving_average(sum(bytes), window=5)`'} `, values: { defaultValue: WINDOW_DEFAULT_VALUE, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index d9ea7926f9ace7..e40cb6cb0d7c09 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -112,21 +112,17 @@ export const cardinalityOperation: OperationDefinition { + return getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .sort() + .map((key) => { + const [description, examples] = tinymathFunctions[key].help.split(`\`\`\``); + return { + label: key, + description: description.replace(/\n/g, '\n\n'), + examples: examples ? `\`\`\`${examples}\`\`\`` : '', + }; + }); + }, [indexPattern]); + const helpGroups: Array<{ label: string; description?: JSX.Element; @@ -72,22 +85,19 @@ function FormulaHelp({ }); helpGroups[0].items.push( - ...getPossibleFunctions(indexPattern) - .filter((key) => key in tinymathFunctions) - .sort() - .map((key) => ({ - label: `${key}`, + ...tinymathFns.map(({ label, description, examples }) => { + return { + label, description: ( <> -

{getFunctionSignatureLabel(key, operationDefinitionMap)}

+

{getFunctionSignatureLabel(label, operationDefinitionMap)}

 - - {tinymathFunctions[key].help.replace(/\n/g, '\n\n')} - + {`${description}${examples}`} ), - })) + }; + }) ); helpGroups.push({ @@ -105,9 +115,11 @@ function FormulaHelp({ items: [], }); + const availableFunctions = getPossibleFunctions(indexPattern); + // Es aggs helpGroups[1].items.push( - ...getPossibleFunctions(indexPattern) + ...availableFunctions .filter( (key) => key in operationDefinitionMap && @@ -123,7 +135,11 @@ function FormulaHelp({ {key}({operationDefinitionMap[key].documentation?.signature}) - + {operationDefinitionMap[key].documentation?.description ? ( + + {operationDefinitionMap[key].documentation!.description} + + ) : null} ), })) @@ -146,7 +162,7 @@ function FormulaHelp({ // Calculations aggs helpGroups[2].items.push( - ...getPossibleFunctions(indexPattern) + ...availableFunctions .filter( (key) => key in operationDefinitionMap && @@ -162,7 +178,11 @@ function FormulaHelp({ {key}({operationDefinitionMap[key].documentation?.signature}) - + {operationDefinitionMap[key].documentation?.description ? ( + + {operationDefinitionMap[key].documentation!.description} + + ) : null} ), checked: @@ -219,10 +239,9 @@ function FormulaHelp({ - - + {i18n.translate('xpack.lens.formulaDocumentation', { + defaultMessage: ` ## How it works Lens formulas let you do math using a combination of Elasticsearch aggregations and @@ -255,45 +274,46 @@ Math functions can take positional arguments, like pow(count(), 3) is the same a Use the symbols +, -, /, and * to perform basic math. `, - description: - 'Text is in markdown. Do not translate function names or field names like sum(bytes)', - })} - /> - - {helpGroups.map((helpGroup, index) => { - return ( -
{ - if (el) { - scrollTargets.current[helpGroup.label] = el; - } - }} - > -

{helpGroup.label}

+ description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + - {helpGroup.description} + {helpGroups.map((helpGroup, index) => { + return ( +
{ + if (el) { + scrollTargets.current[helpGroup.label] = el; + } + }} + > + +

{helpGroup.label}

+ - {helpGroups[index].items.map((helpItem) => { - return ( -
{ - if (el) { - scrollTargets.current[helpItem.label] = el; - } - }} - > - {helpItem.description} -
- ); - })} -
- ); - })} - + {helpGroup.description} + + {helpGroups[index].items.map((helpItem) => { + return ( +
{ + if (el) { + scrollTargets.current[helpItem.label] = el; + } + }} + > + {helpItem.description} +
+ ); + })} +
+ ); + })} @@ -314,7 +334,14 @@ export function getFunctionSignatureLabel( } if (operationDefinitionMap[name]) { const def = operationDefinitionMap[name]; - return `${name}(${def.documentation?.signature})`; // ${firstParam ? firstParam.label : ''})`; + let extraArgs = ''; + if (def.filterable) { + extraArgs += hasFunctionFieldArgument(name) || 'operationParams' in def ? ',' : ''; + extraArgs += i18n.translate('xpack.lens.formula.kqlExtraArguments', { + defaultMessage: '[kql]?: string, [lucene]?: string', + }); + } + return `${name}(${def.documentation?.signature}${extraArgs})`; } return ''; } @@ -339,45 +366,53 @@ function getFunctionArgumentsStringified( export function getHelpTextContent( type: string, operationDefinitionMap: ParamEditorProps['operationDefinitionMap'] -): { description: JSX.Element | string; examples: string[] } { +): { description: string; examples: string[] } { const definition = operationDefinitionMap[type]; const description = definition.documentation?.description ?? ''; // as for the time being just add examples text. // Later will enrich with more information taken from the operation definitions. const examples: string[] = []; - - if (!hasFunctionFieldArgument(type)) { - // ideally this should have the same example automation as the operations below - examples.push(`${type}()`); - return { description, examples }; - } - if (definition.input === 'field') { - const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; - if (mandatoryArgs.length === 0) { - examples.push(`${type}(bytes)`); - } - if (mandatoryArgs.length) { - const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); - examples.push(`${type}(bytes, ${additionalArgs})`); - } - if (definition.operationParams && mandatoryArgs.length !== definition.operationParams.length) { - const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); - examples.push(`${type}(bytes, ${additionalArgs})`); - } - } - if (definition.input === 'fullReference') { - const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; - if (mandatoryArgs.length === 0) { - examples.push(`${type}(sum(bytes))`); + // If the description already contain examples skip it + if (!/Example/.test(description)) { + if (!hasFunctionFieldArgument(type)) { + // ideally this should have the same example automation as the operations below + examples.push(`${type}()`); + return { description, examples }; } - if (mandatoryArgs.length) { - const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); - examples.push(`${type}(sum(bytes), ${additionalArgs})`); + if (definition.input === 'field') { + const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; + if (mandatoryArgs.length === 0) { + examples.push(`${type}(bytes)`); + } + if (mandatoryArgs.length) { + const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); + examples.push(`${type}(bytes, ${additionalArgs})`); + } + if ( + definition.operationParams && + mandatoryArgs.length !== definition.operationParams.length + ) { + const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); + examples.push(`${type}(bytes, ${additionalArgs})`); + } } - if (definition.operationParams && mandatoryArgs.length !== definition.operationParams.length) { - const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); - examples.push(`${type}(sum(bytes), ${additionalArgs})`); + if (definition.input === 'fullReference') { + const mandatoryArgs = definition.operationParams?.filter(({ required }) => required) || []; + if (mandatoryArgs.length === 0) { + examples.push(`${type}(sum(bytes))`); + } + if (mandatoryArgs.length) { + const additionalArgs = getFunctionArgumentsStringified(mandatoryArgs); + examples.push(`${type}(sum(bytes), ${additionalArgs})`); + } + if ( + definition.operationParams && + mandatoryArgs.length !== definition.operationParams.length + ) { + const additionalArgs = getFunctionArgumentsStringified(definition.operationParams); + examples.push(`${type}(sum(bytes), ${additionalArgs})`); + } } } return { description, examples }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 49fc9a06725f25..e76a53b49a7f2c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -276,8 +276,11 @@ function getArgumentSuggestions( ) { possibleOperationNames.push( ...a.operations - .filter((o) => - operation.requiredReferences.some((requirement) => requirement.input.includes(o.type)) + .filter( + (o) => + operation.requiredReferences.some((requirement) => + requirement.input.includes(o.type) + ) && !o.hidden ) .map((o) => o.operationType) ); @@ -350,27 +353,13 @@ export function getSuggestion( insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; if (typeof suggestion !== 'string') { if ('text' in suggestion) break; + label = getFunctionSignatureLabel(suggestion.label, operationDefinitionMap); const tinymathFunction = tinymathFunctions[suggestion.label]; if (tinymathFunction) { - label = `${label}(${tinymathFunction.positionalArguments - .map(({ name }) => name) - .join(', ')})`; detail = 'TinyMath'; kind = monaco.languages.CompletionItemKind.Method; } else { - const def = operationDefinitionMap[suggestion.label]; kind = monaco.languages.CompletionItemKind.Constant; - if (!hasFunctionFieldArgument(suggestion.label) && 'operationParams' in def) { - label = `${label}(${def - .operationParams!.map((p) => `${p.name}=${p.type}`) - .join(', ')})`; - } else if ('operationParams' in def) { - label = `${label}(expression, ${def - .operationParams!.map((p) => `${p.name}=${p.type}`) - .join(', ')})`; - } else { - label = `${label}(expression)`; - } detail = 'Elasticsearch'; // Always put ES functions first sortText = `0${label}`; @@ -420,16 +409,19 @@ function getOperationTypeHelp( name: string, operationDefinitionMap: Record ) { - const { description, examples } = getHelpTextContent(name, operationDefinitionMap); - // const descriptionInMarkdown = description.replace(/\n/g, '\n\n'); - const descriptionInMarkdown = description; - const examplesInMarkdown = `**${i18n.translate('xpack.lens.formulaExampleMarkdown', { - defaultMessage: 'Examples', - })}** - - ${examples.map((example) => `\`${example}\``).join('\n\n')}`; + const { description: descriptionInMarkdown, examples } = getHelpTextContent( + name, + operationDefinitionMap + ); + const examplesInMarkdown = examples.length + ? `\n\n**${i18n.translate('xpack.lens.formulaExampleMarkdown', { + defaultMessage: 'Examples', + })}** + + ${examples.map((example) => `\`${example}\``).join('\n\n')}` + : ''; return { - value: `${descriptionInMarkdown}\n\n${examplesInMarkdown}`, + value: `${descriptionInMarkdown}${examplesInMarkdown}`, }; } @@ -519,7 +511,12 @@ export function getSignatureHelp( if (signatures.length) { return { value: { - signatures, + // remove the documentation + signatures: signatures.map(({ documentation, ...signature }) => ({ + ...signature, + // extract only the first section (usually few lines) + documentation: { value: documentation.value.split('\n\n')[0] }, + })), activeParameter: index, activeSignature: 0, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index bcbdbc53f0a77e..179d10dfbab939 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -130,9 +130,7 @@ Subtracts the first number from the second number. Also works with ${'`-`'} symbol Example: Calculate the range of a field -\`\`\` -subtract(max(bytes), min(bytes)) -\`\`\` +${'`subtract(max(bytes), min(bytes))`'} `, }, multiply: { @@ -151,14 +149,10 @@ Multiplies two numbers. Also works with ${'`*`'} symbol. Example: Calculate price after current tax rate -\`\`\` -sum(bytes) * last_value(tax_rate) -\`\`\` +${'`sum(bytes) * last_value(tax_rate)`'} Example: Calculate price after constant tax rate -\`\`\` -multiply(sum(price), 1.2) -\`\`\` +${'`multiply(sum(price), 1.2)`'} `, }, divide: { @@ -177,9 +171,7 @@ Divides the first number by the second number. Also works with ${'`/`'} symbol Example: Calculate profit margin -\`\`\` -sum(profit) / sum(revenue) -\`\`\` +${'`sum(profit) / sum(revenue)`'} Example: ${'`divide(sum(bytes), 2)`'} `, @@ -208,9 +200,7 @@ Example: Calculate average distance to sea level ${'`abs(average(altitude))`'} Cube root of value. Example: Calculate side length from volume -\`\`\` -cbrt(last_value(volume)) -\`\`\` +${'`cbrt(last_value(volume))`'} `, }, ceil: { @@ -225,9 +215,7 @@ cbrt(last_value(volume)) Ceiling of value, rounds up. Example: Round up price to the next dollar -\`\`\` -ceil(sum(price)) -\`\`\` +${'`ceil(sum(price))`'} `, }, clamp: { @@ -270,9 +258,7 @@ clamp( Calculates the cube of a number. Example: Calculate volume from side length -\`\`\` -cube(last_value(length)) -\`\`\` +${'`cube(last_value(length))`'} `, }, exp: { @@ -301,9 +287,7 @@ ${'`exp(last_value(duration))`'} For positive values, takes the floor. For negative values, takes the ceiling. Example: Rounding towards zero -\`\`\` -fix(sum(profit)) -\`\`\` +${'`fix(sum(profit))`'} `, }, floor: { @@ -317,9 +301,7 @@ fix(sum(profit)) Round down to nearest integer value Example: Round down a price -\`\`\` -floor(sum(price)) -\`\`\` +${'`floor(sum(price))`'} `, }, log: { @@ -370,9 +352,7 @@ log(sum(bytes), 2) Remainder after dividing the function by a number Example: Calculate last three digits of a value -\`\`\` -mod(sum(price), 1000) -\`\`\` +${'`mod(sum(price), 1000)`'} `, }, pow: { @@ -390,9 +370,7 @@ mod(sum(price), 1000) Raises the value to a certain power. The second argument is required Example: Calculate volume based on side length -\`\`\` -pow(last_value(length), 3) -\`\`\` +${'`pow(last_value(length), 3)`'} `, }, round: { @@ -429,9 +407,7 @@ round(sum(bytes), 2) Square root of a positive value only Example: Calculate side length based on area -\`\`\` -sqrt(last_value(area)) -\`\`\` +${'`sqrt(last_value(area))`'} `, }, square: { @@ -445,9 +421,7 @@ sqrt(last_value(area)) Raise the value to the 2nd power Example: Calculate area based on side length -\`\`\` -square(last_value(length)) -\`\`\` +${'`square(last_value(length))`'} `, }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 97ed6eaf50b698..908dcf8e2bbe6c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -268,7 +268,7 @@ export const lastValueOperation: OperationDefinition>({ documentation: { section: 'elasticsearch', signature: i18n.translate('xpack.lens.indexPattern.metric.signature', { - defaultMessage: 'field: string, [kql]?: string, [lucene]?: string', + defaultMessage: 'field: string', }), description: i18n.translate('xpack.lens.indexPattern.metric.documentation', { defaultMessage: ` Returns the {metric} of a field. This function only works for number fields. Example: Get the {metric} of price: -\`\`\` -{metric}(price) -\`\`\` +${'`{metric}(price)`'} Example: Get the {metric} of price for orders from the UK: -\`\`\` -{metric}(price, kql="location:UK") -\`\`\` +${"`{metric}(price, kql='location:UK')`"} `, values: { metric: type, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 6afc9f2f53abb7..2ce8818853943b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -198,16 +198,14 @@ export const percentileOperation: OperationDefinition Date: Mon, 24 May 2021 19:26:18 +0200 Subject: [PATCH 121/185] :sparkles: Add query validation for quotes --- .../definitions/formula/formula.test.tsx | 62 +++++++++++++ .../definitions/formula/validation.ts | 92 ++++++++++++++++++- 2 files changed, 150 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 9862099c3b1ba5..2bee74e94b6e90 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -899,6 +899,68 @@ invalid: " ).toEqual(undefined); }); + it('returns an error for a query not wrapped in single quotes', () => { + const formulas = [ + `count(kql="category.keyword: *")`, + `count(kql=category.keyword: *)`, + `count(lucene="category.keyword: *")`, + `count(lucene=category.keyword: *)`, + `count(lucene=category.keyword: *) + average(bytes)`, + `count(lucene='category.keyword: *') + count(kql=category.keyword: *)`, + `count(lucene='category.keyword: *') + count(kql=category.keyword: *, kql='category.keyword: *')`, + `count(lucene='category.keyword: *') + count(kql="category.keyword: *")`, + `moving_average(count(kql=category.keyword: *), window=7, kql=category.keywork: *)`, + `moving_average( + cumulative_sum( + 7 * clamp(sum(bytes), 0, last_value(memory) + max(memory)) + ), window=10, kql=category.keywork: * + )`, + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual( + expect.arrayContaining([ + expect.stringMatching(`query for the operation must be wrapped in single quotes`), + ]) + ); + } + }); + + it('returns no error for a query wrapped in single quotes but with some whitespaces', () => { + const formulas = [ + `count(kql ='category.keyword: *')`, + `count(kql = 'category.keyword: *')`, + `count(kql = 'category.keyword: *')`, + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + + it('returns an error for multiple queries submitted for the same function', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`count(kql='category.keyword: *', lucene='category.keyword: *')`), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['The operation count contains too many queries']); + }); + it('returns no error if a math operation is passed to fullReference operations', () => { const formulas = [ 'derivative(7+1)', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index fa73cb69b20f04..756d4aedfe6276 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isObject } from 'lodash'; +import { isObject, partition } from 'lodash'; import { i18n } from '@kbn/i18n'; import { parse, TinymathLocation } from '@kbn/tinymath'; import type { TinymathAST, TinymathFunction, TinymathNamedArgument } from '@kbn/tinymath'; @@ -58,6 +58,10 @@ interface ValidationErrors { message: string; type: { operation: string; count: number; params: string }; }; + tooManyQueries: { + message: string; + type: { operation: string }; + }; } type ErrorTypes = keyof ValidationErrors; type ErrorValues = ValidationErrors[K]['type']; @@ -94,11 +98,61 @@ export function hasInvalidOperations( }; } +export const getRawQueryValidationError = (text: string) => { + // try to extract the query context here + const singleLine = text.split('\n').join(''); + const allArgs = singleLine.split(',').filter((args) => /(kql|lucene)/.test(args)); + // no args, so no problem + if (allArgs.length === 0) { + return; + } + // at this point each entry in allArgs may contain one or more + // in the worst case it would be a math chain of count operation + // For instance: count(kql=...) + count(lucene=...) - count(kql=...) + // therefore before partition them, split them by "count" keywork and filter only string with a length + const flattenArgs = allArgs.flatMap((arg) => + arg.split('count').filter((subArg) => /(kql|lucene)/.test(subArg)) + ); + const [kqlQueries, luceneQueries] = partition(flattenArgs, (arg) => /kql/.test(arg)); + const errors = []; + for (const kqlQuery of kqlQueries) { + const result = validateQueryQuotes(kqlQuery, 'kql'); + if (result) { + errors.push(result); + } + } + for (const luceneQuery of luceneQueries) { + const result = validateQueryQuotes(luceneQuery, 'lucene'); + if (result) { + errors.push(result); + } + } + return errors.length ? errors : undefined; +}; + +const validateQueryQuotes = (rawQuery: string, language: 'kql' | 'lucene') => { + // check if the raw argument has the minimal requirements + const [_, rawValue] = rawQuery.split('='); + // it must start with a single quote + if (rawValue.trim()[0] !== "'") { + return i18n.translate('xpack.lens.indexPattern.formulaOperationQueryError', { + defaultMessage: + 'The {language} query for the operation must be wrapped in single quotes: {rawQuery}', + values: { language, rawQuery }, + }); + } +}; + export const getQueryValidationError = ( - query: string, - language: 'kql' | 'lucene', + { value: query, name: language, text }: { value: string; name: 'kql' | 'lucene'; text: string }, indexPattern: IndexPattern ): string | undefined => { + // check if the raw argument has the minimal requirements + const result = validateQueryQuotes(text, language); + // forward the error here is ok? + if (result) { + return result; + } try { if (language === 'kql') { esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query), indexPattern); @@ -203,6 +257,12 @@ function getMessageFromId({ values: { operation: out.operation, count: out.count, params: out.params }, }); break; + case 'tooManyQueries': + message = i18n.translate('xpack.lens.indexPattern.formulaOperationDoubleQueryError', { + defaultMessage: 'The operation {operation} contains too many queries', + values: { operation: out.operation }, + }); + break; // case 'mathRequiresFunction': // message = i18n.translate('xpack.lens.indexPattern.formulaMathRequiresFunctionLabel', { // defaultMessage; 'The function {name} requires an Elasticsearch function', @@ -224,6 +284,12 @@ export function tryToParse( try { root = parse(formula); } catch (e) { + // give it last try + const maybeQueryProblems = getRawQueryValidationError(formula); + if (maybeQueryProblems) { + // need to emulate an error shape here + return { root: null, error: { message: maybeQueryProblems[0], locations: [] } }; + } return { root: null, error: getMessageFromId({ @@ -319,7 +385,7 @@ function getQueryValidationErrors( const errors: ErrorWrapper[] = []; (namedArguments ?? []).forEach((arg) => { if (arg.name === 'kql' || arg.name === 'lucene') { - const message = getQueryValidationError(arg.value, arg.name, indexPattern); + const message = getQueryValidationError(arg, indexPattern); if (message) { errors.push({ message, @@ -331,6 +397,12 @@ function getQueryValidationErrors( return errors; } +function checkSingleQuery(namedArguments: TinymathNamedArgument[] | undefined) { + return namedArguments + ? namedArguments.filter((arg) => arg.name === 'kql' || arg.name === 'lucene').length > 1 + : undefined; +} + function validateNameArguments( node: TinymathFunction, nodeOperation: @@ -383,6 +455,18 @@ function validateNameArguments( if (queryValidationErrors.length) { errors.push(...queryValidationErrors); } + const hasTooManyQueries = checkSingleQuery(namedArguments); + if (hasTooManyQueries) { + errors.push( + getMessageFromId({ + messageId: 'tooManyQueries', + values: { + operation: node.name, + }, + locations: [node.location], + }) + ); + } return errors; } From 97a2457361bc5b7e1c5b6d904891278f14f46504 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 24 May 2021 16:46:06 -0400 Subject: [PATCH 122/185] Usability updates & type fixes --- .../definitions/formula/editor/formula.scss | 11 ++ .../formula/editor/formula_editor.tsx | 36 ++++-- .../formula/editor/formula_help.tsx | 121 ++++++++++-------- .../formula/editor/math_completion.ts | 2 +- 4 files changed, 110 insertions(+), 60 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 56d0960862e7c0..432e17ff7fa79b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -47,9 +47,20 @@ } .lnsFormula__editorContent { + position: relative; height: 201px; } +.lnsFormula__editorPlaceholder { + position: absolute; + top: 80px; + left: 0; + right: 0; + // Matches monaco editor + font-family: Menlo, Monaco, 'Courier New', monospace; + text-align: center; +} + .lnsIndexPatternDimensionEditor-isFullscreen .lnsFormula__editorContent { flex: 1; min-height: 201px; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 0f3355fe3acdc4..c0a773b252aa82 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -17,6 +17,7 @@ import { EuiPopover, EuiText, EuiToolTip, + EuiMarkdownFormat, } from '@elastic/eui'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; @@ -421,6 +422,10 @@ export function FormulaEditor({ [] ); + const closePopover = useCallback(() => { + setIsHelpOpen(false); + }, []); + const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -562,6 +567,18 @@ export function FormulaEditor({ ); }} /> + + {!text ? ( +
+ + {i18n.translate('xpack.lens.formulaPlaceholderText', { + defaultMessage: 'Type a formula by combining functions with math, like:', + })} + + + 
count() + 1
+
+ ) : null}
@@ -606,26 +623,26 @@ export function FormulaEditor({ panelPaddingSize="none" anchorPosition="leftCenter" isOpen={isHelpOpen} - closePopover={() => setIsHelpOpen(false)} + closePopover={() => {}} button={ - setIsHelpOpen(!isHelpOpen)} iconType="help" color="text" - aria-label={i18n.translate( - 'xpack.lens.formula.editorHelpOverlayLabel', - { - defaultMessage: 'Function reference', - } - )} - /> + size="s" + > + {i18n.translate('xpack.lens.formula.editorHelpOverlayLabel', { + defaultMessage: 'Function reference', + })} + } > @@ -664,6 +681,7 @@ export function FormulaEditor({ isFullscreen={isFullscreen} indexPattern={indexPattern} operationDefinitionMap={operationDefinitionMap} + closeHelp={closePopover} />
) : null} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 14e6ed2ffbded6..fecd1c40fa8d28 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -17,6 +17,7 @@ import { EuiListGroup, EuiMarkdownFormat, EuiTitle, + EuiButtonIcon, } from '@elastic/eui'; import { IndexPattern } from '../../../../types'; import { tinymathFunctions } from '../util'; @@ -35,10 +36,12 @@ function FormulaHelp({ indexPattern, operationDefinitionMap, isFullscreen, + closeHelp, }: { indexPattern: IndexPattern; operationDefinitionMap: Record; isFullscreen: boolean; + closeHelp: () => void; }) { const [selectedFunction, setSelectedFunction] = useState(); const scrollTargets = useRef>({}); @@ -49,57 +52,12 @@ function FormulaHelp({ } }, [selectedFunction]); - const tinymathFns = useMemo(() => { - return getPossibleFunctions(indexPattern) - .filter((key) => key in tinymathFunctions) - .sort() - .map((key) => { - const [description, examples] = tinymathFunctions[key].help.split(`\`\`\``); - return { - label: key, - description: description.replace(/\n/g, '\n\n'), - examples: examples ? `\`\`\`${examples}\`\`\`` : '', - }; - }); - }, [indexPattern]); - const helpGroups: Array<{ label: string; description?: JSX.Element; items: Array<{ label: string; description?: JSX.Element }>; }> = []; - helpGroups.push({ - label: i18n.translate('xpack.lens.formulaDocumentation.mathSection', { - defaultMessage: 'Math', - }), - description: ( - - {i18n.translate('xpack.lens.formulaDocumentation.mathSectionDescription', { - defaultMessage: - 'These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.', - })} - - ), - items: [], - }); - - helpGroups[0].items.push( - ...tinymathFns.map(({ label, description, examples }) => { - return { - label, - description: ( - <> - -

{getFunctionSignatureLabel(label, operationDefinitionMap)}

- - {`${description}${examples}`} - - ), - }; - }) - ); - helpGroups.push({ label: i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSection', { defaultMessage: 'Elasticsearch', @@ -118,7 +76,7 @@ function FormulaHelp({ const availableFunctions = getPossibleFunctions(indexPattern); // Es aggs - helpGroups[1].items.push( + helpGroups[0].items.push( ...availableFunctions .filter( (key) => @@ -161,7 +119,7 @@ function FormulaHelp({ }); // Calculations aggs - helpGroups[2].items.push( + helpGroups[1].items.push( ...availableFunctions .filter( (key) => @@ -192,12 +150,75 @@ function FormulaHelp({ })) ); + helpGroups.push({ + label: i18n.translate('xpack.lens.formulaDocumentation.mathSection', { + defaultMessage: 'Math', + }), + description: ( + + {i18n.translate('xpack.lens.formulaDocumentation.mathSectionDescription', { + defaultMessage: + 'These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.', + })} + + ), + items: [], + }); + + const tinymathFns = useMemo(() => { + return getPossibleFunctions(indexPattern) + .filter((key) => key in tinymathFunctions) + .sort() + .map((key) => { + const [description, examples] = tinymathFunctions[key].help.split(`\`\`\``); + return { + label: key, + description: description.replace(/\n/g, '\n\n'), + examples: examples ? `\`\`\`${examples}\`\`\`` : '', + }; + }); + }, [indexPattern]); + + helpGroups[2].items.push( + ...tinymathFns.map(({ label, description, examples }) => { + return { + label, + description: ( + <> + +

{getFunctionSignatureLabel(label, operationDefinitionMap)}

+ + {`${description}${examples}`} + + ), + }; + }) + ); + return ( <> - {i18n.translate('xpack.lens.formulaDocumentation.header', { - defaultMessage: 'Formula reference', - })} + + + {i18n.translate('xpack.lens.formulaDocumentation.header', { + defaultMessage: 'Formula reference', + })} + + + {!isFullscreen ? ( + { + closeHelp(); + }} + aria-label={i18n.translate('xpack.lens.dimensionContainer.close', { + defaultMessage: 'Close', + })} + /> + ) : null} + + diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index e76a53b49a7f2c..0b70109f04138b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -280,7 +280,7 @@ function getArgumentSuggestions( (o) => operation.requiredReferences.some((requirement) => requirement.input.includes(o.type) - ) && !o.hidden + ) && !operationDefinitionMap[o.operationType].hidden ) .map((o) => o.operationType) ); From b84faf9c2b1ddd09e33e90768815c40972708ac5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 14:24:43 +0200 Subject: [PATCH 123/185] add search to formula --- .../definitions/formula/editor/formula.scss | 11 ++ .../formula/editor/formula_help.tsx | 147 ++++++++++++------ 2 files changed, 110 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 432e17ff7fa79b..8e886f22a2558e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -107,6 +107,17 @@ } } +.lnsFormula__docsSidebar { + height: 100%; +} + +.lnsFormula__docsSearch { + border: none; + border-bottom: $euiBorderThin; + box-shadow: none; + border-radius: 0; +} + .lnsFormula__docsNav { @include euiYScroll; background: $euiColorLightestShade; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index fecd1c40fa8d28..0ee81e7f381614 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -18,6 +18,8 @@ import { EuiMarkdownFormat, EuiTitle, EuiButtonIcon, + EuiFieldSearch, + EuiHighlight, } from '@elastic/eui'; import { IndexPattern } from '../../../../types'; import { tinymathFunctions } from '../util'; @@ -58,6 +60,13 @@ function FormulaHelp({ items: Array<{ label: string; description?: JSX.Element }>; }> = []; + helpGroups.push({ + label: i18n.translate('xpack.lens.formulaDocumentationHeading', { + defaultMessage: 'How it works', + }), + items: [], + }); + helpGroups.push({ label: i18n.translate('xpack.lens.formulaDocumentation.elasticsearchSection', { defaultMessage: 'Elasticsearch', @@ -76,7 +85,7 @@ function FormulaHelp({ const availableFunctions = getPossibleFunctions(indexPattern); // Es aggs - helpGroups[0].items.push( + helpGroups[1].items.push( ...availableFunctions .filter( (key) => @@ -119,7 +128,7 @@ function FormulaHelp({ }); // Calculations aggs - helpGroups[1].items.push( + helpGroups[2].items.push( ...availableFunctions .filter( (key) => @@ -179,7 +188,7 @@ function FormulaHelp({ }); }, [indexPattern]); - helpGroups[2].items.push( + helpGroups[3].items.push( ...tinymathFns.map(({ label, description, examples }) => { return { label, @@ -195,6 +204,8 @@ function FormulaHelp({ }) ); + const [searchText, setSearchText] = useState(''); + return ( <> @@ -221,48 +232,87 @@ function FormulaHelp({ - - - {helpGroups.map((helpGroup, index) => { - return ( - - ); - })} + + + + + { + setSearchText(e.target.value); + }} + placeholder={i18n.translate('xpack.lens.formulaSearchPlaceholder', { + defaultMessage: 'Search functions', + })} + /> + + + {helpGroups.map((helpGroup, index) => { + return ( + + ); + })} + + - - {i18n.translate('xpack.lens.formulaDocumentation', { - defaultMessage: ` +
{ + if (el) { + scrollTargets.current[helpGroups[0].label] = el; + } + }} + > + + {i18n.translate('xpack.lens.formulaDocumentation', { + defaultMessage: ` ## How it works Lens formulas let you do math using a combination of Elasticsearch aggregations and @@ -295,12 +345,13 @@ Math functions can take positional arguments, like pow(count(), 3) is the same a Use the symbols +, -, /, and * to perform basic math. `, - description: - 'Text is in markdown. Do not translate function names or field names like sum(bytes)', - })} - + description: + 'Text is in markdown. Do not translate function names or field names like sum(bytes)', + })} + +
- {helpGroups.map((helpGroup, index) => { + {helpGroups.slice(1).map((helpGroup, index) => { return (
{ + {helpGroups[index + 1].items.map((helpItem) => { return (
Date: Tue, 25 May 2021 14:48:31 -0400 Subject: [PATCH 124/185] fix form styles to match designs --- .../definitions/formula/editor/formula.scss | 16 ++++++++++------ .../definitions/formula/editor/formula_help.tsx | 11 +++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 8e886f22a2558e..0a3e78351e9ee7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -108,19 +108,23 @@ } .lnsFormula__docsSidebar { - height: 100%; + background: $euiColorLightestShade; +} + +.lnsFormula__docsSidebarInner { + min-height: 0; + + & > * + * { + border-top: $euiBorderThin; + } } .lnsFormula__docsSearch { - border: none; - border-bottom: $euiBorderThin; - box-shadow: none; - border-radius: 0; + padding: $euiSizeS; } .lnsFormula__docsNav { @include euiYScroll; - background: $euiColorLightestShade; } .lnsFormula__docsNavGroup { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 0ee81e7f381614..89613a33356394 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -238,16 +238,14 @@ function FormulaHelp({ responsive={false} alignItems="stretch" > - + - + { setSearchText(e.target.value); @@ -257,7 +255,8 @@ function FormulaHelp({ })} /> - + + {helpGroups.map((helpGroup, index) => { return (
- {isFullscreen ? ( - children - ) : ( - - -

- {title || - i18n.translate('xpack.lens.chartTitle.unsaved', { - defaultMessage: 'Unsaved visualization', - })} -

- - {children} - - )} + + +

+ {title || + i18n.translate('xpack.lens.chartTitle.unsaved', { + defaultMessage: 'Unsaved visualization', + })} +

+ + {children} + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ed8c163283acdf..712ac618f2ee60 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -18,7 +18,8 @@ import { EuiFormLabel, EuiToolTip, EuiText, - EuiTabbedContent, + EuiTabs, + EuiTab, } from '@elastic/eui'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; @@ -104,15 +105,28 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const { fieldByOperation, operationWithoutField } = operationSupportMatrix; - const setStateWrapper = (layer: IndexPatternLayer) => { - const hasIncompleteColumns = Boolean(layer.incompleteColumns?.[columnId]); + const setStateWrapper = ( + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), + shouldClose?: boolean + ) => { + const hypotheticalLayer = typeof setter === 'function' ? setter(state.layers[layerId]) : setter; + const hasIncompleteColumns = Boolean(hypotheticalLayer.incompleteColumns?.[columnId]); const prevOperationType = - operationDefinitionMap[state.layers[layerId].columns[columnId]?.operationType]?.input; - setState(mergeLayer({ state, layerId, newLayer: layer }), { - shouldReplaceDimension: Boolean(layer.columns[columnId]), - // clear the dimension if there's an incomplete column pending && previous operation was a fullReference operation - shouldRemoveDimension: Boolean(hasIncompleteColumns && prevOperationType === 'fullReference'), - }); + operationDefinitionMap[hypotheticalLayer.columns[columnId]?.operationType]?.input; + setState( + (prevState) => { + const layer = typeof setter === 'function' ? setter(prevState.layers[layerId]) : setter; + return mergeLayer({ state: prevState, layerId, newLayer: layer }); + }, + { + shouldReplaceDimension: Boolean(hypotheticalLayer.columns[columnId]), + // clear the dimension if there's an incomplete column pending && previous operation was a fullReference operation + shouldRemoveDimension: Boolean( + hasIncompleteColumns && prevOperationType === 'fullReference' + ), + shouldClose: Boolean(shouldClose), + } + ); }; const setIsCloseable = (isCloseable: boolean) => { @@ -357,8 +371,20 @@ export function DimensionEditor(props: DimensionEditorProps) { key={index} layer={state.layers[layerId]} columnId={referenceId} - updateLayer={(newLayer: IndexPatternLayer) => { - setState(mergeLayer({ state, layerId, newLayer })); + updateLayer={( + setter: + | IndexPatternLayer + | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), + shouldClose?: boolean + ) => { + setState( + mergeLayer({ + state, + layerId, + newLayer: + typeof setter === 'function' ? setter(state.layers[layerId]) : setter, + }) + ); }} validation={validation} currentIndexPattern={currentIndexPattern} @@ -518,82 +544,77 @@ export function DimensionEditor(props: DimensionEditorProps) { ); - const tabs = [ - { - id: 'quickFunctions', - name: i18n.translate('xpack.lens.indexPattern.quickFunctionsLabel', { - defaultMessage: 'Quick functions', - }), - 'data-test-subj': 'lens-dimensionTabs-quickFunctions', - content: quickFunctions, - }, - { - id: 'formula', - name: i18n.translate('xpack.lens.indexPattern.formulaLabel', { - defaultMessage: 'Formula', - }), - 'data-test-subj': 'lens-dimensionTabs-formula', - content: ParamEditor ? ( - <> - - - ) : ( - <> - ), - }, - ]; + const formulaTab = ParamEditor ? ( + <> + + + ) : ( + <> + ); return (
- {isFullscreen ? ( - tabs[1].content - ) : operationSupportMatrix.operationWithoutField.has('formula') ? ( - { - if ( - selectedTab.id === 'quickFunctions' && - selectedColumn?.operationType === 'formula' - ) { - setQuickFunction(true); - } else if (selectedColumn?.operationType !== 'formula') { - setQuickFunction(false); - const newLayer = insertOrReplaceColumn({ - layer: props.state.layers[props.layerId], - indexPattern: currentIndexPattern, - columnId, - op: 'formula', - visualizationGroups: dimensionGroups, - }); - setStateWrapper(newLayer); - trackUiEvent(`indexpattern_dimension_operation_formula`); - return; - } else if (selectedTab.id === 'formula') { - setQuickFunction(false); - } - }} - size="s" - /> - ) : ( - quickFunctions - )} + {!isFullscreen ? ( + + { + if (selectedColumn?.operationType === 'formula') { + setQuickFunction(true); + } + }} + > + {i18n.translate('xpack.lens.indexPattern.quickFunctionsLabel', { + defaultMessage: 'Quick functions', + })} + + {operationSupportMatrix.operationWithoutField.has('formula') ? ( + { + if (selectedColumn?.operationType !== 'formula') { + setQuickFunction(false); + const newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: 'formula', + visualizationGroups: dimensionGroups, + }); + setStateWrapper(newLayer); + trackUiEvent(`indexpattern_dimension_operation_formula`); + return; + } else { + setQuickFunction(false); + } + }} + > + {i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', + })} + + ) : null} + + ) : null} + + {isFullscreen + ? formulaTab + : selectedOperationDefinition?.type === 'formula' && !temporaryQuickFunction + ? formulaTab + : quickFunctions} {!isFullscreen && !currentFieldIsInvalid && !temporaryQuickFunction && (
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index 2a82f4885f3657..52187a06a3ca75 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -44,8 +44,12 @@ export interface ReferenceEditorProps { selectionStyle: 'full' | 'field' | 'hidden'; validation: RequiredReference; columnId: string; - updateLayer: (newLayer: IndexPatternLayer) => void; + updateLayer: ( + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), + shouldClose?: boolean + ) => void; currentIndexPattern: IndexPattern; + existingFields: IndexPatternPrivateState['existingFields']; dateRange: DateRange; labelAppend?: EuiFormRowProps['labelAppend']; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 432e17ff7fa79b..6dd308798e3109 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -53,12 +53,13 @@ .lnsFormula__editorPlaceholder { position: absolute; - top: 80px; - left: 0; + top: 0; + left: $euiSize; right: 0; + color: $euiTextSubduedColor; // Matches monaco editor font-family: Menlo, Monaco, 'Courier New', monospace; - text-align: center; + pointer-events: none; } .lnsIndexPatternDimensionEditor-isFullscreen .lnsFormula__editorContent { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index c0a773b252aa82..100c85ecc8abf6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -17,8 +17,9 @@ import { EuiPopover, EuiText, EuiToolTip, - EuiMarkdownFormat, + EuiSpacer, } from '@elastic/eui'; +import useUnmount from 'react-use/lib/useUnmount'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; @@ -63,9 +64,7 @@ export function FormulaEditor({ const [text, setText] = useState(currentColumn.params.formula); const [warnings, setWarnings] = useState>([]); const [isHelpOpen, setIsHelpOpen] = useState(isFullscreen); - const editorModel = React.useRef( - monaco.editor.createModel(text ?? '', LANGUAGE_ID) - ); + const editorModel = React.useRef(); const overflowDiv1 = React.useRef(); const disposables = React.useRef([]); const editor1 = React.useRef(); @@ -83,17 +82,33 @@ export function FormulaEditor({ // Clean up the monaco editor and DOM on unmount useEffect(() => { - const model = editorModel.current; - const allDisposables = disposables.current; - const editor1ref = editor1.current; + const model = editorModel; + const allDisposables = disposables; + const editor1ref = editor1; return () => { - model.dispose(); + model.current?.dispose(); overflowDiv1.current?.parentNode?.removeChild(overflowDiv1.current); - editor1ref?.dispose(); - allDisposables?.forEach((d) => d.dispose()); + editor1ref.current?.dispose(); + allDisposables.current?.forEach((d) => d.dispose()); }; }, []); + useUnmount(() => { + // If the text is not synced, update the column. + if (text !== currentColumn.params.formula) { + updateLayer((prevLayer) => { + return regenerateLayerFromAst( + text || '', + prevLayer, + columnId, + currentColumn, + indexPattern, + operationDefinitionMap + ).newLayer; + }, true); + } + }); + useDebounceWithOptions( () => { if (!editorModel.current) return; @@ -422,10 +437,6 @@ export function FormulaEditor({ [] ); - const closePopover = useCallback(() => { - setIsHelpOpen(false); - }, []); - const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -514,12 +525,15 @@ export function FormulaEditor({ { toggleFullscreen(); + // Help text opens when entering full screen, and closes when leaving full screen + setIsHelpOpen(!isFullscreen); trackUiEvent('toggle_formula_fullscreen'); }} iconType={isFullscreen ? 'bolt' : 'fullScreen'} size="xs" color="text" flush="right" + data-test-subj="lnsFormula-fullscreen" > {isFullscreen ? i18n.translate('xpack.lens.formula.fullScreenExitLabel', { @@ -540,10 +554,13 @@ export function FormulaEditor({ ...codeEditorOptions.options, // Shared model and overflow node overflowWidgetsDomNode: overflowDiv1.current, - model: editorModel.current, }} editorDidMount={(editor) => { editor1.current = editor; + const model = editor.getModel(); + if (model) { + editorModel.current = model; + } disposables.current.push( editor.onDidFocusEditorWidget(() => { setTimeout(() => { @@ -570,12 +587,12 @@ export function FormulaEditor({ {!text ? (
- + {i18n.translate('xpack.lens.formulaPlaceholderText', { defaultMessage: 'Type a formula by combining functions with math, like:', })} - - + + 
count() + 1
) : null} @@ -615,7 +632,6 @@ export function FormulaEditor({ content={i18n.translate('xpack.lens.formula.editorHelpOverlayToolTip', { defaultMessage: 'Function reference', })} - delay="long" position="top" > {}} + closePopover={() => setIsHelpOpen(false)} + ownFocus={false} button={ - {i18n.translate('xpack.lens.formula.editorHelpOverlayLabel', { - defaultMessage: 'Function reference', - })} - + /> } > @@ -651,24 +663,33 @@ export function FormulaEditor({ {errorCount || warningCount ? ( - {errorCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaErrorCount', { - defaultMessage: '{count} {count, plural, one {error} other {errors}}', - values: { count: errorCount }, - })} - - ) : null} - {warningCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaWarningCount', { - defaultMessage: '{count} {count, plural, one {warning} other {warnings}}', - values: { count: warningCount }, - })} - - ) : null} + { + editor1.current?.trigger('LENS', 'editor.action.marker.next', {}); + }} + > + {errorCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaErrorCount', { + defaultMessage: '{count} {count, plural, one {error} other {errors}}', + values: { count: errorCount }, + })} + + ) : null} + {warningCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaWarningCount', { + defaultMessage: + '{count} {count, plural, one {warning} other {warnings}}', + values: { count: warningCount }, + })} + + ) : null} + ) : null} @@ -681,7 +702,6 @@ export function FormulaEditor({ isFullscreen={isFullscreen} indexPattern={indexPattern} operationDefinitionMap={operationDefinitionMap} - closeHelp={closePopover} />
) : null} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index fecd1c40fa8d28..5e4a29e37d551f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -17,7 +17,6 @@ import { EuiListGroup, EuiMarkdownFormat, EuiTitle, - EuiButtonIcon, } from '@elastic/eui'; import { IndexPattern } from '../../../../types'; import { tinymathFunctions } from '../util'; @@ -36,12 +35,10 @@ function FormulaHelp({ indexPattern, operationDefinitionMap, isFullscreen, - closeHelp, }: { indexPattern: IndexPattern; operationDefinitionMap: Record; isFullscreen: boolean; - closeHelp: () => void; }) { const [selectedFunction, setSelectedFunction] = useState(); const scrollTargets = useRef>({}); @@ -198,27 +195,9 @@ function FormulaHelp({ return ( <> - - - {i18n.translate('xpack.lens.formulaDocumentation.header', { - defaultMessage: 'Formula reference', - })} - - - {!isFullscreen ? ( - { - closeHelp(); - }} - aria-label={i18n.translate('xpack.lens.dimensionContainer.close', { - defaultMessage: 'Close', - })} - /> - ) : null} - - + {i18n.translate('xpack.lens.formulaDocumentation.header', { + defaultMessage: 'Formula reference', + })} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 796a8ae97deda1..4b11ee6531d298 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -76,6 +76,10 @@ export const formulaOperation: OperationDefinition< : params?.formula : ''; + if (!params.formula || params.isFormulaBroken) { + return []; + } + return [ { type: 'function', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index c54d1b63781425..f929629022e48f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -149,7 +149,10 @@ export { formulaOperation } from './formula/formula'; export interface ParamEditorProps { currentColumn: C; layer: IndexPatternLayer; - updateLayer: (newLayer: IndexPatternLayer) => void; + updateLayer: ( + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), + shouldClose?: boolean + ) => void; toggleFullscreen: () => void; setIsCloseable: (isCloseable: boolean) => void; isFullscreen: boolean; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 63a54cf2f59683..bc5dbd82ce4244 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -306,7 +306,11 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro // Not a StateSetter because we have this unique use case of determining valid columns setState: ( newState: Parameters>[0], - publishToVisualization?: { shouldReplaceDimension?: boolean; shouldRemoveDimension?: boolean } + publishToVisualization?: { + shouldReplaceDimension?: boolean; + shouldRemoveDimension?: boolean; + shouldClose?: boolean; + } ) => void; core: Pick; dateRange: DateRange; diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index 798cb7d3146f55..f8097f243cf17d 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -52,7 +52,63 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await input.type('*'); await PageObjects.header.waitUntilLoadingHasFinished(); - expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14005'); + expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14,005'); + }); + + it('should persist a broken formula on close', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + // Close immediately + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `asdf`, + }); + + expect(await PageObjects.lens.getErrorCount()).to.eql(1); + }); + + it('should keep the formula when entering expanded mode', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + // Close immediately + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count()`, + keepOpen: true, + }); + + await PageObjects.lens.toggleFullscreen(); + + const element = await find.byCssSelector('.monaco-editor'); + expect(await element.getVisibleText()).to.equal('count()'); + }); + + it('should allow an empty formula combined with a valid formula', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count()`, + }); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + }); + + await PageObjects.header.waitUntilLoadingHasFinished(); + expect(await PageObjects.lens.getErrorCount()).to.eql(0); }); it('should duplicate a moving average formula and be a valid table', async () => { diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index f312b380cf6d84..a720e2e8db729c 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -935,6 +935,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click('lens-dimensionTabs-formula'); }, + async toggleFullscreen() { + await testSubjects.click('lnsFormula-fullscreen'); + }, + async typeFormula(formula: string) { await find.byCssSelector('.monaco-editor'); await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); From 232e02d679a3a00e3cf4cbce8e861a6b6f6dada6 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 26 May 2021 19:10:30 -0400 Subject: [PATCH 132/185] Fix documentation tests --- .../formula/editor/math_completion.test.ts | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 9e29160b6747b4..3ed0b246639aac 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -39,6 +39,11 @@ const operationDefinitionMap: Record = { buildColumn: buildGenericColumn('sum'), getPossibleOperationForField: (field: IndexPatternField) => field.type === 'number' ? numericOperation() : null, + documentation: { + section: 'elasticsearch', + signature: 'field: string', + description: 'description', + }, } as unknown) as GenericOperationDefinition, count: ({ type: 'count', @@ -95,14 +100,16 @@ describe('math completion', () => { it('should return a signature for a field-based ES function', () => { expect(unwrapSignatures(getSignatureHelp('sum()', 4, operationDefinitionMap))).toEqual({ - label: 'sum(field)', + label: 'sum(field: string)', + documentation: { value: 'description' }, parameters: [{ label: 'field' }], }); }); it('should return a signature for count', () => { expect(unwrapSignatures(getSignatureHelp('count()', 6, operationDefinitionMap))).toEqual({ - label: 'count()', + label: 'count(undefined)', + documentation: { value: '' }, parameters: [], }); }); @@ -113,7 +120,8 @@ describe('math completion', () => { getSignatureHelp('2 * moving_average(count(), window=)', 35, operationDefinitionMap) ) ).toEqual({ - label: 'moving_average(function, window=number)', + label: expect.stringContaining('moving_average('), + documentation: { value: '' }, parameters: [ { label: 'function' }, { @@ -129,16 +137,21 @@ describe('math completion', () => { unwrapSignatures( getSignatureHelp('2 * moving_average(count())', 25, operationDefinitionMap) ) - ).toEqual({ label: 'count()', parameters: [] }); + ).toEqual({ + label: expect.stringContaining('count('), + parameters: [], + documentation: { value: '' }, + }); }); it('should return a signature for a complex tinymath function', () => { expect( unwrapSignatures(getSignatureHelp('clamp(count(), 5)', 7, operationDefinitionMap)) ).toEqual({ - label: 'clamp(expression, min, max)', + label: expect.stringContaining('clamp('), + documentation: { value: '' }, parameters: [ - { label: 'expression', documentation: '' }, + { label: 'value', documentation: '' }, { label: 'min', documentation: '' }, { label: 'max', documentation: '' }, ], @@ -153,31 +166,43 @@ describe('math completion', () => { it('should show signature for a field-based ES function', () => { expect(getHover('sum()', 2, operationDefinitionMap)).toEqual({ - contents: [{ value: 'sum(field)' }], + contents: [{ value: 'sum(field: string)' }, { value: expect.stringContaining('Example') }], }); }); it('should show signature for count', () => { expect(getHover('count()', 2, operationDefinitionMap)).toEqual({ - contents: [{ value: 'count()' }], + contents: [ + { value: expect.stringContaining('count(') }, + { value: expect.stringContaining('Example') }, + ], }); }); it('should show signature for a function with named parameters', () => { expect(getHover('2 * moving_average(count())', 10, operationDefinitionMap)).toEqual({ - contents: [{ value: 'moving_average(function, window=number)' }], + contents: [ + { value: expect.stringContaining('moving_average(') }, + { value: expect.stringContaining('Example') }, + ], }); }); it('should show signature for an inner function', () => { expect(getHover('2 * moving_average(count())', 22, operationDefinitionMap)).toEqual({ - contents: [{ value: 'count()' }], + contents: [ + { value: expect.stringContaining('count(') }, + { value: expect.stringContaining('Example') }, + ], }); }); it('should show signature for a complex tinymath function', () => { expect(getHover('clamp(count(), 5)', 2, operationDefinitionMap)).toEqual({ - contents: [{ value: 'clamp(expression, min, max)' }], + contents: [ + { value: expect.stringContaining('clamp([value]: number') }, + { value: expect.stringContaining('Example') }, + ], }); }); }); From 759e168ce8da5ffe07defed1931d7a30c1ee66d9 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 27 May 2021 10:01:27 +0200 Subject: [PATCH 133/185] :label: fix type issue --- .../operations/definitions/formula/validation.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 675d7b6458f35a..555a4a261cfc8f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -136,7 +136,7 @@ export const getRawQueryValidationError = (text: string, operations: Record { // check if the raw argument has the minimal requirements - const [_, rawValue] = rawQuery.split('='); + const [, rawValue] = rawQuery.split('='); // it must start with a single quote, and quotes must have a closure if (rawValue.trim()[0] !== "'" || !/'\s*([^']+?)\s*'/.test(rawValue)) { return i18n.translate('xpack.lens.indexPattern.formulaOperationQueryError', { @@ -391,7 +391,10 @@ function getQueryValidationErrors( const errors: ErrorWrapper[] = []; (namedArguments ?? []).forEach((arg) => { if (arg.name === 'kql' || arg.name === 'lucene') { - const message = getQueryValidationError(arg, indexPattern); + const message = getQueryValidationError( + arg as { value: string; name: 'kql' | 'lucene'; text: string }, + indexPattern + ); if (message) { errors.push({ message, From 0542a1072235e3eeae940408020f15782e46ac79 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 27 May 2021 11:22:37 +0200 Subject: [PATCH 134/185] :bug: Remove hidden operations from valid functions list --- .../formula/editor/formula_editor.tsx | 38 +++++++++---------- .../definitions/formula/formula.test.tsx | 28 ++++++++++++++ .../definitions/formula/formula.tsx | 6 ++- .../operations/definitions/formula/parse.ts | 9 ++++- .../operations/definitions/formula/util.ts | 10 ++++- 5 files changed, 66 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 107ccdc9ac5ae8..99826c5f199a56 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -46,6 +46,7 @@ import { trackUiEvent } from '../../../../../lens_ui_telemetry'; import './formula.scss'; import { FormulaIndexPatternColumn } from '../formula'; import { regenerateLayerFromAst } from '../parse'; +import { filterByVisibleOperation } from '../util'; export const MemoizedFormulaEditor = React.memo(FormulaEditor); @@ -69,6 +70,8 @@ export function FormulaEditor({ const disposables = React.useRef([]); const editor1 = React.useRef(); + const visibleOperationsMap = filterByVisibleOperation(operationDefinitionMap); + // The Monaco editor needs to have the overflowDiv in the first render. Using an effect // requires a second render to work, so we are using an if statement to guarantee it happens // on first render @@ -134,16 +137,11 @@ export function FormulaEditor({ let errors: ErrorWrapper[] = []; - const { root, error } = tryToParse(text, operationDefinitionMap); + const { root, error } = tryToParse(text, visibleOperationsMap); if (error) { errors = [error]; } else if (root) { - const validationErrors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap - ); + const validationErrors = runASTValidation(root, layer, indexPattern, visibleOperationsMap); if (validationErrors.length) { errors = validationErrors; } @@ -201,7 +199,7 @@ export function FormulaEditor({ columnId, currentColumn, indexPattern, - operationDefinitionMap + visibleOperationsMap ); updateLayer(newLayer); @@ -209,13 +207,13 @@ export function FormulaEditor({ const markers: monaco.editor.IMarkerData[] = managedColumns .flatMap(([id, column]) => { if (locations[id]) { - const def = operationDefinitionMap[column.operationType]; + const def = visibleOperationsMap[column.operationType]; if (def.getErrorMessage) { const messages = def.getErrorMessage( newLayer, id, indexPattern, - operationDefinitionMap + visibleOperationsMap ); if (messages) { const startPosition = offsetToRowColumn(text, locations[id].min); @@ -304,7 +302,7 @@ export function FormulaEditor({ position: innerText.length - lengthAfterPosition, context, indexPattern, - operationDefinitionMap, + operationDefinitionMap: visibleOperationsMap, data, }); } @@ -314,18 +312,18 @@ export function FormulaEditor({ position: innerText.length - lengthAfterPosition, context, indexPattern, - operationDefinitionMap, + operationDefinitionMap: visibleOperationsMap, data, }); } return { suggestions: aSuggestions.list.map((s) => - getSuggestion(s, aSuggestions.type, wordRange, operationDefinitionMap) + getSuggestion(s, aSuggestions.type, wordRange, visibleOperationsMap) ), }; }, - [indexPattern, operationDefinitionMap, data] + [indexPattern, visibleOperationsMap, data] ); const provideSignatureHelp = useCallback( @@ -347,10 +345,10 @@ export function FormulaEditor({ return getSignatureHelp( model.getValue(), innerText.length - lengthAfterPosition, - operationDefinitionMap + visibleOperationsMap ); }, - [operationDefinitionMap] + [visibleOperationsMap] ); const provideHover = useCallback( @@ -371,10 +369,10 @@ export function FormulaEditor({ return getHover( model.getValue(), innerText.length - lengthAfterPosition, - operationDefinitionMap + visibleOperationsMap ); }, - [operationDefinitionMap] + [visibleOperationsMap] ); const onTypeHandler = useCallback( @@ -654,7 +652,7 @@ export function FormulaEditor({ @@ -701,7 +699,7 @@ export function FormulaEditor({
) : null} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index e4f522732b8c0d..3a1cfc717a66a9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -789,6 +789,34 @@ invalid: " } }); + it('returns an error if formula or math operations are used', () => { + const formulaFormulas = ['formula()', 'formula(bytes)', 'formula(formula())']; + + for (const formula of formulaFormulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operation formula not found']); + } + + const mathFormulas = ['math()', 'math(bytes)', 'math(math())']; + + for (const formula of mathFormulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(['Operation math not found']); + } + }); + it('returns an error if field operation in formula have the wrong first argument', () => { const formulas = [ 'average(7)', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index c83b19d865075d..6e9330532974fa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -13,6 +13,7 @@ import { runASTValidation, tryToParse } from './validation'; import { MemoizedFormulaEditor } from './editor'; import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; +import { filterByVisibleOperation } from './util'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', @@ -50,12 +51,13 @@ export const formulaOperation: OperationDefinition< if (!column.params.formula || !operationDefinitionMap) { return; } - const { root, error } = tryToParse(column.params.formula, operationDefinitionMap); + const visibleOperationsMap = filterByVisibleOperation(operationDefinitionMap); + const { root, error } = tryToParse(column.params.formula, visibleOperationsMap); if (error || !root) { return [error!.message]; } - const errors = runASTValidation(root, layer, indexPattern, operationDefinitionMap); + const errors = runASTValidation(root, layer, indexPattern, visibleOperationsMap); return errors.length ? errors.map(({ message }) => message) : undefined; }, getPossibleOperation() { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index bb74ecfb264f8b..517cf5f1bbf45c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -13,7 +13,12 @@ import { IndexPattern, IndexPatternLayer } from '../../../types'; import { mathOperation } from './math'; import { documentField } from '../../../document_field'; import { runASTValidation, shouldHaveFieldArgument, tryToParse } from './validation'; -import { findVariables, getOperationParams, groupArgsByType } from './util'; +import { + filterByVisibleOperation, + findVariables, + getOperationParams, + groupArgsByType, +} from './util'; import { FormulaIndexPatternColumn } from './formula'; import { getColumnOrder } from '../../layer_helpers'; @@ -169,7 +174,7 @@ export function regenerateLayerFromAst( layer, columnId, indexPattern, - operationDefinitionMap + filterByVisibleOperation(operationDefinitionMap) ); const columns = { ...layer.columns }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts index 179d10dfbab939..dd95ebdec5b8ab 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/util.ts @@ -13,7 +13,7 @@ import type { TinymathNamedArgument, TinymathVariable, } from 'packages/kbn-tinymath'; -import type { OperationDefinition, IndexPatternColumn } from '../index'; +import type { OperationDefinition, IndexPatternColumn, GenericOperationDefinition } from '../index'; import type { GroupedNodes } from './types'; export function groupArgsByType(args: TinymathAST[]) { @@ -464,3 +464,11 @@ export function findVariables(node: TinymathAST | string): TinymathVariable[] { } return node.args.flatMap(findVariables); } + +export function filterByVisibleOperation( + operationDefinitionMap: Record +) { + return Object.fromEntries( + Object.entries(operationDefinitionMap).filter(([, operation]) => !operation.hidden) + ); +} From 195a5a61b7701254468bb1f8665347f048dd94d5 Mon Sep 17 00:00:00 2001 From: dej611 Date: Thu, 27 May 2021 19:54:21 +0200 Subject: [PATCH 135/185] :bug: Fix empty string query edge case --- .../definitions/formula/formula.test.tsx | 21 +++++++++++++++++++ .../definitions/formula/validation.ts | 10 +++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 3a1cfc717a66a9..0da1f1949c5d39 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -927,8 +927,29 @@ invalid: " ).toEqual(undefined); }); + it('returns no error for a query edge case', () => { + const formulas = [ + `count(kql='')`, + `count(lucene='')`, + `moving_average(count(kql=''), window=7)`, + ]; + for (const formula of formulas) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(formula), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); + } + }); + it('returns an error for a query not wrapped in single quotes', () => { const formulas = [ + `count(kql="")`, + `count(kql='")`, + `count(kql="')`, `count(kql="category.keyword: *")`, `count(kql='category.keyword: *")`, `count(kql="category.keyword: *')`, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 555a4a261cfc8f..8286d4130bcc6a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -136,9 +136,15 @@ export const getRawQueryValidationError = (text: string, operations: Record { // check if the raw argument has the minimal requirements - const [, rawValue] = rawQuery.split('='); + const [, rawValue = ''] = rawQuery.split('='); + const cleanedRawValue = rawValue.trim(); // it must start with a single quote, and quotes must have a closure - if (rawValue.trim()[0] !== "'" || !/'\s*([^']+?)\s*'/.test(rawValue)) { + if ( + cleanedRawValue.length && + (cleanedRawValue[0] !== "'" || !/'\s*([^']+?)\s*'/.test(rawValue)) && + // there's a special case when it's valid as two single quote strings + cleanedRawValue !== "''" + ) { return i18n.translate('xpack.lens.indexPattern.formulaOperationQueryError', { defaultMessage: `Single quotes are required for {language}='' at {rawQuery}`, values: { language, rawQuery }, From c51e8e0352a98b6fff306d1c3ef002931aee80bf Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 28 May 2021 14:56:12 +0200 Subject: [PATCH 136/185] :bug: Enable more suggestions + extends validation --- .../definitions/formula/editor/math_completion.ts | 4 ++-- .../operations/definitions/formula/formula.test.tsx | 3 +++ .../operations/definitions/formula/validation.ts | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 0b70109f04138b..d3d2d8a0dbfef5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -312,9 +312,9 @@ export async function getNamedArgumentSuggestions({ } const before = ast.value.split(MARKER)[0]; - // TODO + const suggestions = await data.autocomplete.getQuerySuggestions({ - language: 'kuery', + language: ast.name === 'kql' ? 'kuery' : 'lucene', query: ast.value.split(MARKER)[0], selectionStart: before.length, selectionEnd: before.length, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 0da1f1949c5d39..b4ba3429a36102 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -932,6 +932,9 @@ invalid: " `count(kql='')`, `count(lucene='')`, `moving_average(count(kql=''), window=7)`, + `count(kql='bytes >= 4000')`, + `count(kql='bytes <= 4000')`, + `count(kql='bytes = 4000')`, ]; for (const formula of formulas) { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 8286d4130bcc6a..583c842c351ee2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -136,12 +136,14 @@ export const getRawQueryValidationError = (text: string, operations: Record { // check if the raw argument has the minimal requirements - const [, rawValue = ''] = rawQuery.split('='); - const cleanedRawValue = rawValue.trim(); + // use the rest operator here to handle cases where comparison operations are used in the query + const [, ...rawValue] = rawQuery.split('='); + const fullRawValue = (rawValue || ['']).join(''); + const cleanedRawValue = fullRawValue.trim(); // it must start with a single quote, and quotes must have a closure if ( cleanedRawValue.length && - (cleanedRawValue[0] !== "'" || !/'\s*([^']+?)\s*'/.test(rawValue)) && + (cleanedRawValue[0] !== "'" || !/'\s*([^']+?)\s*'/.test(fullRawValue)) && // there's a special case when it's valid as two single quote strings cleanedRawValue !== "''" ) { From bbf5aac4bd696c6ba13494065924a396e26cf3d1 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 28 May 2021 15:55:09 -0400 Subject: [PATCH 137/185] Fix tests that depended on setState being called without function --- .../dimension_panel/dimension_panel.test.tsx | 775 +++++++++--------- .../formula/editor/formula_editor.tsx | 2 +- .../definitions/formula/validation.ts | 6 +- 3 files changed, 402 insertions(+), 381 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index df5b8bfd53277d..6c3e636b7baa8a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -169,7 +169,9 @@ describe('IndexPatternDimensionEditorPanel', () => { setState = jest.fn().mockImplementation((newState) => { if (wrapper instanceof ReactWrapper) { - wrapper.setProps({ state: newState }); + wrapper.setProps({ + state: typeof newState === 'function' ? newState(wrapper.prop('state')) : newState, + }); } }); @@ -497,26 +499,27 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState).toHaveBeenCalledWith( - { - ...initialState, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - operationType: 'max', - sourceField: 'memory', - params: { format: { id: 'bytes' } }, - // Other parts of this don't matter for this test - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ + ...initialState, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + operationType: 'max', + sourceField: 'memory', + params: { format: { id: 'bytes' } }, + // Other parts of this don't matter for this test + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should switch operations when selecting a field that requires another operation', () => { @@ -531,25 +534,26 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - operationType: 'terms', - sourceField: 'source', - // Other parts of this don't matter for this test - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + operationType: 'terms', + sourceField: 'source', + // Other parts of this don't matter for this test + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should keep the field when switching to another operation compatible for this field', () => { @@ -564,26 +568,27 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - operationType: 'min', - sourceField: 'bytes', - params: { format: { id: 'bytes' } }, - // Other parts of this don't matter for this test - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + operationType: 'min', + sourceField: 'bytes', + params: { format: { id: 'bytes' } }, + // Other parts of this don't matter for this test + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should not set the state if selecting the currently active operation', () => { @@ -637,23 +642,24 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - label: 'Minimum of bytes', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + label: 'Minimum of bytes', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should keep the label on operation change if it is custom', () => { @@ -674,24 +680,25 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - label: 'Custom label', - customLabel: true, - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + label: 'Custom label', + customLabel: true, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should remove customLabel flag if label is set to default', () => { @@ -742,23 +749,24 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - }, - incompleteColumns: { - col1: { operationType: 'terms' }, - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + }, + incompleteColumns: { + col1: { operationType: 'terms' }, }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should show error message in invalid state', () => { @@ -867,20 +875,22 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper = mount(); wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - incompleteColumns: { - col2: { operationType: 'average' }, - }, + + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + incompleteColumns: { + col2: { operationType: 'average' }, }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: false } - ); + }); const comboBox = wrapper .find(EuiComboBox) @@ -892,26 +902,23 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([options![1].options![2]]); }); - expect(setState).toHaveBeenLastCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col2: expect.objectContaining({ - sourceField: 'source', - operationType: 'terms', - // Other parts of this don't matter for this test - }), - }, - columnOrder: ['col2', 'col1'], + expect(setState.mock.calls[1][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col2: expect.objectContaining({ + sourceField: 'source', + operationType: 'terms', + // Other parts of this don't matter for this test + }), }, + columnOrder: ['col2', 'col1'], }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should clean up when transitioning from incomplete reference-based operations to field operation', () => { @@ -938,20 +945,21 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); // Now check that the dimension gets cleaned up on state update - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - incompleteColumns: { - col2: { operationType: 'average' }, - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + incompleteColumns: { + col2: { operationType: 'average' }, }, }, }, - { shouldRemoveDimension: true, shouldReplaceDimension: false } - ); + }); }); it('should select the Records field when count is selected', () => { @@ -975,7 +983,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - const newColumnState = setState.mock.calls[0][0].layers.first.columns.col2; + const newColumnState = setState.mock.calls[0][0](state).layers.first.columns.col2; expect(newColumnState.operationType).toEqual('count'); expect(newColumnState.sourceField).toEqual('Records'); }); @@ -1032,24 +1040,26 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState).toHaveBeenLastCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col1: expect.objectContaining({ - sourceField: 'source', - operationType: 'terms', - }), - }, + expect(setState.mock.calls.length).toEqual(2); + expect(setState.mock.calls[1]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[1][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col1: expect.objectContaining({ + sourceField: 'source', + operationType: 'terms', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); }); @@ -1132,24 +1142,25 @@ describe('IndexPatternDimensionEditorPanel', () => { .find('[data-test-subj="indexPattern-time-scaling-enable"]') .hostNodes() .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: 's', - label: 'Count of records per second', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: 's', + label: 'Count of records per second', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should carry over time scaling to other operation if possible', () => { @@ -1163,24 +1174,25 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: 'h', - label: 'Count of records per hour', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: 'h', + label: 'Count of records per hour', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should not carry over time scaling if the other operation does not support it', () => { @@ -1192,24 +1204,25 @@ describe('IndexPatternDimensionEditorPanel', () => { }); wrapper = mount(); wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: undefined, - label: 'Average of bytes', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: undefined, + label: 'Average of bytes', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to change time scaling', () => { @@ -1225,24 +1238,25 @@ describe('IndexPatternDimensionEditorPanel', () => { .prop('onChange')!(({ target: { value: 'h' }, } as unknown) as ChangeEvent); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: 'h', - label: 'Count of records per hour', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: 'h', + label: 'Count of records per hour', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should not adjust label if it is custom', () => { @@ -1254,24 +1268,25 @@ describe('IndexPatternDimensionEditorPanel', () => { .prop('onChange')!(({ target: { value: 'h' }, } as unknown) as ChangeEvent); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: 'h', - label: 'My label', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: 'h', + label: 'My label', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to remove time scaling', () => { @@ -1284,24 +1299,25 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeScale: undefined, - label: 'Count of records', - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeScale: undefined, + label: 'Count of records', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); }); @@ -1386,26 +1402,27 @@ describe('IndexPatternDimensionEditorPanel', () => { .find('[data-test-subj="indexPattern-filter-by-enable"]') .hostNodes() .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - filter: { - language: 'kuery', - query: '', - }, - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + filter: { + language: 'kuery', + query: '', + }, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should carry over filter to other operation if possible', () => { @@ -1419,23 +1436,24 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - filter: { language: 'kuery', query: 'a: b' }, - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + filter: { language: 'kuery', query: 'a: b' }, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to change filter', () => { @@ -1447,23 +1465,24 @@ describe('IndexPatternDimensionEditorPanel', () => { language: 'kuery', query: 'c: d', }); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - filter: { language: 'kuery', query: 'c: d' }, - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + filter: { language: 'kuery', query: 'c: d' }, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to remove filter', () => { @@ -1478,23 +1497,24 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - filter: undefined, - }), - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + filter: undefined, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); }); @@ -1532,22 +1552,23 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - incompleteColumns: { - col2: { - operationType: 'average', - }, + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + incompleteColumns: { + col2: { + operationType: 'average', }, }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: false } - ); + }); const comboBox = wrapper .find(EuiComboBox) @@ -1558,26 +1579,23 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([options![1].options![0]]); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columnOrder: ['col1', 'col2'], - columns: { - ...state.layers.first.columns, - col2: expect.objectContaining({ - operationType: 'average', - sourceField: 'bytes', - }), - }, - incompleteColumns: {}, + expect(setState.mock.calls[1][0](defaultProps.state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columnOrder: ['col1', 'col2'], + columns: { + ...state.layers.first.columns, + col2: expect.objectContaining({ + operationType: 'average', + sourceField: 'bytes', + }), }, + incompleteColumns: {}, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should select operation directly if only one field is possible', () => { @@ -1601,26 +1619,27 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState).toHaveBeenCalledWith( - { - ...initialState, - layers: { - first: { - ...initialState.layers.first, - columns: { - ...initialState.layers.first.columns, - col2: expect.objectContaining({ - sourceField: 'bytes', - operationType: 'average', - // Other parts of this don't matter for this test - }), - }, - columnOrder: ['col1', 'col2'], + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](initialState)).toEqual({ + ...initialState, + layers: { + first: { + ...initialState.layers.first, + columns: { + ...initialState.layers.first.columns, + col2: expect.objectContaining({ + sourceField: 'bytes', + operationType: 'average', + // Other parts of this don't matter for this test + }), }, + columnOrder: ['col1', 'col2'], }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should select operation directly if only document is possible', () => { @@ -1628,25 +1647,26 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col2: expect.objectContaining({ - operationType: 'count', - // Other parts of this don't matter for this test - }), - }, - columnOrder: ['col1', 'col2'], + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col2: expect.objectContaining({ + operationType: 'count', + // Other parts of this don't matter for this test + }), }, + columnOrder: ['col1', 'col2'], }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should indicate compatible fields when selecting the operation first', () => { @@ -1764,26 +1784,27 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState).toHaveBeenCalledWith( - { - ...state, - layers: { - first: { - ...state.layers.first, - columns: { - ...state.layers.first.columns, - col2: expect.objectContaining({ - operationType: 'range', - sourceField: 'bytes', - // Other parts of this don't matter for this test - }), - }, - columnOrder: ['col1', 'col2'], + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + ]); + expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ + ...state, + layers: { + first: { + ...state.layers.first, + columns: { + ...state.layers.first.columns, + col2: expect.objectContaining({ + operationType: 'range', + sourceField: 'bytes', + // Other parts of this don't matter for this test + }), }, + columnOrder: ['col1', 'col2'], }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should use helper function when changing the function', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 107ccdc9ac5ae8..69b8c872a7b548 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -24,10 +24,10 @@ import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; +import { useDebounceWithOptions } from '../../../../../shared_components'; import { ParamEditorProps } from '../../index'; import { getManagedColumnsFrom } from '../../../layer_helpers'; import { ErrorWrapper, runASTValidation, tryToParse } from '../validation'; -import { useDebounceWithOptions } from '../../helpers'; import { LensMathSuggestion, SUGGESTION_TYPE, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 675d7b6458f35a..4ea4df066091c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -136,7 +136,7 @@ export const getRawQueryValidationError = (text: string, operations: Record { // check if the raw argument has the minimal requirements - const [_, rawValue] = rawQuery.split('='); + const [, rawValue] = rawQuery.split('='); // it must start with a single quote, and quotes must have a closure if (rawValue.trim()[0] !== "'" || !/'\s*([^']+?)\s*'/.test(rawValue)) { return i18n.translate('xpack.lens.indexPattern.formulaOperationQueryError', { @@ -147,11 +147,11 @@ const validateQueryQuotes = (rawQuery: string, language: 'kql' | 'lucene') => { }; export const getQueryValidationError = ( - { value: query, name: language, text }: { value: string; name: 'kql' | 'lucene'; text: string }, + { value: query, name: language, text }: TinymathNamedArgument, indexPattern: IndexPattern ): string | undefined => { // check if the raw argument has the minimal requirements - const result = validateQueryQuotes(text, language); + const result = validateQueryQuotes(text, language as 'kql' | 'lucene'); // forward the error here is ok? if (result) { return result; From 4ab340bc220dadd5205fc484e29f929e85060f52 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 28 May 2021 16:56:22 -0400 Subject: [PATCH 138/185] Error state and text wrapping updates --- .../dimension_panel/dimension_editor.tsx | 2 +- .../definitions/formula/editor/formula.scss | 6 + .../formula/editor/formula_editor.tsx | 122 ++++++++++++------ 3 files changed, 91 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 712ac618f2ee60..a8915ca3fc6cb3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -582,7 +582,7 @@ export function DimensionEditor(props: DimensionEditorProps) { {operationSupportMatrix.operationWithoutField.has('formula') ? ( { if (selectedColumn?.operationType !== 'formula') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index ee0ef129961281..235a6356cad316 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -67,6 +67,12 @@ min-height: 201px; } +.lnsFormula__warningText + .lnsFormula__warningText { + margin-top: $euiSizeS; + border-top: $euiBorderThin; + padding-top: $euiSizeS; +} + .lnsFormula__editorHelp--inline { align-items: center; display: flex; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 229407dd8e38e3..f37aa50862a4ef 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -63,8 +63,11 @@ export function FormulaEditor({ setIsCloseable, }: ParamEditorProps) { const [text, setText] = useState(currentColumn.params.formula); - const [warnings, setWarnings] = useState>([]); + const [warnings, setWarnings] = useState< + Array<{ severity: monaco.MarkerSeverity; message: string }> + >([]); const [isHelpOpen, setIsHelpOpen] = useState(isFullscreen); + const [isWarningOpen, setIsWarningOpen] = useState(false); const editorModel = React.useRef(); const overflowDiv1 = React.useRef(); const disposables = React.useRef([]); @@ -148,6 +151,20 @@ export function FormulaEditor({ } if (errors.length) { + if (currentColumn.params.isFormulaBroken) { + // If the formula is already broken, show the latest error message in the workspace + updateLayer( + regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + visibleOperationsMap + ).newLayer + ); + } + const markers = errors .flatMap((innerError) => { if (innerError.locations.length) { @@ -188,7 +205,7 @@ export function FormulaEditor({ .filter((marker) => marker); monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); - setWarnings(markers.map(({ severity }) => ({ severity }))); + setWarnings(markers.map(({ severity, message }) => ({ severity, message }))); } else { monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); @@ -234,7 +251,7 @@ export function FormulaEditor({ return []; }) .filter((marker) => marker); - setWarnings(markers.map(({ severity }) => ({ severity }))); + setWarnings(markers.map(({ severity, message }) => ({ severity, message }))); monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); } }, @@ -435,6 +452,8 @@ export function FormulaEditor({ [] ); + const isWordWrapped = editor1.current?.getOption(monaco.editor.EditorOption.wordWrap) !== 'off'; + const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -446,7 +465,7 @@ export function FormulaEditor({ lineNumbers: 'off', scrollBeyondLastLine: false, minimap: { enabled: false }, - wordWrap: 'on', + wordWrap: isWordWrapped ? 'on' : 'off', // Disable suggestions that appear when we don't provide a default suggestion wordBasedSuggestions: false, autoIndent: 'brackets', @@ -493,18 +512,31 @@ export function FormulaEditor({ {/* TODO: Replace `bolt` with `wordWrap` icon (after latest EUI is deployed) and hook up button to enable/disable word wrapping. */} { editor1.current?.updateOptions({ wordWrap: @@ -661,33 +693,47 @@ export function FormulaEditor({ {errorCount || warningCount ? ( - { - editor1.current?.trigger('LENS', 'editor.action.marker.next', {}); - }} + setIsWarningOpen(false)} + button={ + { + setIsWarningOpen(!isWarningOpen); + }} + > + {errorCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaErrorCount', { + defaultMessage: + '{count} {count, plural, one {error} other {errors}}', + values: { count: errorCount }, + })} + + ) : null} + {warningCount ? ( + + {' '} + {i18n.translate('xpack.lens.formulaWarningCount', { + defaultMessage: + '{count} {count, plural, one {warning} other {warnings}}', + values: { count: warningCount }, + })} + + ) : null} + + } > - {errorCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaErrorCount', { - defaultMessage: '{count} {count, plural, one {error} other {errors}}', - values: { count: errorCount }, - })} - - ) : null} - {warningCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaWarningCount', { - defaultMessage: - '{count} {count, plural, one {warning} other {warnings}}', - values: { count: warningCount }, - })} - - ) : null} - + {warnings.map(({ message }, index) => ( +
+ {message} +
+ ))} + ) : null} From 32713ddd4426a96afeb3225bdfbb6d5707c91453 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Fri, 28 May 2021 22:57:17 +0200 Subject: [PATCH 139/185] :sparkles: Add new module to CodeEditor for brackets matching (#25) --- packages/kbn-monaco/src/monaco_imports.ts | 1 + src/plugins/kibana_react/public/code_editor/code_editor.tsx | 1 + .../operations/definitions/formula/editor/formula_editor.tsx | 1 + 3 files changed, 3 insertions(+) diff --git a/packages/kbn-monaco/src/monaco_imports.ts b/packages/kbn-monaco/src/monaco_imports.ts index 872ac46352cf31..92ea23347c374c 100644 --- a/packages/kbn-monaco/src/monaco_imports.ts +++ b/packages/kbn-monaco/src/monaco_imports.ts @@ -21,5 +21,6 @@ import 'monaco-editor/esm/vs/editor/contrib/folding/folding.js'; // Needed for f import 'monaco-editor/esm/vs/editor/contrib/suggest/suggestController.js'; // Needed for suggestions import 'monaco-editor/esm/vs/editor/contrib/hover/hover.js'; // Needed for hover import 'monaco-editor/esm/vs/editor/contrib/parameterHints/parameterHints.js'; // Needed for signature +import 'monaco-editor/esm/vs/editor/contrib/bracketMatching/bracketMatching.js'; // Needed for brackets matching highlight export { monaco }; diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index af00980c85284e..251f05950da227 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -187,6 +187,7 @@ export class CodeEditor extends React.Component { wordBasedSuggestions: false, wordWrap: 'on', wrappingIndent: 'indent', + matchBrackets: 'never', ...options, }} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index f37aa50862a4ef..16b08f3e8d5cc2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -472,6 +472,7 @@ export function FormulaEditor({ wrappingIndent: 'none', dimension: { width: 320, height: 200 }, fixedOverflowWidgets: true, + matchBrackets: 'always', }, }; From fc2c3c9218b3c34da5e35b172862e7932d1c6912 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 28 May 2021 17:20:32 -0400 Subject: [PATCH 140/185] Fix type --- .../operations/definitions/formula/validation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 5f442e0cad02bc..7352f045a08f5b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -400,7 +400,7 @@ function getQueryValidationErrors( (namedArguments ?? []).forEach((arg) => { if (arg.name === 'kql' || arg.name === 'lucene') { const message = getQueryValidationError( - arg as { value: string; name: 'kql' | 'lucene'; text: string }, + arg as TinymathNamedArgument & { name: 'kql' | 'lucene' }, indexPattern ); if (message) { From 3637ee6f1ae4da567382d1e3b60520207cee2774 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 31 May 2021 11:14:41 +0200 Subject: [PATCH 141/185] show warning --- .../dimension_panel/dimension_editor.tsx | 34 +++++++++++++++++-- .../formula/editor/formula_editor.tsx | 2 +- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index a8915ca3fc6cb3..0540605eecf12b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -20,6 +20,7 @@ import { EuiText, EuiTabs, EuiTab, + EuiCallOut, } from '@elastic/eui'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; @@ -105,10 +106,22 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const { fieldByOperation, operationWithoutField } = operationSupportMatrix; + const selectedOperationDefinition = + selectedColumn && operationDefinitionMap[selectedColumn.operationType]; + + const [changedFormula, setChangedFormula] = useState( + Boolean(selectedOperationDefinition?.type === 'formula') + ); + const setStateWrapper = ( setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), shouldClose?: boolean ) => { + if (selectedOperationDefinition?.type === 'formula' && !temporaryQuickFunction) { + setChangedFormula(true); + } else { + setChangedFormula(false); + } const hypotheticalLayer = typeof setter === 'function' ? setter(state.layers[layerId]) : setter; const hasIncompleteColumns = Boolean(hypotheticalLayer.incompleteColumns?.[columnId]); const prevOperationType = @@ -133,9 +146,6 @@ export function DimensionEditor(props: DimensionEditorProps) { setState((prevState) => ({ ...prevState, isDimensionClosePrevented: !isCloseable })); }; - const selectedOperationDefinition = - selectedColumn && operationDefinitionMap[selectedColumn.operationType]; - const incompleteInfo = (state.layers[layerId].incompleteColumns ?? {})[columnId]; const incompleteOperation = incompleteInfo?.operationType; const incompleteField = incompleteInfo?.sourceField ?? null; @@ -339,6 +349,24 @@ export function DimensionEditor(props: DimensionEditorProps) { const quickFunctions = ( <>
+ {temporaryQuickFunction && changedFormula && ( + <> + +

+ {i18n.translate('xpack.lens.indexPattern.formulaWarningText', { + defaultMessage: 'Picking a quick function will erase your formula.', + })} +

+ + + + )} {i18n.translate('xpack.lens.indexPattern.functionsLabel', { defaultMessage: 'Select a function', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 16b08f3e8d5cc2..333046e38d5f20 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -257,7 +257,7 @@ export function FormulaEditor({ }, // Make it validate on flyout open in case of a broken formula left over // from a previous edit - { skipFirstRender: text == null }, + { skipFirstRender: true }, 256, [text] ); From ed477054c5560b50ede4245106954509617d65db Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 31 May 2021 11:42:53 +0200 Subject: [PATCH 142/185] keep current quick function --- .../dimension_panel/dimension_editor.tsx | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 0540605eecf12b..5be05338fdc2e2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -22,6 +22,7 @@ import { EuiTab, EuiCallOut, } from '@elastic/eui'; +import useUnmount from 'react-use/lib/useUnmount'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; import { IndexPatternColumn } from '../indexpattern'; @@ -40,7 +41,7 @@ import { mergeLayer } from '../state_helpers'; import { FieldSelect } from './field_select'; import { hasField, fieldIsInvalid } from '../utils'; import { BucketNestingEditor } from './bucket_nesting_editor'; -import { IndexPattern, IndexPatternLayer } from '../types'; +import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; import { trackUiEvent } from '../../lens_ui_telemetry'; import { FormatSelector } from './format_selector'; import { ReferenceEditor } from './reference_editor'; @@ -84,7 +85,7 @@ const LabelInput = ({ value, onChange }: { value: string; onChange: (value: stri export function DimensionEditor(props: DimensionEditorProps) { const { - selectedColumn, + selectedColumn: upstreamSelectedColumn, operationSupportMatrix, state, columnId, @@ -106,6 +107,17 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const { fieldByOperation, operationWithoutField } = operationSupportMatrix; + const [previousQuickFunctionState, setPreviousQuickFunctionState] = useState< + IndexPatternPrivateState | undefined + >(undefined); + + const [temporaryQuickFunction, setQuickFunction] = useState(false); + + const selectedColumn = + temporaryQuickFunction && previousQuickFunctionState + ? previousQuickFunctionState.layers[layerId].columns[columnId] + : upstreamSelectedColumn; + const selectedOperationDefinition = selectedColumn && operationDefinitionMap[selectedColumn.operationType]; @@ -113,12 +125,21 @@ export function DimensionEditor(props: DimensionEditorProps) { Boolean(selectedOperationDefinition?.type === 'formula') ); + useUnmount(() => { + if (temporaryQuickFunction && previousQuickFunctionState) { + setState(previousQuickFunctionState, { + shouldClose: true, + }); + } + }); + const setStateWrapper = ( setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), shouldClose?: boolean ) => { if (selectedOperationDefinition?.type === 'formula' && !temporaryQuickFunction) { setChangedFormula(true); + setPreviousQuickFunctionState(undefined); } else { setChangedFormula(false); } @@ -152,8 +173,6 @@ export function DimensionEditor(props: DimensionEditorProps) { const ParamEditor = selectedOperationDefinition?.paramEditor; - const [temporaryQuickFunction, setQuickFunction] = useState(false); - const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) @@ -615,6 +634,7 @@ export function DimensionEditor(props: DimensionEditorProps) { onClick={() => { if (selectedColumn?.operationType !== 'formula') { setQuickFunction(false); + setPreviousQuickFunctionState(state); const newLayer = insertOrReplaceColumn({ layer: props.state.layers[props.layerId], indexPattern: currentIndexPattern, From e56ef54dc6e6c5563d1201707bfda073582aa26e Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 31 May 2021 15:17:51 +0200 Subject: [PATCH 143/185] :sparkles: Improve suggestions within kql query --- .../formula/editor/formula_editor.tsx | 30 ++++++++-- .../formula/editor/math_completion.ts | 59 +++++++++++-------- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 16b08f3e8d5cc2..05cd1f4e077069 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiButtonIcon, @@ -22,6 +22,8 @@ import { import useUnmount from 'react-use/lib/useUnmount'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; +import { debounce } from 'lodash'; +import type { AutocompleteStart } from '../../../../../../../../../src/plugins/data/public'; import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; import { useDebounceWithOptions } from '../../../../../shared_components'; @@ -73,7 +75,9 @@ export function FormulaEditor({ const disposables = React.useRef([]); const editor1 = React.useRef(); - const visibleOperationsMap = filterByVisibleOperation(operationDefinitionMap); + const visibleOperationsMap = useMemo(() => filterByVisibleOperation(operationDefinitionMap), [ + operationDefinitionMap, + ]); // The Monaco editor needs to have the overflowDiv in the first render. Using an effect // requires a second render to work, so we are using an if statement to guarantee it happens @@ -86,6 +90,14 @@ export function FormulaEditor({ document.body.appendChild(node1); } + const autocompleteServiceDebounced: AutocompleteStart = useMemo( + () => ({ + ...data.autocomplete, + getQuerySuggestion: debounce((args) => data.autocomplete.getQuerySuggestions(args), 256), + }), + [data.autocomplete] + ); + // Clean up the monaco editor and DOM on unmount useEffect(() => { const model = editorModel; @@ -320,7 +332,7 @@ export function FormulaEditor({ context, indexPattern, operationDefinitionMap: visibleOperationsMap, - data, + autocomplete: autocompleteServiceDebounced, }); } } else { @@ -330,17 +342,23 @@ export function FormulaEditor({ context, indexPattern, operationDefinitionMap: visibleOperationsMap, - data, + autocomplete: autocompleteServiceDebounced, }); } return { suggestions: aSuggestions.list.map((s) => - getSuggestion(s, aSuggestions.type, wordRange, visibleOperationsMap) + getSuggestion( + s, + aSuggestions.type, + wordRange, + visibleOperationsMap, + context.triggerCharacter + ) ), }; }, - [indexPattern, visibleOperationsMap, data] + [indexPattern, visibleOperationsMap, autocompleteServiceDebounced] ); const provideSignatureHelp = useCallback( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index d3d2d8a0dbfef5..be61633ecfa414 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -15,7 +15,10 @@ import { TinymathFunction, TinymathNamedArgument, } from '@kbn/tinymath'; -import { DataPublicPluginStart, QuerySuggestion } from 'src/plugins/data/public'; +import type { + AutocompleteStart, + QuerySuggestion, +} from '../../../../../../../../../src/plugins/data/public'; import { IndexPattern } from '../../../../types'; import { memoizedGetAvailableOperationsByMetadata } from '../../../operations'; import { tinymathFunctions, groupArgsByType } from '../util'; @@ -104,14 +107,14 @@ export async function suggest({ context, indexPattern, operationDefinitionMap, - data, + autocomplete, }: { expression: string; position: number; context: monaco.languages.CompletionContext; indexPattern: IndexPattern; operationDefinitionMap: Record; - data: DataPublicPluginStart; + autocomplete: AutocompleteStart; }): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { @@ -129,7 +132,7 @@ export async function suggest({ if (tokenInfo?.parent && (context.triggerCharacter === '=' || isNamedArgument)) { return await getNamedArgumentSuggestions({ ast: tokenAst as TinymathNamedArgument, - data, + autocomplete, indexPattern, }); } else if (tokenInfo?.parent) { @@ -297,38 +300,48 @@ function getArgumentSuggestions( export async function getNamedArgumentSuggestions({ ast, - data, + autocomplete, indexPattern, }: { ast: TinymathNamedArgument; indexPattern: IndexPattern; - data: DataPublicPluginStart; + autocomplete: AutocompleteStart; }) { if (ast.name !== 'kql' && ast.name !== 'lucene') { return { list: [], type: SUGGESTION_TYPE.KQL }; } - if (!data.autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { + if (!autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { return { list: [], type: SUGGESTION_TYPE.KQL }; } - const before = ast.value.split(MARKER)[0]; + const query = ast.value.split(MARKER)[0]; + const position = ast.value.indexOf(MARKER) + 1; - const suggestions = await data.autocomplete.getQuerySuggestions({ + const suggestions = await autocomplete.getQuerySuggestions({ language: ast.name === 'kql' ? 'kuery' : 'lucene', - query: ast.value.split(MARKER)[0], - selectionStart: before.length, - selectionEnd: before.length, + query, + selectionStart: position, + selectionEnd: position, indexPatterns: [indexPattern], boolFilter: [], }); - return { list: suggestions ?? [], type: SUGGESTION_TYPE.KQL }; + return { + list: suggestions ?? [], + type: SUGGESTION_TYPE.KQL, + }; } +const TRIGGER_SUGGESTION_COMMAND = { + title: 'Trigger Suggestion Dialog', + id: 'editor.action.triggerSuggest', +}; + export function getSuggestion( suggestion: LensMathSuggestion, type: SUGGESTION_TYPE, range: monaco.Range, - operationDefinitionMap: Record + operationDefinitionMap: Record, + triggerChar: string | undefined ): monaco.languages.CompletionItem { let kind: monaco.languages.CompletionItemKind = monaco.languages.CompletionItemKind.Method; let label: string = @@ -363,20 +376,14 @@ export function getSuggestion( detail = 'Elasticsearch'; // Always put ES functions first sortText = `0${label}`; - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; + command = TRIGGER_SUGGESTION_COMMAND; } } break; case SUGGESTION_TYPE.NAMED_ARGUMENT: kind = monaco.languages.CompletionItemKind.Keyword; if (label === 'kql' || label === 'lucene') { - command = { - title: 'Trigger Suggestion Dialog', - id: 'editor.action.triggerSuggest', - }; + command = TRIGGER_SUGGESTION_COMMAND; insertText = `${label}='$0'`; insertTextRules = monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet; sortText = `zzz${label}`; @@ -385,8 +392,14 @@ export function getSuggestion( detail = ''; break; case SUGGESTION_TYPE.KQL: + if (triggerChar === ':') { + insertText = `${triggerChar} ${label}`; + } else { + // concatenate KQL suggestion for faster query composition + command = TRIGGER_SUGGESTION_COMMAND; + } if (label.includes(`'`)) { - insertText = label.replaceAll(`'`, "\\'"); + insertText = (insertText || label).replaceAll(`'`, "\\'"); } break; } From fcc775fe5307bd0988914f1f0c07ebd5f468d3de Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 31 May 2021 15:18:29 +0200 Subject: [PATCH 144/185] :camera: Fix snapshot editor test --- .../public/code_editor/__snapshots__/code_editor.test.tsx.snap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/kibana_react/public/code_editor/__snapshots__/code_editor.test.tsx.snap b/src/plugins/kibana_react/public/code_editor/__snapshots__/code_editor.test.tsx.snap index a779ef540d72ec..d4fb5a708e4405 100644 --- a/src/plugins/kibana_react/public/code_editor/__snapshots__/code_editor.test.tsx.snap +++ b/src/plugins/kibana_react/public/code_editor/__snapshots__/code_editor.test.tsx.snap @@ -11,6 +11,7 @@ exports[`is rendered 1`] = ` onChange={[Function]} options={ Object { + "matchBrackets": "never", "minimap": Object { "enabled": false, }, @@ -39,6 +40,7 @@ exports[`is rendered 1`] = ` nodeType="div" onResize={[Function]} querySelector={null} + refreshMode="debounce" refreshRate={1000} skipOnMount={false} targetDomEl={null} From ad50a7e7bcfa3c383f146e01899232095668ad00 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 31 May 2021 18:12:58 +0200 Subject: [PATCH 145/185] :bug: Improved suggestion for single quote and refactored debounce --- .../formula/editor/formula_editor.tsx | 96 ++++++++++++------- .../formula/editor/math_completion.ts | 16 ++-- 2 files changed, 72 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 05cd1f4e077069..4bbe501e9e91e7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -22,8 +22,6 @@ import { import useUnmount from 'react-use/lib/useUnmount'; import { monaco } from '@kbn/monaco'; import classNames from 'classnames'; -import { debounce } from 'lodash'; -import type { AutocompleteStart } from '../../../../../../../../../src/plugins/data/public'; import { CodeEditor } from '../../../../../../../../../src/plugins/kibana_react/public'; import type { CodeEditorProps } from '../../../../../../../../../src/plugins/kibana_react/public'; import { useDebounceWithOptions } from '../../../../../shared_components'; @@ -90,14 +88,6 @@ export function FormulaEditor({ document.body.appendChild(node1); } - const autocompleteServiceDebounced: AutocompleteStart = useMemo( - () => ({ - ...data.autocomplete, - getQuerySuggestion: debounce((args) => data.autocomplete.getQuerySuggestions(args), 256), - }), - [data.autocomplete] - ); - // Clean up the monaco editor and DOM on unmount useEffect(() => { const model = editorModel; @@ -332,7 +322,7 @@ export function FormulaEditor({ context, indexPattern, operationDefinitionMap: visibleOperationsMap, - autocomplete: autocompleteServiceDebounced, + data, }); } } else { @@ -342,7 +332,7 @@ export function FormulaEditor({ context, indexPattern, operationDefinitionMap: visibleOperationsMap, - autocomplete: autocompleteServiceDebounced, + data, }); } @@ -358,7 +348,7 @@ export function FormulaEditor({ ), }; }, - [indexPattern, visibleOperationsMap, autocompleteServiceDebounced] + [indexPattern, visibleOperationsMap, data] ); const provideSignatureHelp = useCallback( @@ -415,16 +405,34 @@ export function FormulaEditor({ if (e.isFlush || e.isRedoing || e.isUndoing) { return; } - if (e.changes.length === 1 && e.changes[0].text === '=') { + if (e.changes.length === 1 && (e.changes[0].text === '=' || e.changes[0].text === "'")) { const currentPosition = e.changes[0].range; if (currentPosition) { - const tokenInfo = getTokenInfo( + let tokenInfo = getTokenInfo( editor.getValue(), monacoPositionToOffset( editor.getValue(), new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) ) ); + + if (!tokenInfo && e.changes[0].text === "'") { + // try again this time replacing the current quote with an escaped quote + const line = editor.getValue(); + const lineEscaped = + line.substring(0, currentPosition.startColumn - 1) + + "\\'" + + line.substring(currentPosition.endColumn); + tokenInfo = getTokenInfo( + lineEscaped, + monacoPositionToOffset( + line, + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) + ) + 1 + ); + } + + const isSingleQuoteCase = /'LENS_MATH_MARKER/; // Make sure that we are only adding kql='' or lucene='', and also // check that the = sign isn't inside the KQL expression like kql='=' if ( @@ -432,28 +440,44 @@ export function FormulaEditor({ typeof tokenInfo.ast === 'number' || tokenInfo.ast.type !== 'namedArgument' || (tokenInfo.ast.name !== 'kql' && tokenInfo.ast.name !== 'lucene') || - tokenInfo.ast.value !== 'LENS_MATH_MARKER' + (tokenInfo.ast.value !== 'LENS_MATH_MARKER' && + !isSingleQuoteCase.test(tokenInfo.ast.value)) ) { return; } - // Timeout is required because otherwise the cursor position is not updated. - setTimeout(() => { + let editOperation = null; + if (e.changes[0].text === '=') { + editOperation = { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn + 1, + endColumn: currentPosition.startColumn + 1, + }, + text: `''`, + }; + } + if (e.changes[0].text === "'") { + editOperation = { + range: { + ...currentPosition, + // Insert after the current char + startColumn: currentPosition.startColumn, + endColumn: currentPosition.startColumn + 1, + }, + text: `\\'`, + }; + } + + // Need to move these sync to prevent race conditions between a fast user typing a single quote + // after an = char + if (editOperation) { editor.executeEdits( 'LENS', + [editOperation], [ - { - range: { - ...currentPosition, - // Insert after the current char - startColumn: currentPosition.startColumn + 1, - endColumn: currentPosition.startColumn + 1, - }, - text: `''`, - }, - ], - [ - // After inserting, move the cursor in between the single quotes + // After inserting, move the cursor in between the single quotes or after the escaped quote new monaco.Selection( currentPosition.startLineNumber, currentPosition.startColumn + 2, @@ -462,8 +486,16 @@ export function FormulaEditor({ ), ] ); - editor.trigger('lens', 'editor.action.triggerSuggest', {}); - }, 0); + + // Timeout is required because otherwise the cursor position is not updated. + setTimeout(() => { + editor.setPosition({ + column: currentPosition.startColumn + 2, + lineNumber: currentPosition.startLineNumber, + }); + editor.trigger('lens', 'editor.action.triggerSuggest', {}); + }, 0); + } } } }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index be61633ecfa414..36387e26def5c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -16,7 +16,7 @@ import { TinymathNamedArgument, } from '@kbn/tinymath'; import type { - AutocompleteStart, + DataPublicPluginStart, QuerySuggestion, } from '../../../../../../../../../src/plugins/data/public'; import { IndexPattern } from '../../../../types'; @@ -107,14 +107,14 @@ export async function suggest({ context, indexPattern, operationDefinitionMap, - autocomplete, + data, }: { expression: string; position: number; context: monaco.languages.CompletionContext; indexPattern: IndexPattern; operationDefinitionMap: Record; - autocomplete: AutocompleteStart; + data: DataPublicPluginStart; }): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { const text = expression.substr(0, position) + MARKER + expression.substr(position); try { @@ -132,7 +132,7 @@ export async function suggest({ if (tokenInfo?.parent && (context.triggerCharacter === '=' || isNamedArgument)) { return await getNamedArgumentSuggestions({ ast: tokenAst as TinymathNamedArgument, - autocomplete, + data, indexPattern, }); } else if (tokenInfo?.parent) { @@ -300,24 +300,24 @@ function getArgumentSuggestions( export async function getNamedArgumentSuggestions({ ast, - autocomplete, + data, indexPattern, }: { ast: TinymathNamedArgument; indexPattern: IndexPattern; - autocomplete: AutocompleteStart; + data: DataPublicPluginStart; }) { if (ast.name !== 'kql' && ast.name !== 'lucene') { return { list: [], type: SUGGESTION_TYPE.KQL }; } - if (!autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { + if (!data.autocomplete.hasQuerySuggestions(ast.name === 'kql' ? 'kuery' : 'lucene')) { return { list: [], type: SUGGESTION_TYPE.KQL }; } const query = ast.value.split(MARKER)[0]; const position = ast.value.indexOf(MARKER) + 1; - const suggestions = await autocomplete.getQuerySuggestions({ + const suggestions = await data.autocomplete.getQuerySuggestions({ language: ast.name === 'kql' ? 'kuery' : 'lucene', query, selectionStart: position, From 8e1934d4d39d97152804a82a0c7c04b615f025d5 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 1 Jun 2021 13:11:01 -0400 Subject: [PATCH 146/185] Fix lodash usage --- .../indexpattern_datasource/indexpattern_suggestions.ts | 4 ++-- .../indexpattern_datasource/operations/layer_helpers.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts index 20732a1a96555d..bb5cf32d7f26c3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { flatten, minBy, pick, mapValues } from 'lodash'; +import { flatten, minBy, pick, mapValues, partition } from 'lodash'; import { i18n } from '@kbn/i18n'; import { generateId } from '../id_generator'; import { DatasourceSuggestion, TableChangeType } from '../types'; @@ -583,7 +583,7 @@ function createSuggestionWithDefaultDateHistogram( function createSimplifiedTableSuggestions(state: IndexPatternPrivateState, layerId: string) { const layer = state.layers[layerId]; - const [availableBucketedColumns, availableMetricColumns] = _.partition( + const [availableBucketedColumns, availableMetricColumns] = partition( layer.columnOrder, (colId) => layer.columns[colId].isBucketed ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index a60dfcc5f34722..3919e908805552 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -850,7 +850,7 @@ function addBucket( visualizationGroups: VisualizationDimensionGroupConfig[], targetGroup?: string ): IndexPatternLayer { - const [buckets, metrics] = _.partition( + const [buckets, metrics] = partition( layer.columnOrder, (colId) => layer.columns[colId].isBucketed ); From dfb8cb507201d960ef14b6268e25dd7f6d0bb8d8 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 1 Jun 2021 15:36:34 -0400 Subject: [PATCH 147/185] Fix tests --- .../components/columns.tsx | 2 +- .../formula/editor/formula_editor.tsx | 36 +++++++++---------- .../definitions/formula/formula.tsx | 34 +++++++++--------- x-pack/test/functional/apps/lens/formula.ts | 25 +++++++++++++ .../test/functional/page_objects/lens_page.ts | 3 +- 5 files changed, 62 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/lens/public/datatable_visualization/components/columns.tsx b/x-pack/plugins/lens/public/datatable_visualization/components/columns.tsx index ba24da8309ed7c..5c53d40f999b77 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/components/columns.tsx +++ b/x-pack/plugins/lens/public/datatable_visualization/components/columns.tsx @@ -51,7 +51,7 @@ export const createGridColumns = ( columnId, }: Pick) => { const rowValue = table.rows[rowIndex][columnId]; - const column = columnsReverseLookup[columnId]; + const column = columnsReverseLookup?.[columnId]; const contentsIsDefined = rowValue != null; const cellContent = formatFactory(column?.meta?.params).convert(rowValue); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 4bbe501e9e91e7..572bb019dfb512 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -446,7 +446,7 @@ export function FormulaEditor({ return; } - let editOperation = null; + let editOperation: monaco.editor.IIdentifiedSingleEditOperation | null = null; if (e.changes[0].text === '=') { editOperation = { range: { @@ -470,25 +470,25 @@ export function FormulaEditor({ }; } - // Need to move these sync to prevent race conditions between a fast user typing a single quote - // after an = char if (editOperation) { - editor.executeEdits( - 'LENS', - [editOperation], - [ - // After inserting, move the cursor in between the single quotes or after the escaped quote - new monaco.Selection( - currentPosition.startLineNumber, - currentPosition.startColumn + 2, - currentPosition.startLineNumber, - currentPosition.startColumn + 2 - ), - ] - ); - - // Timeout is required because otherwise the cursor position is not updated. setTimeout(() => { + editor.executeEdits( + 'LENS', + [editOperation!], + [ + // After inserting, move the cursor in between the single quotes or after the escaped quote + new monaco.Selection( + currentPosition.startLineNumber, + currentPosition.startColumn + 2, + currentPosition.startLineNumber, + currentPosition.startColumn + 2 + ), + ] + ); + + // Need to move these sync to prevent race conditions between a fast user typing a single quote + // after an = char + // Timeout is required because otherwise the cursor position is not updated. editor.setPosition({ column: currentPosition.startColumn + 2, lineNumber: currentPosition.startLineNumber, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 6e9330532974fa..bbf495281004f7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -75,12 +75,8 @@ export const formulaOperation: OperationDefinition< const label = !params?.isFormulaBroken ? useDisplayLabel ? currentColumn.label - : params?.formula - : ''; - - if (!params.formula || params.isFormulaBroken) { - return []; - } + : params?.formula ?? defaultLabel + : defaultLabel; return [ { @@ -88,21 +84,23 @@ export const formulaOperation: OperationDefinition< function: 'mapColumn', arguments: { id: [columnId], - name: [label || ''], + name: [label || defaultLabel], exp: [ { type: 'expression', - chain: [ - { - type: 'function', - function: 'math', - arguments: { - expression: [ - currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``, - ], - }, - }, - ], + chain: currentColumn.references.length + ? [ + { + type: 'function', + function: 'math', + arguments: { + expression: [ + currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``, + ], + }, + }, + ] + : [], }, ], }, diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index f8097f243cf17d..4adf3a6c0a2829 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -12,6 +12,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); const find = getService('find'); const listingTable = getService('listingTable'); + const browser = getService('browser'); describe('lens formula', () => { it('should transition from count to formula', async () => { @@ -55,6 +56,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14,005'); }); + it('should insert single quotes when needed to create valid KQL', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count(kql=`, + keepOpen: true, + }); + + const input = await find.activeElement(); + await input.type(' '); + await input.pressKeys(browser.keys.ARROW_LEFT); + await input.type(`Men's`); + + await PageObjects.common.sleep(100); + + const element = await find.byCssSelector('.monaco-editor'); + expect(await element.getVisibleText()).to.equal(`count(kql=\'Men\\'s \')`); + }); + it('should persist a broken formula on close', async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickVisType('lens'); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index db8752512f1f57..1824da031f0346 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -523,7 +523,8 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); } const errors = await testSubjects.findAll('configuration-failure-error'); - return errors?.length ?? 0; + const expressionErrors = await testSubjects.findAll('expression-failure'); + return (errors?.length ?? 0) + (expressionErrors?.length ?? 0); }, async searchOnChartSwitch(subVisualizationId: string, searchTerm?: string) { From a610e37228494acbcdca83eb529b9f320d10fde2 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 1 Jun 2021 15:49:44 -0400 Subject: [PATCH 148/185] Revert "keep current quick function" This reverts commit ed477054c5560b50ede4245106954509617d65db. --- .../dimension_panel/dimension_editor.tsx | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 919c5f355511a1..82dc32c8cc7ea7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -21,7 +21,6 @@ import { EuiTab, EuiCallOut, } from '@elastic/eui'; -import useUnmount from 'react-use/lib/useUnmount'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { OperationSupportMatrix } from './operation_support'; import { IndexPatternColumn } from '../indexpattern'; @@ -40,7 +39,7 @@ import { mergeLayer } from '../state_helpers'; import { FieldSelect } from './field_select'; import { hasField, fieldIsInvalid } from '../utils'; import { BucketNestingEditor } from './bucket_nesting_editor'; -import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; +import { IndexPattern, IndexPatternLayer } from '../types'; import { trackUiEvent } from '../../lens_ui_telemetry'; import { FormatSelector } from './format_selector'; import { ReferenceEditor } from './reference_editor'; @@ -84,7 +83,7 @@ const LabelInput = ({ value, onChange }: { value: string; onChange: (value: stri export function DimensionEditor(props: DimensionEditorProps) { const { - selectedColumn: upstreamSelectedColumn, + selectedColumn, operationSupportMatrix, state, columnId, @@ -106,17 +105,6 @@ export function DimensionEditor(props: DimensionEditorProps) { }; const { fieldByOperation, operationWithoutField } = operationSupportMatrix; - const [previousQuickFunctionState, setPreviousQuickFunctionState] = useState< - IndexPatternPrivateState | undefined - >(undefined); - - const [temporaryQuickFunction, setQuickFunction] = useState(false); - - const selectedColumn = - temporaryQuickFunction && previousQuickFunctionState - ? previousQuickFunctionState.layers[layerId].columns[columnId] - : upstreamSelectedColumn; - const selectedOperationDefinition = selectedColumn && operationDefinitionMap[selectedColumn.operationType]; @@ -124,21 +112,12 @@ export function DimensionEditor(props: DimensionEditorProps) { Boolean(selectedOperationDefinition?.type === 'formula') ); - useUnmount(() => { - if (temporaryQuickFunction && previousQuickFunctionState) { - setState(previousQuickFunctionState, { - shouldClose: true, - }); - } - }); - const setStateWrapper = ( setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), shouldClose?: boolean ) => { if (selectedOperationDefinition?.type === 'formula' && !temporaryQuickFunction) { setChangedFormula(true); - setPreviousQuickFunctionState(undefined); } else { setChangedFormula(false); } @@ -172,6 +151,8 @@ export function DimensionEditor(props: DimensionEditorProps) { const ParamEditor = selectedOperationDefinition?.paramEditor; + const [temporaryQuickFunction, setQuickFunction] = useState(false); + const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) @@ -633,7 +614,6 @@ export function DimensionEditor(props: DimensionEditorProps) { onClick={() => { if (selectedColumn?.operationType !== 'formula') { setQuickFunction(false); - setPreviousQuickFunctionState(state); const newLayer = insertOrReplaceColumn({ layer: props.state.layers[props.layerId], indexPattern: currentIndexPattern, From 507a421ff4e1833bbe1a6448bfc7db380865c027 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 2 Jun 2021 15:28:49 -0400 Subject: [PATCH 149/185] Improve performance of dispatch by using timeout --- .../config_panel/config_panel.tsx | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 61a19215a2c83d..4105dd2cb7c84d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -61,43 +61,51 @@ export function LayerPanels( ); const updateDatasource = useMemo( () => (datasourceId: string, newState: unknown) => { - dispatch({ - type: 'UPDATE_DATASOURCE_STATE', - updater: (prevState: unknown) => - typeof newState === 'function' ? newState(prevState) : newState, - datasourceId, - clearStagedPreview: false, - }); + // React will synchronously update if this is triggered from a third party component, + // which we don't want. The timeout lets user interaction have priority, then React updates. + setTimeout(() => { + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, + datasourceId, + clearStagedPreview: false, + }); + }, 0); }, [dispatch] ); const updateAll = useMemo( () => (datasourceId: string, newDatasourceState: unknown, newVisualizationState: unknown) => { - dispatch({ - type: 'UPDATE_STATE', - subType: 'UPDATE_ALL_STATES', - updater: (prevState) => { - const updatedDatasourceState = - typeof newDatasourceState === 'function' - ? newDatasourceState(prevState.datasourceStates[datasourceId].state) - : newDatasourceState; - return { - ...prevState, - datasourceStates: { - ...prevState.datasourceStates, - [datasourceId]: { - state: updatedDatasourceState, - isLoading: false, + // React will synchronously update if this is triggered from a third party component, + // which we don't want. The timeout lets user interaction have priority, then React updates. + setTimeout(() => { + dispatch({ + type: 'UPDATE_STATE', + subType: 'UPDATE_ALL_STATES', + updater: (prevState) => { + const updatedDatasourceState = + typeof newDatasourceState === 'function' + ? newDatasourceState(prevState.datasourceStates[datasourceId].state) + : newDatasourceState; + return { + ...prevState, + datasourceStates: { + ...prevState.datasourceStates, + [datasourceId]: { + state: updatedDatasourceState, + isLoading: false, + }, }, - }, - visualization: { - ...prevState.visualization, - state: newVisualizationState, - }, - stagedPreview: undefined, - }; - }, - }); + visualization: { + ...prevState.visualization, + state: newVisualizationState, + }, + stagedPreview: undefined, + }; + }, + }); + }, 0); }, [dispatch] ); From b92c8e614c963b91aa53479febc31c2d9ee75a09 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 2 Jun 2021 15:46:26 -0400 Subject: [PATCH 150/185] Improve memoization of datapanel --- .../editor_frame/editor_frame.tsx | 57 ++++++++----------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 471957b69fa7fc..161b0125a172a3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect, useReducer, useState, useCallback } from 'react'; +import React, { useEffect, useReducer, useState, useCallback, useRef } from 'react'; import { CoreStart } from 'kibana/public'; import { isEqual } from 'lodash'; import { PaletteRegistry } from 'src/plugins/charts/public'; @@ -30,6 +30,7 @@ import { applyVisualizeFieldSuggestions, getTopSuggestionForField, switchToSuggestion, + Suggestion, } from './suggestion_helpers'; import { trackUiEvent } from '../../lens_ui_telemetry'; import { @@ -327,45 +328,37 @@ export function EditorFrame(props: EditorFrameProps) { ] ); - const getSuggestionForField = React.useCallback( - (field: DragDropIdentifier) => { - const { activeDatasourceId, datasourceStates } = state; - const activeVisualizationId = state.visualization.activeId; - const visualizationState = state.visualization.state; - const { visualizationMap, datasourceMap } = props; + // Using a ref to prevent rerenders in the child components while keeping the latest state + const getSuggestionForField = useRef<(field: DragDropIdentifier) => Suggestion | undefined>(); + getSuggestionForField.current = (field: DragDropIdentifier) => { + const { activeDatasourceId, datasourceStates } = state; + const activeVisualizationId = state.visualization.activeId; + const visualizationState = state.visualization.state; + const { visualizationMap, datasourceMap } = props; - if (!field || !activeDatasourceId) { - return; - } + if (!field || !activeDatasourceId) { + return; + } - return getTopSuggestionForField( - datasourceLayers, - activeVisualizationId, - visualizationMap, - visualizationState, - datasourceMap[activeDatasourceId], - datasourceStates, - field - ); - }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [ - state.visualization.state, - props.datasourceMap, - props.visualizationMap, - state.activeDatasourceId, - state.datasourceStates, - ] - ); + return getTopSuggestionForField( + datasourceLayers, + activeVisualizationId, + visualizationMap, + visualizationState, + datasourceMap[activeDatasourceId], + datasourceStates, + field + ); + }; const hasSuggestionForField = useCallback( - (field: DragDropIdentifier) => getSuggestionForField(field) !== undefined, + (field: DragDropIdentifier) => getSuggestionForField.current!(field) !== undefined, [getSuggestionForField] ); const dropOntoWorkspace = useCallback( (field) => { - const suggestion = getSuggestionForField(field); + const suggestion = getSuggestionForField.current!(field); if (suggestion) { trackUiEvent('drop_onto_workspace'); switchToSuggestion(dispatch, suggestion, 'SWITCH_VISUALIZATION'); @@ -436,7 +429,7 @@ export function EditorFrame(props: EditorFrameProps) { core={props.core} plugins={props.plugins} visualizeTriggerFieldContext={visualizeTriggerFieldContext} - getSuggestionForField={getSuggestionForField} + getSuggestionForField={getSuggestionForField.current} /> ) } From 48e22babf0e5d191fb47d9e0287045cd338c69a6 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 2 Jun 2021 18:54:08 -0400 Subject: [PATCH 151/185] Fix escape characters --- .../formula/editor/formula_editor.tsx | 46 +++++++++---------- .../formula/editor/math_completion.test.ts | 12 ++++- .../formula/editor/math_completion.ts | 2 +- x-pack/test/functional/apps/lens/formula.ts | 15 ++++-- .../test/functional/page_objects/lens_page.ts | 1 + 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index e4c5adad171c8b..524c8c7b71d253 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -405,31 +405,25 @@ export function FormulaEditor({ if (e.isFlush || e.isRedoing || e.isUndoing) { return; } - if (e.changes.length === 1 && (e.changes[0].text === '=' || e.changes[0].text === "'")) { + if (e.changes.length === 1) { + const char = e.changes[0].text; + if (char !== '=' && char !== "'") { + return; + } const currentPosition = e.changes[0].range; if (currentPosition) { - let tokenInfo = getTokenInfo( - editor.getValue(), - monacoPositionToOffset( - editor.getValue(), - new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) - ) + const currentText = editor.getValue(); + const offset = monacoPositionToOffset( + currentText, + new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) ); + let tokenInfo = getTokenInfo(currentText, offset + 1); - if (!tokenInfo && e.changes[0].text === "'") { + if (!tokenInfo && char === "'") { // try again this time replacing the current quote with an escaped quote - const line = editor.getValue(); - const lineEscaped = - line.substring(0, currentPosition.startColumn - 1) + - "\\'" + - line.substring(currentPosition.endColumn); - tokenInfo = getTokenInfo( - lineEscaped, - monacoPositionToOffset( - line, - new monaco.Position(currentPosition.startLineNumber, currentPosition.startColumn) - ) + 1 - ); + const line = currentText; + const lineEscaped = line.substring(0, offset) + "\\'" + line.substring(offset + 1); + tokenInfo = getTokenInfo(lineEscaped, offset + 2); } const isSingleQuoteCase = /'LENS_MATH_MARKER/; @@ -447,7 +441,8 @@ export function FormulaEditor({ } let editOperation: monaco.editor.IIdentifiedSingleEditOperation | null = null; - if (e.changes[0].text === '=') { + let cursorOffset = 2; + if (char === '=') { editOperation = { range: { ...currentPosition, @@ -458,7 +453,7 @@ export function FormulaEditor({ text: `''`, }; } - if (e.changes[0].text === "'") { + if (char === "'") { editOperation = { range: { ...currentPosition, @@ -468,6 +463,7 @@ export function FormulaEditor({ }, text: `\\'`, }; + cursorOffset = 3; } if (editOperation) { @@ -479,9 +475,9 @@ export function FormulaEditor({ // After inserting, move the cursor in between the single quotes or after the escaped quote new monaco.Selection( currentPosition.startLineNumber, - currentPosition.startColumn + 2, + currentPosition.startColumn + cursorOffset, currentPosition.startLineNumber, - currentPosition.startColumn + 2 + currentPosition.startColumn + cursorOffset ), ] ); @@ -490,7 +486,7 @@ export function FormulaEditor({ // after an = char // Timeout is required because otherwise the cursor position is not updated. editor.setPosition({ - column: currentPosition.startColumn + 2, + column: currentPosition.startColumn + cursorOffset, lineNumber: currentPosition.startLineNumber, }); editor.trigger('lens', 'editor.action.triggerSuggest', {}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 3ed0b246639aac..1d752bf9878a5c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -12,7 +12,7 @@ import type { IndexPatternField } from '../../../../types'; import type { OperationMetadata } from '../../../../../types'; import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; import { tinymathFunctions } from '../util'; -import { getSignatureHelp, getHover, suggest } from './math_completion'; +import { getSignatureHelp, getHover, suggest, monacoPositionToOffset } from './math_completion'; const buildGenericColumn = (type: string) => { return ({ field }: { field?: IndexPatternField }) => { @@ -366,4 +366,14 @@ describe('math completion', () => { expect(results.list).toEqual(['bytes', 'memory']); }); }); + + describe('monacoPositionToOffset', () => { + it('should work with multi-line strings accounting for newline characters', () => { + const input = `012 +456 +89')`; + expect(input[monacoPositionToOffset(input, new monaco.Position(1, 1))]).toEqual('0'); + expect(input[monacoPositionToOffset(input, new monaco.Position(3, 2))]).toEqual('9'); + }); + }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 36387e26def5c5..0e7d01f0beed2c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -96,7 +96,7 @@ export function monacoPositionToOffset(expression: string, position: monaco.Posi .slice(0, position.lineNumber) .reduce( (prev, current, index) => - prev + index === position.lineNumber - 1 ? position.column : current.length, + prev + (index === position.lineNumber - 1 ? position.column - 1 : current.length + 1), 0 ); } diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index 4adf3a6c0a2829..2aaa08c0941f71 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -56,7 +56,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDatatableCellText(0, 0)).to.eql('14,005'); }); - it('should insert single quotes when needed to create valid KQL', async () => { + it('should insert single quotes and escape when needed to create valid KQL', async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickVisType('lens'); await PageObjects.lens.goToTimeRange(); @@ -69,15 +69,22 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { keepOpen: true, }); - const input = await find.activeElement(); + let input = await find.activeElement(); await input.type(' '); await input.pressKeys(browser.keys.ARROW_LEFT); await input.type(`Men's`); await PageObjects.common.sleep(100); - const element = await find.byCssSelector('.monaco-editor'); - expect(await element.getVisibleText()).to.equal(`count(kql=\'Men\\'s \')`); + let element = await find.byCssSelector('.monaco-editor'); + expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s ')`); + + await PageObjects.lens.typeFormula('count(kql='); + input = await find.activeElement(); + await input.type(`Men\'s `); + + element = await find.byCssSelector('.monaco-editor'); + expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s ')`); }); it('should persist a broken formula on close', async () => { diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index adc77484cb7527..80211e61a14829 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -1014,6 +1014,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await find.byCssSelector('.monaco-editor'); await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); const input = await find.activeElement(); + await input.clearValueWithKeyboard(); await input.type(formula); // Formula is applied on a 250ms timer, won't be applied if we leave too early await PageObjects.common.sleep(500); From b349a1c60e39eb70d23e2519c167ef022064e828 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 3 Jun 2021 10:08:29 +0200 Subject: [PATCH 152/185] fix reduced suggestions --- .../indexpattern_suggestions.test.tsx | 92 +++++++++++++++++++ .../indexpattern_suggestions.ts | 22 +++-- .../operations/layer_helpers.ts | 9 +- 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx index cfadbab2d42907..93ea3069894d32 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx @@ -2496,6 +2496,98 @@ describe('IndexPattern Data Source suggestions', () => { ).toBeTruthy(); }); }); + + it('will leave dangling references in place', () => { + const initialState = testInitialState(); + const state: IndexPatternPrivateState = { + ...initialState, + layers: { + ...initialState.layers, + first: { + ...initialState.layers.first, + columnOrder: ['date', 'ref'], + + columns: { + date: { + label: '', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'timestamp', + params: { interval: 'auto' }, + }, + ref: { + label: '', + dataType: 'number', + isBucketed: false, + operationType: 'cumulative_sum', + references: ['non_existing_metric'], + }, + }, + }, + }, + }; + + const result = getDatasourceSuggestionsFromCurrentState(state); + + // only generate suggestions for top level metrics + expect( + result.filter((suggestion) => suggestion.table.changeType === 'reduced').length + ).toEqual(1); + + // top level "ref" column + expect( + result.some( + (suggestion) => + suggestion.table.changeType === 'reduced' && + isEqual(suggestion.state.layers.first.columnOrder, ['ref']) + ) + ).toBeTruthy(); + }); + + it('will not suggest reduced tables if there is just a referenced top level metric', () => { + const initialState = testInitialState(); + const state: IndexPatternPrivateState = { + ...initialState, + layers: { + ...initialState.layers, + first: { + ...initialState.layers.first, + columnOrder: ['ref', 'metric'], + + columns: { + ref: { + label: '', + dataType: 'number', + isBucketed: false, + operationType: 'math', + params: { + tinymathAst: '', + }, + references: ['metric'], + }, + metric: { + label: '', + dataType: 'number', + isBucketed: false, + operationType: 'count', + sourceField: 'Records', + }, + }, + }, + }, + }; + + const result = getDatasourceSuggestionsFromCurrentState(state); + + expect( + result.filter((suggestion) => suggestion.table.changeType === 'unchanged').length + ).toEqual(1); + + expect( + result.filter((suggestion) => suggestion.table.changeType === 'reduced').length + ).toEqual(0); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts index bb5cf32d7f26c3..cff036db4813bb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts @@ -517,8 +517,11 @@ function createAlternativeMetricSuggestions( ) { const layer = state.layers[layerId]; const suggestions: Array> = []; + const topLevelMetricColumns = layer.columnOrder.filter( + (columnId) => !isReferenced(layer, columnId) + ); - layer.columnOrder.forEach((columnId) => { + topLevelMetricColumns.forEach((columnId) => { const column = layer.columns[columnId]; if (!hasField(column)) { return; @@ -622,13 +625,16 @@ function createSimplifiedTableSuggestions(state: IndexPatternPrivateState, layer }) ) .concat( - topLevelMetricColumns.map((columnId) => { - return { - ...layer, - columnOrder: [columnId, ...getReferencedColumnIds(layer, columnId)], - noBuckets: true, - }; - }) + // if there is just a single top level metric, the unchanged suggestion will take care of this case - only split up if there are multiple metrics or at least one bucket + availableBucketedColumns.length > 0 || topLevelMetricColumns.length > 1 + ? topLevelMetricColumns.map((columnId) => { + return { + ...layer, + columnOrder: [columnId, ...getReferencedColumnIds(layer, columnId)], + noBuckets: true, + }; + }) + : [] ) .map(({ noBuckets, ...updatedLayer }) => { return buildSuggestion({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index d4c0b978f0b99c..9f906aaf2792f5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1229,8 +1229,13 @@ export function getReferencedColumnIds(layer: IndexPatternLayer, columnId: strin function collect(id: string) { const column = layer.columns[id]; if (column && 'references' in column) { - referencedIds.push(...column.references); - column.references.forEach(collect); + const columnReferences = column.references; + // only record references which have created columns yet + const existingReferences = columnReferences.filter((reference) => + Boolean(layer.columns[reference]) + ); + referencedIds.push(...existingReferences); + existingReferences.forEach(collect); } } collect(columnId); From 64260349a257ac3834cd80cf94d34b4819d1f5a4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 3 Jun 2021 10:34:32 +0200 Subject: [PATCH 153/185] fix responsiveness --- .../operations/definitions/formula/editor/formula.scss | 2 +- .../operations/definitions/formula/editor/formula_help.tsx | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index 235a6356cad316..db19c9a78aee4a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -101,7 +101,7 @@ .lnsFormula__docsContent { .lnsFormula__docs--overlay & { height: 40vh; - width: 65vh; + width: #{'min(65vh, 90vw)'}; } .lnsFormula__docs--inline & { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 7384b65d035f4a..afe5471666b22b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -224,6 +224,7 @@ function FormulaHelp({ className="lnsFormula__docsSidebarInner" direction="column" gutterSize="none" + responsive={false} > Date: Thu, 3 Jun 2021 17:38:39 +0200 Subject: [PATCH 154/185] fix unit test --- .../editor_frame/config_panel/config_panel.test.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx index d6e3a7c411b878..1ec48f516bd32d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx @@ -121,19 +121,23 @@ describe('ConfigPanel', () => { expect(component.find(LayerPanel).exists()).toBe(false); }); - it('allow datasources and visualizations to use setters', () => { + it('allow datasources and visualizations to use setters', async () => { const props = getDefaultProps(); const component = mountWithIntl(); const { updateDatasource, updateAll } = component.find(LayerPanel).props(); const updater = () => 'updated'; updateDatasource('ds1', updater); + // wait for one tick so async updater has a chance to trigger + await new Promise((r) => setTimeout(r, 0)); expect(props.dispatch).toHaveBeenCalledTimes(1); expect(props.dispatch.mock.calls[0][0].updater(props.datasourceStates.ds1.state)).toEqual( 'updated' ); updateAll('ds1', updater, props.visualizationState); + // wait for one tick so async updater has a chance to trigger + await new Promise((r) => setTimeout(r, 0)); expect(props.dispatch).toHaveBeenCalledTimes(2); expect(props.dispatch.mock.calls[0][0].updater(props.datasourceStates.ds1.state)).toEqual( 'updated' From 3e9ebdb8999c82fefe3d9b27108e30766beae778 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 3 Jun 2021 18:32:14 -0400 Subject: [PATCH 155/185] Fix autocomplete on nested math --- packages/kbn-tinymath/grammar/grammar.peggy | 44 ++++++---- packages/kbn-tinymath/index.d.ts | 6 +- packages/kbn-tinymath/test/library.test.js | 31 +++++++ .../formula/editor/formula_editor.tsx | 36 ++------ .../formula/editor/math_completion.test.ts | 41 ++++++--- .../formula/editor/math_completion.ts | 84 +++++++++++-------- .../definitions/formula/formula.test.tsx | 2 +- .../definitions/formula/validation.ts | 30 +++---- 8 files changed, 163 insertions(+), 111 deletions(-) diff --git a/packages/kbn-tinymath/grammar/grammar.peggy b/packages/kbn-tinymath/grammar/grammar.peggy index cbcb0b91bfea90..1c6f8c3334c234 100644 --- a/packages/kbn-tinymath/grammar/grammar.peggy +++ b/packages/kbn-tinymath/grammar/grammar.peggy @@ -1,16 +1,16 @@ // tinymath parsing grammar { - function simpleLocation (location) { - // Returns an object representing the position of the function within the expression, - // demarcated by the position of its first character and last character. We calculate these values - // using the offset because the expression could span multiple lines, and we don't want to deal - // with column and line values. - return { - min: location.start.offset, - max: location.end.offset + function simpleLocation (location) { + // Returns an object representing the position of the function within the expression, + // demarcated by the position of its first character and last character. We calculate these values + // using the offset because the expression could span multiple lines, and we don't want to deal + // with column and line values. + return { + min: location.start.offset, + max: location.end.offset + } } - } } start @@ -74,26 +74,34 @@ Expression = AddSubtract AddSubtract - = _ left:MultiplyDivide rest:(('+' / '-') MultiplyDivide)* _ { - return rest.reduce((acc, curr) => ({ + = _ left:MultiplyDivide rest:(('+' / '-') MultiplyDivide)+ _ { + const topLevel = rest.reduce((acc, curr) => ({ type: 'function', name: curr[0] === '+' ? 'add' : 'subtract', args: [acc, curr[1]], - location: simpleLocation(location()), - text: text() - }), left) + }), left); + if (typeof topLevel === 'object') { + topLevel.location = simpleLocation(location()); + topLevel.text = text(); + } + return topLevel; } + / MultiplyDivide MultiplyDivide = _ left:Factor rest:(('*' / '/') Factor)* _ { - return rest.reduce((acc, curr) => ({ + const topLevel = rest.reduce((acc, curr) => ({ type: 'function', name: curr[0] === '*' ? 'multiply' : 'divide', args: [acc, curr[1]], - location: simpleLocation(location()), - text: text() - }), left) + }), left); + if (typeof topLevel === 'object') { + topLevel.location = simpleLocation(location()); + topLevel.text = text(); + } + return topLevel; } + / Factor Factor = Group diff --git a/packages/kbn-tinymath/index.d.ts b/packages/kbn-tinymath/index.d.ts index c3c32a59fa15ad..8e15d86c88fc84 100644 --- a/packages/kbn-tinymath/index.d.ts +++ b/packages/kbn-tinymath/index.d.ts @@ -24,9 +24,11 @@ export interface TinymathLocation { export interface TinymathFunction { type: 'function'; name: string; - text: string; args: TinymathAST[]; - location: TinymathLocation; + // Location is not guaranteed because PEG grammars are not left-recursive + location?: TinymathLocation; + // Text is not guaranteed because PEG grammars are not left-recursive + text?: string; } export interface TinymathVariable { diff --git a/packages/kbn-tinymath/test/library.test.js b/packages/kbn-tinymath/test/library.test.js index bf1c7a9dbc5fb5..bbc8503684fd40 100644 --- a/packages/kbn-tinymath/test/library.test.js +++ b/packages/kbn-tinymath/test/library.test.js @@ -41,6 +41,35 @@ describe('Parser', () => { }); }); + describe('Math', () => { + it('converts basic symbols into left-to-right pairs', () => { + expect(parse('a + b + c - d')).toEqual({ + args: [ + { + name: 'add', + type: 'function', + args: [ + { + name: 'add', + type: 'function', + args: [ + expect.objectContaining({ location: { min: 0, max: 2 } }), + expect.objectContaining({ location: { min: 3, max: 6 } }), + ], + }, + expect.objectContaining({ location: { min: 7, max: 10 } }), + ], + }, + expect.objectContaining({ location: { min: 11, max: 13 } }), + ], + name: 'subtract', + type: 'function', + text: 'a + b + c - d', + location: { min: 0, max: 13 }, + }); + }); + }); + describe('Variables', () => { it('strings', () => { expect(parse('f')).toEqual(variableEqual('f')); @@ -263,6 +292,8 @@ describe('Evaluate', () => { expect(evaluate('5/20')).toEqual(0.25); expect(evaluate('1 + 1 + 2 + 3 + 12')).toEqual(19); expect(evaluate('100 / 10 / 10')).toEqual(1); + expect(evaluate('0 * 1 - 100 / 10 / 10')).toEqual(-1); + expect(evaluate('100 / (10 / 10)')).toEqual(100); }); it('equations with functions', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 524c8c7b71d253..ce7b99fa4405be 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -290,35 +290,23 @@ export function FormulaEditor({ context: monaco.languages.CompletionContext ) => { const innerText = model.getValue(); - const textRange = model.getFullModelRange(); - let wordRange: monaco.Range; let aSuggestions: { list: LensMathSuggestion[]; type: SUGGESTION_TYPE } = { list: [], type: SUGGESTION_TYPE.FIELD, }; - - const lengthAfterPosition = model.getValueLengthInRange({ - startLineNumber: position.lineNumber, - startColumn: position.column, - endLineNumber: textRange.endLineNumber, - endColumn: textRange.endColumn, - }); + const offset = monacoPositionToOffset(innerText, position); if (context.triggerCharacter === '(') { + // Monaco usually inserts the end quote and reports the position is after the end quote + if (innerText.slice(offset - 1, offset + 1) === '()') { + position = position.delta(0, -1); + } const wordUntil = model.getWordAtPosition(position.delta(0, -3)); if (wordUntil) { - wordRange = new monaco.Range( - position.lineNumber, - position.column, - position.lineNumber, - position.column - ); - // Retrieve suggestions for subexpressions - // TODO: make this work for expressions nested more than one level deep aSuggestions = await suggest({ - expression: innerText.substring(0, innerText.length - lengthAfterPosition) + ')', - position: innerText.length - lengthAfterPosition, + expression: innerText, + zeroIndexedOffset: offset, context, indexPattern, operationDefinitionMap: visibleOperationsMap, @@ -328,7 +316,7 @@ export function FormulaEditor({ } else { aSuggestions = await suggest({ expression: innerText, - position: innerText.length - lengthAfterPosition, + zeroIndexedOffset: offset, context, indexPattern, operationDefinitionMap: visibleOperationsMap, @@ -338,13 +326,7 @@ export function FormulaEditor({ return { suggestions: aSuggestions.list.map((s) => - getSuggestion( - s, - aSuggestions.type, - wordRange, - visibleOperationsMap, - context.triggerCharacter - ) + getSuggestion(s, aSuggestions.type, visibleOperationsMap, context.triggerCharacter) ), }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 1d752bf9878a5c..6932ae11b9bda3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { parse } from '@kbn/tinymath'; import { monaco } from '@kbn/monaco'; import { createMockedIndexPattern } from '../../../../mocks'; import { GenericOperationDefinition } from '../../index'; @@ -12,7 +13,13 @@ import type { IndexPatternField } from '../../../../types'; import type { OperationMetadata } from '../../../../../types'; import { dataPluginMock } from '../../../../../../../../../src/plugins/data/public/mocks'; import { tinymathFunctions } from '../util'; -import { getSignatureHelp, getHover, suggest, monacoPositionToOffset } from './math_completion'; +import { + getSignatureHelp, + getHover, + suggest, + monacoPositionToOffset, + getInfoAtZeroIndexedPosition, +} from './math_completion'; const buildGenericColumn = (type: string) => { return ({ field }: { field?: IndexPatternField }) => { @@ -145,8 +152,9 @@ describe('math completion', () => { }); it('should return a signature for a complex tinymath function', () => { + // 15 is the whitespace between the two arguments expect( - unwrapSignatures(getSignatureHelp('clamp(count(), 5)', 7, operationDefinitionMap)) + unwrapSignatures(getSignatureHelp('clamp(count(), 5)', 15, operationDefinitionMap)) ).toEqual({ label: expect.stringContaining('clamp('), documentation: { value: '' }, @@ -213,7 +221,7 @@ describe('math completion', () => { // some typing const results = await suggest({ expression: '', - position: 1, + zeroIndexedOffset: 1, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '', @@ -234,7 +242,7 @@ describe('math completion', () => { it('should list all valid sub-functions for a fullReference', async () => { const results = await suggest({ expression: 'moving_average()', - position: 15, + zeroIndexedOffset: 15, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', @@ -252,7 +260,7 @@ describe('math completion', () => { it('should list all valid named arguments for a fullReference', async () => { const results = await suggest({ expression: 'moving_average(count(),)', - position: 23, + zeroIndexedOffset: 23, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', @@ -267,7 +275,7 @@ describe('math completion', () => { it('should not list named arguments when they are already in use', async () => { const results = await suggest({ expression: 'moving_average(count(), window=5, )', - position: 34, + zeroIndexedOffset: 34, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', @@ -282,7 +290,7 @@ describe('math completion', () => { it('should list all valid positional arguments for a tinymath function used by name', async () => { const results = await suggest({ expression: 'divide(count(), )', - position: 16, + zeroIndexedOffset: 16, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', @@ -303,7 +311,7 @@ describe('math completion', () => { it('should list all valid positional arguments for a tinymath function used with alias', async () => { const results = await suggest({ expression: 'count() / ', - position: 10, + zeroIndexedOffset: 10, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: ',', @@ -324,7 +332,7 @@ describe('math completion', () => { it('should not autocomplete any fields for the count function', async () => { const results = await suggest({ expression: 'count()', - position: 6, + zeroIndexedOffset: 6, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', @@ -339,7 +347,7 @@ describe('math completion', () => { it('should autocomplete and validate the right type of field', async () => { const results = await suggest({ expression: 'sum()', - position: 4, + zeroIndexedOffset: 4, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', @@ -354,7 +362,7 @@ describe('math completion', () => { it('should autocomplete only operations that provide numeric output', async () => { const results = await suggest({ expression: 'last_value()', - position: 11, + zeroIndexedOffset: 11, context: { triggerKind: monaco.languages.CompletionTriggerKind.TriggerCharacter, triggerCharacter: '(', @@ -376,4 +384,15 @@ describe('math completion', () => { expect(input[monacoPositionToOffset(input, new monaco.Position(3, 2))]).toEqual('9'); }); }); + + describe('getInfoAtZeroIndexedPosition', () => { + it('should return the location for a function inside multiple levels of math', () => { + const expression = `count() + 5 + average(LENS_MATH_MARKER)`; + const ast = parse(expression); + expect(getInfoAtZeroIndexedPosition(ast, 22)).toEqual({ + ast: expect.objectContaining({ value: 'LENS_MATH_MARKER' }), + parent: expect.objectContaining({ name: 'average' }), + }); + }); + }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 0e7d01f0beed2c..435d3e7eb8c6a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -52,21 +52,29 @@ function inLocation(cursorPosition: number, location: TinymathLocation) { const MARKER = 'LENS_MATH_MARKER'; -function getInfoAtPosition( +export function getInfoAtZeroIndexedPosition( ast: TinymathAST, - position: number, + zeroIndexedPosition: number, parent?: TinymathFunction ): undefined | { ast: TinymathAST; parent?: TinymathFunction } { if (typeof ast === 'number') { return; } - if (!inLocation(position, ast.location)) { + // +, -, *, and / do not have location any more + if (ast.location && !inLocation(zeroIndexedPosition, ast.location)) { return; } if (ast.type === 'function') { - const [match] = ast.args.map((arg) => getInfoAtPosition(arg, position, ast)).filter((a) => a); + const [match] = ast.args + .map((arg) => getInfoAtZeroIndexedPosition(arg, zeroIndexedPosition, ast)) + .filter((a) => a); if (match) { - return match.parent ? match : { ...match, parent: ast }; + return match; + } else if (ast.location) { + return { ast }; + } else { + // None of the arguments match, but we don't know the position so it's not a match + return; } } return { @@ -103,24 +111,25 @@ export function monacoPositionToOffset(expression: string, position: monaco.Posi export async function suggest({ expression, - position, + zeroIndexedOffset, context, indexPattern, operationDefinitionMap, data, }: { expression: string; - position: number; + zeroIndexedOffset: number; context: monaco.languages.CompletionContext; indexPattern: IndexPattern; operationDefinitionMap: Record; data: DataPublicPluginStart; }): Promise<{ list: LensMathSuggestion[]; type: SUGGESTION_TYPE }> { - const text = expression.substr(0, position) + MARKER + expression.substr(position); + const text = + expression.substr(0, zeroIndexedOffset) + MARKER + expression.substr(zeroIndexedOffset); try { const ast = parse(text); - const tokenInfo = getInfoAtPosition(ast, position); + const tokenInfo = getInfoAtZeroIndexedPosition(ast, zeroIndexedOffset); const tokenAst = tokenInfo?.ast; const isNamedArgument = @@ -279,11 +288,8 @@ function getArgumentSuggestions( ) { possibleOperationNames.push( ...a.operations - .filter( - (o) => - operation.requiredReferences.some((requirement) => - requirement.input.includes(o.type) - ) && !operationDefinitionMap[o.operationType].hidden + .filter((o) => + operation.requiredReferences.some((requirement) => requirement.input.includes(o.type)) ) .map((o) => o.operationType) ); @@ -339,7 +345,6 @@ const TRIGGER_SUGGESTION_COMMAND = { export function getSuggestion( suggestion: LensMathSuggestion, type: SUGGESTION_TYPE, - range: monaco.Range, operationDefinitionMap: Record, triggerChar: string | undefined ): monaco.languages.CompletionItem { @@ -412,7 +417,8 @@ export function getSuggestion( insertTextRules, command, additionalTextEdits: [], - range, + // @ts-expect-error Monaco says this type is required, but provides a default value + range: undefined, sortText, filterText, }; @@ -513,29 +519,33 @@ export function getSignatureHelp( try { const ast = parse(text); - const tokenInfo = getInfoAtPosition(ast, position); + const tokenInfo = getInfoAtZeroIndexedPosition(ast, position); + let signatures: ReturnType = []; + let index = 0; if (tokenInfo?.parent) { const name = tokenInfo.parent.name; // reference equality is fine here because of the way the getInfo function works - const index = tokenInfo.parent.args.findIndex((arg) => arg === tokenInfo.ast); - - const signatures = getSignaturesForFunction(name, operationDefinitionMap); - if (signatures.length) { - return { - value: { - // remove the documentation - signatures: signatures.map(({ documentation, ...signature }) => ({ - ...signature, - // extract only the first section (usually few lines) - documentation: { value: documentation.value.split('\n\n')[0] }, - })), - activeParameter: index, - activeSignature: 0, - }, - dispose: () => {}, - }; - } + index = tokenInfo.parent.args.findIndex((arg) => arg === tokenInfo.ast); + signatures = getSignaturesForFunction(name, operationDefinitionMap); + } else if (typeof tokenInfo?.ast === 'object' && tokenInfo.ast.type === 'function') { + const name = tokenInfo.ast.name; + signatures = getSignaturesForFunction(name, operationDefinitionMap); + } + if (signatures.length) { + return { + value: { + // remove the documentation + signatures: signatures.map(({ documentation, ...signature }) => ({ + ...signature, + // extract only the first section (usually few lines) + documentation: { value: documentation.value.split('\n\n')[0] }, + })), + activeParameter: index, + activeSignature: 0, + }, + dispose: () => {}, + }; } } catch (e) { // do nothing @@ -551,7 +561,7 @@ export function getHover( try { const ast = parse(expression); - const tokenInfo = getInfoAtPosition(ast, position); + const tokenInfo = getInfoAtZeroIndexedPosition(ast, position); if (!tokenInfo || typeof tokenInfo.ast === 'number' || !('name' in tokenInfo.ast)) { return { contents: [] }; @@ -574,7 +584,7 @@ export function getTokenInfo(expression: string, position: number) { try { const ast = parse(text); - return getInfoAtPosition(ast, position); + return getInfoAtZeroIndexedPosition(ast, position); } catch (e) { return; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index b4ba3429a36102..3a66cf302a9e7f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -567,7 +567,7 @@ describe('formula', () => { ).toEqual({ col1X0: { min: 15, max: 29 }, col1X2: { min: 0, max: 41 }, - col1X3: { min: 43, max: 50 }, + col1X3: { min: 42, max: 50 }, }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 7352f045a08f5b..d3613973fa8b9f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -94,7 +94,7 @@ export function hasInvalidOperations( return { // avoid duplicates names: Array.from(new Set(nodes.map(({ name }) => name))), - locations: nodes.map(({ location }) => location), + locations: nodes.map(({ location }) => location).filter((a) => a) as TinymathLocation[], }; } @@ -438,7 +438,7 @@ function validateNameArguments( operation: node.name, params: missingParams.map(({ name }) => name).join(', '), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -451,7 +451,7 @@ function validateNameArguments( operation: node.name, params: wrongTypeParams.map(({ name }) => name).join(', '), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -464,7 +464,7 @@ function validateNameArguments( operation: node.name, params: duplicateParams.join(', '), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -478,7 +478,7 @@ function validateNameArguments( getMessageFromId({ messageId: 'tooManyQueries', values: {}, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -525,7 +525,7 @@ function runFullASTValidation( type: 'field', argument: `math operation`, }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } else { @@ -537,7 +537,7 @@ function runFullASTValidation( type: 'field', argument: getValueOrName(firstArg), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -551,7 +551,7 @@ function runFullASTValidation( values: { operation: node.name, }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -563,7 +563,7 @@ function runFullASTValidation( values: { operation: node.name, }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } else { @@ -594,7 +594,7 @@ function runFullASTValidation( type: 'operation', argument: getValueOrName(firstArg), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } @@ -605,7 +605,7 @@ function runFullASTValidation( values: { operation: node.name, }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } else { @@ -731,7 +731,7 @@ export function validateMathNodes(root: TinymathAST, missingVariableSet: Set name).join(', '), }, - locations: [node.location], + locations: node.location ? [node.location] : [], }) ); } From ae81c0e1ea3402351f0e298e14420bd057898bbc Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Thu, 3 Jun 2021 18:42:33 -0400 Subject: [PATCH 156/185] Show errors and warnings on first render --- .../formula/editor/formula_editor.tsx | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index ce7b99fa4405be..975923e32a5ffa 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -155,16 +155,18 @@ export function FormulaEditor({ if (errors.length) { if (currentColumn.params.isFormulaBroken) { // If the formula is already broken, show the latest error message in the workspace - updateLayer( - regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - visibleOperationsMap - ).newLayer - ); + if (currentColumn.params.formula !== text) { + updateLayer( + regenerateLayerFromAst( + text || '', + layer, + columnId, + currentColumn, + indexPattern, + visibleOperationsMap + ).newLayer + ); + } } const markers = errors @@ -259,7 +261,7 @@ export function FormulaEditor({ }, // Make it validate on flyout open in case of a broken formula left over // from a previous edit - { skipFirstRender: true }, + { skipFirstRender: false }, 256, [text] ); From 2b44b2fe261c418fb47af5acd9bfec985baedb94 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 4 Jun 2021 10:41:51 +0200 Subject: [PATCH 157/185] fix transposing column crash --- .../config_panel/config_panel.tsx | 25 +++++++++++++------ .../editor_frame/config_panel/layer_panel.tsx | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 4105dd2cb7c84d..90b20e8ce0fd4b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -60,20 +60,28 @@ export function LayerPanels( [dispatch, activeVisualization] ); const updateDatasource = useMemo( + () => (datasourceId: string, newState: unknown) => { + // React will synchronously update if this is triggered from a third party component, + // which we don't want. The timeout lets user interaction have priority, then React updates. + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + updater: (prevState: unknown) => + typeof newState === 'function' ? newState(prevState) : newState, + datasourceId, + clearStagedPreview: false, + }); + }, + [dispatch] + ); + const updateDatasourceAsync = useMemo( () => (datasourceId: string, newState: unknown) => { // React will synchronously update if this is triggered from a third party component, // which we don't want. The timeout lets user interaction have priority, then React updates. setTimeout(() => { - dispatch({ - type: 'UPDATE_DATASOURCE_STATE', - updater: (prevState: unknown) => - typeof newState === 'function' ? newState(prevState) : newState, - datasourceId, - clearStagedPreview: false, - }); + updateDatasource(datasourceId, newState); }, 0); }, - [dispatch] + [updateDatasource] ); const updateAll = useMemo( () => (datasourceId: string, newDatasourceState: unknown, newVisualizationState: unknown) => { @@ -134,6 +142,7 @@ export function LayerPanels( visualizationState={visualizationState} updateVisualization={setVisualizationState} updateDatasource={updateDatasource} + updateDatasourceAsync={updateDatasourceAsync} updateAll={updateAll} isOnlyLayer={layerIds.length === 1} onRemoveLayer={() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 137b168a8c4083..3ce5401be0b8a5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -42,6 +42,7 @@ export function LayerPanel( isOnlyLayer: boolean; updateVisualization: StateSetter; updateDatasource: (datasourceId: string, newState: unknown) => void; + updateDatasourceAsync: (datasourceId: string, newState: unknown) => void; updateAll: ( datasourceId: string, newDatasourcestate: unknown, @@ -481,7 +482,7 @@ export function LayerPanel( }) ); } else { - props.updateDatasource(datasourceId, newState); + props.updateDatasourceAsync(datasourceId, newState); } if (!shouldClose) { setActiveDimension({ From d1628b71ca613fb7f7ecc17ea135eb9bfdfaa5ac Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 4 Jun 2021 11:16:57 -0400 Subject: [PATCH 158/185] Update comment --- .../editor_frame/config_panel/config_panel.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index 90b20e8ce0fd4b..81c044af532fbf 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -61,8 +61,6 @@ export function LayerPanels( ); const updateDatasource = useMemo( () => (datasourceId: string, newState: unknown) => { - // React will synchronously update if this is triggered from a third party component, - // which we don't want. The timeout lets user interaction have priority, then React updates. dispatch({ type: 'UPDATE_DATASOURCE_STATE', updater: (prevState: unknown) => From 48d7ff798c42ae5c99c3130918b2fe6bb4fd1014 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 4 Jun 2021 17:38:07 +0200 Subject: [PATCH 159/185] :bug: Fix field error message --- .../operations/definitions/formula/formula.test.tsx | 13 +++++++++++++ .../operations/definitions/formula/validation.ts | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 3a66cf302a9e7f..8468ed4660e80f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -1045,6 +1045,19 @@ invalid: " ).toEqual(['Use only one of kql= or lucene=, not both']); }); + it("returns a clear error when there's a missing field for a function", () => { + for (const fn of ['average', 'terms', 'max', 'sum']) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`${fn}()`), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The first argument for ${fn} should be a field name. Found no field`]); + } + }); + it('returns no error if a math operation is passed to fullReference operations', () => { const formulas = [ 'derivative(7+1)', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index d3613973fa8b9f..bda62875b07d2d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -535,7 +535,11 @@ function runFullASTValidation( values: { operation: node.name, type: 'field', - argument: getValueOrName(firstArg), + argument: + getValueOrName(firstArg) || + i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + defaultMessage: 'no field', + }), }, locations: node.location ? [node.location] : [], }) From 1b97e8b187041168a41452b1f0119e54b32949fc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 4 Jun 2021 17:38:28 +0200 Subject: [PATCH 160/185] fix test types --- .../editor_frame/config_panel/layer_panel.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index 0aa23d8ad947f5..8801be4d8ff017 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -78,6 +78,7 @@ describe('LayerPanel', () => { visualizationState: 'state', updateVisualization: jest.fn(), updateDatasource: jest.fn(), + updateDatasourceAsync: jest.fn(), updateAll: jest.fn(), framePublicAPI: frame, isOnlyLayer: true, From 5cd99fe35fe04a89101b9cd52db39a301081100d Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 4 Jun 2021 17:40:42 +0200 Subject: [PATCH 161/185] :memo: Fix i18n name --- .../operations/definitions/formula/validation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index bda62875b07d2d..4e1f1bef1af467 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -537,7 +537,7 @@ function runFullASTValidation( type: 'field', argument: getValueOrName(firstArg) || - i18n.translate('xpack.lens.indexPattern.formulaOperationWrongFirstArgument', { + i18n.translate('xpack.lens.indexPattern.formulaNoFieldForOperation', { defaultMessage: 'no field', }), }, From 881592aa2e8aefc61c8a5d87941a13394d6c4ee3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Fri, 4 Jun 2021 18:38:52 +0200 Subject: [PATCH 162/185] :lipstick: Manage wordwrap via react component --- .../definitions/formula/editor/formula_editor.tsx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 975923e32a5ffa..bf4ed838c64876 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -68,6 +68,7 @@ export function FormulaEditor({ >([]); const [isHelpOpen, setIsHelpOpen] = useState(isFullscreen); const [isWarningOpen, setIsWarningOpen] = useState(false); + const [isWordWrapped, toggleWordWrap] = useState(false); const editorModel = React.useRef(); const overflowDiv1 = React.useRef(); const disposables = React.useRef([]); @@ -482,8 +483,6 @@ export function FormulaEditor({ [] ); - const isWordWrapped = editor1.current?.getOption(monaco.editor.EditorOption.wordWrap) !== 'off'; - const codeEditorOptions: CodeEditorProps = { languageId: LANGUAGE_ID, value: text ?? '', @@ -570,12 +569,9 @@ export function FormulaEditor({ isSelected={!isWordWrapped} onClick={() => { editor1.current?.updateOptions({ - wordWrap: - editor1.current?.getOption(monaco.editor.EditorOption.wordWrap) === - 'off' - ? 'on' - : 'off', + wordWrap: isWordWrapped ? 'off' : 'on', }); + toggleWordWrap(!isWordWrapped); }} /> From 1d16ef99e57a0308291a3541ffeac36224a58c60 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 4 Jun 2021 16:05:02 -0400 Subject: [PATCH 163/185] Fix selector for palettes that interferes with quick functions --- x-pack/test/functional/page_objects/lens_page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 80211e61a14829..38d80cbe84d79e 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -350,7 +350,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await retry.try(async () => { await testSubjects.click('lns-palettePicker'); const currentPalette = await ( - await find.byCssSelector('[aria-selected=true]') + await find.byCssSelector('[role=option][aria-selected=true]') ).getAttribute('id'); expect(currentPalette).to.equal(palette); }); From fd2ea828c677d12bee726aca197783af11c137b4 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 4 Jun 2021 16:33:18 -0400 Subject: [PATCH 164/185] Use word wrapping by default --- .../operations/definitions/formula/editor/formula_editor.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index bf4ed838c64876..95049eaf6f2f30 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -68,7 +68,7 @@ export function FormulaEditor({ >([]); const [isHelpOpen, setIsHelpOpen] = useState(isFullscreen); const [isWarningOpen, setIsWarningOpen] = useState(false); - const [isWordWrapped, toggleWordWrap] = useState(false); + const [isWordWrapped, toggleWordWrap] = useState(true); const editorModel = React.useRef(); const overflowDiv1 = React.useRef(); const disposables = React.useRef([]); From 6f79509a96b2d357d1e8c26323fb5eef9e9706f3 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 4 Jun 2021 17:58:44 -0400 Subject: [PATCH 165/185] Errors for managed references are handled at the top level --- .../operations/__mocks__/index.ts | 3 ++ .../definitions/formula/formula.test.tsx | 6 ++-- .../definitions/formula/formula.tsx | 21 ++++++++++++- .../operations/layer_helpers.test.ts | 30 +++++++++++++++++++ .../operations/layer_helpers.ts | 14 ++++++++- 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts index 40d7e3ef94ad68..d6429fb67e9a1d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts @@ -17,6 +17,7 @@ jest.spyOn(actualHelpers, 'insertOrReplaceColumn'); jest.spyOn(actualHelpers, 'insertNewColumn'); jest.spyOn(actualHelpers, 'replaceColumn'); jest.spyOn(actualHelpers, 'getErrorMessages'); +jest.spyOn(actualHelpers, 'getColumnOrder'); export const { getAvailableOperationsByMetadata, @@ -48,6 +49,8 @@ export const { resetIncomplete, isOperationAllowedAsReference, canTransition, + isColumnValidAsReference, + getManagedColumnsFrom, } = actualHelpers; export const { adjustTimeScaleLabelSuffix, DEFAULT_TIME_SCALE } = actualTimeScaleUtils; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 8468ed4660e80f..d87bcde65ce6cd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -14,8 +14,10 @@ import { tinymathFunctions } from './util'; jest.mock('../../layer_helpers', () => { return { - getColumnOrder: ({ columns }: { columns: Record }) => - Object.keys(columns), + getColumnOrder: jest.fn(({ columns }: { columns: Record }) => + Object.keys(columns) + ), + getManagedColumnsFrom: jest.fn().mockReturnValue([]), }; }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index bbf495281004f7..3ed50906908762 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -14,6 +14,7 @@ import { MemoizedFormulaEditor } from './editor'; import { regenerateLayerFromAst } from './parse'; import { generateFormula } from './generate'; import { filterByVisibleOperation } from './util'; +import { getManagedColumnsFrom } from '../../layer_helpers'; const defaultLabel = i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', @@ -51,6 +52,7 @@ export const formulaOperation: OperationDefinition< if (!column.params.formula || !operationDefinitionMap) { return; } + const visibleOperationsMap = filterByVisibleOperation(operationDefinitionMap); const { root, error } = tryToParse(column.params.formula, visibleOperationsMap); if (error || !root) { @@ -58,7 +60,24 @@ export const formulaOperation: OperationDefinition< } const errors = runASTValidation(root, layer, indexPattern, visibleOperationsMap); - return errors.length ? errors.map(({ message }) => message) : undefined; + + if (errors.length) { + return errors.map(({ message }) => message); + } + + const managedColumns = getManagedColumnsFrom(columnId, layer.columns); + const innerErrors = managedColumns + .flatMap(([id, col]) => { + const def = visibleOperationsMap[col.operationType]; + if (def?.getErrorMessage) { + const messages = def.getErrorMessage(layer, id, indexPattern, visibleOperationsMap); + return messages ? { message: messages.join(', ') } : []; + } + return []; + }) + .filter((marker) => marker); + + return innerErrors.length ? innerErrors.map(({ message }) => message) : undefined; }, getPossibleOperation() { return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index d85f819b599812..ba3bee415f3f49 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -2694,6 +2694,36 @@ describe('state_helpers', () => { expect(errors).toHaveLength(1); }); + it('should only collect the top level errors from managed references', () => { + const notCalledMock = jest.fn(); + const mock = jest.fn().mockReturnValue(['error 1']); + operationDefinitionMap.testReference.getErrorMessage = notCalledMock; + operationDefinitionMap.managedReference.getErrorMessage = mock; + const errors = getErrorMessages( + { + indexPatternId: '1', + columnOrder: [], + columns: { + col1: + // @ts-expect-error not statically analyzed + { operationType: 'managedReference', references: ['col2'] }, + col2: { + // @ts-expect-error not statically analyzed + operationType: 'testReference', + references: [], + }, + }, + }, + indexPattern, + {}, + '1', + {} + ); + expect(notCalledMock).not.toHaveBeenCalled(); + expect(mock).toHaveBeenCalledTimes(1); + expect(errors).toHaveLength(1); + }); + it('should ignore incompleteColumns when checking for errors', () => { const savedRef = jest.fn().mockReturnValue(['error 1']); const incompleteRef = jest.fn(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 9f906aaf2792f5..fb060bdcfcc88e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1175,8 +1175,20 @@ export function getErrorMessages( } > | undefined { - const errors = Object.entries(layer.columns) + const columns = Object.entries(layer.columns); + const visibleManagedReferences = columns.filter( + ([columnId, column]) => + !isReferenced(layer, columnId) && + operationDefinitionMap[column.operationType].input === 'managedReference' + ); + const skippedColumns = visibleManagedReferences.flatMap(([columnId]) => + getManagedColumnsFrom(columnId, layer.columns).map(([id]) => id) + ); + const errors = columns .flatMap(([columnId, column]) => { + if (skippedColumns.includes(columnId)) { + return; + } const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { return def.getErrorMessage(layer, columnId, indexPattern, operationDefinitionMap); From 2e3cdd1b977153952e20e7fa46f83653612dce92 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Jun 2021 11:36:03 +0200 Subject: [PATCH 166/185] :bug: Move the cursor just next to new inserted text --- .../operations/definitions/formula/editor/formula_editor.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 95049eaf6f2f30..8d17d464f8c7ca 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -426,7 +426,7 @@ export function FormulaEditor({ } let editOperation: monaco.editor.IIdentifiedSingleEditOperation | null = null; - let cursorOffset = 2; + const cursorOffset = 2; if (char === '=') { editOperation = { range: { @@ -448,7 +448,6 @@ export function FormulaEditor({ }, text: `\\'`, }; - cursorOffset = 3; } if (editOperation) { From 1e422fa3ba3efaf50bd42c1786e196127cf07cee Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Jun 2021 14:49:56 +0200 Subject: [PATCH 167/185] :alembic: First pass for performance --- .../lens/public/app_plugin/lens_top_nav.tsx | 251 ++++++++++-------- .../editor_frame/config_panel/layer_panel.tsx | 11 +- .../editor_frame/suggestion_panel.tsx | 3 +- .../dimension_panel/dimension_editor.tsx | 4 +- .../formula/editor/formula_editor.tsx | 74 +++--- 5 files changed, 194 insertions(+), 149 deletions(-) diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index 245e964bbd2e67..2bd539e6715c74 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -7,7 +7,7 @@ import { isEqual } from 'lodash'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useCallback, useMemo } from 'react'; import { TopNavMenuData } from '../../../../../src/plugins/navigation/public'; import { LensAppServices, LensTopNavActions, LensTopNavMenuProps } from './types'; import { downloadMultipleAs } from '../../../../../src/plugins/share/public'; @@ -164,79 +164,152 @@ export const LensTopNavMenu = ({ const unsavedTitle = i18n.translate('xpack.lens.app.unsavedFilename', { defaultMessage: 'unsaved', }); - const topNavConfig = getLensTopNavConfig({ - showSaveAndReturn: Boolean( - isLinkedToOriginatingApp && - // Temporarily required until the 'by value' paradigm is default. - (dashboardFeatureFlag.allowByValueEmbeddables || Boolean(initialInput)) - ), - enableExportToCSV: Boolean(isSaveable && activeData && Object.keys(activeData).length), - isByValueMode: getIsByValueMode(), - allowByValue: dashboardFeatureFlag.allowByValueEmbeddables, - showCancel: Boolean(isLinkedToOriginatingApp), - savingToLibraryPermitted, - savingToDashboardPermitted, - actions: { - exportToCSV: () => { - if (!activeData) { - return; - } - const datatables = Object.values(activeData); - const content = datatables.reduce>( - (memo, datatable, i) => { - // skip empty datatables - if (datatable) { - const postFix = datatables.length > 1 ? `-${i + 1}` : ''; + const topNavConfig = useMemo( + () => + getLensTopNavConfig({ + showSaveAndReturn: Boolean( + isLinkedToOriginatingApp && + // Temporarily required until the 'by value' paradigm is default. + (dashboardFeatureFlag.allowByValueEmbeddables || Boolean(initialInput)) + ), + enableExportToCSV: Boolean(isSaveable && activeData && Object.keys(activeData).length), + isByValueMode: getIsByValueMode(), + allowByValue: dashboardFeatureFlag.allowByValueEmbeddables, + showCancel: Boolean(isLinkedToOriginatingApp), + savingToLibraryPermitted, + savingToDashboardPermitted, + actions: { + exportToCSV: () => { + if (!activeData) { + return; + } + const datatables = Object.values(activeData); + const content = datatables.reduce>( + (memo, datatable, i) => { + // skip empty datatables + if (datatable) { + const postFix = datatables.length > 1 ? `-${i + 1}` : ''; - memo[`${lastKnownDoc?.title || unsavedTitle}${postFix}.csv`] = { - content: exporters.datatableToCSV(datatable, { - csvSeparator: uiSettings.get('csv:separator', ','), - quoteValues: uiSettings.get('csv:quoteValues', true), - formatFactory: data.fieldFormats.deserialize, - }), - type: exporters.CSV_MIME_TYPE, - }; + memo[`${lastKnownDoc?.title || unsavedTitle}${postFix}.csv`] = { + content: exporters.datatableToCSV(datatable, { + csvSeparator: uiSettings.get('csv:separator, ,'), + quoteValues: uiSettings.get('csv:quoteValues', true), + formatFactory: data.fieldFormats.deserialize, + }), + type: exporters.CSV_MIME_TYPE, + }; + } + return memo; + }, + {} + ); + if (content) { + downloadMultipleAs(content); } - return memo; }, - {} - ); - if (content) { - downloadMultipleAs(content); - } - }, - saveAndReturn: () => { - if (savingToDashboardPermitted && lastKnownDoc) { - // disabling the validation on app leave because the document has been saved. - onAppLeave((actions) => { - return actions.default(); - }); - runSave( - { - newTitle: lastKnownDoc.title, - newCopyOnSave: false, - isTitleDuplicateConfirmed: false, - returnToOrigin: true, - }, - { - saveToLibrary: - (initialInput && attributeService.inputIsRefType(initialInput)) ?? false, + saveAndReturn: () => { + if (savingToDashboardPermitted && lastKnownDoc) { + // disabling the validation on app leave because the document has been saved. + onAppLeave((actions) => { + return actions.default(); + }); + runSave( + { + newTitle: lastKnownDoc.title, + newCopyOnSave: false, + isTitleDuplicateConfirmed: false, + returnToOrigin: true, + }, + { + saveToLibrary: + (initialInput && attributeService.inputIsRefType(initialInput)) ?? false, + } + ); } - ); - } - }, - showSaveModal: () => { - if (savingToDashboardPermitted || savingToLibraryPermitted) { - setIsSaveModalVisible(true); - } - }, - cancel: () => { - if (redirectToOrigin) { - redirectToOrigin(); + }, + showSaveModal: () => { + if (savingToDashboardPermitted || savingToLibraryPermitted) { + setIsSaveModalVisible(true); + } + }, + cancel: () => { + if (redirectToOrigin) { + redirectToOrigin(); + } + }, + }, + }), + [ + activeData, + attributeService, + dashboardFeatureFlag.allowByValueEmbeddables, + data.fieldFormats.deserialize, + getIsByValueMode, + initialInput, + isLinkedToOriginatingApp, + isSaveable, + lastKnownDoc, + onAppLeave, + redirectToOrigin, + runSave, + savingToDashboardPermitted, + savingToLibraryPermitted, + setIsSaveModalVisible, + uiSettings, + unsavedTitle, + ] + ); + + const onQuerySubmitWrapped = useCallback( + (payload) => { + const { dateRange, query: newQuery } = payload; + const currentRange = data.query.timefilter.timefilter.getTime(); + if (dateRange.from !== currentRange.from || dateRange.to !== currentRange.to) { + data.query.timefilter.timefilter.setTime(dateRange); + trackUiEvent('app_date_change'); + } else { + // Query has changed, renew the session id. + // Time change will be picked up by the time subscription + dispatchSetState({ searchSessionId: data.search.session.start() }); + trackUiEvent('app_query_change'); + } + if (newQuery) { + if (!isEqual(newQuery, query)) { + dispatchSetState({ query: newQuery }); } - }, + } }, - }); + [data.query.timefilter.timefilter, data.search.session, dispatchSetState, query] + ); + + const onSavedWrapped = useCallback( + (newSavedQuery) => { + dispatchSetState({ savedQuery: newSavedQuery }); + }, + [dispatchSetState] + ); + + const onSavedQueryUpdatedWrapped = useCallback( + (newSavedQuery) => { + const savedQueryFilters = newSavedQuery.attributes.filters || []; + const globalFilters = data.query.filterManager.getGlobalFilters(); + data.query.filterManager.setFilters([...globalFilters, ...savedQueryFilters]); + dispatchSetState({ + query: newSavedQuery.attributes.query, + savedQuery: { ...newSavedQuery }, + }); // Shallow query for reference issues + }, + [data.query.filterManager, dispatchSetState] + ); + + const onClearSavedQueryWrapped = useCallback(() => { + data.query.filterManager.setFilters(data.query.filterManager.getGlobalFilters()); + dispatchSetState({ + filters: data.query.filterManager.getGlobalFilters(), + query: data.query.queryString.getDefaultQuery(), + savedQuery: undefined, + }); + }, [data.query.filterManager, data.query.queryString, dispatchSetState]); return ( { - const { dateRange, query: newQuery } = payload; - const currentRange = data.query.timefilter.timefilter.getTime(); - if (dateRange.from !== currentRange.from || dateRange.to !== currentRange.to) { - data.query.timefilter.timefilter.setTime(dateRange); - trackUiEvent('app_date_change'); - } else { - // Query has changed, renew the session id. - // Time change will be picked up by the time subscription - dispatchSetState({ searchSessionId: data.search.session.start() }); - trackUiEvent('app_query_change'); - } - if (newQuery) { - if (!isEqual(newQuery, query)) { - dispatchSetState({ query: newQuery }); - } - } - }} - onSaved={(newSavedQuery) => { - dispatchSetState({ savedQuery: newSavedQuery }); - }} - onSavedQueryUpdated={(newSavedQuery) => { - const savedQueryFilters = newSavedQuery.attributes.filters || []; - const globalFilters = data.query.filterManager.getGlobalFilters(); - data.query.filterManager.setFilters([...globalFilters, ...savedQueryFilters]); - dispatchSetState({ - query: newSavedQuery.attributes.query, - savedQuery: { ...newSavedQuery }, - }); // Shallow query for reference issues - }} - onClearSavedQuery={() => { - data.query.filterManager.setFilters(data.query.filterManager.getGlobalFilters()); - dispatchSetState({ - filters: data.query.filterManager.getGlobalFilters(), - query: data.query.queryString.getDefaultQuery(), - savedQuery: undefined, - }); - }} + onQuerySubmit={onQuerySubmitWrapped} + onSaved={onSavedWrapped} + onSavedQueryUpdated={onSavedQueryUpdatedWrapped} + onClearSavedQuery={onClearSavedQueryWrapped} indexPatterns={indexPatternsForTopNav} query={query} dateRangeFrom={from} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 3ce5401be0b8a5..1f76143186f35e 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -202,9 +202,16 @@ export function LayerPanel( setNextFocusedButtonId, ]); + const isDimensionPanelOpen = Boolean(activeId); + return ( <> -
+
@@ -412,7 +419,7 @@ export function LayerPanel( (panelRef.current = el)} - isOpen={!!activeId} + isOpen={isDimensionPanelOpen} isFullscreen={isFullscreen} groupLabel={activeGroup?.groupLabel || ''} handleClose={() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_panel.tsx index 0c2eb4f39d8959..8107b6646500d5 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/suggestion_panel.tsx @@ -200,15 +200,16 @@ export function SuggestionPanel({ visualizationState: currentVisualizationState, activeData: frame.activeData, }) - .filter((suggestion) => !suggestion.hide) .filter( ({ + hide, visualizationId, visualizationState: suggestionVisualizationState, datasourceState: suggestionDatasourceState, datasourceId: suggetionDatasourceId, }) => { return ( + !hide && validateDatasourceAndVisualization( suggetionDatasourceId ? datasourceMap[suggetionDatasourceId] : null, suggestionDatasourceState, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ebd4f8ca2fbcce..3d20e324b275eb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -157,11 +157,11 @@ export function DimensionEditor(props: DimensionEditorProps) { const possibleOperations = useMemo(() => { return Object.values(operationDefinitionMap) .filter(({ hidden }) => !hidden) + .filter(({ type }) => fieldByOperation[type]?.size || operationWithoutField.has(type)) .sort((op1, op2) => { return op1.displayName.localeCompare(op2.displayName); }) - .map((def) => def.type) - .filter((type) => fieldByOperation[type]?.size || operationWithoutField.has(type)); + .map((def) => def.type); }, [fieldByOperation, operationWithoutField]); const [filterByOpenInitially, setFilterByOpenInitally] = useState(false); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 95049eaf6f2f30..a02f6bbea065c9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -170,44 +170,42 @@ export function FormulaEditor({ } } - const markers = errors - .flatMap((innerError) => { - if (innerError.locations.length) { - return innerError.locations.map((location) => { - const startPosition = offsetToRowColumn(text, location.min); - const endPosition = offsetToRowColumn(text, location.max); - return { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }; - }); - } else { - // Parse errors return no location info - const startPosition = offsetToRowColumn(text, 0); - const endPosition = offsetToRowColumn(text, text.length - 1); - return [ - { - message: innerError.message, - startColumn: startPosition.column + 1, - startLineNumber: startPosition.lineNumber, - endColumn: endPosition.column + 1, - endLineNumber: endPosition.lineNumber, - severity: - innerError.severity === 'warning' - ? monaco.MarkerSeverity.Warning - : monaco.MarkerSeverity.Error, - }, - ]; - } - }) - .filter((marker) => marker); + const markers = errors.flatMap((innerError) => { + if (innerError.locations.length) { + return innerError.locations.map((location) => { + const startPosition = offsetToRowColumn(text, location.min); + const endPosition = offsetToRowColumn(text, location.max); + return { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }; + }); + } else { + // Parse errors return no location info + const startPosition = offsetToRowColumn(text, 0); + const endPosition = offsetToRowColumn(text, text.length - 1); + return [ + { + message: innerError.message, + startColumn: startPosition.column + 1, + startLineNumber: startPosition.lineNumber, + endColumn: endPosition.column + 1, + endLineNumber: endPosition.lineNumber, + severity: + innerError.severity === 'warning' + ? monaco.MarkerSeverity.Warning + : monaco.MarkerSeverity.Error, + }, + ]; + } + }); monaco.editor.setModelMarkers(editorModel.current, 'LENS', markers); setWarnings(markers.map(({ severity, message }) => ({ severity, message }))); From 0578cc55fe4517b300514903f97a0e38da4288e3 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Jun 2021 16:40:24 +0200 Subject: [PATCH 168/185] :bug: Fix unwanted change --- x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx index 2bd539e6715c74..ecaae04232f8af 100644 --- a/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx +++ b/x-pack/plugins/lens/public/app_plugin/lens_top_nav.tsx @@ -192,7 +192,7 @@ export const LensTopNavMenu = ({ memo[`${lastKnownDoc?.title || unsavedTitle}${postFix}.csv`] = { content: exporters.datatableToCSV(datatable, { - csvSeparator: uiSettings.get('csv:separator, ,'), + csvSeparator: uiSettings.get('csv:separator', ','), quoteValues: uiSettings.get('csv:quoteValues', true), formatFactory: data.fieldFormats.deserialize, }), From 0b7685386e360060d09032bd816c430179167460 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Jun 2021 18:14:28 +0200 Subject: [PATCH 169/185] :zap: Memoize as many combobox props as possible --- .../dimension_panel/format_selector.tsx | 96 +++++++++++-------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx index 3a57579583c90c..ff10810208e706 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFormRow, EuiComboBox, EuiSpacer, EuiRange } from '@elastic/eui'; import { IndexPatternColumn } from '../indexpattern'; @@ -28,6 +28,13 @@ const supportedFormats: Record = { }, }; +const defaultOption = { + value: '', + label: i18n.translate('xpack.lens.indexPattern.defaultFormatLabel', { + defaultMessage: 'Default', + }), +}; + interface FormatSelectorProps { selectedColumn: IndexPatternColumn; onChange: (newFormat?: { id: string; params?: Record }) => void; @@ -37,6 +44,8 @@ interface State { decimalPlaces: number; } +const singleSelectionOption = { asPlainText: true }; + export function FormatSelector(props: FormatSelectorProps) { const { selectedColumn, onChange } = props; @@ -51,13 +60,6 @@ export function FormatSelector(props: FormatSelectorProps) { const selectedFormat = currentFormat?.id ? supportedFormats[currentFormat.id] : undefined; - const defaultOption = { - value: '', - label: i18n.translate('xpack.lens.indexPattern.defaultFormatLabel', { - defaultMessage: 'Default', - }), - }; - const label = i18n.translate('xpack.lens.indexPattern.columnFormatLabel', { defaultMessage: 'Value format', }); @@ -66,6 +68,48 @@ export function FormatSelector(props: FormatSelectorProps) { defaultMessage: 'Decimals', }); + const stableOptions = useMemo( + () => [ + defaultOption, + ...Object.entries(supportedFormats).map(([id, format]) => ({ + value: id, + label: format.title ?? id, + })), + ], + [] + ); + + const onChangeWrapped = useCallback( + (choices) => { + if (choices.length === 0) { + return; + } + + if (!choices[0].value) { + onChange(); + return; + } + onChange({ + id: choices[0].value, + params: { decimals: state.decimalPlaces }, + }); + }, + [onChange, state.decimalPlaces] + ); + + const currentOption = useMemo( + () => + currentFormat + ? [ + { + value: currentFormat.id, + label: selectedFormat?.title ?? currentFormat.id, + }, + ] + : [defaultOption], + [currentFormat, selectedFormat?.title] + ); + return ( <> @@ -76,38 +120,10 @@ export function FormatSelector(props: FormatSelectorProps) { isClearable={false} data-test-subj="indexPattern-dimension-format" aria-label={label} - singleSelection={{ asPlainText: true }} - options={[ - defaultOption, - ...Object.entries(supportedFormats).map(([id, format]) => ({ - value: id, - label: format.title ?? id, - })), - ]} - selectedOptions={ - currentFormat - ? [ - { - value: currentFormat.id, - label: selectedFormat?.title ?? currentFormat.id, - }, - ] - : [defaultOption] - } - onChange={(choices) => { - if (choices.length === 0) { - return; - } - - if (!choices[0].value) { - onChange(); - return; - } - onChange({ - id: choices[0].value, - params: { decimals: state.decimalPlaces }, - }); - }} + singleSelection={singleSelectionOption} + options={stableOptions} + selectedOptions={currentOption} + onChange={onChangeWrapped} /> {currentFormat ? ( <> From affc0b3bb79b61fb5ebab3d086dd85c9212398b9 Mon Sep 17 00:00:00 2001 From: dej611 Date: Mon, 7 Jun 2021 18:24:19 +0200 Subject: [PATCH 170/185] :zap: More memoization --- .../dimension_panel/dimension_editor.tsx | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 3d20e324b275eb..dcab204f432232 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -6,7 +6,7 @@ */ import './dimension_editor.scss'; -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useCallback } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiListGroup, @@ -625,6 +625,24 @@ export function DimensionEditor(props: DimensionEditorProps) { <> ); + const onFormatChange = useCallback( + (newFormat) => { + setState( + mergeLayer({ + state, + layerId, + newLayer: updateColumnParam({ + layer: state.layers[layerId], + columnId, + paramName: 'format', + value: newFormat, + }), + }) + ); + }, + [columnId, layerId, setState, state] + ); + return (
{!isFullscreen ? ( @@ -722,23 +740,7 @@ export function DimensionEditor(props: DimensionEditorProps) { {!isFullscreen && selectedColumn && (selectedColumn.dataType === 'number' || selectedColumn.operationType === 'range') ? ( - { - setState( - mergeLayer({ - state, - layerId, - newLayer: updateColumnParam({ - layer: state.layers[layerId], - columnId, - paramName: 'format', - value: newFormat, - }), - }) - ); - }} - /> + ) : null}
)} From 0762f2f59ae17fb27d94f2fecd0c5cc43625569e Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 7 Jun 2021 13:30:06 -0400 Subject: [PATCH 171/185] Show errors in hover --- .../formula/editor/math_completion.test.ts | 22 +++++-------------- .../formula/editor/math_completion.ts | 7 ++++-- .../definitions/formula/formula.test.tsx | 13 +++++++++++ .../definitions/formula/validation.ts | 6 ++++- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts index 6932ae11b9bda3..9cd748f5759c98 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.test.ts @@ -174,43 +174,31 @@ describe('math completion', () => { it('should show signature for a field-based ES function', () => { expect(getHover('sum()', 2, operationDefinitionMap)).toEqual({ - contents: [{ value: 'sum(field: string)' }, { value: expect.stringContaining('Example') }], + contents: [{ value: 'sum(field: string)' }], }); }); it('should show signature for count', () => { expect(getHover('count()', 2, operationDefinitionMap)).toEqual({ - contents: [ - { value: expect.stringContaining('count(') }, - { value: expect.stringContaining('Example') }, - ], + contents: [{ value: expect.stringContaining('count(') }], }); }); it('should show signature for a function with named parameters', () => { expect(getHover('2 * moving_average(count())', 10, operationDefinitionMap)).toEqual({ - contents: [ - { value: expect.stringContaining('moving_average(') }, - { value: expect.stringContaining('Example') }, - ], + contents: [{ value: expect.stringContaining('moving_average(') }], }); }); it('should show signature for an inner function', () => { expect(getHover('2 * moving_average(count())', 22, operationDefinitionMap)).toEqual({ - contents: [ - { value: expect.stringContaining('count(') }, - { value: expect.stringContaining('Example') }, - ], + contents: [{ value: expect.stringContaining('count(') }], }); }); it('should show signature for a complex tinymath function', () => { expect(getHover('clamp(count(), 5)', 2, operationDefinitionMap)).toEqual({ - contents: [ - { value: expect.stringContaining('clamp([value]: number') }, - { value: expect.stringContaining('Example') }, - ], + contents: [{ value: expect.stringContaining('clamp([value]: number') }], }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts index 435d3e7eb8c6a4..df747e532b38a0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/math_completion.ts @@ -570,8 +570,11 @@ export function getHover( const name = tokenInfo.ast.name; const signatures = getSignaturesForFunction(name, operationDefinitionMap); if (signatures.length) { - const { label, documentation } = signatures[0]; - return { contents: [{ value: label }, documentation] }; + const { label } = signatures[0]; + + return { + contents: [{ value: label }], + }; } } catch (e) { // do nothing diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index d87bcde65ce6cd..e1c722fd9cb38e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -1060,6 +1060,19 @@ invalid: " } }); + it("returns a clear error when there's a missing function for a fullReference operation", () => { + for (const fn of ['cumulative_sum', 'derivative']) { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula(`${fn}()`), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual([`The first argument for ${fn} should be a operation name. Found no operation`]); + } + }); + it('returns no error if a math operation is passed to fullReference operations', () => { const formulas = [ 'derivative(7+1)', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 4e1f1bef1af467..992b8ee2422e90 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -596,7 +596,11 @@ function runFullASTValidation( values: { operation: node.name, type: 'operation', - argument: getValueOrName(firstArg), + argument: + getValueOrName(firstArg) || + i18n.translate('xpack.lens.indexPattern.formulaNoOperation', { + defaultMessage: 'no operation', + }), }, locations: node.location ? [node.location] : [], }) From 304fb647c4b3bc2306f46864100c28e9e9c93ce8 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 7 Jun 2021 14:37:26 -0400 Subject: [PATCH 172/185] Use temporary invalid state when moving away from formula --- .../dimension_panel/dimension_editor.tsx | 102 +++++++++--------- .../formula/editor/formula_editor.tsx | 1 + .../operations/layer_helpers.ts | 12 ++- x-pack/test/functional/apps/lens/formula.ts | 24 +++++ .../test/functional/page_objects/lens_page.ts | 8 +- 5 files changed, 89 insertions(+), 58 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ebd4f8ca2fbcce..7965a62eedbfd1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -109,19 +109,10 @@ export function DimensionEditor(props: DimensionEditorProps) { const selectedOperationDefinition = selectedColumn && operationDefinitionMap[selectedColumn.operationType]; - const [changedFormula, setChangedFormula] = useState( - Boolean(selectedOperationDefinition?.type === 'formula') - ); - const setStateWrapper = ( setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), shouldClose?: boolean ) => { - if (selectedOperationDefinition?.type === 'formula' && !temporaryQuickFunction) { - setChangedFormula(true); - } else { - setChangedFormula(false); - } const hypotheticalLayer = typeof setter === 'function' ? setter(state.layers[layerId]) : setter; const hasIncompleteColumns = Boolean(hypotheticalLayer.incompleteColumns?.[columnId]); const prevOperationType = @@ -247,10 +238,6 @@ export function DimensionEditor(props: DimensionEditorProps) { }`, [`aria-pressed`]: isActive, onClick() { - if (temporaryQuickFunction) { - setQuickFunction(false); - } - if ( operationDefinitionMap[operationType].input === 'none' || operationDefinitionMap[operationType].input === 'managedReference' || @@ -271,37 +258,44 @@ export function DimensionEditor(props: DimensionEditorProps) { visualizationGroups: dimensionGroups, targetGroup: props.groupId, }); + if (temporaryQuickFunction && newLayer.columns[columnId].operationType !== 'formula') { + // Only switch the tab once the formula is fully removed + setQuickFunction(false); + } setStateWrapper(newLayer); trackUiEvent(`indexpattern_dimension_operation_${operationType}`); return; } else if (!selectedColumn || !compatibleWithCurrentField) { const possibleFields = fieldByOperation[operationType] || new Set(); + let newLayer: IndexPatternLayer; if (possibleFields.size === 1) { - setStateWrapper( - insertOrReplaceColumn({ - layer: props.state.layers[props.layerId], - indexPattern: currentIndexPattern, - columnId, - op: operationType, - field: currentIndexPattern.getFieldByName(possibleFields.values().next().value), - visualizationGroups: dimensionGroups, - targetGroup: props.groupId, - }) - ); + newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: operationType, + field: currentIndexPattern.getFieldByName(possibleFields.values().next().value), + visualizationGroups: dimensionGroups, + targetGroup: props.groupId, + }); } else { - setStateWrapper( - insertOrReplaceColumn({ - layer: props.state.layers[props.layerId], - indexPattern: currentIndexPattern, - columnId, - op: operationType, - field: undefined, - visualizationGroups: dimensionGroups, - targetGroup: props.groupId, - }) - ); + newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: operationType, + field: undefined, + visualizationGroups: dimensionGroups, + targetGroup: props.groupId, + }); + // ); } + if (temporaryQuickFunction && newLayer.columns[columnId].operationType !== 'formula') { + // Only switch the tab once the formula is fully removed + setQuickFunction(false); + } + setStateWrapper(newLayer); trackUiEvent(`indexpattern_dimension_operation_${operationType}`); return; } @@ -313,6 +307,9 @@ export function DimensionEditor(props: DimensionEditorProps) { return; } + if (temporaryQuickFunction) { + setQuickFunction(false); + } const newLayer = replaceColumn({ layer: props.state.layers[props.layerId], indexPattern: currentIndexPattern, @@ -350,7 +347,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const quickFunctions = ( <>
- {temporaryQuickFunction && changedFormula && ( + {temporaryQuickFunction && selectedColumn?.operationType === 'formula' && ( <> - - - ) : ( - <> - ); + + ) : null; return (
@@ -631,6 +624,7 @@ export function DimensionEditor(props: DimensionEditorProps) { { if (selectedColumn?.operationType === 'formula') { setQuickFunction(true); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 95049eaf6f2f30..7d1a73f5fb9091 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -115,6 +115,7 @@ export function FormulaEditor({ operationDefinitionMap ).newLayer; }, true); + setIsCloseable(true); } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index fb060bdcfcc88e..b650a2818b2d48 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -362,9 +362,9 @@ export function replaceColumn({ tempLayer = resetIncomplete(tempLayer, columnId); if (previousDefinition.input === 'managedReference') { - // Every transition away from a managedReference resets it, we don't have a way to keep the state + // If the transition is incomplete, leave the managed state until it's finished. tempLayer = deleteColumn({ layer: tempLayer, columnId, indexPattern }); - return insertNewColumn({ + const hypotheticalLayer = insertNewColumn({ layer: tempLayer, columnId, indexPattern, @@ -372,6 +372,14 @@ export function replaceColumn({ field, visualizationGroups, }); + if (hypotheticalLayer.incompleteColumns && hypotheticalLayer.incompleteColumns[columnId]) { + return { + ...layer, + incompleteColumns: hypotheticalLayer.incompleteColumns, + }; + } else { + return hypotheticalLayer; + } } if (operationDefinition.input === 'fullReference') { diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index 2aaa08c0941f71..c6a21b1682351e 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const find = getService('find'); const listingTable = getService('listingTable'); const browser = getService('browser'); + const testSubjects = getService('testSubjects'); describe('lens formula', () => { it('should transition from count to formula', async () => { @@ -170,5 +171,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDatatableCellText(1, 1)).to.eql('222420'); expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('222420'); }); + + it('should keep the formula if the user does not fully transition to a quick function', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'formula', + formula: `count()`, + keepOpen: true, + }); + + await PageObjects.lens.switchToQuickFunctions(); + await testSubjects.click(`lns-indexPatternDimension-min incompatible`); + await PageObjects.common.sleep(1000); + await PageObjects.lens.closeDimensionEditor(); + + expect(await PageObjects.lens.getDimensionTriggerText('lnsDatatable_metrics', 0)).to.eql( + 'count()' + ); + }); }); } diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 38d80cbe84d79e..6abf282ee68e5d 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -1002,6 +1002,10 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await PageObjects.header.waitUntilLoadingHasFinished(); }, + async switchToQuickFunctions() { + await testSubjects.click('lens-dimensionTabs-quickFunctions'); + }, + async switchToFormula() { await testSubjects.click('lens-dimensionTabs-formula'); }, @@ -1011,13 +1015,13 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }, async typeFormula(formula: string) { + // Formula takes time to open + await PageObjects.common.sleep(500); await find.byCssSelector('.monaco-editor'); await find.clickByCssSelectorWhenNotDisabled('.monaco-editor'); const input = await find.activeElement(); await input.clearValueWithKeyboard(); await input.type(formula); - // Formula is applied on a 250ms timer, won't be applied if we leave too early - await PageObjects.common.sleep(500); }, }); } From 57e67f5bf201bd22a99b6e97713bdf3b81d8ce51 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 7 Jun 2021 17:19:51 -0400 Subject: [PATCH 173/185] Remove setActiveDimension and shouldClose, fixed by async setters --- .../editor_frame/config_panel/layer_panel.tsx | 8 ---- .../dimension_panel/dimension_editor.tsx | 7 +-- .../dimension_panel/dimension_panel.test.tsx | 46 +++++++++---------- .../dimension_panel/reference_editor.tsx | 3 +- .../formula/editor/formula_editor.tsx | 4 +- .../operations/definitions/index.ts | 3 +- x-pack/plugins/lens/public/types.ts | 1 - x-pack/test/functional/apps/lens/formula.ts | 8 ++-- 8 files changed, 33 insertions(+), 47 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 3ce5401be0b8a5..a781d25e022eda 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -457,11 +457,9 @@ export function LayerPanel( { shouldReplaceDimension, shouldRemoveDimension, - shouldClose, }: { shouldReplaceDimension?: boolean; shouldRemoveDimension?: boolean; - shouldClose?: boolean; } = {} ) => { if (shouldReplaceDimension || shouldRemoveDimension) { @@ -484,12 +482,6 @@ export function LayerPanel( } else { props.updateDatasourceAsync(datasourceId, newState); } - if (!shouldClose) { - setActiveDimension({ - ...activeDimension, - isNew: false, - }); - } }, }} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 7965a62eedbfd1..3f46ba7ad3fcc6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -110,8 +110,7 @@ export function DimensionEditor(props: DimensionEditorProps) { selectedColumn && operationDefinitionMap[selectedColumn.operationType]; const setStateWrapper = ( - setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), - shouldClose?: boolean + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer) ) => { const hypotheticalLayer = typeof setter === 'function' ? setter(state.layers[layerId]) : setter; const hasIncompleteColumns = Boolean(hypotheticalLayer.incompleteColumns?.[columnId]); @@ -128,7 +127,6 @@ export function DimensionEditor(props: DimensionEditorProps) { shouldRemoveDimension: Boolean( hasIncompleteColumns && prevOperationType === 'fullReference' ), - shouldClose: Boolean(shouldClose), } ); }; @@ -400,8 +398,7 @@ export function DimensionEditor(props: DimensionEditorProps) { updateLayer={( setter: | IndexPatternLayer - | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), - shouldClose?: boolean + | ((prevLayer: IndexPatternLayer) => IndexPatternLayer) ) => { setState( mergeLayer({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index c1ba1d9d12140a..348c8a8e223447 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -504,7 +504,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...initialState, @@ -539,7 +539,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, @@ -573,7 +573,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -647,7 +647,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -685,7 +685,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -754,7 +754,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -881,7 +881,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: false }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -950,7 +950,7 @@ describe('IndexPatternDimensionEditorPanel', () => { // Now check that the dimension gets cleaned up on state update expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: false }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -1046,7 +1046,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls.length).toEqual(2); expect(setState.mock.calls[1]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[1][0](state)).toEqual({ ...state, @@ -1147,7 +1147,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1179,7 +1179,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1209,7 +1209,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1243,7 +1243,7 @@ describe('IndexPatternDimensionEditorPanel', () => { } as unknown) as ChangeEvent); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1273,7 +1273,7 @@ describe('IndexPatternDimensionEditorPanel', () => { } as unknown) as ChangeEvent); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1304,7 +1304,7 @@ describe('IndexPatternDimensionEditorPanel', () => { ); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1597,7 +1597,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1631,7 +1631,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1660,7 +1660,7 @@ describe('IndexPatternDimensionEditorPanel', () => { }); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1692,7 +1692,7 @@ describe('IndexPatternDimensionEditorPanel', () => { ); expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1747,7 +1747,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: false }, ]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, @@ -1814,7 +1814,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](initialState)).toEqual({ ...initialState, @@ -1842,7 +1842,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -1979,7 +1979,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true, shouldClose: false }, + { shouldRemoveDimension: false, shouldReplaceDimension: true }, ]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx index 388b02182fbd67..b0cdf96f928f93 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/reference_editor.tsx @@ -44,8 +44,7 @@ export interface ReferenceEditorProps { validation: RequiredReference; columnId: string; updateLayer: ( - setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), - shouldClose?: boolean + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer) ) => void; currentIndexPattern: IndexPattern; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index 6991929daa2cd2..d194507a9f7109 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -103,6 +103,7 @@ export function FormulaEditor({ }, []); useUnmount(() => { + setIsCloseable(true); // If the text is not synced, update the column. if (text !== currentColumn.params.formula) { updateLayer((prevLayer) => { @@ -114,8 +115,7 @@ export function FormulaEditor({ indexPattern, operationDefinitionMap ).newLayer; - }, true); - setIsCloseable(true); + }); } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index ba8ae118df423d..a7bf415817797d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -151,8 +151,7 @@ export interface ParamEditorProps { currentColumn: C; layer: IndexPatternLayer; updateLayer: ( - setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer), - shouldClose?: boolean + setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer) ) => void; toggleFullscreen: () => void; setIsCloseable: (isCloseable: boolean) => void; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index ed7fb8984be99d..1c20e306b2e8a9 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -308,7 +308,6 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro publishToVisualization?: { shouldReplaceDimension?: boolean; shouldRemoveDimension?: boolean; - shouldClose?: boolean; } ) => void; core: Pick; diff --git a/x-pack/test/functional/apps/lens/formula.ts b/x-pack/test/functional/apps/lens/formula.ts index c6a21b1682351e..e9e5051c006f02 100644 --- a/x-pack/test/functional/apps/lens/formula.ts +++ b/x-pack/test/functional/apps/lens/formula.ts @@ -73,19 +73,19 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { let input = await find.activeElement(); await input.type(' '); await input.pressKeys(browser.keys.ARROW_LEFT); - await input.type(`Men's`); + await input.type(`Men's Clothing`); await PageObjects.common.sleep(100); let element = await find.byCssSelector('.monaco-editor'); - expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s ')`); + expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s Clothing ')`); await PageObjects.lens.typeFormula('count(kql='); input = await find.activeElement(); - await input.type(`Men\'s `); + await input.type(`Men\'s Clothing`); element = await find.byCssSelector('.monaco-editor'); - expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s ')`); + expect(await element.getVisibleText()).to.equal(`count(kql='Men\\'s Clothing')`); }); it('should persist a broken formula on close', async () => { From 199eece3c4686f4eb9a25d927b8fcea464b21dbf Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Mon, 7 Jun 2021 17:39:23 -0400 Subject: [PATCH 174/185] Fix test dependency --- x-pack/plugins/lens/public/mocks.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/lens/public/mocks.tsx b/x-pack/plugins/lens/public/mocks.tsx index 473c170aef2948..07935bb2f241b8 100644 --- a/x-pack/plugins/lens/public/mocks.tsx +++ b/x-pack/plugins/lens/public/mocks.tsx @@ -166,6 +166,9 @@ export function mockDataPlugin(sessionIdSubject = new Subject()) { nowProvider: { get: jest.fn(), }, + fieldFormats: { + deserialize: jest.fn(), + }, } as unknown) as DataPublicPluginStart; } From aca7aba9b8b3dba4752629bad73e1daebc5dc89a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 8 Jun 2021 11:55:59 +0200 Subject: [PATCH 175/185] do not show quick functions tab --- .../dimension_panel/dimension_editor.tsx | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 9ef914b9dd4347..eec5c5345f8d80 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -635,7 +635,7 @@ export function DimensionEditor(props: DimensionEditorProps) { return (
- {!isFullscreen ? ( + {!isFullscreen && operationSupportMatrix.operationWithoutField.has('formula') ? ( - {operationSupportMatrix.operationWithoutField.has('formula') ? ( - { - if (selectedColumn?.operationType !== 'formula') { - setQuickFunction(false); - const newLayer = insertOrReplaceColumn({ - layer: props.state.layers[props.layerId], - indexPattern: currentIndexPattern, - columnId, - op: 'formula', - visualizationGroups: dimensionGroups, - }); - setStateWrapper(newLayer); - trackUiEvent(`indexpattern_dimension_operation_formula`); - return; - } else { - setQuickFunction(false); - } - }} - > - {i18n.translate('xpack.lens.indexPattern.formulaLabel', { - defaultMessage: 'Formula', - })} - - ) : null} + { + if (selectedColumn?.operationType !== 'formula') { + setQuickFunction(false); + const newLayer = insertOrReplaceColumn({ + layer: props.state.layers[props.layerId], + indexPattern: currentIndexPattern, + columnId, + op: 'formula', + visualizationGroups: dimensionGroups, + }); + setStateWrapper(newLayer); + trackUiEvent(`indexpattern_dimension_operation_formula`); + return; + } else { + setQuickFunction(false); + } + }} + > + {i18n.translate('xpack.lens.indexPattern.formulaLabel', { + defaultMessage: 'Formula', + })} + ) : null} @@ -738,6 +736,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
); } + function getErrorMessage( selectedColumn: IndexPatternColumn | undefined, incompleteOperation: boolean, From 16b0d40e19a9685bb7e8fad886bea306d1a71c8c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 8 Jun 2021 12:03:13 +0200 Subject: [PATCH 176/185] increase documentation popover width --- .../operations/definitions/formula/editor/formula.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss index db19c9a78aee4a..14b3fc33efb4e6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula.scss @@ -101,7 +101,7 @@ .lnsFormula__docsContent { .lnsFormula__docs--overlay & { height: 40vh; - width: #{'min(65vh, 90vw)'}; + width: #{'min(75vh, 90vw)'}; } .lnsFormula__docs--inline & { From 9a7b4bc87f2dc059b5681ed7f707cda3fdbecdcb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 8 Jun 2021 16:32:57 +0200 Subject: [PATCH 177/185] fix functional test --- x-pack/test/functional/apps/lens/smokescreen.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/lens/smokescreen.ts b/x-pack/test/functional/apps/lens/smokescreen.ts index 2a4d56bbea791a..4c8d642e7feb93 100644 --- a/x-pack/test/functional/apps/lens/smokescreen.ts +++ b/x-pack/test/functional/apps/lens/smokescreen.ts @@ -172,10 +172,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.existOrFail('indexPattern-dimension-formatDecimals'); + await PageObjects.lens.closeDimensionEditor(); + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( 'Test of label' ); - await PageObjects.lens.closeDimensionEditor(); }); it('should be able to add very long labels and still be able to remove a dimension', async () => { From 7f5dbead745b9496eaa6e4f2187e25ce7c4226b9 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 8 Jun 2021 11:39:45 -0400 Subject: [PATCH 178/185] Call setActiveDimension when updating visualization --- .../editor_frame/config_panel/layer_panel.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 780f28d7ffd49d..136c6a00c5b502 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -486,6 +486,11 @@ export function LayerPanel( prevState: props.visualizationState, }) ); + if (shouldRemoveDimension) { + setActiveDimension(initialActiveDimensionState); + } else { + setActiveDimension({ ...activeDimension, isNew: false }); + } } else { props.updateDatasourceAsync(datasourceId, newState); } From 4b81e5d53a2cce467255e7b28e5c6b18c71d661c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 8 Jun 2021 15:02:59 -0400 Subject: [PATCH 179/185] Simplify handling of flyout with incomplete columns --- .../config_panel/layer_panel.test.tsx | 107 ++++++++++++++++- .../editor_frame/config_panel/layer_panel.tsx | 60 +++++----- .../public/editor_frame_service/mocks.tsx | 1 + .../dimension_panel/dimension_editor.tsx | 14 +-- .../dimension_panel/dimension_panel.test.tsx | 112 ++++-------------- x-pack/plugins/lens/public/types.ts | 3 +- .../test/functional/apps/lens/smokescreen.ts | 51 ++++++++ .../test/functional/page_objects/lens_page.ts | 20 ++-- 8 files changed, 228 insertions(+), 140 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index 8801be4d8ff017..dd1241af14f5af 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -258,7 +258,7 @@ describe('LayerPanel', () => { it('should not update the visualization if the datasource is incomplete', () => { (generateId as jest.Mock).mockReturnValue(`newid`); const updateAll = jest.fn(); - const updateDatasource = jest.fn(); + const updateDatasourceAsync = jest.fn(); mockVisualization.getConfiguration.mockReturnValue({ groups: [ @@ -276,7 +276,7 @@ describe('LayerPanel', () => { const component = mountWithIntl( ); @@ -295,15 +295,88 @@ describe('LayerPanel', () => { mockDatasource.renderDimensionEditor.mock.calls.length - 1 ][1].setState; + act(() => { + stateFn( + { + indexPatternId: '1', + columns: {}, + columnOrder: [], + incompleteColumns: { newId: { operationType: 'count' } }, + }, + { isDimensionComplete: false } + ); + }); + expect(updateAll).not.toHaveBeenCalled(); + expect(updateDatasourceAsync).toHaveBeenCalled(); + act(() => { stateFn({ indexPatternId: '1', columns: {}, columnOrder: [], - incompleteColumns: { newId: { operationType: 'count' } }, }); }); - expect(updateAll).not.toHaveBeenCalled(); + expect(updateAll).toHaveBeenCalled(); + }); + + it('should remove the dimension when the datasource marks it as removed', () => { + const updateAll = jest.fn(); + const updateDatasource = jest.fn(); + + mockVisualization.getConfiguration.mockReturnValue({ + groups: [ + { + groupLabel: 'A', + groupId: 'a', + accessors: [{ columnId: 'y' }], + filterOperations: () => true, + supportsMoreColumns: true, + dataTestSubj: 'lnsGroup', + }, + ], + }); + + const component = mountWithIntl( + + ); + + act(() => { + component.find('[data-test-subj="lnsLayerPanel-dimensionLink"]').first().simulate('click'); + }); + component.update(); + + expect(mockDatasource.renderDimensionEditor).toHaveBeenCalledWith( + expect.any(Element), + expect.objectContaining({ columnId: 'y' }) + ); + const stateFn = + mockDatasource.renderDimensionEditor.mock.calls[ + mockDatasource.renderDimensionEditor.mock.calls.length - 1 + ][1].setState; act(() => { stateFn( @@ -311,11 +384,19 @@ describe('LayerPanel', () => { indexPatternId: '1', columns: {}, columnOrder: [], + incompleteColumns: { y: { operationType: 'average' } }, }, - { shouldReplaceDimension: true } + { + isDimensionComplete: false, + } ); }); expect(updateAll).toHaveBeenCalled(); + expect(mockVisualization.removeDimension).toHaveBeenCalledWith( + expect.objectContaining({ + columnId: 'y', + }) + ); }); it('should keep the DimensionContainer open when configuring a new dimension', () => { @@ -334,6 +415,7 @@ describe('LayerPanel', () => { accessors: [], filterOperations: () => true, supportsMoreColumns: true, + enableDimensionEditor: true, dataTestSubj: 'lnsGroup', }, ], @@ -348,6 +430,7 @@ describe('LayerPanel', () => { accessors: [{ columnId: 'newid' }], filterOperations: () => true, supportsMoreColumns: false, + enableDimensionEditor: true, dataTestSubj: 'lnsGroup', }, ], @@ -360,6 +443,20 @@ describe('LayerPanel', () => { component.update(); expect(component.find('EuiFlyoutHeader').exists()).toBe(true); + + const lastArgs = + mockDatasource.renderDimensionEditor.mock.calls[ + mockDatasource.renderDimensionEditor.mock.calls.length - 1 + ][1]; + + // Simulate what is called by the dimension editor + act(() => { + lastArgs.setState(lastArgs.state, { + isDimensionComplete: true, + }); + }); + + expect(mockVisualization.renderDimensionEditor).toHaveBeenCalled(); }); it('should close the DimensionContainer when the active visualization changes', () => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 136c6a00c5b502..3a299de0fca6a3 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -423,10 +423,11 @@ export function LayerPanel( isFullscreen={isFullscreen} groupLabel={activeGroup?.groupLabel || ''} handleClose={() => { - if (layerDatasource.canCloseDimensionEditor) { - if (!layerDatasource.canCloseDimensionEditor(layerDatasourceState)) { - return false; - } + if ( + layerDatasource.canCloseDimensionEditor && + !layerDatasource.canCloseDimensionEditor(layerDatasourceState) + ) { + return false; } if (layerDatasource.updateStateOnCloseDimension) { const newState = layerDatasource.updateStateOnCloseDimension({ @@ -461,36 +462,37 @@ export function LayerPanel( isFullscreen, setState: ( newState: unknown, - { - shouldReplaceDimension, - shouldRemoveDimension, - }: { - shouldReplaceDimension?: boolean; - shouldRemoveDimension?: boolean; - } = {} + { isDimensionComplete = true }: { isDimensionComplete?: boolean } = {} ) => { - if (shouldReplaceDimension || shouldRemoveDimension) { + if (allAccessors.includes(activeId)) { + if (isDimensionComplete) { + props.updateDatasourceAsync(datasourceId, newState); + } else { + // The datasource can indicate that the previously-valid column is no longer + // complete, which clears the visualization. This keeps the flyout open and reuses + // the previous columnId + props.updateAll( + datasourceId, + newState, + activeVisualization.removeDimension({ + layerId, + columnId: activeId, + prevState: props.visualizationState, + }) + ); + } + } else if (isDimensionComplete) { props.updateAll( datasourceId, newState, - shouldRemoveDimension - ? activeVisualization.removeDimension({ - layerId, - columnId: activeId, - prevState: props.visualizationState, - }) - : activeVisualization.setDimension({ - layerId, - groupId: activeGroup.groupId, - columnId: activeId, - prevState: props.visualizationState, - }) + activeVisualization.setDimension({ + layerId, + groupId: activeGroup.groupId, + columnId: activeId, + prevState: props.visualizationState, + }) ); - if (shouldRemoveDimension) { - setActiveDimension(initialActiveDimensionState); - } else { - setActiveDimension({ ...activeDimension, isNew: false }); - } + setActiveDimension({ ...activeDimension, isNew: false }); } else { props.updateDatasourceAsync(datasourceId, newState); } diff --git a/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx b/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx index 38669d72474df1..1762e7ff20fabb 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/mocks.tsx @@ -57,6 +57,7 @@ export function createMockVisualization(): jest.Mocked { setDimension: jest.fn(), removeDimension: jest.fn(), getErrorMessages: jest.fn((_state) => undefined), + renderDimensionEditor: jest.fn(), }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index eec5c5345f8d80..1dda857c2c3292 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -112,21 +112,21 @@ export function DimensionEditor(props: DimensionEditorProps) { const setStateWrapper = ( setter: IndexPatternLayer | ((prevLayer: IndexPatternLayer) => IndexPatternLayer) ) => { + const prevOperationType = + operationDefinitionMap[state.layers[layerId].columns[columnId]?.operationType]?.input; + const hypotheticalLayer = typeof setter === 'function' ? setter(state.layers[layerId]) : setter; const hasIncompleteColumns = Boolean(hypotheticalLayer.incompleteColumns?.[columnId]); - const prevOperationType = - operationDefinitionMap[hypotheticalLayer.columns[columnId]?.operationType]?.input; setState( (prevState) => { const layer = typeof setter === 'function' ? setter(prevState.layers[layerId]) : setter; return mergeLayer({ state: prevState, layerId, newLayer: layer }); }, { - shouldReplaceDimension: Boolean(hypotheticalLayer.columns[columnId]), - // clear the dimension if there's an incomplete column pending && previous operation was a fullReference operation - shouldRemoveDimension: Boolean( - hasIncompleteColumns && prevOperationType === 'fullReference' - ), + isDimensionComplete: + prevOperationType === 'fullReference' + ? hasIncompleteColumns + : Boolean(hypotheticalLayer.columns[columnId]), } ); }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 348c8a8e223447..72a084ce290d03 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -502,10 +502,7 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...initialState, layers: { @@ -537,10 +534,7 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, layers: { @@ -571,10 +565,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -645,10 +636,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -683,10 +671,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-min"]').simulate('click'); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -752,10 +737,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -881,7 +863,7 @@ describe('IndexPatternDimensionEditorPanel', () => { expect(setState.mock.calls[0]).toEqual([ expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false }, + { isDimensionComplete: false }, ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, @@ -948,10 +930,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); // Now check that the dimension gets cleaned up on state update - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -1044,10 +1023,7 @@ describe('IndexPatternDimensionEditorPanel', () => { }); expect(setState.mock.calls.length).toEqual(2); - expect(setState.mock.calls[1]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[1]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[1][0](state)).toEqual({ ...state, layers: { @@ -1145,10 +1121,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .find('[data-test-subj="indexPattern-time-scaling-enable"]') .hostNodes() .simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1177,10 +1150,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1207,10 +1177,7 @@ describe('IndexPatternDimensionEditorPanel', () => { }); wrapper = mount(); wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1241,10 +1208,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .prop('onChange')!(({ target: { value: 'h' }, } as unknown) as ChangeEvent); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1271,10 +1235,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .prop('onChange')!(({ target: { value: 'h' }, } as unknown) as ChangeEvent); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1302,10 +1263,7 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1595,10 +1553,7 @@ describe('IndexPatternDimensionEditorPanel', () => { .find('[data-test-subj="indexPattern-filter-by-enable"]') .hostNodes() .simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1629,10 +1584,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1658,10 +1610,7 @@ describe('IndexPatternDimensionEditorPanel', () => { language: 'kuery', query: 'c: d', }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1690,10 +1639,7 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1745,10 +1691,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: false }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: false }]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, layers: { @@ -1812,10 +1755,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-average"]').simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](initialState)).toEqual({ ...initialState, layers: { @@ -1840,10 +1780,7 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { @@ -1977,10 +1914,7 @@ describe('IndexPatternDimensionEditorPanel', () => { comboBox.prop('onChange')!([option]); }); - expect(setState.mock.calls[0]).toEqual([ - expect.any(Function), - { shouldRemoveDimension: false, shouldReplaceDimension: true }, - ]); + expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](defaultProps.state)).toEqual({ ...state, layers: { diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 1c20e306b2e8a9..b421d57dae6e1f 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -306,8 +306,7 @@ export type DatasourceDimensionEditorProps = DatasourceDimensionPro setState: ( newState: Parameters>[0], publishToVisualization?: { - shouldReplaceDimension?: boolean; - shouldRemoveDimension?: boolean; + isDimensionComplete?: boolean; } ) => void; core: Pick; diff --git a/x-pack/test/functional/apps/lens/smokescreen.ts b/x-pack/test/functional/apps/lens/smokescreen.ts index 4c8d642e7feb93..418f39682f8b47 100644 --- a/x-pack/test/functional/apps/lens/smokescreen.ts +++ b/x-pack/test/functional/apps/lens/smokescreen.ts @@ -588,6 +588,57 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); + it('should not leave an incomplete column in the visualization config with field-based operation', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'minimum', + }); + + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + undefined + ); + }); + + it('should not leave an incomplete column in the visualization config with reference-based operations', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'moving_average', + field: 'Records', + }); + + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + 'Moving average of Count of records' + ); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-dimensionTrigger', + operation: 'median', + isPreviousIncompatible: true, + keepOpen: true, + }); + + expect(await PageObjects.lens.isDimensionEditorOpen()).to.eql(true); + + await PageObjects.lens.closeDimensionEditor(); + + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + undefined + ); + }); + it('should transition from unique count to last value', async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickVisType('lens'); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 60891caeccfb09..ac522f0c2b26d7 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -389,6 +389,18 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); }, + async isDimensionEditorOpen() { + return await testSubjects.exists('lns-indexPattern-dimensionContainerBack'); + }, + + // closes the dimension editor flyout + async closeDimensionEditor() { + await retry.try(async () => { + await testSubjects.click('lns-indexPattern-dimensionContainerBack'); + await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); + }); + }, + async enableTimeShift() { await testSubjects.click('indexPattern-advanced-popover'); await retry.try(async () => { @@ -408,14 +420,6 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.click('errorFixAction'); }, - // closes the dimension editor flyout - async closeDimensionEditor() { - await retry.try(async () => { - await testSubjects.click('lns-indexPattern-dimensionContainerBack'); - await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); - }); - }, - async isTopLevelAggregation() { return await testSubjects.isEuiSwitchChecked('indexPattern-nesting-switch'); }, From f9a9c9aba5a1c436a8b73f8e887a223cd261ac72 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 8 Jun 2021 18:23:48 -0400 Subject: [PATCH 180/185] Fix test issues --- .../dimension_panel/dimension_editor.tsx | 2 +- .../dimension_panel/dimension_panel.test.tsx | 5 ++++- x-pack/test/functional/apps/lens/smokescreen.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 1dda857c2c3292..198b285bc2ea82 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -125,7 +125,7 @@ export function DimensionEditor(props: DimensionEditorProps) { { isDimensionComplete: prevOperationType === 'fullReference' - ? hasIncompleteColumns + ? !hasIncompleteColumns : Boolean(hypotheticalLayer.columns[columnId]), } ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 72a084ce290d03..7e45b295215631 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -930,7 +930,10 @@ describe('IndexPatternDimensionEditorPanel', () => { .simulate('click'); // Now check that the dimension gets cleaned up on state update - expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); + expect(setState.mock.calls[0]).toEqual([ + expect.any(Function), + { isDimensionComplete: false }, + ]); expect(setState.mock.calls[0][0](state)).toEqual({ ...state, layers: { diff --git a/x-pack/test/functional/apps/lens/smokescreen.ts b/x-pack/test/functional/apps/lens/smokescreen.ts index 418f39682f8b47..5d775f154c9430 100644 --- a/x-pack/test/functional/apps/lens/smokescreen.ts +++ b/x-pack/test/functional/apps/lens/smokescreen.ts @@ -595,7 +595,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.configureDimension({ dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', - operation: 'minimum', + operation: 'min', }); expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( From 93da5010523dbafb0b5751e732de509016c934c3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 9 Jun 2021 11:15:51 +0200 Subject: [PATCH 181/185] add description to formula telemetry --- x-pack/plugins/lens/server/usage/schema.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/server/usage/schema.ts b/x-pack/plugins/lens/server/usage/schema.ts index 62b39ee6793ee2..c3608176717c56 100644 --- a/x-pack/plugins/lens/server/usage/schema.ts +++ b/x-pack/plugins/lens/server/usage/schema.ts @@ -193,7 +193,12 @@ const savedSchema: MakeSchemaFrom = { lnsDatatable: { type: 'long' }, lnsPie: { type: 'long' }, lnsMetric: { type: 'long' }, - formula: { type: 'long' }, + formula: { + type: 'long', + _meta: { + description: 'Number of saved lens visualizations which are using at least one formula', + }, + }, }; export const lensUsageSchema: MakeSchemaFrom = { From fd8bbc931b923af78af960142b1e8b6e4c9457da Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 9 Jun 2021 11:28:20 +0200 Subject: [PATCH 182/185] fix schema --- .../schema/xpack_plugins.json | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index cf436f5dfb661d..8e52450d393b03 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -2692,7 +2692,10 @@ "type": "long" }, "formula": { - "type": "long" + "type": "long", + "_meta": { + "description": "Number of saved lens visualizations which are using at least one formula" + } } } }, @@ -2738,7 +2741,10 @@ "type": "long" }, "formula": { - "type": "long" + "type": "long", + "_meta": { + "description": "Number of saved lens visualizations which are using at least one formula" + } } } }, @@ -2784,7 +2790,10 @@ "type": "long" }, "formula": { - "type": "long" + "type": "long", + "_meta": { + "description": "Number of saved lens visualizations which are using at least one formula" + } } } } From e2e4e6dfece81fbaa044a74d55ca01aa749587d1 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 9 Jun 2021 13:20:38 -0400 Subject: [PATCH 183/185] Update from design feedback --- .../dimension_panel/dimension_editor.scss | 8 ++++ .../dimension_panel/dimension_editor.tsx | 37 ++++++++-------- .../formula/editor/formula_editor.tsx | 43 +++++++++++-------- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index 3fb5e3c39498a6..1b470fdd89a09e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -2,6 +2,14 @@ height: 100%; } +.lnsIndexPatternDimensionEditor__header { + position: sticky; + top: 0; + background: $euiColorEmptyShade; + // Raise it above the elements that are after it in DOM order + z-index: $euiZLevel1; +} + .lnsIndexPatternDimensionEditor-isFullscreen { position: absolute; left: 0; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 198b285bc2ea82..821129b84543af 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -344,25 +344,24 @@ export function DimensionEditor(props: DimensionEditorProps) { const quickFunctions = ( <> -
- {temporaryQuickFunction && selectedColumn?.operationType === 'formula' && ( - <> - + +

+ {i18n.translate('xpack.lens.indexPattern.formulaWarningText', { + defaultMessage: 'Picking a quick function will erase your formula.', })} - color="warning" - > -

- {i18n.translate('xpack.lens.indexPattern.formulaWarningText', { - defaultMessage: 'Picking a quick function will erase your formula.', - })} -

- - - - )} +

+ + + )} +
{i18n.translate('xpack.lens.indexPattern.functionsLabel', { defaultMessage: 'Select a function', @@ -636,7 +635,7 @@ export function DimensionEditor(props: DimensionEditorProps) { return (
{!isFullscreen && operationSupportMatrix.operationWithoutField.has('formula') ? ( - + setIsHelpOpen(false)} ownFocus={false} button={ - setIsHelpOpen(!isHelpOpen)} iconType="help" color="text" size="s" + aria-label={i18n.translate( + 'xpack.lens.formula.editorHelpInlineShowToolTip', + { + defaultMessage: 'Show function reference', + } + )} /> } > @@ -726,36 +732,39 @@ export function FormulaEditor({ { setIsWarningOpen(!isWarningOpen); }} > - {errorCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaErrorCount', { + {errorCount + ? i18n.translate('xpack.lens.formulaErrorCount', { defaultMessage: '{count} {count, plural, one {error} other {errors}}', values: { count: errorCount }, - })} - - ) : null} - {warningCount ? ( - - {' '} - {i18n.translate('xpack.lens.formulaWarningCount', { + }) + : null} + {warningCount + ? i18n.translate('xpack.lens.formulaWarningCount', { defaultMessage: '{count} {count, plural, one {warning} other {warnings}}', values: { count: warningCount }, - })} - - ) : null} + }) + : null} } > - {warnings.map(({ message }, index) => ( + {warnings.map(({ message, severity }, index) => (
- {message} + + {message} +
))} From 28cf06e509e1601ec74fd0a92410cc097144be8c Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 9 Jun 2021 15:33:08 -0400 Subject: [PATCH 184/185] More review comments --- .../dimension_panel/dimension_editor.tsx | 5 +++-- .../operations/definitions/formula/editor/formula_editor.tsx | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 821129b84543af..ed20c8c3d39e4a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -349,13 +349,14 @@ export function DimensionEditor(props: DimensionEditorProps) {

{i18n.translate('xpack.lens.indexPattern.formulaWarningText', { - defaultMessage: 'Picking a quick function will erase your formula.', + defaultMessage: 'To overwrite your formula, select a quick function', })}

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index dbed58e4748338..312ceb116dcede 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -734,6 +734,7 @@ export function FormulaEditor({ className="lnsFormula__editorError" iconType="alert" size="xs" + flush="right" onClick={() => { setIsWarningOpen(!isWarningOpen); }} From 9ec3c9a8ccc906231a5dcce5dd33b7ea9eeb7415 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 9 Jun 2021 17:18:02 -0400 Subject: [PATCH 185/185] Hide callout border from v7 theme --- .../dimension_panel/dimension_editor.scss | 6 ++++++ .../dimension_panel/dimension_editor.tsx | 1 + 2 files changed, 7 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss index 1b470fdd89a09e..874291ae25e341 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.scss @@ -57,3 +57,9 @@ padding-top: 0; padding-bottom: 0; } + +.lnsIndexPatternDimensionEditor__warning { + @include kbnThemeStyle('v7') { + border: none; + } +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ed20c8c3d39e4a..3dd2d4a4ba3f50 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -347,6 +347,7 @@ export function DimensionEditor(props: DimensionEditorProps) { {temporaryQuickFunction && selectedColumn?.operationType === 'formula' && ( <>