diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts index dc9f25b4926359..98a4762e55ebf5 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts @@ -8,7 +8,10 @@ import { DeleteJobsResponsePayload } from './api/ml_cleanup'; import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api'; import { GetMlModuleResponsePayload } from './api/ml_get_module'; import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api'; -import { ValidationIndicesResponsePayload } from '../../../../common/http_api/log_analysis'; +import { + ValidationIndicesResponsePayload, + ValidateLogEntryDatasetsResponsePayload, +} from '../../../../common/http_api/log_analysis'; export interface ModuleDescriptor { moduleId: string; @@ -24,8 +27,15 @@ export interface ModuleDescriptor { ) => Promise; cleanUpModule: (spaceId: string, sourceId: string) => Promise; validateSetupIndices: ( - sourceConfiguration: ModuleSourceConfiguration + indices: string[], + timestampField: string ) => Promise; + validateSetupDatasets: ( + indices: string[], + timestampField: string, + startTime: number, + endTime: number + ) => Promise; } export interface ModuleSourceConfiguration { diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts index d95b24077fa1f8..249560c2208cb2 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts @@ -4,8 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback, useEffect, useMemo, useState, useReducer } from 'react'; - +import { isEqual } from 'lodash'; +import { useCallback, useEffect, useMemo, useReducer, useState } from 'react'; +import { usePrevious } from 'react-use'; import { isExampleDataIndex } from '../../../../common/log_analysis'; import { AvailableIndex, @@ -32,7 +33,7 @@ const fourWeeksInMs = 86400000 * 7 * 4; export const useAnalysisSetupState = ({ cleanUpAndSetUpModule, - moduleDescriptor: { validateSetupIndices }, + moduleDescriptor: { validateSetupDatasets, validateSetupIndices }, setUpModule, sourceConfiguration, }: AnalysisSetupStateArguments) => { @@ -47,6 +48,19 @@ export const useAnalysisSetupState = ({ })) ); + const validIndexNames = useMemo( + () => validatedIndices.filter(index => index.validity === 'valid').map(index => index.name), + [validatedIndices] + ); + + const selectedIndexNames = useMemo( + () => + validatedIndices + .filter(index => index.validity === 'valid' && index.isSelected) + .map(i => i.name), + [validatedIndices] + ); + const setValidatedIndices = useCallback( (valueOrFunc: AvailableIndex[] | ((validateIndices: AvailableIndex[]) => AvailableIndex[])) => dispatchAvailableIndexAction({ @@ -60,7 +74,10 @@ export const useAnalysisSetupState = ({ { cancelPreviousOn: 'resolution', createPromise: async () => { - return await validateSetupIndices(sourceConfiguration); + return await validateSetupIndices( + sourceConfiguration.indices, + sourceConfiguration.timestampField + ); }, onResolve: ({ data: { errors } }) => { dispatchAvailableIndexAction({ @@ -72,21 +89,35 @@ export const useAnalysisSetupState = ({ setValidatedIndices([]); }, }, - [sourceConfiguration.indices] + [sourceConfiguration.indices, sourceConfiguration.timestampField] ); - useEffect(() => { - validateIndices(); - }, [validateIndices]); - - const selectedIndexNames = useMemo( - () => - validatedIndices - .filter(index => index.validity === 'valid' && index.isSelected) - .map(i => i.name), - [validatedIndices] + const [validateDatasetsRequest, validateDatasets] = useTrackedPromise( + { + cancelPreviousOn: 'resolution', + createPromise: async () => { + return await validateSetupDatasets( + validIndexNames, + sourceConfiguration.timestampField, + startTime ?? 0, + endTime ?? Date.now() + ); + }, + onResolve: ({ data: { datasets } }) => { + dispatchAvailableIndexAction({ + type: 'updateWithAvailableDatasets', + availableDatasets: datasets, + }); + }, + }, + [validIndexNames, sourceConfiguration.timestampField, startTime, endTime] ); + const validateIndicesAndDatasets = useCallback(async () => { + await validateIndices(); + await validateDatasets(); + }, [validateDatasets, validateIndices]); + const setUp = useCallback(() => { return setUpModule(selectedIndexNames, startTime, endTime); }, [setUpModule, selectedIndexNames, startTime, endTime]); @@ -98,8 +129,10 @@ export const useAnalysisSetupState = ({ const isValidating = useMemo( () => validateIndicesRequest.state === 'pending' || - validateIndicesRequest.state === 'uninitialized', - [validateIndicesRequest.state] + validateIndicesRequest.state === 'uninitialized' || + validateDatasetsRequest.state === 'pending' || + validateDatasetsRequest.state === 'uninitialized', + [validateDatasetsRequest.state, validateIndicesRequest.state] ); const validationErrors = useMemo(() => { @@ -122,6 +155,28 @@ export const useAnalysisSetupState = ({ }, []); }, [isValidating, validateIndicesRequest.state, selectedIndexNames, validatedIndices]); + const prevStartTime = usePrevious(startTime); + const prevEndTime = usePrevious(endTime); + const prevValidIndexNames = usePrevious(validIndexNames); + + useEffect(() => { + if ( + startTime !== prevStartTime || + endTime !== prevEndTime || + !isEqual(validIndexNames, prevValidIndexNames) + ) { + validateIndicesAndDatasets(); + } + }, [ + endTime, + prevEndTime, + prevStartTime, + prevValidIndexNames, + startTime, + validIndexNames, + validateIndicesAndDatasets, + ]); + return { cleanUpAndSetUp, endTime, @@ -144,7 +199,10 @@ type AvailableIndicesAction = } | { type: 'updateWithValidationErrors'; validationErrors: ValidationIndicesError[] } | { type: 'select'; indexName: string } - | { type: 'updateAvailableDatasets'; indexName: string; availableDatasets: string[] }; + | { + type: 'updateWithAvailableDatasets'; + availableDatasets: Array<{ indexName: string; datasets: string[] }>; + }; const reduceAvailableIndicesState = ( state: AvailableIndex[], @@ -179,26 +237,27 @@ const reduceAvailableIndicesState = ( return { validity: 'valid', name: previousAvailableIndex.name, - isSelected: isExampleDataIndex(previousAvailableIndex.name), - availableDatasets: ['a', 'b', 'c'], + isSelected: !isExampleDataIndex(previousAvailableIndex.name), + availableDatasets: [], datasetFilter: { include: 'all', }, }; } }); - case 'updateAvailableDatasets': + case 'updateWithAvailableDatasets': return state.map(previousAvailableIndex => { - if ( - previousAvailableIndex.name !== action.indexName || - previousAvailableIndex.validity !== 'valid' - ) { + if (previousAvailableIndex.validity !== 'valid') { return previousAvailableIndex; } + const datasetsForIndex = action.availableDatasets + .filter(({ indexName }) => indexName === previousAvailableIndex.name) + .flatMap(({ datasets }) => datasets); + return { ...previousAvailableIndex, - availableDatasets: action.availableDatasets, + availableDatasets: datasetsForIndex, }; }); case 'select': diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/module_descriptor.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/module_descriptor.ts index be7547f2e74cb9..fb3db8466180ea 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/module_descriptor.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/module_descriptor.ts @@ -21,6 +21,7 @@ import { import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api'; import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module'; import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api'; +import { callValidateDatasetsAPI } from '../../../containers/logs/log_analysis/api/validate_datasets'; import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices'; const moduleId = 'logs_ui_categories'; @@ -85,7 +86,7 @@ const cleanUpModule = async (spaceId: string, sourceId: string) => { return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryCategoriesJobTypes); }; -const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => { +const validateSetupIndices = async (indices: string[], timestampField: string) => { return await callValidateIndicesAPI(indices, [ { name: timestampField, @@ -102,6 +103,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon ]); }; +const validateSetupDatasets = async ( + indices: string[], + timestampField: string, + startTime: number, + endTime: number +) => { + return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime); +}; + export const logEntryCategoriesModule: ModuleDescriptor = { moduleId, jobTypes: logEntryCategoriesJobTypes, @@ -111,5 +121,6 @@ export const logEntryCategoriesModule: ModuleDescriptor { return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryRateJobTypes); }; -const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => { +const validateSetupIndices = async (indices: string[], timestampField: string) => { return await callValidateIndicesAPI(indices, [ { name: timestampField, @@ -97,6 +98,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon ]); }; +const validateSetupDatasets = async ( + indices: string[], + timestampField: string, + startTime: number, + endTime: number +) => { + return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime); +}; + export const logEntryRateModule: ModuleDescriptor = { moduleId, jobTypes: logEntryRateJobTypes, @@ -106,5 +116,6 @@ export const logEntryRateModule: ModuleDescriptor = { getModuleDefinition, setUpModule, cleanUpModule, + validateSetupDatasets, validateSetupIndices, };