Skip to content

Commit

Permalink
Load datasets for valid indices
Browse files Browse the repository at this point in the history
  • Loading branch information
@weltenwort
weltenwort committed Apr 29, 2020
1 parent fca3b0c commit 1e2112f
Show file tree
Hide file tree
Showing 4 changed files with 121 additions and 30 deletions.
14 changes: 12 additions & 2 deletions x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ import { DeleteJobsResponsePayload } from './api/ml_cleanup';
import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api';
import { GetMlModuleResponsePayload } from './api/ml_get_module';
import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
import { ValidationIndicesResponsePayload } from '../../../../common/http_api/log_analysis';
import {
ValidationIndicesResponsePayload,
ValidateLogEntryDatasetsResponsePayload,
} from '../../../../common/http_api/log_analysis';

export interface ModuleDescriptor<JobType extends string> {
moduleId: string;
Expand All @@ -24,8 +27,15 @@ export interface ModuleDescriptor<JobType extends string> {
) => Promise<SetupMlModuleResponsePayload>;
cleanUpModule: (spaceId: string, sourceId: string) => Promise<DeleteJobsResponsePayload>;
validateSetupIndices: (
sourceConfiguration: ModuleSourceConfiguration
indices: string[],
timestampField: string
) => Promise<ValidationIndicesResponsePayload>;
validateSetupDatasets: (
indices: string[],
timestampField: string,
startTime: number,
endTime: number
) => Promise<ValidateLogEntryDatasetsResponsePayload>;
}

export interface ModuleSourceConfiguration {
Expand Down
111 changes: 85 additions & 26 deletions x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { useCallback, useEffect, useMemo, useState, useReducer } from 'react';

import { isEqual } from 'lodash';
import { useCallback, useEffect, useMemo, useReducer, useState } from 'react';
import { usePrevious } from 'react-use';
import { isExampleDataIndex } from '../../../../common/log_analysis';
import {
AvailableIndex,
Expand All @@ -32,7 +33,7 @@ const fourWeeksInMs = 86400000 * 7 * 4;

export const useAnalysisSetupState = <JobType extends string>({
cleanUpAndSetUpModule,
moduleDescriptor: { validateSetupIndices },
moduleDescriptor: { validateSetupDatasets, validateSetupIndices },
setUpModule,
sourceConfiguration,
}: AnalysisSetupStateArguments<JobType>) => {
Expand All @@ -47,6 +48,19 @@ export const useAnalysisSetupState = <JobType extends string>({
}))
);

const validIndexNames = useMemo(
() => validatedIndices.filter(index => index.validity === 'valid').map(index => index.name),
[validatedIndices]
);

const selectedIndexNames = useMemo(
() =>
validatedIndices
.filter(index => index.validity === 'valid' && index.isSelected)
.map(i => i.name),
[validatedIndices]
);

const setValidatedIndices = useCallback(
(valueOrFunc: AvailableIndex[] | ((validateIndices: AvailableIndex[]) => AvailableIndex[])) =>
dispatchAvailableIndexAction({
Expand All @@ -60,7 +74,10 @@ export const useAnalysisSetupState = <JobType extends string>({
{
cancelPreviousOn: 'resolution',
createPromise: async () => {
return await validateSetupIndices(sourceConfiguration);
return await validateSetupIndices(
sourceConfiguration.indices,
sourceConfiguration.timestampField
);
},
onResolve: ({ data: { errors } }) => {
dispatchAvailableIndexAction({
Expand All @@ -72,21 +89,35 @@ export const useAnalysisSetupState = <JobType extends string>({
setValidatedIndices([]);
},
},
[sourceConfiguration.indices]
[sourceConfiguration.indices, sourceConfiguration.timestampField]
);

useEffect(() => {
validateIndices();
}, [validateIndices]);

const selectedIndexNames = useMemo(
() =>
validatedIndices
.filter(index => index.validity === 'valid' && index.isSelected)
.map(i => i.name),
[validatedIndices]
const [validateDatasetsRequest, validateDatasets] = useTrackedPromise(
{
cancelPreviousOn: 'resolution',
createPromise: async () => {
return await validateSetupDatasets(
validIndexNames,
sourceConfiguration.timestampField,
startTime ?? 0,
endTime ?? Date.now()
);
},
onResolve: ({ data: { datasets } }) => {
dispatchAvailableIndexAction({
type: 'updateWithAvailableDatasets',
availableDatasets: datasets,
});
},
},
[validIndexNames, sourceConfiguration.timestampField, startTime, endTime]
);

const validateIndicesAndDatasets = useCallback(async () => {
await validateIndices();
await validateDatasets();
}, [validateDatasets, validateIndices]);

const setUp = useCallback(() => {
return setUpModule(selectedIndexNames, startTime, endTime);
}, [setUpModule, selectedIndexNames, startTime, endTime]);
Expand All @@ -98,8 +129,10 @@ export const useAnalysisSetupState = <JobType extends string>({
const isValidating = useMemo(
() =>
validateIndicesRequest.state === 'pending' ||
validateIndicesRequest.state === 'uninitialized',
[validateIndicesRequest.state]
validateIndicesRequest.state === 'uninitialized' ||
validateDatasetsRequest.state === 'pending' ||
validateDatasetsRequest.state === 'uninitialized',
[validateDatasetsRequest.state, validateIndicesRequest.state]
);

const validationErrors = useMemo<ValidationIndicesUIError[]>(() => {
Expand All @@ -122,6 +155,28 @@ export const useAnalysisSetupState = <JobType extends string>({
}, []);
}, [isValidating, validateIndicesRequest.state, selectedIndexNames, validatedIndices]);

const prevStartTime = usePrevious(startTime);
const prevEndTime = usePrevious(endTime);
const prevValidIndexNames = usePrevious(validIndexNames);

useEffect(() => {
if (
startTime !== prevStartTime ||
endTime !== prevEndTime ||
!isEqual(validIndexNames, prevValidIndexNames)
) {
validateIndicesAndDatasets();
}
}, [
endTime,
prevEndTime,
prevStartTime,
prevValidIndexNames,
startTime,
validIndexNames,
validateIndicesAndDatasets,
]);

return {
cleanUpAndSetUp,
endTime,
Expand All @@ -144,7 +199,10 @@ type AvailableIndicesAction =
}
| { type: 'updateWithValidationErrors'; validationErrors: ValidationIndicesError[] }
| { type: 'select'; indexName: string }
| { type: 'updateAvailableDatasets'; indexName: string; availableDatasets: string[] };
| {
type: 'updateWithAvailableDatasets';
availableDatasets: Array<{ indexName: string; datasets: string[] }>;
};

const reduceAvailableIndicesState = (
state: AvailableIndex[],
Expand Down Expand Up @@ -179,26 +237,27 @@ const reduceAvailableIndicesState = (
return {
validity: 'valid',
name: previousAvailableIndex.name,
isSelected: isExampleDataIndex(previousAvailableIndex.name),
availableDatasets: ['a', 'b', 'c'],
isSelected: !isExampleDataIndex(previousAvailableIndex.name),
availableDatasets: [],
datasetFilter: {
include: 'all',
},
};
}
});
case 'updateAvailableDatasets':
case 'updateWithAvailableDatasets':
return state.map(previousAvailableIndex => {
if (
previousAvailableIndex.name !== action.indexName ||
previousAvailableIndex.validity !== 'valid'
) {
if (previousAvailableIndex.validity !== 'valid') {
return previousAvailableIndex;
}

const datasetsForIndex = action.availableDatasets
.filter(({ indexName }) => indexName === previousAvailableIndex.name)
.flatMap(({ datasets }) => datasets);

return {
...previousAvailableIndex,
availableDatasets: action.availableDatasets,
availableDatasets: datasetsForIndex,
};
});
case 'select':
Expand Down
13 changes: 12 additions & 1 deletion x-pack/plugins/infra/public/pages/logs/log_entry_categories/module_descriptor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
import { callValidateDatasetsAPI } from '../../../containers/logs/log_analysis/api/validate_datasets';
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';

const moduleId = 'logs_ui_categories';
Expand Down Expand Up @@ -85,7 +86,7 @@ const cleanUpModule = async (spaceId: string, sourceId: string) => {
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryCategoriesJobTypes);
};

const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => {
const validateSetupIndices = async (indices: string[], timestampField: string) => {
return await callValidateIndicesAPI(indices, [
{
name: timestampField,
Expand All @@ -102,6 +103,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon
]);
};

const validateSetupDatasets = async (
indices: string[],
timestampField: string,
startTime: number,
endTime: number
) => {
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
};

export const logEntryCategoriesModule: ModuleDescriptor<LogEntryCategoriesJobType> = {
moduleId,
jobTypes: logEntryCategoriesJobTypes,
Expand All @@ -111,5 +121,6 @@ export const logEntryCategoriesModule: ModuleDescriptor<LogEntryCategoriesJobTyp
getModuleDefinition,
setUpModule,
cleanUpModule,
validateSetupDatasets,
validateSetupIndices,
};
13 changes: 12 additions & 1 deletion x-pack/plugins/infra/public/pages/logs/log_entry_rate/module_descriptor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
import { callValidateDatasetsAPI } from '../../../containers/logs/log_analysis/api/validate_datasets';
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';

const moduleId = 'logs_ui_analysis';
Expand Down Expand Up @@ -84,7 +85,7 @@ const cleanUpModule = async (spaceId: string, sourceId: string) => {
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryRateJobTypes);
};

const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => {
const validateSetupIndices = async (indices: string[], timestampField: string) => {
return await callValidateIndicesAPI(indices, [
{
name: timestampField,
Expand All @@ -97,6 +98,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon
]);
};

const validateSetupDatasets = async (
indices: string[],
timestampField: string,
startTime: number,
endTime: number
) => {
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
};

export const logEntryRateModule: ModuleDescriptor<LogEntryRateJobType> = {
moduleId,
jobTypes: logEntryRateJobTypes,
Expand All @@ -106,5 +116,6 @@ export const logEntryRateModule: ModuleDescriptor<LogEntryRateJobType> = {
getModuleDefinition,
setUpModule,
cleanUpModule,
validateSetupDatasets,
validateSetupIndices,
};

0 comments on commit 1e2112f

Please sign in to comment.