[Logs UI] Add dataset filter to ML module setup screen (#64470)

This adds the ability to filter the datasets to be processed by the ML jobs on the setup screen.

x-pack/plugins/infra/common/http_api/log_analysis/validation/datasets.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+export const LOG_ANALYSIS_VALIDATE_DATASETS_PATH =
+  '/api/infra/log_analysis/validation/log_entry_datasets';
+
+/**
+ * Request types
+ */
+export const validateLogEntryDatasetsRequestPayloadRT = rt.type({
+  data: rt.type({
+    indices: rt.array(rt.string),
+    timestampField: rt.string,
+    startTime: rt.number,
+    endTime: rt.number,
+  }),
+});
+
+export type ValidateLogEntryDatasetsRequestPayload = rt.TypeOf<
+  typeof validateLogEntryDatasetsRequestPayloadRT
+>;
+
+/**
+ * Response types
+ * */
+const logEntryDatasetsEntryRT = rt.strict({
+  indexName: rt.string,
+  datasets: rt.array(rt.string),
+});
+
+export const validateLogEntryDatasetsResponsePayloadRT = rt.type({
+  data: rt.type({
+    datasets: rt.array(logEntryDatasetsEntryRT),
+  }),
+});
+
+export type ValidateLogEntryDatasetsResponsePayload = rt.TypeOf<
+  typeof validateLogEntryDatasetsResponsePayloadRT
+>;

x-pack/plugins/infra/common/http_api/log_analysis/validation/index.ts

@@ -4,4 +4,5 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export * from './datasets';
 export * from './log_entry_rate_indices';

x-pack/plugins/infra/common/log_analysis/job_parameters.ts

@@ -21,17 +21,73 @@ export const getJobId = (spaceId: string, sourceId: string, jobType: string) =>
 export const getDatafeedId = (spaceId: string, sourceId: string, jobType: string) =>
   `datafeed-${getJobId(spaceId, sourceId, jobType)}`;
 
-export const jobSourceConfigurationRT = rt.type({
+export const datasetFilterRT = rt.union([
+  rt.strict({
+    type: rt.literal('includeAll'),
+  }),
+  rt.strict({
+    type: rt.literal('includeSome'),
+    datasets: rt.array(rt.string),
+  }),
+]);
+
+export type DatasetFilter = rt.TypeOf<typeof datasetFilterRT>;
+
+export const jobSourceConfigurationRT = rt.partial({
   indexPattern: rt.string,
   timestampField: rt.string,
   bucketSpan: rt.number,
+  datasetFilter: datasetFilterRT,
 });
 
 export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;
 
 export const jobCustomSettingsRT = rt.partial({
   job_revision: rt.number,
-  logs_source_config: rt.partial(jobSourceConfigurationRT.props),
+  logs_source_config: jobSourceConfigurationRT,
 });
 
 export type JobCustomSettings = rt.TypeOf<typeof jobCustomSettingsRT>;
+
+export const combineDatasetFilters = (
+  firstFilter: DatasetFilter,
+  secondFilter: DatasetFilter
+): DatasetFilter => {
+  if (firstFilter.type === 'includeAll' && secondFilter.type === 'includeAll') {
+    return {
+      type: 'includeAll',
+    };
+  }
+
+  const includedDatasets = new Set([
+    ...(firstFilter.type === 'includeSome' ? firstFilter.datasets : []),
+    ...(secondFilter.type === 'includeSome' ? secondFilter.datasets : []),
+  ]);
+
+  return {
+    type: 'includeSome',
+    datasets: [...includedDatasets],
+  };
+};
+
+export const filterDatasetFilter = (
+  datasetFilter: DatasetFilter,
+  predicate: (dataset: string) => boolean
+): DatasetFilter => {
+  if (datasetFilter.type === 'includeAll') {
+    return datasetFilter;
+  } else {
+    const newDatasets = datasetFilter.datasets.filter(predicate);
+
+    if (newDatasets.length > 0) {
+      return {
+        type: 'includeSome',
+        datasets: newDatasets,
+      };
+    } else {
+      return {
+        type: 'includeAll',
+      };
+    }
+  }
+};

x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/analysis_setup_indices_form.tsx

@@ -4,56 +4,41 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { EuiCode, EuiDescribedFormGroup, EuiFormRow, EuiCheckbox, EuiToolTip } from '@elastic/eui';
+import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
-import React, { useCallback, useMemo } from 'react';
-
+import React, { useCallback } from 'react';
 import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
-import { ValidatedIndex, ValidationIndicesUIError } from './validation';
+import { IndexSetupRow } from './index_setup_row';
+import { AvailableIndex } from './validation';
 
 export const AnalysisSetupIndicesForm: React.FunctionComponent<{
   disabled?: boolean;
-  indices: ValidatedIndex[];
+  indices: AvailableIndex[];
   isValidating: boolean;
-  onChangeSelectedIndices: (selectedIndices: ValidatedIndex[]) => void;
+  onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
   valid: boolean;
 }> = ({ disabled = false, indices, isValidating, onChangeSelectedIndices, valid }) => {
-  const handleCheckboxChange = useCallback(
-    (event: React.ChangeEvent<HTMLInputElement>) => {
+  const changeIsIndexSelected = useCallback(
+    (indexName: string, isSelected: boolean) => {
       onChangeSelectedIndices(
         indices.map(index => {
-          const checkbox = event.currentTarget;
-          return index.name === checkbox.id ? { ...index, isSelected: checkbox.checked } : index;
+          return index.name === indexName ? { ...index, isSelected } : index;
         })
       );
     },
     [indices, onChangeSelectedIndices]
   );
 
-  const choices = useMemo(
-    () =>
-      indices.map(index => {
-        const checkbox = (
-          <EuiCheckbox
-            key={index.name}
-            id={index.name}
-            label={<EuiCode>{index.name}</EuiCode>}
-            onChange={handleCheckboxChange}
-            checked={index.validity === 'valid' && index.isSelected}
-            disabled={disabled || index.validity === 'invalid'}
-          />
-        );
-
-        return index.validity === 'valid' ? (
-          checkbox
-        ) : (
-          <div key={index.name}>
-            <EuiToolTip content={formatValidationError(index.errors)}>{checkbox}</EuiToolTip>
-          </div>
-        );
-      }),
-    [disabled, handleCheckboxChange, indices]
+  const changeDatasetFilter = useCallback(
+    (indexName: string, datasetFilter) => {
+      onChangeSelectedIndices(
+        indices.map(index => {
+          return index.name === indexName ? { ...index, datasetFilter } : index;
+        })
+      );
+    },
+    [indices, onChangeSelectedIndices]
   );
 
   return (
@@ -69,13 +54,23 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
       description={
         <FormattedMessage
           id="xpack.infra.analysisSetup.indicesSelectionDescription"
-          defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries."
+          defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries. You can also choose to only include a certain subset of datasets. Note that the dataset filter applies to all selected indices."
         />
       }
     >
       <LoadingOverlayWrapper isLoading={isValidating}>
         <EuiFormRow fullWidth isInvalid={!valid} label={indicesSelectionLabel} labelType="legend">
-          <>{choices}</>
+          <>
+            {indices.map(index => (
+              <IndexSetupRow
+                index={index}
+                isDisabled={disabled}
+                key={index.name}
+                onChangeIsSelected={changeIsIndexSelected}
+                onChangeDatasetFilter={changeDatasetFilter}
+              />
+            ))}
+          </>
         </EuiFormRow>
       </LoadingOverlayWrapper>
     </EuiDescribedFormGroup>
@@ -85,51 +80,3 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
 const indicesSelectionLabel = i18n.translate('xpack.infra.analysisSetup.indicesSelectionLabel', {
   defaultMessage: 'Indices',
 });
-
-const formatValidationError = (errors: ValidationIndicesUIError[]): React.ReactNode => {
-  return errors.map(error => {
-    switch (error.error) {
-      case 'INDEX_NOT_FOUND':
-        return (
-          <p key={`${error.error}-${error.index}`}>
-            <FormattedMessage
-              id="xpack.infra.analysisSetup.indicesSelectionIndexNotFound"
-              defaultMessage="No indices match the pattern {index}"
-              values={{ index: <EuiCode>{error.index}</EuiCode> }}
-            />
-          </p>
-        );
-
-      case 'FIELD_NOT_FOUND':
-        return (
-          <p key={`${error.error}-${error.index}-${error.field}`}>
-            <FormattedMessage
-              id="xpack.infra.analysisSetup.indicesSelectionNoTimestampField"
-              defaultMessage="At least one index matching {index} lacks a required field {field}."
-              values={{
-                index: <EuiCode>{error.index}</EuiCode>,
-                field: <EuiCode>{error.field}</EuiCode>,
-              }}
-            />
-          </p>
-        );
-
-      case 'FIELD_NOT_VALID':
-        return (
-          <p key={`${error.error}-${error.index}-${error.field}`}>
-            <FormattedMessage
-              id="xpack.infra.analysisSetup.indicesSelectionTimestampNotValid"
-              defaultMessage="At least one index matching {index} has a field called {field} without the correct type."
-              values={{
-                index: <EuiCode>{error.index}</EuiCode>,
-                field: <EuiCode>{error.field}</EuiCode>,
-              }}
-            />
-          </p>
-        );
-
-      default:
-        return '';
-    }
-  });
-};

x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx

@@ -0,0 +1,88 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  EuiFilterButton,
+  EuiFilterGroup,
+  EuiPopover,
+  EuiPopoverTitle,
+  EuiSelectable,
+  EuiSelectableOption,
+} from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import React, { useCallback, useMemo } from 'react';
+import { DatasetFilter } from '../../../../../common/log_analysis';
+import { useVisibilityState } from '../../../../utils/use_visibility_state';
+
+export const IndexSetupDatasetFilter: React.FC<{
+  availableDatasets: string[];
+  datasetFilter: DatasetFilter;
+  isDisabled?: boolean;
+  onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
+}> = ({ availableDatasets, datasetFilter, isDisabled, onChangeDatasetFilter }) => {
+  const { isVisible, hide, show } = useVisibilityState(false);
+
+  const changeDatasetFilter = useCallback(
+    (options: EuiSelectableOption[]) => {
+      const selectedDatasets = options
+        .filter(({ checked }) => checked === 'on')
+        .map(({ label }) => label);
+
+      onChangeDatasetFilter(
+        selectedDatasets.length === 0
+          ? { type: 'includeAll' }
+          : { type: 'includeSome', datasets: selectedDatasets }
+      );
+    },
+    [onChangeDatasetFilter]
+  );
+
+  const selectableOptions: EuiSelectableOption[] = useMemo(
+    () =>
+      availableDatasets.map(datasetName => ({
+        label: datasetName,
+        checked:
+          datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName)
+            ? 'on'
+            : undefined,
+      })),
+    [availableDatasets, datasetFilter]
+  );
+
+  const datasetFilterButton = (
+    <EuiFilterButton disabled={isDisabled} isSelected={isVisible} onClick={show}>
+      <FormattedMessage
+        id="xpack.infra.analysisSetup.indexDatasetFilterIncludeAllButtonLabel"
+        defaultMessage="{includeType, select, includeAll {All datasets} includeSome {{includedDatasetCount, plural, one {# dataset} other {# datasets}}}}"
+        values={{
+          includeType: datasetFilter.type,
+          includedDatasetCount:
+            datasetFilter.type === 'includeSome' ? datasetFilter.datasets.length : 0,
+        }}
+      />
+    </EuiFilterButton>
+  );
+
+  return (
+    <EuiFilterGroup>
+      <EuiPopover
+        button={datasetFilterButton}
+        closePopover={hide}
+        isOpen={isVisible}
+        panelPaddingSize="none"
+      >
+        <EuiSelectable onChange={changeDatasetFilter} options={selectableOptions} searchable>
+          {(list, search) => (
+            <div>
+              <EuiPopoverTitle>{search}</EuiPopoverTitle>
+              {list}
+            </div>
+          )}
+        </EuiSelectable>
+      </EuiPopover>
+    </EuiFilterGroup>
+  );
+};