diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 8ff75b25388b0f..4fff99b09d4adc 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -7,6 +7,7 @@ import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter'; import { Filter, EsQueryConfig } from 'src/plugins/data/public'; import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { ExceptionListItemSchema } from '../shared_imports'; describe('get_filter', () => { describe('getQueryFilter', () => { @@ -919,19 +920,27 @@ describe('get_filter', () => { dateFormatTZ: 'Zulu', }; test('it should build a filter without chunking exception items', () => { - const exceptionFilter = buildExceptionFilter( - [ - { language: 'kuery', query: 'host.name: linux and some.field: value' }, - { language: 'kuery', query: 'user.name: name' }, + const exceptionItem1: ExceptionListItemSchema = { + ...getExceptionListItemSchemaMock(), + entries: [ + { field: 'host.name', operator: 'included', type: 'match', value: 'linux' }, + { field: 'some.field', operator: 'included', type: 'match', value: 'value' }, ], - { + }; + const exceptionItem2: ExceptionListItemSchema = { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }], + }; + const exceptionFilter = buildExceptionFilter({ + lists: [exceptionItem1, exceptionItem2], + config, + excludeExceptions: true, + chunkSize: 2, + indexPattern: { fields: [], title: 'auditbeat-*', }, - config, - true, - 2 - ); + }); expect(exceptionFilter).toEqual({ meta: { alias: null, @@ -949,7 +958,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'host.name': 'linux', }, }, @@ -961,7 +970,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'some.field': 'value', }, }, @@ -976,7 +985,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'user.name': 'name', }, }, @@ -990,20 +999,31 @@ describe('get_filter', () => { }); test('it should properly chunk exception items', () => { - const exceptionFilter = buildExceptionFilter( - [ - { language: 'kuery', query: 'host.name: linux and some.field: value' }, - { language: 'kuery', query: 'user.name: name' }, - { language: 'kuery', query: 'file.path: /safe/path' }, + const exceptionItem1: ExceptionListItemSchema = { + ...getExceptionListItemSchemaMock(), + entries: [ + { field: 'host.name', operator: 'included', type: 'match', value: 'linux' }, + { field: 'some.field', operator: 'included', type: 'match', value: 'value' }, ], - { + }; + const exceptionItem2: ExceptionListItemSchema = { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }], + }; + const exceptionItem3: ExceptionListItemSchema = { + ...getExceptionListItemSchemaMock(), + entries: [{ field: 'file.path', operator: 'included', type: 'match', value: '/safe/path' }], + }; + const exceptionFilter = buildExceptionFilter({ + lists: [exceptionItem1, exceptionItem2, exceptionItem3], + config, + excludeExceptions: true, + chunkSize: 2, + indexPattern: { fields: [], title: 'auditbeat-*', }, - config, - true, - 2 - ); + }); expect(exceptionFilter).toEqual({ meta: { alias: null, @@ -1024,7 +1044,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'host.name': 'linux', }, }, @@ -1036,7 +1056,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'some.field': 'value', }, }, @@ -1051,7 +1071,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'user.name': 'name', }, }, @@ -1069,7 +1089,7 @@ describe('get_filter', () => { minimum_should_match: 1, should: [ { - match: { + match_phrase: { 'file.path': '/safe/path', }, }, diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 73638fc48f3813..fcea90402d89dd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -6,7 +6,6 @@ import { Filter, - Query, IIndexPattern, isFilterDisabled, buildEsQuery, @@ -18,15 +17,10 @@ import { } from '../../../lists/common/schemas'; import { ESBoolQuery } from '../typed_json'; import { buildExceptionListQueries } from './build_exceptions_query'; -import { - Query as QueryString, - Language, - Index, - TimestampOverrideOrUndefined, -} from './schemas/common/schemas'; +import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas'; export const getQueryFilter = ( - query: QueryString, + query: Query, language: Language, filters: Array>, index: Index, @@ -53,19 +47,18 @@ export const getQueryFilter = ( * buildEsQuery, this allows us to offer nested queries * regardless */ - const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists }); - if (exceptionQueries.length > 0) { - // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), - // allowing us to make 1024-item chunks of exception list items. - // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a - // very conservative value. - const exceptionFilter = buildExceptionFilter( - exceptionQueries, - indexPattern, - config, - excludeExceptions, - 1024 - ); + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + const exceptionFilter = buildExceptionFilter({ + lists, + config, + excludeExceptions, + chunkSize: 1024, + indexPattern, + }); + if (exceptionFilter !== undefined) { enabledFilters.push(exceptionFilter); } const initialQuery = { query, language }; @@ -101,15 +94,17 @@ export const buildEqlSearchRequest = ( ignoreFilterIfFieldNotInIndex: false, dateFormatTZ: 'Zulu', }; - const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); - let exceptionFilter: Filter | undefined; - if (exceptionQueries.length > 0) { - // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), - // allowing us to make 1024-item chunks of exception list items. - // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a - // very conservative value. - exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); - } + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + const exceptionFilter = buildExceptionFilter({ + lists: exceptionLists, + config, + excludeExceptions: true, + chunkSize: 1024, + indexPattern, + }); const indexString = index.join(); const requestFilter: unknown[] = [ { @@ -154,13 +149,23 @@ export const buildEqlSearchRequest = ( } }; -export const buildExceptionFilter = ( - exceptionQueries: Query[], - indexPattern: IIndexPattern, - config: EsQueryConfig, - excludeExceptions: boolean, - chunkSize: number -) => { +export const buildExceptionFilter = ({ + lists, + config, + excludeExceptions, + chunkSize, + indexPattern, +}: { + lists: Array; + config: EsQueryConfig; + excludeExceptions: boolean; + chunkSize: number; + indexPattern?: IIndexPattern; +}) => { + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists }); + if (exceptionQueries.length === 0) { + return undefined; + } const exceptionFilter: Filter = { meta: { alias: null, diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx index b2f8426413b128..0bbe4c71ef5a88 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +/* eslint complexity: ["error", 30]*/ + import React, { memo, useEffect, useState, useCallback, useMemo } from 'react'; import styled, { css } from 'styled-components'; import { @@ -53,6 +55,7 @@ import { import { ErrorInfo, ErrorCallout } from '../error_callout'; import { ExceptionsBuilderExceptionItem } from '../types'; import { useFetchIndex } from '../../../containers/source'; +import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs'; export interface AddExceptionModalProps { ruleName: string; @@ -108,7 +111,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({ const { http } = useKibana().services; const [errorsExist, setErrorExists] = useState(false); const [comment, setComment] = useState(''); - const { rule: maybeRule } = useRuleAsync(ruleId); + const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId); const [shouldCloseAlert, setShouldCloseAlert] = useState(false); const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false); const [shouldDisableBulkClose, setShouldDisableBulkClose] = useState(false); @@ -124,8 +127,22 @@ export const AddExceptionModal = memo(function AddExceptionModal({ const [isSignalIndexPatternLoading, { indexPatterns: signalIndexPatterns }] = useFetchIndex( memoSignalIndexName ); - const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices); + const memoMlJobIds = useMemo( + () => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []), + [maybeRule] + ); + const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds); + + const memoRuleIndices = useMemo(() => { + if (jobs.length > 0) { + return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : []; + } else { + return ruleIndices; + } + }, [jobs, ruleIndices]); + + const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices); const onError = useCallback( (error: Error): void => { addError(error, { title: i18n.ADD_EXCEPTION_ERROR }); @@ -364,6 +381,8 @@ export const AddExceptionModal = memo(function AddExceptionModal({ !isSignalIndexPatternLoading && !isLoadingExceptionList && !isIndexPatternLoading && + !isRuleLoading && + !mlJobLoading && ruleExceptionList && ( <> diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx index ab0c566aa55c63..e97f745d6f979a 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx @@ -47,6 +47,7 @@ import { } from '../helpers'; import { Loader } from '../../loader'; import { ErrorInfo, ErrorCallout } from '../error_callout'; +import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs'; interface EditExceptionModalProps { ruleName: string; @@ -100,7 +101,7 @@ export const EditExceptionModal = memo(function EditExceptionModal({ const { http } = useKibana().services; const [comment, setComment] = useState(''); const [errorsExist, setErrorExists] = useState(false); - const { rule: maybeRule } = useRuleAsync(ruleId); + const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId); const [updateError, setUpdateError] = useState(null); const [hasVersionConflict, setHasVersionConflict] = useState(false); const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false); @@ -117,7 +118,21 @@ export const EditExceptionModal = memo(function EditExceptionModal({ memoSignalIndexName ); - const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices); + const memoMlJobIds = useMemo( + () => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []), + [maybeRule] + ); + const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds); + + const memoRuleIndices = useMemo(() => { + if (jobs.length > 0) { + return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : []; + } else { + return ruleIndices; + } + }, [jobs, ruleIndices]); + + const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices); const handleExceptionUpdateError = useCallback( (error: Error, statusCode: number | null, message: string | null) => { @@ -280,69 +295,75 @@ export const EditExceptionModal = memo(function EditExceptionModal({ {(addExceptionIsLoading || isIndexPatternLoading || isSignalIndexLoading) && ( )} - {!isSignalIndexLoading && !addExceptionIsLoading && !isIndexPatternLoading && ( - <> - - {isRuleEQLSequenceStatement && ( - <> - - - - )} - {i18n.EXCEPTION_BUILDER_INFO} - - - - - - - - - - - + + {isRuleEQLSequenceStatement && ( + <> + + + + )} + {i18n.EXCEPTION_BUILDER_INFO} + + - - {exceptionListType === 'endpoint' && ( - <> - - - {i18n.ENDPOINT_QUARANTINE_TEXT} - - - )} - - - )} + + + + + + + + + + + {exceptionListType === 'endpoint' && ( + <> + + + {i18n.ENDPOINT_QUARANTINE_TEXT} + + + )} + + + )} {updateError != null && ( => + http.fetch('/api/ml/jobs/jobs', { + method: 'POST', + body: JSON.stringify({ jobIds }), + asSystemRequest: true, + signal, + }); diff --git a/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_jobs.ts b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_jobs.ts new file mode 100644 index 00000000000000..4d7b342773d6c5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_jobs.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useEffect, useState } from 'react'; +import { useAsync, withOptionalSignal } from '../../../../shared_imports'; +import { getJobs } from '../api/get_jobs'; +import { CombinedJobWithStats } from '../../../../../../ml/common/types/anomaly_detection_jobs'; + +import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions'; +import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license'; +import { useAppToasts } from '../../../hooks/use_app_toasts'; +import { useHttp } from '../../../lib/kibana'; +import { useMlCapabilities } from './use_ml_capabilities'; +import * as i18n from '../translations'; + +const _getJobs = withOptionalSignal(getJobs); + +export const useGetJobs = () => useAsync(_getJobs); + +export interface UseGetInstalledJobReturn { + loading: boolean; + jobs: CombinedJobWithStats[]; + isMlUser: boolean; + isLicensed: boolean; +} + +export const useGetInstalledJob = (jobIds: string[]): UseGetInstalledJobReturn => { + const [jobs, setJobs] = useState([]); + const { addError } = useAppToasts(); + const mlCapabilities = useMlCapabilities(); + const http = useHttp(); + const { error, loading, result, start } = useGetJobs(); + + const isMlUser = hasMlUserPermissions(mlCapabilities); + const isLicensed = hasMlLicense(mlCapabilities); + + useEffect(() => { + if (isMlUser && isLicensed && jobIds.length > 0) { + start({ http, jobIds }); + } + }, [http, isMlUser, isLicensed, start, jobIds]); + + useEffect(() => { + if (result) { + setJobs(result); + } + }, [result]); + + useEffect(() => { + if (error) { + addError(error, { title: i18n.SIEM_JOB_FETCH_FAILURE }); + } + }, [addError, error]); + + return { isLicensed, isMlUser, jobs, loading }; +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx index cf8204478a9551..9eb0a97a1c9a21 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx @@ -21,7 +21,6 @@ import { TimelineId } from '../../../../../common/types/timeline'; import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants'; import { Status, Type } from '../../../../../common/detection_engine/schemas/common/schemas'; import { isThresholdRule } from '../../../../../common/detection_engine/utils'; -import { isMlRule } from '../../../../../common/machine_learning/helpers'; import { timelineActions } from '../../../../timelines/store/timeline'; import { EventsTd, EventsTdContent } from '../../../../timelines/components/timeline/styles'; import { DEFAULT_ICON_BUTTON_WIDTH } from '../../../../timelines/components/timeline/helpers'; @@ -75,11 +74,17 @@ const AlertContextMenuComponent: React.FC = ({ '', [ecsRowData] ); - const ruleIndices = useMemo( - (): string[] => - (ecsRowData.signal?.rule && ecsRowData.signal.rule.index) ?? DEFAULT_INDEX_PATTERN, - [ecsRowData] - ); + const ruleIndices = useMemo((): string[] => { + if ( + ecsRowData.signal?.rule && + ecsRowData.signal.rule.index && + ecsRowData.signal.rule.index.length > 0 + ) { + return ecsRowData.signal.rule.index; + } else { + return DEFAULT_INDEX_PATTERN; + } + }, [ecsRowData]); const { addWarning } = useAppToasts(); @@ -321,7 +326,7 @@ const AlertContextMenuComponent: React.FC = ({ const areExceptionsAllowed = useMemo((): boolean => { const ruleTypes = getOr([], 'signal.rule.type', ecsRowData); const [ruleType] = ruleTypes as Type[]; - return !isMlRule(ruleType) && !isThresholdRule(ruleType); + return !isThresholdRule(ruleType); }, [ecsRowData]); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx index 2479a260872be7..40b73fc7d158c8 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx @@ -8,7 +8,6 @@ import { EuiAccordion, EuiFlexItem, EuiSpacer, EuiFormRow } from '@elastic/eui'; import React, { FC, memo, useCallback, useEffect, useState } from 'react'; import styled from 'styled-components'; -import { isMlRule } from '../../../../../common/machine_learning/helpers'; import { isThresholdRule } from '../../../../../common/detection_engine/utils'; import { RuleStepProps, @@ -76,10 +75,7 @@ const StepAboutRuleComponent: FC = ({ const [severityValue, setSeverityValue] = useState(initialState.severity.value); const [indexPatternLoading, { indexPatterns }] = useFetchIndex(defineRuleData?.index ?? []); - const canUseExceptions = - defineRuleData?.ruleType && - !isMlRule(defineRuleData.ruleType) && - !isThresholdRule(defineRuleData.ruleType); + const canUseExceptions = defineRuleData?.ruleType && !isThresholdRule(defineRuleData.ruleType); const { form } = useForm({ defaultValue: initialState, diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx index 4866824f882cfc..d04980d7648310 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx @@ -80,7 +80,6 @@ import { DEFAULT_INDEX_PATTERN } from '../../../../../../common/constants'; import { useFullScreen } from '../../../../../common/containers/use_full_screen'; import { Display } from '../../../../../hosts/pages/display'; import { ExceptionListTypeEnum, ExceptionListIdentifiers } from '../../../../../shared_imports'; -import { isMlRule } from '../../../../../../common/machine_learning/helpers'; import { isThresholdRule } from '../../../../../../common/detection_engine/utils'; import { useRuleAsync } from '../../../../containers/detection_engine/rules/use_rule_async'; import { showGlobalFilters } from '../../../../../timelines/components/timeline/helpers'; @@ -104,7 +103,7 @@ enum RuleDetailTabs { } const getRuleDetailsTabs = (rule: Rule | null) => { - const canUseExceptions = rule && !isMlRule(rule.type) && !isThresholdRule(rule.type); + const canUseExceptions = rule && !isThresholdRule(rule.type); return [ { id: RuleDetailTabs.alerts, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 0a38bdc790b412..764604a7937886 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -150,8 +150,8 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig export const sampleDocWithSortId = ( someUuid: string = sampleIdGuid, - ip?: string, - destIp?: string + ip?: string | string[], + destIp?: string | string[] ): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', @@ -502,8 +502,8 @@ export const repeatedSearchResultsWithSortId = ( total: number, pageSize: number, guids: string[], - ips?: string[], - destIps?: string[] + ips?: Array, + destIps?: Array ): SignalSearchResponse => ({ took: 10, timed_out: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 5c2dfa62e59516..d530fe10c6498b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -6,7 +6,6 @@ import { flow, omit } from 'lodash/fp'; import set from 'set-value'; -import { SearchResponse } from 'elasticsearch'; import { Logger } from '../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerts/server'; @@ -15,6 +14,7 @@ import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { AnomalyResults, Anomaly } from '../../machine_learning'; import { BuildRuleMessage } from './rule_messages'; +import { SearchResponse } from '../../types'; interface BulkCreateMlSignalsParams { actions: RuleAlertAction[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts index 3334cc17b90502..01e7e7160e1ae5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts @@ -400,6 +400,87 @@ describe('filterEventsAgainstList', () => { '9.9.9.9', ]).toEqual(ipVals); }); + + it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => { + const exceptionItem = getExceptionListItemSchemaMock(); + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'included', + type: 'list', + list: { + id: 'ci-badguys-again.txt', + type: 'ip', + }, + }, + ]; + + // this call represents an exception list with a value list containing ['2.2.2.2'] + (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getListItemResponseMock(), value: '2.2.2.2' }, + ]); + // this call represents an exception list with a value list containing ['4.4.4.4'] + (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getListItemResponseMock(), value: '4.4.4.4' }, + ]); + + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient, + exceptionsList: [exceptionItem], + eventSearchResult: repeatedSearchResultsWithSortId( + 3, + 3, + someGuids.slice(0, 3), + [ + ['1.1.1.1', '1.1.1.1'], + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ], + [ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ['3.3.3.3', '4.4.4.4'], + ] + ), + buildRuleMessage, + }); + expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + ]); + expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '4.4.4.4', + ]); + expect(res.hits.hits.length).toEqual(2); + + // @ts-expect-error + const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip); + expect([ + ['1.1.1.1', '1.1.1.1'], + ['1.1.1.1', '2.2.2.2'], + ]).toEqual(sourceIpVals); + // @ts-expect-error + const destIpVals = res.hits.hits.map((item) => item._source.destination.ip); + expect([ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ]).toEqual(destIpVals); + }); }); describe('operator type is excluded', () => { it('should respond with empty list if no items match value list', async () => { @@ -463,5 +544,86 @@ describe('filterEventsAgainstList', () => { ); expect(res.hits.hits.length).toEqual(2); }); + + it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => { + const exceptionItem = getExceptionListItemSchemaMock(); + exceptionItem.entries = [ + { + field: 'source.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys.txt', + type: 'ip', + }, + }, + { + field: 'destination.ip', + operator: 'excluded', + type: 'list', + list: { + id: 'ci-badguys-again.txt', + type: 'ip', + }, + }, + ]; + + // this call represents an exception list with a value list containing ['2.2.2.2'] + (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getListItemResponseMock(), value: '2.2.2.2' }, + ]); + // this call represents an exception list with a value list containing ['4.4.4.4'] + (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([ + { ...getListItemResponseMock(), value: '4.4.4.4' }, + ]); + + const res = await filterEventsAgainstList({ + logger: mockLogger, + listClient, + exceptionsList: [exceptionItem], + eventSearchResult: repeatedSearchResultsWithSortId( + 3, + 3, + someGuids.slice(0, 3), + [ + ['1.1.1.1', '1.1.1.1'], + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ], + [ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ['3.3.3.3', '4.4.4.4'], + ] + ), + buildRuleMessage, + }); + expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2); + expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + ]); + expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([ + '1.1.1.1', + '2.2.2.2', + '3.3.3.3', + '4.4.4.4', + ]); + expect(res.hits.hits.length).toEqual(2); + + // @ts-expect-error + const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip); + expect([ + ['1.1.1.1', '2.2.2.2'], + ['2.2.2.2', '3.3.3.3'], + ]).toEqual(sourceIpVals); + // @ts-expect-error + const destIpVals = res.hits.hits.map((item) => item._source.destination.ip); + expect([ + ['2.2.2.2', '3.3.3.3'], + ['3.3.3.3', '4.4.4.4'], + ]).toEqual(destIpVals); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index 908d073fbeabd0..1c13de16d9b1e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -7,7 +7,6 @@ import { get } from 'lodash/fp'; import { Logger } from 'src/core/server'; import { ListClient } from '../../../../../lists/server'; -import { SignalSearchResponse } from './types'; import { BuildRuleMessage } from './rule_messages'; import { EntryList, @@ -17,16 +16,23 @@ import { } from '../../../../../lists/common/schemas'; import { hasLargeValueList } from '../../../../common/detection_engine/utils'; import { SearchTypes } from '../../../../common/detection_engine/types'; +import { SearchResponse } from '../../types'; -interface FilterEventsAgainstList { - listClient: ListClient; - exceptionsList: ExceptionListItemSchema[]; - logger: Logger; - eventSearchResult: SignalSearchResponse; - buildRuleMessage: BuildRuleMessage; -} +// narrow unioned type to be single +const isStringableType = (val: SearchTypes): val is string | number | boolean => + ['string', 'number', 'boolean'].includes(typeof val); + +const isStringableArray = (val: SearchTypes): val is Array => { + if (!Array.isArray(val)) { + return false; + } + // TS does not allow .every to be called on val as-is, even though every type in the union + // is an array. https://github.com/microsoft/TypeScript/issues/36390 + // @ts-expect-error + return val.every((subVal) => isStringableType(subVal)); +}; -export const createSetToFilterAgainst = async ({ +export const createSetToFilterAgainst = async ({ events, field, listId, @@ -35,7 +41,7 @@ export const createSetToFilterAgainst = async ({ logger, buildRuleMessage, }: { - events: SignalSearchResponse['hits']['hits']; + events: SearchResponse['hits']['hits']; field: string; listId: string; listType: Type; @@ -43,13 +49,14 @@ export const createSetToFilterAgainst = async ({ logger: Logger; buildRuleMessage: BuildRuleMessage; }): Promise> => { - // narrow unioned type to be single - const isStringableType = (val: SearchTypes) => - ['string', 'number', 'boolean'].includes(typeof val); const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => { const valueField = get(field, searchResultItem._source); - if (valueField != null && isStringableType(valueField)) { - acc.add(valueField.toString()); + if (valueField != null) { + if (isStringableType(valueField)) { + acc.add(valueField.toString()); + } else if (isStringableArray(valueField)) { + valueField.forEach((subVal) => acc.add(subVal.toString())); + } } return acc; }, new Set()); @@ -71,13 +78,19 @@ export const createSetToFilterAgainst = async ({ return matchedListItemsSet; }; -export const filterEventsAgainstList = async ({ +export const filterEventsAgainstList = async ({ listClient, exceptionsList, logger, eventSearchResult, buildRuleMessage, -}: FilterEventsAgainstList): Promise => { +}: { + listClient: ListClient; + exceptionsList: ExceptionListItemSchema[]; + logger: Logger; + eventSearchResult: SearchResponse; + buildRuleMessage: BuildRuleMessage; +}): Promise> => { try { if (exceptionsList == null || exceptionsList.length === 0) { logger.debug(buildRuleMessage('about to return original search result')); @@ -108,9 +121,9 @@ export const filterEventsAgainstList = async ({ }); // now that we have all the exception items which are value lists (whether single entry or have multiple entries) - const res = await valueListExceptionItems.reduce>( + const res = await valueListExceptionItems.reduce['hits']['hits']>>( async ( - filteredAccum: Promise, + filteredAccum: Promise['hits']['hits']>, exceptionItem: ExceptionListItemSchema ) => { // 1. acquire the values from the specified fields to check @@ -152,15 +165,23 @@ export const filterEventsAgainstList = async ({ const vals = fieldAndSetTuples.map((tuple) => { const eventItem = get(tuple.field, item._source); if (tuple.operator === 'included') { - // only create a signal if the event is not in the value list + // only create a signal if the field value is not in the value list if (eventItem != null) { - return !tuple.matchedSet.has(eventItem); + if (isStringableType(eventItem)) { + return !tuple.matchedSet.has(eventItem); + } else if (isStringableArray(eventItem)) { + return !eventItem.some((val) => tuple.matchedSet.has(val)); + } } return true; } else if (tuple.operator === 'excluded') { - // only create a signal if the event is in the value list + // only create a signal if the field value is in the value list if (eventItem != null) { - return tuple.matchedSet.has(eventItem); + if (isStringableType(eventItem)) { + return tuple.matchedSet.has(eventItem); + } else if (isStringableArray(eventItem)) { + return eventItem.some((val) => tuple.matchedSet.has(val)); + } } return true; } @@ -175,10 +196,10 @@ export const filterEventsAgainstList = async ({ const toReturn = filteredEvents; return toReturn; }, - Promise.resolve(eventSearchResult.hits.hits) + Promise.resolve['hits']['hits']>(eventSearchResult.hits.hits) ); - const toReturn: SignalSearchResponse = { + const toReturn: SearchResponse = { took: eventSearchResult.took, timed_out: eventSearchResult.timed_out, _shards: eventSearchResult._shards, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts index 94b73fce79f0c5..ec653f088b5231 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts @@ -5,6 +5,7 @@ */ import dateMath from '@elastic/datemath'; +import { ExceptionListItemSchema } from '../../../../../lists/common'; import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../../src/core/server'; import { MlPluginSetup } from '../../../../../ml/server'; @@ -18,6 +19,7 @@ export const findMlSignals = async ({ anomalyThreshold, from, to, + exceptionItems, }: { ml: MlPluginSetup; request: KibanaRequest; @@ -26,6 +28,7 @@ export const findMlSignals = async ({ anomalyThreshold: number; from: string; to: string; + exceptionItems: ExceptionListItemSchema[]; }): Promise => { const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient); const params = { @@ -33,6 +36,7 @@ export const findMlSignals = async ({ threshold: anomalyThreshold, earliestMs: dateMath.parse(from)?.valueOf() ?? 0, latestMs: dateMath.parse(to)?.valueOf() ?? 0, + exceptionItems, }; return getAnomalies(params, mlAnomalySearch); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index f0b1825c7cc99c..d6bdc14a92b40e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -66,6 +66,7 @@ import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk import { createThreatSignals } from './threat_mapping/create_threat_signals'; import { getIndexVersion } from '../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; +import { filterEventsAgainstList } from './filter_events_with_list'; export const signalRulesAlertType = ({ logger, @@ -242,9 +243,18 @@ export const signalRulesAlertType = ({ anomalyThreshold, from, to, + exceptionItems: exceptionItems ?? [], + }); + + const filteredAnomalyResults = await filterEventsAgainstList({ + listClient, + exceptionsList: exceptionItems ?? [], + logger, + eventSearchResult: anomalyResults, + buildRuleMessage, }); - const anomalyCount = anomalyResults.hits.hits.length; + const anomalyCount = filteredAnomalyResults.hits.hits.length; if (anomalyCount) { logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`)); } @@ -257,7 +267,7 @@ export const signalRulesAlertType = ({ } = await bulkCreateMlSignals({ actions, throttle, - someResult: anomalyResults, + someResult: filteredAnomalyResults, ruleParams: params, services, logger, @@ -276,15 +286,16 @@ export const signalRulesAlertType = ({ }); // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } const shardFailures = - (anomalyResults._shards as typeof anomalyResults._shards & { failures: [] }).failures ?? - []; + (filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & { + failures: []; + }).failures ?? []; const searchErrors = createErrorsFromShard({ errors: shardFailures, }); result = mergeReturns([ result, createSearchAfterReturnType({ - success: success && anomalyResults._shards.failed === 0, + success: success && filteredAnomalyResults._shards.failed === 0, errors: [...errors, ...searchErrors], createdSignalsCount: createdItemsCount, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], diff --git a/x-pack/plugins/security_solution/server/lib/machine_learning/index.test.ts b/x-pack/plugins/security_solution/server/lib/machine_learning/index.test.ts index 63e3f3487e482a..d08b5e649451cc 100644 --- a/x-pack/plugins/security_solution/server/lib/machine_learning/index.test.ts +++ b/x-pack/plugins/security_solution/server/lib/machine_learning/index.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getAnomalies, AnomaliesSearchParams } from '.'; const getFiltersFromMock = (mock: jest.Mock) => { @@ -23,6 +24,7 @@ describe('getAnomalies', () => { threshold: 5, earliestMs: 1588517231429, latestMs: 1588617231429, + exceptionItems: [getExceptionListItemSchemaMock(), getExceptionListItemSchemaMock()], }; }); diff --git a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts index 34e004d817fe77..ec801f6c49ae76 100644 --- a/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts +++ b/x-pack/plugins/security_solution/server/lib/machine_learning/index.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SearchResponse } from 'elasticsearch'; import { RequestParams } from '@elastic/elasticsearch'; +import { ExceptionListItemSchema } from '../../../../lists/common'; +import { buildExceptionFilter } from '../../../common/detection_engine/get_query_filter'; import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server'; +import { SearchResponse } from '../types'; export { Anomaly }; export type AnomalyResults = SearchResponse; @@ -21,6 +23,7 @@ export interface AnomaliesSearchParams { threshold: number; earliestMs: number; latestMs: number; + exceptionItems: ExceptionListItemSchema[]; maxRecords?: number; } @@ -49,6 +52,17 @@ export const getAnomalies = async ( }, }, ], + must_not: buildExceptionFilter({ + lists: params.exceptionItems, + config: { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }, + excludeExceptions: true, + chunkSize: 1024, + })?.query, }, }, sort: [{ record_score: { order: 'desc' } }],