diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index e7af3d484dfbd2..a1d7d03f313db2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -78,6 +78,8 @@ describe('eql_executor', () => { logger, searchAfterSize, bulkCreate: jest.fn(), + wrapHits: jest.fn(), + wrapSequences: jest.fn(), }); expect(response.warningMessages.length).toEqual(1); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index a187b730696829..e08f519e9761a3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -21,18 +21,19 @@ import { isOutdated } from '../../migrations/helpers'; import { getIndexVersion } from '../../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template'; import { EqlRuleParams } from '../../schemas/rule_schemas'; -import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body'; import { getInputIndex } from '../get_input_output_index'; -import { filterDuplicateSignals } from '../filter_duplicate_signals'; + import { AlertAttributes, BulkCreate, + WrapHits, + WrapSequences, EqlSignalSearchResponse, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, - WrappedSignalHit, + SimpleHit, } from '../types'; -import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils'; +import { createSearchAfterReturnType, makeFloatString } from '../utils'; export const eqlExecutor = async ({ rule, @@ -43,6 +44,8 @@ export const eqlExecutor = async ({ logger, searchAfterSize, bulkCreate, + wrapHits, + wrapSequences, }: { rule: SavedObject>; tuple: RuleRangeTuple; @@ -52,6 +55,8 @@ export const eqlExecutor = async ({ logger: Logger; searchAfterSize: number; bulkCreate: BulkCreate; + wrapHits: WrapHits; + wrapSequences: WrapSequences; }): Promise => { const result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; @@ -104,27 +109,18 @@ export const eqlExecutor = async ({ const eqlSignalSearchEnd = performance.now(); const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart); result.searchAfterTimes = [eqlSearchDuration]; - let newSignals: WrappedSignalHit[] | undefined; + let newSignals: SimpleHit[] | undefined; if (response.hits.sequences !== undefined) { - newSignals = response.hits.sequences.reduce( - (acc: WrappedSignalHit[], sequence) => - acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)), - [] - ); + newSignals = wrapSequences(response.hits.sequences); } else if (response.hits.events !== undefined) { - newSignals = filterDuplicateSignals( - rule.id, - response.hits.events.map((event) => - wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex) - ) - ); + newSignals = wrapHits(response.hits.events); } else { throw new Error( 'eql query response should have either `sequences` or `events` but had neither' ); } - if (newSignals.length > 0) { + if (newSignals?.length) { const insertResult = await bulkCreate(newSignals); result.bulkCreateTimes.push(insertResult.bulkCreateDuration); result.createdSignalsCount += insertResult.createdItemsCount; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts index 5c4af83c3b03e3..0098d50fc01efa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts @@ -36,11 +36,13 @@ const mockSignals = [ ]; describe('filterDuplicateSignals', () => { - it('filters duplicate signals', () => { - expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1); - }); + describe('detection engine implementation', () => { + it('filters duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId1, mockSignals, false).length).toEqual(1); + }); - it('does not filter non-duplicate signals', () => { - expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2); + it('does not filter non-duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId3, mockSignals, false).length).toEqual(2); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts index a648c053062894..0b9859fad76883 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts @@ -5,10 +5,26 @@ * 2.0. */ -import { WrappedSignalHit } from './types'; +import { SimpleHit, WrappedSignalHit } from './types'; -export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { - return signals.filter( - (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) - ); +const isWrappedSignalHit = ( + signals: SimpleHit[], + isRuleRegistryEnabled: boolean +): signals is WrappedSignalHit[] => { + return !isRuleRegistryEnabled; +}; + +export const filterDuplicateSignals = ( + ruleId: string, + signals: SimpleHit[], + isRuleRegistryEnabled: boolean +) => { + if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) { + return signals.filter( + (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); + } else { + // TODO: filter duplicate signals for RAC + return []; + } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 184b49c2d6c7b9..21c1402861e6e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -66,7 +66,10 @@ describe('searchAfterAndBulkCreate', () => { buildRuleMessage, false ); - wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX }); + wrapHits = wrapHitsFactory({ + ruleSO, + signalsIndex: DEFAULT_SIGNALS_INDEX, + }); }); test('should return success with number of searches less than max signals', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index bb1e50c14d4014..32bd6d71bfb1da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -67,6 +67,7 @@ import { } from '../schemas/rule_schemas'; import { bulkCreateFactory } from './bulk_create_factory'; import { wrapHitsFactory } from './wrap_hits_factory'; +import { wrapSequencesFactory } from './wrap_sequences_factory'; export const signalRulesAlertType = ({ logger, @@ -233,6 +234,11 @@ export const signalRulesAlertType = ({ signalsIndex: params.outputIndex, }); + const wrapSequences = wrapSequencesFactory({ + ruleSO: savedObject, + signalsIndex: params.outputIndex, + }); + if (isMlRule(type)) { const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); for (const tuple of tuples) { @@ -313,6 +319,8 @@ export const signalRulesAlertType = ({ searchAfterSize, bulkCreate, logger, + wrapHits, + wrapSequences, }); } } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 8a6ce91b2575ab..c399454b9888be 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -25,6 +25,7 @@ import { BaseHit, RuleAlertAction, SearchTypes, + EqlSequence, } from '../../../../common/detection_engine/types'; import { ListClient } from '../../../../../lists/server'; import { Logger, SavedObject } from '../../../../../../../src/core/server'; @@ -257,9 +258,11 @@ export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise(docs: Array>) => Promise>; -export type WrapHits = ( - hits: Array> -) => Array>; +export type SimpleHit = BaseHit<{ '@timestamp': string }>; + +export type WrapHits = (hits: Array>) => SimpleHit[]; + +export type WrapSequences = (sequences: Array>) => SimpleHit[]; export interface SearchAfterAndBulkCreateParams { tuple: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts index 3f3e4ef3631bd9..d5c05bc890332e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts @@ -5,12 +5,7 @@ * 2.0. */ -import { - SearchAfterAndBulkCreateParams, - SignalSourceHit, - WrapHits, - WrappedSignalHit, -} from './types'; +import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types'; import { generateId } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { filterDuplicateSignals } from './filter_duplicate_signals'; @@ -25,11 +20,15 @@ export const wrapHitsFactory = ({ const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ { _index: signalsIndex, - // TODO: bring back doc._version - _id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''), - _source: buildBulkBody(ruleSO, doc as SignalSourceHit), + _id: generateId( + doc._index, + doc._id, + String(doc._version), + ruleSO.attributes.params.ruleId ?? '' + ), + _source: buildBulkBody(ruleSO, doc), }, ]); - return filterDuplicateSignals(ruleSO.id, wrappedDocs); + return filterDuplicateSignals(ruleSO.id, wrappedDocs, false); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts new file mode 100644 index 00000000000000..c53ea7b7ebe729 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types'; +import { buildSignalGroupFromSequence } from './build_bulk_body'; + +export const wrapSequencesFactory = ({ + ruleSO, + signalsIndex, +}: { + ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; + signalsIndex: string; +}): WrapSequences => (sequences) => + sequences.reduce( + (acc: WrappedSignalHit[], sequence) => [ + ...acc, + ...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex), + ], + [] + ); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index e8fe54150ea147..4bcbcb71d048c4 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -211,8 +211,10 @@ export class Plugin implements IPlugin core.getStartServices().then(([coreStart]) => coreStart); @@ -296,7 +298,7 @@ export class Plugin implements IPlugin { parents: [ { rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it - id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -242,7 +242,7 @@ export default ({ getService }: FtrProviderContext) => { }, { rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it - id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -252,7 +252,7 @@ export default ({ getService }: FtrProviderContext) => { depth: 2, parent: { rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it - id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1265,7 +1265,7 @@ export default ({ getService }: FtrProviderContext) => { parents: [ { rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it - id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1280,7 +1280,7 @@ export default ({ getService }: FtrProviderContext) => { }, { rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it - id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1290,7 +1290,7 @@ export default ({ getService }: FtrProviderContext) => { depth: 2, parent: { rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it - id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1423,7 +1423,7 @@ export default ({ getService }: FtrProviderContext) => { parents: [ { rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it - id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1438,7 +1438,7 @@ export default ({ getService }: FtrProviderContext) => { }, { rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it - id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33', type: 'signal', index: '.siem-signals-default-000001', depth: 1, @@ -1448,7 +1448,7 @@ export default ({ getService }: FtrProviderContext) => { depth: 2, parent: { rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it - id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33', type: 'signal', index: '.siem-signals-default-000001', depth: 1,