diff --git a/docs/user/production-considerations/alerting-production-considerations.asciidoc b/docs/user/production-considerations/alerting-production-considerations.asciidoc index e3d343475e175f..59c8a4bfa6d154 100644 --- a/docs/user/production-considerations/alerting-production-considerations.asciidoc +++ b/docs/user/production-considerations/alerting-production-considerations.asciidoc @@ -56,14 +56,10 @@ Predicting the buffer required to account for actions depends heavily on the rul experimental[] -Alerts and actions log activity in a set of "event log" indices. These indices are configured with an index lifecycle management (ILM) policy, which you can customize. The default policy rolls over the index when it reaches 50GB, or after 30 days. Indices over 90 days old are deleted. +Alerts and actions log activity in a set of "event log" data streams, one per Kibana version, named `.kibana-event-log-{VERSION}`. These data streams are configured with a lifecycle data retention of 90 days. This can be updated to other values via the standard data stream lifecycle APIs. Note that the event log data contains the data shown in the alerting pages in {kib}, so reducing the data retention period will result in less data being available to view. -The name of the index policy is `kibana-event-log-policy`. {kib} creates the index policy on startup, if it doesn't already exist. The index policy can be customized for your environment, but {kib} never modifies the index policy after creating it. - -Because {kib} uses the documents to display historic data, you should set the delete phase longer than you would like the historic data to be shown. For example, if you would like to see one month's worth of historic data, you should set the delete phase to at least one month. - -For more information on index lifecycle management, see: -{ref}/index-lifecycle-management.html[Index Lifecycle Policies]. +For more information on data stream lifecycle management, see: +{ref}/data-stream-lifecycle.html[Data stream lifecycle]. [float] [[alerting-circuit-breakers]] diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts index 6659f0b19ebeb4..2a5582347db74f 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts @@ -11,8 +11,6 @@ const createClusterClientMock = () => { const mock: jest.Mocked = { indexDocument: jest.fn(), indexDocuments: jest.fn(), - doesIlmPolicyExist: jest.fn(), - createIlmPolicy: jest.fn(), doesIndexTemplateExist: jest.fn(), createIndexTemplate: jest.fn(), doesDataStreamExist: jest.fn(), diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index 151a65573125ca..6f36af2be1f784 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -165,56 +165,6 @@ describe('buffering documents', () => { }); }); -describe('doesIlmPolicyExist', () => { - // ElasticsearchError can be a bit random in shape, we need an any here - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const notFoundError = new Error('Not found') as any; - notFoundError.statusCode = 404; - - test('should call cluster with proper arguments', async () => { - await clusterClientAdapter.doesIlmPolicyExist('foo'); - expect(clusterClient.transport.request).toHaveBeenCalledWith({ - method: 'GET', - path: '/_ilm/policy/foo', - }); - }); - - test('should return false when 404 error is returned by Elasticsearch', async () => { - clusterClient.transport.request.mockRejectedValue(notFoundError); - await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(false); - }); - - test('should throw error when error is not 404', async () => { - clusterClient.transport.request.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.doesIlmPolicyExist('foo') - ).rejects.toThrowErrorMatchingInlineSnapshot(`"error checking existance of ilm policy: Fail"`); - }); - - test('should return true when no error is thrown', async () => { - await expect(clusterClientAdapter.doesIlmPolicyExist('foo')).resolves.toEqual(true); - }); -}); - -describe('createIlmPolicy', () => { - test('should call cluster client with given policy', async () => { - clusterClient.transport.request.mockResolvedValue({ success: true }); - await clusterClientAdapter.createIlmPolicy('foo', { args: true }); - expect(clusterClient.transport.request).toHaveBeenCalledWith({ - method: 'PUT', - path: '/_ilm/policy/foo', - body: { args: true }, - }); - }); - - test('should throw error when call cluster client throws', async () => { - clusterClient.transport.request.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.createIlmPolicy('foo', { args: true }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"error creating ilm policy: Fail"`); - }); -}); - describe('doesIndexTemplateExist', () => { test('should call cluster with proper arguments', async () => { await clusterClientAdapter.doesIndexTemplateExist('foo'); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 8807e34cfedf30..27a86b839d6c9f 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -178,36 +178,6 @@ export class ClusterClientAdapter { - const request = { - method: 'GET', - path: `/_ilm/policy/${policyName}`, - }; - try { - const esClient = await this.elasticsearchClientPromise; - await esClient.transport.request(request); - } catch (err) { - if (err.statusCode === 404) return false; - throw new Error(`error checking existance of ilm policy: ${err.message}`); - } - return true; - } - - public async createIlmPolicy(policyName: string, policy: Record): Promise { - this.logger.info(`Installing ILM policy ${policyName}`); - const request = { - method: 'PUT', - path: `/_ilm/policy/${policyName}`, - body: policy, - }; - try { - const esClient = await this.elasticsearchClientPromise; - await esClient.transport.request(request); - } catch (err) { - throw new Error(`error creating ilm policy: ${err.message}`); - } - } - public async doesIndexTemplateExist(name: string): Promise { try { const esClient = await this.elasticsearchClientPromise; diff --git a/x-pack/plugins/event_log/server/es/context.test.ts b/x-pack/plugins/event_log/server/es/context.test.ts index 67ea2a95151f2c..681b927478d816 100644 --- a/x-pack/plugins/event_log/server/es/context.test.ts +++ b/x-pack/plugins/event_log/server/es/context.test.ts @@ -57,13 +57,12 @@ describe('createEsContext', () => { expect(esNames).toStrictEqual({ base: 'test-index', dataStream: 'test-index-event-log-1.2.3', - ilmPolicy: 'test-index-event-log-policy', indexPattern: 'test-index-event-log-*', indexTemplate: 'test-index-event-log-1.2.3-template', }); }); - test('should return exist false for esAdapter ilm policy, index template and data stream before initialize', async () => { + test('should return exist false for esAdapter index template and data stream before initialize', async () => { const context = createEsContext({ logger, indexNameRoot: 'test1', @@ -84,7 +83,7 @@ describe('createEsContext', () => { expect(doesIndexTemplateExist).toBeFalsy(); }); - test('should return exist true for esAdapter ilm policy, index template and data stream after initialize', async () => { + test('should return exist true for esAdapter index template and data stream after initialize', async () => { const context = createEsContext({ logger, indexNameRoot: 'test2', @@ -94,11 +93,6 @@ describe('createEsContext', () => { elasticsearchClient.indices.existsTemplate.mockResponse(true); context.initialize(); - const doesIlmPolicyExist = await context.esAdapter.doesIlmPolicyExist( - context.esNames.ilmPolicy - ); - expect(doesIlmPolicyExist).toBeTruthy(); - elasticsearchClient.indices.getDataStream.mockResolvedValue(GetDataStreamsResponse); const doesDataStreamExist = await context.esAdapter.doesDataStreamExist( context.esNames.dataStream diff --git a/x-pack/plugins/event_log/server/es/documents.test.ts b/x-pack/plugins/event_log/server/es/documents.test.ts index 814596f751c613..71b75ee3ca3dc1 100644 --- a/x-pack/plugins/event_log/server/es/documents.test.ts +++ b/x-pack/plugins/event_log/server/es/documents.test.ts @@ -5,19 +5,9 @@ * 2.0. */ -import { getIndexTemplate, getIlmPolicy } from './documents'; +import { getIndexTemplate } from './documents'; import { getEsNames } from './names'; -describe('getIlmPolicy()', () => { - test('returns the basic structure of an ilm policy', () => { - expect(getIlmPolicy()).toMatchObject({ - policy: { - phases: {}, - }, - }); - }); -}); - describe('getIndexTemplate()', () => { const kibanaVersion = '1.2.3'; const esNames = getEsNames('XYZ', kibanaVersion); @@ -27,7 +17,6 @@ describe('getIndexTemplate()', () => { expect(indexTemplate.index_patterns).toEqual([esNames.dataStream]); expect(indexTemplate.template.settings.number_of_shards).toBeGreaterThanOrEqual(0); expect(indexTemplate.template.settings.auto_expand_replicas).toBe('0-1'); - expect(indexTemplate.template.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy); expect(indexTemplate.template.mappings).toMatchObject({}); }); }); diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts index deaa8349971f99..0f654f80ad55b8 100644 --- a/x-pack/plugins/event_log/server/es/documents.ts +++ b/x-pack/plugins/event_log/server/es/documents.ts @@ -25,7 +25,9 @@ export function getIndexTemplate(esNames: EsNames) { hidden: true, number_of_shards: 1, auto_expand_replicas: '0-1', - 'index.lifecycle.name': esNames.ilmPolicy, + }, + lifecycle: { + data_retention: '90d', }, mappings, }, @@ -33,33 +35,3 @@ export function getIndexTemplate(esNames: EsNames) { return indexTemplateBody; } - -// returns the body of an ilm policy used in an ES PUT _ilm/policy call -export function getIlmPolicy() { - return { - policy: { - _meta: { - description: - 'ilm policy the Kibana event log, created initially by Kibana, but updated by the user, not Kibana', - managed: false, - }, - phases: { - hot: { - actions: { - rollover: { - max_size: '50GB', - max_age: '30d', - // max_docs: 1, // you know, for testing - }, - }, - }, - delete: { - min_age: '90d', - actions: { - delete: {}, - }, - }, - }, - }, - }; -} diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index 30e220313b26bc..7c81ae80b88239 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -83,7 +83,6 @@ describe('initializeEs', () => { `error getting existing index templates - Fail` ); expect(esContext.esAdapter.setLegacyIndexTemplateToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index templates throws an error`, async () => { @@ -124,7 +123,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar-template\" index template to hidden - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should update existing index settings if any exist and are not hidden`, async () => { @@ -207,7 +205,6 @@ describe('initializeEs', () => { expect(esContext.esAdapter.getExistingIndices).toHaveBeenCalled(); expect(esContext.logger.error).toHaveBeenCalledWith(`error getting existing indices - Fail`); expect(esContext.esAdapter.setIndexToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index settings throws an error`, async () => { @@ -251,7 +248,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar-000001\" index to hidden - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should update existing index aliases if any exist and are not hidden`, async () => { @@ -300,7 +296,6 @@ describe('initializeEs', () => { `error getting existing index aliases - Fail` ); expect(esContext.esAdapter.setIndexAliasToHidden).not.toHaveBeenCalled(); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); }); test(`should continue initialization if updating existing index aliases throws an error`, async () => { @@ -336,23 +331,6 @@ describe('initializeEs', () => { expect(esContext.logger.error).toHaveBeenCalledWith( `error setting existing \"foo-bar\" index aliases - Fail` ); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - }); - - test(`should create ILM policy if it doesn't exist`, async () => { - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(false); - - await initializeEs(esContext); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - expect(esContext.esAdapter.createIlmPolicy).toHaveBeenCalled(); - }); - - test(`shouldn't create ILM policy if it exists`, async () => { - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true); - - await initializeEs(esContext); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalled(); - expect(esContext.esAdapter.createIlmPolicy).not.toHaveBeenCalled(); }); test(`should create index template if it doesn't exist`, async () => { @@ -463,30 +441,10 @@ describe('retries', () => { esContext.esAdapter.getExistingLegacyIndexTemplates.mockResolvedValue({}); esContext.esAdapter.getExistingIndices.mockResolvedValue({}); esContext.esAdapter.getExistingIndexAliases.mockResolvedValue({}); - esContext.esAdapter.doesIlmPolicyExist.mockResolvedValue(true); esContext.esAdapter.doesIndexTemplateExist.mockResolvedValue(true); esContext.esAdapter.doesDataStreamExist.mockResolvedValue(true); }); - test('createIlmPolicyIfNotExists with 1 retry', async () => { - esContext.esAdapter.doesIlmPolicyExist.mockRejectedValueOnce(new Error('retry 1')); - - const timeStart = performance.now(); - await initializeEs(esContext); - const timeElapsed = Math.ceil(performance.now() - timeStart); - - expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY); - - expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(2); - expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1); - - const prefix = `eventLog initialization operation failed and will be retried: createIlmPolicyIfNotExists`; - expect(esContext.logger.warn).toHaveBeenCalledTimes(1); - expect(esContext.logger.warn).toHaveBeenCalledWith(`${prefix}; 4 more times; error: retry 1`); - }); - test('createIndexTemplateIfNotExists with 2 retries', async () => { esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2a')); esContext.esAdapter.doesIndexTemplateExist.mockRejectedValueOnce(new Error('retry 2b')); @@ -498,7 +456,6 @@ describe('retries', () => { expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2)); expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(3); expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(1); @@ -524,7 +481,6 @@ describe('retries', () => { expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY * (1 + 2 + 4 + 8)); expect(esContext.esAdapter.getExistingLegacyIndexTemplates).toHaveBeenCalledTimes(1); - expect(esContext.esAdapter.doesIlmPolicyExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesIndexTemplateExist).toHaveBeenCalledTimes(1); expect(esContext.esAdapter.doesDataStreamExist).toHaveBeenCalledTimes(5); expect(esContext.esAdapter.createDataStream).toHaveBeenCalledTimes(0); diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts index 6eb4d5736a4a1d..cf737cbf035c6a 100644 --- a/x-pack/plugins/event_log/server/es/init.ts +++ b/x-pack/plugins/event_log/server/es/init.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { asyncForEach } from '@kbn/std'; import { groupBy } from 'lodash'; import pRetry, { FailedAttemptError } from 'p-retry'; -import { getIlmPolicy, getIndexTemplate } from './documents'; +import { getIndexTemplate } from './documents'; import { EsContext } from './context'; const MAX_RETRY_DELAY = 30000; @@ -33,7 +33,6 @@ async function initializeEsResources(esContext: EsContext) { // today, setExistingAssetsToHidden() never throws, but just in case ... await retry(steps.setExistingAssetsToHidden); - await retry(steps.createIlmPolicyIfNotExists); await retry(steps.createIndexTemplateIfNotExists); await retry(steps.createDataStreamIfNotExists); @@ -202,18 +201,6 @@ class EsInitializationSteps { await this.setExistingIndexAliasesToHidden(); } - async createIlmPolicyIfNotExists(): Promise { - const exists = await this.esContext.esAdapter.doesIlmPolicyExist( - this.esContext.esNames.ilmPolicy - ); - if (!exists) { - await this.esContext.esAdapter.createIlmPolicy( - this.esContext.esNames.ilmPolicy, - getIlmPolicy() - ); - } - } - async createIndexTemplateIfNotExists(): Promise { const exists = await this.esContext.esAdapter.doesIndexTemplateExist( this.esContext.esNames.indexTemplate diff --git a/x-pack/plugins/event_log/server/es/names.mock.ts b/x-pack/plugins/event_log/server/es/names.mock.ts index 138d99aa706eae..837abe9dd413b6 100644 --- a/x-pack/plugins/event_log/server/es/names.mock.ts +++ b/x-pack/plugins/event_log/server/es/names.mock.ts @@ -11,7 +11,6 @@ const createNamesMock = () => { const mock: jest.Mocked = { base: '.kibana', dataStream: '.kibana-event-log-8.0.0', - ilmPolicy: 'kibana-event-log-policy', indexPattern: '.kibana-event-log-*', indexTemplate: '.kibana-event-log-8.0.0-template', }; diff --git a/x-pack/plugins/event_log/server/es/names.test.ts b/x-pack/plugins/event_log/server/es/names.test.ts index 0a05d560b9c94c..63d1ad9d398a7d 100644 --- a/x-pack/plugins/event_log/server/es/names.test.ts +++ b/x-pack/plugins/event_log/server/es/names.test.ts @@ -18,16 +18,7 @@ describe('getEsNames()', () => { const esNames = getEsNames(base, kibanaVersion); expect(esNames.base).toEqual(base); expect(esNames.dataStream).toEqual(`${base}-event-log-${kibanaVersion}`); - expect(esNames.ilmPolicy).toEqual(`${base}-event-log-policy`); expect(esNames.indexPattern).toEqual(`${base}-event-log-*`); expect(esNames.indexTemplate).toEqual(`${base}-event-log-${kibanaVersion}-template`); }); - - test('ilm policy name does not contain dot prefix', () => { - const base = '.XYZ'; - const kibanaVersion = '1.2.3'; - - const esNames = getEsNames(base, kibanaVersion); - expect(esNames.ilmPolicy).toEqual('XYZ-event-log-policy'); - }); }); diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index d807e53c6abbbf..0e48ca911b95a9 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -10,7 +10,6 @@ const EVENT_LOG_NAME_SUFFIX = `-event-log`; export interface EsNames { base: string; dataStream: string; - ilmPolicy: string; indexPattern: string; indexTemplate: string; } @@ -19,13 +18,9 @@ export function getEsNames(baseName: string, kibanaVersion: string): EsNames { const EVENT_LOG_VERSION_SUFFIX = `-${kibanaVersion.toLocaleLowerCase()}`; const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`; const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`; - const eventLogPolicyName = `${ - baseName.startsWith('.') ? baseName.substring(1) : baseName - }${EVENT_LOG_NAME_SUFFIX}-policy`; return { base: baseName, dataStream: eventLogNameWithVersion, - ilmPolicy: `${eventLogPolicyName}`, indexPattern: `${eventLogName}-*`, indexTemplate: `${eventLogNameWithVersion}-template`, };