[APM] Fixes support for APM data streams in Security and Timelines UIs (

elastic#105334) * addes support for apm data streams (traces-apm*) to timelines * [APM] Adds support for apm data streams (traces-apm*) to security solution (elastic#94702) * fix unit tests * reverting prepackaged_rules changes to be bumped later # Conflicts: # x-pack/plugins/timelines/server/search_strategy/index_fields/index.test.ts
ogupte · Jul 14, 2021 · be2ef68 · be2ef68
1 parent 2dacdae
commit be2ef68
Show file tree

Hide file tree

Showing 55 changed files with 171 additions and 21 deletions.
diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
@@ -110,6 +110,7 @@ export const APP_EVENT_FILTERS_PATH = `${APP_PATH}${EVENT_FILTERS_PATH}`;
 /** The comma-delimited list of Elasticsearch indices from which the SIEM app collects events */
 export const DEFAULT_INDEX_PATTERN = [
   'apm-*-transaction*',
+  'traces-apm*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',

diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts
@@ -98,6 +98,7 @@ export interface MachineLearningRule {
 
 export const getIndexPatterns = (): string[] => [
   'apm-*-transaction*',
+  'traces-apm*',
   'auditbeat-*',
   'endgame-*',
   'filebeat-*',

diff --git a/...lic/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap b/...lic/common/components/drag_and_drop/__snapshots__/drag_drop_context_wrapper.test.tsx.snap
diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts b/x-pack/plugins/security_solution/public/common/components/event_details/__mocks__/index.ts
@@ -406,6 +406,7 @@ export const mockAlertDetailsData = [
     field: 'signal.rule.index',
     values: [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',
@@ -415,6 +416,7 @@ export const mockAlertDetailsData = [
     ],
     originalValue: [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',

diff --git a/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/sourcerer/index.test.tsx
@@ -39,6 +39,7 @@ const mockOptions = [
   { label: 'filebeat-*', value: 'filebeat-*' },
   { label: 'logs-*', value: 'logs-*' },
   { label: 'packetbeat-*', value: 'packetbeat-*' },
+  { label: 'traces-apm*', value: 'traces-apm*' },
   { label: 'winlogbeat-*', value: 'winlogbeat-*' },
 ];
 

diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts
@@ -23,6 +23,7 @@ describe('Sourcerer selectors', () => {
         'filebeat-*',
         'logs-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
         '-*elastic-cloud-logs-*',
       ]);
@@ -42,6 +43,7 @@ describe('Sourcerer selectors', () => {
         'endgame-*',
         'filebeat-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
       ]);
     });
@@ -64,6 +66,7 @@ describe('Sourcerer selectors', () => {
         'filebeat-*',
         'logs-endpoint.event-*',
         'packetbeat-*',
+        'traces-apm*',
         'winlogbeat-*',
       ]);
     });

diff --git a/...ck/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts b/...ck/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts
@@ -175,6 +175,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -414,6 +415,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -619,6 +621,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',
@@ -822,6 +825,7 @@ export const alertsMock: AlertSearchResponse<unknown, unknown> = {
               immutable: false,
               index: [
                 'apm-*-transaction*',
+                'traces-apm*',
                 'auditbeat-*',
                 'endgame-*',
                 'filebeat-*',

diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/mock.ts
@@ -20,6 +20,7 @@ export const savedRuleMock: Rule = {
   id: '12345678987654321',
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',

diff --git a/...s/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx b/...s/security_solution/public/detections/containers/detection_engine/rules/use_rule.test.tsx
@@ -54,6 +54,7 @@ describe('useRule', () => {
           immutable: false,
           index: [
             'apm-*-transaction*',
+            'traces-apm*',
             'auditbeat-*',
             'endgame-*',
             'filebeat-*',

diff --git a/...ity_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx b/...ity_solution/public/detections/containers/detection_engine/rules/use_rule_status.test.tsx
@@ -43,6 +43,7 @@ const testRule: Rule = {
   immutable: false,
   index: [
     'apm-*-transaction*',
+    'traces-apm*',
     'auditbeat-*',
     'endgame-*',
     'filebeat-*',

diff --git a/...ution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx b/...ution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.test.tsx
@@ -62,6 +62,7 @@ describe('useRuleWithFallback', () => {
             "immutable": false,
             "index": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",
@@ -125,6 +126,7 @@ describe('useRuleWithFallback', () => {
             "immutable": false,
             "index": Array [
               "apm-*-transaction*",
+              "traces-apm*",
               "auditbeat-*",
               "endgame-*",
               "filebeat-*",

diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/network/components/embeddables/__mocks__/mock.ts
@@ -14,6 +14,7 @@ export const mockIndexPatternIds: IndexPatternMapping[] = [
 
 export const mockAPMIndexPatternIds: IndexPatternMapping[] = [
   { title: 'apm-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
+  { title: 'traces-apm*,logs-apm*,metrics-apm*,apm-*', id: '8c7323ac-97ad-4b53-ac0a-40f8f691a918' },
 ];
 
 export const mockSourceLayer = {
@@ -183,6 +184,11 @@ export const mockClientLayer = {
   joins: [],
 };
 
+const mockApmDataStreamClientLayer = {
+  ...mockClientLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Client Point',
+};
+
 export const mockServerLayer = {
   sourceDescriptor: {
     id: 'uuid.v4()',
@@ -238,6 +244,11 @@ export const mockServerLayer = {
   query: { query: '', language: 'kuery' },
 };
 
+const mockApmDataStreamServerLayer = {
+  ...mockServerLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Server Point',
+};
+
 export const mockLineLayer = {
   sourceDescriptor: {
     type: 'ES_PEW_PEW',
@@ -365,6 +376,10 @@ export const mockClientServerLineLayer = {
   type: 'VECTOR',
   query: { query: '', language: 'kuery' },
 };
+const mockApmDataStreamClientServerLineLayer = {
+  ...mockClientServerLineLayer,
+  label: 'traces-apm*,logs-apm*,metrics-apm*,apm-* | Line',
+};
 
 export const mockLayerList = [
   {
@@ -421,6 +436,9 @@ export const mockLayerListMixed = [
   mockClientServerLineLayer,
   mockServerLayer,
   mockClientLayer,
+  mockApmDataStreamClientServerLineLayer,
+  mockApmDataStreamServerLayer,
+  mockApmDataStreamClientLayer,
 ];
 
 export const mockAPMIndexPattern: IndexPatternSavedObject = {
@@ -468,6 +486,15 @@ export const mockAPMTransactionIndexPattern: IndexPatternSavedObject = {
   },
 };
 
+export const mockAPMTracesDataStreamIndexPattern: IndexPatternSavedObject = {
+  id: 'traces-apm*',
+  type: 'index-pattern',
+  _version: 'abc',
+  attributes: {
+    title: 'traces-apm*',
+  },
+};
+
 export const mockGlobIndexPattern: IndexPatternSavedObject = {
   id: '*',
   type: 'index-pattern',

diff --git a/...ins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx b/...ins/security_solution/public/network/components/embeddables/embedded_map_helpers.test.tsx
@@ -12,6 +12,7 @@ import {
   mockAPMIndexPattern,
   mockAPMRegexIndexPattern,
   mockAPMTransactionIndexPattern,
+  mockAPMTracesDataStreamIndexPattern,
   mockAuditbeatIndexPattern,
   mockCCSGlobIndexPattern,
   mockCommaFilebeatAuditbeatCCSGlobIndexPattern,
@@ -69,6 +70,7 @@ describe('embedded_map_helpers', () => {
   describe('findMatchingIndexPatterns', () => {
     const siemDefaultIndices = [
       'apm-*-transaction*',
+      'traces-apm*',
       'auditbeat-*',
       'endgame-*',
       'filebeat-*',
@@ -102,11 +104,16 @@ describe('embedded_map_helpers', () => {
 
     test('finds exact glob-matched index patterns ', () => {
       const matchingIndexPatterns = findMatchingIndexPatterns({
-        kibanaIndexPatterns: [mockAPMTransactionIndexPattern, mockFilebeatIndexPattern],
+        kibanaIndexPatterns: [
+          mockAPMTransactionIndexPattern,
+          mockAPMTracesDataStreamIndexPattern,
+          mockFilebeatIndexPattern,
+        ],
         siemDefaultIndices,
       });
       expect(matchingIndexPatterns).toEqual([
         mockAPMTransactionIndexPattern,
+        mockAPMTracesDataStreamIndexPattern,
         mockFilebeatIndexPattern,
       ]);
     });

diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts b/x-pack/plugins/security_solution/public/network/components/embeddables/map_config.ts
@@ -61,6 +61,21 @@ export const SUM_OF_DESTINATION_BYTES = 'sum_of_destination.bytes';
 export const SUM_OF_CLIENT_BYTES = 'sum_of_client.bytes';
 export const SUM_OF_SERVER_BYTES = 'sum_of_server.bytes';
 
+const APM_LAYER_FIELD_MAPPING = {
+  source: {
+    metricField: 'client.bytes',
+    geoField: 'client.geo.location',
+    tooltipProperties: Object.keys(clientFieldMappings),
+    label: i18n.CLIENT_LAYER,
+  },
+  destination: {
+    metricField: 'server.bytes',
+    geoField: 'server.geo.location',
+    tooltipProperties: Object.keys(serverFieldMappings),
+    label: i18n.SERVER_LAYER,
+  },
+};
+
 // Mapping to fields for creating specific layers for a given index pattern
 // e.g. The apm-* index pattern needs layers for client/server instead of source/destination
 export const lmc: LayerMappingCollection = {
@@ -78,20 +93,8 @@ export const lmc: LayerMappingCollection = {
       label: i18n.DESTINATION_LAYER,
     },
   },
-  'apm-*': {
-    source: {
-      metricField: 'client.bytes',
-      geoField: 'client.geo.location',
-      tooltipProperties: Object.keys(clientFieldMappings),
-      label: i18n.CLIENT_LAYER,
-    },
-    destination: {
-      metricField: 'server.bytes',
-      geoField: 'server.geo.location',
-      tooltipProperties: Object.keys(serverFieldMappings),
-      label: i18n.SERVER_LAYER,
-    },
-  },
+  'apm-*': APM_LAYER_FIELD_MAPPING,
+  'traces-apm*,logs-apm*,metrics-apm*,apm-*': APM_LAYER_FIELD_MAPPING,
 };
 
 /**

diff --git a/...ublic/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap b/...ublic/timelines/components/timeline/body/column_headers/__snapshots__/index.test.tsx.snap
diff --git a/...onents/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap b/...onents/timeline/body/renderers/suricata/__snapshots__/suricata_row_renderer.test.tsx.snap
diff --git a/...imelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap b/...imelines/components/timeline/body/renderers/zeek/__snapshots__/zeek_details.test.tsx.snap
diff --git a/...nes/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap b/...nes/components/timeline/body/renderers/zeek/__snapshots__/zeek_row_renderer.test.tsx.snap
diff --git a/...ion/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json b/...ion/server/lib/detection_engine/scripts/roles_users/detections_admin/detections_role.json
@@ -8,6 +8,7 @@
           ".lists*",
           ".items*",
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/...rity_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json b/...rity_solution/server/lib/detection_engine/scripts/roles_users/hunter/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/...on/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json b/...on/server/lib/detection_engine/scripts/roles_users/platform_engineer/detections_role.json
@@ -9,6 +9,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/...solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json b/...solution/server/lib/detection_engine/scripts/roles_users/rule_author/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/...solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json b/...solution/server/lib/detection_engine/scripts/roles_users/soc_manager/detections_role.json
@@ -5,6 +5,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/..._solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json b/..._solution/server/lib/detection_engine/scripts/roles_users/t1_analyst/detections_role.json
@@ -6,6 +6,7 @@
       {
         "names": [
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/..._solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json b/..._solution/server/lib/detection_engine/scripts/roles_users/t2_analyst/detections_role.json
@@ -8,6 +8,7 @@
           ".lists*",
           ".items*",
           "apm-*-transaction*",
+          "traces-apm*",
           "auditbeat-*",
           "endgame-*",
           "filebeat-*",

diff --git a/...ity_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json b/...ity_solution/server/lib/detection_engine/scripts/rules/patches/simplest_updated_name.json
@@ -8,6 +8,7 @@
   "from": "now-360s",
   "index": [
     "apm-*-transaction*",
+    "traces-apm*",
     "auditbeat-*",
     "endgame-*",
     "filebeat-*",

diff --git a/...urity_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json b/...urity_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json
@@ -3,6 +3,7 @@
   "enabled": false,
   "index": [
     "apm-*-transaction*",
+    "traces-apm*",
     "auditbeat-*",
     "endgame-*",
     "filebeat-*",