[Ingest] Allow to enable monitoring of elastic agent (#63598) (#64836)

x-pack/plugins/ingest_manager/common/constants/agent_config.ts

@@ -14,6 +14,7 @@ export const DEFAULT_AGENT_CONFIG = {
   status: AgentConfigStatus.Active,
   datasources: [],
   is_default: true,
+  monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>,
 };
 
 export const DEFAULT_AGENT_CONFIGS_PACKAGES = [DefaultPackages.system];

x-pack/plugins/ingest_manager/common/types/models/agent_config.ts

@@ -23,6 +23,7 @@ export interface NewAgentConfig {
   namespace?: string;
   description?: string;
   is_default?: boolean;
+  monitoring_enabled?: Array<'logs' | 'metrics'>;
 }
 
 export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes {
@@ -60,4 +61,12 @@ export interface FullAgentConfig {
   };
   datasources: FullAgentConfigDatasource[];
   revision?: number;
+  settings?: {
+    monitoring: {
+      use_output?: string;
+      enabled: boolean;
+      metrics: boolean;
+      logs: boolean;
+    };
+  };
 }

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx

@@ -18,6 +18,7 @@ import {
   EuiText,
   EuiComboBox,
   EuiIconTip,
+  EuiCheckboxGroup,
 } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
@@ -30,7 +31,7 @@ interface ValidationResults {
 
 const StyledEuiAccordion = styled(EuiAccordion)`
   .ingest-active-button {
-    color: ${props => props.theme.eui.euiColorPrimary}};
+    color: ${props => props.theme.eui.euiColorPrimary};
   }
 `;
 
@@ -244,6 +245,68 @@ export const AgentConfigForm: React.FunctionComponent<Props> = ({
             )}
           </EuiFlexItem>
         </EuiFlexGroup>
+        <EuiSpacer size="m" />
+        <EuiFlexGroup>
+          <EuiFlexItem>
+            <EuiText>
+              <h4>
+                <FormattedMessage
+                  id="xpack.ingestManager.agentConfigForm.monitoringLabel"
+                  defaultMessage="Agent monitoring"
+                />
+              </h4>
+            </EuiText>
+            <EuiSpacer size="m" />
+            <EuiText size="s">
+              <FormattedMessage
+                id="xpack.ingestManager.agentConfigForm.monitoringDescription"
+                defaultMessage="Collect data about your agents for debugging and tracking performance."
+              />
+            </EuiText>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiCheckboxGroup
+              options={[
+                {
+                  id: 'logs',
+                  label: i18n.translate(
+                    'xpack.ingestManager.agentConfigForm.monitoringLogsFieldLabel',
+                    { defaultMessage: 'Collect agent logs' }
+                  ),
+                },
+                {
+                  id: 'metrics',
+                  label: i18n.translate(
+                    'xpack.ingestManager.agentConfigForm.monitoringMetricsFieldLabel',
+                    { defaultMessage: 'Collect agent metrics' }
+                  ),
+                },
+              ]}
+              idToSelectedMap={(agentConfig.monitoring_enabled || []).reduce(
+                (acc: { logs: boolean; metrics: boolean }, key) => {
+                  acc[key] = true;
+                  return acc;
+                },
+                { logs: false, metrics: false }
+              )}
+              onChange={id => {
+                if (id !== 'logs' && id !== 'metrics') {
+                  return;
+                }
+
+                const hasLogs =
+                  agentConfig.monitoring_enabled && agentConfig.monitoring_enabled.indexOf(id) >= 0;
+
+                const previousValues = agentConfig.monitoring_enabled || [];
+                updateAgentConfig({
+                  monitoring_enabled: hasLogs
+                    ? previousValues.filter(type => type !== id)
+                    : [...previousValues, id],
+                });
+              }}
+            />
+          </EuiFlexItem>
+        </EuiFlexGroup>
       </StyledEuiAccordion>
     </EuiForm>
   );

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx

@@ -34,6 +34,7 @@ export const CreateAgentConfigFlyout: React.FunctionComponent<Props> = ({ onClos
     description: '',
     namespace: '',
     is_default: undefined,
+    monitoring_enabled: ['logs', 'metrics'],
   });
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [withSysMonitoring, setWithSysMonitoring] = useState<boolean>(true);

x-pack/plugins/ingest_manager/server/saved_objects.ts

@@ -128,6 +128,7 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
         updated_on: { type: 'keyword' },
         updated_by: { type: 'keyword' },
         revision: { type: 'integer' },
+        monitoring_enabled: { type: 'keyword' },
       },
     },
   },

x-pack/plugins/ingest_manager/server/services/agent_config.test.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { savedObjectsClientMock } from 'src/core/server/mocks';
+import { agentConfigService } from './agent_config';
+import { Output } from '../types';
+
+function getSavedObjectMock(configAttributes: any) {
+  const mock = savedObjectsClientMock.create();
+
+  mock.get.mockImplementation(async (type: string, id: string) => {
+    return {
+      type,
+      id,
+      references: [],
+      attributes: configAttributes,
+    };
+  });
+
+  return mock;
+}
+
+jest.mock('./output', () => {
+  return {
+    outputService: {
+      getDefaultOutputId: () => 'test-id',
+      get: (): Output => {
+        return {
+          id: 'test-id',
+          is_default: true,
+          name: 'default',
+          // @ts-ignore
+          type: 'elasticsearch',
+          hosts: ['http://127.0.0.1:9201'],
+        };
+      },
+    },
+  };
+});
+
+describe('agent config', () => {
+  describe('getFullConfig', () => {
+    it('should return a config without monitoring if not monitoring is not enabled', async () => {
+      const soClient = getSavedObjectMock({
+        revision: 1,
+      });
+      const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+      expect(config).toMatchObject({
+        id: 'config',
+        outputs: {
+          default: {
+            type: 'elasticsearch',
+            hosts: ['http://127.0.0.1:9201'],
+            ca_sha256: undefined,
+            api_key: undefined,
+          },
+        },
+        datasources: [],
+        revision: 1,
+        settings: {
+          monitoring: {
+            enabled: false,
+            logs: false,
+            metrics: false,
+          },
+        },
+      });
+    });
+
+    it('should return a config with monitoring if monitoring is enabled for logs', async () => {
+      const soClient = getSavedObjectMock({
+        revision: 1,
+        monitoring_enabled: ['logs'],
+      });
+      const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+      expect(config).toMatchObject({
+        id: 'config',
+        outputs: {
+          default: {
+            type: 'elasticsearch',
+            hosts: ['http://127.0.0.1:9201'],
+            ca_sha256: undefined,
+            api_key: undefined,
+          },
+        },
+        datasources: [],
+        revision: 1,
+        settings: {
+          monitoring: {
+            use_output: 'default',
+            enabled: true,
+            logs: true,
+            metrics: false,
+          },
+        },
+      });
+    });
+
+    it('should return a config with monitoring if monitoring is enabled for metrics', async () => {
+      const soClient = getSavedObjectMock({
+        revision: 1,
+        monitoring_enabled: ['metrics'],
+      });
+      const config = await agentConfigService.getFullConfig(soClient, 'config');
+
+      expect(config).toMatchObject({
+        id: 'config',
+        outputs: {
+          default: {
+            type: 'elasticsearch',
+            hosts: ['http://127.0.0.1:9201'],
+            ca_sha256: undefined,
+            api_key: undefined,
+          },
+        },
+        datasources: [],
+        revision: 1,
+        settings: {
+          monitoring: {
+            use_output: 'default',
+            enabled: true,
+            logs: false,
+            metrics: true,
+          },
+        },
+      });
+    });
+  });
+});

x-pack/plugins/ingest_manager/server/services/agent_config.ts

@@ -301,28 +301,49 @@ class AgentConfigService {
     if (!config) {
       return null;
     }
+    const defaultOutput = await outputService.get(
+      soClient,
+      await outputService.getDefaultOutputId(soClient)
+    );
 
     const agentConfig: FullAgentConfig = {
       id: config.id,
       outputs: {
         // TEMPORARY as we only support a default output
-        ...[
-          await outputService.get(soClient, await outputService.getDefaultOutputId(soClient)),
-        ].reduce((outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => {
-          outputs[name] = {
-            type,
-            hosts,
-            ca_sha256,
-            api_key,
-            ...outputConfig,
-          };
-          return outputs;
-        }, {} as FullAgentConfig['outputs']),
+        ...[defaultOutput].reduce(
+          (outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => {
+            outputs[name] = {
+              type,
+              hosts,
+              ca_sha256,
+              api_key,
+              ...outputConfig,
+            };
+            return outputs;
+          },
+          {} as FullAgentConfig['outputs']
+        ),
       },
       datasources: (config.datasources as Datasource[])
         .filter(datasource => datasource.enabled)
         .map(ds => storedDatasourceToAgentDatasource(ds)),
       revision: config.revision,
+      ...(config.monitoring_enabled && config.monitoring_enabled.length > 0
+        ? {
+            settings: {
+              monitoring: {
+                use_output: defaultOutput.name,
+                enabled: true,
+                logs: config.monitoring_enabled.indexOf('logs') >= 0,
+                metrics: config.monitoring_enabled.indexOf('metrics') >= 0,
+              },
+            },
+          }
+        : {
+            settings: {
+              monitoring: { enabled: false, logs: false, metrics: false },
+            },
+          }),
     };
 
     return agentConfig;

x-pack/plugins/ingest_manager/server/types/models/agent_config.ts

@@ -11,6 +11,9 @@ const AgentConfigBaseSchema = {
   name: schema.string(),
   namespace: schema.maybe(schema.string()),
   description: schema.maybe(schema.string()),
+  monitoring_enabled: schema.maybe(
+    schema.arrayOf(schema.oneOf([schema.literal('logs'), schema.literal('metrics')]))
+  ),
 };
 
 export const NewAgentConfigSchema = schema.object({