[7.x] [APM] Fleet: Introduce API for uploading source maps for RUM (#101623)

x-pack/plugins/apm/kibana.json

@@ -43,4 +43,4 @@
     "ml",
     "observability"
   ]
-}
+}

x-pack/plugins/apm/server/lib/fleet/register_fleet_policy_callbacks.ts

@@ -12,6 +12,7 @@ import { APMPluginStartDependencies } from '../../types';
 import { ExternalCallback } from '../../../../fleet/server';
 import { AGENT_NAME } from '../../../common/elasticsearch_fieldnames';
 import { AgentConfiguration } from '../../../common/agent_configuration/configuration_types';
+import { getPackagePolicyWithSourceMap, listArtifacts } from './source_maps';
 
 export async function registerFleetPolicyCallbacks({
   plugins,
@@ -31,7 +32,7 @@ export async function registerFleetPolicyCallbacks({
 
   // Registers a callback invoked when a policy is created to populate the APM
   // integration policy with pre-existing agent configurations
-  registerAgentConfigExternalCallback({
+  registerPackagePolicyExternalCallback({
     fleetPluginStart,
     callbackName: 'packagePolicyCreate',
     plugins,
@@ -42,7 +43,7 @@ export async function registerFleetPolicyCallbacks({
 
   // Registers a callback invoked when a policy is updated to populate the APM
   // integration policy with existing agent configurations
-  registerAgentConfigExternalCallback({
+  registerPackagePolicyExternalCallback({
     fleetPluginStart,
     callbackName: 'packagePolicyUpdate',
     plugins,
@@ -53,11 +54,11 @@ export async function registerFleetPolicyCallbacks({
 }
 
 type ExternalCallbackParams = Parameters<ExternalCallback[1]>;
-type PackagePolicy = ExternalCallbackParams[0];
+export type PackagePolicy = ExternalCallbackParams[0];
 type Context = ExternalCallbackParams[1];
 type Request = ExternalCallbackParams[2];
 
-function registerAgentConfigExternalCallback({
+function registerPackagePolicyExternalCallback({
   fleetPluginStart,
   callbackName,
   plugins,
@@ -91,16 +92,17 @@ function registerAgentConfigExternalCallback({
       ruleDataClient,
     });
     const agentConfigurations = await listConfigurations({ setup });
+    const artifacts = await listArtifacts({ fleetPluginStart });
     return getPackagePolicyWithAgentConfigurations(
-      packagePolicy,
+      getPackagePolicyWithSourceMap({ packagePolicy, artifacts }),
       agentConfigurations
     );
   };
 
   fleetPluginStart.registerExternalCallback(callbackName, callbackFn);
 }
 
-const APM_SERVER = 'apm-server';
+export const APM_SERVER = 'apm-server';
 
 // Immutable function applies the given package policy with a set of agent configurations
 export function getPackagePolicyWithAgentConfigurations(

x-pack/plugins/apm/server/lib/fleet/source_maps.test.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ArtifactSourceMap,
+  getPackagePolicyWithSourceMap,
+} from './source_maps';
+
+const packagePolicy = {
+  id: '123',
+  version: 'WzMxNDI2LDFd',
+  name: 'apm-1',
+  description: '',
+  namespace: 'default',
+  policy_id: '7a87c160-c961-11eb-81e2-f7327d61c92a',
+  enabled: true,
+  output_id: '',
+  inputs: [
+    {
+      policy_template: 'apmserver',
+      streams: [],
+      vars: {},
+      type: 'apm',
+      enabled: true,
+      compiled_input: {
+        'apm-server': {
+          capture_personal_data: true,
+          max_event_size: 307200,
+          api_key: { limit: 100, enabled: false },
+          default_service_environment: null,
+          host: 'localhost:8200',
+          kibana: { api_key: null },
+          secret_token: null,
+        },
+      },
+    },
+  ],
+  package: { name: 'apm', title: 'Elastic APM', version: '0.2.0' },
+  created_at: '2021-06-16T14:54:32.195Z',
+  created_by: 'elastic',
+};
+
+const artifacts = [
+  {
+    type: 'sourcemap',
+    identifier: 'service_name-1.0.0',
+    relative_url: '/api/fleet/artifacts/service_name-1.0.0/my-id-1',
+    body: {
+      serviceName: 'service_name',
+      serviceVersion: '1.0.0',
+      bundleFilepath: 'http://localhost:3000/static/js/main.chunk.js',
+      sourceMap: {
+        version: 3,
+        file: 'static/js/main.chunk.js',
+        sources: ['foo'],
+        sourcesContent: ['foo'],
+        mappings: 'foo',
+        sourceRoot: '',
+      },
+    },
+    created: '2021-06-16T15:03:55.049Z',
+    id: 'apm:service_name-1.0.0-my-id-1',
+    compressionAlgorithm: 'zlib',
+    decodedSha256: 'my-id-1',
+    decodedSize: 9440,
+    encodedSha256: 'sha123',
+    encodedSize: 2622,
+    encryptionAlgorithm: 'none',
+    packageName: 'apm',
+  },
+  {
+    type: 'sourcemap',
+    identifier: 'service_name-2.0.0',
+    relative_url: '/api/fleet/artifacts/service_name-2.0.0/my-id-2',
+    body: {
+      serviceName: 'service_name',
+      serviceVersion: '2.0.0',
+      bundleFilepath: 'http://localhost:3000/static/js/main.chunk.js',
+      sourceMap: {
+        version: 3,
+        file: 'static/js/main.chunk.js',
+        sources: ['foo'],
+        sourcesContent: ['foo'],
+        mappings: 'foo',
+        sourceRoot: '',
+      },
+    },
+    created: '2021-06-16T15:03:55.049Z',
+    id: 'apm:service_name-2.0.0-my-id-2',
+    compressionAlgorithm: 'zlib',
+    decodedSha256: 'my-id-2',
+    decodedSize: 9440,
+    encodedSha256: 'sha456',
+    encodedSize: 2622,
+    encryptionAlgorithm: 'none',
+    packageName: 'apm',
+  },
+] as ArtifactSourceMap[];
+
+describe('Source maps', () => {
+  describe('getPackagePolicyWithSourceMap', () => {
+    it('returns unchanged package policy when artifacts is empty', () => {
+      const updatedPackagePolicy = getPackagePolicyWithSourceMap({
+        packagePolicy,
+        artifacts: [],
+      });
+      expect(updatedPackagePolicy).toEqual(packagePolicy);
+    });
+    it('adds source maps into the package policy', () => {
+      const updatedPackagePolicy = getPackagePolicyWithSourceMap({
+        packagePolicy,
+        artifacts,
+      });
+      expect(updatedPackagePolicy.inputs[0].config).toEqual({
+        'apm-server': {
+          value: {
+            rum: {
+              source_mapping: {
+                metadata: [
+                  {
+                    'service.name': 'service_name',
+                    'service.version': '1.0.0',
+                    'bundle.filepath':
+                      'http://localhost:3000/static/js/main.chunk.js',
+                    'sourcemap.url':
+                      '/api/fleet/artifacts/service_name-1.0.0/my-id-1',
+                  },
+                  {
+                    'service.name': 'service_name',
+                    'service.version': '2.0.0',
+                    'bundle.filepath':
+                      'http://localhost:3000/static/js/main.chunk.js',
+                    'sourcemap.url':
+                      '/api/fleet/artifacts/service_name-2.0.0/my-id-2',
+                  },
+                ],
+              },
+            },
+          },
+        },
+      });
+    });
+  });
+});

x-pack/plugins/apm/server/lib/fleet/source_maps.ts

@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import * as t from 'io-ts';
+import {
+  CoreSetup,
+  CoreStart,
+  ElasticsearchClient,
+  SavedObjectsClientContract,
+} from 'kibana/server';
+import { promisify } from 'util';
+import { unzip } from 'zlib';
+import { Artifact } from '../../../../fleet/server';
+import { sourceMapRt } from '../../routes/source_maps';
+import { APMPluginStartDependencies } from '../../types';
+import { getApmPackgePolicies } from './get_apm_package_policies';
+import { APM_SERVER, PackagePolicy } from './register_fleet_policy_callbacks';
+
+export interface ApmArtifactBody {
+  serviceName: string;
+  serviceVersion: string;
+  bundleFilepath: string;
+  sourceMap: t.TypeOf<typeof sourceMapRt>;
+}
+export type ArtifactSourceMap = Omit<Artifact, 'body'> & {
+  body: ApmArtifactBody;
+};
+
+export type FleetPluginStart = NonNullable<APMPluginStartDependencies['fleet']>;
+
+const doUnzip = promisify(unzip);
+
+function decodeArtifacts(artifacts: Artifact[]): Promise<ArtifactSourceMap[]> {
+  return Promise.all(
+    artifacts.map(async (artifact) => {
+      const body = await doUnzip(Buffer.from(artifact.body, 'base64'));
+      return {
+        ...artifact,
+        body: JSON.parse(body.toString()) as ApmArtifactBody,
+      };
+    })
+  );
+}
+
+function getApmArtifactClient(fleetPluginStart: FleetPluginStart) {
+  return fleetPluginStart.createArtifactsClient('apm');
+}
+
+export async function listArtifacts({
+  fleetPluginStart,
+}: {
+  fleetPluginStart: FleetPluginStart;
+}) {
+  const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
+  const artifacts = await apmArtifactClient.listArtifacts({
+    kuery: 'type: sourcemap',
+  });
+
+  return decodeArtifacts(artifacts.items);
+}
+
+export async function createApmArtifact({
+  apmArtifactBody,
+  fleetPluginStart,
+}: {
+  apmArtifactBody: ApmArtifactBody;
+  fleetPluginStart: FleetPluginStart;
+}) {
+  const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
+  const identifier = `${apmArtifactBody.serviceName}-${apmArtifactBody.serviceVersion}`;
+
+  return apmArtifactClient.createArtifact({
+    type: 'sourcemap',
+    identifier,
+    content: JSON.stringify(apmArtifactBody),
+  });
+}
+
+export async function deleteApmArtifact({
+  id,
+  fleetPluginStart,
+}: {
+  id: string;
+  fleetPluginStart: FleetPluginStart;
+}) {
+  const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
+  return apmArtifactClient.deleteArtifact(id);
+}
+
+export function getPackagePolicyWithSourceMap({
+  packagePolicy,
+  artifacts,
+}: {
+  packagePolicy: PackagePolicy;
+  artifacts: ArtifactSourceMap[];
+}) {
+  if (!artifacts.length) {
+    return packagePolicy;
+  }
+  const [firstInput, ...restInputs] = packagePolicy.inputs;
+  return {
+    ...packagePolicy,
+    inputs: [
+      {
+        ...firstInput,
+        config: {
+          ...firstInput.config,
+          [APM_SERVER]: {
+            value: {
+              ...firstInput?.config?.[APM_SERVER].value,
+              rum: {
+                source_mapping: {
+                  metadata: artifacts.map((artifact) => ({
+                    'service.name': artifact.body.serviceName,
+                    'service.version': artifact.body.serviceVersion,
+                    'bundle.filepath': artifact.body.bundleFilepath,
+                    'sourcemap.url': artifact.relative_url,
+                  })),
+                },
+              },
+            },
+          },
+        },
+      },
+      ...restInputs,
+    ],
+  };
+}
+
+export async function updateSourceMapsOnFleetPolicies({
+  core,
+  fleetPluginStart,
+  savedObjectsClient,
+  elasticsearchClient,
+}: {
+  core: { setup: CoreSetup; start: () => Promise<CoreStart> };
+  fleetPluginStart: FleetPluginStart;
+  savedObjectsClient: SavedObjectsClientContract;
+  elasticsearchClient: ElasticsearchClient;
+}) {
+  const artifacts = await listArtifacts({ fleetPluginStart });
+
+  const apmFleetPolicies = await getApmPackgePolicies({
+    core,
+    fleetPluginStart,
+  });
+
+  return Promise.all(
+    apmFleetPolicies.items.map(async (item) => {
+      const {
+        id,
+        revision,
+        updated_at: updatedAt,
+        updated_by: updatedBy,
+        ...packagePolicy
+      } = item;
+
+      const updatedPackagePolicy = getPackagePolicyWithSourceMap({
+        packagePolicy,
+        artifacts,
+      });
+
+      await fleetPluginStart.packagePolicyService.update(
+        savedObjectsClient,
+        elasticsearchClient,
+        id,
+        updatedPackagePolicy
+      );
+    })
+  );
+}