From adb527691decb93c2fe3f23487db62a230ed0c93 Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 13 May 2021 00:43:48 -0700 Subject: [PATCH 1/9] first pass of basic osquery usage stats collection --- x-pack/plugins/osquery/kibana.json | 1 + x-pack/plugins/osquery/server/plugin.ts | 6 + .../routes/action/create_action_route.ts | 59 +++++---- .../osquery/server/routes/usage/index.ts | 8 ++ .../osquery/server/routes/usage/recorder.ts | 48 +++++++ x-pack/plugins/osquery/server/types.ts | 2 + .../plugins/osquery/server/usage/collector.ts | 39 ++++++ .../plugins/osquery/server/usage/constants.ts | 8 ++ .../plugins/osquery/server/usage/fetchers.ts | 121 ++++++++++++++++++ x-pack/plugins/osquery/server/usage/index.ts | 15 +++ x-pack/plugins/osquery/server/usage/types.ts | 59 +++++++++ .../schema/xpack_plugins.json | 47 +++++++ 12 files changed, 390 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugins/osquery/server/routes/usage/index.ts create mode 100644 x-pack/plugins/osquery/server/routes/usage/recorder.ts create mode 100644 x-pack/plugins/osquery/server/usage/collector.ts create mode 100644 x-pack/plugins/osquery/server/usage/constants.ts create mode 100644 x-pack/plugins/osquery/server/usage/fetchers.ts create mode 100644 x-pack/plugins/osquery/server/usage/index.ts create mode 100644 x-pack/plugins/osquery/server/usage/types.ts diff --git a/x-pack/plugins/osquery/kibana.json b/x-pack/plugins/osquery/kibana.json index 2c5c0708dc68e3..86a4d817de40e4 100644 --- a/x-pack/plugins/osquery/kibana.json +++ b/x-pack/plugins/osquery/kibana.json @@ -10,6 +10,7 @@ "kibanaVersion": "kibana", "optionalPlugins": [ "home", + "usageCollection", "lens" ], "requiredBundles": [ diff --git a/x-pack/plugins/osquery/server/plugin.ts b/x-pack/plugins/osquery/server/plugin.ts index 31f611c5f1d31d..ae779a9788238e 100644 --- a/x-pack/plugins/osquery/server/plugin.ts +++ b/x-pack/plugins/osquery/server/plugin.ts @@ -18,6 +18,7 @@ import { OsqueryPluginSetup, OsqueryPluginStart, SetupPlugins, StartPlugins } fr import { defineRoutes } from './routes'; import { osquerySearchStrategyProvider } from './search_strategy/osquery'; import { initSavedObjects } from './saved_objects'; +import { initUsageCollectors } from './usage'; import { OsqueryAppContext, OsqueryAppContextService } from './lib/osquery_app_context_services'; import { ConfigType } from './config'; @@ -48,6 +49,11 @@ export class OsqueryPlugin implements Plugin { diff --git a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts index 970a786b930b29..2c33f54711388c 100644 --- a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts +++ b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts @@ -18,6 +18,8 @@ import { CreateActionRequestBodySchema, } from '../../../common/schemas/routes/action/create_action_request_body_schema'; +import {getUsageRecorder} from '../usage' + export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => { router.post( { @@ -39,34 +41,45 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon osqueryContext, agentSelection ); - + const usageRecorder = getUsageRecorder() + usageRecorder.incrementCallCount('live_query') if (!selectedAgents.length) { + usageRecorder.incrementErrorCount('live_query') return response.badRequest({ body: new Error('No agents found for selection') }); } - const action = { - action_id: uuid.v4(), - '@timestamp': moment().toISOString(), - expiration: moment().add(1, 'days').toISOString(), - type: 'INPUT_ACTION', - input_type: 'osquery', - agents: selectedAgents, - data: { - id: uuid.v4(), - query: request.body.query, - }, - }; - const actionResponse = await esClient.index<{}, {}>({ - index: '.fleet-actions', - body: action, - }); + try { + const action = { + action_id: uuid.v4(), + '@timestamp': moment().toISOString(), + expiration: moment().add(1, 'days').toISOString(), + type: 'INPUT_ACTION', + input_type: 'osquery', + agents: selectedAgents, + data: { + id: uuid.v4(), + query: request.body.query, + }, + }; + const actionResponse = await esClient.index<{}, {}>({ + index: '.fleet-actions', + body: action, + }); - return response.ok({ - body: { - response: actionResponse, - actions: [action], - }, - }); + return response.ok({ + body: { + response: actionResponse, + actions: [action], + }, + }); + } catch (error) { + usageRecorder.incrementErrorCount('live_query') + return response.customError({ + statusCode: 500, + body: new Error(`Error occurred whlie processing ${error}`) , + }); + + } } ); }; diff --git a/x-pack/plugins/osquery/server/routes/usage/index.ts b/x-pack/plugins/osquery/server/routes/usage/index.ts new file mode 100644 index 00000000000000..f242f9636feee0 --- /dev/null +++ b/x-pack/plugins/osquery/server/routes/usage/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './recorder' \ No newline at end of file diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts new file mode 100644 index 00000000000000..59a175cfecc5c7 --- /dev/null +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface RouteUsageMetric { + call_count: number; + error_count: number; +} + +// TODO: use ES for this recording +class UsageRecorder { + private counts = new Map() + public incrementCallCount(route: string, increment: number = 1) { + const count = this.counts.get(route) ?? 0 + this.counts.set(route, count + increment) + } + public getCallCount(route: string): number { + return this.counts.get(route) ?? 0; + } + + private errors = new Map() + public incrementErrorCount(route: string, increment: number = 1) { + const count = this.errors.get(route) ?? 0 + this.errors.set(route, count + increment) + } + public getErrorCount(route: string): number { + return this.errors.get(route) ?? 0; + } + + public getRouteMetric(route: string): RouteUsageMetric { + return { + call_count: this.getCallCount(route), + error_count: this.getErrorCount(route) + } + } +} + +let usageRecorder: UsageRecorder; + +export const getUsageRecorder = (): UsageRecorder => { + if (usageRecorder == null) { + usageRecorder = new UsageRecorder() + } + return usageRecorder +} \ No newline at end of file diff --git a/x-pack/plugins/osquery/server/types.ts b/x-pack/plugins/osquery/server/types.ts index 1882e52074660e..667fba2bc98e24 100644 --- a/x-pack/plugins/osquery/server/types.ts +++ b/x-pack/plugins/osquery/server/types.ts @@ -11,6 +11,7 @@ import { PluginStart as DataPluginStart, } from '../../../../src/plugins/data/server'; import { FleetStartContract } from '../../fleet/server'; +import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; import { PluginSetupContract } from '../../features/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -19,6 +20,7 @@ export interface OsqueryPluginSetup {} export interface OsqueryPluginStart {} export interface SetupPlugins { + usageCollection?: UsageCollectionSetup; actions: ActionsPlugin['setup']; data: DataPluginSetup; features: PluginSetupContract; diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts new file mode 100644 index 00000000000000..82a2e4e8807332 --- /dev/null +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup } from '../../../../../src/core/server'; +import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; +import { getBeatUsage, getLiveQueryUsage } from './fetchers'; +import { CollectorDependencies, usageSchema } from './types'; + +export type RegisterCollector = (deps: CollectorDependencies) => void; +export async function getInternalSavedObjectsClient(core: CoreSetup) { + return core.getStartServices().then(async ([coreStart]) => { + return coreStart.savedObjects.createInternalRepository(); + }); +} + +export const registerCollector: RegisterCollector = ({ usageCollection }) => { + if (!usageCollection) { + return; + } + const collector = usageCollection.makeUsageCollector({ + type: 'osquery', + schema: usageSchema, + isReady: () => true, + fetch: async ({ esClient }: CollectorFetchContext): Promise => { + return { + beat_metrics: { + usage: await getBeatUsage(esClient), + }, + live_query_usage: getLiveQueryUsage(), + }; + }, + }); + + usageCollection.registerCollector(collector); +}; diff --git a/x-pack/plugins/osquery/server/usage/constants.ts b/x-pack/plugins/osquery/server/usage/constants.ts new file mode 100644 index 00000000000000..f463380ee0b331 --- /dev/null +++ b/x-pack/plugins/osquery/server/usage/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const METRICS_INDICES = 'logs-elastic_agent.osquerybeat*'; diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts new file mode 100644 index 00000000000000..31be97a769eebf --- /dev/null +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SingleBucketAggregate, + TopHitsAggregate, + ValueAggregate, +} from '@elastic/elasticsearch/api/types'; +import {getUsageRecorder} from '../routes/usage' +import { ElasticsearchClient } from '../../../../../src/core/server'; +import { METRICS_INDICES } from './constants'; + +export interface MetricEntry { + max?: number; + latest?: number; + avg?: number; +} + +export interface BeatMetricAggregation { + rss: MetricEntry; + cpuMs: MetricEntry; +} + +// TODO: pipe this through ES +export function getLiveQueryUsage() { + const usageRecorder = getUsageRecorder() + return usageRecorder.getRouteMetric('live_query') +} + +export async function getBeatUsage(esClient: ElasticsearchClient) { + // is there a better way to get these aggregates? + // needs a time window limit to make sure the reports are fresh + // XXX: these aggregates conflate agents, they should be broken out by id + // XXX: currently cpu is recorded as a duration rather than a load % + const { body: metricResponse } = await esClient.search({ + body: { + size: 0, + aggs: { + lastDay: { + filter: { + range: { + '@timestamp': { + gte: 'now-24h', + lte: 'now', + }, + }, + }, + aggs: { + latest: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + size: 1, + }, + }, + max_rss: { + max: { + field: 'monitoring.metrics.beat.memstats.rss', + }, + }, + avg_rss: { + avg: { + field: 'monitoring.metrics.beat.memstats.rss', + }, + }, + max_cpu: { + max: { + field: 'monitoring.metrics.beat.cpu.total.time.ms', + }, + }, + avg_cpu: { + avg: { + field: 'monitoring.metrics.beat.cpu.total.time.ms', + }, + }, + }, + }, + }, + }, + index: METRICS_INDICES, + }); + const lastDayAggs = metricResponse.aggregations?.lastDay as SingleBucketAggregate; + const result: BeatMetricAggregation = { + rss: {}, + cpuMs: {}, + }; + + // XXX: discrimating the union types gets hairy when attempting to genericize, figure out a fix! + if ('max_rss' in lastDayAggs) { + result.rss.max = (lastDayAggs.max_rss as ValueAggregate).value + } + + if ('avg_rss' in lastDayAggs) { + result.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value + } + + if ('max_cpu' in lastDayAggs) { + result.cpuMs.max = (lastDayAggs.max_cpu as ValueAggregate).value + } + + if ('avg_cpu' in lastDayAggs) { + result.cpuMs.avg = (lastDayAggs.max_cpu as ValueAggregate).value + } + + if ('latest' in lastDayAggs) { + const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring.metrics.beat; + result.cpuMs.latest = latest.cpu.total.time.ms; + result.rss.latest = latest.memstats.rss; + } + + return result; +} diff --git a/x-pack/plugins/osquery/server/usage/index.ts b/x-pack/plugins/osquery/server/usage/index.ts new file mode 100644 index 00000000000000..2982ae92a5bbe2 --- /dev/null +++ b/x-pack/plugins/osquery/server/usage/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CollectorDependencies } from './types'; +import { registerCollector } from './collector'; + +export type InitUsageCollectors = (deps: CollectorDependencies) => void; + +export const initUsageCollectors: InitUsageCollectors = (dependencies) => { + registerCollector(dependencies); +}; diff --git a/x-pack/plugins/osquery/server/usage/types.ts b/x-pack/plugins/osquery/server/usage/types.ts new file mode 100644 index 00000000000000..dd2121a5529801 --- /dev/null +++ b/x-pack/plugins/osquery/server/usage/types.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup } from 'src/core/server'; +import { OsqueryAppContext } from '../lib/osquery_app_context_services'; +import { MakeSchemaFrom } from '../../../../../src/plugins/usage_collection/server'; +import { SetupPlugins } from '../types'; + +export type CollectorDependencies = { + osqueryContext: OsqueryAppContext; + core: CoreSetup; +} & Pick; + +export const usageSchema: MakeSchemaFrom = { + query_metrics: { + live_query_usage: { + call_count: { + type: 'long', + }, + error_count: { + type: 'long', + }, + }, + }, + beat_metrics: { + usage: { + cpu: { + // TODO?: break out into system/user usage + latest: { + type: 'long', + }, + max: { + type: 'long', + }, + avg: { + type: 'long', + }, + }, + memory: { + rss: { + // TODO?: add a dimension on these for agent instance + latest: { + type: 'long', + }, + max: { + type: 'long', + }, + avg: { + type: 'long', + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 7c96dce3fac7f5..5313be5c9cc306 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3598,6 +3598,53 @@ } } }, + "osquery": { + "_meta": { + "description": "TODO: UPDATE THIS TO MATCH EXPORTED METRICS SCHEMA BEFORE PR" + }, + "properties": { + "query_metrics": { + "properties": { + "live_query_usage": { + "properties": { + "call_count": { + "type": "long", + "_meta": { + "description": "Number of ad hoc queries dispatched" + } + }, + "error_count": { + "type": "long", + "_meta": { + "description": "Number of ad hoc queries which resulted in an error" + } + } + } + } + } + }, + "beat_metrics": { + "properties": { + "usage": { + "properties": { + "cpu": { + "type": "long", + "_meta": { + "description": "CPU usage reported by osquery beat" + } + }, + "rss": { + "type": "long", + "_meta": { + "description": "Memory usage reported by osquery beat" + } + } + } + } + } + } + } + }, "ml": { "properties": { "alertRules": { From 821eee49f058c878555191bf359e808aa052183a Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 26 May 2021 21:06:07 -0700 Subject: [PATCH 2/9] updates, linting --- .../routes/action/create_action_route.ts | 13 ++--- .../osquery/server/routes/usage/index.ts | 2 +- .../osquery/server/routes/usage/recorder.ts | 58 +++++++++---------- .../plugins/osquery/server/usage/collector.ts | 2 + .../plugins/osquery/server/usage/fetchers.ts | 23 ++++---- 5 files changed, 50 insertions(+), 48 deletions(-) diff --git a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts index 2c33f54711388c..a44f7a803a9429 100644 --- a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts +++ b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts @@ -18,7 +18,7 @@ import { CreateActionRequestBodySchema, } from '../../../common/schemas/routes/action/create_action_request_body_schema'; -import {getUsageRecorder} from '../usage' +import { getUsageRecorder } from '../usage'; export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => { router.post( @@ -41,10 +41,10 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon osqueryContext, agentSelection ); - const usageRecorder = getUsageRecorder() - usageRecorder.incrementCallCount('live_query') + const usageRecorder = getUsageRecorder(); + usageRecorder.incrementCallCount('live_query'); if (!selectedAgents.length) { - usageRecorder.incrementErrorCount('live_query') + usageRecorder.incrementErrorCount('live_query'); return response.badRequest({ body: new Error('No agents found for selection') }); } @@ -73,12 +73,11 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon }, }); } catch (error) { - usageRecorder.incrementErrorCount('live_query') + usageRecorder.incrementErrorCount('live_query'); return response.customError({ statusCode: 500, - body: new Error(`Error occurred whlie processing ${error}`) , + body: new Error(`Error occurred whlie processing ${error}`), }); - } } ); diff --git a/x-pack/plugins/osquery/server/routes/usage/index.ts b/x-pack/plugins/osquery/server/routes/usage/index.ts index f242f9636feee0..c96f94643ef7e8 100644 --- a/x-pack/plugins/osquery/server/routes/usage/index.ts +++ b/x-pack/plugins/osquery/server/routes/usage/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export * from './recorder' \ No newline at end of file +export * from './recorder'; diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts index 59a175cfecc5c7..02903041dc97e6 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -6,43 +6,43 @@ */ export interface RouteUsageMetric { - call_count: number; - error_count: number; + call_count: number; + error_count: number; } // TODO: use ES for this recording class UsageRecorder { - private counts = new Map() - public incrementCallCount(route: string, increment: number = 1) { - const count = this.counts.get(route) ?? 0 - this.counts.set(route, count + increment) - } - public getCallCount(route: string): number { - return this.counts.get(route) ?? 0; - } + private counts = new Map(); + public incrementCallCount(route: string, increment = 1) { + const count = this.counts.get(route) ?? 0; + this.counts.set(route, count + increment); + } + public getCallCount(route: string): number { + return this.counts.get(route) ?? 0; + } - private errors = new Map() - public incrementErrorCount(route: string, increment: number = 1) { - const count = this.errors.get(route) ?? 0 - this.errors.set(route, count + increment) - } - public getErrorCount(route: string): number { - return this.errors.get(route) ?? 0; - } + private errors = new Map(); + public incrementErrorCount(route: string, increment = 1) { + const count = this.errors.get(route) ?? 0; + this.errors.set(route, count + increment); + } + public getErrorCount(route: string): number { + return this.errors.get(route) ?? 0; + } - public getRouteMetric(route: string): RouteUsageMetric { - return { - call_count: this.getCallCount(route), - error_count: this.getErrorCount(route) - } - } + public getRouteMetric(route: string): RouteUsageMetric { + return { + call_count: this.getCallCount(route), + error_count: this.getErrorCount(route), + }; + } } let usageRecorder: UsageRecorder; export const getUsageRecorder = (): UsageRecorder => { - if (usageRecorder == null) { - usageRecorder = new UsageRecorder() - } - return usageRecorder -} \ No newline at end of file + if (usageRecorder == null) { + usageRecorder = new UsageRecorder(); + } + return usageRecorder; +}; diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index 82a2e4e8807332..f0599e7bfe6de6 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -21,10 +21,12 @@ export const registerCollector: RegisterCollector = ({ usageCollection }) => { if (!usageCollection) { return; } + // eslint-disable-next-line @typescript-eslint/no-explicit-any const collector = usageCollection.makeUsageCollector({ type: 'osquery', schema: usageSchema, isReady: () => true, + // eslint-disable-next-line @typescript-eslint/no-explicit-any fetch: async ({ esClient }: CollectorFetchContext): Promise => { return { beat_metrics: { diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index 31be97a769eebf..a7edbc5619bf6b 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -6,11 +6,11 @@ */ import { - SingleBucketAggregate, - TopHitsAggregate, - ValueAggregate, + SingleBucketAggregate, + TopHitsAggregate, + ValueAggregate, } from '@elastic/elasticsearch/api/types'; -import {getUsageRecorder} from '../routes/usage' +import { getUsageRecorder } from '../routes/usage'; import { ElasticsearchClient } from '../../../../../src/core/server'; import { METRICS_INDICES } from './constants'; @@ -27,8 +27,8 @@ export interface BeatMetricAggregation { // TODO: pipe this through ES export function getLiveQueryUsage() { - const usageRecorder = getUsageRecorder() - return usageRecorder.getRouteMetric('live_query') + const usageRecorder = getUsageRecorder(); + return usageRecorder.getRouteMetric('live_query'); } export async function getBeatUsage(esClient: ElasticsearchClient) { @@ -96,23 +96,24 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { // XXX: discrimating the union types gets hairy when attempting to genericize, figure out a fix! if ('max_rss' in lastDayAggs) { - result.rss.max = (lastDayAggs.max_rss as ValueAggregate).value + result.rss.max = (lastDayAggs.max_rss as ValueAggregate).value; } if ('avg_rss' in lastDayAggs) { - result.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value + result.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value; } if ('max_cpu' in lastDayAggs) { - result.cpuMs.max = (lastDayAggs.max_cpu as ValueAggregate).value + result.cpuMs.max = (lastDayAggs.max_cpu as ValueAggregate).value; } if ('avg_cpu' in lastDayAggs) { - result.cpuMs.avg = (lastDayAggs.max_cpu as ValueAggregate).value + result.cpuMs.avg = (lastDayAggs.max_cpu as ValueAggregate).value; } if ('latest' in lastDayAggs) { - const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring.metrics.beat; + const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring + .metrics.beat; result.cpuMs.latest = latest.cpu.total.time.ms; result.rss.latest = latest.memstats.rss; } From cab8d8d8120b99607a6b141401e91e1ea9f5b606 Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 3 Jun 2021 02:05:45 -0700 Subject: [PATCH 3/9] updated exported metrics --- x-pack/plugins/osquery/common/types.ts | 3 +- .../routes/action/create_action_route.ts | 9 +- .../osquery/server/routes/usage/recorder.ts | 78 ++++++----- .../routes/usage/saved_object_mappings.ts | 28 ++++ .../plugins/osquery/server/saved_objects.ts | 3 + .../plugins/osquery/server/usage/collector.ts | 22 ++- .../plugins/osquery/server/usage/fetchers.ts | 128 ++++++++++++++++-- x-pack/plugins/osquery/server/usage/types.ts | 32 ++++- 8 files changed, 243 insertions(+), 60 deletions(-) create mode 100644 x-pack/plugins/osquery/server/routes/usage/saved_object_mappings.ts diff --git a/x-pack/plugins/osquery/common/types.ts b/x-pack/plugins/osquery/common/types.ts index 11c418a51fc7cb..111c05d68dd021 100644 --- a/x-pack/plugins/osquery/common/types.ts +++ b/x-pack/plugins/osquery/common/types.ts @@ -7,7 +7,8 @@ export const savedQuerySavedObjectType = 'osquery-saved-query'; export const packSavedObjectType = 'osquery-pack'; -export type SavedObjectType = 'osquery-saved-query' | 'osquery-pack'; +export const usageMetricSavedObjectType = 'osquery-usage-metric'; +export type SavedObjectType = 'osquery-saved-query' | 'osquery-pack' | 'osquery-usage-metric'; /** * This makes any optional property the same as Required would but also has the diff --git a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts index a44f7a803a9429..86e871f0411608 100644 --- a/x-pack/plugins/osquery/server/routes/action/create_action_route.ts +++ b/x-pack/plugins/osquery/server/routes/action/create_action_route.ts @@ -18,7 +18,7 @@ import { CreateActionRequestBodySchema, } from '../../../common/schemas/routes/action/create_action_request_body_schema'; -import { getUsageRecorder } from '../usage'; +import { incrementCount } from '../usage'; export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => { router.post( @@ -41,10 +41,9 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon osqueryContext, agentSelection ); - const usageRecorder = getUsageRecorder(); - usageRecorder.incrementCallCount('live_query'); + incrementCount(soClient, 'live_query'); if (!selectedAgents.length) { - usageRecorder.incrementErrorCount('live_query'); + incrementCount(soClient, 'live_query', 'errors'); return response.badRequest({ body: new Error('No agents found for selection') }); } @@ -73,7 +72,7 @@ export const createActionRoute = (router: IRouter, osqueryContext: OsqueryAppCon }, }); } catch (error) { - usageRecorder.incrementErrorCount('live_query'); + incrementCount(soClient, 'live_query', 'errors'); return response.customError({ statusCode: 500, body: new Error(`Error occurred whlie processing ${error}`), diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts index 02903041dc97e6..6ba5507528791c 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -5,44 +5,56 @@ * 2.0. */ +import { SavedObjectsClientContract } from 'kibana/server'; +import { usageMetricSavedObjectType } from '../../../common/types'; + export interface RouteUsageMetric { - call_count: number; - error_count: number; + queries: number; + errors: number; } -// TODO: use ES for this recording -class UsageRecorder { - private counts = new Map(); - public incrementCallCount(route: string, increment = 1) { - const count = this.counts.get(route) ?? 0; - this.counts.set(route, count + increment); - } - public getCallCount(route: string): number { - return this.counts.get(route) ?? 0; - } +export type RouteString = 'live_query'; - private errors = new Map(); - public incrementErrorCount(route: string, increment = 1) { - const count = this.errors.get(route) ?? 0; - this.errors.set(route, count + increment); - } - public getErrorCount(route: string): number { - return this.errors.get(route) ?? 0; - } +export async function createMetricObjects(soClient: SavedObjectsClientContract) { + const res = await Promise.allSettled( + ['live_query'].map(async (route) => { + try { + await soClient.get(usageMetricSavedObjectType, route); + } catch (e) { + await soClient.create( + usageMetricSavedObjectType, + { + errors: 0, + count: 0, + }, + { + id: route, + } + ); + } + }) + ); + return !res.some((e) => e.status === 'rejected'); +} - public getRouteMetric(route: string): RouteUsageMetric { - return { - call_count: this.getCallCount(route), - error_count: this.getErrorCount(route), - }; - } +export async function getCount(soClient: SavedObjectsClientContract, route: RouteString) { + return await soClient.get(usageMetricSavedObjectType, route); } -let usageRecorder: UsageRecorder; +export async function incrementCount( + soClient: SavedObjectsClientContract, + route: RouteString, + key: 'errors' | 'count' = 'count', + increment = 1 +) { + const metric = await soClient.get<{ count: number; errors: number }>( + usageMetricSavedObjectType, + route + ); + metric.attributes[key] += increment; + await soClient.update(usageMetricSavedObjectType, route, metric.attributes); +} -export const getUsageRecorder = (): UsageRecorder => { - if (usageRecorder == null) { - usageRecorder = new UsageRecorder(); - } - return usageRecorder; -}; +export async function getRouteMetric(soClient: SavedObjectsClientContract, route: RouteString) { + return (await getCount(soClient, route)).attributes; +} diff --git a/x-pack/plugins/osquery/server/routes/usage/saved_object_mappings.ts b/x-pack/plugins/osquery/server/routes/usage/saved_object_mappings.ts new file mode 100644 index 00000000000000..92709f92d9e5f0 --- /dev/null +++ b/x-pack/plugins/osquery/server/routes/usage/saved_object_mappings.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsType } from '../../../../../../src/core/server'; + +import { usageMetricSavedObjectType } from '../../../common/types'; + +export const usageMetricSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + count: { + type: 'long', + }, + errors: { + type: 'long', + }, + }, +}; + +export const usageMetricType: SavedObjectsType = { + name: usageMetricSavedObjectType, + hidden: false, + namespaceType: 'single', + mappings: usageMetricSavedObjectMappings, +}; diff --git a/x-pack/plugins/osquery/server/saved_objects.ts b/x-pack/plugins/osquery/server/saved_objects.ts index c6be1098cedb2e..9f93ea5ccd6de1 100644 --- a/x-pack/plugins/osquery/server/saved_objects.ts +++ b/x-pack/plugins/osquery/server/saved_objects.ts @@ -9,6 +9,7 @@ import { CoreSetup } from '../../../../src/core/server'; import { OsqueryAppContext } from './lib/osquery_app_context_services'; import { savedQueryType, packType } from './lib/saved_query/saved_object_mappings'; +import { usageMetricType } from './routes/usage/saved_object_mappings'; const types = [savedQueryType, packType]; @@ -20,6 +21,8 @@ export const initSavedObjects = ( ) => { const config = osqueryContext.config(); + savedObjects.registerType(usageMetricType); + if (config.savedQueries) { savedObjects.registerType(savedQueryType); } diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index f0599e7bfe6de6..53da9df0d98744 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -5,9 +5,10 @@ * 2.0. */ -import { CoreSetup } from '../../../../../src/core/server'; +import { CoreSetup, SavedObjectsClientContract } from '../../../../../src/core/server'; import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; -import { getBeatUsage, getLiveQueryUsage } from './fetchers'; +import { createMetricObjects } from '../routes/usage'; +import { getBeatUsage, getLiveQueryUsage, getPolicyLevelUsage } from './fetchers'; import { CollectorDependencies, usageSchema } from './types'; export type RegisterCollector = (deps: CollectorDependencies) => void; @@ -17,7 +18,7 @@ export async function getInternalSavedObjectsClient(core: CoreSetup) { }); } -export const registerCollector: RegisterCollector = ({ usageCollection }) => { +export const registerCollector: RegisterCollector = ({ core, osqueryContext, usageCollection }) => { if (!usageCollection) { return; } @@ -25,14 +26,25 @@ export const registerCollector: RegisterCollector = ({ usageCollection }) => { const collector = usageCollection.makeUsageCollector({ type: 'osquery', schema: usageSchema, - isReady: () => true, + isReady: async () => { + const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); + const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; + return await createMetricObjects(savedObjectsClient); + }, // eslint-disable-next-line @typescript-eslint/no-explicit-any fetch: async ({ esClient }: CollectorFetchContext): Promise => { + const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); + const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; return { beat_metrics: { usage: await getBeatUsage(esClient), }, - live_query_usage: getLiveQueryUsage(), + live_query_usage: await getLiveQueryUsage(savedObjectsClient, esClient), + ...(await getPolicyLevelUsage( + esClient, + savedObjectsClient, + osqueryContext.service.getPackagePolicyService() + )), }; }, }); diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index a7edbc5619bf6b..400de4c3c062fa 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -10,8 +10,11 @@ import { TopHitsAggregate, ValueAggregate, } from '@elastic/elasticsearch/api/types'; -import { getUsageRecorder } from '../routes/usage'; -import { ElasticsearchClient } from '../../../../../src/core/server'; +import { PackagePolicyServiceInterface } from '../../../fleet/server'; +import { getRouteMetric } from '../routes/usage'; +import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../src/core/server'; +import { ListResult, PackagePolicy, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../fleet/common'; +import { OSQUERY_INTEGRATION_NAME } from '../../common'; import { METRICS_INDICES } from './constants'; export interface MetricEntry { @@ -25,17 +28,121 @@ export interface BeatMetricAggregation { cpuMs: MetricEntry; } -// TODO: pipe this through ES -export function getLiveQueryUsage() { - const usageRecorder = getUsageRecorder(); - return usageRecorder.getRouteMetric('live_query'); +interface PolicyLevelUsage { + scheduled_queries?: {}; + agent_info?: {}; +} + +export async function getPolicyLevelUsage( + esClient: ElasticsearchClient, + soClient: SavedObjectsClientContract, + packagePolicyService?: PackagePolicyServiceInterface +): Promise { + if (!packagePolicyService) { + return {}; + } + const packagePolicies = await packagePolicyService.list(soClient, { + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${OSQUERY_INTEGRATION_NAME}`, + perPage: 10_000, + }); + + const result: PolicyLevelUsage = { + scheduled_queries: getScheduledQueryUsage(packagePolicies), + // TODO: figure out how to support dynamic keys in metrics + // packageVersions: getPackageVersions(packagePolicies), + }; + const agentResponse = await esClient.search({ + body: { + size: 0, + aggs: { + policied: { + filter: { + terms: { + policy_id: packagePolicies.items.map((p) => p.policy_id), + }, + }, + }, + }, + }, + index: '.fleet-agents', + }); + if (agentResponse.statusCode === 200) { + result.agent_info = { + enrolled: (agentResponse.body.aggregations?.policied as SingleBucketAggregate).doc_count, + }; + } + return result; +} + +export function getPackageVersions(packagePolicies: ListResult) { + return packagePolicies.items.reduce((acc, item) => { + if (item.package) { + acc[item.package.version] = (acc[item.package.version] ?? 0) + 1; + } + return acc; + }, {} as { [version: string]: number }); +} + +interface ScheduledQueryUsageMetrics { + queryGroups: { + total: number; + empty: number; + }; +} + +export function getScheduledQueryUsage(packagePolicies: ListResult) { + return packagePolicies.items.reduce( + (acc, item) => { + ++acc.queryGroups.total; + if (item.inputs.length === 0) { + ++acc.queryGroups.empty; + } + return acc; + }, + { + queryGroups: { + total: 0, + empty: 0, + }, + } as ScheduledQueryUsageMetrics + ); +} + +export async function getLiveQueryUsage( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient +) { + const { body: metricResponse } = await esClient.search({ + body: { + size: 0, + aggs: { + queries: { + filter: { + term: { + input_type: 'osquery', + }, + }, + }, + }, + }, + index: '.fleet-actions', + }); + const esQueries = (metricResponse.aggregations?.queries as SingleBucketAggregate).doc_count; + const result = { + session: await getRouteMetric(soClient, 'live_query'), + // getting error stats out of ES is difficult due to a lack of error info on .fleet-actions + // and a lack of indexable osquery specific info on .fleet-actions-results + cumulative: { + queries: esQueries, + }, + }; + + return result; } export async function getBeatUsage(esClient: ElasticsearchClient) { - // is there a better way to get these aggregates? - // needs a time window limit to make sure the reports are fresh - // XXX: these aggregates conflate agents, they should be broken out by id - // XXX: currently cpu is recorded as a duration rather than a load % + // ???: currently cpu is recorded as a duration rather than a load %. this might make it difficult to reason about the metrics in parallel systems. + // ???: these metrics would be more actionable with some facets of them (e.g. platform, architecture, etc) const { body: metricResponse } = await esClient.search({ body: { size: 0, @@ -94,7 +201,6 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { cpuMs: {}, }; - // XXX: discrimating the union types gets hairy when attempting to genericize, figure out a fix! if ('max_rss' in lastDayAggs) { result.rss.max = (lastDayAggs.max_rss as ValueAggregate).value; } diff --git a/x-pack/plugins/osquery/server/usage/types.ts b/x-pack/plugins/osquery/server/usage/types.ts index dd2121a5529801..597cf291d613d3 100644 --- a/x-pack/plugins/osquery/server/usage/types.ts +++ b/x-pack/plugins/osquery/server/usage/types.ts @@ -15,16 +15,38 @@ export type CollectorDependencies = { core: CoreSetup; } & Pick; +// eslint-disable-next-line @typescript-eslint/no-explicit-any export const usageSchema: MakeSchemaFrom = { - query_metrics: { - live_query_usage: { - call_count: { + live_query_usage: { + // session is an awkward name for this + session: { + count: { type: 'long', }, - error_count: { + errors: { type: 'long', }, }, + cumulative: { + queries: { + type: 'long', + }, + }, + }, + scheduled_queries: { + queryGroups: { + total: { + type: 'long', + }, + empty: { + type: 'long', + }, + }, + }, + agent_info: { + enrolled: { + type: 'long', + }, }, beat_metrics: { usage: { @@ -42,7 +64,7 @@ export const usageSchema: MakeSchemaFrom = { }, memory: { rss: { - // TODO?: add a dimension on these for agent instance + // ???: add a dimension on these for agent instance latest: { type: 'long', }, From 838ae1d3166d58e295d268cbc16ea284940dfabc Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 3 Jun 2021 22:49:50 -0700 Subject: [PATCH 4/9] clean up comments, add description fields to metric fields --- .../plugins/osquery/server/usage/fetchers.ts | 2 - x-pack/plugins/osquery/server/usage/types.ts | 39 +++++- .../schema/xpack_plugins.json | 121 ++++++++++++------ 3 files changed, 121 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index 400de4c3c062fa..fd6e0ca379dd1e 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -141,8 +141,6 @@ export async function getLiveQueryUsage( } export async function getBeatUsage(esClient: ElasticsearchClient) { - // ???: currently cpu is recorded as a duration rather than a load %. this might make it difficult to reason about the metrics in parallel systems. - // ???: these metrics would be more actionable with some facets of them (e.g. platform, architecture, etc) const { body: metricResponse } = await esClient.search({ body: { size: 0, diff --git a/x-pack/plugins/osquery/server/usage/types.ts b/x-pack/plugins/osquery/server/usage/types.ts index 597cf291d613d3..689e2c46a81332 100644 --- a/x-pack/plugins/osquery/server/usage/types.ts +++ b/x-pack/plugins/osquery/server/usage/types.ts @@ -18,18 +18,26 @@ export type CollectorDependencies = { // eslint-disable-next-line @typescript-eslint/no-explicit-any export const usageSchema: MakeSchemaFrom = { live_query_usage: { - // session is an awkward name for this session: { count: { type: 'long', + _meta: { + description: 'Number of osquery action requests', + }, }, errors: { type: 'long', + _meta: { + description: 'Number of osquery action requests that resulted in errors', + }, }, }, cumulative: { queries: { type: 'long', + _meta: { + description: 'Number of osquery actions stored in Elasticsearch', + }, }, }, }, @@ -37,42 +45,67 @@ export const usageSchema: MakeSchemaFrom = { queryGroups: { total: { type: 'long', + _meta: { + description: 'Number of osquery policies/query groups', + }, }, empty: { type: 'long', + _meta: { + description: 'Number of empty osquery policies/query groups', + }, }, }, }, agent_info: { enrolled: { type: 'long', + _meta: { + description: 'Number of agents enrolled in a policy with an osquery integration', + }, }, }, beat_metrics: { usage: { cpu: { - // TODO?: break out into system/user usage latest: { type: 'long', + _meta: { + description: 'Latest cpu usage sample in ms', + }, }, max: { type: 'long', + _meta: { + description: 'Max cpu usage sample over 24 hours in ms', + }, }, avg: { type: 'long', + _meta: { + description: 'Mean cpu usage over 24 hours in ms', + }, }, }, memory: { rss: { - // ???: add a dimension on these for agent instance latest: { type: 'long', + _meta: { + description: 'Latest resident set size sample', + }, }, max: { type: 'long', + _meta: { + description: 'Max resident set size sample over 24 hours', + }, }, avg: { type: 'long', + _meta: { + description: 'Mean resident set size sample over 24 hours', + }, }, }, }, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 5313be5c9cc306..6ed8290e90ac99 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3599,45 +3599,94 @@ } }, "osquery": { - "_meta": { - "description": "TODO: UPDATE THIS TO MATCH EXPORTED METRICS SCHEMA BEFORE PR" - }, - "properties": { - "query_metrics": { - "properties": { - "live_query_usage": { - "properties": { - "call_count": { - "type": "long", - "_meta": { - "description": "Number of ad hoc queries dispatched" - } - }, - "error_count": { - "type": "long", - "_meta": { - "description": "Number of ad hoc queries which resulted in an error" - } - } - } + "live_query_usage": { + "session": { + "count": { + "type": "long", + "_meta": { + "description": "Number of osquery action requests" + } + }, + "errors": { + "type": "long", + "_meta": { + "description": "Number of osquery action requests that resulted in errors" } } }, - "beat_metrics": { - "properties": { - "usage": { - "properties": { - "cpu": { - "type": "long", - "_meta": { - "description": "CPU usage reported by osquery beat" - } - }, - "rss": { - "type": "long", - "_meta": { - "description": "Memory usage reported by osquery beat" - } + "cumulative": { + "queries": { + "type": "long", + "_meta": { + "description": "Number of osquery actions stored in Elasticsearch" + } + } + } + }, + "scheduled_queries": { + "queryGroups": { + "total": { + "type": "long", + "_meta": { + "description": "Number of osquery policies/query groups" + } + }, + "empty": { + "type": "long", + "_meta": { + "description": "Number of empty osquery policies/query groups" + } + } + } + }, + "agent_info": { + "enrolled": { + "type": "long", + "_meta": { + "description": "Number of agents enrolled in a policy with an osquery integration" + } + } + }, + "beat_metrics": { + "usage": { + "cpu": { + "latest": { + "type": "long", + "_meta": { + "description": "Latest cpu usage sample in ms" + } + }, + "max": { + "type": "long", + "_meta": { + "description": "Max cpu usage sample over 24 hours in ms" + } + }, + "avg": { + "type": "long", + "_meta": { + "description": "Mean cpu usage over 24 hours in ms" + } + } + }, + "memory": { + "rss": { + "latest": { + "type": "long", + "_meta": { + "description": "Latest resident set size sample" + } + }, + "max": { + "type": "long", + "_meta": { + "description": "Max resident set size sample over 24 hours" + } + }, + "avg": { + "type": "long", + "_meta": { + "description": "Mean resident set size sample over 24 hours" } } } From d01db684a859c94ac416067a3ad226e2d670b104 Mon Sep 17 00:00:00 2001 From: bryan Date: Fri, 4 Jun 2021 05:51:33 -0700 Subject: [PATCH 5/9] reworked types --- .../osquery/server/routes/usage/recorder.ts | 3 +- .../plugins/osquery/server/usage/collector.ts | 5 +- .../plugins/osquery/server/usage/fetchers.ts | 55 +++++++++---------- x-pack/plugins/osquery/server/usage/types.ts | 47 ++++++++++++++++ 4 files changed, 76 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts index 6ba5507528791c..b0d0f6ed00b750 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -7,6 +7,7 @@ import { SavedObjectsClientContract } from 'kibana/server'; import { usageMetricSavedObjectType } from '../../../common/types'; +import { LiveQuerySessionUsage } from '../../usage/types'; export interface RouteUsageMetric { queries: number; @@ -38,7 +39,7 @@ export async function createMetricObjects(soClient: SavedObjectsClientContract) } export async function getCount(soClient: SavedObjectsClientContract, route: RouteString) { - return await soClient.get(usageMetricSavedObjectType, route); + return await soClient.get(usageMetricSavedObjectType, route); } export async function incrementCount( diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index 53da9df0d98744..67fef22af25213 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -9,7 +9,7 @@ import { CoreSetup, SavedObjectsClientContract } from '../../../../../src/core/s import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; import { createMetricObjects } from '../routes/usage'; import { getBeatUsage, getLiveQueryUsage, getPolicyLevelUsage } from './fetchers'; -import { CollectorDependencies, usageSchema } from './types'; +import { CollectorDependencies, usageSchema, UsageData } from './types'; export type RegisterCollector = (deps: CollectorDependencies) => void; export async function getInternalSavedObjectsClient(core: CoreSetup) { @@ -31,8 +31,7 @@ export const registerCollector: RegisterCollector = ({ core, osqueryContext, usa const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; return await createMetricObjects(savedObjectsClient); }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - fetch: async ({ esClient }: CollectorFetchContext): Promise => { + fetch: async ({ esClient }: CollectorFetchContext): Promise => { const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; return { diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index fd6e0ca379dd1e..c3ab365e3dca4d 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -16,21 +16,11 @@ import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../ import { ListResult, PackagePolicy, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../fleet/common'; import { OSQUERY_INTEGRATION_NAME } from '../../common'; import { METRICS_INDICES } from './constants'; - -export interface MetricEntry { - max?: number; - latest?: number; - avg?: number; -} - -export interface BeatMetricAggregation { - rss: MetricEntry; - cpuMs: MetricEntry; -} +import { AgentInfo, BeatMetricsUsage, LiveQueryUsage } from './types'; interface PolicyLevelUsage { - scheduled_queries?: {}; - agent_info?: {}; + scheduled_queries?: ScheduledQueryUsageMetrics; + agent_info?: AgentInfo; } export async function getPolicyLevelUsage( @@ -66,9 +56,10 @@ export async function getPolicyLevelUsage( }, index: '.fleet-agents', }); - if (agentResponse.statusCode === 200) { + const policied = agentResponse.body.aggregations?.policied as SingleBucketAggregate; + if (policied && typeof policied.doc_count === 'number') { result.agent_info = { - enrolled: (agentResponse.body.aggregations?.policied as SingleBucketAggregate).doc_count, + enrolled: policied.doc_count, }; } return result; @@ -127,15 +118,17 @@ export async function getLiveQueryUsage( }, index: '.fleet-actions', }); - const esQueries = (metricResponse.aggregations?.queries as SingleBucketAggregate).doc_count; - const result = { + const result: LiveQueryUsage = { session: await getRouteMetric(soClient, 'live_query'), + }; + const esQueries = metricResponse.aggregations?.queries as SingleBucketAggregate; + if (esQueries && typeof esQueries.doc_count === 'number') { // getting error stats out of ES is difficult due to a lack of error info on .fleet-actions // and a lack of indexable osquery specific info on .fleet-actions-results - cumulative: { - queries: esQueries, - }, - }; + result.cumulative = { + queries: esQueries.doc_count, + }; + } return result; } @@ -194,32 +187,34 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { index: METRICS_INDICES, }); const lastDayAggs = metricResponse.aggregations?.lastDay as SingleBucketAggregate; - const result: BeatMetricAggregation = { - rss: {}, - cpuMs: {}, + const result: BeatMetricsUsage = { + memory: { + rss: {}, + }, + cpu: {}, }; if ('max_rss' in lastDayAggs) { - result.rss.max = (lastDayAggs.max_rss as ValueAggregate).value; + result.memory.rss.max = (lastDayAggs.max_rss as ValueAggregate).value; } if ('avg_rss' in lastDayAggs) { - result.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value; + result.memory.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value; } if ('max_cpu' in lastDayAggs) { - result.cpuMs.max = (lastDayAggs.max_cpu as ValueAggregate).value; + result.cpu.max = (lastDayAggs.max_cpu as ValueAggregate).value; } if ('avg_cpu' in lastDayAggs) { - result.cpuMs.avg = (lastDayAggs.max_cpu as ValueAggregate).value; + result.cpu.avg = (lastDayAggs.max_cpu as ValueAggregate).value; } if ('latest' in lastDayAggs) { const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring .metrics.beat; - result.cpuMs.latest = latest.cpu.total.time.ms; - result.rss.latest = latest.memstats.rss; + result.cpu.latest = latest.cpu.total.time.ms; + result.memory.rss.latest = latest.memstats.rss; } return result; diff --git a/x-pack/plugins/osquery/server/usage/types.ts b/x-pack/plugins/osquery/server/usage/types.ts index 689e2c46a81332..24168b854e723f 100644 --- a/x-pack/plugins/osquery/server/usage/types.ts +++ b/x-pack/plugins/osquery/server/usage/types.ts @@ -15,6 +15,53 @@ export type CollectorDependencies = { core: CoreSetup; } & Pick; +export interface LiveQuerySessionUsage { + count: number; + errors: number; +} +export interface LiveQueryCumulativeUsage { + queries: number; +} + +export interface LiveQueryUsage { + session: LiveQuerySessionUsage; + cumulative?: LiveQueryCumulativeUsage; +} + +export interface ScheduledQueryUsage { + queryGroups: { + total: number; + empty: number; + }; +} +export interface AgentInfo { + enrolled: number; +} + +export interface MetricEntry { + max?: number; + latest?: number; + avg?: number; +} + +export interface BeatMetricsUsage { + cpu: MetricEntry; + memory: { + rss: MetricEntry; + }; +} + +export interface BeatMetrics { + usage: BeatMetricsUsage; +} + +export interface UsageData { + live_query_usage?: LiveQueryUsage; + scheduled_queries?: ScheduledQueryUsage; + agent_info?: AgentInfo; + beat_metrics?: BeatMetrics; +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any export const usageSchema: MakeSchemaFrom = { live_query_usage: { From d6b904de255c9c27d1c844bbab8481706053b63c Mon Sep 17 00:00:00 2001 From: bryan Date: Fri, 4 Jun 2021 06:05:40 -0700 Subject: [PATCH 6/9] actually use the updated types --- .../plugins/osquery/server/usage/collector.ts | 3 +- x-pack/plugins/osquery/server/usage/types.ts | 3 +- .../schema/xpack_plugins.json | 216 ++++++++++-------- 3 files changed, 122 insertions(+), 100 deletions(-) diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index 67fef22af25213..7aa35fe87fd24e 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -22,8 +22,7 @@ export const registerCollector: RegisterCollector = ({ core, osqueryContext, usa if (!usageCollection) { return; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const collector = usageCollection.makeUsageCollector({ + const collector = usageCollection.makeUsageCollector({ type: 'osquery', schema: usageSchema, isReady: async () => { diff --git a/x-pack/plugins/osquery/server/usage/types.ts b/x-pack/plugins/osquery/server/usage/types.ts index 24168b854e723f..6ad7295438a1f9 100644 --- a/x-pack/plugins/osquery/server/usage/types.ts +++ b/x-pack/plugins/osquery/server/usage/types.ts @@ -62,8 +62,7 @@ export interface UsageData { beat_metrics?: BeatMetrics; } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const usageSchema: MakeSchemaFrom = { +export const usageSchema: MakeSchemaFrom = { live_query_usage: { session: { count: { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 6ed8290e90ac99..a4f590b4b9cb98 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -3598,102 +3598,6 @@ } } }, - "osquery": { - "live_query_usage": { - "session": { - "count": { - "type": "long", - "_meta": { - "description": "Number of osquery action requests" - } - }, - "errors": { - "type": "long", - "_meta": { - "description": "Number of osquery action requests that resulted in errors" - } - } - }, - "cumulative": { - "queries": { - "type": "long", - "_meta": { - "description": "Number of osquery actions stored in Elasticsearch" - } - } - } - }, - "scheduled_queries": { - "queryGroups": { - "total": { - "type": "long", - "_meta": { - "description": "Number of osquery policies/query groups" - } - }, - "empty": { - "type": "long", - "_meta": { - "description": "Number of empty osquery policies/query groups" - } - } - } - }, - "agent_info": { - "enrolled": { - "type": "long", - "_meta": { - "description": "Number of agents enrolled in a policy with an osquery integration" - } - } - }, - "beat_metrics": { - "usage": { - "cpu": { - "latest": { - "type": "long", - "_meta": { - "description": "Latest cpu usage sample in ms" - } - }, - "max": { - "type": "long", - "_meta": { - "description": "Max cpu usage sample over 24 hours in ms" - } - }, - "avg": { - "type": "long", - "_meta": { - "description": "Mean cpu usage over 24 hours in ms" - } - } - }, - "memory": { - "rss": { - "latest": { - "type": "long", - "_meta": { - "description": "Latest resident set size sample" - } - }, - "max": { - "type": "long", - "_meta": { - "description": "Max resident set size sample over 24 hours" - } - }, - "avg": { - "type": "long", - "_meta": { - "description": "Mean resident set size sample over 24 hours" - } - } - } - } - } - } - }, "ml": { "properties": { "alertRules": { @@ -3827,6 +3731,126 @@ } } }, + "osquery": { + "properties": { + "live_query_usage": { + "properties": { + "session": { + "properties": { + "count": { + "type": "long", + "_meta": { + "description": "Number of osquery action requests" + } + }, + "errors": { + "type": "long", + "_meta": { + "description": "Number of osquery action requests that resulted in errors" + } + } + } + }, + "cumulative": { + "properties": { + "queries": { + "type": "long", + "_meta": { + "description": "Number of osquery actions stored in Elasticsearch" + } + } + } + } + } + }, + "scheduled_queries": { + "properties": { + "queryGroups": { + "properties": { + "total": { + "type": "long", + "_meta": { + "description": "Number of osquery policies/query groups" + } + }, + "empty": { + "type": "long", + "_meta": { + "description": "Number of empty osquery policies/query groups" + } + } + } + } + } + }, + "agent_info": { + "properties": { + "enrolled": { + "type": "long", + "_meta": { + "description": "Number of agents enrolled in a policy with an osquery integration" + } + } + } + }, + "beat_metrics": { + "properties": { + "usage": { + "properties": { + "cpu": { + "properties": { + "latest": { + "type": "long", + "_meta": { + "description": "Latest cpu usage sample in ms" + } + }, + "max": { + "type": "long", + "_meta": { + "description": "Max cpu usage sample over 24 hours in ms" + } + }, + "avg": { + "type": "long", + "_meta": { + "description": "Mean cpu usage over 24 hours in ms" + } + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "latest": { + "type": "long", + "_meta": { + "description": "Latest resident set size sample" + } + }, + "max": { + "type": "long", + "_meta": { + "description": "Max resident set size sample over 24 hours" + } + }, + "avg": { + "type": "long", + "_meta": { + "description": "Mean resident set size sample over 24 hours" + } + } + } + } + } + } + } + } + } + } + } + }, "reporting": { "properties": { "csv": { From eb20cf82851fb2dbd309eee6228e674b2e468504 Mon Sep 17 00:00:00 2001 From: bryan Date: Mon, 7 Jun 2021 00:44:13 -0700 Subject: [PATCH 7/9] added tests around the route usage recoder functions --- .../server/routes/usage/recorder.test.ts | 135 ++++++++++++++++++ .../osquery/server/routes/usage/recorder.ts | 16 ++- 2 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/osquery/server/routes/usage/recorder.test.ts diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts new file mode 100644 index 00000000000000..aa5f550234fcb0 --- /dev/null +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '../../../../../../src/core/server/mocks'; + +import { usageMetricSavedObjectType } from '../../../common/types'; + +import { + CounterValue, + createMetricObjects, + getRouteMetric, + incrementCount, + RouteString, + routeStrings, +} from './recorder'; + +const savedObjectsClient = savedObjectsClientMock.create(); + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function checkGetCalls(calls: any[]) { + expect(calls.length).toEqual(routeStrings.length); + for (let i = 0; i < routeStrings.length; ++i) { + expect(calls[i]).toEqual([usageMetricSavedObjectType, routeStrings[i]]); + } +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function checkCreateCalls(calls: any[], expectedCallRoutes: string[] = routeStrings) { + expect(calls.length).toEqual(expectedCallRoutes.length); + for (let i = 0; i < expectedCallRoutes.length; ++i) { + expect(calls[i][0]).toEqual(usageMetricSavedObjectType); + expect(calls[i][2].id).toEqual(expectedCallRoutes[i]); + } +} + +describe('Usage metric recorder', () => { + describe('Metric initalizer', () => { + const get = savedObjectsClient.get as jest.Mock; + const create = savedObjectsClient.create as jest.Mock; + afterEach(() => { + get.mockClear(); + create.mockClear(); + }); + it('should seed route metrics objects', async () => { + get.mockRejectedValueOnce('stub value'); + create.mockReturnValueOnce('stub value'); + const result = await createMetricObjects(savedObjectsClient); + checkGetCalls(get.mock.calls); + checkCreateCalls(create.mock.calls); + expect(result).toBe(true); + }); + + it('should handle previously seeded objects properly', async () => { + get.mockReturnValueOnce('stub value'); + create.mockRejectedValueOnce('stub value'); + const result = await createMetricObjects(savedObjectsClient); + checkGetCalls(get.mock.calls); + checkCreateCalls(create.mock.calls, []); + expect(result).toBe(true); + }); + + it('should report failure to create the metrics object', async () => { + get.mockRejectedValueOnce('stub value'); + create.mockRejectedValueOnce('stub value'); + const result = await createMetricObjects(savedObjectsClient); + checkGetCalls(get.mock.calls); + checkCreateCalls(create.mock.calls); + expect(result).toBe(false); + }); + }); + + describe('Incrementation', () => { + let counterMap: { [key: string]: CounterValue }; + const get = savedObjectsClient.get as jest.Mock; + const update = savedObjectsClient.update as jest.Mock; + update.mockImplementation( + async (objectType: string, route: RouteString, newVal: CounterValue) => { + counterMap[`${objectType}-${route}`] = newVal; + } + ); + get.mockImplementation(async (objectType: string, route: RouteString) => ({ + attributes: counterMap[`${objectType}-${route}`], + })); + beforeEach(() => { + counterMap = routeStrings.reduce((acc, route) => { + acc[`${usageMetricSavedObjectType}-${route}`] = { + count: 0, + errors: 0, + }; + return acc; + }, {} as { [key: string]: CounterValue }); + get.mockClear(); + update.mockClear(); + }); + it('should increment the route counter', async () => { + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 0, + errors: 0, + }); + await incrementCount(savedObjectsClient, 'live_query'); + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 1, + errors: 0, + }); + }); + + it('should allow incrementing the error counter', async () => { + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 0, + errors: 0, + }); + await incrementCount(savedObjectsClient, 'live_query', 'errors'); + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 0, + errors: 1, + }); + }); + + it('should allow adjustment of the increment', async () => { + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 0, + errors: 0, + }); + await incrementCount(savedObjectsClient, 'live_query', 'count', 2); + expect(await getRouteMetric(savedObjectsClient, 'live_query')).toEqual({ + count: 2, + errors: 0, + }); + }); + }); +}); diff --git a/x-pack/plugins/osquery/server/routes/usage/recorder.ts b/x-pack/plugins/osquery/server/routes/usage/recorder.ts index b0d0f6ed00b750..9f5e7cd1d56e0a 100644 --- a/x-pack/plugins/osquery/server/routes/usage/recorder.ts +++ b/x-pack/plugins/osquery/server/routes/usage/recorder.ts @@ -16,9 +16,11 @@ export interface RouteUsageMetric { export type RouteString = 'live_query'; +export const routeStrings: RouteString[] = ['live_query']; + export async function createMetricObjects(soClient: SavedObjectsClientContract) { const res = await Promise.allSettled( - ['live_query'].map(async (route) => { + routeStrings.map(async (route) => { try { await soClient.get(usageMetricSavedObjectType, route); } catch (e) { @@ -42,16 +44,18 @@ export async function getCount(soClient: SavedObjectsClientContract, route: Rout return await soClient.get(usageMetricSavedObjectType, route); } +export interface CounterValue { + count: number; + errors: number; +} + export async function incrementCount( soClient: SavedObjectsClientContract, route: RouteString, - key: 'errors' | 'count' = 'count', + key: keyof CounterValue = 'count', increment = 1 ) { - const metric = await soClient.get<{ count: number; errors: number }>( - usageMetricSavedObjectType, - route - ); + const metric = await soClient.get(usageMetricSavedObjectType, route); metric.attributes[key] += increment; await soClient.update(usageMetricSavedObjectType, route, metric.attributes); } From be2e6f0e62b9ca5935cad40e0dd62b4465a1ac3e Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 9 Jun 2021 23:38:54 -0700 Subject: [PATCH 8/9] review comments --- x-pack/plugins/osquery/server/usage/collector.ts | 8 +++----- x-pack/plugins/osquery/server/usage/fetchers.ts | 6 ++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/osquery/server/usage/collector.ts b/x-pack/plugins/osquery/server/usage/collector.ts index 7aa35fe87fd24e..9b690be6df0f16 100644 --- a/x-pack/plugins/osquery/server/usage/collector.ts +++ b/x-pack/plugins/osquery/server/usage/collector.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CoreSetup, SavedObjectsClientContract } from '../../../../../src/core/server'; +import { CoreSetup, SavedObjectsClient } from '../../../../../src/core/server'; import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; import { createMetricObjects } from '../routes/usage'; import { getBeatUsage, getLiveQueryUsage, getPolicyLevelUsage } from './fetchers'; @@ -26,13 +26,11 @@ export const registerCollector: RegisterCollector = ({ core, osqueryContext, usa type: 'osquery', schema: usageSchema, isReady: async () => { - const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); - const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; + const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core)); return await createMetricObjects(savedObjectsClient); }, fetch: async ({ esClient }: CollectorFetchContext): Promise => { - const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); - const savedObjectsClient = (internalSavedObjectsClient as unknown) as SavedObjectsClientContract; + const savedObjectsClient = new SavedObjectsClient(await getInternalSavedObjectsClient(core)); return { beat_metrics: { usage: await getBeatUsage(esClient), diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index c3ab365e3dca4d..b55f0ee6917572 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -213,8 +213,10 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { if ('latest' in lastDayAggs) { const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring .metrics.beat; - result.cpu.latest = latest.cpu.total.time.ms; - result.memory.rss.latest = latest.memstats.rss; + if (latest) { + result.cpu.latest = latest.cpu.total.time.ms; + result.memory.rss.latest = latest.memstats.rss; + } } return result; From bcb61381224f0440e755391146489c8ada1ba091 Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 10 Jun 2021 01:03:48 -0700 Subject: [PATCH 9/9] update aggregate types --- .../plugins/osquery/server/usage/fetchers.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/osquery/server/usage/fetchers.ts b/x-pack/plugins/osquery/server/usage/fetchers.ts index b55f0ee6917572..5f5b282331bece 100644 --- a/x-pack/plugins/osquery/server/usage/fetchers.ts +++ b/x-pack/plugins/osquery/server/usage/fetchers.ts @@ -6,9 +6,9 @@ */ import { - SingleBucketAggregate, - TopHitsAggregate, - ValueAggregate, + AggregationsSingleBucketAggregate, + AggregationsTopHitsAggregate, + AggregationsValueAggregate, } from '@elastic/elasticsearch/api/types'; import { PackagePolicyServiceInterface } from '../../../fleet/server'; import { getRouteMetric } from '../routes/usage'; @@ -56,7 +56,7 @@ export async function getPolicyLevelUsage( }, index: '.fleet-agents', }); - const policied = agentResponse.body.aggregations?.policied as SingleBucketAggregate; + const policied = agentResponse.body.aggregations?.policied as AggregationsSingleBucketAggregate; if (policied && typeof policied.doc_count === 'number') { result.agent_info = { enrolled: policied.doc_count, @@ -121,7 +121,7 @@ export async function getLiveQueryUsage( const result: LiveQueryUsage = { session: await getRouteMetric(soClient, 'live_query'), }; - const esQueries = metricResponse.aggregations?.queries as SingleBucketAggregate; + const esQueries = metricResponse.aggregations?.queries as AggregationsSingleBucketAggregate; if (esQueries && typeof esQueries.doc_count === 'number') { // getting error stats out of ES is difficult due to a lack of error info on .fleet-actions // and a lack of indexable osquery specific info on .fleet-actions-results @@ -186,7 +186,7 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { }, index: METRICS_INDICES, }); - const lastDayAggs = metricResponse.aggregations?.lastDay as SingleBucketAggregate; + const lastDayAggs = metricResponse.aggregations?.lastDay as AggregationsSingleBucketAggregate; const result: BeatMetricsUsage = { memory: { rss: {}, @@ -195,24 +195,24 @@ export async function getBeatUsage(esClient: ElasticsearchClient) { }; if ('max_rss' in lastDayAggs) { - result.memory.rss.max = (lastDayAggs.max_rss as ValueAggregate).value; + result.memory.rss.max = (lastDayAggs.max_rss as AggregationsValueAggregate).value; } if ('avg_rss' in lastDayAggs) { - result.memory.rss.avg = (lastDayAggs.max_rss as ValueAggregate).value; + result.memory.rss.avg = (lastDayAggs.max_rss as AggregationsValueAggregate).value; } if ('max_cpu' in lastDayAggs) { - result.cpu.max = (lastDayAggs.max_cpu as ValueAggregate).value; + result.cpu.max = (lastDayAggs.max_cpu as AggregationsValueAggregate).value; } if ('avg_cpu' in lastDayAggs) { - result.cpu.avg = (lastDayAggs.max_cpu as ValueAggregate).value; + result.cpu.avg = (lastDayAggs.max_cpu as AggregationsValueAggregate).value; } if ('latest' in lastDayAggs) { - const latest = (lastDayAggs.latest as TopHitsAggregate).hits.hits[0]?._source?.monitoring - .metrics.beat; + const latest = (lastDayAggs.latest as AggregationsTopHitsAggregate).hits.hits[0]?._source + ?.monitoring.metrics.beat; if (latest) { result.cpu.latest = latest.cpu.total.time.ms; result.memory.rss.latest = latest.memstats.rss;