From 744821cfa6ca4f19c3c85d8ff629cff6b25c85fd Mon Sep 17 00:00:00 2001 From: Steven Stern Date: Thu, 15 May 2025 13:57:49 -0700 Subject: [PATCH 1/3] Add a new method to track custom events for AI operations --- .../__tests__/LDAIConfigTrackerImpl.test.ts | 88 +++++++++++++++++++ .../server-ai/src/LDAIConfigTrackerImpl.ts | 14 +++ .../src/api/config/LDAIConfigTracker.ts | 31 +++++++ 3 files changed, 133 insertions(+) diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 483d097889..e87a08d49a 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -113,6 +113,80 @@ it('tracks negative feedback', () => { ); }); +it('tracks custom event with data', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); + const customData = { property: 'value', count: 42 }; + tracker.trackCustomEvent('test-event', customData); + + expect(mockTrack).toHaveBeenCalledWith( + '$ld:ai:custom:test-event', + testContext, + { configKey, variationKey, version, custom: customData }, + undefined, + ); +}); + +it('tracks custom event with metric value', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); + tracker.trackCustomEvent('test-event', undefined, 123.45); + + expect(mockTrack).toHaveBeenCalledWith( + '$ld:ai:custom:test-event', + testContext, + { configKey, variationKey, version, custom: undefined }, + 123.45, + ); +}); + +it('tracks custom event with both data and metric value', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); + const customData = { property: 'value' }; + tracker.trackCustomEvent('test-event', customData, 123.45); + + expect(mockTrack).toHaveBeenCalledWith( + '$ld:ai:custom:test-event', + testContext, + { configKey, variationKey, version, custom: customData }, + 123.45, + ); +}); + +it('tracks custom event with no data or metric value', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + configKey, + variationKey, + version, + testContext, + ); + tracker.trackCustomEvent('test-event'); + + expect(mockTrack).toHaveBeenCalledWith( + '$ld:ai:custom:test-event', + testContext, + { configKey, variationKey, version, custom: undefined }, + undefined, + ); +}); + it('tracks success', () => { const tracker = new LDAIConfigTrackerImpl( mockLdClient, @@ -429,6 +503,8 @@ it('summarizes tracked metrics', () => { }); tracker.trackFeedback({ kind: LDFeedbackKind.Positive }); tracker.trackSuccess(); + tracker.trackCustomEvent('event1', { test: 'data' }); + tracker.trackCustomEvent('event2', undefined, 42); const summary = tracker.getSummary(); @@ -443,6 +519,18 @@ it('summarizes tracked metrics', () => { kind: 'positive', }, success: true, + customEvents: [ + { + name: 'event1', + data: { test: 'data' }, + metricValue: undefined + }, + { + name: 'event2', + data: undefined, + metricValue: 42 + } + ] }); }); diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index 0972a5eee5..dda949f2c3 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -62,6 +62,20 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { } } + trackCustomEvent(eventName: string, data?: any, metricValue?: number): void { + if (!this._trackedMetrics.customEvents) { + this._trackedMetrics.customEvents = []; + } + this._trackedMetrics.customEvents.push({ + name: eventName, + data, + metricValue, + }); + + const trackData = { ...this._getTrackData(), custom: data }; + this._ldClient.track(`$ld:ai:custom:${eventName}`, this._context, trackData, metricValue); + } + trackSuccess(): void { this._trackedMetrics.success = true; this._ldClient.track('$ld:ai:generation', this._context, this._getTrackData(), 1); diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts index 2f92aa9386..39db7d122b 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts @@ -28,6 +28,26 @@ export interface LDAIMetricSummary { * Time to first token for this generation. */ timeToFirstTokenMs?: number; + + /** + * Custom events tracked for this operation. + */ + customEvents?: Array<{ + /** + * The name of the custom event. + */ + name: string; + + /** + * Optional additional information associated with the event. + */ + data?: any; + + /** + * Optional numeric value for analytics. + */ + metricValue?: number; + }>; } /** @@ -67,6 +87,17 @@ export interface LDAIConfigTracker { */ trackFeedback(feedback: { kind: LDFeedbackKind }): void; + /** + * Track a custom event related to AI operations. + * + * This method allows for tracking arbitrary custom events specific to AI operations. + * + * @param eventName The name of the custom event. + * @param data Optional additional information to associate with the event. + * @param metricValue A numeric value that can be used for analytics. Can be omitted if not needed. + */ + trackCustomEvent(eventName: string, data?: any, metricValue?: number): void; + /** * Track the time to first token for this generation. * From 28d2d546eff88bf0a5cd808abfa12c345dac8e12 Mon Sep 17 00:00:00 2001 From: Steven Stern Date: Thu, 15 May 2025 15:15:00 -0700 Subject: [PATCH 2/3] Fix lint errors --- .../sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index e87a08d49a..869cd5fb36 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -523,14 +523,14 @@ it('summarizes tracked metrics', () => { { name: 'event1', data: { test: 'data' }, - metricValue: undefined + metricValue: undefined, }, { name: 'event2', data: undefined, - metricValue: 42 - } - ] + metricValue: 42, + }, + ], }); }); From 15041f0a5a31d36bc0dc5f37b642d8e72e9ea9c4 Mon Sep 17 00:00:00 2001 From: Steven Stern Date: Thu, 15 May 2025 16:08:00 -0700 Subject: [PATCH 3/3] More lint fixes --- packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts index 39db7d122b..713a84928c 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts @@ -28,7 +28,7 @@ export interface LDAIMetricSummary { * Time to first token for this generation. */ timeToFirstTokenMs?: number; - + /** * Custom events tracked for this operation. */ @@ -37,12 +37,12 @@ export interface LDAIMetricSummary { * The name of the custom event. */ name: string; - + /** * Optional additional information associated with the event. */ data?: any; - + /** * Optional numeric value for analytics. */