AD model performance benchmark

This PR adds a HCAD model performance benchmark so that we can compare model performance across versions. Regarding benchmark data, we randomly generated synthetic data with known anomalies inserted throughout the signal. In particular, these are one/two/four dimensional data where each dimension is a noisy cosine wave. Anomalies are inserted into one dimension with 0.003 probability. Anomalies across each dimension can be independent or dependent. We have approximately 5000 observations per data set. The data set is generated using the same random seed so the result is comparable across versions. We also backported opensearch-project#600 so that we can capture the performance data in CI output. Testing done: * added unit tests to run the benchmark. Signed-off-by: Kaituo Li <kaituo@amazon.com>
kaituo · Nov 15, 2022 · 3aadcad · 3aadcad
1 parent e7c2b05
commit 3aadcad
Show file tree

Hide file tree

Showing 11 changed files with 1,175 additions and 638 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,33 @@
+name: Run AD benchmark
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [17]
+      fail-fast: false
+
+    name: Run Anomaly detection model performance benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true
+          ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -45,6 +45,9 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and
 8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
 9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
 10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
+11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output
+12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output
+
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 

diff --git a/build.gradle b/build.gradle
@@ -38,10 +38,14 @@ buildscript {
                 '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + plugin_no_snapshot + '.zip'
         anomaly_detection_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
                 '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-anomaly-detection-' + plugin_no_snapshot + '.zip'
-        bwcOpenDistroADDownload = 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/' +
-                'opendistro-anomaly-detection/opendistro-anomaly-detection-1.13.0.0.zip'
-        bwcOpenDistroJSDownload = 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/' +
-                'opendistro-job-scheduler/opendistro-job-scheduler-1.13.0.0.zip'
+        bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' +
+                'opensearch-anomaly-detection-1.1.0.0.zip'
+        bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/' +
+                'opensearch-job-scheduler-1.1.0.0.zip'
+        // gradle build won't print logs during test by default unless there is a failure.
+        // It is useful to record intermediately information like prediction precision and recall.
+        // This option turn on log printing during tests. 
+        printLogs = "true" == System.getProperty("test.logs", "false")
     }
 
     repositories {
@@ -179,6 +183,12 @@ test {
     }
     include '**/*Tests.class'
     systemProperty 'tests.security.manager', 'false'
+
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.ml.HCADModelPerfTests"
+        }
+    }
 }
 
 task integTest(type: RestIntegTestTask) {
@@ -224,6 +234,12 @@ integTest {
         }
     }
 
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.e2e.SingleStreamModelPerfIT"
+        }
+    }
+
     // The 'doFirst' delays till execution time.
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
@@ -244,6 +260,12 @@ integTest {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
     }
 
+    if (printLogs) {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
 }
 
 testClusters.integTest {
@@ -331,7 +353,7 @@ task integTestRemote(type: RestIntegTestTask) {
     }
 }
 
-String bwcVersion = "1.13.0.0"
+String bwcVersion = "1.1.0.0"
 String baseName = "adBwcCluster"
 String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
 String bwcJobSchedulerPath = bwcFilePath + "job-scheduler/"
@@ -341,7 +363,7 @@ String bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["7.10.2", opensearch_version]
+            versions = ["1.1.0", opensearch_version]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override
@@ -353,7 +375,7 @@ String bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
                                 project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
                             }
                             project.mkdir bwcJobSchedulerPath + bwcVersion
-                            ant.get(src: bwcOpenDistroJSDownload,
+                            ant.get(src: bwcOpenSearchJSDownload,
                                     dest: bwcJobSchedulerPath + bwcVersion,
                                     httpusecaches: false)
                             return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
@@ -371,7 +393,7 @@ String bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
                                 project.delete(files("$project.rootDir/$bwcFilePath/anomaly-detection/$bwcVersion"))
                             }
                             project.mkdir bwcAnomalyDetectionPath + bwcVersion
-                            ant.get(src: bwcOpenDistroADDownload,
+                            ant.get(src: bwcOpenSearchADDownload,
                                     dest: bwcAnomalyDetectionPath + bwcVersion,
                                     httpusecaches: false)
                             return fileTree(bwcAnomalyDetectionPath + bwcVersion).getSingleFile()
@@ -757,3 +779,13 @@ validateNebulaPom.enabled = false
 tasks.withType(licenseHeaders.class) {
     additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
 }
+
+// show test results so that we can record information like precion/recall results of correctness testing.
+if (printLogs) {
+    test {
+        testLogging {
+            showStandardStreams = true
+            outputs.upToDateWhen {false}
+        }
+    }
+}
diff --git a/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java
@@ -70,6 +70,7 @@ public class ADBackwardsCompatibilityIT extends OpenSearchRestTestCase {
     private List<String> runningRealtimeDetectors;
     private List<String> historicalDetectors;
 
+    @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
@@ -143,8 +144,8 @@ public void testBackwardsCompatibility() throws Exception {
                         categoryFieldSize
                     );
                     assertEquals(totalDocsPerCategory * categoryFieldSize * 2, getDocCountOfIndex(client(), dataIndexName));
-                    Assert.assertTrue(pluginNames.contains("opendistro-anomaly-detection"));
-                    Assert.assertTrue(pluginNames.contains("opendistro-job-scheduler"));
+                    Assert.assertTrue(pluginNames.contains("opensearch-anomaly-detection"));
+                    Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler"));
 
                     // Create single entity detector and start realtime job
                     createRealtimeAnomalyDetectorsAndStart(SINGLE_ENTITY_DETECTOR);
@@ -403,8 +404,7 @@ private List<String> createRealtimeAnomalyDetectorsAndStart(ADRestTestUtils.Dete
         }
     }
 
-    @SuppressWarnings("unchecked")
-    private void createHistoricalAnomalyDetectorsAndStart() throws Exception {
+    private List<String> createHistoricalAnomalyDetectorsAndStart() throws Exception {
         // only support single entity for historical detector
         Response historicalSingleFlowDetectorResponse = createAnomalyDetector(
             client(),
@@ -418,16 +418,7 @@ private void createHistoricalAnomalyDetectorsAndStart() throws Exception {
             null,
             true
         );
-        List<String> historicalDetectorStartResult = startAnomalyDetector(historicalSingleFlowDetectorResponse, true);
-        String detectorId = historicalDetectorStartResult.get(0);
-        String taskId = historicalDetectorStartResult.get(1);
-        deleteRunningDetector(detectorId);
-        waitUntilTaskDone(client(), detectorId);
-        List<ADTask> adTasks = searchLatestAdTaskOfDetector(client(), detectorId, ADTaskType.HISTORICAL.name());
-        assertEquals(1, adTasks.size());
-        assertEquals(taskId, adTasks.get(0).getTaskId());
-        int adResultCount = countADResultOfDetector(client(), detectorId, taskId);
-        assertTrue(adResultCount > 0);
+        return startAnomalyDetector(historicalSingleFlowDetectorResponse, true);
     }
 
     private void deleteRunningDetector(String detectorId) {