From 8ac65ac975ad97214a3c4870ecc3b605e33b0f3a Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 7 Aug 2024 21:32:05 +0530 Subject: [PATCH] rename count to value_count Signed-off-by: Bharathwaj G --- .../index/mapper/StarTreeMapperIT.java | 4 +- .../compositeindex/datacube/MetricStat.java | 2 +- .../startree/StarTreeIndexSettings.java | 2 +- .../aggregators/CountValueAggregator.java | 2 +- .../aggregators/ValueAggregatorFactory.java | 4 +- .../util/ByteArrayBackedBitsetTests.java | 56 +++++++++++++++++++ .../StarTreeDocValuesFormatTests.java | 2 +- .../CountValueAggregatorTests.java | 2 +- .../MetricAggregatorInfoTests.java | 8 +-- .../builder/AbstractStarTreeBuilderTests.java | 22 ++++---- .../index/mapper/StarTreeMapperTests.java | 6 +- 11 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 1cabb8b617ce3..6f5b4bba481dd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -267,7 +267,7 @@ public void testValidCompositeIndex() { assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( MetricStat.AVG, - MetricStat.COUNT, + MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.MAX, MetricStat.MIN @@ -351,7 +351,7 @@ public void testUpdateIndexWhenMappingIsSame() { assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( MetricStat.AVG, - MetricStat.COUNT, + MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.MAX, MetricStat.MIN diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java index fbde296b15f7e..df3b2229d2c5b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -17,7 +17,7 @@ */ @ExperimentalApi public enum MetricStat { - COUNT("count"), + VALUE_COUNT("value_count"), AVG("avg"), SUM("sum"), MIN("min"), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java index a2ac545be3cc9..6535f8ed11da3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -97,7 +97,7 @@ public class StarTreeIndexSettings { "index.composite_index.star_tree.field.default.metrics", Arrays.asList( MetricStat.AVG.toString(), - MetricStat.COUNT.toString(), + MetricStat.VALUE_COUNT.toString(), MetricStat.SUM.toString(), MetricStat.MAX.toString(), MetricStat.MIN.toString() diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index ed159ee2efb7b..8ea7d2b2dfc48 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -26,7 +26,7 @@ public CountValueAggregator(StarTreeNumericType starTreeNumericType) { @Override public MetricStat getAggregationType() { - return MetricStat.COUNT; + return MetricStat.VALUE_COUNT; } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 240bbd37a53ee..a7ed38cb234f7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -30,7 +30,7 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType, Sta // other metric types (count, min, max, avg) will be supported in the future case SUM: return new SumValueAggregator(starTreeNumericType); - case COUNT: + case VALUE_COUNT: return new CountValueAggregator(starTreeNumericType); default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); @@ -48,7 +48,7 @@ public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationT // other metric types (count, min, max, avg) will be supported in the future case SUM: return SumValueAggregator.VALUE_AGGREGATOR_TYPE; - case COUNT: + case VALUE_COUNT: return CountValueAggregator.VALUE_AGGREGATOR_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); diff --git a/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java b/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java new file mode 100644 index 0000000000000..6750a9e48f033 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/util/ByteArrayBackedBitsetTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RandomAccessInput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Tests for {@link ByteArrayBackedBitset} + */ +public class ByteArrayBackedBitsetTests extends OpenSearchTestCase { + public void testWriteAndReadNullBitSets() throws IOException { + for (int k = 0; k < 10; k++) { + int randomArraySize = randomIntBetween(2, 300); + int randomIndex1 = randomIntBetween(0, (randomArraySize - 1) * 8); + int randomIndex2 = randomIntBetween(0, (randomArraySize - 1) * 8); + testWriteAndReadBitset(randomArraySize, randomIndex1, randomIndex2); + } + } + + private static void testWriteAndReadBitset(int randomArraySize, int randomIndex1, int randomIndex2) throws IOException { + ByteArrayBackedBitset bitset = new ByteArrayBackedBitset(randomArraySize); + Path basePath = createTempDir("OffHeapTests"); + FSDirectory fsDirectory = FSDirectory.open(basePath); + String TEST_FILE = "test_file"; + IndexOutput indexOutput = fsDirectory.createOutput(TEST_FILE, IOContext.DEFAULT); + bitset.set(randomIndex1); + bitset.set(randomIndex2); + bitset.write(indexOutput); + indexOutput.close(); + + IndexInput in = fsDirectory.openInput(TEST_FILE, IOContext.DEFAULT); + RandomAccessInput randomAccessInput = in.randomAccessSlice(0, in.length()); + ByteArrayBackedBitset bitset1 = new ByteArrayBackedBitset(randomAccessInput, 0, randomArraySize); + for (int i = 0; i < (randomArraySize * 8); i++) { + if (randomIndex1 == i || randomIndex2 == i) { + assertTrue(bitset1.get(i)); + } else { + assertFalse(bitset1.get(i)); + } + } + } +} diff --git a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java index 049d91bc42d9c..6fa88215cad48 100644 --- a/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java +++ b/server/src/test/java/org/opensearch/index/codec/composite/datacube/startree/StarTreeDocValuesFormatTests.java @@ -126,7 +126,7 @@ private XContentBuilder getExpandedMapping(String dim, String metric) throws IOE b.field("name", "field"); b.startArray("stats"); b.value("sum"); - b.value("count"); // TODO : THIS TEST FAILS. + b.value("value_count"); b.endArray(); b.endObject(); b.endArray(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java index 8e6e9e9974646..169ad2acd01c9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -16,7 +16,7 @@ public class CountValueAggregatorTests extends OpenSearchTestCase { private final CountValueAggregator aggregator = new CountValueAggregator(StarTreeNumericType.LONG); public void testGetAggregationType() { - assertEquals(MetricStat.COUNT.getTypeName(), aggregator.getAggregationType().getTypeName()); + assertEquals(MetricStat.VALUE_COUNT.getTypeName(), aggregator.getAggregationType().getTypeName()); } public void testGetAggregatedValueType() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 73e6aeb44cfd7..62671ffa03b82 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -27,12 +27,12 @@ public void testConstructor() { public void testCountStarConstructor() { MetricAggregatorInfo pair = new MetricAggregatorInfo( - MetricStat.COUNT, + MetricStat.VALUE_COUNT, "anything", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE ); - assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals(MetricStat.VALUE_COUNT, pair.getMetricStat()); assertEquals("anything", pair.getField()); } @@ -62,7 +62,7 @@ public void testEquals() { assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) + new MetricAggregatorInfo(MetricStat.VALUE_COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE) ); assertNotEquals( pair1, @@ -100,7 +100,7 @@ public void testCompareTo() { IndexNumericFieldData.NumericType.DOUBLE ); MetricAggregatorInfo pair3 = new MetricAggregatorInfo( - MetricStat.COUNT, + MetricStat.VALUE_COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 131d7444ff91c..30d67a4e7b0a3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -92,7 +92,7 @@ public void setup() throws IOException { metrics = List.of( new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM)), - new Metric("field6", List.of(MetricStat.COUNT)) + new Metric("field6", List.of(MetricStat.VALUE_COUNT)) ); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); @@ -1231,7 +1231,7 @@ private static StarTreeField getStarTreeFieldWithMultipleMetrics() { Dimension d1 = new NumericDimension("field1"); Dimension d2 = new NumericDimension("field3"); Metric m1 = new Metric("field2", List.of(MetricStat.SUM)); - Metric m2 = new Metric("field2", List.of(MetricStat.COUNT)); + Metric m2 = new Metric("field2", List.of(MetricStat.VALUE_COUNT)); List dims = List.of(d1, d2); List metrics = List.of(m1, m2); StarTreeFieldConfiguration c = new StarTreeFieldConfiguration( @@ -1313,7 +1313,7 @@ public void testMergeFlowWithCount() throws IOException { List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1393,7 +1393,7 @@ public void testMergeFlowWithDifferentDocsFromSegments() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1455,7 +1455,7 @@ public void testMergeFlowNumSegmentsDocs() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1515,7 +1515,7 @@ public void testMergeFlowWithMissingDocs() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1577,7 +1577,7 @@ public void testMergeFlowWithMissingDocsWithZero() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1640,7 +1640,7 @@ public void testMergeFlowWithMissingDocsWithZeroComplexCase() throws IOException List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1707,7 +1707,7 @@ public void testMergeFlowWithMissingDocsInSecondDim() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1770,7 +1770,7 @@ public void testMergeFlowWithDocsMissingAtTheEnd() throws IOException { List metricsList2 = List.of(5L, 6L, 7L, 8L, 9L); List metricsWithField2 = List.of(0, 1, 2, 3, 4); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), @@ -1824,7 +1824,7 @@ public void testMergeFlowWithEmptyFieldsInOneSegment() throws IOException { List metricsList = List.of(0L, 1L, 2L, 3L, 4L, 5L, 6L); List metricsWithField = List.of(0, 1, 2, 3, 4, 5, 6); - StarTreeField sf = getStarTreeField(MetricStat.COUNT); + StarTreeField sf = getStarTreeField(MetricStat.VALUE_COUNT); StarTreeValues starTreeValues = getStarTreeValues( getSortedNumericMock(dimList, docsWithField), getSortedNumericMock(dimList2, docsWithField2), diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index 132d2ff5a566a..3fa97825cdfc6 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -94,7 +94,7 @@ public void testValidStarTreeDefaults() throws IOException { assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); List expectedMetrics = Arrays.asList( MetricStat.AVG, - MetricStat.COUNT, + MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.MAX, MetricStat.MIN @@ -223,11 +223,11 @@ public void testMetric() { assertEquals(metric1, metric2); List m2 = new ArrayList<>(); m2.add(MetricStat.MAX); - m2.add(MetricStat.COUNT); + m2.add(MetricStat.VALUE_COUNT); metric2 = new Metric("name", m2); assertNotEquals(metric1, metric2); - assertEquals(MetricStat.COUNT, MetricStat.fromTypeName("count")); + assertEquals(MetricStat.VALUE_COUNT, MetricStat.fromTypeName("value_count")); assertEquals(MetricStat.MAX, MetricStat.fromTypeName("max")); assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min")); assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum"));