Star Tree Meta and Data Writers (#15295) (#15490)

---------
Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
(cherry picked from commit 8629279)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactory.java

@@ -34,17 +34,15 @@ public static DocValuesConsumer getDocValuesConsumerForCompositeCodec(
         String metaCodec,
         String metaExtension
     ) throws IOException {
-        try (
-            Lucene90DocValuesConsumerWrapper lucene90DocValuesConsumerWrapper = new Lucene90DocValuesConsumerWrapper(
-                state,
-                dataCodec,
-                dataExtension,
-                metaCodec,
-                metaExtension
-            )
-        ) {
-            return lucene90DocValuesConsumerWrapper.getLucene90DocValuesConsumer();
-        }
+        Lucene90DocValuesConsumerWrapper lucene90DocValuesConsumerWrapper = new Lucene90DocValuesConsumerWrapper(
+            state,
+            dataCodec,
+            dataExtension,
+            metaCodec,
+            metaExtension
+        );
+        return lucene90DocValuesConsumerWrapper.getLucene90DocValuesConsumer();
+
     }
 
 }

server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java

@@ -40,17 +40,14 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec(
 
         switch (compositeCodec) {
             case Composite99Codec.COMPOSITE_INDEX_CODEC_NAME:
-                try (
-                    Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
-                        state,
-                        dataCodec,
-                        dataExtension,
-                        metaCodec,
-                        metaExtension
-                    )
-                ) {
-                    return lucene90DocValuesProducerWrapper.getLucene90DocValuesProducer();
-                }
+                Lucene90DocValuesProducerWrapper lucene90DocValuesProducerWrapper = new Lucene90DocValuesProducerWrapper(
+                    state,
+                    dataCodec,
+                    dataExtension,
+                    metaCodec,
+                    metaExtension
+                );
+                return lucene90DocValuesProducerWrapper.getLucene90DocValuesProducer();
             default:
                 throw new IllegalStateException("Invalid composite codec " + "[" + compositeCodec + "]");
         }

server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+/**
+ * This class contains constants used in the Composite Index implementation.
+ */
+public class CompositeIndexConstants {
+
+    /**
+     * The magic marker value used for sanity checks in the Composite Index implementation.
+     */
+    public static final long COMPOSITE_FIELD_MARKER = 0xC0950513F1E1DL; // Composite Field
+
+    /**
+     * Represents the key to fetch number of non-star aggregated segment documents.
+     */
+    public static final String SEGMENT_DOCS_COUNT = "segmentDocsCount";
+
+}

server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexMetadata.java

@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex;
+
+import org.opensearch.index.mapper.CompositeMappedFieldType;
+
+/**
+ * This class represents the metadata of a Composite Index, which includes information about
+ * the composite field name, type, and the specific metadata for the type of composite field
+ * (e.g., Star Tree metadata).
+ *
+ * @opensearch.experimental
+ */
+public class CompositeIndexMetadata {
+
+    private final String compositeFieldName;
+    private final CompositeMappedFieldType.CompositeFieldType compositeFieldType;
+
+    /**
+     * Constructs a CompositeIndexMetadata object with the provided composite field name and type.
+     *
+     * @param compositeFieldName the name of the composite field
+     * @param compositeFieldType the type of the composite field
+     */
+    public CompositeIndexMetadata(String compositeFieldName, CompositeMappedFieldType.CompositeFieldType compositeFieldType) {
+        this.compositeFieldName = compositeFieldName;
+        this.compositeFieldType = compositeFieldType;
+    }
+
+    /**
+     * Returns the name of the composite field.
+     *
+     * @return the composite field name
+     */
+    public String getCompositeFieldName() {
+        return compositeFieldName;
+    }
+
+    /**
+     * Returns the type of the composite field.
+     *
+     * @return the composite field type
+     */
+    public CompositeMappedFieldType.CompositeFieldType getCompositeFieldType() {
+        return compositeFieldType;
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java

@@ -20,37 +20,43 @@
  */
 @ExperimentalApi
 public enum MetricStat {
-    VALUE_COUNT("value_count"),
-    SUM("sum"),
-    MIN("min"),
-    MAX("max"),
-    AVG("avg", VALUE_COUNT, SUM),
-    DOC_COUNT("doc_count", true);
+    VALUE_COUNT("value_count", 0),
+    SUM("sum", 1),
+    MIN("min", 2),
+    MAX("max", 3),
+    AVG("avg", 4, VALUE_COUNT, SUM),
+    DOC_COUNT("doc_count", true, 5);
 
     private final String typeName;
     private final MetricStat[] baseMetrics;
+    private final int metricOrdinal;
 
     // System field stats cannot be used as input for user metric types
     private final boolean isSystemFieldStat;
 
-    MetricStat(String typeName) {
-        this(typeName, false);
+    MetricStat(String typeName, int metricOrdinal) {
+        this(typeName, false, metricOrdinal);
     }
 
-    MetricStat(String typeName, MetricStat... baseMetrics) {
-        this(typeName, false, baseMetrics);
+    MetricStat(String typeName, int metricOrdinal, MetricStat... baseMetrics) {
+        this(typeName, false, metricOrdinal, baseMetrics);
     }
 
-    MetricStat(String typeName, boolean isSystemFieldStat, MetricStat... baseMetrics) {
+    MetricStat(String typeName, boolean isSystemFieldStat, int metricOrdinal, MetricStat... baseMetrics) {
         this.typeName = typeName;
         this.isSystemFieldStat = isSystemFieldStat;
         this.baseMetrics = baseMetrics;
+        this.metricOrdinal = metricOrdinal;
     }
 
     public String getTypeName() {
         return typeName;
     }
 
+    public int getMetricOrdinal() {
+        return metricOrdinal;
+    }
+
     /**
      * Return the list of metrics that this metric is derived from
      * For example, AVG is derived from COUNT and SUM
@@ -76,4 +82,13 @@ public static MetricStat fromTypeName(String typeName) {
         }
         throw new IllegalArgumentException("Invalid metric stat: " + typeName);
     }
+
+    public static MetricStat fromMetricOrdinal(int metricOrdinal) {
+        for (MetricStat metric : MetricStat.values()) {
+            if (metric.getMetricOrdinal() == metricOrdinal) {
+                return metric;
+            }
+        }
+        throw new IllegalArgumentException("Invalid metric stat: " + metricOrdinal);
+    }
 }

server/src/main/java/org/opensearch/index/compositeindex/datacube/ReadDimension.java

@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.compositeindex.datacube;
+
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.mapper.CompositeDataCubeFieldType;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Represents a dimension for reconstructing StarTreeField from file formats during searches and merges.
+ *
+ * @opensearch.experimental
+ */
+public class ReadDimension implements Dimension {
+    public static final String READ = "read";
+    private final String field;
+
+    public ReadDimension(String field) {
+        this.field = field;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CompositeDataCubeFieldType.NAME, field);
+        builder.field(CompositeDataCubeFieldType.TYPE, READ);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ReadDimension dimension = (ReadDimension) o;
+        return Objects.equals(field, dimension.getField());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field);
+    }
+}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java

@@ -56,19 +56,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @ExperimentalApi
     public enum StarTreeBuildMode {
         // TODO : remove onheap support unless this proves useful
-        ON_HEAP("onheap"),
-        OFF_HEAP("offheap");
+        ON_HEAP("onheap", (byte) 0),
+        OFF_HEAP("offheap", (byte) 1);
 
         private final String typeName;
+        private final byte buildModeOrdinal;
 
-        StarTreeBuildMode(String typeName) {
+        StarTreeBuildMode(String typeName, byte buildModeOrdinal) {
             this.typeName = typeName;
+            this.buildModeOrdinal = buildModeOrdinal;
         }
 
         public String getTypeName() {
             return typeName;
         }
 
+        public byte getBuildModeOrdinal() {
+            return buildModeOrdinal;
+        }
+
         public static StarTreeBuildMode fromTypeName(String typeName) {
             for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) {
                 if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) {
@@ -77,6 +83,16 @@ public static StarTreeBuildMode fromTypeName(String typeName) {
             }
             throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName));
         }
+
+        public static StarTreeBuildMode fromBuildModeOrdinal(byte buildModeOrdinal) {
+            for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) {
+                if (starTreeBuildMode.getBuildModeOrdinal() == buildModeOrdinal) {
+                    return starTreeBuildMode;
+                }
+            }
+            throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", buildModeOrdinal));
+        }
+
     }
 
     public int maxLeafDocs() {