From a59f1e4ea73235c1c7343e319fec225acac0683f Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Tue, 5 Mar 2024 13:10:10 -0800
Subject: [PATCH 01/13] Add initial implementation of DerivedFieldMapper class

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/DerivedFieldMapper.java      | 173 ++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
new file mode 100644
index 0000000000000..7ca71afa717b1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -0,0 +1,173 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
+import org.opensearch.common.Nullable;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.Script;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A field mapper for derived fields
+ *
+ * @opensearch.internal
+ */
+public class DerivedFieldMapper extends ParametrizedFieldMapper {
+
+    public static final String CONTENT_TYPE = "derived_field";
+
+    /**
+     * Default parameters for the boolean field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Defaults {
+        public static final FieldType FIELD_TYPE = new FieldType();
+
+        static {
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    private static DerivedFieldMapper toType(FieldMapper in) {
+        return (DerivedFieldMapper) in;
+    }
+
+    /**
+     * Builder for this field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Builder extends ParametrizedFieldMapper.Builder {
+
+        // TODO: The type of parameter may change here if the actual underlying FieldType object is needed
+        private final Parameter<String> type = Parameter.stringParam(
+            "type",
+            false,
+            m -> toType(m).type,
+            "text"
+        );
+
+        private final Parameter<Script> script = new Parameter<>(
+            "script",
+            false,
+            () -> null,
+            (n, c, o) -> o == null ? null : Script.parse(o),
+            m -> toType(m).script
+        );
+
+        public Builder(String name) { super(name); }
+
+        @Override
+        protected List<Parameter<?>> getParameters() {
+            return Arrays.asList(type, script);
+        }
+
+        @Override
+        public DerivedFieldMapper build(BuilderContext context) {
+            MappedFieldType ft = new DerivedFieldType(buildFullName(context));
+            return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
+        }
+    }
+
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+
+    /**
+     * Field type for derived field mapper
+     *
+     * @opensearch.internal
+     */
+    public static final class DerivedFieldType extends MappedFieldType {
+
+        public DerivedFieldType(
+            String name,
+            boolean isIndexed,
+            boolean isStored,
+            boolean hasDocValues,
+            Map<String, String> meta
+        ) {
+            super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.NONE, meta);
+        }
+
+        public DerivedFieldType(String name) { this(name, false, false, false, Collections.emptyMap()); }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format ) {
+            if (format != null) {
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+            }
+
+            // TODO alongside parseCreateField
+            return new SourceValueFetcher(name(), context) {
+                @Override
+                protected Object parseSourceValue(Object value) {
+                    return null;
+                }
+            };
+        }
+
+        @Override
+        public Query termQuery(Object value, @Nullable QueryShardContext context) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support term queries");
+        }
+
+        @Override
+        public boolean isAggregatable() { return false; }
+    }
+
+    private final String type;
+
+    private final Script script;
+
+    protected DerivedFieldMapper(
+        String simpleName,
+        MappedFieldType mappedFieldType,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        Builder builder
+    ) {
+        super(simpleName, mappedFieldType, multiFields, copyTo);
+        this.type = builder.type.getValue();
+        this.script = builder.script.getValue();
+    }
+
+    @Override
+    public DerivedFieldType fieldType() { return (DerivedFieldType) super.fieldType(); }
+
+    @Override
+    protected void parseCreateField(ParseContext context) throws IOException {
+        // TODO
+    }
+
+    @Override
+    public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+}

From 9a4a1e0fab21ca55f06714fb305e35b15a9d1ef1 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Tue, 5 Mar 2024 13:19:20 -0800
Subject: [PATCH 02/13] Add support for parsing derived fields in mappings

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/DerivedFieldMapper.java      |  7 ++-
 .../opensearch/index/mapper/ObjectMapper.java | 58 +++++++++++++++++++
 .../org/opensearch/indices/IndicesModule.java |  2 +
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index 7ca71afa717b1..21abb2b645328 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -121,7 +121,8 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            // TODO alongside parseCreateField
+            // TODO Return to this during Query implementation. The derived fields don't typically exist in _source but
+            //  we may want fetch the field name from source if a 'script' is not provided.
             return new SourceValueFetcher(name(), context) {
                 @Override
                 protected Object parseSourceValue(Object value) {
@@ -160,7 +161,9 @@ protected DerivedFieldMapper(
 
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
-        // TODO
+        // Leaving this empty as the parsing should be handled via the Builder when root object is parsed.
+        // The context would not contain anything in this case since the DerivedFieldMapper is not indexed or stored.
+        throw new UnsupportedOperationException("should not be invoked");
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index 39b0ec224b57c..ab8bbc2fb4823 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -291,6 +291,15 @@ protected static boolean parseObjectOrDocumentTypeProperties(
             } else if (fieldName.equals("enabled")) {
                 builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled"));
                 return true;
+            } else if (fieldName.equals("derived")) {
+                if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
+                    // nothing to do here, empty (to support "derived: []" case)
+                } else if (!(fieldNode instanceof Map)) {
+                    throw new OpenSearchParseException("derived must be a map type");
+                } else {
+                    parseDerived(builder, (Map<String, Object>) fieldNode, parserContext);
+                }
+                return true;
             } else if (fieldName.equals("properties")) {
                 if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
                     // nothing to do here, empty (to support "properties: []" case)
@@ -349,6 +358,55 @@ protected static void parseNested(
             }
         }
 
+        protected static void parseDerived(ObjectMapper.Builder objBuilder, Map<String, Object> derivedNode, ParserContext parserContext) {
+            Iterator<Map.Entry<String, Object>> iterator = derivedNode.entrySet().iterator();
+            while (iterator.hasNext()) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                // Should accept empty arrays, as a work around for when the
+                // user can't provide an empty Map. (PHP for example)
+                boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
+
+                if (entry.getValue() instanceof Map) {
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> node = (Map<String, Object>) entry.getValue();
+
+                    // Derived fields are a bit unique in that the 'type' attribute does not map to the TypeParser
+                    // like it would for traditional fields in properties.
+                    // So in this case, the DerivedFieldMapper's TypeParser will explicitly be used
+                    Mapper.TypeParser typeParser = parserContext.typeParser(DerivedFieldMapper.CONTENT_TYPE);
+                    String[] fieldNameParts = fieldName.split("\\.");
+                    // field name is just ".", which is invalid
+                    if (fieldNameParts.length < 1) {
+                        throw new MapperParsingException("Invalid field name " + fieldName);
+                    }
+                    String realFieldName = fieldNameParts[fieldNameParts.length - 1];
+                    Mapper.Builder<?> fieldBuilder = typeParser.parse(realFieldName, node, parserContext);
+                    for (int i = fieldNameParts.length - 2; i >= 0; --i) {
+                        ObjectMapper.Builder<?> intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]);
+                        intermediate.add(fieldBuilder);
+                        fieldBuilder = intermediate;
+                    }
+                    objBuilder.add(fieldBuilder);
+                    node.remove("type");
+                    DocumentMapperParser.checkNoRemainingFields(fieldName, node, parserContext.indexVersionCreated());
+                    iterator.remove();
+                } else if (isEmptyList) {
+                    iterator.remove();
+                } else {
+                    throw new MapperParsingException(
+                        "Expected map for property [derived_fields] on field [" + fieldName + "] but got a " + fieldName.getClass()
+                    );
+                }
+            }
+
+            DocumentMapperParser.checkNoRemainingFields(
+                derivedNode,
+                parserContext.indexVersionCreated(),
+                "DocType mapping definition has unsupported parameters: "
+            );
+        }
+
         protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java
index b86e98f4ebcbc..243291405906c 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java
@@ -48,6 +48,7 @@
 import org.opensearch.index.mapper.CompletionFieldMapper;
 import org.opensearch.index.mapper.DataStreamFieldMapper;
 import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.FieldAliasMapper;
 import org.opensearch.index.mapper.FieldNamesFieldMapper;
@@ -168,6 +169,7 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
         mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
         mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
         mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER);
+        mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER);
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {

From 3cdff10ad257433f9d90941d514c9715d942466c Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Thu, 7 Mar 2024 12:52:32 -0800
Subject: [PATCH 03/13] Add tests for DerivedFieldMapper

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/DerivedFieldMapper.java      |   4 +
 .../index/mapper/DerivedFieldMapperTests.java | 108 ++++++++++++++++++
 .../index/mapper/MapperServiceTestCase.java   |  20 +++-
 3 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index 21abb2b645328..b625a418839a2 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -173,4 +173,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    public String getType() { return type; }
+
+    public Script getScript() { return script; }
 }
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
new file mode 100644
index 0000000000000..dee6876a634ce
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+
+public class DerivedFieldMapperTests extends MapperTestCase {
+
+    protected boolean supportsOrIgnoresBoost() {
+        return false;
+    }
+
+    // Overriding fieldMapping to make it create derived mappings by default.
+    // This way, the parent tests are checking the right behavior for this Mapper.
+    @Override
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, true);
+    }
+
+    // Overriding this to do nothing since derived fields are not written in the documents
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {}
+
+    @Override
+    protected void minimalMapping(XContentBuilder b) throws IOException {
+        b.field("type", "keyword");
+    }
+
+    protected void registerParameters(ParameterChecker checker) throws IOException {
+        // TODO Any conflicts or updates to check for here? Parameters index, store, doc_values and boost are not
+        //  supported for DerivedFieldMapper (we explicitly set these values on initialization)
+    }
+
+    // TESTCASE: testDefaults() (no script provided and assuming field is not in the source either)
+    //  TODO: Might not need this (no default that really deviates it from testSerialization()
+
+    public void testSerialization() throws IOException {
+
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"field\":{\"type\":\"keyword\"}}", builder.toString());
+    }
+
+    public void testParsesScript() throws IOException {
+
+        String scriptString = "{\"source\": \"doc['test'].value\"}";
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.field("script", scriptString);
+        }));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals(Script.parse(scriptString), derivedFieldMapper.getScript());
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"type\": \"keyword\",\"script\": {\"source\": \"emit(doc['test'].value)\"}}", builder.toString());
+    }
+
+    //  TODO: Is there a case where we want to allow the field to be defined in both 'derived' and 'properties'?
+    //    If the user wants to move a field they were testing as derived to be indexed they should be able to update
+    //    the mappings in index template to move the field from 'derived' to 'properties' and it should take affect
+    //    during the next index rollover (so even in this case, the field is only defined in one or the other).
+    public void testFieldInDerivedAndProperties() throws IOException {
+        MapperParsingException ex = expectThrows(
+            MapperParsingException.class,
+            () -> createDocumentMapper(topMapping(b -> {
+                b.startObject("derived");
+                b.startObject("field");
+                b.field("type", "keyword");
+                b.endObject();
+                b.endObject();
+                b.startObject("properties");
+                b.startObject("field");
+                b.field("type", "keyword");
+                b.endObject();
+                b.endObject();
+            }))
+        );
+        assertEquals(
+            "Field [field] is defined more than once",
+            ex.getMessage()
+        );
+    }
+
+    // TODO TESTCASE: testWithFieldInSource() (derived field with that field present in source)
+    //  This is more checking search behavior so may need to revisit this after query implementation
+
+}
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
index a65ce3cbdd380..f90aa5d3ff11a 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
@@ -225,13 +225,31 @@ protected final XContentBuilder mapping(CheckedConsumer<XContentBuilder, IOExcep
         return builder.endObject().endObject().endObject();
     }
 
+    protected final XContentBuilder derivedMapping(CheckedConsumer<XContentBuilder, IOException> buildFields) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("derived");
+        buildFields.accept(builder);
+        return builder.endObject().endObject().endObject();
+    }
+
     protected final XContentBuilder dynamicMapping(Mapping dynamicMapping) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
         dynamicMapping.toXContent(builder, ToXContent.EMPTY_PARAMS);
         return builder.endObject();
     }
 
-    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+    protected XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, false);
+    }
+
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField, Boolean isDerived) throws IOException {
+        if (isDerived) {
+            return derivedMapping(b -> {
+                b.startObject("field");
+                buildField.accept(b);
+                b.endObject();
+            });
+        }
+
         return mapping(b -> {
             b.startObject("field");
             buildField.accept(b);

From d601d82fdf671c0c61bc20bb855e42fcaaf70da2 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Thu, 7 Mar 2024 12:53:03 -0800
Subject: [PATCH 04/13] Add test for parsing derived mappings in
 ObjectMapperTests

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/ObjectMapperTests.java       | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index be947935dc4ea..cf35d03c338b1 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.mapper.ObjectMapper.Dynamic;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.script.Script;
 import org.opensearch.test.InternalSettingsPlugin;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
@@ -440,6 +441,51 @@ public void testEmptyName() throws Exception {
         assertThat(e.getMessage(), containsString("name cannot be empty string"));
     }
 
+    public void testDerivedFields() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("tweet")
+            .startObject("derived")
+            .startObject("derived_field_name1")
+            .field("type", "text")
+            .endObject()
+            .startObject("derived_field_name2")
+            .field("type", "keyword")
+            .field("script", "{\"source\": \"doc['test'].value\"}")
+            .endObject()
+            .endObject()
+            .startObject("properties")
+            .startObject("field_name")
+            .field("type", "text")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+
+        DocumentMapper documentMapper = createIndex("test")
+            .mapperService()
+            .documentMapperParser()
+            .parse("tweet", new CompressedXContent(mapping));
+
+        Mapper mapper = documentMapper.root().getMapper("derived_field_name1");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("text", derivedFieldMapper.getType());
+        assertNull(derivedFieldMapper.getScript());
+
+        mapper = documentMapper.root().getMapper("derived_field_name2");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("keyword", derivedFieldMapper.getType());
+        assertEquals(Script.parse("{\"source\": \"doc['test'].value\"}"), derivedFieldMapper.getScript());
+
+        // Check that field in properties was parsed correctly as well
+        mapper = documentMapper.root().getMapper("field_name");
+        assertNotNull(mapper);
+        assertEquals("text", mapper.typeName());
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);

From 16717ee279b726ead2e2ea9f64f8611e32026b73 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Fri, 8 Mar 2024 10:24:11 -0800
Subject: [PATCH 05/13] Update serialization for DerivedFieldMapper to work as
 expected

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/DerivedFieldMapper.java          | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index b625a418839a2..a4ff914cd657d 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
 import org.opensearch.common.Nullable;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.script.Script;
 import org.opensearch.search.lookup.SearchLookup;
@@ -73,7 +74,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             () -> null,
             (n, c, o) -> o == null ? null : Script.parse(o),
             m -> toType(m).script
-        );
+        ).setSerializerCheck((id, ic, value) -> value != null);
 
         public Builder(String name) { super(name); }
 
@@ -136,6 +137,11 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support term queries");
         }
 
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exist queries");
+        }
+
         @Override
         public boolean isAggregatable() { return false; }
     }
@@ -174,6 +180,13 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        getMergeBuilder().toXContent(builder, includeDefaults);
+        multiFields.toXContent(builder, params);
+        copyTo.toXContent(builder, params);
+    }
+
     public String getType() { return type; }
 
     public Script getScript() { return script; }

From 1e103e0a998e106b07da6e9b65ff95bfe0ce3470 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Fri, 8 Mar 2024 10:25:28 -0800
Subject: [PATCH 06/13] Update toXContent in ObjectMapper so serialization
 accounts for derived field mappings

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../opensearch/index/mapper/ObjectMapper.java | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index ab8bbc2fb4823..33c4b3393f7d2 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -721,7 +721,19 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
         doXContent(builder, params);
 
         // sort the mappers so we get consistent serialization format
-        Mapper[] sortedMappers = mappers.values().stream().toArray(size -> new Mapper[size]);
+        Mapper[] derivedSortedMappers = mappers.values().stream()
+            .filter(m -> m instanceof DerivedFieldMapper)
+            .toArray(size -> new Mapper[size]);
+        Arrays.sort(derivedSortedMappers, new Comparator<Mapper>() {
+            @Override
+            public int compare(Mapper o1, Mapper o2) {
+                return o1.name().compareTo(o2.name());
+            }
+        });
+
+        Mapper[] sortedMappers = mappers.values().stream()
+            .filter(m -> !(m instanceof DerivedFieldMapper))
+            .toArray(size -> new Mapper[size]);
         Arrays.sort(sortedMappers, new Comparator<Mapper>() {
             @Override
             public int compare(Mapper o1, Mapper o2) {
@@ -730,6 +742,17 @@ public int compare(Mapper o1, Mapper o2) {
         });
 
         int count = 0;
+        for (Mapper mapper : derivedSortedMappers) {
+            if (count++ == 0) {
+                builder.startObject("derived");
+            }
+            mapper.toXContent(builder, params);
+        }
+        if (count > 0) {
+            builder.endObject();
+        }
+
+        count = 0;
         for (Mapper mapper : sortedMappers) {
             if (!(mapper instanceof MetadataFieldMapper)) {
                 if (count++ == 0) {

From c8a57880c9d7d3432d2d82b2a7a17c1d4c5ec011 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Fri, 8 Mar 2024 10:26:44 -0800
Subject: [PATCH 07/13] Do not remove 'type' from fieldNode prematurely when it
 is a parameter in the case of DerivedFieldMapper

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../org/opensearch/index/mapper/ParametrizedFieldMapper.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
index f4723b6178137..cde351f923767 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
@@ -670,7 +670,9 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
                     deprecatedParamsMap.put(deprecatedName, param);
                 }
             }
-            String type = (String) fieldNode.remove("type");
+            String type = (String) fieldNode.get("type");
+            if (paramsMap.get("type") == null) { fieldNode.remove("type"); }
+
             for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 final String propName = entry.getKey();

From 40ea1ade0df870126abc1bb3f64638156c02483e Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Fri, 8 Mar 2024 10:27:09 -0800
Subject: [PATCH 08/13] Fix tests

Signed-off-by: Mohammad Qureshi <qreshi@amazon.com>
---
 .../index/mapper/DerivedFieldMapperTests.java | 26 ++++++++++++++-----
 .../index/mapper/ObjectMapperTests.java       |  6 +++--
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
index dee6876a634ce..92311584a5972 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
@@ -22,6 +22,10 @@ protected boolean supportsOrIgnoresBoost() {
         return false;
     }
 
+    protected boolean supportsMeta() {
+        return false;
+    }
+
     // Overriding fieldMapping to make it create derived mappings by default.
     // This way, the parent tests are checking the right behavior for this Mapper.
     @Override
@@ -43,8 +47,11 @@ protected void registerParameters(ParameterChecker checker) throws IOException {
         //  supported for DerivedFieldMapper (we explicitly set these values on initialization)
     }
 
-    // TESTCASE: testDefaults() (no script provided and assuming field is not in the source either)
-    //  TODO: Might not need this (no default that really deviates it from testSerialization()
+    // TODO: Can update this once the query implementation is completed
+    //  This is also being left blank because the super assertExistsQuery is trying to parse
+    //  an empty source and fails.
+    @Override
+    protected void assertExistsQuery(MapperService mapperService) {}
 
     public void testSerialization() throws IOException {
 
@@ -60,10 +67,12 @@ public void testSerialization() throws IOException {
 
     public void testParsesScript() throws IOException {
 
-        String scriptString = "{\"source\": \"doc['test'].value\"}";
+        String scriptString = "doc['test'].value";
         DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
             b.field("type", "keyword");
-            b.field("script", scriptString);
+            b.startObject("script");
+            b.field("source", scriptString);
+            b.endObject();
         }));
         Mapper mapper = defaultMapper.mappers().getMapper("field");
         assertTrue(mapper instanceof DerivedFieldMapper);
@@ -73,11 +82,11 @@ public void testParsesScript() throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
         mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        assertEquals("{\"type\": \"keyword\",\"script\": {\"source\": \"emit(doc['test'].value)\"}}", builder.toString());
+        assertEquals("{\"field\":{\"type\":\"keyword\",\"script\":{\"source\":\"doc['test'].value\",\"lang\":\"painless\"}}}", builder.toString());
     }
 
     //  TODO: Is there a case where we want to allow the field to be defined in both 'derived' and 'properties'?
-    //    If the user wants to move a field they were testing as derived to be indexed they should be able to update
+    //    If the user wants to move a field they were testing as derived to be indexed, they should be able to update
     //    the mappings in index template to move the field from 'derived' to 'properties' and it should take affect
     //    during the next index rollover (so even in this case, the field is only defined in one or the other).
     public void testFieldInDerivedAndProperties() throws IOException {
@@ -96,8 +105,11 @@ public void testFieldInDerivedAndProperties() throws IOException {
                 b.endObject();
             }))
         );
+        // TODO: Do we want to handle this as a different error? As it stands, it fails as a merge conflict which makes sense.
+        //  If it didn't fail here, it would hit the MapperParsingException for the field being defined more than once
+        //  when MappingLookup is initialized
         assertEquals(
-            "Field [field] is defined more than once",
+            "Failed to parse mapping [_doc]: mapper [field] cannot be changed from type [derived_field] to [keyword]",
             ex.getMessage()
         );
     }
diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index cf35d03c338b1..66a2faa89aedc 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -451,7 +451,9 @@ public void testDerivedFields() throws Exception {
             .endObject()
             .startObject("derived_field_name2")
             .field("type", "keyword")
-            .field("script", "{\"source\": \"doc['test'].value\"}")
+            .startObject("script")
+            .field("source", "doc['test'].value")
+            .endObject()
             .endObject()
             .endObject()
             .startObject("properties")
@@ -478,7 +480,7 @@ public void testDerivedFields() throws Exception {
         assertTrue(mapper instanceof DerivedFieldMapper);
         derivedFieldMapper = (DerivedFieldMapper) mapper;
         assertEquals("keyword", derivedFieldMapper.getType());
-        assertEquals(Script.parse("{\"source\": \"doc['test'].value\"}"), derivedFieldMapper.getScript());
+        assertEquals(Script.parse("doc['test'].value"), derivedFieldMapper.getScript());
 
         // Check that field in properties was parsed correctly as well
         mapper = documentMapper.root().getMapper("field_name");

From 5575d146a7511568f5333827c2caad725ca38069 Mon Sep 17 00:00:00 2001
From: Mohammad Qureshi <qreshi@amazon.com>
Date: Mon, 11 Mar 2024 15:08:58 -0700
Subject: [PATCH 09/13] Apply spotless to fix Gradle check

---
 .../org/opensearch/index/mapper/MapperServiceTestCase.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
index f90aa5d3ff11a..658e505fb47bd 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
@@ -241,7 +241,8 @@ protected XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOExcept
         return fieldMapping(buildField, false);
     }
 
-    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField, Boolean isDerived) throws IOException {
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField, Boolean isDerived)
+        throws IOException {
         if (isDerived) {
             return derivedMapping(b -> {
                 b.startObject("field");

From 1b8318ed4ecdf1fefe44623efcf1fdb8ea28ff52 Mon Sep 17 00:00:00 2001
From: Rishabh Maurya <rishabhmaurya05@gmail.com>
Date: Wed, 13 Mar 2024 22:17:22 -0700
Subject: [PATCH 10/13] Query execution logic for Derived Fields

---
 .../index/mapper/DerivedFieldMapper.java      | 173 ++++++++++++++++--
 .../index/mapper/IpFieldMapper.java           |   2 +-
 .../index/mapper/KeywordFieldMapper.java      |   2 +-
 .../index/query/DerivedFieldQuery.java        | 144 +++++++++++++++
 .../index/query/DerivedFieldScript.java       | 137 ++++++++++++++
 .../index/query/DerivedFieldsContext.java     |  89 +++++++++
 .../org/opensearch/script/ScriptModule.java   |   4 +-
 7 files changed, 534 insertions(+), 17 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
 create mode 100644 server/src/main/java/org/opensearch/index/query/DerivedFieldScript.java
 create mode 100644 server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index a4ff914cd657d..e2a204ef7aa2a 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -8,20 +8,33 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.geo.ShapeRelation;
+import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.time.DateMathParser;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.query.DerivedFieldQuery;
+import org.opensearch.index.query.DerivedFieldScript;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.script.Script;
 import org.opensearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * A field mapper for derived fields
@@ -59,7 +72,6 @@ private static DerivedFieldMapper toType(FieldMapper in) {
      * @opensearch.internal
      */
     public static class Builder extends ParametrizedFieldMapper.Builder {
-
         // TODO: The type of parameter may change here if the actual underlying FieldType object is needed
         private final Parameter<String> type = Parameter.stringParam(
             "type",
@@ -76,7 +88,10 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
             m -> toType(m).script
         ).setSerializerCheck((id, ic, value) -> value != null);
 
-        public Builder(String name) { super(name); }
+        public Builder(String name) {
+            super(name);
+        }
+
 
         @Override
         protected List<Parameter<?>> getParameters() {
@@ -85,11 +100,49 @@ protected List<Parameter<?>> getParameters() {
 
         @Override
         public DerivedFieldMapper build(BuilderContext context) {
-            MappedFieldType ft = new DerivedFieldType(buildFullName(context));
+            FieldMapper typeFieldMapper;
+            Function<Object, IndexableField> fieldFunction;
+            switch (type.getValue()) {
+                // TODO: add logic all supported type in derived fields
+                // TODO: should we support mapping settings exposed by a given field type from derived fields too?
+                // for example, support `format` for date type?
+                case KeywordFieldMapper.CONTENT_TYPE:
+                    FieldType dummyFieldType = new FieldType();
+                    dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+                    KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name());
+                    KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
+                    keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+                    typeFieldMapper = new KeywordFieldMapper(
+                        name(), dummyFieldType, keywordFieldType,
+                        keywordBuilder.multiFieldsBuilder.build(this, context),
+                        keywordBuilder.copyTo.build(),
+                        keywordBuilder
+                    );
+                    fieldFunction = o -> new KeywordField(name(), (String) o, Field.Store.NO);
+                    break;
+                case "long":
+                    // ignoreMalformed?
+                    NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
+                    typeFieldMapper = longBuilder.build(context);
+                    fieldFunction = o -> new LongField(name(), Long.parseLong(o.toString()), Field.Store.NO);
+                    break;
+                case "double":
+                    // ignoreMalformed?
+                    NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
+                    typeFieldMapper = doubleBuilder.build(context);
+                    fieldFunction = o -> new DoubleField(name(), Double.parseDouble(o.toString()), Field.Store.NO);
+                    break;
+                default:
+                    throw new IllegalArgumentException("Field [" + name() + "] of type [" + type + "] isn't supported " +
+                        "in Derived field context.");
+
+            }
+            MappedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), typeFieldMapper, fieldFunction);
             return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
         }
     }
 
+
     public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
 
     /**
@@ -98,18 +151,35 @@ public DerivedFieldMapper build(BuilderContext context) {
      * @opensearch.internal
      */
     public static final class DerivedFieldType extends MappedFieldType {
+        private final String type;
+
+        private final Script script;
+
+        FieldMapper typeFieldMapper;
+
+        private final Function<Object, IndexableField> fieldFunction;
 
         public DerivedFieldType(
             String name,
+            String type,
+            Script script,
             boolean isIndexed,
             boolean isStored,
             boolean hasDocValues,
-            Map<String, String> meta
+            Map<String, String> meta,
+            FieldMapper typeFieldMapper,
+            Function<Object, IndexableField> fieldFunction
         ) {
-            super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.NONE, meta);
+            super(name, isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
+            this.type = type;
+            this.script = script;
+            this.typeFieldMapper = typeFieldMapper;
+            this.fieldFunction = fieldFunction;
         }
 
-        public DerivedFieldType(String name) { this(name, false, false, false, Collections.emptyMap()); }
+        public DerivedFieldType(String name, String type, Script script, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
+            this(name, type, script, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
+        }
 
         @Override
         public String typeName() {
@@ -117,7 +187,7 @@ public String typeName() {
         }
 
         @Override
-        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format ) {
+        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
@@ -127,14 +197,78 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL
             return new SourceValueFetcher(name(), context) {
                 @Override
                 protected Object parseSourceValue(Object value) {
-                    return null;
+                    return value;
                 }
             };
         }
 
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support term queries");
+            Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
+            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
+        }
+
+        @Override
+        public Query prefixQuery(
+            String value,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            boolean caseInsensitive,
+            QueryShardContext context
+        ) {
+            Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
+            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
+        }
+
+        @Override
+        public Query wildcardQuery(
+            String value,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            boolean caseInsensitive,
+            QueryShardContext context
+        ) {
+            Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
+            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
+        }
+
+        @Override
+        public Query regexpQuery(
+            String value,
+            int syntaxFlags,
+            int matchFlags,
+            int maxDeterminizedStates,
+            @Nullable MultiTermQuery.RewriteMethod method,
+            QueryShardContext context
+        ) {
+            Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags,
+                maxDeterminizedStates, method, context);
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
+            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
+        }
+
+        // TODO: Override all types of queries which can be supported by all types supported within derived fields.
+        @Override
+        public Query rangeQuery(
+            Object lowerTerm,
+            Object upperTerm,
+            boolean includeLower,
+            boolean includeUpper,
+            ShapeRelation relation,
+            ZoneId timeZone,
+            DateMathParser parser,
+            QueryShardContext context
+        ) {
+            Query query = typeFieldMapper.mappedFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper,
+                relation, timeZone, parser, context);
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
+            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
         }
 
         @Override
@@ -143,9 +277,12 @@ public Query existsQuery(QueryShardContext context) {
         }
 
         @Override
-        public boolean isAggregatable() { return false; }
+        public boolean isAggregatable() {
+            return false;
+        }
     }
 
+
     private final String type;
 
     private final Script script;
@@ -163,7 +300,9 @@ protected DerivedFieldMapper(
     }
 
     @Override
-    public DerivedFieldType fieldType() { return (DerivedFieldType) super.fieldType(); }
+    public DerivedFieldType fieldType() {
+        return (DerivedFieldType) super.fieldType();
+    }
 
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
@@ -173,7 +312,9 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); }
+    public ParametrizedFieldMapper.Builder getMergeBuilder() {
+        return new Builder(simpleName()).init(this);
+    }
 
     @Override
     protected String contentType() {
@@ -187,7 +328,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         copyTo.toXContent(builder, params);
     }
 
-    public String getType() { return type; }
+    public String getType() {
+        return type;
+    }
 
-    public Script getScript() { return script; }
+    public Script getScript() {
+        return script;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
index db8da8a949d6f..b657a41ff3819 100644
--- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
@@ -450,7 +450,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
     private final boolean ignoreMalformedByDefault;
     private final Version indexCreatedVersion;
 
-    private IpFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) {
+    protected IpFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) {
         super(simpleName, mappedFieldType, multiFields, copyTo);
         this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault;
         this.indexed = builder.indexed.getValue();
diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
index c14b2c92c89c3..42b974734e5e7 100644
--- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
@@ -218,7 +218,7 @@ protected List<Parameter<?>> getParameters() {
             );
         }
 
-        private KeywordFieldType buildFieldType(BuilderContext context, FieldType fieldType) {
+        protected KeywordFieldType buildFieldType(BuilderContext context, FieldType fieldType) {
             NamedAnalyzer normalizer = Lucene.KEYWORD_ANALYZER;
             NamedAnalyzer searchAnalyzer = Lucene.KEYWORD_ANALYZER;
             String normalizerName = this.normalizer.getValue();
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
new file mode 100644
index 0000000000000..b5e8d5e3f782d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
@@ -0,0 +1,144 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.query;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * DerivedFieldQuery used for querying derived fields.
+ */
+public class DerivedFieldQuery extends Query {
+    private final Query filter;
+    private final DerivedFieldScript.LeafFactory derivedFieldScriptFactory;
+    private final Analyzer indexAnalyzer;
+    private final Function<Object, IndexableField> fieldFunction;
+    private final QueryShardContext context;
+
+    /**
+     * @param query                     query to execute against memory index
+     * @param derivedFieldScriptFactory derived field script's LeafFactory instance
+     * @param indexAnalyzer             index analyzer to use while building memory index
+     * @param fieldFunction             IndexableField generator
+     * @param context                   QueryShardContext
+     */
+    public DerivedFieldQuery(Query query, DerivedFieldScript.LeafFactory derivedFieldScriptFactory,
+                             Analyzer indexAnalyzer, Function<Object, IndexableField> fieldFunction,
+                             QueryShardContext context) {
+        this.filter = query;
+        this.derivedFieldScriptFactory = derivedFieldScriptFactory;
+        this.indexAnalyzer = indexAnalyzer;
+        this.context = context;
+        this.fieldFunction = fieldFunction;
+        if (!context.documentMapper("").sourceMapper().enabled()) {
+            throw new IllegalArgumentException(
+                "DerivedFieldQuery error: unable to fetch fields from _source field: _source is disabled in the mappings "
+                    + "for index ["
+                    + context.index().getName()
+                    + "]"
+            );
+        }
+    }
+
+    @Override
+    public void visit(QueryVisitor visitor) {
+
+    }
+
+    @Override
+    public Query rewrite(IndexSearcher indexSearcher) throws IOException {
+        Query rewritten = indexSearcher.rewrite(filter);
+        if (rewritten == filter) {
+            return this;
+        }
+        return new DerivedFieldQuery(rewritten, derivedFieldScriptFactory, indexAnalyzer, fieldFunction, context);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+        return new ConstantScoreWeight(this, boost) {
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
+                DerivedFieldScript derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
+                TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
+                    @Override
+                    public boolean matches() {
+                        derivedFieldScript.setDocument(approximation.docID());
+                        Object value = derivedFieldScript.execute();
+                        // TODO: in case of errors from script, should it be ignored and treated as missing field
+                        //  by using a configurable setting?
+                        MemoryIndex memoryIndex = new MemoryIndex();
+                        // TODO add support for multi-field with use of emit()
+                        memoryIndex.addField(fieldFunction.apply(value), indexAnalyzer);
+                        float score = memoryIndex.search(filter);
+                        return score > 0.0f;
+                    }
+
+                    @Override
+                    public float matchCost() {
+                        // TODO: how can we compute this?
+                        return 1000f;
+                    }
+                };
+                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+            }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                // TODO: Change this to true when we can assume that scripts are pure functions
+                // ie. the return value is always the same given the same conditions and may not
+                // depend on the current timestamp, other documents, etc.
+                return false;
+            }
+        };
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (sameClassAs(o) == false) {
+            return false;
+        }
+        DerivedFieldQuery other = (DerivedFieldQuery) o;
+        return Objects.equals(this.filter, other.filter)
+            && Objects.equals(this.derivedFieldScriptFactory, other.derivedFieldScriptFactory)
+            && Objects.equals(this.fieldFunction, other.fieldFunction)
+            && Objects.equals(this.indexAnalyzer, other.indexAnalyzer)
+            && Objects.equals(this.context, other.context);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(classHash(), filter, derivedFieldScriptFactory, fieldFunction, indexAnalyzer, context);
+    }
+
+    @Override
+    public String toString(String f) {
+        return "DerivedFieldQuery (filter query: [ " + filter.toString(f) + "])";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldScript.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldScript.java
new file mode 100644
index 0000000000000..08fc79e9da1ba
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldScript.java
@@ -0,0 +1,137 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.index.query;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.opensearch.common.logging.DeprecationLogger;
+import org.opensearch.index.fielddata.ScriptDocValues;
+import org.opensearch.script.DynamicMap;
+import org.opensearch.script.ScriptContext;
+import org.opensearch.script.ScriptFactory;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Script for derived fields
+ *
+ * @opensearch.internal
+ */
+public abstract class DerivedFieldScript {
+
+    public static final String[] PARAMETERS = {};
+
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("derived_field", Factory.class);
+
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DynamicMap.class);
+
+    private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of(
+        "doc", value -> value,
+        "_doc", value -> {
+            deprecationLogger.deprecate(
+                "derived-field__doc",
+                "Accessing variable [doc] via [params._doc] from within an terms-set-query-script "
+                    + "is deprecated in favor of directly accessing [doc]."
+            );
+            return value;
+        },
+        "_source", value -> ((SourceLookup) value).loadSourceIfNeeded()
+    );
+
+    /**
+     * The generic runtime parameters for the script.
+     */
+    private final Map<String, Object> params;
+
+    /**
+     * A leaf lookup for the bound segment this script will operate on.
+     */
+    private final LeafSearchLookup leafLookup;
+
+    public DerivedFieldScript(Map<String, Object> params, SearchLookup lookup, LeafReaderContext leafContext) {
+        Map<String, Object> parameters = new HashMap<>(params);
+        this.leafLookup = lookup.getLeafSearchLookup(leafContext);
+        parameters.putAll(leafLookup.asMap());
+        this.params = new DynamicMap(parameters, PARAMS_FUNCTIONS);
+    }
+
+    protected DerivedFieldScript() {
+        params = null;
+        leafLookup = null;
+    }
+
+    /**
+     * Return the parameters for this script.
+     */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    /**
+     * The doc lookup for the Lucene segment this script was created for.
+     */
+    public Map<String, ScriptDocValues<?>> getDoc() {
+        return leafLookup.doc();
+    }
+
+    /**
+     * Set the current document to run the script on next.
+     */
+    public void setDocument(int docid) {
+        leafLookup.setDocument(docid);
+    }
+
+    public abstract Object execute();
+
+    /**
+     * A factory to construct {@link DerivedFieldScript} instances.
+     *
+     * @opensearch.internal
+     */
+    public interface LeafFactory {
+        DerivedFieldScript newInstance(LeafReaderContext ctx) throws IOException;
+    }
+
+    /**
+     * A factory to construct stateful {@link DerivedFieldScript} factories for a specific index.
+     *
+     * @opensearch.internal
+     */
+    public interface Factory extends ScriptFactory {
+        LeafFactory newFactory(Map<String, Object> params, SearchLookup lookup);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java
new file mode 100644
index 0000000000000..d18556003d71b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java
@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.index.query;
+
+import org.opensearch.common.annotation.PublicApi;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Context used for script fields
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.13.0")
+public class DerivedFieldsContext {
+
+    /**
+     * Script field use in the script fields context
+     *
+     * @opensearch.api
+     */
+    @PublicApi(since = "2.13.0")
+    public static class DerivedField {
+        private final String name;
+        private final DerivedFieldScript.LeafFactory script;
+        private final boolean ignoreException;
+
+        public DerivedField(String name, DerivedFieldScript.LeafFactory script, boolean ignoreException) {
+            this.name = name;
+            this.script = script;
+            this.ignoreException = ignoreException;
+        }
+
+        public String name() {
+            return name;
+        }
+
+        public DerivedFieldScript.LeafFactory script() {
+            return this.script;
+        }
+
+        public boolean ignoreException() {
+            return ignoreException;
+        }
+    }
+
+    private List<DerivedField> fields = new ArrayList<>();
+
+    public DerivedFieldsContext() {}
+
+    public void add(DerivedField field) {
+        this.fields.add(field);
+    }
+
+    public List<DerivedField> fields() {
+        return this.fields;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/script/ScriptModule.java b/server/src/main/java/org/opensearch/script/ScriptModule.java
index a192e9553016b..20eccd82e43d2 100644
--- a/server/src/main/java/org/opensearch/script/ScriptModule.java
+++ b/server/src/main/java/org/opensearch/script/ScriptModule.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.index.query.DerivedFieldScript;
 import org.opensearch.index.query.IntervalFilterScript;
 import org.opensearch.plugins.ScriptPlugin;
 import org.opensearch.search.aggregations.pipeline.MovingFunctionScript;
@@ -78,7 +79,8 @@ public class ScriptModule {
             ScriptedMetricAggContexts.MapScript.CONTEXT,
             ScriptedMetricAggContexts.CombineScript.CONTEXT,
             ScriptedMetricAggContexts.ReduceScript.CONTEXT,
-            IntervalFilterScript.CONTEXT
+            IntervalFilterScript.CONTEXT,
+            DerivedFieldScript.CONTEXT
         ).collect(Collectors.toMap(c -> c.name, Function.identity()));
     }
 

From 848cf49b242d2c577fffe8269763d7f2ceb64d82 Mon Sep 17 00:00:00 2001
From: Rishabh Maurya <rishabhmaurya05@gmail.com>
Date: Thu, 14 Mar 2024 23:19:06 -0700
Subject: [PATCH 11/13] Add support for query time derived fields and retrieval
 in fetch phase

---
 .../index/mapper/DerivedFieldMapper.java      | 25 ++++++++++++++++---
 .../index/mapper/DocumentMapperParser.java    |  2 +-
 .../index/query/DerivedFieldQuery.java        |  2 +-
 .../index/query/QueryShardContext.java        | 19 +++++++++++++-
 .../org/opensearch/search/SearchService.java  |  9 +++++++
 .../search/builder/SearchSourceBuilder.java   | 24 ++++++++++++++++++
 .../search/fetch/subphase/FieldFetcher.java   |  6 +++++
 7 files changed, 81 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index e2a204ef7aa2a..5885d08ec9ce4 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.document.LongField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
 import org.opensearch.common.Nullable;
@@ -27,6 +28,7 @@
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.script.Script;
 import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.time.ZoneId;
@@ -43,7 +45,7 @@
  */
 public class DerivedFieldMapper extends ParametrizedFieldMapper {
 
-    public static final String CONTENT_TYPE = "derived_field";
+    public static final String CONTENT_TYPE = "derived";
 
     /**
      * Default parameters for the boolean field mapper
@@ -191,14 +193,31 @@ public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchL
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
+            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+            DerivedFieldScript.LeafFactory derivedFieldScriptFactory = factory.newFactory(script.getParams(), context.lookup());
 
-            // TODO Return to this during Query implementation. The derived fields don't typically exist in _source but
-            //  we may want fetch the field name from source if a 'script' is not provided.
             return new SourceValueFetcher(name(), context) {
+                DerivedFieldScript derivedFieldScript;
+
+                @Override
+                public List<Object> fetchValues(SourceLookup lookup) {
+                    derivedFieldScript.setDocument(lookup.docId());
+                    // TODO: remove List.of() when derivedFieldScript.execute() returns list of objects.
+                    return List.of(derivedFieldScript.execute());
+                }
+
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;
                 }
+
+                public void setNextReader(LeafReaderContext context) {
+                    try {
+                        derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
             };
         }
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
index fa736f84895eb..7c69b679e5a55 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
@@ -133,7 +133,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th
     }
 
     @SuppressWarnings({ "unchecked" })
-    private DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
+    public DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
         if (type == null) {
             throw new MapperParsingException("Failed to derive type");
         }
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
index b5e8d5e3f782d..db98aaee90fc4 100644
--- a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
@@ -91,7 +91,7 @@ public boolean matches() {
                         // TODO: in case of errors from script, should it be ignored and treated as missing field
                         //  by using a configurable setting?
                         MemoryIndex memoryIndex = new MemoryIndex();
-                        // TODO add support for multi-field with use of emit()
+                        // TODO add support for multi-field with use of emit() once available
                         memoryIndex.addField(fieldFunction.apply(value), indexAnalyzer);
                         float score = memoryIndex.search(filter);
                         return score > 0.0f;
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
index f3b392559d33e..ed4b9af14ff65 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
@@ -58,6 +58,7 @@
 import org.opensearch.index.cache.bitset.BitsetFilterCache;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.Mapper;
@@ -119,6 +120,8 @@ public class QueryShardContext extends QueryRewriteContext {
     private final ValuesSourceRegistry valuesSourceRegistry;
     private BitSetProducer parentFilter;
 
+    private DocumentMapper derivedFieldMappers;
+
     public QueryShardContext(
         int shardId,
         IndexSettings indexSettings,
@@ -264,6 +267,7 @@ private QueryShardContext(
         this.fullyQualifiedIndex = fullyQualifiedIndex;
         this.allowExpensiveQueries = allowExpensiveQueries;
         this.valuesSourceRegistry = valuesSourceRegistry;
+        derivedFieldMappers = null;
     }
 
     private void reset() {
@@ -395,6 +399,14 @@ public ValuesSourceRegistry getValuesSourceRegistry() {
         return valuesSourceRegistry;
     }
 
+    public void setDerivedFieldMappers(DocumentMapper derivedFieldMappers) {
+        this.derivedFieldMappers = derivedFieldMappers;
+    }
+
+    public DocumentMapper getDerivedFieldsMapper() {
+        return derivedFieldMappers;
+    }
+
     public void setAllowUnmappedFields(boolean allowUnmappedFields) {
         this.allowUnmappedFields = allowUnmappedFields;
     }
@@ -404,7 +416,12 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
     }
 
     MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) {
-        if (fieldMapping != null || allowUnmappedFields) {
+        if (fieldMapping != null) {
+            return fieldMapping;
+        } else if(derivedFieldMappers != null && derivedFieldMappers.mappers() != null
+            && derivedFieldMappers.mappers().getMapper(name) != null) {
+            return ((DerivedFieldMapper) derivedFieldMappers.mappers().getMapper(name)).fieldType();
+        } else if (allowUnmappedFields) {
             return fieldMapping;
         } else if (mapUnmappedFieldAsString) {
             TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index 62eb597e387e6..690430d49bea4 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -77,6 +77,8 @@
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.Engine;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.query.InnerHitContextBuilder;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.MatchNoneQueryBuilder;
@@ -1066,6 +1068,13 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
             // might end up with incorrect state since we are using now() or script services
             // during rewrite and normalized / evaluate templates etc.
             QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
+            if (request.source().derivedFields() != null && request.source().size() != 0) {
+                Map<String, Object> derivedFieldObject = new HashMap<>();
+                derivedFieldObject.put(DerivedFieldMapper.CONTENT_TYPE, request.source().derivedFields());
+                DocumentMapper documentMapper = searchContext.mapperService().documentMapperParser().parse(DerivedFieldMapper.CONTENT_TYPE, derivedFieldObject);
+                context.setDerivedFieldMappers(documentMapper);
+                searchContext.getQueryShardContext().setDerivedFieldMappers(documentMapper);
+            }
             Rewriteable.rewrite(request.getRewriteable(), context, true);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
index 1a5a9dc6d1f03..ae49bd16d5b40 100644
--- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
@@ -51,6 +51,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentHelper;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryRewriteContext;
 import org.opensearch.index.query.Rewriteable;
@@ -113,6 +114,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields");
     public static final ParseField FETCH_FIELDS_FIELD = new ParseField("fields");
     public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
+    public static final ParseField DERIVED_FIELDS_FIELD = new ParseField(DerivedFieldMapper.CONTENT_TYPE);
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
     public static final ParseField SORT_FIELD = new ParseField("sort");
@@ -192,6 +194,7 @@ public static HighlightBuilder highlight() {
     private StoredFieldsContext storedFieldsContext;
     private List<FieldAndFormat> docValueFields;
     private List<ScriptField> scriptFields;
+    private Map<String, Object> derivedFields;
     private FetchSourceContext fetchSourceContext;
     private List<FieldAndFormat> fetchFields;
 
@@ -247,6 +250,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             scriptFields = in.readList(ScriptField::new);
         }
+        if (in.readBoolean()) {
+            derivedFields = in.readMap();
+        }
         size = in.readVInt();
         if (in.readBoolean()) {
             int size = in.readVInt();
@@ -310,6 +316,11 @@ public void writeTo(StreamOutput out) throws IOException {
         if (hasScriptFields) {
             out.writeList(scriptFields);
         }
+        boolean hasDerivedFields = derivedFields != null;
+        out.writeBoolean(hasDerivedFields);
+        if (hasDerivedFields) {
+            out.writeMap(derivedFields);
+        }
         out.writeVInt(size);
         boolean hasSorts = sorts != null;
         out.writeBoolean(hasSorts);
@@ -955,6 +966,10 @@ public List<ScriptField> scriptFields() {
         return scriptFields;
     }
 
+    public Map<String, Object> derivedFields() {
+        return derivedFields;
+    }
+
     /**
      * Sets the boost a specific index or alias will receive when the query is executed
      * against it.
@@ -1119,6 +1134,7 @@ private SearchSourceBuilder shallowCopy(
         rewrittenBuilder.queryBuilder = queryBuilder;
         rewrittenBuilder.rescoreBuilders = rescoreBuilders;
         rewrittenBuilder.scriptFields = scriptFields;
+        rewrittenBuilder.derivedFields = derivedFields;
         rewrittenBuilder.searchAfterBuilder = searchAfterBuilder;
         rewrittenBuilder.sliceBuilder = slice;
         rewrittenBuilder.size = size;
@@ -1220,6 +1236,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         scriptFields.add(new ScriptField(parser));
                     }
+                } else if (DERIVED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    derivedFields = parser.map();
                 } else if (INDICES_BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     deprecationLogger.deprecate(
                         "indices_boost_object_format",
@@ -1434,6 +1452,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
             builder.endObject();
         }
 
+        if (derivedFields != null) {
+            builder.field(DERIVED_FIELDS_FIELD.getPreferredName(), derivedFields);
+        }
+
         if (sorts != null) {
             builder.startArray(SORT_FIELD.getPreferredName());
             for (SortBuilder<?> sort : sorts) {
@@ -1772,6 +1794,7 @@ public int hashCode() {
             queryBuilder,
             rescoreBuilders,
             scriptFields,
+            derivedFields,
             size,
             sorts,
             searchAfterBuilder,
@@ -1815,6 +1838,7 @@ public boolean equals(Object obj) {
             && Objects.equals(queryBuilder, other.queryBuilder)
             && Objects.equals(rescoreBuilders, other.rescoreBuilders)
             && Objects.equals(scriptFields, other.scriptFields)
+            && Objects.equals(derivedFields, other.derivedFields)
             && Objects.equals(size, other.size)
             && Objects.equals(sorts, other.sorts)
             && Objects.equals(searchAfterBuilder, other.searchAfterBuilder)
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/opensearch/search/fetch/subphase/FieldFetcher.java
index e2402c03c0bd6..2c1b90f7da9a6 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/FieldFetcher.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/FieldFetcher.java
@@ -34,6 +34,8 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.opensearch.common.document.DocumentField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.ValueFetcher;
 import org.opensearch.index.query.QueryShardContext;
@@ -66,6 +68,10 @@ public static FieldFetcher create(QueryShardContext context, SearchLookup search
             Collection<String> concreteFields = context.simpleMatchToIndexNames(fieldPattern);
             for (String field : concreteFields) {
                 MappedFieldType ft = context.getFieldType(field);
+                if (ft == null && context.getDerivedFieldsMapper() != null) {
+                    DocumentMapper derivedFieldsMapper = context.getDerivedFieldsMapper();
+                    ft = ((DerivedFieldMapper) derivedFieldsMapper.mappers().getMapper(field)).fieldType();
+                }
                 if (ft == null || context.isMetadataField(field)) {
                     continue;
                 }

From 4cb430308bce3b32ab610b95ff5a9058f6b051be Mon Sep 17 00:00:00 2001
From: Rishabh Maurya <rishabhmaurya05@gmail.com>
Date: Fri, 15 Mar 2024 16:42:31 -0700
Subject: [PATCH 12/13] Code refactor to split classes

---
 .../index/mapper/DerivedFieldMapper.java      | 220 +-----------------
 .../mapper/DerivedFieldSupportedTypes.java    | 101 ++++++++
 .../index/mapper/DerivedFieldType.java        | 155 ++++++++++++
 .../mapper/DerivedFieldValueFetcher.java      |  54 +++++
 .../index/query/DerivedFieldQuery.java        |  80 ++++---
 5 files changed, 353 insertions(+), 257 deletions(-)
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
 create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java

diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index 5885d08ec9ce4..c02b618bfccd5 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -8,34 +8,15 @@
 
 package org.opensearch.index.mapper;
 
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.KeywordField;
-import org.apache.lucene.document.LongField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.Query;
-import org.opensearch.common.Nullable;
-import org.opensearch.common.geo.ShapeRelation;
-import org.opensearch.common.lucene.Lucene;
-import org.opensearch.common.time.DateMathParser;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.index.query.DerivedFieldQuery;
-import org.opensearch.index.query.DerivedFieldScript;
-import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.script.Script;
-import org.opensearch.search.lookup.SearchLookup;
-import org.opensearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
-import java.time.ZoneId;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Function;
 
 /**
@@ -94,7 +75,6 @@ public Builder(String name) {
             super(name);
         }
 
-
         @Override
         protected List<Parameter<?>> getParameters() {
             return Arrays.asList(type, script);
@@ -102,208 +82,16 @@ protected List<Parameter<?>> getParameters() {
 
         @Override
         public DerivedFieldMapper build(BuilderContext context) {
-            FieldMapper typeFieldMapper;
-            Function<Object, IndexableField> fieldFunction;
-            switch (type.getValue()) {
-                // TODO: add logic all supported type in derived fields
-                // TODO: should we support mapping settings exposed by a given field type from derived fields too?
-                // for example, support `format` for date type?
-                case KeywordFieldMapper.CONTENT_TYPE:
-                    FieldType dummyFieldType = new FieldType();
-                    dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
-                    KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name());
-                    KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
-                    keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
-                    typeFieldMapper = new KeywordFieldMapper(
-                        name(), dummyFieldType, keywordFieldType,
-                        keywordBuilder.multiFieldsBuilder.build(this, context),
-                        keywordBuilder.copyTo.build(),
-                        keywordBuilder
-                    );
-                    fieldFunction = o -> new KeywordField(name(), (String) o, Field.Store.NO);
-                    break;
-                case "long":
-                    // ignoreMalformed?
-                    NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
-                    typeFieldMapper = longBuilder.build(context);
-                    fieldFunction = o -> new LongField(name(), Long.parseLong(o.toString()), Field.Store.NO);
-                    break;
-                case "double":
-                    // ignoreMalformed?
-                    NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
-                    typeFieldMapper = doubleBuilder.build(context);
-                    fieldFunction = o -> new DoubleField(name(), Double.parseDouble(o.toString()), Field.Store.NO);
-                    break;
-                default:
-                    throw new IllegalArgumentException("Field [" + name() + "] of type [" + type + "] isn't supported " +
-                        "in Derived field context.");
-
-            }
-            MappedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), typeFieldMapper, fieldFunction);
+            FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
+            Function<Object, IndexableField> fieldFunction =
+                DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type.getValue(), name);
+            DerivedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), fieldMapper, fieldFunction);
             return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
         }
     }
 
-
     public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
-
-    /**
-     * Field type for derived field mapper
-     *
-     * @opensearch.internal
-     */
-    public static final class DerivedFieldType extends MappedFieldType {
-        private final String type;
-
-        private final Script script;
-
-        FieldMapper typeFieldMapper;
-
-        private final Function<Object, IndexableField> fieldFunction;
-
-        public DerivedFieldType(
-            String name,
-            String type,
-            Script script,
-            boolean isIndexed,
-            boolean isStored,
-            boolean hasDocValues,
-            Map<String, String> meta,
-            FieldMapper typeFieldMapper,
-            Function<Object, IndexableField> fieldFunction
-        ) {
-            super(name, isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
-            this.type = type;
-            this.script = script;
-            this.typeFieldMapper = typeFieldMapper;
-            this.fieldFunction = fieldFunction;
-        }
-
-        public DerivedFieldType(String name, String type, Script script, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
-            this(name, type, script, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
-        }
-
-        @Override
-        public String typeName() {
-            return CONTENT_TYPE;
-        }
-
-        @Override
-        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
-            if (format != null) {
-                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
-            }
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScriptFactory = factory.newFactory(script.getParams(), context.lookup());
-
-            return new SourceValueFetcher(name(), context) {
-                DerivedFieldScript derivedFieldScript;
-
-                @Override
-                public List<Object> fetchValues(SourceLookup lookup) {
-                    derivedFieldScript.setDocument(lookup.docId());
-                    // TODO: remove List.of() when derivedFieldScript.execute() returns list of objects.
-                    return List.of(derivedFieldScript.execute());
-                }
-
-                @Override
-                protected Object parseSourceValue(Object value) {
-                    return value;
-                }
-
-                public void setNextReader(LeafReaderContext context) {
-                    try {
-                        derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
-                    } catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                }
-            };
-        }
-
-        @Override
-        public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
-            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
-        }
-
-        @Override
-        public Query prefixQuery(
-            String value,
-            @Nullable MultiTermQuery.RewriteMethod method,
-            boolean caseInsensitive,
-            QueryShardContext context
-        ) {
-            Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
-            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
-        }
-
-        @Override
-        public Query wildcardQuery(
-            String value,
-            @Nullable MultiTermQuery.RewriteMethod method,
-            boolean caseInsensitive,
-            QueryShardContext context
-        ) {
-            Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
-            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
-        }
-
-        @Override
-        public Query regexpQuery(
-            String value,
-            int syntaxFlags,
-            int matchFlags,
-            int maxDeterminizedStates,
-            @Nullable MultiTermQuery.RewriteMethod method,
-            QueryShardContext context
-        ) {
-            Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags,
-                maxDeterminizedStates, method, context);
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
-            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
-        }
-
-        // TODO: Override all types of queries which can be supported by all types supported within derived fields.
-        @Override
-        public Query rangeQuery(
-            Object lowerTerm,
-            Object upperTerm,
-            boolean includeLower,
-            boolean includeUpper,
-            ShapeRelation relation,
-            ZoneId timeZone,
-            DateMathParser parser,
-            QueryShardContext context
-        ) {
-            Query query = typeFieldMapper.mappedFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper,
-                relation, timeZone, parser, context);
-            DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
-            DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
-            return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
-        }
-
-        @Override
-        public Query existsQuery(QueryShardContext context) {
-            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exist queries");
-        }
-
-        @Override
-        public boolean isAggregatable() {
-            return false;
-        }
-    }
-
-
     private final String type;
-
     private final Script script;
 
     protected DerivedFieldMapper(
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
new file mode 100644
index 0000000000000..c70a79441cbf3
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
@@ -0,0 +1,101 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.opensearch.common.lucene.Lucene;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public enum DerivedFieldSupportedTypes {
+    KEYWORD (
+        "keyword",
+        (name, context) -> {
+            FieldType dummyFieldType = new FieldType();
+            dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+            KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name);
+            KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
+            keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            return new KeywordFieldMapper(
+                name, dummyFieldType, keywordFieldType,
+                keywordBuilder.multiFieldsBuilder.build(keywordBuilder, context),
+                keywordBuilder.copyTo.build(),
+                keywordBuilder
+            );
+        },
+        name -> o -> new KeywordField(name, (String) o, Field.Store.NO)
+    ),
+    LONG (
+        "long",
+        (name, context) -> {
+            NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
+            return longBuilder.build(context);
+        },
+        name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO)
+    ),
+    DOUBLE (
+        "double",
+        (name, context) -> {
+            NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
+            return doubleBuilder.build(context);
+        },
+        name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO)
+    );
+    // TODO: add logic all supported type in derived fields
+    // TODO: should we support mapping settings exposed by a given field type from derived fields too?
+    // for example, support `format` for date type?
+    final String name;
+    private final BiFunction<String, Mapper.BuilderContext, FieldMapper> builder;
+
+    private final Function<String, Function<Object, IndexableField>> indexableFieldBuilder;
+
+    DerivedFieldSupportedTypes(String name, BiFunction<String, Mapper.BuilderContext, FieldMapper> builder,
+                               Function<String, Function<Object, IndexableField>> indexableFieldBuilder){
+        this.name = name;
+        this.builder = builder;
+        this.indexableFieldBuilder = indexableFieldBuilder;
+    }
+
+    public String getName() {
+        return name;
+    }
+    private FieldMapper getFieldMapper(String name, Mapper.BuilderContext context) {
+        return builder.apply(name, context);
+    }
+
+    private Function<Object, IndexableField> getIndexableFieldGenerator(String name) {
+        return indexableFieldBuilder.apply(name);
+    }
+    private static final Map<String, DerivedFieldSupportedTypes> enumMap = Arrays.stream(DerivedFieldSupportedTypes.values())
+        .collect(Collectors.toMap(DerivedFieldSupportedTypes::getName, enumValue -> enumValue));
+
+    public static FieldMapper getFieldMapperFromType(String type, String name, Mapper.BuilderContext context) {
+        if (!enumMap.containsKey(type)) {
+            throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+        }
+        return enumMap.get(type).getFieldMapper(name, context);
+    }
+
+    public static Function<Object, IndexableField> getIndexableFieldGeneratorType(String type, String name) {
+        if (!enumMap.containsKey(type)) {
+            throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+        }
+        return enumMap.get(type).getIndexableFieldGenerator(name);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
new file mode 100644
index 0000000000000..53887e85bd0cd
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
@@ -0,0 +1,155 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.Query;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.geo.ShapeRelation;
+import org.opensearch.common.time.DateMathParser;
+import org.opensearch.index.query.DerivedFieldQuery;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.Script;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.time.ZoneId;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
+
+
+/**
+ * MappedFieldType for {@link DerivedFieldMapper}
+ * Contains logic to different type of queries on derived fields
+ * @opensearch.internal
+ */
+public final class DerivedFieldType extends MappedFieldType {
+    private final String type;
+
+    private final Script script;
+
+    FieldMapper typeFieldMapper;
+
+    private final Function<Object, IndexableField> indexableFieldGenerator;
+
+    public DerivedFieldType(
+        String name,
+        String type,
+        Script script,
+        boolean isIndexed,
+        boolean isStored,
+        boolean hasDocValues,
+        Map<String, String> meta,
+        FieldMapper typeFieldMapper,
+        Function<Object, IndexableField> fieldFunction
+    ) {
+        super(name, isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
+        this.type = type;
+        this.script = script;
+        this.typeFieldMapper = typeFieldMapper;
+        this.indexableFieldGenerator = fieldFunction;
+    }
+
+    public DerivedFieldType(String name, String type, Script script, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
+        this(name, type, script, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
+    }
+
+    @Override
+    public String typeName() {
+        return DerivedFieldMapper.CONTENT_TYPE;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    @Override
+    public DerivedFieldValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+        return new DerivedFieldValueFetcher(context, script);
+    }
+
+    @Override
+    public Query termQuery(Object value, @Nullable QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
+        DerivedFieldValueFetcher valueFetcher =  new DerivedFieldValueFetcher(context, script);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), indexableFieldGenerator, typeFieldMapper.mappedFieldType.indexAnalyzer());
+    }
+
+    @Override
+    public Query prefixQuery(
+        String value,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        boolean caseInsensitive,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
+        DerivedFieldValueFetcher valueFetcher =  new DerivedFieldValueFetcher(context, script);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), indexableFieldGenerator, typeFieldMapper.mappedFieldType.indexAnalyzer());
+    }
+
+    @Override
+    public Query wildcardQuery(
+        String value,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        boolean caseInsensitive,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
+        DerivedFieldValueFetcher valueFetcher =  new DerivedFieldValueFetcher(context, script);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), indexableFieldGenerator, typeFieldMapper.mappedFieldType.indexAnalyzer());
+    }
+
+    @Override
+    public Query regexpQuery(
+        String value,
+        int syntaxFlags,
+        int matchFlags,
+        int maxDeterminizedStates,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags,
+            maxDeterminizedStates, method, context);
+        DerivedFieldValueFetcher valueFetcher =  new DerivedFieldValueFetcher(context, script);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), indexableFieldGenerator, typeFieldMapper.mappedFieldType.indexAnalyzer());
+    }
+
+    // TODO: Override all types of queries which can be supported by all types supported within derived fields.
+    @Override
+    public Query rangeQuery(
+        Object lowerTerm,
+        Object upperTerm,
+        boolean includeLower,
+        boolean includeUpper,
+        ShapeRelation relation,
+        ZoneId timeZone,
+        DateMathParser parser,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper,
+            relation, timeZone, parser, context);
+        DerivedFieldValueFetcher valueFetcher =  new DerivedFieldValueFetcher(context, script);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), indexableFieldGenerator, typeFieldMapper.mappedFieldType.indexAnalyzer());
+    }
+
+    @Override
+    public Query existsQuery(QueryShardContext context) {
+        throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exist queries");
+    }
+
+    @Override
+    public boolean isAggregatable() {
+        return false;
+    }
+}
+
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
new file mode 100644
index 0000000000000..dd11b2ddd8f5f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
@@ -0,0 +1,54 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.opensearch.index.query.DerivedFieldScript;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.Script;
+import org.opensearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * ValueFetcher used by Derived Fields.
+ */
+public final class DerivedFieldValueFetcher implements ValueFetcher {
+    private DerivedFieldScript derivedFieldScript;
+    private final DerivedFieldScript.LeafFactory derivedFieldScriptFactory;
+
+    DerivedFieldValueFetcher(QueryShardContext context, Script script) {
+        if (!context.documentMapper("").sourceMapper().enabled()) {
+            throw new IllegalArgumentException(
+                "DerivedFieldQuery error: unable to fetch fields from _source field: _source is disabled in the mappings "
+                    + "for index ["
+                    + context.index().getName()
+                    + "]"
+            );
+        }
+        DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
+        derivedFieldScriptFactory = factory.newFactory(script.getParams(), context.lookup());
+    }
+
+    @Override
+    public List<Object> fetchValues(SourceLookup lookup) {
+        derivedFieldScript.setDocument(lookup.docId());
+        // TODO: remove List.of() when derivedFieldScript.execute() returns list of objects.
+        return List.of(derivedFieldScript.execute());
+    }
+
+    public void setNextReader(LeafReaderContext context) {
+        try {
+            derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
index db98aaee90fc4..948f26be6db90 100644
--- a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
@@ -22,44 +22,39 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
+import org.opensearch.index.mapper.DerivedFieldValueFetcher;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Objects;
 import java.util.function.Function;
 
 /**
  * DerivedFieldQuery used for querying derived fields.
  */
-public class DerivedFieldQuery extends Query {
-    private final Query filter;
-    private final DerivedFieldScript.LeafFactory derivedFieldScriptFactory;
+public final class DerivedFieldQuery extends Query {
+    private final Query query;
+    private final DerivedFieldValueFetcher valueFetcher;
+    private final SearchLookup searchLookup;
+    private final Function<Object, IndexableField> indexableFieldGenerator;
     private final Analyzer indexAnalyzer;
-    private final Function<Object, IndexableField> fieldFunction;
-    private final QueryShardContext context;
 
     /**
-     * @param query                     query to execute against memory index
-     * @param derivedFieldScriptFactory derived field script's LeafFactory instance
-     * @param indexAnalyzer             index analyzer to use while building memory index
-     * @param fieldFunction             IndexableField generator
-     * @param context                   QueryShardContext
+     *
+     * @param query
+     * @param valueFetcher
+     * @param searchLookup
+     * @param indexableFieldGenerator
      */
-    public DerivedFieldQuery(Query query, DerivedFieldScript.LeafFactory derivedFieldScriptFactory,
-                             Analyzer indexAnalyzer, Function<Object, IndexableField> fieldFunction,
-                             QueryShardContext context) {
-        this.filter = query;
-        this.derivedFieldScriptFactory = derivedFieldScriptFactory;
+    public DerivedFieldQuery(Query query, DerivedFieldValueFetcher valueFetcher, SearchLookup searchLookup,
+                             Function<Object, IndexableField> indexableFieldGenerator, Analyzer indexAnalyzer) {
+        this.query = query;
+        this.valueFetcher = valueFetcher;
+        this.searchLookup = searchLookup;
+        this.indexableFieldGenerator = indexableFieldGenerator;
         this.indexAnalyzer = indexAnalyzer;
-        this.context = context;
-        this.fieldFunction = fieldFunction;
-        if (!context.documentMapper("").sourceMapper().enabled()) {
-            throw new IllegalArgumentException(
-                "DerivedFieldQuery error: unable to fetch fields from _source field: _source is disabled in the mappings "
-                    + "for index ["
-                    + context.index().getName()
-                    + "]"
-            );
-        }
     }
 
     @Override
@@ -69,31 +64,34 @@ public void visit(QueryVisitor visitor) {
 
     @Override
     public Query rewrite(IndexSearcher indexSearcher) throws IOException {
-        Query rewritten = indexSearcher.rewrite(filter);
-        if (rewritten == filter) {
+        Query rewritten = indexSearcher.rewrite(query);
+        if (rewritten == query) {
             return this;
         }
-        return new DerivedFieldQuery(rewritten, derivedFieldScriptFactory, indexAnalyzer, fieldFunction, context);
+        return new DerivedFieldQuery(rewritten, valueFetcher, searchLookup, indexableFieldGenerator, indexAnalyzer);
     }
 
     @Override
     public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+
         return new ConstantScoreWeight(this, boost) {
             @Override
             public Scorer scorer(LeafReaderContext context) throws IOException {
                 DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
-                DerivedFieldScript derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
+                valueFetcher.setNextReader(context);
+                LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(context);
                 TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
                     @Override
                     public boolean matches() {
-                        derivedFieldScript.setDocument(approximation.docID());
-                        Object value = derivedFieldScript.execute();
+                        leafSearchLookup.source().setSegmentAndDocument(context, approximation.docID());
+                        List<Object> values = valueFetcher.fetchValues(leafSearchLookup.source());
                         // TODO: in case of errors from script, should it be ignored and treated as missing field
                         //  by using a configurable setting?
                         MemoryIndex memoryIndex = new MemoryIndex();
-                        // TODO add support for multi-field with use of emit() once available
-                        memoryIndex.addField(fieldFunction.apply(value), indexAnalyzer);
-                        float score = memoryIndex.search(filter);
+                        for (Object value : values) {
+                            memoryIndex.addField(indexableFieldGenerator.apply(value), indexAnalyzer);
+                        }
+                        float score = memoryIndex.search(query);
                         return score > 0.0f;
                     }
 
@@ -125,20 +123,20 @@ public boolean equals(Object o) {
             return false;
         }
         DerivedFieldQuery other = (DerivedFieldQuery) o;
-        return Objects.equals(this.filter, other.filter)
-            && Objects.equals(this.derivedFieldScriptFactory, other.derivedFieldScriptFactory)
-            && Objects.equals(this.fieldFunction, other.fieldFunction)
-            && Objects.equals(this.indexAnalyzer, other.indexAnalyzer)
-            && Objects.equals(this.context, other.context);
+        return Objects.equals(this.query, other.query)
+            && Objects.equals(this.valueFetcher, other.valueFetcher)
+            && Objects.equals(this.searchLookup, other.searchLookup)
+            && Objects.equals(this.indexableFieldGenerator, other.indexableFieldGenerator)
+            && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(classHash(), filter, derivedFieldScriptFactory, fieldFunction, indexAnalyzer, context);
+        return Objects.hash(classHash(), query, valueFetcher, searchLookup, indexableFieldGenerator, indexableFieldGenerator);
     }
 
     @Override
     public String toString(String f) {
-        return "DerivedFieldQuery (filter query: [ " + filter.toString(f) + "])";
+        return "DerivedFieldQuery (Query: [ " + query.toString(f) + "])";
     }
 }

From 4bb0dc3a26466bb5fec27d7b70af1575e4283e90 Mon Sep 17 00:00:00 2001
From: Rishabh Maurya <rishabhmaurya05@gmail.com>
Date: Fri, 15 Mar 2024 16:48:28 -0700
Subject: [PATCH 13/13] remove unused file

---
 .../index/query/DerivedFieldsContext.java     | 89 -------------------
 1 file changed, 89 deletions(-)
 delete mode 100644 server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java

diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java
deleted file mode 100644
index d18556003d71b..0000000000000
--- a/server/src/main/java/org/opensearch/index/query/DerivedFieldsContext.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/*
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
- */
-
-package org.opensearch.index.query;
-
-import org.opensearch.common.annotation.PublicApi;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Context used for script fields
- *
- * @opensearch.api
- */
-@PublicApi(since = "2.13.0")
-public class DerivedFieldsContext {
-
-    /**
-     * Script field use in the script fields context
-     *
-     * @opensearch.api
-     */
-    @PublicApi(since = "2.13.0")
-    public static class DerivedField {
-        private final String name;
-        private final DerivedFieldScript.LeafFactory script;
-        private final boolean ignoreException;
-
-        public DerivedField(String name, DerivedFieldScript.LeafFactory script, boolean ignoreException) {
-            this.name = name;
-            this.script = script;
-            this.ignoreException = ignoreException;
-        }
-
-        public String name() {
-            return name;
-        }
-
-        public DerivedFieldScript.LeafFactory script() {
-            return this.script;
-        }
-
-        public boolean ignoreException() {
-            return ignoreException;
-        }
-    }
-
-    private List<DerivedField> fields = new ArrayList<>();
-
-    public DerivedFieldsContext() {}
-
-    public void add(DerivedField field) {
-        this.fields.add(field);
-    }
-
-    public List<DerivedField> fields() {
-        return this.fields;
-    }
-}