Auto-convert V6 configuration instances into V7 configuration instances (for OpenSearch 2.x only)

src/integrationTest/java/org/opensearch/security/legacy/LegacyConfigV6AutoConversionTest.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security.legacy;
+
+import java.util.Map;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class LegacyConfigV6AutoConversionTest {
+    static final TestSecurityConfig LEGACY_CONFIG = new TestSecurityConfig()//
+        .rawConfigurationDocumentYaml(
+            "config",
+            "opendistro_security:\n"
+                + "  dynamic:\n"
+                + "    authc:\n"
+                + "      basic_internal_auth_domain:\n"
+                + "        http_enabled: true\n"
+                + "        order: 4\n"
+                + "        http_authenticator:\n"
+                + "          type: basic\n"
+                + "          challenge: true\n"
+                + "        authentication_backend:\n"
+                + "          type: intern\n"
+        )//
+        .rawConfigurationDocumentYaml(
+            "internalusers",
+            "admin:\n"
+                + "  readonly: true\n"
+                + "  hash: $2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG\n"
+                + "  roles:\n"
+                + "  - admin\n"
+                + "  attributes:\n"
+                + "    attribute1: value1\n"
+        )//
+        .rawConfigurationDocumentYaml(
+            "roles",
+            "all_access_role:\n"
+                + "  readonly: true\n"
+                + "  cluster:\n"
+                + "  - UNLIMITED\n"
+                + "  indices:\n"
+                + "    '*':\n"
+                + "      '*':\n"
+                + "      - UNLIMITED\n"
+                + "  tenants:\n"
+                + "    admin_tenant: RW\n"
+        )//
+        .rawConfigurationDocumentYaml("rolesmapping", "all_access_role:\n" + "  readonly: true\n" + "  backendroles:\n" + "  - admin")//
+        .rawConfigurationDocumentYaml("actiongroups", "dummy:\n" + "  permissions: []");
+
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
+        .config(LEGACY_CONFIG)
+        .nodeSettings(Map.of("plugins.security.restapi.roles_enabled.0", "all_access_role"))
+        .build();
+
+    @Test
+    public void authc() {
+        try (TestRestClient client = cluster.getRestClient("admin", "admin")) {
+            TestRestClient.HttpResponse response = client.get("_opendistro/_security/authinfo");
+            Assert.assertEquals(response.getBody(), 200, response.getStatusCode());
+            Assert.assertEquals(response.getBody(), true, response.getBooleanFromJsonBody("/tenants/admin"));
+        }
+    }
+
+    @Test
+    public void configRestApiReturnsV6Config() {
+        try (TestRestClient client = cluster.getRestClient("admin", "admin")) {
+            TestRestClient.HttpResponse response = client.get("_opendistro/_security/api/roles/all_access_role");
+            Assert.assertEquals(response.getBody(), 200, response.getStatusCode());
+            Assert.assertEquals(
+                "Expected v6 format",
+                "{\"all_access_role\":{\"readonly\":true,\"hidden\":false,\"cluster\":[\"UNLIMITED\"],\"tenants\":{\"admin_tenant\":\"RW\"},\"indices\":{\"*\":{\"*\":[\"UNLIMITED\"]}}}}",
+                response.getBody()
+            );
+        }
+    }
+
+    /**
+     * This must be the last test executed, as it changes the config index
+     */
+    @Test
+    public void zzz_migrateApi() {
+        try (TestRestClient client = cluster.getRestClient("admin", "admin")) {
+            TestRestClient.HttpResponse response = client.post("_opendistro/_security/api/migrate");
+            Assert.assertEquals(response.getBody(), 200, response.getStatusCode());
+            Assert.assertEquals(response.getBody(), "Migration completed.", response.getTextFromJsonBody("/message"));
+            response = client.get("_opendistro/_security/api/roles/all_access_role");
+            Assert.assertEquals(response.getBody(), 200, response.getStatusCode());
+            Assert.assertEquals(
+                "Expected v7 format",
+                "Migrated from v6 (all types mapped)",
+                response.getTextFromJsonBody("/all_access_role/description")
+            );
+        }
+    }
+
+}

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -43,6 +43,9 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -94,6 +97,13 @@ public class TestSecurityConfig {
 
     private Map<String, ActionGroup> actionGroups = new LinkedHashMap<>();
 
+    /**
+     * A map from document id to a string containing config JSON.
+     * If this is not null, it will be used ALTERNATIVELY to all other configuration contained in this class.
+     * Can be used to simulate invalid configuration or legacy configuration.
+     */
+    private Map<String, String> rawConfigurationDocuments;
+
     private String indexName = ".opendistro_security";
 
     public TestSecurityConfig() {
@@ -212,6 +222,27 @@ public List<ActionGroup> actionGroups() {
         return List.copyOf(actionGroups.values());
     }
 
+    /**
+     * Specifies raw document content for the configuration index as YAML document. If this method is used,
+     * then ONLY the raw documents will be written to the configuration index. Any other configuration specified
+     * by the roles() or users() method will be ignored.
+     * Can be used to simulate invalid configuration or legacy configuration.
+     */
+    public TestSecurityConfig rawConfigurationDocumentYaml(String configTypeId, String configDocumentAsYaml) {
+        try {
+            if (this.rawConfigurationDocuments == null) {
+                this.rawConfigurationDocuments = new LinkedHashMap<>();
+            }
+
+            JsonNode node = new ObjectMapper(new YAMLFactory()).readTree(configDocumentAsYaml);
+
+            this.rawConfigurationDocuments.put(configTypeId, new ObjectMapper().writeValueAsString(node));
+            return this;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public static class Config implements ToXContentObject {
         private boolean anonymousAuth;
 
@@ -964,15 +995,24 @@ public void initIndex(Client client) {
         }
         client.admin().indices().create(new CreateIndexRequest(indexName).settings(settings)).actionGet();
 
-        writeSingleEntryConfigToIndex(client, CType.CONFIG, config);
-        if (auditConfiguration != null) {
-            writeSingleEntryConfigToIndex(client, CType.AUDIT, "config", auditConfiguration);
+        if (rawConfigurationDocuments == null) {
+            writeSingleEntryConfigToIndex(client, CType.CONFIG, config);
+            if (auditConfiguration != null) {
+                writeSingleEntryConfigToIndex(client, CType.AUDIT, "config", auditConfiguration);
+            }
+            writeConfigToIndex(client, CType.ROLES, roles);
+            writeConfigToIndex(client, CType.INTERNALUSERS, internalUsers);
+            writeConfigToIndex(client, CType.ROLESMAPPING, rolesMapping);
+            writeEmptyConfigToIndex(client, CType.ACTIONGROUPS);
+            writeEmptyConfigToIndex(client, CType.TENANTS);
+        } else {
+            // Write raw configuration alternatively to the normal configuration
+
+            for (Map.Entry<String, String> entry : this.rawConfigurationDocuments.entrySet()) {
+                writeConfigToIndex(client, entry.getKey(), entry.getValue());
+            }
         }
-        writeConfigToIndex(client, CType.ROLES, roles);
-        writeConfigToIndex(client, CType.INTERNALUSERS, internalUsers);
-        writeConfigToIndex(client, CType.ROLESMAPPING, rolesMapping);
-        writeEmptyConfigToIndex(client, CType.ACTIONGROUPS);
-        writeEmptyConfigToIndex(client, CType.TENANTS);
+
     }
 
     public void updateInternalUsersConfiguration(Client client, List<User> users) {
@@ -987,11 +1027,11 @@ static String hashPassword(final String clearTextPassword) {
         return passwordHasher.hash(clearTextPassword.toCharArray());
     }
 
-    private void writeEmptyConfigToIndex(Client client, CType configType) {
+    private void writeEmptyConfigToIndex(Client client, CType<?> configType) {
         writeConfigToIndex(client, configType, Collections.emptyMap());
     }
 
-    private void writeConfigToIndex(Client client, CType configType, Map<String, ? extends ToXContentObject> config) {
+    private void writeConfigToIndex(Client client, CType<?> configType, Map<String, ? extends ToXContentObject> config) {
         try {
             String json = configToJson(configType, config);
 
@@ -1008,11 +1048,23 @@ private void writeConfigToIndex(Client client, CType configType, Map<String, ? e
         }
     }
 
+    private void writeConfigToIndex(Client client, String documentId, String jsonString) {
+        try {
+            log.info("Writing raw security configuration into index {}:\n{}", documentId, jsonString);
+
+            BytesReference bytesReference = toByteReference(jsonString);
+            client.index(new IndexRequest(indexName).id(documentId).setRefreshPolicy(IMMEDIATE).source(documentId, bytesReference))
+                .actionGet();
+        } catch (Exception e) {
+            throw new RuntimeException("Error while initializing config for " + indexName, e);
+        }
+    }
+
     private static BytesReference toByteReference(String string) throws UnsupportedEncodingException {
         return BytesReference.fromByteBuffer(ByteBuffer.wrap(string.getBytes("utf-8")));
     }
 
-    private void updateConfigInIndex(Client client, CType configType, Map<String, ? extends ToXContentObject> config) {
+    private void updateConfigInIndex(Client client, CType<?> configType, Map<String, ? extends ToXContentObject> config) {
         try {
             String json = configToJson(configType, config);
             BytesReference bytesReference = toByteReference(json);
@@ -1025,7 +1077,7 @@ private void updateConfigInIndex(Client client, CType configType, Map<String, ?
         }
     }
 
-    private static String configToJson(CType configType, Map<String, ? extends ToXContentObject> config) throws IOException {
+    private static String configToJson(CType<?> configType, Map<String, ? extends ToXContentObject> config) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
 
         builder.startObject();
@@ -1043,11 +1095,11 @@ private static String configToJson(CType configType, Map<String, ? extends ToXCo
         return builder.toString();
     }
 
-    private void writeSingleEntryConfigToIndex(Client client, CType configType, ToXContentObject config) {
+    private void writeSingleEntryConfigToIndex(Client client, CType<?> configType, ToXContentObject config) {
         writeSingleEntryConfigToIndex(client, configType, configType.toLCString(), config);
     }
 
-    private void writeSingleEntryConfigToIndex(Client client, CType configType, String configurationRoot, ToXContentObject config) {
+    private void writeSingleEntryConfigToIndex(Client client, CType<?> configType, String configurationRoot, ToXContentObject config) {
         try {
             XContentBuilder builder = XContentFactory.jsonBuilder();
 

src/integrationTest/resources/log4j2-test.properties

@@ -46,4 +46,4 @@ logger.backendreg.appenderRef.capturing.ref = logCapturingAppender
 #logger.ldap.name=com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend
 logger.ldap.name=com.amazon.dlic.auth.ldap.backend
 logger.ldap.level=TRACE
-logger.ldap.appenderRef.capturing.ref = logCapturingAppender
+logger.ldap.appenderRef.capturing.ref = logCapturingAppender

src/main/java/org/opensearch/security/DefaultObjectMapper.java

@@ -266,6 +266,22 @@
         }
     }
 
+    @SuppressWarnings("removal")
+    public static <T> T convertValue(JsonNode jsonNode, JavaType jt) throws IOException {
+
+        final SecurityManager sm = System.getSecurityManager();
+
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> objectMapper.convertValue(jsonNode, jt));
+        } catch (final PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
     public static TypeFactory getTypeFactory() {
         return objectMapper.getTypeFactory();
     }

src/main/java/org/opensearch/security/configuration/ConfigurationChangeListener.java

@@ -26,11 +26,6 @@
 
 package org.opensearch.security.configuration;
 
-import java.util.Map;
-
-import org.opensearch.security.securityconf.impl.CType;
-import org.opensearch.security.securityconf.impl.SecurityDynamicConfiguration;
-
 /**
  * Callback function on change particular configuration
  */
@@ -39,5 +34,5 @@ public interface ConfigurationChangeListener {
     /**
      * @param configuration not null updated configuration on that was subscribe current listener
      */
-    void onChange(Map<CType, SecurityDynamicConfiguration<?>> typeToConfig);
+    void onChange(ConfigurationMap typeToConfig);
 }