[Backport 2.6] [#4421] [YCQL] Enable LDAP based authentication

Summary:
In terms of supported functionality - YCQL will support all options that are
allowed in YSQL's LDAP auth. Broadly, this includes the simple bind and
search + bind mode.

Instead of a full blown file based auth config like ysql_hba.conf in YSQL (where
the config supports many features apart from LDAP), we chose to allow LDAP
configuration using a set of gflags. This is simpler to do now. In case we later
plan to support more auth based rules based on remote ip, keyspace name, etc,
we can add a similar auth config file for YCQL.

Most code is almost just a copy paste from src/postgres/src/backend/libpq/auth.c -
  1. InitializeLDAPConnection()
  2. CheckLDAPAuth()
  3. errdetail_for_ldap() is logic present in LDAPError class

Given the minimal functionality needed, a copy paste from auth.c is a simpler
and less error-prone than making functions in auth.c generic enough to be used
from both postgres and YCQL proxy.

One difference is that - LDAP_DEPRECATED is removed and so some interface
calls to ldap library are different.

Original commit: https://phabricator.dev.yugabyte.com/D12095 / d8c7713

Test Plan:
Jenkins: rebase: 2.6, urgent
./yb_build.sh --java-test org.yb.cql.TestLDAPAuth

Reviewers: dmitry, mihnea

Reviewed By: mihnea

Subscribers: yql

Differential Revision: https://phabricator.dev.yugabyte.com/D12544

java/yb-cql/pom.xml

@@ -71,5 +71,22 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.directory.server</groupId>
+      <artifactId>apacheds-all</artifactId>
+      <version>2.0.0-M22</version>
+      <exclusions>
+        <!-- exclude additional LDIF schema files to avoid conflicts through
+          multiple copies -->
+        <exclusion>
+          <groupId>org.apache.directory.shared</groupId>
+          <artifactId>shared-ldap-schema</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.directory.api</groupId>
+          <artifactId>api-ldap-schema-data</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 </project>

java/yb-cql/src/test/java/org/yb/cql/BaseAuthenticationCQLTest.java

@@ -113,8 +113,11 @@ public void checkConnectivityWithMessage(boolean usingAuth,
       assertFalse(expectFailure);
     } catch (com.datastax.driver.core.exceptions.AuthenticationException e) {
       // If we're expecting a failure, we should be in here.
-      assertTrue(expectFailure);
-      assertTrue(e.getMessage().contains(expectedMessage));
+      assertTrue(e.getMessage(), expectFailure);
+      if (!e.getMessage().contains(expectedMessage)) {
+        LOG.info("Expecting '" + expectedMessage + "' contained in '" + e.getMessage() + "'");
+        assertTrue(false);
+      }
     }
   }
 

java/yb-cql/src/test/java/org/yb/cql/BaseCQLTest.java

@@ -457,7 +457,7 @@ protected void dropRoles() throws Exception {
       String roleName = row.getString("role");
       if (!DEFAULT_ROLE.equals(roleName)) {
         LOG.info("Dropping role " + roleName);
-        session.execute("DROP ROLE " + roleName);
+        session.execute("DROP ROLE '" + roleName + "'");
       }
     }
   }

java/yb-cql/src/test/java/org/yb/cql/TestLDAPAuth.java

@@ -0,0 +1,294 @@
+// Copyright (c) YugaByte, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied.  See the License for the specific language governing permissions and limitations
+// under the License.
+//
+package org.yb.cql;
+
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.datastax.driver.core.ProtocolOptions;
+
+import org.apache.directory.server.annotations.CreateLdapServer;
+import org.apache.directory.server.annotations.CreateTransport;
+import org.apache.directory.server.core.annotations.ApplyLdifs;
+import org.apache.directory.server.core.annotations.CreateDS;
+import org.apache.directory.server.core.annotations.CreatePartition;
+import org.apache.directory.server.core.integ.CreateLdapServerRule;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yb.YBTestRunner;
+
+// TODO(Piyush): Test with TLS and LDAPS
+
+@RunWith(value=YBTestRunner.class)
+@CreateDS(name = "myDS",
+    partitions = {
+        @CreatePartition(name = "test", suffix = "dc=myorg,dc=com")
+    })
+@CreateLdapServer(transports = {
+  @CreateTransport(protocol = "LDAP", address = "localhost", port=10389)})
+@ApplyLdifs({
+  "dn: dc=myorg,dc=com",
+  "objectClass: domain",
+  "objectClass: top",
+  "dc: myorg",
+  "dn: ou=Users,dc=myorg,dc=com",
+  "objectClass: organizationalUnit",
+  "objectClass: top",
+  "ou: Users",
+  "dn: cn=admin,ou=Users,dc=myorg,dc=com",
+  "objectClass: inetOrgPerson",
+  "objectClass: organizationalPerson",
+  "objectClass: person",
+  "objectClass: top",
+  "cn: admin",
+  "sn: Ldap",
+  "uid: adminUid",
+  "userPassword: adminPasswd",
+  "dn: cn=testUser1,ou=Users,dc=myorg,dc=com",
+  "objectClass: inetOrgPerson",
+  "objectClass: organizationalPerson",
+  "objectClass: person",
+  "objectClass: top",
+  "cn: testUser1",
+  "sn: Ldap",
+  "uid: testUser1Uid",
+  "userPassword: 12345",
+  "dn: cn=testUserNonUnique,ou=Users,dc=myorg,dc=com",
+  "objectClass: inetOrgPerson",
+  "objectClass: organizationalPerson",
+  "objectClass: person",
+  "objectClass: top",
+  "cn: testUserNonUnique",
+  "sn: Ldap",
+  "uid: testUserNonUniqueUid",
+  "userPassword: 12345",
+  "dn: cn=testUserNonUnique,dc=myorg,dc=com",
+  "objectClass: inetOrgPerson",
+  "objectClass: organizationalPerson",
+  "objectClass: person",
+  "objectClass: top",
+  "cn: testUserNonUnique",
+  "sn: Ldap",
+  "uid: testUserNonUniqueUid",
+  "userPassword: 12345",
+})
+public class TestLDAPAuth extends BaseAuthenticationCQLTest {
+  private static final Logger LOG = LoggerFactory.getLogger(TestLDAPAuth.class);
+
+  @ClassRule
+  public static CreateLdapServerRule serverRule = new CreateLdapServerRule();
+
+  private void recreateMiniCluster(Map<String, String> extraTserverFlag) throws Exception {
+    destroyMiniCluster();
+    Map<String, String> tserverFlags = new HashMap<>();
+    tserverFlags.put("use_cassandra_authentication", "true");
+    tserverFlags.put("ycql_use_ldap", "true");
+    tserverFlags.put("ycql_ldap_users_to_skip_csv", "cassandra");
+    tserverFlags.put("ycql_ldap_server", "ldap://localhost:10389");
+    tserverFlags.put("ycql_ldap_tls", "false");
+    tserverFlags.put("vmodule", "cql_processor=4");
+    tserverFlags.putAll(extraTserverFlag);
+    createMiniCluster(Collections.emptyMap(), tserverFlags);
+    setUpCqlClient();
+  }
+
+  @Test
+  public void incorrectLDAPServer() throws Exception {
+    Map<String, String> extraTserverFlagMap = getTServerFlags();
+    extraTserverFlagMap.put("ycql_ldap_server", "ldap://localhost:1039");
+    extraTserverFlagMap.put("ycql_ldap_user_prefix", "cn=");
+    extraTserverFlagMap.put("ycql_ldap_user_suffix", ",ou=Users,dc=myorg,dc=com");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */, "Can't contact LDAP server");
+    session.execute("DROP ROLE 'testUser1'");
+  }
+
+  @Test
+  public void simpleBindMode() throws Exception {
+    // Test with incorrect user prefix
+    Map<String, String> extraTserverFlagMap = new HashMap<String, String>();
+    extraTserverFlagMap.put("ycql_ldap_user_prefix", "dummy=");
+    extraTserverFlagMap.put("ycql_ldap_user_suffix", ",ou=Users,dc=myorg,dc=com");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */,
+      "Failed to authenticate using LDAP: Provided username testUser1 and/or password are " +
+      "incorrect");
+
+    // Test with incorrect user suffix
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_user_prefix", "cn=");
+    extraTserverFlagMap.put("ycql_ldap_user_suffix", ",dc=myorg,dc=com");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */,
+      "Failed to authenticate using LDAP: Provided username testUser1 and/or password are " +
+      "incorrect");
+
+    // Test with correct prefix and suffix
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_user_prefix", "cn=");
+    extraTserverFlagMap.put("ycql_ldap_user_suffix", ",ou=Users,dc=myorg,dc=com");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      false /* expectFailure */, "");
+
+    // Incorrect username/password
+    checkConnectivityWithMessage(true, "testUser1", "1234", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */,
+      "Failed to authenticate using LDAP: Provided username testUser1 and/or password are " +
+      "incorrect");
+
+    checkConnectivityWithMessage(true, "testUser2", "12345", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */,
+      "Provided username testUser2 and/or password are incorrect");
+
+    checkConnectivityWithMessage(true, "testUser1", "", ProtocolOptions.Compression.NONE,
+      true /* expectFailure */,
+      "Failed to authenticate using LDAP: Internal error");
+
+    session.execute("DROP ROLE 'testUser1'");
+  }
+
+  @Test
+  public void searchBindMode() throws Exception {
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    // Test with incorrect bind user dn
+    Map<String, String> extraTserverFlagMap = getTServerFlags();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=dummy,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "adminPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "cn");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "could not perform initial LDAP bind for ldapbinddn 'cn=dummy,ou=Users,dc=myorg,dc=com'");
+
+    // Test with incorrect bind password
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=admin,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "dummyPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "cn");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "could not perform initial LDAP bind for ldapbinddn 'cn=admin,ou=Users,dc=myorg,dc=com'");
+
+    // Test with non-existant base dn
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=admin,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "adminPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "ou=dummy,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "cn");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "could not search LDAP for filter '(cn=testUser1)' on server " +
+      "'ldap://localhost:10389': No such object LDAP diagnostics: " +
+      "NO_SUCH_OBJECT: failed for MessageType : SEARCH_REQUEST");
+
+    // Test with incorrect incorrect search attribute
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=admin,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "adminPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "dummy");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    checkConnectivityWithMessage(true, "testUser1", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "LDAP user 'testUser1' does not exist. LDAP search for filter '(dummy=testUser1)' on " +
+      "server 'ldap://localhost:10389' returned no entries.");
+
+    // Test with all correct - bind db, bind password, base dn, search attribute
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=admin,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "adminPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "cn");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    // Test with incorrect user
+    checkConnectivityWithMessage(true, "dummy", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "Provided username dummy and/or password are incorrect");
+
+    // Test with incorrect user and password
+    checkConnectivityWithMessage(true, "testUser1", "1234", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "Failed to authenticate using LDAP: Provided username testUser1 and/or " +
+      "password are incorrect");
+
+    // Test with correct password
+    checkConnectivity(true, "testUser1", "12345", false /* expectFailure */);
+
+    // Test with prefix of base dn
+    extraTserverFlagMap.clear();
+    extraTserverFlagMap.put("ycql_ldap_bind_dn", "cn=admin,ou=Users,dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_bind_passwd", "adminPasswd");
+    extraTserverFlagMap.put("ycql_ldap_base_dn", "dc=myorg,dc=com");
+    extraTserverFlagMap.put("ycql_ldap_search_attribute", "cn");
+    recreateMiniCluster(extraTserverFlagMap);
+    session.execute("CREATE ROLE 'testUser1' WITH LOGIN = true");
+
+    // Test with incorrect user
+    checkConnectivityWithMessage(true, "dummy", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "Provided username dummy and/or password are incorrect");
+
+    // Test with incorrect user password
+    checkConnectivityWithMessage(true, "testUser1", "1234", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "Failed to authenticate using LDAP: Provided username testUser1 and/or " +
+      "password are incorrect");
+
+    // Test with correct password
+    checkConnectivity(true, "testUser1", "12345", false /* expectFailure */);
+
+    session.execute("CREATE ROLE 'test*User1' WITH LOGIN = true");
+    // Test with user name that has characters that are not allowed
+    checkConnectivityWithMessage(true, "test*User1", "12345", ProtocolOptions.Compression.NONE,
+      true, /* expectFailure */
+      "invalid character in user name for LDAP authentication");
+
+    session.execute("CREATE ROLE 'testUserNonUnique' WITH LOGIN = true");
+    // Test with more than one user name matching search criteria
+    checkConnectivityWithMessage(true, "testUserNonUnique", "12345",
+      ProtocolOptions.Compression.NONE, true, /* expectFailure */
+      "LDAP user 'testUserNonUnique' is not unique, 2 entries exist.");
+  }
+}

src/yb/yql/cql/cqlserver/CMakeLists.txt

@@ -56,7 +56,9 @@ target_link_libraries(yb-cql
   yb_client
   cql_service_proto
   server_common
-  server_process)
+  server_process
+  ldap
+  lber)
 
 # Tests
 set(YB_TEST_LINK_LIBS yb-cql integration-tests ${YB_MIN_TEST_LIBS})