From 85ac64e5c1cc8a160ceb95c89b1b2382f6bbe323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 8 Aug 2024 11:05:03 +0300 Subject: [PATCH 1/2] MDEV-26851 : Provide means to verify Galera using TLS from SQL level Implement INFORMATION_SCHEMA.WSREP_CONNECTIONS plugin using wsrep_connection_monitor service. --- cmake/wsrep.cmake | 1 + .../galera_3nodes/r/galera_connections.result | 75 +++++ .../r/galera_connections_ssl.result | 84 ++++++ .../galera_3nodes/t/galera_connections.cnf | 5 + .../galera_3nodes/t/galera_connections.test | 59 ++++ .../t/galera_connections_ssl.cnf | 14 + .../t/galera_connections_ssl.test | 62 +++++ sql/CMakeLists.txt | 12 +- sql/wsrep_connection_monitor_plugin.cc | 262 ++++++++++++++++++ sql/wsrep_connection_monitor_service.cc | 74 +++++ sql/wsrep_connection_monitor_service.h | 49 ++++ sql/wsrep_schema.cc | 16 +- sql/wsrep_schema.h | 20 +- sql/wsrep_server_state.cc | 4 + wsrep-lib | 2 +- 15 files changed, 726 insertions(+), 13 deletions(-) create mode 100644 mysql-test/suite/galera_3nodes/r/galera_connections.result create mode 100644 mysql-test/suite/galera_3nodes/r/galera_connections_ssl.result create mode 100644 mysql-test/suite/galera_3nodes/t/galera_connections.cnf create mode 100644 mysql-test/suite/galera_3nodes/t/galera_connections.test create mode 100644 mysql-test/suite/galera_3nodes/t/galera_connections_ssl.cnf create mode 100644 mysql-test/suite/galera_3nodes/t/galera_connections_ssl.test create mode 100644 sql/wsrep_connection_monitor_plugin.cc create mode 100644 sql/wsrep_connection_monitor_service.cc create mode 100644 sql/wsrep_connection_monitor_service.h diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index a01a1d68f6418..93762e1b46696 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -59,6 +59,7 @@ Then restart the build. "Disable building dbsim for wsrep-lib") endif() INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/include) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26) SET(old_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) diff --git a/mysql-test/suite/galera_3nodes/r/galera_connections.result b/mysql-test/suite/galera_3nodes/r/galera_connections.result new file mode 100644 index 0000000000000..b2c76e598007e --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_connections.result @@ -0,0 +1,75 @@ +connection node_2; +connection node_1; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_3; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_1; +CREATE TABLE t1(a int not null primary key) engine=innodb; +INSERT INTO t1 VALUES (1),(2),(3),(4); +connection node_2; +SELECT * FROM t1; +a +1 +2 +3 +4 +connection node_3; +SELECT * from t1; +a +1 +2 +3 +4 +connection node_1; +SHOW CREATE TABLE INFORMATION_SCHEMA.WSREP_CONNECTIONS; +Table Create Table +wsrep_connections CREATE TEMPORARY TABLE `wsrep_connections` ( + `connection_id` bigint(21) unsigned NOT NULL, + `connection_scheme` varchar(3) NOT NULL, + `local_address` varchar(256) NOT NULL, + `remote_uuid` varchar(256) NOT NULL, + `remote_address` varchar(256) NOT NULL +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_2; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_3; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_1; +DROP TABLE t1; diff --git a/mysql-test/suite/galera_3nodes/r/galera_connections_ssl.result b/mysql-test/suite/galera_3nodes/r/galera_connections_ssl.result new file mode 100644 index 0000000000000..a11e0060b3ad6 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_connections_ssl.result @@ -0,0 +1,84 @@ +connection node_2; +connection node_1; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_3; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_1; +CREATE TABLE t1(a int not null primary key) engine=innodb; +INSERT INTO t1 VALUES (1),(2),(3),(4); +connection node_2; +SELECT * FROM t1; +a +1 +2 +3 +4 +connection node_3; +SELECT * from t1; +a +1 +2 +3 +4 +connection node_1; +SHOW CREATE TABLE INFORMATION_SCHEMA.WSREP_CONNECTIONS; +Table Create Table +wsrep_connections CREATE TEMPORARY TABLE `wsrep_connections` ( + `connection_id` bigint(21) unsigned NOT NULL, + `connection_scheme` varchar(3) NOT NULL, + `local_address` varchar(256) NOT NULL, + `remote_uuid` varchar(256) NOT NULL, + `remote_address` varchar(256) NOT NULL +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_2; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_3; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; +EXPECT_3 +3 +connection node_1; +DROP TABLE t1; diff --git a/mysql-test/suite/galera_3nodes/t/galera_connections.cnf b/mysql-test/suite/galera_3nodes/t/galera_connections.cnf new file mode 100644 index 0000000000000..317094cea72d0 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_connections.cnf @@ -0,0 +1,5 @@ +!include ../galera_3nodes.cnf + +[mysqld] +sql-safe-updates=1 +wsrep-debug=1 diff --git a/mysql-test/suite/galera_3nodes/t/galera_connections.test b/mysql-test/suite/galera_3nodes/t/galera_connections.test new file mode 100644 index 0000000000000..129587d74307e --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_connections.test @@ -0,0 +1,59 @@ +--source include/galera_cluster.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--connection node_1 +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--source ../galera/include/auto_increment_offset_save.inc + +--connection node_1 +CREATE TABLE t1(a int not null primary key) engine=innodb; +INSERT INTO t1 VALUES (1),(2),(3),(4); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * FROM t1; + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * from t1; + +# +# Table contents is not deterministic only number of connections is +# +--connection node_1 +SHOW CREATE TABLE INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +# +# Two of the connections should be from exactly same node as in cluster members table, one is naturally this node +# +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_3 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_1 +DROP TABLE t1; diff --git a/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.cnf b/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.cnf new file mode 100644 index 0000000000000..67de1f6009027 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.cnf @@ -0,0 +1,14 @@ +!include ../galera_3nodes.cnf + +[mysqld] +wsrep-debug=1 +loose-galera-connections-ssl=1 + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' + +[mysqld.3] +wsrep_provider_options='base_port=@mysqld.3.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' diff --git a/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.test b/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.test new file mode 100644 index 0000000000000..761e942c5fea0 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_connections_ssl.test @@ -0,0 +1,62 @@ +--source include/galera_cluster.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--connection node_1 +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--source ../galera/include/auto_increment_offset_save.inc + +--connection node_1 +CREATE TABLE t1(a int not null primary key) engine=innodb; +INSERT INTO t1 VALUES (1),(2),(3),(4); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * FROM t1; + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * from t1; + +# +# Table contents is not deterministic only number of connections is +# +--connection node_1 +SHOW CREATE TABLE INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +# +# Two of the connections should be from exactly same node as in cluster members table, one is naturally this node +# +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_3 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_1 +DROP TABLE t1; diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 0195555efaf39..fd2eaaf82df48 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -18,6 +18,8 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY) SET(WSREP_SOURCES wsrep_client_service.cc + wsrep_connection_monitor_plugin.cc + wsrep_connection_monitor_service.cc wsrep_high_priority_service.cc wsrep_server_service.cc wsrep_storage_service.cc @@ -41,13 +43,19 @@ IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY) ) MYSQL_ADD_PLUGIN(wsrep_provider ${WSREP_SOURCES} DEFAULT NOT_EMBEDDED LINK_LIBRARIES wsrep-lib wsrep_api_v26) MYSQL_ADD_PLUGIN(wsrep ${WSREP_SOURCES} MANDATORY NOT_EMBEDDED LINK_LIBRARIES wsrep-lib wsrep_api_v26) + MYSQL_ADD_PLUGIN(wsrep_connection_monitor ${WSREP_SOURCES} MANDATORY NOT_EMBEDDED LINK_LIBRARIES wsrep-lib wsrep_api_v26) + # wsrep and wsrep_connection_monitor plugin needs some wsrep symbols from inside mysqld + # we have to remove -fvisibility=hidden from these IF(VISIBILITY_HIDDEN_FLAG AND TARGET wsrep) - # wsrep_info plugin needs some wsrep symbols from inside mysqld - # we have to remove -fvisibility=hidden from wsrep GET_TARGET_PROPERTY(f wsrep COMPILE_FLAGS) STRING(REPLACE "${VISIBILITY_HIDDEN_FLAG}" "" f ${f}) SET_TARGET_PROPERTIES(wsrep PROPERTIES COMPILE_FLAGS "${f}") ENDIF() + IF(VISIBILITY_HIDDEN_FLAG AND TARGET wsrep_connection_monitor) + GET_TARGET_PROPERTY(f wsrep_connection_monitor COMPILE_FLAGS) + STRING(REPLACE "${VISIBILITY_HIDDEN_FLAG}" "" f ${f}) + SET_TARGET_PROPERTIES(wsrep_connection_monitor PROPERTIES COMPILE_FLAGS "${f}") + ENDIF() ELSE() ADD_LIBRARY(wsrep STATIC wsrep_dummy.cc) ADD_DEPENDENCIES(wsrep GenError) diff --git a/sql/wsrep_connection_monitor_plugin.cc b/sql/wsrep_connection_monitor_plugin.cc new file mode 100644 index 0000000000000..5be3aa2e351eb --- /dev/null +++ b/sql/wsrep_connection_monitor_plugin.cc @@ -0,0 +1,262 @@ +/* Copyright 2024 Codership Oy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + +/* +*/ + +#include "my_global.h" +#include "mysqld_error.h" +#include +#include "sql_plugin.h" +#include "sql_priv.h" +#include "sql_class.h" +#include "set_var.h" +#include "sql_acl.h" +#include "sql_i_s.h" +#include "wsrep_mysqld.h" +#include "wsrep_schema.h" + +#include +#include + +#include "wsrep_connection_monitor_service.h" + +#ifdef WITH_WSREP +static bool connection_monitor_plugin_enabled= false; + +/** Mutex protecting in-memory cache of the wsrep connections */ +std::mutex wsrep_connections_mutex; + +/** Wsrep connections in-memory cache */ +std::map wsrep_connections; + +bool wsrep_connection_monitor_plugin_enabled() +{ + return connection_monitor_plugin_enabled; +} + +static int wsrep_connection_monitor_plugin_deinit(void *p) +{ + return 0; +} + +namespace Show { +static ST_FIELD_INFO wsrep_connections_fields_info[]= +{ +#define CONNECTION_ID 0 + Column ("connection_id", ULonglong(), NOT_NULL), +#define CONNECTION_SCHEME 1 + Column ("connection_scheme", Varchar(3), NOT_NULL), +#define LOCAL_ADDRESS 2 + Column ("local_address", Varchar(256), NOT_NULL), +#define REMOTE_UUID 3 + Column ("remote_uuid", Varchar(256), NOT_NULL), +#define REMOTE_ADDRESS 4 + Column ("remote_address", Varchar(256), NOT_NULL), + + CEnd() +}; +} // namespace Show + + +#define OK(expr) \ + if ((expr) != 0) \ + { \ + return (1); \ + } + +static int store_string(Field* field, const std::string &str) +{ + if (str.empty()) + { + field->set_null(); + return 0; + } + + field->set_notnull(); + return field->store(str.c_str(), uint(str.size()), system_charset_info); +} + +/** + Return Galera connections in-memory cache + + @return reference to list of Galera connections +*/ +static std::map& get_connections(void) +{ + return wsrep_connections; +} + +static int fill_wsrep_connections(THD *thd, TABLE_LIST *tables, Item *) +{ + // Require wsrep enabled and deny access to non-superusers + if (!WSREP(thd) || check_global_access(thd, PROCESS_ACL)) + return 0; + + // Wsrep should be inited + if (!wsrep_inited) + { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_CANT_FIND_SYSTEM_REC, + "Galera: SELECTing from " + "INFORMATION_SCHEMA.wsrep_connections but " + "the wsrep is not inited"); + return 0; + } + + Field** fields= tables->table->field; + std::lock_guard wsrep_connections_lock(wsrep_connections_mutex); + std::map& wsrep_connections= get_connections(); + + std::map::iterator c = wsrep_connections.begin(); + while( c != wsrep_connections.end()) + { + wsrep_connection_t &conn = c->second; + OK(fields[CONNECTION_ID]->store(conn.connection_id)); + OK(store_string(fields[CONNECTION_SCHEME], conn.scheme)); + OK(store_string(fields[LOCAL_ADDRESS], conn.local_address)); + OK(store_string(fields[REMOTE_UUID], conn.remote_uuid)); + OK(store_string(fields[REMOTE_ADDRESS], conn.remote_address)); + OK(schema_table_store_record(thd, tables->table)); + c++; + } + + return 0; +} + +/** Bind the dynamic table INFORMATION_SCHEMA.wsrep_connections +@return 0 on success */ +static int wsrep_connections_init(void* p) +{ + ST_SCHEMA_TABLE* schema; + + connection_monitor_plugin_enabled= true; + + schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = Show::wsrep_connections_fields_info; + schema->fill_table = fill_wsrep_connections; + + return 0; +} + +static struct st_mysql_information_schema plugin_descriptor = +{ + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +/** version number reported by SHOW PLUGINS */ +constexpr unsigned i_s_version= MYSQL_VERSION_MAJOR << 8 | MYSQL_VERSION_MINOR; + +maria_declare_plugin(wsrep_connection_monitor) +{ + /* the plugin type (a MYSQL_XXX_PLUGIN value) */ + /* int */ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + + /* pointer to type-specific plugin descriptor */ + /* void* */ + &plugin_descriptor, + + /* plugin name */ + /* const char* */ + "wsrep_connections", + + /* plugin author (for SHOW PLUGINS) */ + /* const char* */ + "Codership Oy", + + /* general descriptive text (for SHOW PLUGINS) */ + /* const char* */ + "Provides information about Galera connections", + + /* the plugin license (PLUGIN_LICENSE_XXX) */ + /* int */ + PLUGIN_LICENSE_GPL, + + /* the function to invoke when plugin is loaded */ + /* int (*)(void*); */ + wsrep_connections_init, + + /* the function to invoke when plugin is unloaded */ + /* int (*)(void*); */ + wsrep_connection_monitor_plugin_deinit, + + i_s_version, nullptr, nullptr, PACKAGE_VERSION, + MariaDB_PLUGIN_MATURITY_STABLE +} +maria_declare_plugin_end; + +bool wsrep_connection_monitor_connect(wsrep_connection_key_t id, + const std::string &scheme, + const std::string &local_addr, + const std::string &remote_uuid, + const std::string &remote_addr) +{ + uintptr_t key = (uintptr_t)(id); + + std::lock_guard wsrep_lock(wsrep_connections_mutex); + std::map &wsrep_connections = get_connections(); + std::map::iterator i = wsrep_connections.find(key); + + WSREP_DEBUG("wsrep_connection_add: %llu : %s %s %s %s", + key, scheme.c_str(), local_addr.c_str(), + remote_uuid.c_str(), remote_addr.c_str()); + + // Not found : add + if (i == wsrep_connections.end()) + { + WSREP_DEBUG("wsrep_connection_add: key %lld not found add %s %s %s %s", key, + scheme.c_str(), local_addr.c_str(), remote_uuid.c_str(), remote_addr.c_str()); + wsrep_connection_t new_connection {key, scheme, local_addr, remote_uuid, remote_addr}; + wsrep_connections.insert(std::pair(key, new_connection)); + } + else + { + // found, update + wsrep_connection_t &old_conn= i->second; + WSREP_DEBUG("wsrep_connection_add: key %lld found %s %s %s %s new %s %s %s", + key, old_conn.scheme.c_str(), old_conn.local_address.c_str(), + old_conn.remote_uuid.c_str(), old_conn.remote_address.c_str(), + remote_uuid.c_str(), remote_addr.c_str(), local_addr.c_str()); + old_conn.remote_uuid= remote_uuid; + old_conn.scheme= scheme; + old_conn.remote_address= remote_addr; + old_conn.local_address= local_addr; + } + return true; +} + +bool wsrep_connection_monitor_disconnect(wsrep_connection_key_t id) +{ + uintptr_t key = (uintptr_t)id; + std::lock_guard wsrep_lock(wsrep_connections_mutex); + std::map &wsrep_connections= get_connections(); + WSREP_DEBUG("wsrep_connection_remove: %lld", key); + + std::map::iterator c = wsrep_connections.find(key); + + if (c != wsrep_connections.end()) + { + wsrep_connection_t conn= c->second; + WSREP_DEBUG("wsrep_connection_remove: found for %llu : %s %s %s %s", + key, + conn.scheme.c_str(), conn.local_address.c_str(), + conn.remote_uuid.c_str(), conn.remote_address.c_str()); + wsrep_connections.erase(c); + } + return true; +} + +#endif /* WITH_WSREP */ diff --git a/sql/wsrep_connection_monitor_service.cc b/sql/wsrep_connection_monitor_service.cc new file mode 100644 index 0000000000000..78b6442068141 --- /dev/null +++ b/sql/wsrep_connection_monitor_service.cc @@ -0,0 +1,74 @@ +/* Copyright 2024 Codership Oy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "wsrep_connection_monitor_service.h" + +#include "my_global.h" +#include "wsrep_mysqld.h" +#include "wsrep_priv.h" +#include "wsrep_schema.h" + +#include +#include +#include + +class Wsrep_connection_monitor_service : public wsrep::connection_monitor_service +{ +public: + bool connection_monitor_connect_cb( + wsrep_connection_key_t id, + const wsrep::const_buffer& scheme, + const wsrep::const_buffer& local_addr, + const wsrep::const_buffer& remote_uuid, + const wsrep::const_buffer& remote_addr) WSREP_NOEXCEPT override; + + bool connection_monitor_disconnect_cb( + wsrep_connection_key_t id) WSREP_NOEXCEPT override; +}; + +bool Wsrep_connection_monitor_service::connection_monitor_connect_cb ( + wsrep_connection_key_t id, + const wsrep::const_buffer& scheme, + const wsrep::const_buffer& local_addr, + const wsrep::const_buffer& remote_uuid, + const wsrep::const_buffer& remote_addr) + WSREP_NOEXCEPT +{ + std::string remote(remote_uuid.data()); + std::string lscheme(scheme.data()); + std::string raddr(remote_addr.data()); + std::string laddr(local_addr.data()); + + return wsrep_connection_monitor_connect(id, lscheme, laddr, remote, raddr); +} + +bool Wsrep_connection_monitor_service::connection_monitor_disconnect_cb ( + wsrep_connection_key_t id) WSREP_NOEXCEPT +{ + return wsrep_connection_monitor_disconnect(id); +} + +std::unique_ptr monitor_entrypoint; + +wsrep::connection_monitor_service* wsrep_connection_monitor_service_init() +{ + monitor_entrypoint = std::unique_ptr(new Wsrep_connection_monitor_service); + return monitor_entrypoint.get(); +} + +void wsrep_connection_monitor_service_deinit() +{ + monitor_entrypoint.reset(); +} diff --git a/sql/wsrep_connection_monitor_service.h b/sql/wsrep_connection_monitor_service.h new file mode 100644 index 0000000000000..594540326b6ce --- /dev/null +++ b/sql/wsrep_connection_monitor_service.h @@ -0,0 +1,49 @@ +/* Copyright 2024 Codership Oy + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* + Implementation of wsrep connection monitor servie. + */ + +#ifndef WSREP_PROVIDER_CONNECITION_MONITOR_H +#define WSREP_PROVIDER_CONNECTION_MONITOR_H + +#include "wsrep/connection_monitor_service.hpp" +#include +#include + +wsrep::connection_monitor_service* wsrep_connection_monitor_service_init(); + +void wsrep_connection_monitor_service_deinit(); + +bool wsrep_connection_monitor_connect(wsrep_connection_key_t id, + const std::string &scheme, + const std::string &local_addr, + const std::string &remote_uuid, + const std::string &remote_addr); + +bool wsrep_connection_monitor_disconnect(wsrep_connection_key_t id); + +/** Structure holding information of one Galera connection */ +typedef struct +{ + const uintptr_t connection_id; + std::string scheme; + std::string local_address; + std::string remote_uuid; + std::string remote_address; +} wsrep_connection_t; + +#endif /* WSREP_PROVIDER_CONNECTION_MONITOR_H */ diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc index e7b3a8580c3ac..4698b4b93df22 100644 --- a/sql/wsrep_schema.cc +++ b/sql/wsrep_schema.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 Codership Oy +/* Copyright (C) 2015-2024 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ #include #include - +#include #define WSREP_SCHEMA "mysql" #define WSREP_STREAMING_TABLE "wsrep_streaming_log" #define WSREP_CLUSTER_TABLE "wsrep_cluster" @@ -612,9 +612,8 @@ static int scan_member(TABLE* table, } try { - members.push_back(Wsrep_view::member(member_id, - member_name, - member_incoming)); + Wsrep_view::member member(member_id, member_name, member_incoming); + members.push_back(member); } catch (...) { WSREP_ERROR("Caught exception while scanning members table"); @@ -694,7 +693,7 @@ Wsrep_schema::Wsrep_schema() = default; Wsrep_schema::~Wsrep_schema() = default; -static void wsrep_init_thd_for_schema(THD *thd) +void wsrep_init_thd_for_schema(THD *thd) { thd->security_ctx->skip_grants(); thd->system_thread= SYSTEM_THREAD_GENERIC; @@ -720,6 +719,11 @@ static void wsrep_init_thd_for_schema(THD *thd) static bool wsrep_schema_ready= false; +bool Wsrep_schema::inited() +{ + return wsrep_schema_ready; +} + int Wsrep_schema::init() { DBUG_ENTER("Wsrep_schema::init()"); diff --git a/sql/wsrep_schema.h b/sql/wsrep_schema.h index c9004d076bd80..ed9b76986a858 100644 --- a/sql/wsrep_schema.h +++ b/sql/wsrep_schema.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2023 Codership Oy +/* Copyright (C) 2015-2024 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,6 +23,9 @@ #include "mysqld.h" #include "wsrep_mysqld.h" +#include +#include + /* Forward decls */ @@ -44,18 +47,25 @@ class Wsrep_schema Wsrep_schema(); ~Wsrep_schema(); - /* + /** Initialize wsrep schema. Storage engines must be running before calling this function. */ int init(); - /* + /** + Has wsrep schema inited. + + @return true if inited, false if not + */ + bool inited(); + + /** Store wsrep view info into wsrep schema. */ int store_view(THD*, const Wsrep_view& view); - /* + /** Restore view info from stable storage. */ Wsrep_view restore_view(THD* thd, const Wsrep_id& own_id) const; @@ -163,6 +173,8 @@ class Wsrep_schema extern Wsrep_schema* wsrep_schema; +extern void wsrep_init_thd_for_schema(THD *thd); + extern LEX_CSTRING WSREP_LEX_SCHEMA; extern LEX_CSTRING WSREP_LEX_STREAMING; extern LEX_CSTRING WSREP_LEX_CLUSTER; diff --git a/sql/wsrep_server_state.cc b/sql/wsrep_server_state.cc index 7bfe0d6c0815e..2fbfbabef00e4 100644 --- a/sql/wsrep_server_state.cc +++ b/sql/wsrep_server_state.cc @@ -18,6 +18,7 @@ #include "wsrep_server_state.h" #include "wsrep_allowlist_service.h" #include "wsrep_event_service.h" +#include "wsrep_connection_monitor_service.h" #include "wsrep_binlog.h" /* init/deinit group commit */ #include "wsrep_plugin.h" /* make/destroy sysvar helpers */ @@ -147,12 +148,15 @@ void Wsrep_server_state::init_provider_services() { m_provider_services.allowlist_service= wsrep_allowlist_service_init(); m_provider_services.event_service= Wsrep_event_service::instance(); + m_provider_services.connection_monitor_service= wsrep_connection_monitor_service_init(); } void Wsrep_server_state::deinit_provider_services() { if (m_provider_services.allowlist_service) wsrep_allowlist_service_deinit(); + if (m_provider_services.connection_monitor_service) + wsrep_connection_monitor_service_deinit(); m_provider_services= wsrep::provider::services(); } diff --git a/wsrep-lib b/wsrep-lib index 31db8476768ba..a85cd0ba4f587 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit 31db8476768ba68296ad91b6785bb06a6a9abf71 +Subproject commit a85cd0ba4f587906c5183bd0668ccb0e96c083e2 From 98ce5105c943ec147b4b529f74d3aa18f3321577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 14 Oct 2024 10:03:15 +0300 Subject: [PATCH 2/2] WIP: Add SSL/TLS info --- .../r/galera_connections_ist.result | 126 +++++++++++++++++ .../t/galera_connections_ist.test | 133 ++++++++++++++++++ mysql-test/suite/galera_3nodes/t/jan.cnf | 14 ++ mysql-test/suite/galera_3nodes/t/jan.test | 63 +++++++++ sql/wsrep_connection_monitor_plugin.cc | 79 ++++++++--- sql/wsrep_connection_monitor_service.cc | 23 +++ sql/wsrep_connection_monitor_service.h | 10 ++ wsrep-lib | 2 +- 8 files changed, 426 insertions(+), 24 deletions(-) create mode 100644 mysql-test/suite/galera_3nodes/r/galera_connections_ist.result create mode 100644 mysql-test/suite/galera_3nodes/t/galera_connections_ist.test create mode 100644 mysql-test/suite/galera_3nodes/t/jan.cnf create mode 100644 mysql-test/suite/galera_3nodes/t/jan.test diff --git a/mysql-test/suite/galera_3nodes/r/galera_connections_ist.result b/mysql-test/suite/galera_3nodes/r/galera_connections_ist.result new file mode 100644 index 0000000000000..a831238052a1c --- /dev/null +++ b/mysql-test/suite/galera_3nodes/r/galera_connections_ist.result @@ -0,0 +1,126 @@ +connection node_2; +connection node_1; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +connection node_3; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_1; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); +INSERT INTO t1 VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'a'), (5, 'a'),(6, 'a'); +connection node_2; +SELECT * FROM t1; +f1 f2 +1 a +2 a +3 a +4 a +5 a +6 a +connection node_3; +SELECT * from t1; +f1 f2 +1 a +2 a +3 a +4 a +5 a +6 a +connection node_1; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +# Disconnect node_2 +connection node_2; +SET SESSION wsrep_sync_wait=0; +Unloading wsrep provider ... +SET GLOBAL wsrep_cluster_address = ''; +connection node_1; +UPDATE t1 SET f2 = 'b' WHERE f1 > 1; +# We should now have only 1 connection +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_1 +1 +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_1 +1 +connection node_3; +# We should now have only 1 connection +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_1 +1 +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_1 +1 +connection node_1; +UPDATE t1 SET f2 = 'c' WHERE f1 > 2; +connection node_2; +Loading wsrep_provider ... +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; +UPDATE t1 SET f2 = 'd' WHERE f1 > 3; +# We should see IST connection +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +connection node_2; +SET GLOBAL wsrep_provider_options = 'signal=recv_IST_after_apply_trx'; +connection node_1; +connection node_1; +SELECT * FROM t1; +f1 f2 +1 a +2 b +3 c +4 d +5 d +6 d +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +connection node_2; +SELECT * FROM t1; +f1 f2 +1 a +2 b +3 c +4 d +5 d +6 d +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +connection node_3; +SELECT * FROM t1; +f1 f2 +1 a +2 b +3 c +4 d +5 d +6 d +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +EXPECT_2 +2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +EXPECT_2 +2 +connection node_1; +DROP TABLE t1; +disconnect node_2; +disconnect node_1; diff --git a/mysql-test/suite/galera_3nodes/t/galera_connections_ist.test b/mysql-test/suite/galera_3nodes/t/galera_connections_ist.test new file mode 100644 index 0000000000000..a0a4716b07d46 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/galera_connections_ist.test @@ -0,0 +1,133 @@ +--source include/galera_cluster.inc +--source include/big_test.inc +--source include/have_debug.inc +--source include/galera_have_debug_sync.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--connection node_1 +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--source ../galera/include/auto_increment_offset_save.inc + +--connection node_1 +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1)); +INSERT INTO t1 VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'a'), (5, 'a'),(6, 'a'); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc +SELECT * FROM t1; + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc +SELECT * from t1; + +# +# Table contents is not deterministic only number of connections is +# +--connection node_1 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + + +--echo # Disconnect node_2 +--connection node_2 +SET SESSION wsrep_sync_wait=0; +--source suite/galera/include/galera_stop_replication.inc + +--connection node_1 +UPDATE t1 SET f2 = 'b' WHERE f1 > 1; + +# Wait until node_2 has left +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--echo # We should now have only 1 connection +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--echo # We should now have only 1 connection +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_1 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_1 +UPDATE t1 SET f2 = 'c' WHERE f1 > 2; + +--connection node_2 +# Write file to make mysql-test-run.pl expect the crash, but don't start it +--let $_expect_file_name= `select regexp_replace(@@tmpdir, '^.*/','')` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/$_expect_file_name.expect +--write_line wait $_expect_file_name + +--let KILL_NODE_PIDFILE = `SELECT @@pid_file` + +# ... and restart provider to force IST +--echo Loading wsrep_provider ... +--disable_query_log +# Make sure IST will block ... +--let $galera_sync_point = recv_IST_after_apply_trx +--source include/galera_set_sync_point.inc +--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; +--enable_query_log + +# wait for the IST to execute one transaction +--let $galera_sync_point = recv_IST_after_apply_trx +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--connection node_1 +# Perform DML while IST is in progress +UPDATE t1 SET f2 = 'd' WHERE f1 > 3; + +--echo # We should see IST connection +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_2 +# release the local transaction to continue with commit +--let $galera_sync_point = recv_IST_after_apply_trx +--source include/galera_signal_sync_point.inc + +--source include/wait_until_ready.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--connection node_1 +SELECT * FROM t1; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_2 +SELECT * FROM t1; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_3 +SELECT * FROM t1; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; + +--connection node_1 +DROP TABLE t1; + +# Restore original auto_increment_offset values. +--source ../galera/include/auto_increment_offset_restore.inc + +--source include/galera_end.inc diff --git a/mysql-test/suite/galera_3nodes/t/jan.cnf b/mysql-test/suite/galera_3nodes/t/jan.cnf new file mode 100644 index 0000000000000..67de1f6009027 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/jan.cnf @@ -0,0 +1,14 @@ +!include ../galera_3nodes.cnf + +[mysqld] +wsrep-debug=1 +loose-galera-connections-ssl=1 + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' + +[mysqld.3] +wsrep_provider_options='base_port=@mysqld.3.#galera_port;socket.ssl=yes;socket.ssl_ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem' diff --git a/mysql-test/suite/galera_3nodes/t/jan.test b/mysql-test/suite/galera_3nodes/t/jan.test new file mode 100644 index 0000000000000..a17d00d7c4306 --- /dev/null +++ b/mysql-test/suite/galera_3nodes/t/jan.test @@ -0,0 +1,63 @@ +--source include/galera_cluster.inc + +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 +--connection node_3 +--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +--connection node_1 +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--source ../galera/include/auto_increment_offset_save.inc + +--connection node_1 +CREATE TABLE t1(a int not null primary key) engine=innodb; +INSERT INTO t1 VALUES (1),(2),(3),(4); + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * FROM t1; + +--connection node_3 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc +SELECT * from t1; + +# +# Table contents is not deterministic only number of connections is +# +--connection node_1 +SHOW CREATE TABLE INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT * FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +# +# Two of the connections should be from exactly same node as in cluster members table, one is naturally this node +# +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_2 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_3 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='ssl'; +SELECT COUNT(*) AS EXPECT_0 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS WHERE CONNECTION_SCHEME='tcp'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.WSREP_CONNECTIONS,mysql.wsrep_cluster_members WHERE REMOTE_UUID = NODE_UUID; +SELECT COUNT(*) AS EXPECT_3 FROM mysql.wsrep_cluster_members; + +--connection node_1 +DROP TABLE t1; diff --git a/sql/wsrep_connection_monitor_plugin.cc b/sql/wsrep_connection_monitor_plugin.cc index 5be3aa2e351eb..98d51e66c9780 100644 --- a/sql/wsrep_connection_monitor_plugin.cc +++ b/sql/wsrep_connection_monitor_plugin.cc @@ -65,6 +65,14 @@ static ST_FIELD_INFO wsrep_connections_fields_info[]= Column ("remote_uuid", Varchar(256), NOT_NULL), #define REMOTE_ADDRESS 4 Column ("remote_address", Varchar(256), NOT_NULL), +#define CHIPHER 5 + Column ("chipher", Varchar(256), NOT_NULL), +#define CERTIFICATE_SUBJECT 6 + Column ("certificate_subject", Varchar(256), NOT_NULL), +#define CERTIFICATE_ISSUER 7 + Column ("certificate_issuer", Varchar(256), NOT_NULL), +#define CERTIFICATE_VERSION 8 + Column ("certificate_version", Varchar(256), NOT_NULL), CEnd() }; @@ -89,16 +97,6 @@ static int store_string(Field* field, const std::string &str) return field->store(str.c_str(), uint(str.size()), system_charset_info); } -/** - Return Galera connections in-memory cache - - @return reference to list of Galera connections -*/ -static std::map& get_connections(void) -{ - return wsrep_connections; -} - static int fill_wsrep_connections(THD *thd, TABLE_LIST *tables, Item *) { // Require wsrep enabled and deny access to non-superusers @@ -118,9 +116,9 @@ static int fill_wsrep_connections(THD *thd, TABLE_LIST *tables, Item *) Field** fields= tables->table->field; std::lock_guard wsrep_connections_lock(wsrep_connections_mutex); - std::map& wsrep_connections= get_connections(); - std::map::iterator c = wsrep_connections.begin(); + + WSREP_DEBUG(":::JAN:::FILL_WSREP_CONNECTIONS"); while( c != wsrep_connections.end()) { wsrep_connection_t &conn = c->second; @@ -129,6 +127,10 @@ static int fill_wsrep_connections(THD *thd, TABLE_LIST *tables, Item *) OK(store_string(fields[LOCAL_ADDRESS], conn.local_address)); OK(store_string(fields[REMOTE_UUID], conn.remote_uuid)); OK(store_string(fields[REMOTE_ADDRESS], conn.remote_address)); + OK(store_string(fields[CHIPHER], conn.chipher)); + OK(store_string(fields[CERTIFICATE_SUBJECT], conn.certificate_subject)); + OK(store_string(fields[CERTIFICATE_ISSUER], conn.certificate_issuer)); + OK(store_string(fields[CERTIFICATE_VERSION], conn.version)); OK(schema_table_store_record(thd, tables->table)); c++; } @@ -207,19 +209,19 @@ bool wsrep_connection_monitor_connect(wsrep_connection_key_t id, uintptr_t key = (uintptr_t)(id); std::lock_guard wsrep_lock(wsrep_connections_mutex); - std::map &wsrep_connections = get_connections(); std::map::iterator i = wsrep_connections.find(key); WSREP_DEBUG("wsrep_connection_add: %llu : %s %s %s %s", - key, scheme.c_str(), local_addr.c_str(), - remote_uuid.c_str(), remote_addr.c_str()); + key, scheme.c_str(), local_addr.c_str(), + remote_uuid.c_str(), remote_addr.c_str()); // Not found : add if (i == wsrep_connections.end()) { WSREP_DEBUG("wsrep_connection_add: key %lld not found add %s %s %s %s", key, - scheme.c_str(), local_addr.c_str(), remote_uuid.c_str(), remote_addr.c_str()); - wsrep_connection_t new_connection {key, scheme, local_addr, remote_uuid, remote_addr}; + scheme.c_str(), local_addr.c_str(), remote_uuid.c_str(), remote_addr.c_str()); + wsrep_connection_t new_connection {key, scheme, local_addr, remote_uuid, remote_addr, + "","","",""}; wsrep_connections.insert(std::pair(key, new_connection)); } else @@ -227,9 +229,10 @@ bool wsrep_connection_monitor_connect(wsrep_connection_key_t id, // found, update wsrep_connection_t &old_conn= i->second; WSREP_DEBUG("wsrep_connection_add: key %lld found %s %s %s %s new %s %s %s", - key, old_conn.scheme.c_str(), old_conn.local_address.c_str(), - old_conn.remote_uuid.c_str(), old_conn.remote_address.c_str(), - remote_uuid.c_str(), remote_addr.c_str(), local_addr.c_str()); + key, old_conn.scheme.c_str(), old_conn.local_address.c_str(), + old_conn.remote_uuid.c_str(), old_conn.remote_address.c_str(), + remote_uuid.c_str(), remote_addr.c_str(), local_addr.c_str()); + old_conn.remote_uuid= remote_uuid; old_conn.scheme= scheme; old_conn.remote_address= remote_addr; @@ -242,7 +245,6 @@ bool wsrep_connection_monitor_disconnect(wsrep_connection_key_t id) { uintptr_t key = (uintptr_t)id; std::lock_guard wsrep_lock(wsrep_connections_mutex); - std::map &wsrep_connections= get_connections(); WSREP_DEBUG("wsrep_connection_remove: %lld", key); std::map::iterator c = wsrep_connections.find(key); @@ -251,12 +253,43 @@ bool wsrep_connection_monitor_disconnect(wsrep_connection_key_t id) { wsrep_connection_t conn= c->second; WSREP_DEBUG("wsrep_connection_remove: found for %llu : %s %s %s %s", - key, - conn.scheme.c_str(), conn.local_address.c_str(), + key, + conn.scheme.c_str(), conn.local_address.c_str(), conn.remote_uuid.c_str(), conn.remote_address.c_str()); wsrep_connections.erase(c); } return true; } +bool wsrep_connection_monitor_ssl_info(wsrep_connection_key_t id, + const std::string &chipher, + const std::string &certificate_subject, + const std::string &certificate_issuer, + const std::string &version) +{ + uintptr_t key = (uintptr_t)(id); + + std::lock_guard wsrep_lock(wsrep_connections_mutex); + std::map::iterator i = wsrep_connections.find(key); + + // Not found : add + if (i != wsrep_connections.end()) + { + // found, update + wsrep_connection_t &old_conn= i->second; + WSREP_DEBUG("wsrep_connection_ssl_info: key %lld %s %s %s %s : %s %s %s %s", + key, old_conn.scheme.c_str(), old_conn.local_address.c_str(), + old_conn.remote_uuid.c_str(), old_conn.remote_address.c_str(), + chipher.c_str(), certificate_subject.c_str(), + certificate_issuer.c_str(), version.c_str()); + old_conn.chipher= chipher; + old_conn.certificate_subject= certificate_subject; + old_conn.certificate_issuer= certificate_issuer; + old_conn.version= version; + } + else + WSREP_DEBUG("::JAN::not found %lld", key); + return true; +} + #endif /* WITH_WSREP */ diff --git a/sql/wsrep_connection_monitor_service.cc b/sql/wsrep_connection_monitor_service.cc index 78b6442068141..1bc904f25d21f 100644 --- a/sql/wsrep_connection_monitor_service.cc +++ b/sql/wsrep_connection_monitor_service.cc @@ -36,6 +36,13 @@ class Wsrep_connection_monitor_service : public wsrep::connection_monitor_servic bool connection_monitor_disconnect_cb( wsrep_connection_key_t id) WSREP_NOEXCEPT override; + + bool connection_monitor_ssl_info_cb( + wsrep_connection_key_t id, + const wsrep::const_buffer& chipher, + const wsrep::const_buffer& certificate_subject, + const wsrep::const_buffer& certificate_issuer, + const wsrep::const_buffer& version) WSREP_NOEXCEPT override; }; bool Wsrep_connection_monitor_service::connection_monitor_connect_cb ( @@ -60,6 +67,22 @@ bool Wsrep_connection_monitor_service::connection_monitor_disconnect_cb ( return wsrep_connection_monitor_disconnect(id); } +bool Wsrep_connection_monitor_service::connection_monitor_ssl_info_cb ( + wsrep_connection_key_t id, + const wsrep::const_buffer& chipher, + const wsrep::const_buffer& certificate_subject, + const wsrep::const_buffer& certificate_issuer, + const wsrep::const_buffer& version) + WSREP_NOEXCEPT +{ + std::string ch(chipher.data()); + std::string subject(certificate_subject.data()); + std::string issuer(certificate_issuer.data()); + std::string vers(version.data()); + + return wsrep_connection_monitor_ssl_info(id, ch, subject, issuer, vers); +} + std::unique_ptr monitor_entrypoint; wsrep::connection_monitor_service* wsrep_connection_monitor_service_init() diff --git a/sql/wsrep_connection_monitor_service.h b/sql/wsrep_connection_monitor_service.h index 594540326b6ce..bac3fdf9fe630 100644 --- a/sql/wsrep_connection_monitor_service.h +++ b/sql/wsrep_connection_monitor_service.h @@ -36,6 +36,12 @@ bool wsrep_connection_monitor_connect(wsrep_connection_key_t id, bool wsrep_connection_monitor_disconnect(wsrep_connection_key_t id); +bool wsrep_connection_monitor_ssl_info(wsrep_connection_key_t id, + const std::string &chipher, + const std::string &certificate_subject, + const std::string &certificate_issuer, + const std::string &version); + /** Structure holding information of one Galera connection */ typedef struct { @@ -44,6 +50,10 @@ typedef struct std::string local_address; std::string remote_uuid; std::string remote_address; + std::string chipher; + std::string certificate_subject; + std::string certificate_issuer; + std::string version; } wsrep_connection_t; #endif /* WSREP_PROVIDER_CONNECTION_MONITOR_H */ diff --git a/wsrep-lib b/wsrep-lib index a85cd0ba4f587..328faa2cbb02c 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit a85cd0ba4f587906c5183bd0668ccb0e96c083e2 +Subproject commit 328faa2cbb02c4f76268abcf0de163b7c347747d