Skip to content

Commit ef77a17

Browse files
janlindstromsjaakola
janlindstrom
authored and
sjaakola
committed
MDEV-31905 GTID inconsistency
This commit fixes GTID inconsistency which was injected by mariabackup SST. Donor node now writes new info file: donor_galera_info, which is streamed along the mariabackup donation to the joiner node. The donor_galera_info file contains both GTID and gtid domain_id, and joiner will use these to initialize the GTID state. Commit has new mtr test case: galera_3nodes.galera_gtid_consistency, which exercises potentially harmful mariabackup SST scenarios. The test has also scenario with IST joining.
1 parent ee5cadd commit ef77a17

17 files changed

+683
-12
lines changed

‎extra/mariabackup/backup_copy.cc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1618,6 +1618,7 @@ ibx_copy_incremental_over_full()
16181618
NULL};
16191619
const char *sup_files[] = {"xtrabackup_binlog_info",
16201620
"xtrabackup_galera_info",
1621+
"donor_galera_info",
16211622
"xtrabackup_slave_info",
16221623
"xtrabackup_info",
16231624
"ib_lru_dump",

‎extra/mariabackup/backup_copy.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
/* special files */
1010
#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info"
1111
#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info"
12+
#define XTRABACKUP_DONOR_GALERA_INFO "donor_galera_info"
1213
#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info"
1314
#define XTRABACKUP_INFO "xtrabackup_info"
1415

‎extra/mariabackup/backup_mysql.cc

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,7 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection)
14291429
{
14301430
char *state_uuid = NULL, *state_uuid55 = NULL;
14311431
char *last_committed = NULL, *last_committed55 = NULL;
1432+
char *domain_id = NULL, *domain_id55 = NULL;
14321433
bool result;
14331434

14341435
mysql_variable status[] = {
@@ -1439,6 +1440,12 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection)
14391440
{NULL, NULL}
14401441
};
14411442

1443+
mysql_variable value[] = {
1444+
{"Wsrep_gtid_domain_id", &domain_id},
1445+
{"wsrep_gtid_domain_id", &domain_id55},
1446+
{NULL, NULL}
1447+
};
1448+
14421449
/* When backup locks are supported by the server, we should skip
14431450
creating xtrabackup_galera_info file on the backup stage, because
14441451
wsrep_local_state_uuid and wsrep_last_committed will be inconsistent
@@ -1457,9 +1464,26 @@ write_galera_info(ds_ctxt *datasink, MYSQL *connection)
14571464
goto cleanup;
14581465
}
14591466

1467+
read_mysql_variables(connection, "SHOW VARIABLES LIKE 'wsrep%'", value, true);
1468+
1469+
if (domain_id == NULL && domain_id55 == NULL) {
1470+
msg("Warning: failed to get master wsrep state from SHOW VARIABLES.");
1471+
result = true;
1472+
goto cleanup;
1473+
}
1474+
14601475
result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO,
1461-
"%s:%s\n", state_uuid ? state_uuid : state_uuid55,
1462-
last_committed ? last_committed : last_committed55);
1476+
"%s:%s %s\n", state_uuid ? state_uuid : state_uuid55,
1477+
last_committed ? last_committed : last_committed55,
1478+
domain_id ? domain_id : domain_id55);
1479+
1480+
if (result)
1481+
{
1482+
result= datasink->backup_file_printf(XTRABACKUP_DONOR_GALERA_INFO,
1483+
"%s:%s %s\n", state_uuid ? state_uuid : state_uuid55,
1484+
last_committed ? last_committed : last_committed55,
1485+
domain_id ? domain_id : domain_id55);
1486+
}
14631487
if (result)
14641488
{
14651489
write_current_binlog_file(datasink, connection);

‎extra/mariabackup/wsrep.cc

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ permission notice:
5353

5454
/*! Name of file where Galera info is stored on recovery */
5555
#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info"
56+
#define XB_GALERA_DONOR_INFO_FILENAME "donor_galera_info"
5657

5758
/***********************************************************************
5859
Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that
@@ -67,7 +68,7 @@ xb_write_galera_info(bool incremental_prepare)
6768
long long seqno;
6869
MY_STAT statinfo;
6970

70-
/* Do not overwrite existing an existing file to be compatible with
71+
/* Do not overwrite an existing file to be compatible with
7172
servers with older server versions */
7273
if (!incremental_prepare &&
7374
my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) {
@@ -101,10 +102,11 @@ xb_write_galera_info(bool incremental_prepare)
101102

102103
seqno = wsrep_xid_seqno(&xid);
103104

104-
msg("mariabackup: Recovered WSREP position: %s:%lld\n",
105-
uuid_str, (long long) seqno);
105+
msg("mariabackup: Recovered WSREP position: %s:%lld domain_id: %lld\n",
106+
uuid_str, (long long) seqno, (long long)wsrep_get_domain_id());
106107

107-
if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) {
108+
if (fprintf(fp, "%s:%lld %lld", uuid_str, (long long) seqno,
109+
(long long)wsrep_get_domain_id()) < 0) {
108110

109111
die(
110112
"could not write to " XB_GALERA_INFO_FILENAME

‎include/mysql/service_wsrep.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ extern struct wsrep_service_st {
9595
void (*wsrep_thd_kill_LOCK_func)(const MYSQL_THD thd);
9696
void (*wsrep_thd_kill_UNLOCK_func)(const MYSQL_THD thd);
9797
void (*wsrep_thd_set_wsrep_PA_unsafe_func)(MYSQL_THD thd);
98+
uint32 (*wsrep_get_domain_id_func)();
9899
} *wsrep_service;
99100

100101
#define MYSQL_SERVICE_WSREP_INCLUDED
@@ -144,6 +145,7 @@ extern struct wsrep_service_st {
144145
#define wsrep_thd_set_ignored_error(T,V) wsrep_service->wsrep_thd_set_ignored_error_func(T,V)
145146
#define wsrep_report_bf_lock_wait(T,I) wsrep_service->wsrep_report_bf_lock_wait(T,I)
146147
#define wsrep_thd_set_PA_unsafe(T) wsrep_service->wsrep_thd_set_PA_unsafe_func(T)
148+
#define wsrep_get_domain_id(T) wsrep_service->wsrep_get_domain_id_func(T)
147149
#else
148150

149151
#define MYSQL_SERVICE_WSREP_STATIC_INCLUDED
@@ -253,5 +255,6 @@ extern "C" void wsrep_report_bf_lock_wait(const THD *thd,
253255
unsigned long long trx_id);
254256
/* declare parallel applying unsafety for the THD */
255257
extern "C" void wsrep_thd_set_PA_unsafe(MYSQL_THD thd);
258+
extern "C" uint32 wsrep_get_domain_id();
256259
#endif
257260
#endif /* MYSQL_SERVICE_WSREP_INCLUDED */

‎mysql-test/include/galera_sst_method.combinations

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[rsync]
2+
wsrep-sst-method=rsync
3+
4+
[mariabackup]
5+
wsrep_sst_method=mariabackup
6+
7+
8+

‎mysql-test/include/galera_sst_method.inc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# The goal of including this file is to enable galera_sst_method combinations
2+
# (see include/galera_sst_method.combinations)
3+
4+
--source include/have_innodb.inc

‎mysql-test/suite/galera_3nodes/r/galera_gtid_consistency.result

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
connection node_2;
2+
connection node_1;
3+
connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
4+
connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2;
5+
set wsrep_sync_wait=0;
6+
connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
7+
set wsrep_sync_wait=0;
8+
connection node_1;
9+
CREATE PROCEDURE insert_row (IN node varchar(10), IN repeat_count int)
10+
BEGIN
11+
DECLARE current_num int;
12+
SET current_num = 0;
13+
WHILE current_num < repeat_count do
14+
INSERT INTO t1(node, name) VALUES (node, UUID());
15+
SET current_num = current_num + 1;
16+
END WHILE;
17+
END|
18+
CREATE TABLE t1 (id bigint not null primary key auto_increment, node VARCHAR(10), name VARCHAR(64)) ENGINE=innodb;
19+
# node_1
20+
show variables like '%gtid_binlog_pos%';
21+
Variable_name Value
22+
gtid_binlog_pos 1111-1-2
23+
connection node_2;
24+
# node_2
25+
show variables like '%gtid_binlog_pos%';
26+
Variable_name Value
27+
gtid_binlog_pos 1111-1-2
28+
connection node_3;
29+
# node_3
30+
show variables like '%gtid_binlog_pos%';
31+
Variable_name Value
32+
gtid_binlog_pos 1111-1-2
33+
connection node_1;
34+
CALL insert_row('node1', 500);;
35+
connection node_2;
36+
CALL insert_row('node2', 500);;
37+
connection node_3;
38+
CALL insert_row('node3', 500);;
39+
connection node_2;
40+
# Shutdown node_2, force SST
41+
connection node_2b;
42+
# Wait until node_2 leaves cluster
43+
connection node_1b;
44+
connection node_1;
45+
connection node_3;
46+
connection node_1;
47+
CALL insert_row('node1', 500);
48+
connection node_3;
49+
CALL insert_row('node3', 500);
50+
CREATE TABLE t2(i int primary key) engine=innodb;
51+
connection node_2;
52+
# Restart node_2
53+
# restart
54+
connection node_1b;
55+
# Wait until node_2 is back in cluster
56+
# node2 has joined
57+
# GTID in node1
58+
show variables like 'wsrep_gtid_domain_id';
59+
Variable_name Value
60+
wsrep_gtid_domain_id 1111
61+
show variables like '%gtid_binlog_pos%';
62+
Variable_name Value
63+
gtid_binlog_pos 1111-1-2503
64+
connection node_2;
65+
# GTID in node2
66+
show variables like 'wsrep_gtid_domain_id';
67+
Variable_name Value
68+
wsrep_gtid_domain_id 1111
69+
show variables like '%gtid_binlog_pos%';
70+
Variable_name Value
71+
gtid_binlog_pos 1111-1-2503
72+
connection node_3;
73+
# GTID in node3
74+
show variables like 'wsrep_gtid_domain_id';
75+
Variable_name Value
76+
wsrep_gtid_domain_id 1111
77+
show variables like '%gtid_binlog_pos%';
78+
Variable_name Value
79+
gtid_binlog_pos 1111-1-2503
80+
# Shutdown node_3
81+
connection node_3;
82+
SET GLOBAL wsrep_provider_options = 'gmcast.isolate = 1';
83+
# Wait until node_3 leaves cluster
84+
connection node_1b;
85+
connection node_1;
86+
CALL insert_row('node1', 50);
87+
CREATE TABLE t3(i int primary key) engine=innodb;
88+
connection node_3;
89+
# Rejoin node_3
90+
SET GLOBAL wsrep_provider_options = 'gmcast.isolate = 0';
91+
connection node_1b;
92+
# Wait until node_3 is back in cluster
93+
# node3 has joined
94+
connection node_1;
95+
# GTID in node1
96+
show variables like 'wsrep_gtid_domain_id';
97+
Variable_name Value
98+
wsrep_gtid_domain_id 1111
99+
show variables like '%gtid_binlog_pos%';
100+
Variable_name Value
101+
gtid_binlog_pos 1111-1-2554
102+
connection node_2;
103+
# GTID in node2
104+
show variables like 'wsrep_gtid_domain_id';
105+
Variable_name Value
106+
wsrep_gtid_domain_id 1111
107+
show variables like '%gtid_binlog_pos%';
108+
Variable_name Value
109+
gtid_binlog_pos 1111-1-2554
110+
connection node_3;
111+
# GTID in node3
112+
show variables like 'wsrep_gtid_domain_id';
113+
Variable_name Value
114+
wsrep_gtid_domain_id 1111
115+
show variables like '%gtid_binlog_pos%';
116+
Variable_name Value
117+
gtid_binlog_pos 1111-1-2554
118+
# One by one shutdown all nodes
119+
connection node_3;
120+
# shutdown node_3
121+
connection node_2;
122+
# wait until node_3 is out of cluster
123+
# shutdown node_2
124+
connection node_1;
125+
# wait until node_2 is out of cluster
126+
# shutdown node_1
127+
# Bootstrap from node_1
128+
connection node_1;
129+
# restart: --wsrep_new_cluster
130+
show variables like 'wsrep_gtid_domain_id';
131+
Variable_name Value
132+
wsrep_gtid_domain_id 1111
133+
show variables like '%gtid_binlog_pos%';
134+
Variable_name Value
135+
gtid_binlog_pos 1111-1-2554
136+
ANALYZE TABLE t2;
137+
Table Op Msg_type Msg_text
138+
test.t2 analyze status Engine-independent statistics collected
139+
test.t2 analyze status OK
140+
CALL insert_row('node1', 100);;
141+
# Restart node_2
142+
connection node_2;
143+
# restart
144+
connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
145+
set wsrep_sync_wait=0;
146+
connection node_1c;
147+
# wait until node_1 and node_2 are in cluster
148+
connection node_2;
149+
ALTER TABLE t2 ADD COLUMN (k int);
150+
CALL insert_row('node2', 100);;
151+
# Restart node_3
152+
connection node_3;
153+
# restart
154+
connection node_1c;
155+
# wait until all nodes are back in cluster
156+
after cluster restart
157+
connection node_2;
158+
connection node_1;
159+
connection node_1;
160+
node1 GTID
161+
show variables like 'wsrep_gtid_domain_id';
162+
Variable_name Value
163+
wsrep_gtid_domain_id 1111
164+
show variables like '%gtid_binlog_pos%';
165+
Variable_name Value
166+
gtid_binlog_pos 1111-1-2756
167+
connection node_2;
168+
node2 GTID
169+
show variables like 'wsrep_gtid_domain_id';
170+
Variable_name Value
171+
wsrep_gtid_domain_id 1111
172+
show variables like '%gtid_binlog_pos%';
173+
Variable_name Value
174+
gtid_binlog_pos 1111-1-2756
175+
connection node_3;
176+
node3 GTID
177+
show variables like 'wsrep_gtid_domain_id';
178+
Variable_name Value
179+
wsrep_gtid_domain_id 1111
180+
show variables like '%gtid_binlog_pos%';
181+
Variable_name Value
182+
gtid_binlog_pos 1111-1-2756
183+
connection node_1;
184+
table size in node1
185+
SELECT COUNT(*) FROM t1;
186+
COUNT(*)
187+
2750
188+
connection node_2;
189+
table size in node2
190+
SELECT COUNT(*) FROM t1;
191+
COUNT(*)
192+
2750
193+
connection node_3;
194+
table size in node3
195+
SELECT COUNT(*) FROM t1;
196+
COUNT(*)
197+
2750
198+
connection node_2;
199+
call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node");
200+
call mtr.add_suppression("WSREP: Sending JOIN failed:.*");
201+
connection node_3;
202+
call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node");
203+
call mtr.add_suppression("WSREP: Sending JOIN failed:.*");
204+
# cleanup
205+
connection node_1;
206+
DROP PROCEDURE insert_row;
207+
DROP TABLE t1;
208+
DROP TABLE t2;
209+
DROP TABLE t3;
210+
connection node_3;
211+
connection node_2;
212+
disconnect node_3;
213+
disconnect node_2b;
214+
disconnect node_1b;
215+
disconnect node_1c;

‎mysql-test/suite/galera_3nodes/t/MDEV-29171.test

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@
44
#
55

66
--source include/galera_cluster.inc
7-
--source include/have_innodb.inc
7+
--source include/galera_sst_method.inc
8+
--source include/force_restart.inc
89

910
#
1011
# Initially wsrep gtid domain id is 100
1112
#
13+
1214
--connection node_1
1315
select @@wsrep_gtid_domain_id,@@wsrep_node_name;
1416

0 commit comments

Comments
 (0)