MDEV-18426: Most of the mtr tests in the galera_3nodes suite fail

Most of the mtr tests in the galera_3nodes suite fail
for a variety of reasons with a variety of errors.

This patch fixes several substantial flaws
in the galera_3nodes suite tests and in the mtr framework
service files, adapting the tests from galera_3nodes
for the current version of MariaDB.

This patch also synchronizes some galera_3nodes-related
files with the latest changes made for MDEV-17835 (v2 patch)
and for MDEV-18379 in other branches (10.2 and 10.3).

Closes #1161

mysql-test/suite/galera/include/galera_resume.inc → mysql-test/include/galera_resume.inc

@@ -3,7 +3,7 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -18 $mysqld_pid");
+        system("kill -SIGCONT $mysqld_pid");
         exit(0);
 EOF
 

mysql-test/suite/galera_3nodes/include/galera_suspend.inc → mysql-test/include/galera_suspend.inc

@@ -9,6 +9,6 @@
         my $pid_filename = $ENV{'_SUSPEND_NODE_PIDFILE'};
         my $mysqld_pid = `cat $pid_filename`;
         chomp($mysqld_pid);
-        system("kill -19 $mysqld_pid");
+        system("kill -SIGSTOP $mysqld_pid");
         exit(0);
 EOF

mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf

@@ -9,6 +9,7 @@ innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
 wsrep_gtid_mode=1
 gtid_ignore_duplicates
+auto_increment_increment=3
 
 wsrep-on=1
 wsrep-provider=@ENV.WSREP_PROVIDER

mysql-test/suite/galera_3nodes/galera_3nodes.cnf

@@ -5,6 +5,7 @@
 binlog-format=row
 innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
+auto_increment_increment=3
 
 wsrep-on=1
 wsrep-provider=@ENV.WSREP_PROVIDER

mysql-test/suite/galera_3nodes/r/galera_pc_weight.result

@@ -1,8 +1,10 @@
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
+1
 SET GLOBAL wsrep_provider_options = 'pc.weight=3';
 SELECT VARIABLE_VALUE = 5 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 5
+1
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
 SET SESSION wsrep_sync_wait=0;
 SET SESSION wsrep_on=OFF;
@@ -12,6 +14,7 @@ Variable_name	Value
 wsrep_cluster_size	2
 SHOW STATUS LIKE 'wsrep_cluster_weight';
 Variable_name	Value
+wsrep_cluster_weight	0
 SHOW STATUS LIKE 'wsrep_cluster_status';
 Variable_name	Value
 wsrep_cluster_status	non-Primary
@@ -35,6 +38,7 @@ Variable_name	Value
 wsrep_cluster_size	2
 SHOW STATUS LIKE 'wsrep_cluster_weight';
 Variable_name	Value
+wsrep_cluster_weight	0
 SHOW STATUS LIKE 'wsrep_cluster_status';
 Variable_name	Value
 wsrep_cluster_status	non-Primary
@@ -52,6 +56,7 @@ Variable_name	Value
 wsrep_local_state_comment	Initialized
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
+1
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 VARIABLE_VALUE = 'Primary'
 1
@@ -70,12 +75,14 @@ VARIABLE_VALUE = 'Synced'
 SET GLOBAL wsrep_provider_options = 'pc.weight=1';
 SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 1
+1
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 3
 1
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
+1
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 VARIABLE_VALUE = 'Primary'
 1
@@ -96,6 +103,7 @@ VARIABLE_VALUE = 3
 1
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
+1
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 VARIABLE_VALUE = 'Primary'
 1
@@ -116,6 +124,7 @@ VARIABLE_VALUE = 3
 1
 SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_weight';
 VARIABLE_VALUE = 3
+1
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 VARIABLE_VALUE = 'Primary'
 1

mysql-test/suite/galera_3nodes/r/galera_safe_to_bootstrap.result

@@ -16,7 +16,17 @@ Killing server ...
 safe_to_bootstrap: 1
 safe_to_bootstrap: 0
 safe_to_bootstrap: 0
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 Table	Create Table

mysql-test/suite/galera_3nodes/suite.pm

@@ -81,11 +81,11 @@ sub skip_combinations {
   my %skip = ();
   $skip{'include/have_filekeymanagement.inc'} = 'needs file_key_management plugin'
              unless $ENV{FILE_KEY_MANAGEMENT_SO};
-  $skip{'include/have_mariabackup.inc'} = 'Need mariabackup'
+  $skip{'suite/galera/include/have_mariabackup.inc'} = 'Need mariabackup'
              unless which(mariabackup);
-  $skip{'include/have_mariabackup.inc'} = 'Need ss'
+  $skip{'suite/galera/include/have_mariabackup.inc'} = 'Need ss'
              unless which(ss);
-  $skip{'include/have_mariabackup.inc'} = 'Need socat or nc'
+  $skip{'suite/galera/include/have_mariabackup.inc'} = 'Need socat or nc'
              unless $ENV{MTR_GALERA_TFMT};
   %skip;
 }

mysql-test/suite/galera_3nodes/t/galera_garbd.test

@@ -7,10 +7,20 @@
 --source include/have_innodb.inc
 --source include/big_test.inc
 
---echo Killing node #3 to free ports for garbd ...
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3
 --source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--echo Killing node #3 to free ports for garbd ...
+--connection node_3
+--let $gp3 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_3 = `SELECT SUBSTR('$gp3', 1, LOCATE(';', '$gp3') - 1)`
 --source include/shutdown_mysqld.inc
 
 --connection node_1
@@ -18,7 +28,9 @@
 --source include/wait_condition.inc
 
 --echo Starting garbd ...
---exec `dirname $WSREP_PROVIDER`/garb/garbd --address "gcomm://127.0.0.1:$NODE_GALERAPORT_1" --group my_wsrep_cluster --options 'base_port=$NODE_GALERAPORT_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
+--let $gp1 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+--let $galera_port_1 = `SELECT SUBSTR('$gp1', 1, LOCATE(';', '$gp1') - 1)`
+--exec `dirname $WSREP_PROVIDER`/../../bin/garb/garbd --address "gcomm://127.0.0.1:$galera_port_1" --group my_wsrep_cluster --options 'base_port=$galera_port_3' > $MYSQL_TMP_DIR/garbd.log 2>&1 &
 
 --sleep 5
 
@@ -32,7 +44,7 @@ INSERT INTO t1 VALUES (1);
 SELECT COUNT(*) = 1 FROM t1;
 
 --echo Killing garbd ...
---exec pkill --oldest --full garbd.*$NODE_GALERAPORT_3
+--exec pkill --oldest --full garbd.*$galera_port_3
 
 --sleep 5
 
@@ -51,6 +63,8 @@ DROP TABLE t1;
 --connection node_3
 --source include/start_mysqld.inc
 
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
 
 # Workaround for galera#101
 

mysql-test/suite/galera_3nodes/t/galera_innobackupex_backup.test

@@ -4,7 +4,7 @@
 
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
---source include/have_mariabackup.inc
+--source suite/galera/include/have_mariabackup.inc
 
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3

mysql-test/suite/galera_3nodes/t/galera_ipv6_mariabackup.test

@@ -1,6 +1,6 @@
 --source include/galera_cluster.inc
 --source include/check_ipv6.inc
---source include/have_mariabackup.inc
+--source suite/galera/include/have_mariabackup.inc
 
 # Confirm that initial handshake happened over ipv6
 

mysql-test/suite/galera_3nodes/t/galera_ipv6_mysqldump.opt

@@ -0,0 +1 @@
+--bind-address=::

mysql-test/suite/galera_3nodes/t/galera_ipv6_rsync.opt

@@ -0,0 +1 @@
+--bind-address=::

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf

@@ -1,11 +1,11 @@
 !include ../galera_3nodes.cnf
 
 [mysqld.1]
-wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.2]
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 
 [mysqld.3]
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;pc.ignore_sb=true;gcache.size=1M'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
 

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test

@@ -17,6 +17,13 @@
 --let $galera_server_number = 3
 --source include/galera_connect.inc
 
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 
@@ -99,3 +106,9 @@ SELECT LENGTH(f1) = 512 * 1024 FROM t2;
 CALL mtr.add_suppression("WSREP: Unsupported protocol downgrade: incremental data collection disabled");
 
 DROP TABLE t1, t2;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc
+
+--let $galera_cluster_size=3
+--source include/galera_end.inc

mysql-test/suite/galera_3nodes/t/galera_safe_to_bootstrap.test

@@ -1,8 +1,19 @@
 #
 # Test the safe_to_bootstrap in grastate.dat 
 #
-
 --source include/galera_cluster.inc
+
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 
 #
@@ -47,8 +58,6 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 #
 # Shut down one more node
 #
-
---connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
 --connection node_3
 --source include/shutdown_mysqld.inc
 
@@ -129,9 +138,9 @@ SET SESSION wsrep_on = OFF;
 #
 
 --error 1
---exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.2 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 --error 1
---exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster | grep 'This node is not safe to bootstrap the cluster'
+--exec $MYSQLD --defaults-group-suffix=.3 --defaults-file=$MYSQLTEST_VARDIR/my.cnf --wsrep-new-cluster --wsrep-cluster-address='gcomm://' | grep 'This node is not safe to bootstrap the cluster'
 
 #
 # Attempt to bootstrap starting from node #1, should succeed
@@ -154,10 +163,23 @@ SET SESSION wsrep_on = OFF;
 --source include/wait_condition.inc
 
 --connection node_2
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 
 --connection node_3
+CALL mtr.add_suppression("WSREP: no nodes coming from prim view, prim not possible");
+CALL mtr.add_suppression("WSREP: It may not be safe to bootstrap the cluster from this node");
+CALL mtr.add_suppression("WSREP: wsrep::connect(.*) failed: 7");
+CALL mtr.add_suppression("Aborting");
+CALL mtr.add_suppression("WSREP: moving position backwards: [0-9]+ -> 0");
 CALL mtr.add_suppression("Failed to prepare for incremental state transfer");
 SHOW CREATE TABLE t1;
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc

mysql-test/suite/galera_3nodes/t/galera_var_dirty_reads2.test

@@ -5,6 +5,17 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
+--let $galera_connection_name = node_3
+--let $galera_server_number = 3
+--source include/galera_connect.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
+--connection node_1
 CREATE TABLE t1 (f1 INTEGER);
 INSERT INTO t1 VALUES (1);
 
@@ -110,3 +121,6 @@ SET GLOBAL wsrep_provider_options='gmcast.isolate=0';
 --source include/wait_condition.inc
 
 DROP TABLE t1;
+
+# Restore original auto_increment_offset values.
+--source ../galera/include/auto_increment_offset_restore.inc

scripts/wsrep_sst_mariabackup.sh

@@ -520,7 +520,7 @@ kill_xtrabackup()
 setup_ports()
 {
     if [[ "$WSREP_SST_OPT_ROLE"  == "donor" ]];then
-        if [[ ${WSREP_SST_OPT_ADDR:0:1} == '[' ]];then
+        if [ "${WSREP_SST_OPT_ADDR#\[}" != "$WSREP_SST_OPT_ADDR" ]; then
             remain=$(echo $WSREP_SST_OPT_ADDR | awk -F '\\][:/]' '{ print $2 }')
             REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F '\\]:' '{ print $1 }')"]"
             SST_PORT=$(echo $remain | awk -F '[:/]' '{ print $1 }')
@@ -533,7 +533,7 @@ setup_ports()
             sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }')
         fi
     else
-        if [[ ${WSREP_SST_OPT_ADDR:0:1} == '[' ]];then
+        if [ "${WSREP_SST_OPT_ADDR#\[}" != "$WSREP_SST_OPT_ADDR" ]; then
             SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F '\\]:' '{ print $2 }')
         else
             SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }')
@@ -952,7 +952,7 @@ then
     if [ -z "${SST_PORT}" ]
     then
         SST_PORT=4444
-        if [[ ${ADDR:0:1} == '[' ]];then
+        if [ "${ADDR#\[}" != "$ADDR" ]; then
             ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F '\\]:' '{ print $1 }')]:${SST_PORT}"
         else
             ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}"

scripts/wsrep_sst_rsync.sh

@@ -364,7 +364,7 @@ then
     rm -rf "$RSYNC_PID"
 
     ADDR=$WSREP_SST_OPT_ADDR
-    if [[ ${ADDR:0:1} == '[' ]]; then
+    if [ "${ADDR#\[}" != "$ADDR" ]; then
         RSYNC_PORT=$(echo $ADDR | awk -F '\\]:' '{ print $2 }')
         RSYNC_ADDR=$(echo $ADDR | awk -F '\\]:' '{ print $1 }')"]"
     else