MDEV-4605: Failing to load GTID slave position from rpl.gtid_slave_pos

There were several cases where the slave GTID position was not loaded
correctly before being used. This caused various failures such as
corrupting the position at slave start and empty values of
@@gtid_slave_pos and @@gtid_current_pos.

Fixed by adding more checks for loaded position, and by always loading
the position at server startup.

mysql-test/suite/rpl/r/rpl_gtid_stop_start.result

@@ -50,5 +50,15 @@ a
 3
 4
 5
+*** Test that @@gtid_slave_pos and @@gtid_current_pos are correctly loaded even if slave threads have not started. ***
+SET @slave_pos2= @@GLOBAL.gtid_slave_pos;
+SET @current_pos2= @@GLOBAL.gtid_current_pos;
+SELECT IF(@slave_pos1=@slave_pos2, "OK", CONCAT(@slave_pos1, " != ", @slave_pos2));
+IF(@slave_pos1=@slave_pos2, "OK", CONCAT(@slave_pos1, " != ", @slave_pos2))
+OK
+SELECT IF(@current_pos1=@current_pos2, "OK", CONCAT(@current_pos1, " != ", @current_pos2));
+IF(@current_pos1=@current_pos2, "OK", CONCAT(@current_pos1, " != ", @current_pos2))
+OK
+include/start_slave.inc
 DROP TABLE t1;
 include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_gtid_stop_start.test

@@ -93,6 +93,31 @@ INSERT INTO t1 VALUES(5);
 --source include/wait_condition.inc
 SELECT * FROM t1 ORDER BY a;
 
+--echo *** Test that @@gtid_slave_pos and @@gtid_current_pos are correctly loaded even if slave threads have not started. ***
+--let $slave_pos1= `SELECT @@GLOBAL.gtid_slave_pos`
+--let $current_pos1= `SELECT @@GLOBAL.gtid_current_pos`
+
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+wait
+EOF
+--shutdown_server 30
+--source include/wait_until_disconnected.inc
+
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+restart: --skip-slave-start=1 --skip-log-bin
+EOF
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--disable_query_log
+eval SET @slave_pos1= "$slave_pos1";
+eval SET @current_pos1= "$current_pos1";
+--enable_query_log
+SET @slave_pos2= @@GLOBAL.gtid_slave_pos;
+SET @current_pos2= @@GLOBAL.gtid_current_pos;
+SELECT IF(@slave_pos1=@slave_pos2, "OK", CONCAT(@slave_pos1, " != ", @slave_pos2));
+SELECT IF(@current_pos1=@current_pos2, "OK", CONCAT(@current_pos1, " != ", @current_pos2));
+--source include/start_slave.inc
+
 
 --connection server_1
 DROP TABLE t1;

sql/mysqld.cc

@@ -931,7 +931,8 @@ static PSI_cond_info all_server_conds[]=
 
 PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert,
   key_thread_handle_manager, key_thread_main,
-  key_thread_one_connection, key_thread_signal_hand;
+  key_thread_one_connection, key_thread_signal_hand,
+  key_thread_slave_init;
 
 static PSI_thread_info all_server_threads[]=
 {
@@ -956,7 +957,8 @@ static PSI_thread_info all_server_threads[]=
   { &key_thread_handle_manager, "manager", PSI_FLAG_GLOBAL},
   { &key_thread_main, "main", PSI_FLAG_GLOBAL},
   { &key_thread_one_connection, "one_connection", 0},
-  { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL}
+  { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL},
+  { &key_thread_slave_init, "slave_init", PSI_FLAG_GLOBAL}
 };
 
 PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest,

sql/mysqld.h

@@ -283,7 +283,7 @@ extern PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy;
 
 extern PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert,
   key_thread_handle_manager, key_thread_kill_server, key_thread_main,
-  key_thread_one_connection, key_thread_signal_hand;
+  key_thread_one_connection, key_thread_signal_hand, key_thread_slave_init;
 
 extern PSI_file_key key_file_binlog, key_file_binlog_index, key_file_casetest,
   key_file_dbopt, key_file_des_key_file, key_file_ERRMSG, key_select_to_file,

sql/rpl_rli.cc

@@ -1425,7 +1425,10 @@ rpl_load_gtid_slave_state(THD *thd)
 
   bitmap_set_all(table->read_set);
   if ((err= table->file->ha_rnd_init_with_error(1)))
+  {
+    table->file->print_error(err, MYF(0));
     goto end;
+  }
   table_scanned= true;
   for (;;)
   {
@@ -1440,7 +1443,10 @@ rpl_load_gtid_slave_state(THD *thd)
       else if (err == HA_ERR_END_OF_FILE)
         break;
       else
+      {
+        table->file->print_error(err, MYF(0));
         goto end;
+      }
     }
     domain_id= (ulonglong)table->field[0]->val_int();
     sub_id= (ulonglong)table->field[1]->val_int();
@@ -1465,6 +1471,7 @@ rpl_load_gtid_slave_state(THD *thd)
       if (!(entry= (struct local_element *)my_malloc(sizeof(*entry),
                                                      MYF(MY_WME))))
       {
+        my_error(ER_OUTOFMEMORY, MYF(0), (int)sizeof(*entry));
         err= 1;
         goto end;
       }
@@ -1475,12 +1482,18 @@ rpl_load_gtid_slave_state(THD *thd)
       if ((err= my_hash_insert(&hash, (uchar *)entry)))
       {
         my_free(entry);
+        my_error(ER_OUT_OF_RESOURCES, MYF(0));
         goto end;
       }
     }
   }
 
   rpl_global_gtid_slave_state.lock();
+  if (rpl_global_gtid_slave_state.loaded)
+  {
+    rpl_global_gtid_slave_state.unlock();
+    goto end;
+  }
   for (i= 0; i < hash.records; ++i)
   {
     entry= (struct local_element *)my_hash_element(&hash, i);
@@ -1490,14 +1503,15 @@ rpl_load_gtid_slave_state(THD *thd)
                                                  entry->gtid.seq_no)))
     {
       rpl_global_gtid_slave_state.unlock();
+      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       goto end;
     }
     if (opt_bin_log &&
         mysql_bin_log.bump_seq_no_counter_if_needed(entry->gtid.domain_id,
                                                     entry->gtid.seq_no))
     {
-      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       rpl_global_gtid_slave_state.unlock();
+      my_error(ER_OUT_OF_RESOURCES, MYF(0));
       goto end;
     }
   }

sql/share/errmsg-utf8.txt

@@ -6538,7 +6538,7 @@ ER_GTID_OPEN_TABLE_FAILED
 ER_GTID_POSITION_NOT_FOUND_IN_BINLOG
 	eng "Connecting slave requested to start from GTID %u-%u-%llu, which is not in the master's binlog"
 ER_CANNOT_LOAD_SLAVE_GTID_STATE
-	eng "Failed to load replication slave GTID state from table %s.%s"
+	eng "Failed to load replication slave GTID position from table %s.%s"
 ER_MASTER_GTID_POS_CONFLICTS_WITH_BINLOG
 	eng "Specified GTID %u-%u-%llu conflicts with the binary log which contains a more recent GTID %u-%u-%llu. If MASTER_GTID_POS=CURRENT_POS is used, the binlog position will override the new value of @@gtid_slave_pos."
 ER_MASTER_GTID_POS_MISSING_DOMAIN

sql/slave.cc

@@ -253,6 +253,66 @@ static void init_slave_psi_keys(void)
 }
 #endif /* HAVE_PSI_INTERFACE */
 
+
+static bool slave_init_thread_running;
+
+
+pthread_handler_t
+handle_slave_init(void *arg __attribute__((unused)))
+{
+  THD *thd;
+
+  my_thread_init();
+  thd= new THD;
+  thd->thread_stack= (char*) &thd;           /* Set approximate stack start */
+  mysql_mutex_lock(&LOCK_thread_count);
+  thd->thread_id= thread_id++;
+  mysql_mutex_unlock(&LOCK_thread_count);
+  thd->store_globals();
+
+  thd_proc_info(thd, "Loading slave GTID position from table");
+  if (rpl_load_gtid_slave_state(thd))
+    sql_print_warning("Failed to load slave replication state from table "
+                      "%s.%s: %u: %s", "mysql",
+                      rpl_gtid_slave_state_table_name.str,
+                      thd->stmt_da->sql_errno(), thd->stmt_da->message());
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  delete thd;
+  mysql_mutex_unlock(&LOCK_thread_count);
+  my_thread_end();
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  slave_init_thread_running= false;
+  mysql_cond_signal(&COND_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  return 0;
+}
+
+
+static int
+run_slave_init_thread()
+{
+  pthread_t th;
+
+  slave_init_thread_running= true;
+  if (mysql_thread_create(key_thread_slave_init, &th, NULL,
+                          handle_slave_init, NULL))
+  {
+    sql_print_error("Failed to create thread while initialising slave");
+    return 1;
+  }
+
+  mysql_mutex_lock(&LOCK_thread_count);
+  while (slave_init_thread_running)
+    mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  return 0;
+}
+
+
 /* Initialize slave structures */
 
 int init_slave()
@@ -264,6 +324,9 @@ int init_slave()
   init_slave_psi_keys();
 #endif
 
+  if (run_slave_init_thread())
+    return 1;
+
   /*
     This is called when mysqld starts. Before client connections are
     accepted. However bootstrap may conflict with us if it does START SLAVE.
@@ -3454,6 +3517,16 @@ pthread_handler_t handle_slave_io(void *arg)
   /* This must be called before run any binlog_relay_io hooks */
   my_pthread_setspecific_ptr(RPL_MASTER_INFO, mi);
 
+  /* Load the set of seen GTIDs, if we did not already. */
+  if (rpl_load_gtid_slave_state(thd))
+  {
+    mi->report(ERROR_LEVEL, thd->stmt_da->sql_errno(), 
+                "Unable to load replication GTID slave state from mysql.%s: %s",
+                rpl_gtid_slave_state_table_name.str, thd->stmt_da->message());
+    goto err;
+  }
+
+
   if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi)))
   {
     mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR,

sql/sys_vars.cc

@@ -1262,6 +1262,12 @@ Sys_var_gtid_binlog_pos::global_value_ptr(THD *thd, LEX_STRING *base)
   String str(buf, sizeof(buf), system_charset_info);
   char *p;
 
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return NULL;
+  }
   str.length(0);
   if ((opt_bin_log && mysql_bin_log.append_state_pos(&str)) ||
       !(p= thd->strmake(str.ptr(), str.length())))
@@ -1308,6 +1314,14 @@ Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var)
   bool running;
 
   DBUG_ASSERT(var->type == OPT_GLOBAL);
+
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return true;
+  }
+
   mysql_mutex_lock(&LOCK_active_mi);
   running= master_info_index->give_error_if_slave_running();
   mysql_mutex_unlock(&LOCK_active_mi);
@@ -1366,6 +1380,13 @@ Sys_var_gtid_slave_pos::global_value_ptr(THD *thd, LEX_STRING *base)
   String str;
   char *p;
 
+  if (!rpl_global_gtid_slave_state.loaded)
+  {
+    my_error(ER_CANNOT_LOAD_SLAVE_GTID_STATE, MYF(0), "mysql",
+             rpl_gtid_slave_state_table_name.str);
+    return NULL;
+  }
+
   str.length(0);
   if (rpl_append_gtid_state(&str, false) ||
       !(p= thd->strmake(str.ptr(), str.length())))