Merge 10.6 into 10.7

mysql-test/main/range.result

@@ -3635,6 +3635,25 @@ SELECT * FROM t1 LEFT JOIN t2 ON a = pk WHERE b >= 0 AND pk IS NULL;
 a	pk	b
 DROP TABLE t1, t2;
 SET @@optimizer_switch= @save_optimizer_switch;
+
+# MDEV-28858 Wrong result with table elimination combined with
+# not_null_range_scan
+#
+CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES  (10,1),(null,2);
+CREATE TABLE t2 (pk INT PRIMARY KEY) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+SET @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch= 'not_null_range_scan=on';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+b
+2
+SET optimizer_switch= 'not_null_range_scan=off';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+b
+2
+SET @@optimizer_switch=@save_optimizer_switch;
+drop table t1,t2;
 #
 # End of 10.5 tests
 #

mysql-test/main/range.test

@@ -2492,6 +2492,24 @@ DROP TABLE t1, t2;
 
 SET @@optimizer_switch= @save_optimizer_switch;
 
+--echo
+--echo # MDEV-28858 Wrong result with table elimination combined with
+--echo # not_null_range_scan
+--echo #
+
+CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES  (10,1),(null,2);
+CREATE TABLE t2 (pk INT PRIMARY KEY) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+
+SET @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch= 'not_null_range_scan=on';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+SET optimizer_switch= 'not_null_range_scan=off';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+SET @@optimizer_switch=@save_optimizer_switch;
+drop table t1,t2;
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

mysql-test/main/range_mrr_icp.result

@@ -3624,6 +3624,25 @@ SELECT * FROM t1 LEFT JOIN t2 ON a = pk WHERE b >= 0 AND pk IS NULL;
 a	pk	b
 DROP TABLE t1, t2;
 SET @@optimizer_switch= @save_optimizer_switch;
+
+# MDEV-28858 Wrong result with table elimination combined with
+# not_null_range_scan
+#
+CREATE TABLE t1 (a INT, b INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES  (10,1),(null,2);
+CREATE TABLE t2 (pk INT PRIMARY KEY) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (1),(2);
+SET @save_optimizer_switch= @@optimizer_switch;
+SET optimizer_switch= 'not_null_range_scan=on';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+b
+2
+SET optimizer_switch= 'not_null_range_scan=off';
+SELECT t1.b FROM t1 LEFT JOIN t2 ON t1.a = t2.pk WHERE t1.a IS NULL ORDER BY t1.b;
+b
+2
+SET @@optimizer_switch=@save_optimizer_switch;
+drop table t1,t2;
 #
 # End of 10.5 tests
 #

mysql-test/suite/encryption/r/innodb-redo-badkey.result

@@ -6,6 +6,8 @@ call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]
 call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
 call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t[12]\\.ibd'");
 call mtr.add_suppression("InnoDB: Failed to read page .* from file '.*'");
+call mtr.add_suppression("InnoDB: OPT_PAGE_CHECKSUM mismatch");
+call mtr.add_suppression("InnoDB: Set innodb_force_recovery=1 to ignore corruption");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed");
 call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space=");

mysql-test/suite/encryption/t/innodb-redo-badkey.test

@@ -15,6 +15,8 @@ call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]
 call mtr.add_suppression("InnoDB: Unable to decompress .*.test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]");
 call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t[12]\\.ibd'");
 call mtr.add_suppression("InnoDB: Failed to read page .* from file '.*'");
+call mtr.add_suppression("InnoDB: OPT_PAGE_CHECKSUM mismatch");
+call mtr.add_suppression("InnoDB: Set innodb_force_recovery=1 to ignore corruption");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed");
 # for innodb_checksum_algorithm=full_crc32 only

sql/sql_select.cc

@@ -29974,11 +29974,12 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond,
 
   DBUG_ENTER("build_notnull_conds_for_range_scans");
 
-  for (JOIN_TAB *s= join->join_tab + join->const_tables ;
+  for (JOIN_TAB *s= join->join_tab;
        s < join->join_tab + join->table_count ; s++)
   {
     /* Clear all needed bitmaps to mark found fields */
-    if (allowed & s->table->map)
+    if ((allowed & s->table->map) &&
+        !(s->table->map && join->const_table_map))
       bitmap_clear_all(&s->table->tmp_set);
   }
 
@@ -29993,17 +29994,18 @@ bool build_notnull_conds_for_range_scans(JOIN *join, Item *cond,
     For each table t from 'allowed' build a conjunction of NOT NULL predicates
     constructed for all found fields if they are included in some indexes.
     If the construction of the conjunction succeeds attach the formula to
-    t->table->notnull_cond. The condition will be used to look for complementary
-    range scans.
+    t->table->notnull_cond. The condition will be used to look for
+    complementary range scans.
   */
-  for (JOIN_TAB *s= join->join_tab + join->const_tables ;
+  for (JOIN_TAB *s= join->join_tab ;
        s < join->join_tab + join->table_count ; s++)
   {
     TABLE *tab= s->table;
     List<Item> notnull_list;
     Item *notnull_cond= 0;
 
-    if (!(allowed & tab->map))
+    if (!(allowed & tab->map) ||
+        (s->table->map && join->const_table_map))
       continue;
 
     for (Field** field_ptr= tab->field; *field_ptr; field_ptr++)

storage/innobase/CMakeLists.txt

@@ -236,7 +236,6 @@ SET(INNOBASE_SOURCES
 	include/trx0i_s.h
 	include/trx0purge.h
 	include/trx0rec.h
-	include/trx0rec.inl
 	include/trx0roll.h
 	include/trx0rseg.h
 	include/trx0sys.h

storage/innobase/include/trx0purge.h

@@ -33,10 +33,6 @@ Created 3/26/1996 Heikki Tuuri
 
 #include <queue>
 
-/** A dummy undo record used as a return value when we have a whole undo log
-which needs no purge */
-extern trx_undo_rec_t	trx_purge_dummy_rec;
-
 /** Prepend the history list with an undo log.
 Remove the undo log segment from the rseg slot if it is too big for reuse.
 @param[in]	trx		transaction

storage/innobase/include/trx0rec.h

@@ -24,8 +24,7 @@ Transaction undo log record
 Created 3/26/1996 Heikki Tuuri
 *******************************************************/
 
-#ifndef trx0rec_h
-#define trx0rec_h
+#pragma once
 
 #include "trx0types.h"
 #include "row0types.h"
@@ -37,29 +36,31 @@ Created 3/26/1996 Heikki Tuuri
 
 /***********************************************************************//**
 Copies the undo record to the heap.
-@return own: copy of undo log record */
-UNIV_INLINE
-trx_undo_rec_t*
-trx_undo_rec_copy(
-/*==============*/
-	const trx_undo_rec_t*	undo_rec,	/*!< in: undo log record */
-	mem_heap_t*		heap);		/*!< in: heap where copied */
-/**********************************************************************//**
-Reads the undo log record type.
-@return record type */
-UNIV_INLINE
-ulint
-trx_undo_rec_get_type(
-/*==================*/
-	const trx_undo_rec_t*	undo_rec);	/*!< in: undo log record */
+@param undo_rec   record in an undo log page
+@param heap       memory heap
+@return copy of undo_rec
+@retval nullptr if the undo log record is corrupted */
+inline trx_undo_rec_t* trx_undo_rec_copy(const trx_undo_rec_t *undo_rec,
+                                         mem_heap_t *heap)
+{
+  const size_t offset= ut_align_offset(undo_rec, srv_page_size);
+  const size_t end= mach_read_from_2(undo_rec);
+  if (end <= offset || end >= srv_page_size - FIL_PAGE_DATA_END)
+    return nullptr;
+  const size_t len= end - offset;
+  trx_undo_rec_t *rec= static_cast<trx_undo_rec_t*>
+    (mem_heap_dup(heap, undo_rec, len));
+  mach_write_to_2(rec, len);
+  return rec;
+}
+
 /**********************************************************************//**
 Reads the undo log record number.
 @return undo no */
-UNIV_INLINE
-undo_no_t
-trx_undo_rec_get_undo_no(
-/*=====================*/
-	const trx_undo_rec_t*	undo_rec);	/*!< in: undo log record */
+inline undo_no_t trx_undo_rec_get_undo_no(const trx_undo_rec_t *undo_rec)
+{
+  return mach_u64_read_much_compressed(undo_rec + 3);
+}
 
 /**********************************************************************//**
 Returns the start of the undo record data area. */
@@ -345,7 +346,3 @@ inline table_id_t trx_undo_rec_get_table_id(const trx_undo_rec_t *rec)
   mach_read_next_much_compressed(&rec);
   return mach_read_next_much_compressed(&rec);
 }
-
-#include "trx0rec.inl"
-
-#endif /* trx0rec_h */

storage/innobase/include/trx0rec.inl

@@ -1,73 +0,0 @@
-/*****************************************************************************
-
-Copyright (c) 1996, 2014, Oracle and/or its affiliates. All Rights Reserved.
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
-
-*****************************************************************************/
-
-/**************************************************//**
-@file include/trx0rec.ic
-Transaction undo log record
-
-Created 3/26/1996 Heikki Tuuri
-*******************************************************/
-
-/**********************************************************************//**
-Reads from an undo log record the record type.
-@return record type */
-UNIV_INLINE
-ulint
-trx_undo_rec_get_type(
-/*==================*/
-	const trx_undo_rec_t*	undo_rec)	/*!< in: undo log record */
-{
-	return(mach_read_from_1(undo_rec + 2) & (TRX_UNDO_CMPL_INFO_MULT - 1));
-}
-
-/**********************************************************************//**
-Reads the undo log record number.
-@return undo no */
-UNIV_INLINE
-undo_no_t
-trx_undo_rec_get_undo_no(
-/*=====================*/
-	const trx_undo_rec_t*	undo_rec)	/*!< in: undo log record */
-{
-	const byte*	ptr;
-
-	ptr = undo_rec + 3;
-
-	return(mach_u64_read_much_compressed(ptr));
-}
-
-/***********************************************************************//**
-Copies the undo record to the heap.
-@return own: copy of undo log record */
-UNIV_INLINE
-trx_undo_rec_t*
-trx_undo_rec_copy(
-/*==============*/
-	const trx_undo_rec_t*	undo_rec,	/*!< in: undo log record */
-	mem_heap_t*		heap)		/*!< in: heap where copied */
-{
-	ulint		len;
-
-	len = mach_read_from_2(undo_rec)
-		- ut_align_offset(undo_rec, srv_page_size);
-	ut_ad(len < srv_page_size);
-	trx_undo_rec_t* rec = static_cast<trx_undo_rec_t*>(
-		mem_heap_dup(heap, undo_rec, len));
-	mach_write_to_2(rec, len);
-	return rec;
-}

storage/innobase/include/trx0undo.h

@@ -360,7 +360,7 @@ public:
   page_id_t get_page_id() const { return page_id; }
 
   /** Handle the DML undo log and apply it on online indexes */
-  void apply_undo_rec();
+  inline void apply_undo_rec();
 
   ~UndorecApplier()
   {

storage/innobase/log/log0recv.cc

@@ -1194,17 +1194,14 @@ inline size_t recv_sys_t::files_size()
 }
 
 /** Process a file name from a FILE_* record.
-@param[in,out]	name		file name
+@param[in]	name		file name
 @param[in]	len		length of the file name
 @param[in]	space_id	the tablespace ID
 @param[in]	deleted		whether this is a FILE_DELETE record
 @param[in]	lsn		lsn of the redo log
-@param[in]	store		whether the redo log has to
-				stored */
-static
-void
-fil_name_process(char* name, ulint len, uint32_t space_id,
-		 bool deleted, lsn_t lsn, store_t *store)
+@param[in]	store		whether the redo log has to be stored */
+static void fil_name_process(const char *name, ulint len, uint32_t space_id,
+                             bool deleted, lsn_t lsn, const store_t *store)
 {
 	if (srv_operation == SRV_OPERATION_BACKUP
 	    || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER) {
@@ -1226,13 +1223,17 @@ fil_name_process(char* name, ulint len, uint32_t space_id,
 
 	file_name_t&	f = p.first->second;
 
-	if (deleted) {
-		/* Got FILE_DELETE */
-		if (auto d = deferred_spaces.find(static_cast<uint32_t>(
-							  space_id))) {
+	if (auto d = deferred_spaces.find(space_id)) {
+		if (deleted) {
 			d->deleted = true;
+			goto got_deleted;
 		}
+		goto reload;
+	}
 
+	if (deleted) {
+got_deleted:
+		/* Got FILE_DELETE */
 		if (!p.second && f.status != file_name_t::DELETED) {
 			f.status = file_name_t::DELETED;
 			if (f.space != NULL) {
@@ -1244,6 +1245,7 @@ fil_name_process(char* name, ulint len, uint32_t space_id,
 		ut_ad(f.space == NULL);
 	} else if (p.second // the first FILE_MODIFY or FILE_RENAME
 		   || f.name != fname.name) {
+reload:
 		fil_space_t*	space;
 
 		/* Check if the tablespace file exists and contains
@@ -1254,8 +1256,7 @@ fil_name_process(char* name, ulint len, uint32_t space_id,
 		case FIL_LOAD_OK:
 			ut_ad(space != NULL);
 
-			deferred_spaces.remove(
-				static_cast<uint32_t>(space_id));
+			deferred_spaces.remove(space_id);
 			if (!f.space) {
 				if (f.size
 				    || f.flags != f.initial_flags) {
@@ -1314,9 +1315,7 @@ same_space:
 			/** Skip the deferred spaces
 			when lsn is already processed */
 			if (*store != store_t::STORE_IF_EXISTS) {
-				deferred_spaces.add(
-					static_cast<uint32_t>(space_id),
-					name, lsn);
+				deferred_spaces.add(space_id, name, lsn);
 			}
 			break;
 		case FIL_LOAD_INVALID:

storage/innobase/row/row0purge.cc

@@ -1114,7 +1114,7 @@ row_purge(
 	trx_undo_rec_t*	undo_rec,	/*!< in: record to purge */
 	que_thr_t*	thr)		/*!< in: query thread */
 {
-	if (undo_rec != &trx_purge_dummy_rec) {
+	if (undo_rec != reinterpret_cast<trx_undo_rec_t*>(-1)) {
 		bool	updated_extern;
 
 		while (row_purge_parse_undo_rec(

storage/innobase/row/row0undo.cc

@@ -342,7 +342,11 @@ static bool row_undo_rec_get(undo_node_t* node)
 					   node->heap);
 	mtr.commit();
 
-	switch (trx_undo_rec_get_type(node->undo_rec)) {
+	if (UNIV_UNLIKELY(!node->undo_rec)) {
+		return false;
+	}
+
+	switch (node->undo_rec[2] & (TRX_UNDO_CMPL_INFO_MULT - 1)) {
 	case TRX_UNDO_INSERT_METADATA:
 		/* This record type was introduced in MDEV-11369
 		instant ADD COLUMN, which was implemented after
@@ -356,13 +360,12 @@ static bool row_undo_rec_get(undo_node_t* node)
 	case TRX_UNDO_INSERT_REC:
 	case TRX_UNDO_EMPTY:
 		node->roll_ptr |= 1ULL << ROLL_PTR_INSERT_FLAG_POS;
-		node->state = undo == temp
+		node->state = is_temp
 			? UNDO_INSERT_TEMPORARY : UNDO_INSERT_PERSISTENT;
 		break;
 	default:
-		node->state = undo == temp
+		node->state = is_temp
 			? UNDO_UPDATE_TEMPORARY : UNDO_UPDATE_PERSISTENT;
-		break;
 	}
 
 	trx->undo_no = node->undo_no = trx_undo_rec_get_undo_no(

storage/innobase/trx/trx0purge.cc

@@ -55,10 +55,6 @@ ulong		srv_max_purge_lag_delay = 0;
 /** The global data structure coordinating a purge */
 purge_sys_t	purge_sys;
 
-/** A dummy undo record used as a return value when we have a whole undo log
-which needs no purge */
-trx_undo_rec_t	trx_purge_dummy_rec;
-
 #ifdef UNIV_DEBUG
 my_bool		srv_purge_view_update_only_debug;
 #endif /* UNIV_DEBUG */
@@ -1022,7 +1018,9 @@ TRANSACTIONAL_TARGET static void trx_purge_choose_next_log()
 
 /***********************************************************************//**
 Gets the next record to purge and updates the info in the purge system.
-@return copy of an undo log record or pointer to the dummy undo log record */
+@return copy of an undo log record
+@retval -1 if there is nothing to purge
+@retval nullptr on corruption */
 static
 trx_undo_rec_t*
 trx_purge_get_next_rec(
@@ -1048,11 +1046,10 @@ trx_purge_get_next_rec(
 		/* Look for the next undo log and record to purge */
 
 		trx_purge_choose_next_log();
-
-		return(&trx_purge_dummy_rec);
+		return reinterpret_cast<trx_undo_rec_t*>(-1);
 	}
 
-	mtr_start(&mtr);
+	mtr.start();
 
 	const buf_block_t* undo_page
 		= buf_page_get_gen(page_id, 0, RW_S_LATCH, nullptr,
@@ -1060,7 +1057,7 @@ trx_purge_get_next_rec(
 	if (UNIV_UNLIKELY(!undo_page)) {
 corrupted:
 		mtr.commit();
-		return &trx_purge_dummy_rec;
+		return nullptr;
 	}
 
 	const buf_block_t* rec2_page = undo_page;
@@ -1105,16 +1102,16 @@ corrupted:
 	trx_undo_rec_t*	rec_copy = trx_undo_rec_copy(undo_page->page.frame
 						     + offset, heap);
 
-	mtr_commit(&mtr);
-
-	return(rec_copy);
+	mtr.commit();
+	return rec_copy;
 }
 
 /********************************************************************//**
 Fetches the next undo log record from the history list to purge. It must be
 released with the corresponding release function.
-@return copy of an undo log record or pointer to trx_purge_dummy_rec,
-if the whole undo log can skipped in purge; NULL if none left */
+@return copy of an undo log record
+@retval -1 if the whole undo log can skipped in purge
+@retval nullptr if nothing is left, or on corruption */
 static MY_ATTRIBUTE((warn_unused_result))
 trx_undo_rec_t*
 trx_purge_fetch_next_rec(
@@ -1130,13 +1127,12 @@ trx_purge_fetch_next_rec(
 		if (!purge_sys.next_stored) {
 			DBUG_PRINT("ib_purge",
 				   ("no logs left in the history list"));
-			return(NULL);
+			return nullptr;
 		}
 	}
 
 	if (purge_sys.tail.trx_no >= purge_sys.low_limit_no()) {
-
-		return(NULL);
+		return nullptr;
 	}
 
 	/* fprintf(stderr, "Thread %lu purging trx %llu undo record %llu\n",
@@ -1152,7 +1148,7 @@ trx_purge_fetch_next_rec(
 	/* The following call will advance the stored values of the
 	purge iterator. */
 
-	return(trx_purge_get_next_rec(n_pages_handled, heap));
+	return trx_purge_get_next_rec(n_pages_handled, heap);
 }
 
 /** Run a purge batch.
@@ -1229,7 +1225,8 @@ trx_purge_attach_undo_recs(ulint n_purge_threads)
 
 		if (purge_rec.undo_rec == NULL) {
 			break;
-		} else if (purge_rec.undo_rec == &trx_purge_dummy_rec) {
+		} else if (purge_rec.undo_rec
+			   == reinterpret_cast<trx_undo_rec_t*>(-1)) {
 			continue;
 		}
 

storage/innobase/trx/trx0undo.cc

@@ -307,8 +307,10 @@ inline void UndorecApplier::assign_rec(const buf_block_t &block,
   this->undo_rec= trx_undo_rec_copy(block.page.frame + offset, heap);
 }
 
-void UndorecApplier::apply_undo_rec()
+inline void UndorecApplier::apply_undo_rec()
 {
+  if (!undo_rec)
+    return;
   bool updated_extern= false;
   undo_no_t undo_no= 0;
   table_id_t table_id= 0;

tpool/tpool_generic.cc

@@ -327,21 +327,22 @@ public:
     int m_period;
     std::mutex m_mtx;
     bool m_on;
-    std::atomic<bool> m_running;
+    std::atomic<int> m_running;
 
     void run()
     {
       /*
         In rare cases, multiple callbacks can be scheduled,
-        e.g with set_time(0,0) in a loop.
-        We do not allow parallel execution, as user is not prepared.
+        at the same time,. e.g with set_time(0,0) in a loop.
+        We do not allow parallel execution, since it is against the expectations.
       */
-      bool expected = false;
-      if (!m_running.compare_exchange_strong(expected, true))
+      if (m_running.fetch_add(1, std::memory_order_acquire) > 0)
         return;
-
-      m_callback(m_data);
-      m_running = false;
+      do
+      {
+        m_callback(m_data);
+      }
+      while (m_running.fetch_sub(1, std::memory_order_release) != 1);
 
       if (m_pool && m_period)
       {