MDEV-36613 Incorrect undo logging for indexes on virtual columns

Starting with mysql/mysql-server@02f8eaa9988dadb73dd68630dd82393cfa19bfb8
and commit 2e814d4702 the index ID of
indexes on virtual columns was being encoded insufficiently in
InnoDB undo log records.  Only the least significant 32 bits were
being written.  This could lead to some corruption of the affected
indexes on ROLLBACK, as well as to missed chances to remove some
history from such indexes when purging the history of committed
transactions that included DELETE or an UPDATE in the indexes.

dict_hdr_create(): In debug instrumented builds, initialize the
DICT_HDR_INDEX_ID close to the 32-bit barrier, instead of initializing
it to DICT_HDR_FIRST_ID (10).  This will allow the changed code to
be exercised while running ./mtr --suite=gcol,vcol.

trx_undo_log_v_idx(): Encode large index->id in a similar way as
mysql/mysql-server@e00328b4d068c7485ac2ffe27207ed1f462c718d
but using a different implementation.

trx_undo_read_v_idx_low(): Decode large index->id in a similar way
as mach_u64_read_much_compressed().

Reviewed by: Debarun Banerjee

mysql-test/suite/gcol/r/innodb_virtual_basic.result

@@ -86,6 +86,8 @@ delete from t where a =13;
 DROP INDEX idx1 ON t;
 DROP INDEX idx2 ON t;
 DROP TABLE t;
+# restart
+set default_storage_engine=innodb;
 /* Test large BLOB data */
 CREATE TABLE `t` (
 `a` BLOB,

mysql-test/suite/gcol/t/innodb_virtual_basic.test

@@ -1,6 +1,6 @@
 --source include/have_innodb.inc
 --source include/have_partition.inc
---source include/big_test.inc
+--source include/not_embedded.inc
 
 call mtr.add_suppression("\\[Warning\\] InnoDB: Compute virtual");
 
@@ -66,6 +66,41 @@ DROP INDEX idx1 ON t;
 DROP INDEX idx2 ON t;
 DROP TABLE t;
 
+let MYSQLD_DATADIR=`select @@datadir`;
+let PAGE_SIZE=`select @@innodb_page_size`;
+--source include/shutdown_mysqld.inc
+perl;
+do "$ENV{MTR_SUITE_DIR}/../innodb/include/crc32.pl";
+my $file = "$ENV{MYSQLD_DATADIR}/ibdata1";
+open(FILE, "+<$file") || die "Unable to open $file";
+binmode FILE;
+my $ps= $ENV{PAGE_SIZE};
+my $page;
+die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps;
+my $full_crc32 = unpack("N",substr($page,54,4)) & 0x10; # FIL_SPACE_FLAGS
+sysseek(FILE, 7*$ps, 0) || die "Unable to seek $file\n";
+die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps;
+substr($page,54,4)=pack("N",0xc001cafe); # 32 MSB of 64-bit DICT_HDR_INDEX_ID
+my $polynomial = 0x82f63b78; # CRC-32C
+if ($full_crc32)
+{
+    my $ck = mycrc32(substr($page, 0, $ps-4), 0, $polynomial);
+    substr($page, $ps-4, 4) = pack("N", $ck);
+}
+else
+{
+    my $ck= pack("N",mycrc32(substr($page, 4, 22), 0, $polynomial) ^
+		 mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial));
+    substr($page,0,4)=$ck;
+    substr($page,$ps-8,4)=$ck;
+}
+sysseek(FILE, 7*$ps, 0) || die "Unable to rewind $file\n";
+syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n";
+close(FILE) || die "Unable to close $file";
+EOF
+--source include/start_mysqld.inc
+set default_storage_engine=innodb;
+
 /* Test large BLOB data */
 CREATE TABLE `t` (
   `a` BLOB,

storage/innobase/trx/trx0rec.cc

@@ -148,7 +148,9 @@ trx_undo_log_v_idx(
 	ulint n_idx = 0;
 	for (const auto& v_index : vcol->v_indexes) {
 		n_idx++;
-		/* FIXME: index->id is 64 bits! */
+		if (uint32_t hi= uint32_t(v_index.index->id >> 32)) {
+			size += 1 + mach_get_compressed_size(hi);
+		}
 		size += mach_get_compressed_size(uint32_t(v_index.index->id));
 		size += mach_get_compressed_size(v_index.nth_field);
 	}
@@ -175,10 +177,14 @@ trx_undo_log_v_idx(
 	ptr += mach_write_compressed(ptr, n_idx);
 
 	for (const auto& v_index : vcol->v_indexes) {
-		ptr += mach_write_compressed(
-			/* FIXME: index->id is 64 bits! */
-			ptr, uint32_t(v_index.index->id));
-
+		/* This is compatible with
+		ptr += mach_u64_write_much_compressed(ptr, v_index.index-id)
+		(the added "if" statement is fixing an old regression). */
+		if (uint32_t hi= uint32_t(v_index.index->id >> 32)) {
+			*ptr++ = 0xff;
+			ptr += mach_write_compressed(ptr, hi);
+		}
+		ptr += mach_write_compressed(ptr, uint32_t(v_index.index->id));
 		ptr += mach_write_compressed(ptr, v_index.nth_field);
 	}
 
@@ -217,7 +223,15 @@ trx_undo_read_v_idx_low(
 	dict_index_t*	clust_index = dict_table_get_first_index(table);
 
 	for (ulint i = 0; i < num_idx; i++) {
-		index_id_t	id = mach_read_next_compressed(&ptr);
+		index_id_t	id = 0;
+		/* This is like mach_u64_read_much_compressed(),
+		but advancing ptr to the next field. */
+		if (*ptr == 0xff) {
+			ptr++;
+			id = mach_read_next_compressed(&ptr);
+			id <<= 32;
+		}
+		id |= mach_read_next_compressed(&ptr);
 		ulint		pos = mach_read_next_compressed(&ptr);
 		dict_index_t*	index = dict_table_get_next_index(clust_index);