Mirrors
/
mariadb
mirror of https://github.com/MariaDB/server
3
0
Fork 0
Code Releases Activity
Browse Source

MDEV-8708: InnoDB temp file encryption 

Added encryption support for online alter table where InnoDB temporary
files are used. Added similar support also for tables containing
full text-indexes.

Made sure that table remains encrypted during discard and import
tablespace.
pull/100/head
Jan Lindström 10 years ago
parent
42574427e6
commit
509b836623
22 changed files with 1739 additions and 254 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 136
      mysql-test/suite/encryption/r/innodb_encryption_discard_import.result
  2. 176
      mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result
  3. 8
      mysql-test/suite/encryption/t/innodb_encryption_discard_import.opt
  4. 211
      mysql-test/suite/encryption/t/innodb_encryption_discard_import.test
  5. 8
      mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt
  6. 191
      mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test
  7. 83
      storage/innobase/fil/fil0crypt.cc
  8. 69
      storage/innobase/fil/fil0fil.cc
  9. 2
      storage/innobase/handler/ha_innodb.cc
  10. 14
      storage/innobase/include/fil0crypt.h
  11. 6
      storage/innobase/include/row0ftsort.h
  12. 23
      storage/innobase/include/row0merge.h
  13. 84
      storage/innobase/row/row0ftsort.cc
  14. 353
      storage/innobase/row/row0merge.cc
  15. 76
      storage/xtradb/fil/fil0crypt.cc
  16. 69
      storage/xtradb/fil/fil0fil.cc
  17. 2
      storage/xtradb/handler/ha_innodb.cc
  18. 16
      storage/xtradb/include/fil0crypt.h
  19. 6
      storage/xtradb/include/row0ftsort.h
  20. 23
      storage/xtradb/include/row0merge.h
  21. 84
      storage/xtradb/row/row0ftsort.cc
  22. 353
      storage/xtradb/row/row0merge.cc

136
mysql-test/suite/encryption/r/innodb_encryption_discard_import.result
View File

@ -0,0 +1,136 @@
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
set current_num = current_num + 1;
end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(10000);
commit;
set autocommit=1;
# Wait max 10 min for key encryption threads to encrypt all spaces
# tablespaces should be now encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
FLUSH TABLE t1, t2, t3 FOR EXPORT;
# List before copying files
t1.cfg
t1.frm
t1.ibd
t2.cfg
t2.frm
t2.ibd
t3.cfg
t3.frm
t3.ibd
UNLOCK TABLES;
# Restarting server
# Done restarting server
# List before t1 DISCARD
t1.frm
t1.ibd
t2.frm
t2.ibd
t3.frm
t3.ibd
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
# Discarded tablespaces should be encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# List after t1 DISCARD
t1.frm
t2.frm
t3.frm
# Tablespaces should be still encrypted
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
ALTER TABLE t1 IMPORT TABLESPACE;
ALTER TABLE t2 IMPORT TABLESPACE;
ALTER TABLE t3 IMPORT TABLESPACE;
# tablespaces should remain encrypted after import
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
ALTER TABLE t1 ENGINE InnoDB;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
ALTER TABLE t2 ENGINE InnoDB;
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
ALTER TABLE t3 ENGINE InnoDB;
SHOW CREATE TABLE t3;
Table Create Table
t3 CREATE TABLE `t3` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes
# Wait max 10 min for key encryption threads to encrypt all spaces
# Tablespaces should be encrypted after alter table
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# Restarting server
# Done restarting server
# Verify that tables are still usable
SELECT COUNT(1) FROM t1;
COUNT(1)
10000
SELECT COUNT(1) FROM t2;
COUNT(1)
10000
SELECT COUNT(1) FROM t3;
COUNT(1)
10000
# Tablespaces should be encrypted after restart
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3;

176
mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result
View File

@ -0,0 +1,176 @@
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t4 (id INT, a VARCHAR(255)) engine=InnoDB;
CREATE TABLE t5 (id INT NOT NULL PRIMARY KEY, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t6 (id INT, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB;
CREATE TABLE t7 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
insert into t4 values (current_num,repeat('repeat',42));
insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22));
insert into t6 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('mangled old',22));
insert into t7 values (current_num,repeat('mysql',42));
set current_num = current_num + 1;
end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
commit;
set autocommit=1;
# Wait max 10 min for key encryption threads to encrypt all spaces
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
ALTER TABLE t1 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t2 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t3 ADD COLUMN c int default 5;
ALTER TABLE t4 ADD COLUMN c int default 5;
ALTER TABLE t3 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t4 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t5 ADD FULLTEXT(a);
ALTER TABLE t6 ADD FULLTEXT(a);
ALTER TABLE t7 ADD KEY a(a), ADD key b(b);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t3;
Table Create Table
t3 CREATE TABLE `t3` (
`id` int(11) DEFAULT NULL,
`a` varchar(255) DEFAULT NULL,
`c` int(11) DEFAULT '5',
KEY `a` (`a`),
KEY `c` (`c`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t4;
Table Create Table
t4 CREATE TABLE `t4` (
`id` int(11) DEFAULT NULL,
`a` varchar(255) DEFAULT NULL,
`c` int(11) DEFAULT '5',
KEY `a` (`a`),
KEY `c` (`c`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t5;
Table Create Table
t5 CREATE TABLE `t5` (
`id` int(11) NOT NULL,
`a` text,
`b` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
FULLTEXT KEY `b` (`b`),
FULLTEXT KEY `a` (`a`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes
SHOW CREATE TABLE t6;
Table Create Table
t6 CREATE TABLE `t6` (
`id` int(11) DEFAULT NULL,
`a` text,
`b` varchar(255) DEFAULT NULL,
FULLTEXT KEY `b` (`b`),
FULLTEXT KEY `a` (`a`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
SHOW CREATE TABLE t7;
Table Create Table
t7 CREATE TABLE `t7` (
`id` int(11) NOT NULL,
`a` varchar(255) DEFAULT NULL,
`b` int(11) DEFAULT '2',
PRIMARY KEY (`id`),
KEY `a` (`a`),
KEY `b` (`b`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
# Restarting server
# Done restarting server
select count(1) from t1;
count(1)
15000
select count(1) from t2;
count(1)
15000
select count(1) from t3;
count(1)
15000
select count(1) from t4;
count(1)
15000
select count(1) from t5;
count(1)
15000
select count(1) from t6;
count(1)
15000
select count(1) from t7;
count(1)
15000
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 ... on expecting NOT FOUND
NOT FOUND /temp/ in t2.ibd
# t3 ... on expecting NOT FOUND
NOT FOUND /barfoo/ in t3.ibd
# t4 ... on expecting NOT FOUND
NOT FOUND /repeat/ in t4.ibd
# t5 ... on expecting NOT FOUND
NOT FOUND /author/ in t5.ibd
# t6 ... on expecting NOT FOUND
NOT FOUND /mangled/ in t6.ibd
# t7 ... on expecting NOT FOUND
NOT FOUND /mysql/ in t7.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;

8
mysql-test/suite/encryption/t/innodb_encryption_discard_import.opt
View File

@ -0,0 +1,8 @@
--innodb-encrypt-tables=ON
--innodb-encrypt-log=ON
--innodb-encryption-rotate-key-age=15
--innodb-encryption-threads=4
--innodb-tablespaces-encryption
--innodb-max-dirty-pages-pct=0.001

211
mysql-test/suite/encryption/t/innodb_encryption_discard_import.test
View File

@ -0,0 +1,211 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/not_valgrind.inc
-- source include/not_embedded.inc
-- source include/not_windows.inc
--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
--let $MYSQLD_DATADIR = `SELECT @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t1_IBD_1 = $MYSQLD_TMPDIR/t1.ibd
--let t2_IBD_1 = $MYSQLD_TMPDIR/t2.ibd
--let t3_IBD_1 = $MYSQLD_TMPDIR/t3.ibd
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(10000);
commit;
set autocommit=1;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--echo # tablespaces should be now encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
FLUSH TABLE t1, t2, t3 FOR EXPORT;
--echo # List before copying files
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg
--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd
--copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2.cfg
--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2.ibd
--copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3.cfg
--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3.ibd
UNLOCK TABLES;
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
--echo # List before t1 DISCARD
--list_files $MYSQLD_DATADIR/test
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
--sleep 5
--echo # Discarded tablespaces should be encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD_1
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD_1
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD_1
-- source include/search_pattern_in_file.inc
--echo # List after t1 DISCARD
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg
--copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd
--copy_file $MYSQLD_TMPDIR/t2.cfg $MYSQLD_DATADIR/test/t2.cfg
--copy_file $MYSQLD_TMPDIR/t2.ibd $MYSQLD_DATADIR/test/t2.ibd
--copy_file $MYSQLD_TMPDIR/t3.cfg $MYSQLD_DATADIR/test/t3.cfg
--copy_file $MYSQLD_TMPDIR/t3.ibd $MYSQLD_DATADIR/test/t3.ibd
--sleep 5
--echo # Tablespaces should be still encrypted
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 IMPORT TABLESPACE;
ALTER TABLE t2 IMPORT TABLESPACE;
ALTER TABLE t3 IMPORT TABLESPACE;
--sleep 5
--echo # tablespaces should remain encrypted after import
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 ENGINE InnoDB;
SHOW CREATE TABLE t1;
ALTER TABLE t2 ENGINE InnoDB;
SHOW CREATE TABLE t2;
ALTER TABLE t3 ENGINE InnoDB;
SHOW CREATE TABLE t3;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--echo # Tablespaces should be encrypted after alter table
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
--echo # Verify that tables are still usable
SELECT COUNT(1) FROM t1;
SELECT COUNT(1) FROM t2;
SELECT COUNT(1) FROM t3;
--sleep 5
--echo # Tablespaces should be encrypted after restart
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--echo # t3 ... on expecting NOT FOUND
--let SEARCH_PATTERN=barfoo
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log

8
mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt
View File

@ -0,0 +1,8 @@
--innodb-encrypt-tables=ON
--innodb-encrypt-log=ON
--innodb-encryption-rotate-key-age=15
--innodb-encryption-threads=4
--innodb-tablespaces-encryption
--innodb-max-dirty-pages-pct=0.001

191
mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test
View File

@ -0,0 +1,191 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
-- source include/not_valgrind.inc
-- source include/not_embedded.inc
-- source include/not_windows.inc
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
--let $MYSQLD_DATADIR=`select @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd
--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd
--let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd
--let t7_IBD = $MYSQLD_DATADIR/test/t7.ibd
CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB;
CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t4 (id INT, a VARCHAR(255)) engine=InnoDB;
CREATE TABLE t5 (id INT NOT NULL PRIMARY KEY, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB encrypted=yes;
CREATE TABLE t6 (id INT, a TEXT(500), b VARCHAR(255), FULLTEXT(b)) ENGINE=InnoDB;
CREATE TABLE t7 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values (current_num,repeat('foobar',42));
insert into t2 values (current_num,repeat('temp', 42));
insert into t3 values (current_num,repeat('barfoo',42));
insert into t4 values (current_num,repeat('repeat',42));
insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22));
insert into t6 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('mangled old',22));
insert into t7 values (current_num,repeat('mysql',42));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(15000);
commit;
set autocommit=1;
--echo # Wait max 10 min for key encryption threads to encrypt all spaces
--let $wait_timeout= 600
--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
--source include/wait_condition.inc
--sleep 5
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 ADD COLUMN b int default 2;
ALTER TABLE t2 ADD COLUMN b int default 2;
ALTER TABLE t7 ADD COLUMN b int default 2;
ALTER TABLE t1 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t2 ADD KEY a(a), ADD KEY b(b);
ALTER TABLE t3 ADD COLUMN c int default 5;
ALTER TABLE t4 ADD COLUMN c int default 5;
ALTER TABLE t3 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t4 ADD KEY (a), ADD KEY c(c);
ALTER TABLE t5 ADD FULLTEXT(a);
ALTER TABLE t6 ADD FULLTEXT(a);
ALTER TABLE t7 ADD KEY a(a), ADD key b(b);
SHOW CREATE TABLE t1;
SHOW CREATE TABLE t2;
SHOW CREATE TABLE t3;
SHOW CREATE TABLE t4;
SHOW CREATE TABLE t5;
SHOW CREATE TABLE t6;
SHOW CREATE TABLE t7;
--sleep 5
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
--echo # Restarting server
-- source include/restart_mysqld.inc
--echo # Done restarting server
select count(1) from t1;
select count(1) from t2;
select count(1) from t3;
select count(1) from t4;
select count(1) from t5;
select count(1) from t6;
select count(1) from t7;
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=temp
--echo # t2 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t3 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=repeat
--echo # t4 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=author
--echo # t5 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t5_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mangled
--echo # t6 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t6_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t7 ... on expecting NOT FOUND
-- let SEARCH_FILE=$t7_IBD
-- source include/search_pattern_in_file.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1, t2, t3, t4, t5, t6, t7;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log

83
storage/innobase/fil/fil0crypt.cc
View File

@ -244,9 +244,6 @@ fil_space_merge_crypt_data(
ut_a(dst->type == CRYPT_SCHEME_UNENCRYPTED ||
dst->type == CRYPT_SCHEME_1);
/* no support for changing iv (yet?) */
ut_a(memcmp(src->iv, dst->iv, sizeof(src->iv)) == 0);
dst->encryption = src->encryption;
dst->type = src->type;
dst->min_key_version = src->min_key_version;
@ -287,7 +284,7 @@ fil_space_read_crypt_data(
page[offset + 4],
page[offset + 5]);
#endif
/* Create data is not stored. */
/* Crypt data is not stored. */
return NULL;
}
@ -370,7 +367,7 @@ fil_space_destroy_crypt_data(
mutex_enter(&(*crypt_data)->mutex);
(*crypt_data)->inited = false;
mutex_exit(&(*crypt_data)->mutex);
mutex_free(&(*crypt_data)->mutex);
mutex_free(& (*crypt_data)->mutex);
memset(*crypt_data, 0, sizeof(fil_space_crypt_t));
free(*crypt_data);
(*crypt_data) = NULL;
@ -555,42 +552,22 @@ fil_space_clear_crypt_data(
}
/******************************************************************
Encrypt a page */
Encrypt a buffer */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
row format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
uint key_version;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
key_version = fil_crypt_get_latest_key_version(crypt_data);
uint key_version = fil_crypt_get_latest_key_version(crypt_data);
if (key_version == ENCRYPTION_KEY_VERSION_INVALID) {
ib_logf(IB_LOG_LEVEL_FATAL,
@ -599,6 +576,7 @@ fil_space_encrypt(
ut_error;
}
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint header_len = FIL_PAGE_DATA;
@ -661,6 +639,45 @@ fil_space_encrypt(
return dst_frame;
}
/******************************************************************
Encrypt a page */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame);
return tmp;
}
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read
@return true if fil space has encryption data. */
@ -1075,7 +1092,7 @@ fil_crypt_start_encrypting_space(
do
{
if (fil_crypt_is_closing(space) ||
fil_space_found_by_id(space)) {
fil_space_found_by_id(space) == NULL) {
break;
}
@ -2326,11 +2343,11 @@ fil_crypt_threads_init()
fil_crypt_threads_event = os_event_create();
mutex_create(fil_crypt_threads_mutex_key,
&fil_crypt_threads_mutex, SYNC_NO_ORDER_CHECK);
fil_crypt_threads_inited = true;
uint cnt = srv_n_fil_crypt_threads;
srv_n_fil_crypt_threads = 0;
fil_crypt_set_thread_cnt(cnt);
fil_crypt_threads_inited = true;
}
/*********************************************************************

69
storage/innobase/fil/fil0fil.cc
View File

@ -6418,8 +6418,16 @@ fil_iterate(
ut_ad(!(n_bytes % iter.page_size));
byte* readptr = io_buffer;
if (iter.crypt_data != NULL) {
byte* writeptr = io_buffer;
bool encrypted = false;
/* Use additional crypt io buffer if tablespace is encrypted */
if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
encrypted = true;
readptr = iter.crypt_io_buffer;
writeptr = iter.crypt_io_buffer;
}
if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) {
@ -6432,12 +6440,15 @@ fil_iterate(
bool updated = false;
os_offset_t page_off = offset;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
bool decrypted = false;
for (ulint i = 0; i < n_pages_read; ++i) {
ulint size = iter.page_size;
if (iter.crypt_data != NULL) {
ulint size = iter.page_size;
bool decrypted = fil_space_decrypt(
/* If tablespace is encrypted, we need to decrypt
the page. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
io_buffer + i * size, //dst
iter.page_size,
@ -6468,6 +6479,32 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
/* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the
buffer that is in buffer pool. */
if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size);
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN);
byte* tmp = fil_encrypt_buf(
iter.crypt_data,
space,
offset,
lsn,
src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst);
if (tmp == src) {
/* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size);
}
}
page_off += iter.page_size;
block->frame += iter.page_size;
}
@ -6475,7 +6512,7 @@ fil_iterate(
/* A page was updated in the set, write back to disk. */
if (updated
&& !os_file_write(
iter.filepath, iter.file, io_buffer,
iter.filepath, iter.file, writeptr,
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
@ -6637,28 +6674,6 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
/* clear crypt data from page 0 and write it back */
os_file_read(file, page, 0, UNIV_PAGE_SIZE);
fil_space_clear_crypt_data(page, crypt_data_offset);
lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
if (callback.get_zip_size() == 0) {
buf_flush_init_for_writing(
page, 0, lsn);
} else {
buf_flush_update_zip_checksum(
page, callback.get_zip_size(), lsn);
}
if (!os_file_write(
iter.filepath, iter.file, page,
0, iter.page_size)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"os_file_write() failed");
return(DB_IO_ERROR);
}
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);

2
storage/innobase/handler/ha_innodb.cc
View File

@ -11976,6 +11976,8 @@ ha_innobase::discard_or_import_tablespace(
| HA_STATUS_CONST
| HA_STATUS_VARIABLE
| HA_STATUS_AUTO);
fil_crypt_set_encrypt_tables(srv_encrypt_tables);
}
}

14
storage/innobase/include/fil0crypt.h
View File

@ -381,6 +381,20 @@ fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
/******************************************************************
Encrypt a buffer */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
/******************************************************************
Calculate post encryption checksum

6
storage/innobase/include/row0ftsort.h
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2012, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -71,6 +72,7 @@ struct fts_psort_common_t {
store Doc ID during sort, if
Doc ID will not be big enough
to use 8 bytes value */
fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */
};
struct fts_psort_t {
@ -83,6 +85,10 @@ struct fts_psort_t {
/*!< buffer to write to file */
row_merge_block_t* block_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
row_merge_block_t* crypt_block[FTS_NUM_AUX_INDEX];
/*!< buffer to crypt data */
row_merge_block_t* crypt_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
ulint child_status; /*!< child thread status */
ulint state; /*!< parent thread state */
fts_doc_list_t fts_doc_list; /*!< doc list to process */

23
storage/innobase/include/row0merge.h
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -351,7 +352,11 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf); /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Empty a sort buffer.
@return sort buffer */
@ -386,7 +391,10 @@ row_merge_sort(
int* tmpfd, /*!< in/out: temporary file handle */
const bool update_progress, /*!< in: update progress status variable or not */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull));
/*********************************************************************//**
Allocate a sort buffer.
@ -424,7 +432,11 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf); /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Read a merge record.
@return pointer to next record, or NULL on I/O error or end of list */
@ -441,6 +453,9 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull, warn_unused_result));
#endif /* row0merge.h */

84
storage/innobase/row/row0ftsort.cc
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -37,7 +38,8 @@ Created 10/13/2010 Jimmy Yang
do { \
b[N] = row_merge_read_rec( \
block[N], buf[N], b[N], index, \
fd[N], &foffs[N], &mrec[N], offsets[N]); \
fd[N], &foffs[N], &mrec[N], offsets[N], \
crypt_data, crypt_block[N], space); \
if (UNIV_UNLIKELY(!b[N])) { \
if (mrec[N]) { \
goto exit; \
@ -191,6 +193,8 @@ row_fts_psort_info_init(
fts_psort_t* merge_info = NULL;
ulint block_size;
ibool ret = TRUE;
fil_space_crypt_t* crypt_data = NULL;
bool encrypted = false;
block_size = 3 * srv_sort_buf_size;
@ -219,6 +223,19 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
common_info->crypt_data = crypt_data;
encrypted = true;
} else {
/* Not needed */
common_info->crypt_data = NULL;
crypt_data = NULL;
}
/* There will be FTS_NUM_AUX_INDEX number of "sort buckets" for
each parallel sort thread. Each "sort bucket" holds records for
@ -256,6 +273,29 @@ row_fts_psort_info_init(
ut_align(
psort_info[j].block_alloc[i], 1024));
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if (encrypted) {
/* Need to align memory for O_DIRECT write */
psort_info[j].crypt_alloc[i] =
static_cast<row_merge_block_t*>(ut_malloc(
block_size + 1024));
psort_info[j].crypt_block[i] =
static_cast<row_merge_block_t*>(
ut_align(
psort_info[j].crypt_alloc[i], 1024));
if (!psort_info[j].crypt_block[i]) {
ret = FALSE;
goto func_exit;
}
} else {
psort_info[j].crypt_alloc[i] = NULL;
psort_info[j].crypt_block[i] = NULL;
}
if (!psort_info[j].merge_block[i]) {
ret = FALSE;
goto func_exit;
@ -313,6 +353,11 @@ row_fts_psort_info_destroy(
if (psort_info[j].block_alloc[i]) {
ut_free(psort_info[j].block_alloc[i]);
}
if (psort_info[j].crypt_alloc[i]) {
ut_free(psort_info[j].crypt_alloc[i]);
}
mem_free(psort_info[j].merge_file[i]);
}
@ -595,6 +640,7 @@ fts_parallel_tokenization(
ibool processed = FALSE;
merge_file_t** merge_file;
row_merge_block_t** block;
row_merge_block_t** crypt_block;
int tmpfd[FTS_NUM_AUX_INDEX];
ulint mycount[FTS_NUM_AUX_INDEX];
ib_uint64_t total_rec = 0;
@ -609,6 +655,7 @@ fts_parallel_tokenization(
fts_tokenize_ctx_t t_ctx;
ulint retried = 0;
dberr_t error = DB_SUCCESS;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(psort_info);
@ -630,6 +677,8 @@ fts_parallel_tokenization(
? DATA_VARCHAR : DATA_VARMYSQL;
block = psort_info->merge_block;
crypt_block = psort_info->crypt_block;
crypt_data = psort_info->psort_common->crypt_data;
zip_size = dict_table_zip_size(table);
row_merge_fts_get_next_doc_item(psort_info, &doc_item);
@ -724,7 +773,10 @@ loop:
if (!row_merge_write(merge_file[t_ctx.buf_used]->fd,
merge_file[t_ctx.buf_used]->offset++,
block[t_ctx.buf_used])) {
block[t_ctx.buf_used],
crypt_data,
crypt_block[t_ctx.buf_used],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
@ -817,13 +869,21 @@ exit:
if (merge_file[i]->offset != 0) {
if (!row_merge_write(merge_file[i]->fd,
merge_file[i]->offset++,
block[i])) {
block[i],
crypt_data,
crypt_block[i],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
UNIV_MEM_INVALID(block[i][0],
srv_sort_buf_size);
if (crypt_block[i]) {
UNIV_MEM_INVALID(crypt_block[i][0],
srv_sort_buf_size);
}
}
buf[i] = row_merge_buf_empty(buf[i]);
@ -848,7 +908,10 @@ exit:
error = row_merge_sort(psort_info->psort_common->trx,
psort_info->psort_common->dup,
merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */);
merge_file[i], block[i], &tmpfd[i],
false, 0.0/* pct_progress */, 0.0/* pct_cost */,
crypt_data, crypt_block[i], table->space);
if (error != DB_SUCCESS) {
close(tmpfd[i]);
goto func_exit;
@ -1352,6 +1415,7 @@ row_fts_merge_insert(
mrec_buf_t** buf;
int* fd;
byte** block;
byte** crypt_block;
const mrec_t** mrec;
ulint count = 0;
int* sel_tree;
@ -1359,6 +1423,8 @@ row_fts_merge_insert(
ulint start;
fts_psort_insert_t ins_ctx;
ulint count_diag = 0;
fil_space_crypt_t* crypt_data = NULL;
ulint space;
ut_ad(index);
ut_ad(table);
@ -1371,6 +1437,7 @@ row_fts_merge_insert(
ins_ctx.trx->op_info = "inserting index entries";
ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size;
crypt_data = psort_info[0].psort_common->crypt_data;
heap = mem_heap_create(500 + sizeof(mrec_buf_t));
@ -1385,6 +1452,8 @@ row_fts_merge_insert(
fd = (int*) mem_heap_alloc(heap, sizeof(*fd) * fts_sort_pll_degree);
block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
crypt_block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
mrec = (const mrec_t**) mem_heap_alloc(
heap, sizeof(*mrec) * fts_sort_pll_degree);
sel_tree = (int*) mem_heap_alloc(
@ -1405,6 +1474,7 @@ row_fts_merge_insert(
offsets[i][0] = num;
offsets[i][1] = dict_index_get_n_fields(index);
block[i] = psort_info[i].merge_block[id];
crypt_block[i] = psort_info[i].crypt_block[id];
b[i] = psort_info[i].merge_block[id];
fd[i] = psort_info[i].merge_file[id]->fd;
foffs[i] = 0;
@ -1447,6 +1517,7 @@ row_fts_merge_insert(
ins_ctx.fts_table.table_id = table->id;
ins_ctx.fts_table.parent = index->table->name;
ins_ctx.fts_table.table = index->table;
space = table->space;
for (i = 0; i < fts_sort_pll_degree; i++) {
if (psort_info[i].merge_file[id]->n_rec == 0) {
@ -1459,7 +1530,10 @@ row_fts_merge_insert(
if (psort_info[i].merge_file[id]->offset > 0
&& (!row_merge_read(
fd[i], foffs[i],
(row_merge_block_t*) block[i]))) {
(row_merge_block_t*) block[i],
crypt_data,
(row_merge_block_t*) crypt_block[i],
space))) {
error = DB_CORRUPTION;
goto exit;
}

353
storage/innobase/row/row0merge.cc
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -41,6 +42,7 @@ Completed by Sunny Bains and Marko Makela
#include "handler0alter.h"
#include "ha_prototypes.h"
#include "math.h" /* log() */
#include "fil0crypt.h"
float my_log2f(float n)
{
@ -76,6 +78,92 @@ UNIV_INTERN char srv_disable_sort_file_cache;
/* Maximum pending doc memory limit in bytes for a fts tokenization thread */
#define FTS_PENDING_DOC_MEMORY_LIMIT 1000000
/* Reserve free space from every block for key_version */
#define ROW_MERGE_RESERVE_SIZE 4
/******************************************************//**
Encrypt a merge block. */
static
void
row_merge_encrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
key_version = encryption_key_get_latest_version(crypt_data->key_id);
/* Store key_version at the begining of the input buffer */
mach_write_to_4((byte *)crypted_buf, key_version);
int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
}
/******************************************************//**
Decrypt a merge block. */
static
bool
row_merge_decrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
/* Read key_version from begining of the buffer */
key_version = mach_read_from_4((byte *)input_buf);
if (key_version == 0) {
/* block not encrypted */
return false;
}
int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
return true;
}
#ifdef UNIV_DEBUG
/******************************************************//**
Display a merge tuple. */
@ -193,7 +281,7 @@ row_merge_buf_create(
ulint buf_size;
mem_heap_t* heap;
max_tuples = srv_sort_buf_size
max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)
/ ut_max(1, dict_index_get_min_size(index));
buf_size = (sizeof *buf);
@ -608,8 +696,8 @@ row_merge_buf_add(
ut_ad(data_size < srv_sort_buf_size);
/* Reserve one byte for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= srv_sort_buf_size - 1) {
/* Reserve bytes for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) {
DBUG_RETURN(0);
}
@ -781,7 +869,7 @@ row_merge_buf_write(
{
const dict_index_t* index = buf->index;
ulint n_fields= dict_index_get_n_fields(index);
byte* b = &block[0];
byte* b = &block[ROW_MERGE_RESERVE_SIZE];
for (ulint i = 0; i < buf->n_tuples; i++) {
const mtuple_t* entry = &buf->tuples[i];
@ -800,7 +888,7 @@ row_merge_buf_write(
/* Write an "end-of-chunk" marker. */
ut_a(b < &block[srv_sort_buf_size]);
ut_a(b == &block[0] + buf->total_size);
ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE);
*b++ = 0;
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
@ -857,7 +945,10 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf) /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size;
ibool success;
@ -881,6 +972,13 @@ row_merge_read(
success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf,
ofs, srv_sort_buf_size);
/* For encrypted tables, decrypt data after reading and copy data */
if (crypt_data && crypt_buf) {
if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) {
memcpy(buf, crypt_buf, srv_sort_buf_size);
}
}
#ifdef POSIX_FADV_DONTNEED
/* Each block is read exactly once. Free up the file cache. */
posix_fadvise(fd, ofs, srv_sort_buf_size, POSIX_FADV_DONTNEED);
@ -903,18 +1001,32 @@ UNIV_INTERN
ibool
row_merge_write(
/*============*/
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf) /*!< in: data */
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
size_t buf_len = srv_sort_buf_size;
os_offset_t ofs = buf_len * (os_offset_t) offset;
ibool ret;
void* out_buf = (void *)buf;
DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE););
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), buf, ofs, buf_len);
/* For encrypted tables, encrypt data before writing */
if (crypt_data && crypt_buf) {
row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf);
out_buf = crypt_buf;
} else {
/* Mark block unencrypted */
mach_write_to_4((byte *)out_buf, 0);
}
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), out_buf, ofs, buf_len);
#ifdef UNIV_DEBUG
if (row_merge_print_block_write) {
@ -948,7 +1060,10 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint data_size;
@ -966,6 +1081,10 @@ row_merge_read_rec(
ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE
+ dict_index_get_n_fields(index));
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
extra_size = *b++;
if (UNIV_UNLIKELY(!extra_size)) {
@ -985,7 +1104,8 @@ row_merge_read_rec(
/* Read another byte of extra_size. */
if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) {
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
err_exit:
/* Signal I/O error. */
*mrec = b;
@ -993,7 +1113,7 @@ err_exit:
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
}
extra_size = (extra_size & 0x7f) << 8;
@ -1014,13 +1134,14 @@ err_exit:
ut_ad(avail_size < sizeof *buf);
memcpy(*buf, b, avail_size);
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the record. */
memcpy(*buf + avail_size, b, extra_size - avail_size);
@ -1075,13 +1196,14 @@ err_exit:
offsets[3] = (ulint) index;
#endif /* UNIV_DEBUG */
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the rest of the record. */
memcpy(*buf + avail_size, b, extra_size + data_size - avail_size);
@ -1157,7 +1279,10 @@ row_merge_write_rec(
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
const mrec_t* mrec, /*!< in: record to write */
const ulint* offsets)/*!< in: offsets of mrec */
const ulint* offsets,/*!< in: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint size;
@ -1178,6 +1303,10 @@ row_merge_write_rec(
size = extra_size + (extra_size >= 0x80)
+ rec_offs_data_size(offsets);
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) {
/* The record spans two blocks.
Copy it to the temporary buffer first. */
@ -1192,14 +1321,15 @@ row_merge_write_rec(
record to the head of the new block. */
memcpy(b, buf[0], avail_size);
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
/* Copy the rest. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
memcpy(b, buf[0] + avail_size, size - avail_size);
b += size - avail_size;
} else {
@ -1218,10 +1348,13 @@ static
byte*
row_merge_write_eof(
/*================*/
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs) /*!< in/out: file offset */
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ut_ad(block);
ut_ad(b >= &block[0]);
@ -1234,20 +1367,27 @@ row_merge_write_eof(
}
#endif /* UNIV_DEBUG */
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
*b++ = 0;
UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]);
UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size);
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
to avoid bogus warnings. */
memset(b, 0xff, &block[srv_sort_buf_size] - b);
#endif /* UNIV_DEBUG_VALGRIND */
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
return(&block[0]);
}
@ -1292,7 +1432,11 @@ row_merge_read_clustered_index(
ULINT_UNDEFINED if none is added */
ib_sequence_t& sequence,/*!< in/out: autoinc sequence */
row_merge_block_t* block, /*!< in/out: file buffer */
float pct_cost) /*!< in: percent of task weight out of total alter job */
float pct_cost, /*!< in: percent of task weight
out of total alter job */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block)/*!< in: in/out: crypted file
buffer */
{
dict_index_t* clust_index; /* Clustered index */
mem_heap_t* row_heap; /* Heap memory to create
@ -1314,9 +1458,10 @@ row_merge_read_clustered_index(
ib_int64_t sig_count = 0;
mem_heap_t* conv_heap = NULL;
float curr_progress;
float curr_progress = 0.0;
ib_int64_t read_rows = 0;
ib_int64_t table_total_rows;
ib_int64_t table_total_rows = 0;
DBUG_ENTER("row_merge_read_clustered_index");
ut_ad((old_table == new_table) == !col_map);
@ -1807,14 +1952,15 @@ write_buffers:
row_merge_buf_write(buf, file, block);
if (!row_merge_write(file->fd, file->offset++,
block)) {
if (!row_merge_write(file->fd, file->offset++, block,
crypt_data, crypt_block, new_table->space)) {
err = DB_TEMP_FILE_WRITE_FAILURE;
trx->error_key_num = i;
break;
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
merge_buf[i] = row_merge_buf_empty(buf);
if (UNIV_LIKELY(row != NULL)) {
@ -1983,14 +2129,21 @@ wait_again:
b2 = row_merge_write_rec(&block[2 * srv_sort_buf_size], \
&buf[2], b2, \
of->fd, &of->offset, \
mrec##N, offsets##N); \
mrec##N, offsets##N, \
crypt_data, \
&crypt_block[2 * srv_sort_buf_size], \
space); \
if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \
goto corrupt; \
} \
b##N = row_merge_read_rec(&block[N * srv_sort_buf_size],\
&buf[N], b##N, INDEX, \
file->fd, foffs##N, \
&mrec##N, offsets##N); \
&mrec##N, offsets##N, \
crypt_data, \
&crypt_block[N * srv_sort_buf_size], \
space); \
\
if (UNIV_UNLIKELY(!b##N)) { \
if (mrec##N) { \
goto corrupt; \
@ -2015,7 +2168,11 @@ row_merge_blocks(
source list in the file */
ulint* foffs1, /*!< in/out: offset of second
source list in the file */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block,/*!< in: in/out: crypted file
buffer */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2046,8 +2203,10 @@ row_merge_blocks(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size],
crypt_data, &crypt_block[srv_sort_buf_size], space)) {
corrupt:
mem_heap_free(heap);
return(DB_CORRUPTION);
@ -2059,11 +2218,15 @@ corrupt:
b0 = row_merge_read_rec(
&block[0], &buf[0], b0, dup->index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
b1 = row_merge_read_rec(
&block[srv_sort_buf_size],
&buf[srv_sort_buf_size], b1, dup->index,
file->fd, foffs1, &mrec1, offsets1);
file->fd, foffs1, &mrec1, offsets1,
crypt_data, &crypt_block[srv_sort_buf_size], space);
if (UNIV_UNLIKELY(!b0 && mrec0)
|| UNIV_UNLIKELY(!b1 && mrec1)) {
@ -2105,8 +2268,11 @@ done0:
done1:
mem_heap_free(heap);
b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset);
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space);
return(b2 ? DB_SUCCESS : DB_CORRUPTION);
}
@ -2121,7 +2287,10 @@ row_merge_blocks_copy(
const merge_file_t* file, /*!< in: input file */
row_merge_block_t* block, /*!< in/out: 3 buffers */
ulint* foffs0, /*!< in/out: input file offset */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2148,7 +2317,8 @@ row_merge_blocks_copy(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)) {
corrupt:
mem_heap_free(heap);
return(FALSE);
@ -2159,7 +2329,9 @@ corrupt:
b2 = &block[2 * srv_sort_buf_size];
b0 = row_merge_read_rec(&block[0], &buf[0], b0, index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
if (UNIV_UNLIKELY(!b0 && mrec0)) {
goto corrupt;
@ -2178,8 +2350,10 @@ done0:
(*foffs0)++;
mem_heap_free(heap);
return(row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset)
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space)
!= NULL);
}
@ -2199,9 +2373,12 @@ row_merge(
int* tmpfd, /*!< in/out: temporary file handle */
ulint* num_run,/*!< in/out: Number of runs remain
to be merged */
ulint* run_offset) /*!< in/out: Array contains the
ulint* run_offset, /*!< in/out: Array contains the
first offset number for each merge
run */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint foffs0; /*!< first input offset */
ulint foffs1; /*!< second input offset */
@ -2214,6 +2391,10 @@ row_merge(
UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size);
if (crypt_block) {
UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size);
}
ut_ad(ihalf < file->offset);
of.fd = *tmpfd;
@ -2244,7 +2425,8 @@ row_merge(
run_offset[n_run++] = of.offset;
error = row_merge_blocks(dup, file, block,
&foffs0, &foffs1, &of);
&foffs0, &foffs1, &of,
crypt_data, crypt_block, space);
if (error != DB_SUCCESS) {
return(error);
@ -2264,7 +2446,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs0, &of)) {
&foffs0, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2281,7 +2464,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs1, &of)) {
&foffs1, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2336,7 +2520,10 @@ row_merge_sort(
const float pct_progress,
/*!< in: total progress percent
until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const ulint half = file->offset / 2;
ulint num_runs;
@ -2388,7 +2575,8 @@ row_merge_sort(
sql_print_information("InnoDB: Online DDL : merge-sorting current run %lu estimated %lu runs", cur_run, num_runs);
error = row_merge(trx, dup, file, block, tmpfd,
&num_runs, run_offset);
&num_runs, run_offset,
crypt_data, crypt_block, space);
if(update_progress) {
merge_count++;
@ -2471,7 +2659,11 @@ row_merge_insert_index_tuples(
row_merge_block_t* block, /*!< in/out: file buffer */
const ib_int64_t table_total_rows, /*!< in: total rows of old table */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent
*/
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const byte* b;
mem_heap_t* heap;
@ -2502,9 +2694,10 @@ row_merge_insert_index_tuples(
offsets[1] = dict_index_get_n_fields(index);
}
b = block;
b = &block[0];
if (!row_merge_read(fd, foffs, block)) {
if (!row_merge_read(fd, foffs, block,
crypt_data, crypt_block, space)) {
error = DB_CORRUPTION;
} else {
buf = static_cast<mrec_buf_t*>(
@ -2520,7 +2713,8 @@ row_merge_insert_index_tuples(
mtr_t mtr;
b = row_merge_read_rec(block, buf, b, index,
fd, &foffs, &mrec, offsets);
fd, &foffs, &mrec, offsets,
crypt_data, crypt_block, space);
if (UNIV_UNLIKELY(!b)) {
/* End of list, or I/O error */
if (mrec) {
@ -3661,6 +3855,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
ulint block_size;
ulint i;
ulint j;
@ -3671,6 +3866,7 @@ row_merge_build_indexes(
fts_psort_t* merge_info = NULL;
ib_int64_t sig_count = 0;
bool fts_psort_initiated = false;
fil_space_crypt_t * crypt_data = NULL;
float total_static_cost = 0;
float total_dynamic_cost = 0;
@ -3695,6 +3891,27 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
crypt_block = static_cast<row_merge_block_t*>(
os_mem_alloc_large(&block_size));
if (crypt_block == NULL) {
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
} else {
/* Not needed */
crypt_data = NULL;
}
trx_start_if_not_started_xa(trx);
merge_files = static_cast<merge_file_t*>(
@ -3776,10 +3993,11 @@ row_merge_build_indexes(
secondary index entries for merge sort */
error = row_merge_read_clustered_index(
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost);
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost,
crypt_data, crypt_block);
pct_progress += pct_cost;
@ -3799,6 +4017,10 @@ row_merge_build_indexes(
/* Now we have files containing index entries ready for
sorting and inserting. */
DBUG_EXECUTE_IF(
"ib_merge_wait_after_read",
os_thread_sleep(20000000);); /* 20 sec */
for (i = 0; i < n_indexes; i++) {
dict_index_t* sort_idx = indexes[i];
@ -3892,8 +4114,10 @@ wait_again:
buf, (i+1), n_indexes, pct_cost);
error = row_merge_sort(
trx, &dup, &merge_files[i],
block, &tmpfd, true, pct_progress, pct_cost);
trx, &dup, &merge_files[i],
block, &tmpfd, true,
pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
@ -3901,6 +4125,10 @@ wait_again:
" merge-sorting index %s (%lu / %lu)",
buf, (i+1), n_indexes);
DBUG_EXECUTE_IF(
"ib_merge_wait_after_sort",
os_thread_sleep(20000000);); /* 20 sec */
if (error == DB_SUCCESS) {
pct_cost = (COST_BUILD_INDEX_STATIC +
(total_dynamic_cost * merge_files[i].offset /
@ -3916,7 +4144,8 @@ wait_again:
error = row_merge_insert_index_tuples(
trx->id, sort_idx, old_table,
merge_files[i].fd, block,
merge_files[i].n_rec, pct_progress, pct_cost);
merge_files[i].n_rec, pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
sql_print_information("InnoDB: Online DDL : "
@ -3990,6 +4219,10 @@ func_exit:
mem_free(merge_files);
os_mem_free_large(block, block_size);
if (crypt_block) {
os_mem_free_large(crypt_block, block_size);
}
DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID);
if (online && old_table == new_table && error != DB_SUCCESS) {

76
storage/xtradb/fil/fil0crypt.cc
View File

@ -244,9 +244,6 @@ fil_space_merge_crypt_data(
ut_a(dst->type == CRYPT_SCHEME_UNENCRYPTED ||
dst->type == CRYPT_SCHEME_1);
/* no support for changing iv (yet?) */
ut_a(memcmp(src->iv, dst->iv, sizeof(src->iv)) == 0);
dst->encryption = src->encryption;
dst->type = src->type;
dst->min_key_version = src->min_key_version;
@ -555,42 +552,22 @@ fil_space_clear_crypt_data(
}
/******************************************************************
Encrypt a page */
Encrypt a buffer */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
row format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
uint key_version;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
key_version = fil_crypt_get_latest_key_version(crypt_data);
uint key_version = fil_crypt_get_latest_key_version(crypt_data);
if (key_version == ENCRYPTION_KEY_VERSION_INVALID) {
ib_logf(IB_LOG_LEVEL_FATAL,
@ -599,6 +576,7 @@ fil_space_encrypt(
ut_error;
}
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint header_len = FIL_PAGE_DATA;
@ -661,6 +639,45 @@ fil_space_encrypt(
return dst_frame;
}
/******************************************************************
Encrypt a page */
UNIV_INTERN
byte*
fil_space_encrypt(
/*==============*/
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame) /*!< in: outbut buffer */
{
fil_space_crypt_t* crypt_data = NULL;
ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
|| orig_page_type==FIL_PAGE_TYPE_XDES) {
/* File space header or extent descriptor do not need to be
encrypted. */
return src_frame;
}
/* Get crypt data from file space */
crypt_data = fil_space_get_crypt_data(space);
if (crypt_data == NULL) {
return src_frame;
}
ut_ad(crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF);
byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame);
return tmp;
}
/*********************************************************************
Check if extra buffer shall be allocated for decrypting after read
@return true if fil space has encryption data. */
@ -2330,6 +2347,7 @@ fil_crypt_threads_init()
uint cnt = srv_n_fil_crypt_threads;
srv_n_fil_crypt_threads = 0;
fil_crypt_set_thread_cnt(cnt);
fil_crypt_threads_inited = true;
}
/*********************************************************************

69
storage/xtradb/fil/fil0fil.cc
View File

@ -6475,8 +6475,16 @@ fil_iterate(
ut_ad(!(n_bytes % iter.page_size));
byte* readptr = io_buffer;
if (iter.crypt_data != NULL) {
byte* writeptr = io_buffer;
bool encrypted = false;
/* Use additional crypt io buffer if tablespace is encrypted */
if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
encrypted = true;
readptr = iter.crypt_io_buffer;
writeptr = iter.crypt_io_buffer;
}
if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) {
@ -6489,12 +6497,15 @@ fil_iterate(
bool updated = false;
os_offset_t page_off = offset;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
bool decrypted = false;
for (ulint i = 0; i < n_pages_read; ++i) {
ulint size = iter.page_size;
if (iter.crypt_data != NULL) {
ulint size = iter.page_size;
bool decrypted = fil_space_decrypt(
/* If tablespace is encrypted, we need to decrypt
the page. */
if (encrypted) {
decrypted = fil_space_decrypt(
iter.crypt_data,
io_buffer + i * size, //dst
iter.page_size,
@ -6525,6 +6536,32 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
/* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the
buffer that is in buffer pool. */
if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size);
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN);
byte* tmp = fil_encrypt_buf(
iter.crypt_data,
space,
offset,
lsn,
src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst);
if (tmp == src) {
/* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size);
}
}
page_off += iter.page_size;
block->frame += iter.page_size;
}
@ -6532,7 +6569,7 @@ fil_iterate(
/* A page was updated in the set, write back to disk. */
if (updated
&& !os_file_write(
iter.filepath, iter.file, io_buffer,
iter.filepath, iter.file, writeptr,
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
@ -6694,28 +6731,6 @@ fil_tablespace_iterate(
mem_free(io_buffer);
if (iter.crypt_data != NULL) {
/* clear crypt data from page 0 and write it back */
os_file_read(file, page, 0, UNIV_PAGE_SIZE);
fil_space_clear_crypt_data(page, crypt_data_offset);
lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
if (callback.get_zip_size() == 0) {
buf_flush_init_for_writing(
page, 0, lsn);
} else {
buf_flush_update_zip_checksum(
page, callback.get_zip_size(), lsn);
}
if (!os_file_write(
iter.filepath, iter.file, page,
0, iter.page_size)) {
ib_logf(IB_LOG_LEVEL_ERROR,
"os_file_write() failed");
return(DB_IO_ERROR);
}
mem_free(crypt_io_buffer);
iter.crypt_io_buffer = NULL;
fil_space_destroy_crypt_data(&iter.crypt_data);

2
storage/xtradb/handler/ha_innodb.cc
View File

@ -12465,6 +12465,8 @@ ha_innobase::discard_or_import_tablespace(
| HA_STATUS_CONST
| HA_STATUS_VARIABLE
| HA_STATUS_AUTO);
fil_crypt_set_encrypt_tables(srv_encrypt_tables);
}
}

16
storage/xtradb/include/fil0crypt.h
View File

@ -381,6 +381,21 @@ fil_crypt_set_encrypt_tables(
/*=========================*/
uint val); /*!< in: New srv_encrypt_tables setting */
/******************************************************************
Encrypt a buffer */
UNIV_INTERN
byte*
fil_encrypt_buf(
/*============*/
fil_space_crypt_t* crypt_data, /*!< in: crypt data */
ulint space, /*!< in: Space id */
ulint offset, /*!< in: Page offset */
lsn_t lsn, /*!< in: lsn */
byte* src_frame, /*!< in: Source page to be encrypted */
ulint zip_size, /*!< in: compressed size if
row_format compressed */
byte* dst_frame); /*!< in: outbut buffer */
/******************************************************************
Calculate post encryption checksum
@return page checksum or BUF_NO_CHECKSUM_MAGIC
@ -392,6 +407,7 @@ fil_crypt_calculate_checksum(
ulint zip_size, /*!< in: zip_size or 0 */
byte* dst_frame); /*!< in: page where to calculate */
#ifndef UNIV_NONINL
#include "fil0crypt.ic"
#endif

6
storage/xtradb/include/row0ftsort.h
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2012, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -71,6 +72,7 @@ struct fts_psort_common_t {
store Doc ID during sort, if
Doc ID will not be big enough
to use 8 bytes value */
fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */
};
struct fts_psort_t {
@ -83,6 +85,10 @@ struct fts_psort_t {
/*!< buffer to write to file */
row_merge_block_t* block_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
row_merge_block_t* crypt_block[FTS_NUM_AUX_INDEX];
/*!< buffer to crypt data */
row_merge_block_t* crypt_alloc[FTS_NUM_AUX_INDEX];
/*!< buffer to allocated */
ulint child_status; /*!< child thread status */
ulint state; /*!< parent thread state */
fts_doc_list_t fts_doc_list; /*!< doc list to process */

23
storage/xtradb/include/row0merge.h
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -351,7 +352,11 @@ row_merge_write(
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf); /*!< in: data */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Empty a sort buffer.
@return sort buffer */
@ -386,7 +391,10 @@ row_merge_sort(
int* tmpfd, /*!< in/out: temporary file handle */
const bool update_progress, /*!< in: update progress status variable or not */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull));
/*********************************************************************//**
Allocate a sort buffer.
@ -424,7 +432,11 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf); /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space); /*!< in: space id */
/********************************************************************//**
Read a merge record.
@return pointer to next record, or NULL on I/O error or end of list */
@ -441,6 +453,9 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
__attribute__((nonnull, warn_unused_result));
#endif /* row0merge.h */

84
storage/xtradb/row/row0ftsort.cc
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2010, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -38,7 +39,8 @@ Created 10/13/2010 Jimmy Yang
do { \
b[N] = row_merge_read_rec( \
block[N], buf[N], b[N], index, \
fd[N], &foffs[N], &mrec[N], offsets[N]); \
fd[N], &foffs[N], &mrec[N], offsets[N], \
crypt_data, crypt_block[N], space); \
if (UNIV_UNLIKELY(!b[N])) { \
if (mrec[N]) { \
goto exit; \
@ -194,6 +196,8 @@ row_fts_psort_info_init(
fts_psort_t* merge_info = NULL;
ulint block_size;
ibool ret = TRUE;
fil_space_crypt_t* crypt_data = NULL;
bool encrypted = false;
block_size = 3 * srv_sort_buf_size;
@ -222,6 +226,19 @@ row_fts_psort_info_init(
common_info->sort_event = os_event_create();
common_info->merge_event = os_event_create();
common_info->opt_doc_id_size = opt_doc_id_size;
crypt_data = fil_space_get_crypt_data(new_table->space);
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
common_info->crypt_data = crypt_data;
encrypted = true;
} else {
/* Not needed */
common_info->crypt_data = NULL;
crypt_data = NULL;
}
/* There will be FTS_NUM_AUX_INDEX number of "sort buckets" for
each parallel sort thread. Each "sort bucket" holds records for
@ -259,6 +276,29 @@ row_fts_psort_info_init(
ut_align(
psort_info[j].block_alloc[i], 1024));
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if (encrypted) {
/* Need to align memory for O_DIRECT write */
psort_info[j].crypt_alloc[i] =
static_cast<row_merge_block_t*>(ut_malloc(
block_size + 1024));
psort_info[j].crypt_block[i] =
static_cast<row_merge_block_t*>(
ut_align(
psort_info[j].crypt_alloc[i], 1024));
if (!psort_info[j].crypt_block[i]) {
ret = FALSE;
goto func_exit;
}
} else {
psort_info[j].crypt_alloc[i] = NULL;
psort_info[j].crypt_block[i] = NULL;
}
if (!psort_info[j].merge_block[i]) {
ret = FALSE;
goto func_exit;
@ -316,6 +356,11 @@ row_fts_psort_info_destroy(
if (psort_info[j].block_alloc[i]) {
ut_free(psort_info[j].block_alloc[i]);
}
if (psort_info[j].crypt_alloc[i]) {
ut_free(psort_info[j].crypt_alloc[i]);
}
mem_free(psort_info[j].merge_file[i]);
}
@ -598,6 +643,7 @@ fts_parallel_tokenization(
ibool processed = FALSE;
merge_file_t** merge_file;
row_merge_block_t** block;
row_merge_block_t** crypt_block;
int tmpfd[FTS_NUM_AUX_INDEX];
ulint mycount[FTS_NUM_AUX_INDEX];
ib_uint64_t total_rec = 0;
@ -612,6 +658,7 @@ fts_parallel_tokenization(
fts_tokenize_ctx_t t_ctx;
ulint retried = 0;
dberr_t error = DB_SUCCESS;
fil_space_crypt_t* crypt_data = NULL;
ut_ad(psort_info);
@ -633,6 +680,8 @@ fts_parallel_tokenization(
? DATA_VARCHAR : DATA_VARMYSQL;
block = psort_info->merge_block;
crypt_block = psort_info->crypt_block;
crypt_data = psort_info->psort_common->crypt_data;
zip_size = dict_table_zip_size(table);
row_merge_fts_get_next_doc_item(psort_info, &doc_item);
@ -727,7 +776,10 @@ loop:
if (!row_merge_write(merge_file[t_ctx.buf_used]->fd,
merge_file[t_ctx.buf_used]->offset++,
block[t_ctx.buf_used])) {
block[t_ctx.buf_used],
crypt_data,
crypt_block[t_ctx.buf_used],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
@ -820,13 +872,21 @@ exit:
if (merge_file[i]->offset != 0) {
if (!row_merge_write(merge_file[i]->fd,
merge_file[i]->offset++,
block[i])) {
block[i],
crypt_data,
crypt_block[i],
table->space)) {
error = DB_TEMP_FILE_WRITE_FAILURE;
goto func_exit;
}
UNIV_MEM_INVALID(block[i][0],
srv_sort_buf_size);
if (crypt_block[i]) {
UNIV_MEM_INVALID(crypt_block[i][0],
srv_sort_buf_size);
}
}
buf[i] = row_merge_buf_empty(buf[i]);
@ -851,7 +911,10 @@ exit:
error = row_merge_sort(psort_info->psort_common->trx,
psort_info->psort_common->dup,
merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */);
merge_file[i], block[i], &tmpfd[i],
false, 0.0/* pct_progress */, 0.0/* pct_cost */,
crypt_data, crypt_block[i], table->space);
if (error != DB_SUCCESS) {
close(tmpfd[i]);
goto func_exit;
@ -1355,6 +1418,7 @@ row_fts_merge_insert(
mrec_buf_t** buf;
int* fd;
byte** block;
byte** crypt_block;
const mrec_t** mrec;
ulint count = 0;
int* sel_tree;
@ -1362,6 +1426,8 @@ row_fts_merge_insert(
ulint start;
fts_psort_insert_t ins_ctx;
ulint count_diag = 0;
fil_space_crypt_t* crypt_data = NULL;
ulint space;
ut_ad(index);
ut_ad(table);
@ -1374,6 +1440,7 @@ row_fts_merge_insert(
ins_ctx.trx->op_info = "inserting index entries";
ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size;
crypt_data = psort_info[0].psort_common->crypt_data;
heap = mem_heap_create(500 + sizeof(mrec_buf_t));
@ -1388,6 +1455,8 @@ row_fts_merge_insert(
fd = (int*) mem_heap_alloc(heap, sizeof(*fd) * fts_sort_pll_degree);
block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
crypt_block = (byte**) mem_heap_alloc(
heap, sizeof(*block) * fts_sort_pll_degree);
mrec = (const mrec_t**) mem_heap_alloc(
heap, sizeof(*mrec) * fts_sort_pll_degree);
sel_tree = (int*) mem_heap_alloc(
@ -1408,6 +1477,7 @@ row_fts_merge_insert(
offsets[i][0] = num;
offsets[i][1] = dict_index_get_n_fields(index);
block[i] = psort_info[i].merge_block[id];
crypt_block[i] = psort_info[i].crypt_block[id];
b[i] = psort_info[i].merge_block[id];
fd[i] = psort_info[i].merge_file[id]->fd;
foffs[i] = 0;
@ -1450,6 +1520,7 @@ row_fts_merge_insert(
ins_ctx.fts_table.table_id = table->id;
ins_ctx.fts_table.parent = index->table->name;
ins_ctx.fts_table.table = index->table;
space = table->space;
for (i = 0; i < fts_sort_pll_degree; i++) {
if (psort_info[i].merge_file[id]->n_rec == 0) {
@ -1462,7 +1533,10 @@ row_fts_merge_insert(
if (psort_info[i].merge_file[id]->offset > 0
&& (!row_merge_read(
fd[i], foffs[i],
(row_merge_block_t*) block[i]))) {
(row_merge_block_t*) block[i],
crypt_data,
(row_merge_block_t*) crypt_block[i],
space))) {
error = DB_CORRUPTION;
goto exit;
}

353
storage/xtradb/row/row0merge.cc
View File

@ -1,6 +1,7 @@
/*****************************************************************************
Copyright (c) 2005, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2015, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -41,6 +42,7 @@ Completed by Sunny Bains and Marko Makela
#include "handler0alter.h"
#include "ha_prototypes.h"
#include "math.h" /* log2() */
#include "fil0crypt.h"
float my_log2f(float n)
{
@ -76,6 +78,92 @@ UNIV_INTERN char srv_disable_sort_file_cache;
/* Maximum pending doc memory limit in bytes for a fts tokenization thread */
#define FTS_PENDING_DOC_MEMORY_LIMIT 1000000
/* Reserve free space from every block for key_version */
#define ROW_MERGE_RESERVE_SIZE 4
/******************************************************//**
Encrypt a merge block. */
static
void
row_merge_encrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
key_version = encryption_key_get_latest_version(crypt_data->key_id);
/* Store key_version at the begining of the input buffer */
mach_write_to_4((byte *)crypted_buf, key_version);
int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
}
/******************************************************//**
Decrypt a merge block. */
static
bool
row_merge_decrypt_buf(
/*==================*/
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
ulint offset, /*!< in: offset where to
write */
ulint space, /*!< in: tablespace id */
const byte* input_buf, /*!< in: input buffer */
byte* crypted_buf) /*!< out: crypted buffer */
{
uint key_version;
uint dstlen=0;
os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset;
/* Read key_version from begining of the buffer */
key_version = mach_read_from_4((byte *)input_buf);
if (key_version == 0) {
/* block not encrypted */
return false;
}
int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE,
srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE,
crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen,
crypt_data, key_version,
space, ofs, 0);
if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %lu buf: %p buflen: %d."
" return-code: %d. Can't continue!\n",
input_buf, (size_t)srv_sort_buf_size,
crypted_buf, dstlen, rc);
ut_error;
}
return (true);
}
#ifdef UNIV_DEBUG
/******************************************************//**
Display a merge tuple. */
@ -193,7 +281,7 @@ row_merge_buf_create(
ulint buf_size;
mem_heap_t* heap;
max_tuples = srv_sort_buf_size
max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)
/ ut_max(1, dict_index_get_min_size(index));
buf_size = (sizeof *buf);
@ -610,8 +698,8 @@ row_merge_buf_add(
ut_ad(data_size < srv_sort_buf_size);
/* Reserve one byte for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= srv_sort_buf_size - 1) {
/* Reserve bytes for the end marker of row_merge_block_t. */
if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) {
DBUG_RETURN(0);
}
@ -783,7 +871,7 @@ row_merge_buf_write(
{
const dict_index_t* index = buf->index;
ulint n_fields= dict_index_get_n_fields(index);
byte* b = &block[0];
byte* b = &block[ROW_MERGE_RESERVE_SIZE];
for (ulint i = 0; i < buf->n_tuples; i++) {
const mtuple_t* entry = &buf->tuples[i];
@ -802,7 +890,7 @@ row_merge_buf_write(
/* Write an "end-of-chunk" marker. */
ut_a(b < &block[srv_sort_buf_size]);
ut_a(b == &block[0] + buf->total_size);
ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE);
*b++ = 0;
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
@ -859,7 +947,10 @@ row_merge_read(
ulint offset, /*!< in: offset where to read
in number of row_merge_block_t
elements */
row_merge_block_t* buf) /*!< out: data */
row_merge_block_t* buf, /*!< out: data */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size;
ibool success;
@ -883,6 +974,13 @@ row_merge_read(
success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf,
ofs, srv_sort_buf_size);
/* For encrypted tables, decrypt data after reading and copy data */
if (crypt_data && crypt_buf) {
if( row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) {
memcpy(buf, crypt_buf, srv_sort_buf_size);
}
}
#ifdef POSIX_FADV_DONTNEED
/* Each block is read exactly once. Free up the file cache. */
posix_fadvise(fd, ofs, srv_sort_buf_size, POSIX_FADV_DONTNEED);
@ -905,18 +1003,32 @@ UNIV_INTERN
ibool
row_merge_write(
/*============*/
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf) /*!< in: data */
int fd, /*!< in: file descriptor */
ulint offset, /*!< in: offset where to write,
in number of row_merge_block_t elements */
const void* buf, /*!< in: data */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
void* crypt_buf, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
size_t buf_len = srv_sort_buf_size;
os_offset_t ofs = buf_len * (os_offset_t) offset;
ibool ret;
void* out_buf = (void *)buf;
DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE););
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), buf, ofs, buf_len);
/* For encrypted tables, encrypt data before writing */
if (crypt_data && crypt_buf) {
row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf);
out_buf = crypt_buf;
} else {
/* Mark block unencrypted */
mach_write_to_4((byte *)out_buf, 0);
}
ret = os_file_write("(merge)", OS_FILE_FROM_FD(fd), out_buf, ofs, buf_len);
#ifdef UNIV_DEBUG
if (row_merge_print_block_write) {
@ -950,7 +1062,10 @@ row_merge_read_rec(
const mrec_t** mrec, /*!< out: pointer to merge record,
or NULL on end of list
(non-NULL on I/O error) */
ulint* offsets)/*!< out: offsets of mrec */
ulint* offsets,/*!< out: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint data_size;
@ -968,6 +1083,10 @@ row_merge_read_rec(
ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE
+ dict_index_get_n_fields(index));
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
extra_size = *b++;
if (UNIV_UNLIKELY(!extra_size)) {
@ -987,7 +1106,8 @@ row_merge_read_rec(
/* Read another byte of extra_size. */
if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) {
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
err_exit:
/* Signal I/O error. */
*mrec = b;
@ -995,7 +1115,7 @@ err_exit:
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
}
extra_size = (extra_size & 0x7f) << 8;
@ -1016,13 +1136,14 @@ err_exit:
ut_ad(avail_size < sizeof *buf);
memcpy(*buf, b, avail_size);
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the record. */
memcpy(*buf + avail_size, b, extra_size - avail_size);
@ -1077,13 +1198,14 @@ err_exit:
offsets[3] = (ulint) index;
#endif /* UNIV_DEBUG */
if (!row_merge_read(fd, ++(*foffs), block)) {
if (!row_merge_read(fd, ++(*foffs), block,
crypt_data, crypt_block, space)) {
goto err_exit;
}
/* Wrap around to the beginning of the buffer. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
/* Copy the rest of the record. */
memcpy(*buf + avail_size, b, extra_size + data_size - avail_size);
@ -1159,7 +1281,10 @@ row_merge_write_rec(
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
const mrec_t* mrec, /*!< in: record to write */
const ulint* offsets)/*!< in: offsets of mrec */
const ulint* offsets,/*!< in: offsets of mrec */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint extra_size;
ulint size;
@ -1180,6 +1305,10 @@ row_merge_write_rec(
size = extra_size + (extra_size >= 0x80)
+ rec_offs_data_size(offsets);
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) {
/* The record spans two blocks.
Copy it to the temporary buffer first. */
@ -1194,14 +1323,15 @@ row_merge_write_rec(
record to the head of the new block. */
memcpy(b, buf[0], avail_size);
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
/* Copy the rest. */
b = &block[0];
b = &block[ROW_MERGE_RESERVE_SIZE];
memcpy(b, buf[0] + avail_size, size - avail_size);
b += size - avail_size;
} else {
@ -1220,10 +1350,13 @@ static
byte*
row_merge_write_eof(
/*================*/
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs) /*!< in/out: file offset */
row_merge_block_t* block, /*!< in/out: file buffer */
byte* b, /*!< in: pointer to end of block */
int fd, /*!< in: file descriptor */
ulint* foffs, /*!< in/out: file offset */
fil_space_crypt_t* crypt_data, /*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ut_ad(block);
ut_ad(b >= &block[0]);
@ -1236,20 +1369,27 @@ row_merge_write_eof(
}
#endif /* UNIV_DEBUG */
if (b == &block[0]) {
b+= ROW_MERGE_RESERVE_SIZE;
}
*b++ = 0;
UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]);
UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size);
#ifdef UNIV_DEBUG_VALGRIND
/* The rest of the block is uninitialized. Initialize it
to avoid bogus warnings. */
memset(b, 0xff, &block[srv_sort_buf_size] - b);
#endif /* UNIV_DEBUG_VALGRIND */
if (!row_merge_write(fd, (*foffs)++, block)) {
if (!row_merge_write(fd, (*foffs)++, block,
crypt_data, crypt_block, space)) {
return(NULL);
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
return(&block[0]);
}
@ -1294,7 +1434,11 @@ row_merge_read_clustered_index(
ULINT_UNDEFINED if none is added */
ib_sequence_t& sequence,/*!< in/out: autoinc sequence */
row_merge_block_t* block, /*!< in/out: file buffer */
float pct_cost) /*!< in: percent of task weight out of total alter job */
float pct_cost, /*!< in: percent of task weight
out of total alter job */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block)/*!< in: in/out: crypted file
buffer */
{
dict_index_t* clust_index; /* Clustered index */
mem_heap_t* row_heap; /* Heap memory to create
@ -1316,9 +1460,10 @@ row_merge_read_clustered_index(
ib_int64_t sig_count = 0;
mem_heap_t* conv_heap = NULL;
float curr_progress;
float curr_progress = 0.0;
ib_int64_t read_rows = 0;
ib_int64_t table_total_rows;
ib_int64_t table_total_rows = 0;
DBUG_ENTER("row_merge_read_clustered_index");
ut_ad((old_table == new_table) == !col_map);
@ -1815,14 +1960,15 @@ write_buffers:
row_merge_buf_write(buf, file, block);
if (!row_merge_write(file->fd, file->offset++,
block)) {
if (!row_merge_write(file->fd, file->offset++, block,
crypt_data, crypt_block, new_table->space)) {
err = DB_TEMP_FILE_WRITE_FAILURE;
trx->error_key_num = i;
break;
}
UNIV_MEM_INVALID(&block[0], srv_sort_buf_size);
merge_buf[i] = row_merge_buf_empty(buf);
if (UNIV_LIKELY(row != NULL)) {
@ -1991,14 +2137,21 @@ wait_again:
b2 = row_merge_write_rec(&block[2 * srv_sort_buf_size], \
&buf[2], b2, \
of->fd, &of->offset, \
mrec##N, offsets##N); \
mrec##N, offsets##N, \
crypt_data, \
&crypt_block[2 * srv_sort_buf_size], \
space); \
if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \
goto corrupt; \
} \
b##N = row_merge_read_rec(&block[N * srv_sort_buf_size],\
&buf[N], b##N, INDEX, \
file->fd, foffs##N, \
&mrec##N, offsets##N); \
&mrec##N, offsets##N, \
crypt_data, \
&crypt_block[N * srv_sort_buf_size], \
space); \
\
if (UNIV_UNLIKELY(!b##N)) { \
if (mrec##N) { \
goto corrupt; \
@ -2023,7 +2176,11 @@ row_merge_blocks(
source list in the file */
ulint* foffs1, /*!< in/out: offset of second
source list in the file */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */
row_merge_block_t* crypt_block,/*!< in: in/out: crypted file
buffer */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2054,8 +2211,10 @@ row_merge_blocks(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)
|| !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size],
crypt_data, &crypt_block[srv_sort_buf_size], space)) {
corrupt:
mem_heap_free(heap);
return(DB_CORRUPTION);
@ -2067,11 +2226,15 @@ corrupt:
b0 = row_merge_read_rec(
&block[0], &buf[0], b0, dup->index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
b1 = row_merge_read_rec(
&block[srv_sort_buf_size],
&buf[srv_sort_buf_size], b1, dup->index,
file->fd, foffs1, &mrec1, offsets1);
file->fd, foffs1, &mrec1, offsets1,
crypt_data, &crypt_block[srv_sort_buf_size], space);
if (UNIV_UNLIKELY(!b0 && mrec0)
|| UNIV_UNLIKELY(!b1 && mrec1)) {
@ -2113,8 +2276,11 @@ done0:
done1:
mem_heap_free(heap);
b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset);
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space);
return(b2 ? DB_SUCCESS : DB_CORRUPTION);
}
@ -2129,7 +2295,10 @@ row_merge_blocks_copy(
const merge_file_t* file, /*!< in: input file */
row_merge_block_t* block, /*!< in/out: 3 buffers */
ulint* foffs0, /*!< in/out: input file offset */
merge_file_t* of) /*!< in/out: output file */
merge_file_t* of, /*!< in/out: output file */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */
@ -2156,7 +2325,8 @@ row_merge_blocks_copy(
/* Write a record and read the next record. Split the output
file in two halves, which can be merged on the following pass. */
if (!row_merge_read(file->fd, *foffs0, &block[0])) {
if (!row_merge_read(file->fd, *foffs0, &block[0],
crypt_data, &crypt_block[0], space)) {
corrupt:
mem_heap_free(heap);
return(FALSE);
@ -2167,7 +2337,9 @@ corrupt:
b2 = &block[2 * srv_sort_buf_size];
b0 = row_merge_read_rec(&block[0], &buf[0], b0, index,
file->fd, foffs0, &mrec0, offsets0);
file->fd, foffs0, &mrec0, offsets0,
crypt_data, &crypt_block[0], space);
if (UNIV_UNLIKELY(!b0 && mrec0)) {
goto corrupt;
@ -2186,8 +2358,10 @@ done0:
(*foffs0)++;
mem_heap_free(heap);
return(row_merge_write_eof(&block[2 * srv_sort_buf_size],
b2, of->fd, &of->offset)
b2, of->fd, &of->offset,
crypt_data, &crypt_block[2 * srv_sort_buf_size], space)
!= NULL);
}
@ -2207,9 +2381,12 @@ row_merge(
int* tmpfd, /*!< in/out: temporary file handle */
ulint* num_run,/*!< in/out: Number of runs remain
to be merged */
ulint* run_offset) /*!< in/out: Array contains the
ulint* run_offset, /*!< in/out: Array contains the
first offset number for each merge
run */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
ulint foffs0; /*!< first input offset */
ulint foffs1; /*!< second input offset */
@ -2222,6 +2399,10 @@ row_merge(
UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size);
if (crypt_block) {
UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size);
}
ut_ad(ihalf < file->offset);
of.fd = *tmpfd;
@ -2252,7 +2433,8 @@ row_merge(
run_offset[n_run++] = of.offset;
error = row_merge_blocks(dup, file, block,
&foffs0, &foffs1, &of);
&foffs0, &foffs1, &of,
crypt_data, crypt_block, space);
if (error != DB_SUCCESS) {
return(error);
@ -2272,7 +2454,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs0, &of)) {
&foffs0, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2289,7 +2472,8 @@ row_merge(
run_offset[n_run++] = of.offset;
if (!row_merge_blocks_copy(dup->index, file, block,
&foffs1, &of)) {
&foffs1, &of,
crypt_data, crypt_block, space)) {
return(DB_CORRUPTION);
}
}
@ -2344,7 +2528,10 @@ row_merge_sort(
const float pct_progress,
/*!< in: total progress percent
until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent */
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const ulint half = file->offset / 2;
ulint num_runs;
@ -2396,7 +2583,8 @@ row_merge_sort(
sql_print_information("InnoDB: Online DDL : merge-sorting current run %lu estimated %lu runs", cur_run, num_runs);
error = row_merge(trx, dup, file, block, tmpfd,
&num_runs, run_offset);
&num_runs, run_offset,
crypt_data, crypt_block, space);
if(update_progress) {
merge_count++;
@ -2479,7 +2667,11 @@ row_merge_insert_index_tuples(
row_merge_block_t* block, /*!< in/out: file buffer */
const ib_int64_t table_total_rows, /*!< in: total rows of old table */
const float pct_progress, /*!< in: total progress percent until now */
const float pct_cost) /*!< in: current progress percent */
const float pct_cost, /*!< in: current progress percent
*/
fil_space_crypt_t* crypt_data,/*!< in: table crypt data */
row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */
ulint space) /*!< in: space id */
{
const byte* b;
mem_heap_t* heap;
@ -2510,9 +2702,10 @@ row_merge_insert_index_tuples(
offsets[1] = dict_index_get_n_fields(index);
}
b = block;
b = &block[0];
if (!row_merge_read(fd, foffs, block)) {
if (!row_merge_read(fd, foffs, block,
crypt_data, crypt_block, space)) {
error = DB_CORRUPTION;
} else {
buf = static_cast<mrec_buf_t*>(
@ -2528,7 +2721,8 @@ row_merge_insert_index_tuples(
mtr_t mtr;
b = row_merge_read_rec(block, buf, b, index,
fd, &foffs, &mrec, offsets);
fd, &foffs, &mrec, offsets,
crypt_data, crypt_block, space);
if (UNIV_UNLIKELY(!b)) {
/* End of list, or I/O error */
if (mrec) {
@ -3669,6 +3863,7 @@ row_merge_build_indexes(
{
merge_file_t* merge_files;
row_merge_block_t* block;
row_merge_block_t* crypt_block;
ulint block_size;
ulint i;
ulint j;
@ -3679,6 +3874,7 @@ row_merge_build_indexes(
fts_psort_t* merge_info = NULL;
ib_int64_t sig_count = 0;
bool fts_psort_initiated = false;
fil_space_crypt_t * crypt_data = NULL;
float total_static_cost = 0;
float total_dynamic_cost = 0;
@ -3703,6 +3899,27 @@ row_merge_build_indexes(
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
/* Get crypt data from tablespace if present. */
crypt_data = fil_space_get_crypt_data(new_table->space);
crypt_block = NULL;
/* If tablespace is encrypted, allocate additional buffer for
encryption/decryption. */
if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
(srv_encrypt_tables &&
crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
crypt_block = static_cast<row_merge_block_t*>(
os_mem_alloc_large(&block_size, FALSE));
if (crypt_block == NULL) {
DBUG_RETURN(DB_OUT_OF_MEMORY);
}
} else {
/* Not needed */
crypt_data = NULL;
}
trx_start_if_not_started_xa(trx);
merge_files = static_cast<merge_file_t*>(
@ -3784,10 +4001,11 @@ row_merge_build_indexes(
secondary index entries for merge sort */
error = row_merge_read_clustered_index(
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost);
trx, table, old_table, new_table, online, indexes,
fts_sort_idx, psort_info, merge_files, key_numbers,
n_indexes, add_cols, col_map,
add_autoinc, sequence, block, pct_cost,
crypt_data, crypt_block);
pct_progress += pct_cost;
@ -3807,6 +4025,10 @@ row_merge_build_indexes(
/* Now we have files containing index entries ready for
sorting and inserting. */
DBUG_EXECUTE_IF(
"ib_merge_wait_after_read",
os_thread_sleep(20000000);); /* 20 sec */
for (i = 0; i < n_indexes; i++) {
dict_index_t* sort_idx = indexes[i];
@ -3900,8 +4122,10 @@ wait_again:
buf, (i+1), n_indexes, pct_cost);
error = row_merge_sort(
trx, &dup, &merge_files[i],
block, &tmpfd, true, pct_progress, pct_cost);
trx, &dup, &merge_files[i],
block, &tmpfd, true,
pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
@ -3909,6 +4133,10 @@ wait_again:
" merge-sorting index %s (%lu / %lu)",
buf, (i+1), n_indexes);
DBUG_EXECUTE_IF(
"ib_merge_wait_after_sort",
os_thread_sleep(20000000);); /* 20 sec */
if (error == DB_SUCCESS) {
pct_cost = (COST_BUILD_INDEX_STATIC +
(total_dynamic_cost * merge_files[i].offset /
@ -3924,7 +4152,8 @@ wait_again:
error = row_merge_insert_index_tuples(
trx->id, sort_idx, old_table,
merge_files[i].fd, block,
merge_files[i].n_rec, pct_progress, pct_cost);
merge_files[i].n_rec, pct_progress, pct_cost,
crypt_data, crypt_block, new_table->space);
pct_progress += pct_cost;
sql_print_information("InnoDB: Online DDL : "
@ -3998,6 +4227,10 @@ func_exit:
mem_free(merge_files);
os_mem_free_large(block, block_size);
if (crypt_block) {
os_mem_free_large(crypt_block, block_size);
}
DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID);
if (online && old_table == new_table && error != DB_SUCCESS) {

Write Preview
Loading…
Cancel
Save