From d7d589dc01f6d70d1518b74d46fd3b75e76267f5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 23 Mar 2015 21:23:00 +0100 Subject: [PATCH] Further fixes (copied from d7d589dc) --- CMakeLists.txt | 1 - dbug/dbug.c | 50 -- include/my_aes.h | 2 +- include/my_crypt.h | 3 - include/my_crypt_key_management.h | 10 +- include/mysql/plugin.h | 3 - libmysqld/CMakeLists.txt | 1 - mysql-test/include/bootstrap.cnf | 2 - mysql-test/include/have_innodb_encryption.inc | 7 + mysql-test/mysql-test-run.pl | 4 +- .../r/information_schema_all_engines.result | 2 +- mysql-test/r/mysqld--help.result | 9 + .../suite/funcs_1/r/is_engines_innodb.result | 2 +- .../suite/innodb/r/innodb_monitor.result | 3 + .../t/innodb-page_encryption-32k-master.opt | 5 +- .../suite/innodb/t/innodb-page_encryption.opt | 4 +- .../t/innodb-page_encryption_compression.opt | 4 +- .../innodb/t/innodb_bug14147491-master.opt | 2 +- mysql-test/suite/innodb/t/innodb_bug60049.opt | 1 - .../suite/innodb/t/innodb_encryption.opt | 9 +- .../innodb/t/innodb_information_schema.opt | 1 - .../t/innodb_information_schema_buffer.opt | 1 - mysql-test/suite/innodb/t/innodb_scrub.opt | 16 +- .../innodb/t/innodb_scrub_background.opt | 16 +- .../innodb/t/innodb_scrub_compressed.opt | 17 +- .../t/innodb_cmp_drop_table-master.opt | 2 +- mysql-test/suite/maria/maria3.result | 2 +- mysql-test/suite/sys_vars/r/all_vars.result | 1 - ...ug_use_static_encryption_keys_basic.result | 3 + .../r/encryption_algorithm_basic.result | 7 + .../suite/sys_vars/r/sysvars_debug.result | 28 + .../suite/sys_vars/r/sysvars_innodb.result | 58 +- .../sys_vars/r/sysvars_server_embedded.result | 28 + .../r/sysvars_server_notembedded.result | 32 +- .../t/danger_danger_use_dbug_keys_basic.test | 3 - ...> debug_encryption_key_version_basic.test} | 2 +- ...ebug_use_static_encryption_keys_basic.test | 3 + .../t/encryption_algorithm_basic.test | 13 + .../t/innodb_checksum_algorithm_basic.opt | 1 - .../innodb_data_encryption_filekey_basic.test | 1 + ...db_data_encryption_providername_basic.test | 4 +- ...db_data_encryption_providertype_basic.test | 1 + ...odb_data_encryption_providerurl_basic.test | 1 + mysql-test/t/mysqld--help.test | 3 +- mysys/my_default.c | 5 +- mysys/my_thr_init.c | 6 +- mysys_ssl/my_crypt_key_management.cc | 27 +- mysys_ssl/my_crypt_key_management_impl.cc | 5 + .../example_key_management_plugin.cc | 20 +- .../file_key_management_plugin/CMakeLists.txt | 3 +- plugin/file_key_management_plugin/EncKeys.cc | 679 ++++++++++-------- plugin/file_key_management_plugin/EncKeys.h | 2 +- .../KeySingleton.cc | 58 +- .../file_key_management_plugin/KeySingleton.h | 2 +- .../file_key_management_plugin.cc | 49 +- sql/mysqld.cc | 6 +- sql/mysqld.h | 4 +- sql/sql_acl.cc | 9 - sql/sql_plugin.cc | 97 +-- sql/sql_plugin.h | 4 +- sql/sql_select.cc | 8 +- sql/sys_vars.cc | 35 +- storage/innobase/CMakeLists.txt | 1 - storage/innobase/buf/buf0buf.cc | 60 +- storage/innobase/buf/buf0rea.cc | 2 +- storage/innobase/enc/EncKeys.cc | 433 ----------- storage/innobase/enc/KeySingleton.cc | 68 -- storage/innobase/fil/fil0crypt.cc | 34 +- storage/innobase/fil/fil0pagecompress.cc | 13 +- storage/innobase/handler/ha_innodb.cc | 6 +- storage/innobase/handler/ha_innodb.h | 2 - storage/innobase/include/buf0buf.h | 3 +- storage/innobase/include/fil0fil.h | 4 +- storage/innobase/log/log0crypt.cc | 6 +- storage/innobase/log/log0recv.cc | 4 +- storage/innobase/os/os0file.cc | 82 +-- storage/maria/ma_bitmap.c | 8 +- storage/maria/ma_blockrec.c | 108 +-- storage/maria/ma_create.c | 2 +- storage/maria/ma_crypt.c | 6 +- storage/maria/ma_crypt.h | 2 +- storage/maria/ma_write.c | 2 +- storage/maria/maria_def.h | 1 + storage/xtradb/buf/buf0rea.cc | 2 +- storage/xtradb/fil/fil0crypt.cc | 33 +- storage/xtradb/fil/fil0pagecompress.cc | 15 +- storage/xtradb/handler/ha_innodb.cc | 4 +- storage/xtradb/include/buf0buf.h | 3 +- storage/xtradb/include/fil0fil.h | 4 +- storage/xtradb/log/log0crypt.cc | 6 +- storage/xtradb/log/log0recv.cc | 4 +- storage/xtradb/os/os0file.cc | 238 ++---- .../test_innodb_log_encryption.sh | 24 +- 93 files changed, 953 insertions(+), 1614 deletions(-) delete mode 100644 mysql-test/include/bootstrap.cnf create mode 100644 mysql-test/include/have_innodb_encryption.inc delete mode 100644 mysql-test/suite/innodb/t/innodb_bug60049.opt delete mode 100644 mysql-test/suite/innodb/t/innodb_information_schema.opt create mode 100644 mysql-test/suite/sys_vars/r/debug_use_static_encryption_keys_basic.result create mode 100644 mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result delete mode 100644 mysql-test/suite/sys_vars/t/danger_danger_use_dbug_keys_basic.test rename mysql-test/suite/sys_vars/t/{danger_danger_dbug_crypto_key_version_basic.test => debug_encryption_key_version_basic.test} (58%) create mode 100644 mysql-test/suite/sys_vars/t/debug_use_static_encryption_keys_basic.test create mode 100644 mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test delete mode 100644 mysql-test/suite/sys_vars/t/innodb_checksum_algorithm_basic.opt delete mode 100644 storage/innobase/enc/EncKeys.cc delete mode 100644 storage/innobase/enc/KeySingleton.cc rename {google => tests}/test_innodb_log_encryption.sh (75%) diff --git a/CMakeLists.txt b/CMakeLists.txt index fac472d59ac..196b60731da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -323,7 +323,6 @@ SET(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON) ADD_DEFINITIONS(-DHAVE_CONFIG_H) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include) - # Add bundled or system zlib. MYSQL_CHECK_ZLIB_WITH_COMPRESS() # Add bundled yassl/taocrypt or system openssl. diff --git a/dbug/dbug.c b/dbug/dbug.c index ad72e542ada..dffd7a44cd8 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -85,7 +85,6 @@ #undef SAFE_MUTEX #include #include -#include #ifndef DBUG_OFF @@ -2185,55 +2184,6 @@ const char* _db_get_func_(void) return cs->func; } -#ifdef EXTRA_DEBUG - -#define simple_isprint(A) ((A) >= ' ' && (A) <= 126) - -void dump_buffer(FILE *stream, unsigned n, const unsigned char* buf) -{ - int on_this_line = 0; - int counter = 0; - int cc =0; - char ch =0; - - fflush(stream); - fprintf(stream, "%06X: ", counter); - while (n-- > 0) { - fprintf(stream, "%02X ", *buf++); - on_this_line += 1; - if (on_this_line == 16 || n == 0) { - int i; - fprintf(stream, " "); - cc = on_this_line; - if (cc != 16) { - - - for (i = on_this_line; i < 16; i++) { - fprintf(stream," " ); - } - } - for (i = on_this_line; i > 0; i--) { - ch = simple_isprint(buf[-i]) ? buf[-i] : '.'; - fprintf(stream,"%c",ch); - } - - fprintf(stream,"\n" ); - - on_this_line = 0; - if (n!=0) fprintf(stream, "%06X: ", ++counter); - - - } else { - counter++; - } - } - fprintf( stream, "\n"); - fflush(stream); -} - -#endif - - #else /* diff --git a/include/my_aes.h b/include/my_aes.h index 8ec36add4d1..1616d79d70a 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -125,7 +125,7 @@ my_aes_encrypt_dynamic_type get_aes_encrypt_func(enum enum_my_aes_encryption_alg my_bool my_aes_init_dynamic_encrypt(enum enum_my_aes_encryption_algorithm method); -extern enum enum_my_aes_encryption_algorithm current_aes_dynamic_method; +extern MYSQL_PLUGIN_IMPORT enum enum_my_aes_encryption_algorithm current_aes_dynamic_method; diff --git a/include/my_crypt.h b/include/my_crypt.h index 758a544c3f6..c6b5e734b5e 100644 --- a/include/my_crypt.h +++ b/include/my_crypt.h @@ -3,11 +3,8 @@ #ifndef MYSYS_MY_CRYPT_H_ #define MYSYS_MY_CRYPT_H_ -/* We expect same result code from encryption functions as in my_aes.h */ #include -typedef int Crypt_result; - #if !defined(HAVE_YASSL) && defined(HAVE_OPENSSL) #define HAVE_EncryptAes128Ctr diff --git a/include/my_crypt_key_management.h b/include/my_crypt_key_management.h index 99643d654e1..3da0ab2b90e 100644 --- a/include/my_crypt_key_management.h +++ b/include/my_crypt_key_management.h @@ -7,15 +7,15 @@ #include "mysql/psi/psi.h" #ifndef DBUG_OFF -extern my_bool opt_danger_danger_use_dbug_keys; +extern my_bool debug_use_static_encryption_keys; #ifdef HAVE_PSI_INTERFACE -extern PSI_rwlock_key key_LOCK_dbug_crypto_key_version; +extern PSI_rwlock_key key_LOCK_dbug_encryption_key_version; #endif -extern mysql_rwlock_t LOCK_dbug_crypto_key_version; -extern uint opt_danger_danger_dbug_crypto_key_version; -#endif +extern mysql_rwlock_t LOCK_dbug_encryption_key_version; +extern uint opt_debug_encryption_key_version; +#endif /* DBUG_OFF */ C_MODE_START /** diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index e5d6ed7d13b..df74be0209d 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -178,9 +178,6 @@ enum enum_mysql_show_type SHOW_always_last }; - - - /* backward compatibility mapping. */ #define SHOW_INT SHOW_UINT #define SHOW_LONG SHOW_ULONG diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 7dbc8cd70f5..3826fbbc9bf 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -26,7 +26,6 @@ ${PCRE_INCLUDES} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} ${SSL_INTERNAL_INCLUDE_DIRS} -${CMAKE_SOURCE_DIR}/sql/backup ) SET(GEN_SOURCES diff --git a/mysql-test/include/bootstrap.cnf b/mysql-test/include/bootstrap.cnf deleted file mode 100644 index 3cff256b1b0..00000000000 --- a/mysql-test/include/bootstrap.cnf +++ /dev/null @@ -1,2 +0,0 @@ -[mysqld] -loose-innodb-encrypt-tables=OFF diff --git a/mysql-test/include/have_innodb_encryption.inc b/mysql-test/include/have_innodb_encryption.inc new file mode 100644 index 00000000000..f81d1527d50 --- /dev/null +++ b/mysql-test/include/have_innodb_encryption.inc @@ -0,0 +1,7 @@ +# +# Ensure we have innodb encryption incompiled + +if (`select count(*)=0 from information_schema.global_variables where variable_name="innodb_data_encryption_providertype"`) +{ + --skip Test requires InnoDB encryption. +} diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index e31ccce5bc7..47f227209c5 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1850,7 +1850,7 @@ sub collect_mysqld_features { # my $args; mtr_init_args(\$args); - mtr_add_arg($args, "--defaults-file=%s", "include/bootstrap.cnf"); + mtr_add_arg($args, "--no-defaults"); mtr_add_arg($args, "--datadir=."); mtr_add_arg($args, "--basedir=%s", $basedir); mtr_add_arg($args, "--lc-messages-dir=%s", $path_language); @@ -3109,7 +3109,7 @@ sub mysql_install_db { my $args; mtr_init_args(\$args); - mtr_add_arg($args, "--defaults-file=%s", "include/bootstrap.cnf"); + mtr_add_arg($args, "--no-defaults"); mtr_add_arg($args, "--bootstrap"); mtr_add_arg($args, "--basedir=%s", $install_basedir); mtr_add_arg($args, "--datadir=%s", $install_datadir); diff --git a/mysql-test/r/information_schema_all_engines.result b/mysql-test/r/information_schema_all_engines.result index c6c2c2ed308..7b3e3a5f328 100644 --- a/mysql-test/r/information_schema_all_engines.result +++ b/mysql-test/r/information_schema_all_engines.result @@ -417,5 +417,5 @@ Wildcard: inf_rmation_schema | information_schema | SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') GROUP BY TABLE_SCHEMA; table_schema count(*) -information_schema 57 +information_schema 59 mysql 30 diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index ce1198a994a..1a6897d4de2 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -162,6 +162,13 @@ The following options may be given as the first argument: --div-precision-increment=# Precision of the result of '/' operator will be increased on that value + --encrypt-tmp-disk-tables + Encrypt tmp disk tables (created as part of query + execution) + --encryption-algorithm=name + Which encryption algorithm to use for table encryption. + aes_cbc is the recommended one.. One of: none, aes_ecb, + aes_cbc, aes_ctr --event-scheduler[=name] Enable the event scheduler. Possible values are ON, OFF, and DISABLED (keep the event scheduler completely @@ -1141,6 +1148,8 @@ delayed-insert-limit 100 delayed-insert-timeout 300 delayed-queue-size 1000 div-precision-increment 4 +encrypt-tmp-disk-tables FALSE +encryption-algorithm none event-scheduler OFF expensive-subquery-limit 100 expire-logs-days 0 diff --git a/mysql-test/suite/funcs_1/r/is_engines_innodb.result b/mysql-test/suite/funcs_1/r/is_engines_innodb.result index 5713b417cd1..b8c6399fe94 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_innodb.result +++ b/mysql-test/suite/funcs_1/r/is_engines_innodb.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'InnoDB'; ENGINE InnoDB SUPPORT YES -COMMENT Supports transactions, row-level locking, and foreign keys +COMMENT Supports transactions, row-level locking, foreign keys and encryption for tables TRANSACTIONS YES XA YES SAVEPOINTS YES diff --git a/mysql-test/suite/innodb/r/innodb_monitor.result b/mysql-test/suite/innodb/r/innodb_monitor.result index 84668cf49d0..02e72ae6a80 100644 --- a/mysql-test/suite/innodb/r/innodb_monitor.result +++ b/mysql-test/suite/innodb/r/innodb_monitor.result @@ -180,6 +180,9 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled +compress_pages_page_encrypted disabled +compress_pages_page_decrypted disabled +compress_pages_page_encryption_error disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption-32k-master.opt b/mysql-test/suite/innodb/t/innodb-page_encryption-32k-master.opt index a2d8b7ab11a..723c0b360fd 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption-32k-master.opt +++ b/mysql-test/suite/innodb/t/innodb-page_encryption-32k-master.opt @@ -1 +1,4 @@ ---default_storage_engine=InnoDB --innodb-buffer-pool-size=24M --file_key_management_plugin_filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt \ No newline at end of file +--default-storage-engine=InnoDB +--encryption-algorithm=aes_cbs +--file-key-management-plugin-filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt +--innodb-buffer-pool-size=24M diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption.opt b/mysql-test/suite/innodb/t/innodb-page_encryption.opt index ca1d5d1ef18..f3aa00059a8 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption.opt +++ b/mysql-test/suite/innodb/t/innodb-page_encryption.opt @@ -1 +1,3 @@ ---enable-file_key_management_plugin --file_key_management_plugin_filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt +--enable-file-key-management-plugin +--encryption-algorithm=aes_cbs +--file-key-management-plugin-filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption_compression.opt b/mysql-test/suite/innodb/t/innodb-page_encryption_compression.opt index da28287f7a4..f3aa00059a8 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption_compression.opt +++ b/mysql-test/suite/innodb/t/innodb-page_encryption_compression.opt @@ -1 +1,3 @@ ---enable-file_key_management_plugin --file_key_management_plugin_filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt \ No newline at end of file +--enable-file-key-management-plugin +--encryption-algorithm=aes_cbs +--file-key-management-plugin-filename=$MYSQL_TEST_DIR/suite/innodb/include/keys.txt diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt b/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt index 7a5772a48ae..6b82baca147 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt +++ b/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt @@ -1 +1 @@ ---innodb_file_per_table=1 --skip-stack-trace --skip-core-file --innodb-encrypt-tables=FALSE +--innodb_file_per_table=1 --skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/innodb/t/innodb_bug60049.opt b/mysql-test/suite/innodb/t/innodb_bug60049.opt deleted file mode 100644 index 5c9cc32f4d6..00000000000 --- a/mysql-test/suite/innodb/t/innodb_bug60049.opt +++ /dev/null @@ -1 +0,0 @@ ---innodb-encrypt-tables=FALSE diff --git a/mysql-test/suite/innodb/t/innodb_encryption.opt b/mysql-test/suite/innodb/t/innodb_encryption.opt index ead690cf43e..e2fc72eb5a8 100644 --- a/mysql-test/suite/innodb/t/innodb_encryption.opt +++ b/mysql-test/suite/innodb/t/innodb_encryption.opt @@ -1 +1,8 @@ ---enable-example_key_management_plugin --loose-encrypt-tmp-disk-tables=ON --loose-aria-encrypt-tables=ON --loose-innodb-encryption-threads=4 --loose-innodb-encryption-rotate-key-age=15 --loose-innodb-encrypt-tables=ON --innodb_tablespaces_encryption \ No newline at end of file +--enable-example-key-management-plugin +--encrypt-tmp-disk-tables=ON +--aria-encrypt-tables=ON +--innodb-encryption-threads=4 +--innodb-encryption-rotate-key-age=15 +--innodb-encrypt-tables=ON +--innodb-tablespaces-encryption +--encryption-algorithm=aes_ctr diff --git a/mysql-test/suite/innodb/t/innodb_information_schema.opt b/mysql-test/suite/innodb/t/innodb_information_schema.opt deleted file mode 100644 index 5c9cc32f4d6..00000000000 --- a/mysql-test/suite/innodb/t/innodb_information_schema.opt +++ /dev/null @@ -1 +0,0 @@ ---innodb-encrypt-tables=FALSE diff --git a/mysql-test/suite/innodb/t/innodb_information_schema_buffer.opt b/mysql-test/suite/innodb/t/innodb_information_schema_buffer.opt index c8ebc2c606c..9e43eb47f97 100644 --- a/mysql-test/suite/innodb/t/innodb_information_schema_buffer.opt +++ b/mysql-test/suite/innodb/t/innodb_information_schema_buffer.opt @@ -1,4 +1,3 @@ --loose-innodb-buffer-pool-stats --loose-innodb-buffer-page --loose-innodb-buffer-page-lru ---loose-innodb-encrypt-tables=FALSE diff --git a/mysql-test/suite/innodb/t/innodb_scrub.opt b/mysql-test/suite/innodb/t/innodb_scrub.opt index d701024b138..fd165b269b5 100644 --- a/mysql-test/suite/innodb/t/innodb_scrub.opt +++ b/mysql-test/suite/innodb/t/innodb_scrub.opt @@ -1,2 +1,14 @@ ---innodb_file_per_table=1 --innodb_file_format=Barracuda --innodb_encrypt_tables=0 --innodb-encryption_threads=0 --innodb-immediate-scrub-data-uncompressed=ON --innodb-background-scrub-data-uncompressed=OFF --innodb-background-scrub-data-compressed=OFF --loose-innodb-scrub-force-testing=ON --enable-example_key_management_plugin --loose-encrypt-tmp-disk-tables=ON --loose-aria-encrypt-tables=ON --loose-innodb-encryption-threads=4 --loose-innodb-encryption-rotate-key-age=15 --loose-innodb-encrypt-tables=ON - +--enable-example-key-management-plugin +--innodb-background-scrub-data-compressed=OFF +--innodb-background-scrub-data-uncompressed=OFF +--innodb-encrypt-tables=0 +--innodb-encryption-threads=0 +--innodb-file-format=Barracuda +--innodb-file-per-table=1 +--innodb-immediate-scrub-data-uncompressed=ON +--loose-aria-encrypt-tables=ON +--loose-encrypt-tmp-disk-tables=ON +--loose-innodb-encrypt-tables=ON +--loose-innodb-encryption-rotate-key-age=15 +--loose-innodb-encryption-threads=4 +--loose-innodb-scrub-force-testing=ON diff --git a/mysql-test/suite/innodb/t/innodb_scrub_background.opt b/mysql-test/suite/innodb/t/innodb_scrub_background.opt index 031bb071874..e1fc6d82626 100644 --- a/mysql-test/suite/innodb/t/innodb_scrub_background.opt +++ b/mysql-test/suite/innodb/t/innodb_scrub_background.opt @@ -1 +1,15 @@ ---innodb_file_per_table=1 --innodb_file_format=Barracuda --innodb_encrypt_tables=0 --innodb-encryption_threads=0 --innodb-immediate-scrub-data-uncompressed=OFF --innodb-background-scrub-data-uncompressed=ON --innodb-background-scrub-data-compressed=ON --loose-innodb-scrub-force-testing=ON --enable-example_key_management_plugin --loose-encrypt-tmp-disk-tables=ON --loose-aria-encrypt-tables=ON --loose-innodb-encryption-threads=4 --loose-innodb-encryption-rotate-key-age=15 --loose-innodb-encrypt-tables=ON --innodb_tablespaces_scrubbing +--enable-example-key-management-plugin +--innodb-background-scrub-data-compressed=ON +--innodb-background-scrub-data-uncompressed=ON +--innodb-encrypt-tables=0 +--innodb-encryption-threads=0 +--innodb-file-format=Barracuda +--innodb-file-per-table=1 +--innodb-immediate-scrub-data-uncompressed=OFF +--innodb-tablespaces-scrubbing +--loose-aria-encrypt-tables=ON +--loose-encrypt-tmp-disk-tables=ON +--loose-innodb-encrypt-tables=ON +--loose-innodb-encryption-rotate-key-age=15 +--loose-innodb-encryption-threads=4 +--loose-innodb-scrub-force-testing=ON diff --git a/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt b/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt index 2d1e1a58480..b956866e648 100644 --- a/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt +++ b/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt @@ -1,2 +1,15 @@ ---innodb_file_per_table=1 --innodb_file_format=Barracuda --innodb_encrypt_tables=off --innodb-immediate-scrub-data-uncompressed=ON --innodb-background-scrub-data-uncompressed=ON --innodb-background-scrub-data-compressed=ON --loose-innodb-scrub-force-testing=ON --innodb-encryption-threads=0 --enable-example_key_management_plugin --loose-encrypt-tmp-disk-tables=ON --loose-aria-encrypt-tables=ON --loose-innodb-encryption-threads=4 --loose-innodb-encryption-rotate-key-age=15 --loose-innodb-encrypt-tables=ON --innodb_tablespaces_scrubbing - +--enable-example-key-management-plugin +--innodb-background-scrub-data-compressed=ON +--innodb-background-scrub-data-uncompressed=ON +--innodb-encrypt-tables=off +--innodb-encryption-threads=0 +--innodb-file-format=Barracuda +--innodb-file-per-table=1 +--innodb-immediate-scrub-data-uncompressed=ON +--innodb-tablespaces-scrubbing +--loose-aria-encrypt-tables=ON +--loose-encrypt-tmp-disk-tables=ON +--loose-innodb-encrypt-tables=ON +--loose-innodb-encryption-rotate-key-age=15 +--loose-innodb-encryption-threads=4 +--loose-innodb-scrub-force-testing=ON diff --git a/mysql-test/suite/innodb_zip/t/innodb_cmp_drop_table-master.opt b/mysql-test/suite/innodb_zip/t/innodb_cmp_drop_table-master.opt index 1759b221aad..a9a3d8c3db8 100644 --- a/mysql-test/suite/innodb_zip/t/innodb_cmp_drop_table-master.opt +++ b/mysql-test/suite/innodb_zip/t/innodb_cmp_drop_table-master.opt @@ -1 +1 @@ ---innodb-buffer-pool-size=8M --innodb-encrypt-tables=FALSE +--innodb-buffer-pool-size=8M diff --git a/mysql-test/suite/maria/maria3.result b/mysql-test/suite/maria/maria3.result index 00675478521..feb5fa82cd4 100644 --- a/mysql-test/suite/maria/maria3.result +++ b/mysql-test/suite/maria/maria3.result @@ -305,7 +305,7 @@ Variable_name Value aria_block_size 8192 aria_checkpoint_interval 30 aria_checkpoint_log_activity 1048576 -aria_encrypt_tables ON +aria_encrypt_tables OFF aria_force_start_after_recovery_failures 0 aria_group_commit none aria_group_commit_interval 0 diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index 975f2a341e8..840da8405c3 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -11,6 +11,5 @@ select distinct variable_name as `there should be *no* variables listed below:` left join t1 on variable_name=test_name where test_name is null; there should be *no* variables listed below: strict_password_validation -encrypt_algorithm drop table t1; drop table t2; diff --git a/mysql-test/suite/sys_vars/r/debug_use_static_encryption_keys_basic.result b/mysql-test/suite/sys_vars/r/debug_use_static_encryption_keys_basic.result new file mode 100644 index 00000000000..a0d4f45cdbf --- /dev/null +++ b/mysql-test/suite/sys_vars/r/debug_use_static_encryption_keys_basic.result @@ -0,0 +1,3 @@ +show global variables like "debug_use_static_encryption_keys"; +Variable_name Value +debug_use_static_encryption_keys OFF diff --git a/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result b/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result new file mode 100644 index 00000000000..a9101b0f950 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result @@ -0,0 +1,7 @@ +select @@global.encryption_algorithm; +@@global.encryption_algorithm +none +select @@session.encryption_algorithm; +ERROR HY000: Variable 'encryption_algorithm' is a GLOBAL variable +set global encryption_algorithm="none"; +ERROR HY000: Variable 'encryption_algorithm' is a read only variable diff --git a/mysql-test/suite/sys_vars/r/sysvars_debug.result b/mysql-test/suite/sys_vars/r/sysvars_debug.result index a46e135af0a..b7f169dc22e 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_debug.result +++ b/mysql-test/suite/sys_vars/r/sysvars_debug.result @@ -57,6 +57,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME DEBUG_ENCRYPTION_KEY_VERSION +SESSION_VALUE NULL +GLOBAL_VALUE 0 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 0 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT Encryption key version. Only to be used in internal testing. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME DEBUG_MUTEX_DEADLOCK_DETECTOR SESSION_VALUE NULL GLOBAL_VALUE ON @@ -99,3 +113,17 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT NULL +VARIABLE_NAME DEBUG_USE_STATIC_ENCRYPTION_KEYS +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Enable use of nonrandom encryption keys. Only to be used in internal testing +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY YES +COMMAND_LINE_ARGUMENT OPTIONAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 8a769b5fb8f..966187a61ab 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -537,62 +537,6 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME INNODB_DATA_ENCRYPTION_FILEKEY -SESSION_VALUE NULL -GLOBAL_VALUE -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE VARCHAR -VARIABLE_COMMENT Key to encrypt / decrypt the keyfile. -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST NULL -READ_ONLY YES -COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME INNODB_DATA_ENCRYPTION_PROVIDERNAME -SESSION_VALUE NULL -GLOBAL_VALUE -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE VARCHAR -VARIABLE_COMMENT Name of keyfile or keyserver. -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST NULL -READ_ONLY YES -COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME INNODB_DATA_ENCRYPTION_PROVIDERTYPE -SESSION_VALUE NULL -GLOBAL_VALUE 1 -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE 1 -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Use table or column encryption / decryption. Default is 0 for no use, 1 for keyfile and 2 for keyserver. -NUMERIC_MIN_VALUE 0 -NUMERIC_MAX_VALUE 2 -NUMERIC_BLOCK_SIZE 0 -ENUM_VALUE_LIST NULL -READ_ONLY YES -COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME INNODB_DATA_ENCRYPTION_PROVIDERURL -SESSION_VALUE NULL -GLOBAL_VALUE -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE VARCHAR -VARIABLE_COMMENT Path or URL for keyfile or keyserver. -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST NULL -READ_ONLY YES -COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_DATA_FILE_PATH SESSION_VALUE NULL GLOBAL_VALUE ibdata1:12M:autoextend @@ -824,7 +768,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE OFF VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Encrypt tables +VARIABLE_COMMENT Encrypt all tables in the storage engine NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index 2bf6814633f..48e5fd621e9 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -679,6 +679,34 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME ENCRYPTION_ALGORITHM +SESSION_VALUE NULL +GLOBAL_VALUE none +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE none +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE ENUM +VARIABLE_COMMENT Which encryption algorithm to use for table encryption. aes_cbc is the recommended one. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST none,aes_ecb,aes_cbc,aes_ctr +READ_ONLY YES +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME ENCRYPT_TMP_DISK_TABLES +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Encrypt tmp disk tables (created as part of query execution) +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME ERROR_COUNT SESSION_VALUE 0 GLOBAL_VALUE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 25c5c725283..00f11a87792 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -455,34 +455,6 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME DANGER_DANGER_DBUG_CRYPTO_KEY_VERSION -SESSION_VALUE NULL -GLOBAL_VALUE 0 -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE 0 -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Crypto key version for debugging only -NUMERIC_MIN_VALUE 0 -NUMERIC_MAX_VALUE 4294967295 -NUMERIC_BLOCK_SIZE 1 -ENUM_VALUE_LIST NULL -READ_ONLY NO -COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME DANGER_DANGER_USE_DBUG_KEYS -SESSION_VALUE NULL -GLOBAL_VALUE OFF -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE OFF -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Enable use of nonrandom keys for crypto -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST OFF,ON -READ_ONLY YES -COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME DATADIR SESSION_VALUE NULL GLOBAL_VALUE PATH @@ -721,14 +693,14 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME ENCRYPT_ALGORITHM +VARIABLE_NAME ENCRYPTION_ALGORITHM SESSION_VALUE NULL GLOBAL_VALUE none GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE none VARIABLE_SCOPE GLOBAL VARIABLE_TYPE ENUM -VARIABLE_COMMENT Which encryption algorithm to use for table encryption +VARIABLE_COMMENT Which encryption algorithm to use for table encryption. aes_cbc is the recommended one. NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/t/danger_danger_use_dbug_keys_basic.test b/mysql-test/suite/sys_vars/t/danger_danger_use_dbug_keys_basic.test deleted file mode 100644 index 99c97cbb722..00000000000 --- a/mysql-test/suite/sys_vars/t/danger_danger_use_dbug_keys_basic.test +++ /dev/null @@ -1,3 +0,0 @@ - -# This is just to satisfy all_vars -select 1; diff --git a/mysql-test/suite/sys_vars/t/danger_danger_dbug_crypto_key_version_basic.test b/mysql-test/suite/sys_vars/t/debug_encryption_key_version_basic.test similarity index 58% rename from mysql-test/suite/sys_vars/t/danger_danger_dbug_crypto_key_version_basic.test rename to mysql-test/suite/sys_vars/t/debug_encryption_key_version_basic.test index 99c97cbb722..007724b0966 100644 --- a/mysql-test/suite/sys_vars/t/danger_danger_dbug_crypto_key_version_basic.test +++ b/mysql-test/suite/sys_vars/t/debug_encryption_key_version_basic.test @@ -1,3 +1,3 @@ - +--source include/have_debug.inc # This is just to satisfy all_vars select 1; diff --git a/mysql-test/suite/sys_vars/t/debug_use_static_encryption_keys_basic.test b/mysql-test/suite/sys_vars/t/debug_use_static_encryption_keys_basic.test new file mode 100644 index 00000000000..2e0d51e89b7 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/debug_use_static_encryption_keys_basic.test @@ -0,0 +1,3 @@ +# This is just to satisfy all_vars +--source include/have_debug.inc +show global variables like "debug_use_static_encryption_keys"; diff --git a/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test b/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test new file mode 100644 index 00000000000..065453eba34 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test @@ -0,0 +1,13 @@ +# bool global + +# exists as global only +# +select @@global.encryption_algorithm; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +select @@session.encryption_algorithm; + +# +# show that it's not writable +# +--error 1238 +set global encryption_algorithm="none"; diff --git a/mysql-test/suite/sys_vars/t/innodb_checksum_algorithm_basic.opt b/mysql-test/suite/sys_vars/t/innodb_checksum_algorithm_basic.opt deleted file mode 100644 index 48218bb3e15..00000000000 --- a/mysql-test/suite/sys_vars/t/innodb_checksum_algorithm_basic.opt +++ /dev/null @@ -1 +0,0 @@ ---innodb-encrypt-tables=FALSE --innodb-encryption-threads=0 diff --git a/mysql-test/suite/sys_vars/t/innodb_data_encryption_filekey_basic.test b/mysql-test/suite/sys_vars/t/innodb_data_encryption_filekey_basic.test index c88b928122b..a35be702c25 100644 --- a/mysql-test/suite/sys_vars/t/innodb_data_encryption_filekey_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_data_encryption_filekey_basic.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_innodb_encryption.inc SELECT @start_data_encryption_filekey; diff --git a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providername_basic.test b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providername_basic.test index 4c2be584ab9..80e8282af8f 100644 --- a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providername_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providername_basic.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_innodb_encryption.inc SELECT @start_data_encryption_providername; @@ -8,6 +9,3 @@ SELECT COUNT(@@GLOBAL.innodb_data_encryption_providername); # This variable is read only variable --error 1238 SET @@GLOBAL.innodb_data_encryption_providername='key.txt'; - - - diff --git a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providertype_basic.test b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providertype_basic.test index 385e1ede4cb..7e841a27d27 100644 --- a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providertype_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providertype_basic.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_innodb_encryption.inc SELECT @start_data_encryption_providertype; diff --git a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providerurl_basic.test b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providerurl_basic.test index a18e1bcfc0d..d742fe2aa06 100644 --- a/mysql-test/suite/sys_vars/t/innodb_data_encryption_providerurl_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_data_encryption_providerurl_basic.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_innodb_encryption.inc SELECT @start_data_encryption_providerurl; diff --git a/mysql-test/t/mysqld--help.test b/mysql-test/t/mysqld--help.test index 83f58171333..3ffcd31092d 100644 --- a/mysql-test/t/mysqld--help.test +++ b/mysql-test/t/mysqld--help.test @@ -30,7 +30,8 @@ perl; thread-concurrency super-large-pages mutex-deadlock-detector connect null-audit aria oqgraph sphinx thread-handling test-sql-discovery rpl-semi-sync query-cache-info - query-response-time metadata-lock-info locales wsrep/; + query-response-time metadata-lock-info locales wsrep + file-key-management/; # And substitute the content some environment variables with their # names: diff --git a/mysys/my_default.c b/mysys/my_default.c index 87258a3b27e..e40e24fd5bc 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -102,8 +102,7 @@ static const char *f_extensions[]= { ".cnf", 0 }; #define NEWLINE "\n" #endif -static int handle_default_option(void *in_ctx, const char *group_name, - const char *option); +static int handle_default_option(void *, const char *, const char *); /* This structure defines the context that we pass to callback @@ -917,7 +916,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, end= remove_end_comment(ptr); if ((value= strchr(ptr, '='))) - end= value; /* Option without argument */ + end= value; for ( ; my_isspace(&my_charset_latin1,end[-1]) ; end--) ; if (!value) { diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 17bfdfc25fb..d49a2eff970 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -67,8 +67,8 @@ static void my_thread_init_common_mutex(void) mysql_mutex_init(key_LOCK_localtime_r, &LOCK_localtime_r, MY_MUTEX_INIT_SLOW); #endif #ifndef DBUG_OFF - mysql_rwlock_init(key_LOCK_dbug_crypto_key_version, - &LOCK_dbug_crypto_key_version); + mysql_rwlock_init(key_LOCK_dbug_encryption_key_version, + &LOCK_dbug_encryption_key_version); #endif } @@ -85,7 +85,7 @@ void my_thread_destroy_common_mutex(void) mysql_mutex_destroy(&LOCK_localtime_r); #endif #ifndef DBUG_OFF - mysql_rwlock_destroy(&LOCK_dbug_crypto_key_version); + mysql_rwlock_destroy(&LOCK_dbug_encryption_key_version); #endif } diff --git a/mysys_ssl/my_crypt_key_management.cc b/mysys_ssl/my_crypt_key_management.cc index 94199403b4a..69efed32567 100644 --- a/mysys_ssl/my_crypt_key_management.cc +++ b/mysys_ssl/my_crypt_key_management.cc @@ -1,18 +1,16 @@ #include #include #include -#ifdef __linux__ -#include -#endif #ifndef DBUG_OFF -my_bool opt_danger_danger_use_dbug_keys = 0; +#include +my_bool debug_use_static_encryption_keys = 0; #ifdef HAVE_PSI_INTERFACE -PSI_rwlock_key key_LOCK_dbug_crypto_key_version; +PSI_rwlock_key key_LOCK_dbug_encryption_key_version; #endif -mysql_rwlock_t LOCK_dbug_crypto_key_version; -unsigned int opt_danger_danger_dbug_crypto_key_version = 0; +mysql_rwlock_t LOCK_dbug_encryption_key_version; +unsigned int opt_debug_encryption_key_version = 0; #endif /** @@ -43,10 +41,10 @@ struct CryptoKeyFuncs_t cryptoKeyFuncs = { extern "C" int GetLatestCryptoKeyVersion() { #ifndef DBUG_OFF - if (opt_danger_danger_use_dbug_keys) { - mysql_rwlock_rdlock(&LOCK_dbug_crypto_key_version); - unsigned int res = opt_danger_danger_dbug_crypto_key_version; - mysql_rwlock_unlock(&LOCK_dbug_crypto_key_version); + if (debug_use_static_encryption_keys) { + mysql_rwlock_rdlock(&LOCK_dbug_encryption_key_version); + unsigned int res = opt_debug_encryption_key_version; + mysql_rwlock_unlock(&LOCK_dbug_encryption_key_version); return res; } #endif @@ -67,15 +65,14 @@ int GetCryptoKeySize(unsigned int version) { extern "C" int GetCryptoKey(unsigned int version, unsigned char* key, unsigned int size) { #ifndef DBUG_OFF - if (opt_danger_danger_use_dbug_keys) { + if (debug_use_static_encryption_keys) { memset(key, 0, size); // Just don't support tiny keys, no point anyway. - if (size < sizeof(version)) { + if (size < 4) { return 1; } - version = htonl(version); - memcpy(key, &version, sizeof(version)); + mi_int4store(key, version); return 0; } #endif diff --git a/mysys_ssl/my_crypt_key_management_impl.cc b/mysys_ssl/my_crypt_key_management_impl.cc index ece4b72c372..af2077d8d15 100644 --- a/mysys_ssl/my_crypt_key_management_impl.cc +++ b/mysys_ssl/my_crypt_key_management_impl.cc @@ -4,26 +4,31 @@ int GetLatestCryptoKeyVersionImpl() { abort(); + return 0; /* Keep compiler happy */ } unsigned int HasCryptoKeyImpl(unsigned int version) { abort(); + return 0; /* Keep compiler happy */ } int GetCryptoKeySizeImpl(unsigned int version) { abort(); + return 0; /* Keep compiler happy */ } int GetCryptoKeyImpl(unsigned int version, unsigned char* key, unsigned int key_size) { abort(); + return 0; /* Keep compiler happy */ } int GetCryptoIVImpl(unsigned int version, unsigned char* key, unsigned int key_size) { abort(); + return 0; /* Keep compiler happy */ } diff --git a/plugin/example_key_management_plugin/example_key_management_plugin.cc b/plugin/example_key_management_plugin/example_key_management_plugin.cc index 5785d264c60..cdc32e87aad 100644 --- a/plugin/example_key_management_plugin/example_key_management_plugin.cc +++ b/plugin/example_key_management_plugin/example_key_management_plugin.cc @@ -2,18 +2,20 @@ #include #include +#include #include #include #include +#include /* rotate key randomly between 45 and 90 seconds */ #define KEY_ROTATION_MIN 45 #define KEY_ROTATION_MAX 90 -static unsigned int seed = 0; +static struct my_rnd_struct seed; static unsigned int key_version = 0; static unsigned int next_key_version = 0; -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t mutex; static int @@ -25,7 +27,7 @@ get_latest_key_version() { key_version = now; unsigned int interval = KEY_ROTATION_MAX - KEY_ROTATION_MIN; - next_key_version = now + KEY_ROTATION_MIN + rand_r(&seed) % interval; + next_key_version = now + KEY_ROTATION_MIN + my_rnd(&seed) * interval; } pthread_mutex_unlock(&mutex); @@ -78,11 +80,13 @@ static int get_iv(unsigned int keyID, unsigned char* dstbuf, unsigned buflen) static int example_key_management_plugin_init(void *p) { /* init */ - seed = time(0); + my_rnd_init(&seed, time(0), 0); get_latest_key_version(); my_aes_init_dynamic_encrypt(MY_AES_ALGORITHM_CTR); + pthread_mutex_init(&mutex, NULL); + struct CryptoKeyFuncs_t func; func.getLatestCryptoKeyVersionFunc = get_latest_key_version; func.hasCryptoKeyFunc = has_key_func; @@ -95,9 +99,7 @@ static int example_key_management_plugin_init(void *p) static int example_key_management_plugin_deinit(void *p) { - /** - * don't uninstall... - */ + pthread_mutex_destroy(&mutex); return 0; } @@ -116,8 +118,8 @@ maria_declare_plugin(example_key_management_plugin) "Jonas Oreland", "Example key management plugin", PLUGIN_LICENSE_GPL, - example_key_management_plugin_init, /* Plugin Init */ - example_key_management_plugin_deinit, /* Plugin Deinit */ + example_key_management_plugin_init, + example_key_management_plugin_deinit, 0x0100 /* 1.0 */, NULL, /* status variables */ NULL, /* system variables */ diff --git a/plugin/file_key_management_plugin/CMakeLists.txt b/plugin/file_key_management_plugin/CMakeLists.txt index 49196448e2b..d5412abf714 100644 --- a/plugin/file_key_management_plugin/CMakeLists.txt +++ b/plugin/file_key_management_plugin/CMakeLists.txt @@ -1,3 +1,4 @@ SET(FILE_KEY_MANAGEMENT_PLUGIN_SOURCES file_key_management_plugin.cc EncKeys.cc KeySingleton.cc) -MYSQL_ADD_PLUGIN(FILE_KEY_MANAGEMENT_PLUGIN ${FILE_KEY_MANAGEMENT_PLUGIN_SOURCES} DEFAULT) +MYSQL_ADD_PLUGIN(FILE_KEY_MANAGEMENT_PLUGIN ${FILE_KEY_MANAGEMENT_PLUGIN_SOURCES} DEFAULT + LINK_LIBRARIES pcre) diff --git a/plugin/file_key_management_plugin/EncKeys.cc b/plugin/file_key_management_plugin/EncKeys.cc index c43d261926a..266a88d2c3f 100644 --- a/plugin/file_key_management_plugin/EncKeys.cc +++ b/plugin/file_key_management_plugin/EncKeys.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. +/* Copyright (C) 2014 eperi GmbH. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,37 +28,50 @@ file_key_management_plugin_filekey = secret file_key_management_plugin_encryption_method = aes_cbc ... -Optional configuration value file_key_management_plugin_encryption_method determines the method used for encryption. + +Optional configuration value +file_key_management_plugin_encryption_method determines the method +used for encryption. Supported are aes_cbc, aes_ecb or aes_ctr. aes_cbc is default. The plug-in sets the default aes encryption/decryption method to the given method. The keys are read from a file. -The filename is set up via the file_key_management_plugin_filename configuration value. -file_key_management_plugin_filename is used to configure the absolute path to this file. +The filename is set up via the file_key_management_plugin_filename +configuration value. +file_key_management_plugin_filename is used to configure the absolute +path to this file. Examples: file_key_management_plugin_filename = \\\\unc\\keys.enc (windows share) file_key_management_plugin_filename = e:/tmp/keys.enc (windows path) file_key_management_plugin_filename = /tmp/keys.enc (linux path) -The key file contains AES keys and initialization vectors as hex-encoded Strings. +The key file contains AES keys and initialization vectors as +hex-encoded Strings. Supported are keys of size 128, 192 or 256 bits. IV consists of 16 bytes. Example: 1;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 -1 is the key identifier which can be used for table creation, a 16 byte IV follows, and finally a 16 byte AES key. +1 is the key identifier which can be used for table creation, a 16 +byte IV follows, and finally a 16 byte AES key. 255 entries are supported. -The key file should be encrypted and the key to decrypt the file can be given with the -optional file_key_management_plugin_filekey parameter. +The key file should be encrypted and the key to decrypt the file can +be given with the optional file_key_management_plugin_filekey +parameter. + +The file key can also be located if FILE: is prepended to the +key. Then the following part is interpreted as absolute path to the +file containing the file key. This file can optionally be encrypted, +currently with a fix key. -The file key can also be located if FILE: is prepended to the key. Then the following part is interpreted -as absolute path to the file containing the file key. This file can optionally be encrypted, currently with a fix key. Example: + file_key_management_plugin_filekey = FILE:y:/secret256.enc -If the key file can not be read at server startup, for example if the file key is not present, -page_encryption feature is not availabe and access to page_encryption tables is not possible. +If the key file can not be read at server startup, for example if the +file key is not present, page_encryption feature is not availabe and +access to page_encryption tables is not possible. Open SSL command line utility can be used to create an encrypted key file. Examples: @@ -66,14 +79,15 @@ openssl enc –aes-256-cbc –md sha1 –k secret –in keys.txt –out keys.enc openssl enc –aes-256-cbc –md sha1 –k –in secret –out secret.enc Created 09/15/2014 - ***********************************************************************/ +***********************************************************************/ + #ifdef __WIN__ #define PCRE_STATIC 1 #endif -#include "EncKeys.h" #include - +#include /* For sql_print_error */ +#include "EncKeys.h" #include #include #include @@ -81,332 +95,385 @@ openssl enc –aes-256-cbc –md sha1 –k –in secret –out secr #include #include +const char* EncKeys::strMAGIC= "Salted__"; +const int EncKeys::magicSize= 8;//strlen(strMAGIC); // 8 byte +const char* EncKeys::newLine= "\n"; - - -const char* EncKeys::strMAGIC = "Salted__"; -const int EncKeys::magicSize = 8;//strlen(strMAGIC); // 8 byte -const char* EncKeys::newLine = "\n"; - -const char* EncKeys::errorNoKeyId = "KeyID = %u not found or with error. Check the key and the log file.\n"; -const char* EncKeys::errorInMatches = "Wrong match of the keyID in line %u, see the template.\n"; -const char* EncKeys::errorExceedKeyFileSize = "The size of the key file %s exceeds " +const char* EncKeys::errorNoKeyId= "KeyID %u not found or with error. Check the key and the log file.\n"; +const char* EncKeys::errorInMatches= "Wrong match of the keyID in line %u, see the template.\n"; +const char* EncKeys::errorExceedKeyFileSize= "The size of the key file %s exceeds " "the maximum allowed of %u bytes.\n"; -const char* EncKeys::errorExceedKeySize = "The key size exceeds the maximum allowed size of %u in line %u.\n"; -const char* EncKeys::errorEqualDoubleKey = "More than one identical key with keyID = %u found" +const char* EncKeys::errorExceedKeySize= "The key size exceeds the maximum allowed size of %u in line %u.\n"; +const char* EncKeys::errorEqualDoubleKey= "More than one identical key with keyID %u found" " in lines %u and %u.\nDelete one of them in the key file.\n"; -const char* EncKeys::errorUnequalDoubleKey = "More than one not identical key with keyID = %u found" +const char* EncKeys::errorUnequalDoubleKey= "More than one not identical key with keyID %u found" " in lines %u and %u.\nChoose the right one and delete the other in the key file.\n" "I'll take the key from line %u\n"; -const char* EncKeys::errorNoInitializedKey = "The key could not be initialized.\n"; -const char* EncKeys::errorNotImplemented = "Initializing keys through key server is not" +#define errorNoInitializedKey "The key could not be initialized.\n" +const char* EncKeys::errorNotImplemented= "Initializing keys through key server is not" " yet implemented.\nYou can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorOpenFile = "Could not open %s for reading. You can not read encrypted tables or columns.\n\n"; -const char* EncKeys::errorReadingFile = "Could not read from %s. You can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorFileSize = "Could not get the file size from %s. You can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorFalseFileKey = "Wrong encryption / decryption key for keyfile '%s'.\n"; +const char* EncKeys::errorOpenFile= "Could not open %s for reading. You can not read encrypted tables or columns.\n\n"; +const char* EncKeys::errorReadingFile= "Could not read from %s. You can not read encrypted tables or columns\n\n"; +const char* EncKeys::errorFileSize= "Could not get the file size from %s. You can not read encrypted tables or columns\n\n"; +const char* EncKeys::errorFalseFileKey= "Wrong encryption / decryption key for keyfile '%s'.\n"; /* read this from a secret source in some later version */ -const char* EncKeys::initialPwd = "lg28s9ac5ffa537fd8798875c98e190df289da7e047c05"; - -EncKeys::EncKeys() { - countKeys = keyLineInKeyFile = 0; - for (int ii = 0; ii < MAX_KEYS; ii++) { - keys[ii].id = 0; - keys[ii].iv = keys[ii].key = NULL; - } - oneKey = NULL; +const char* EncKeys::initialPwd= "lg28s9ac5ffa537fd8798875c98e190df289da7e047c05"; + +EncKeys::EncKeys() +{ + countKeys= keyLineInKeyFile= 0; + for (int ii= 0; ii < MAX_KEYS; ii++) { + keys[ii].id= 0; + keys[ii].iv= keys[ii].key= NULL; + } + oneKey= NULL; } -EncKeys::~EncKeys() { - for (int ii = MAX_KEYS - 1; ii >= 0 ; ii--) { - delete[] keys[ii].iv; keys[ii].iv = NULL; - delete[] keys[ii].key; keys[ii].key = NULL; - } +EncKeys::~EncKeys() +{ + for (int ii= MAX_KEYS - 1; ii >= 0 ; ii--) { + delete[] keys[ii].iv; + keys[ii].iv= NULL; + delete[] keys[ii].key; + keys[ii].key= NULL; + } } -bool EncKeys::initKeys(const char *filename, const char *filekey) { - if (filename==NULL) - return ERROR_OPEN_FILE; - - const char *MAGIC = "FILE:"; - const short MAGIC_LEN = 5; - - char *secret = (char*) malloc(MAX_SECRET_SIZE +1 * sizeof(char)); - - if(filekey != NULL) - { - //If secret starts with FILE: interpret the secret as filename. - if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) { - int fk_len = strlen(filekey); - char *secretfile = (char*)malloc( (1 + fk_len - MAGIC_LEN)* sizeof(char)); - memcpy(secretfile, filekey+MAGIC_LEN, fk_len - MAGIC_LEN); - secretfile[fk_len-MAGIC_LEN] = '\0'; - parseSecret(secretfile, secret); - free(secretfile); - } else - { - sprintf(secret, "%s", filekey); - } - } - - int ret = parseFile((const char *)filename, 254, secret); - free(secret); - return (ret==NO_ERROR_KEY_FILE_PARSE_OK); + +bool EncKeys::initKeys(const char *filename, const char *filekey) +{ + if (filename==NULL) + return false; + + const char *MAGIC= "FILE:"; + const short MAGIC_LEN= 5; + + char *secret= (char*) malloc(MAX_SECRET_SIZE +1 * sizeof(char)); + + if (filekey != NULL) + { + //If secret starts with FILE: interpret the secret as filename. + if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) + { + int fk_len= strlen(filekey); + char *secretfile= (char*)malloc((1 + fk_len - MAGIC_LEN)* sizeof(char)); + memcpy(secretfile, filekey+MAGIC_LEN, fk_len - MAGIC_LEN); + secretfile[fk_len-MAGIC_LEN]= '\0'; + parseSecret(secretfile, secret); + free(secretfile); + } else + { + sprintf(secret, "%s", filekey); + } + } + + int ret= parseFile((const char *)filename, 254, secret); + free(secret); + return (ret==NO_ERROR_KEY_FILE_PARSE_OK); } + /* - * secret is limited to MAX_SECRET_SIZE characters - */ -void EncKeys::parseSecret( const char *secretfile, char *secret ) { - int maxSize = (MAX_SECRET_SIZE +16 + magicSize*2) ; - char* buf = (char*)malloc((maxSize) * sizeof(char)); - char* _initPwd = (char*)malloc((strlen(initialPwd)+1) * sizeof(char)); - FILE *fp = fopen(secretfile, "rb"); - fseek(fp, 0L, SEEK_END); - long file_size = ftell(fp); - rewind(fp); - int bytes_to_read = (maxSize >= file_size)? file_size:(maxSize); - fread(buf, 1, bytes_to_read, fp); - if (memcmp(buf, strMAGIC, magicSize)) { - bytes_to_read = (bytes_to_read>MAX_SECRET_SIZE) ? MAX_SECRET_SIZE : bytes_to_read; - memcpy(secret, buf, bytes_to_read); - secret[bytes_to_read] = '\0'; - } else { - unsigned char salt[magicSize]; - unsigned char *key = new unsigned char[keySize32]; - unsigned char *iv = new unsigned char[ivSize16]; - memcpy(&salt, buf + magicSize, magicSize); - memcpy(_initPwd, initialPwd, strlen(initialPwd)); - _initPwd[strlen(initialPwd)]= '\0'; - my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); - uint32 d_size = 0; - my_aes_decrypt_dynamic_type func = get_aes_decrypt_func(MY_AES_ALGORITHM_CBC); - int res = (* func)((const uchar*)buf + 2 * magicSize, bytes_to_read - 2 * magicSize, - (uchar*)secret, &d_size, (const uchar*)key, keySize32, iv, ivSize16, 0); - if (d_size>EncKeys::MAX_SECRET_SIZE) { - d_size = EncKeys::MAX_SECRET_SIZE; - } - secret[d_size] = '\0'; - delete[] key; - delete[] iv; - } - free(buf); - free(_initPwd); - fclose(fp); + secret is limited to MAX_SECRET_SIZE characters +*/ + +void EncKeys::parseSecret(const char *secretfile, char *secret) +{ + size_t maxSize= (MAX_SECRET_SIZE +16 + magicSize*2) ; + char* buf= (char*)malloc((maxSize) * sizeof(char)); + char* _initPwd= (char*)malloc((strlen(initialPwd)+1) * sizeof(char)); + FILE *fp= fopen(secretfile, "rb"); + fseek(fp, 0L, SEEK_END); + long file_size= ftell(fp); + rewind(fp); + size_t bytes_to_read= ((maxSize >= (size_t) file_size) ? (size_t) file_size : + maxSize); + bytes_to_read= fread(buf, 1, bytes_to_read, fp); + if (memcmp(buf, strMAGIC, magicSize)) + { + bytes_to_read= (bytes_to_read>MAX_SECRET_SIZE) ? MAX_SECRET_SIZE : + bytes_to_read; + memcpy(secret, buf, bytes_to_read); + secret[bytes_to_read]= '\0'; + } + else + { + unsigned char salt[magicSize]; + unsigned char *key= new unsigned char[keySize32]; + unsigned char *iv= new unsigned char[ivSize16]; + memcpy(&salt, buf + magicSize, magicSize); + memcpy(_initPwd, initialPwd, strlen(initialPwd)); + _initPwd[strlen(initialPwd)]= '\0'; + my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); + uint32 d_size= 0; + my_aes_decrypt_dynamic_type func= get_aes_decrypt_func(MY_AES_ALGORITHM_CBC); + int re= (* func)((const uchar*)buf + 2 * magicSize, + bytes_to_read - 2 * magicSize, + (uchar*)secret, &d_size, (const uchar*)key, keySize32, + iv, ivSize16, 0); + if (re) + d_size= 0; + if (d_size>EncKeys::MAX_SECRET_SIZE) + { + d_size= EncKeys::MAX_SECRET_SIZE; + } + secret[d_size]= '\0'; + delete[] key; + delete[] iv; + } + free(buf); + free(_initPwd); + fclose(fp); } + /** * Returns a struct keyentry with the asked 'id' or NULL. */ -keyentry *EncKeys::getKeys(int id) { - if (KEY_MIN <= id && KEY_MAX >= id && (&keys[id - 1])->iv) - { - return &keys[id - 1]; - } -#ifdef UNIV_DEBUG - else { - - fprintf(stderr, errorNoKeyId, id); - return NULL; - } -#endif //UNIV_DEBUG +keyentry *EncKeys::getKeys(int id) +{ + if (KEY_MIN <= id && KEY_MAX >= id && (&keys[id - 1])->iv) + { + return &keys[id - 1]; + } +#ifndef DBUG_OFF + else + { + sql_print_error(errorNoKeyId, id); + } +#endif + return NULL; } /** - * Get the keys from the key file and decrypt it with the key . - * Store the keys with id smaller then in an array of structs keyentry. - * Returns NO_ERROR_PARSE_OK or an appropriate error code. + Get the keys from the key file and decrypt it with the + key . Store the keys with id smaller then in an + array of structs keyentry. Returns NO_ERROR_PARSE_OK or an + appropriate error code. */ -int EncKeys::parseFile(const char* filename, const uint32 maxKeyId, const char *secret) { - int errorCode = 0; - char *buffer = decryptFile(filename, secret, &errorCode); - uint32 id = 0; - - if (NO_ERROR_PARSE_OK != errorCode) return errorCode; - else errorCode = NO_ERROR_KEY_FILE_PARSE_OK; - - char *line = strtok(buffer, newLine); - while ( NULL != line) { - keyLineInKeyFile++; - switch (parseLine(line, maxKeyId)) { - case NO_ERROR_PARSE_OK: - id = oneKey->id; - keys[oneKey->id - 1] = *oneKey; - delete(oneKey); - countKeys++; - fprintf(stderr, "Line: %u --> ", keyLineInKeyFile); printKeyEntry(id); - break; - case ERROR_ID_TOO_BIG: - fprintf(stderr, errorExceedKeySize, KEY_MAX, keyLineInKeyFile); - fprintf(stderr, " ---> %s\n", line); - errorCode = ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS; - break; - case ERROR_NOINITIALIZEDKEY: - fprintf(stderr, errorNoInitializedKey); - fprintf(stderr, " ----> %s\n", line); - errorCode = ERROR_KEY_FILE_PARSE_NULL; - break; - case ERROR_WRONG_NUMBER_OF_MATCHES: - fprintf(stderr, errorInMatches, keyLineInKeyFile); - fprintf(stderr, " -----> %s\n", line); - errorCode = ERROR_KEY_FILE_PARSE_NULL; - break; - case NO_ERROR_KEY_GREATER_THAN_ASKED: - fprintf(stderr, "No asked key in line %lu: %s\n", keyLineInKeyFile, line); - break; - case NO_ERROR_ISCOMMENT: - fprintf(stderr, "Is comment in line %lu: %s\n", keyLineInKeyFile, line); - default: - break; - } - line = strtok(NULL, newLine); - } - - free(line); line = NULL; - delete[] buffer; buffer = NULL; - return errorCode; + +int EncKeys::parseFile(const char* filename, const uint32 maxKeyId, + const char *secret) +{ + int errorCode= 0; + char *buffer= decryptFile(filename, secret, &errorCode); + uint32 id= 0; + + if (errorCode != NO_ERROR_PARSE_OK) + return errorCode; + errorCode= NO_ERROR_KEY_FILE_PARSE_OK; + + char *line= strtok(buffer, newLine); + while (NULL != line) + { + keyLineInKeyFile++; + switch (parseLine(line, maxKeyId)) { + case NO_ERROR_PARSE_OK: + id= oneKey->id; + keys[oneKey->id - 1]= *oneKey; + delete(oneKey); + countKeys++; + break; + case ERROR_ID_TOO_BIG: + sql_print_error(errorExceedKeySize, KEY_MAX, + keyLineInKeyFile); + sql_print_error(" ---> %s\n", line); + errorCode= ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS; + break; + case ERROR_NOINITIALIZEDKEY: + sql_print_error(errorNoInitializedKey); + sql_print_error(" ----> %s\n", line); + errorCode= ERROR_KEY_FILE_PARSE_NULL; + break; + case ERROR_WRONG_NUMBER_OF_MATCHES: + sql_print_error(errorInMatches, keyLineInKeyFile); + sql_print_error(" -----> %s\n", line); + errorCode= ERROR_KEY_FILE_PARSE_NULL; + break; + case NO_ERROR_KEY_GREATER_THAN_ASKED: + sql_print_error("No asked key in line %u: %s\n", + keyLineInKeyFile, line); + break; + case NO_ERROR_ISCOMMENT: + sql_print_error("Is comment in line %u: %s\n", + keyLineInKeyFile, line); + default: + break; + } + line= strtok(NULL, newLine); + } + + free(line); + line= NULL; + delete[] buffer; + buffer= NULL; + return errorCode; } -int EncKeys::parseLine(const char *line, const uint32 maxKeyId) { - int ret = NO_ERROR_PARSE_OK; - if (isComment(line)) - ret = NO_ERROR_ISCOMMENT; - else { - const char *error_p = NULL; - int offset; - pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", - 0, &error_p, &offset, NULL); - if ( NULL != error_p) - fprintf(stderr, "Error: %s\nOffset: %d\n", error_p, offset); - - int m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; - int rc = pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); - pcre_free(pattern); - if (4 == rc) { - char lin[MAX_KEY_LINE_SIZE + 1]; - strncpy( lin, line, MAX_KEY_LINE_SIZE); - lin[MAX_KEY_LINE_SIZE] = '\0'; - char *substring_start = lin + ovector[2]; - int substr_length = ovector[3] - ovector[2]; - if (3 < substr_length) - ret = ERROR_ID_TOO_BIG; - else { - char buffer[4]; - sprintf(buffer, "%.*s", substr_length, substring_start); - uint32 id = atoi(buffer); - if (0 == id) ret = ERROR_NOINITIALIZEDKEY; - else if (KEY_MAX < id) ret = ERROR_ID_TOO_BIG; - else if (maxKeyId < id) ret = NO_ERROR_KEY_GREATER_THAN_ASKED; - else { - oneKey = new keyentry; - oneKey->id = id; - substring_start = lin + ovector[4]; - substr_length = ovector[5] - ovector[4]; - oneKey->iv = new char[substr_length + 1]; - sprintf(oneKey->iv, "%.*s", substr_length, substring_start); - substring_start = lin + ovector[6]; - substr_length = ovector[7] - ovector[6]; - oneKey->key = new char[substr_length + 1]; - sprintf(oneKey->key, "%.*s", substr_length, substring_start); - } - } - } - else - ret = ERROR_WRONG_NUMBER_OF_MATCHES; - } - return ret; + +int EncKeys::parseLine(const char *line, const uint32 maxKeyId) +{ + int ret= NO_ERROR_PARSE_OK; + if (isComment(line)) + ret= NO_ERROR_ISCOMMENT; + else + { + const char *error_p= NULL; + int offset; + pcre *pattern= pcre_compile( + "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", + 0, &error_p, &offset, NULL); + if (NULL != error_p) + sql_print_error("Error: %s\nOffset: %d\n", error_p, offset); + + int m_len= (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; + int rc= pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, + MAX_OFFSETS_IN_PCRE_PATTERNS); + pcre_free(pattern); + if (4 == rc) + { + char lin[MAX_KEY_LINE_SIZE + 1]; + strncpy(lin, line, MAX_KEY_LINE_SIZE); + lin[MAX_KEY_LINE_SIZE]= '\0'; + char *substring_start= lin + ovector[2]; + int substr_length= ovector[3] - ovector[2]; + if (3 < substr_length) + ret= ERROR_ID_TOO_BIG; + else + { + char buffer[4]; + sprintf(buffer, "%.*s", substr_length, substring_start); + uint32 id= atoi(buffer); + if (0 == id) ret= ERROR_NOINITIALIZEDKEY; + else if (KEY_MAX < id) ret= ERROR_ID_TOO_BIG; + else if (maxKeyId < id) ret= NO_ERROR_KEY_GREATER_THAN_ASKED; + else + { + oneKey= new keyentry; + oneKey->id= id; + substring_start= lin + ovector[4]; + substr_length= ovector[5] - ovector[4]; + oneKey->iv= new char[substr_length + 1]; + sprintf(oneKey->iv, "%.*s", substr_length, substring_start); + substring_start= lin + ovector[6]; + substr_length= ovector[7] - ovector[6]; + oneKey->key= new char[substr_length + 1]; + sprintf(oneKey->key, "%.*s", substr_length, substring_start); + } + } + } + else + ret= ERROR_WRONG_NUMBER_OF_MATCHES; + } + return ret; } /** - * Decrypt the key file 'filename' if it is encrypted with the key 'secret'. - * Store the content of the decrypted file in 'buffer'. The buffer has to be freed - * in the calling function. + Decrypt the key file 'filename' if it is encrypted with the key + 'secret'. Store the content of the decrypted file in 'buffer'. The + buffer has to be freed in the calling function. */ -char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorCode) { - *errorCode = NO_ERROR_PARSE_OK; - fprintf(stderr, "Reading %s\n\n", filename); - FILE *fp = fopen(filename, "rb"); - if (NULL == fp) { - fprintf(stderr, errorOpenFile, filename); - *errorCode = ERROR_OPEN_FILE; - return NULL; - } - - if (fseek(fp, 0L, SEEK_END)) { - *errorCode = ERROR_READING_FILE; - return NULL; - } - long file_size = ftell(fp); // get the file size - if (MAX_KEY_FILE_SIZE < file_size) { - fprintf(stderr, errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); - *errorCode = ERROR_KEY_FILE_TOO_BIG; - fclose(fp); - return NULL; - } - else if (-1L == file_size) { - fprintf(stderr, errorFileSize, filename); - *errorCode = ERROR_READING_FILE; - return NULL; - } - - rewind(fp); - //Read file into buffer - uchar *buffer = new uchar[file_size + 1]; - size_t read_bytes = fread(buffer, 1, file_size, fp); - buffer[file_size] = '\0'; - fclose(fp); - //Check for file encryption - if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. - unsigned char salt[magicSize]; - unsigned char *key = new unsigned char[keySize32]; - unsigned char *iv = new unsigned char[ivSize16]; - uchar *decrypted = new uchar[file_size]; - memcpy(&salt, buffer + magicSize, magicSize); - my_bytes_to_key((unsigned char *) salt, secret, key, iv); - uint32 d_size = 0; - my_aes_decrypt_dynamic_type func = get_aes_decrypt_func(MY_AES_ALGORITHM_CBC); - int res = (* func)((const uchar*)buffer + 2 * magicSize, file_size - 2 * magicSize, - decrypted, &d_size, (const uchar*) key, keySize32, iv, ivSize16, 0); - if(0 != res) { - *errorCode = ERROR_FALSE_FILE_KEY; - delete[] buffer; buffer = NULL; - fprintf(stderr, errorFalseFileKey, filename); - } - else { - memcpy(buffer, decrypted, d_size); - buffer[d_size] = '\0'; - } - - delete[] decrypted; decrypted = NULL; - delete[] key; key = NULL; - delete[] iv; iv = NULL; - } - return (char*) buffer; + +char* EncKeys::decryptFile(const char* filename, const char *secret, + int *errorCode) +{ + *errorCode= NO_ERROR_PARSE_OK; + FILE *fp= fopen(filename, "rb"); + if (NULL == fp) + { + sql_print_error(errorOpenFile, filename); + *errorCode= ERROR_OPEN_FILE; + return NULL; + } + + if (fseek(fp, 0L, SEEK_END)) + { + *errorCode= ERROR_READING_FILE; + return NULL; + } + long file_size= ftell(fp); // get the file size + if (MAX_KEY_FILE_SIZE < file_size) + { + sql_print_error(errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); + *errorCode= ERROR_KEY_FILE_TOO_BIG; + fclose(fp); + return NULL; + } + else if (-1L == file_size) + { + sql_print_error(errorFileSize, filename); + *errorCode= ERROR_READING_FILE; + return NULL; + } + + rewind(fp); + //Read file into buffer + uchar *buffer= new uchar[file_size + 1]; + file_size= fread(buffer, 1, file_size, fp); + buffer[file_size]= '\0'; + fclose(fp); + //Check for file encryption + if (0 == memcmp(buffer, strMAGIC, magicSize)) + { + //If file is encrypted, decrypt it first. + unsigned char salt[magicSize]; + unsigned char *key= new unsigned char[keySize32]; + unsigned char *iv= new unsigned char[ivSize16]; + uchar *decrypted= new uchar[file_size]; + memcpy(&salt, buffer + magicSize, magicSize); + my_bytes_to_key((unsigned char *) salt, secret, key, iv); + uint32 d_size= 0; + my_aes_decrypt_dynamic_type func= get_aes_decrypt_func(MY_AES_ALGORITHM_CBC); + int res= (* func)((const uchar*)buffer + 2 * magicSize, + file_size - 2 * magicSize, + decrypted, &d_size, (const uchar*) key, keySize32, + iv, ivSize16, 0); + if(0 != res) + { + *errorCode= ERROR_FALSE_FILE_KEY; + delete[] buffer; buffer= NULL; + sql_print_error(errorFalseFileKey, filename); + } + else + { + memcpy(buffer, decrypted, d_size); + buffer[d_size]= '\0'; + } + + delete[] decrypted; decrypted= NULL; + delete[] key; key= NULL; + delete[] iv; iv= NULL; + } + return (char*) buffer; } -bool EncKeys::isComment(const char *line) { - const char *error_p; - int offset, m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; - pcre *pattern = pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); - int rc = pcre_exec( pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); - pcre_free(pattern); - if (0 > rc) return false; - else return true; +bool EncKeys::isComment(const char *line) +{ + const char *error_p; + int offset, m_len= (int) strlen(line), + ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; + pcre *pattern= pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); + int rc= pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, + MAX_OFFSETS_IN_PCRE_PATTERNS); + pcre_free(pattern); + return (rc >= 0); } -void EncKeys::printKeyEntry( uint32 id) +void EncKeys::printKeyEntry(uint32 id) { -#ifdef UNIV_DEBUG - keyentry *entry = getKeys(id); - if( NULL == entry) { - fprintf(stderr, "No such keyID=%lu\n",id); - } - else { - fprintf(stderr, "Key: id:%3lu \tiv:%lu bytes\tkey:%lu bytes\n", entry->id, strlen(entry->iv)/2, strlen(entry->key)/2); - } -#endif //UNIV_DEBUG +#ifndef DBUG_OFF + keyentry *entry= getKeys(id); + if (NULL == entry) + { + sql_print_error("No such keyID: %u\n",id); + } + else + { + sql_print_error("Key: id: %3u\tiv:%lu bytes\tkey:%lu bytes\n", + entry->id, strlen(entry->iv)/2, strlen(entry->key)/2); + } +#endif /* DBUG_OFF */ } diff --git a/plugin/file_key_management_plugin/EncKeys.h b/plugin/file_key_management_plugin/EncKeys.h index c2b56047a6e..c0ab98b8e9d 100644 --- a/plugin/file_key_management_plugin/EncKeys.h +++ b/plugin/file_key_management_plugin/EncKeys.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. +/* Copyright (C) 2014 eperi GmbH. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/plugin/file_key_management_plugin/KeySingleton.cc b/plugin/file_key_management_plugin/KeySingleton.cc index 891fe4335f9..936a7cf71a9 100644 --- a/plugin/file_key_management_plugin/KeySingleton.cc +++ b/plugin/file_key_management_plugin/KeySingleton.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. +/* Copyright (C) 2014 eperi GmbH. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,44 +21,44 @@ Created 09/13/2014 ***********************************************************************/ +#include +#include #include "KeySingleton.h" -#include - bool KeySingleton::instanceInited = false; KeySingleton KeySingleton::theInstance; EncKeys KeySingleton::encKeys; - - -KeySingleton & KeySingleton::getInstance() { -#ifdef UNIV_DEBUG - if( !instanceInited) { - fprintf(stderr, "Encryption / decryption keys were not initialized. " - "You can not read encrypted tables or columns\n"); - } -#endif UNIV_DEBUG - return theInstance; +KeySingleton & KeySingleton::getInstance() +{ +#ifndef DBUG_OFF + if( !instanceInited) + { + sql_print_error("Encryption / decryption keys were not initialized. " + "You can not read encrypted tables or columns\n"); + } +#endif /* DBUG_OFF */ + return theInstance; } -KeySingleton & KeySingleton::getInstance(const char *filename, const char *filekey) { - - if(instanceInited) return theInstance; - instanceInited = encKeys.initKeys(filename, filekey); - if( !instanceInited) { - fprintf(stderr, "Could not initialize any of the encryption / decryption keys. " - "You can not read encrypted tables\n\n"); - fflush(stderr); - } - - return theInstance; +KeySingleton & KeySingleton::getInstance(const char *filename, + const char *filekey) +{ + if (!instanceInited) + { + if (!(instanceInited = encKeys.initKeys(filename, filekey))) + sql_print_error("Could not initialize any of the encryption / " + "decryption keys. You can not read encrypted tables"); + } + return theInstance; } -keyentry *KeySingleton::getKeys(int id) { - return encKeys.getKeys(id); +keyentry *KeySingleton::getKeys(int id) +{ + return encKeys.getKeys(id); } -bool KeySingleton::hasKey(int id) { - return encKeys.getKeys(id) != NULL; +bool KeySingleton::hasKey(int id) +{ + return encKeys.getKeys(id) != NULL; } - diff --git a/plugin/file_key_management_plugin/KeySingleton.h b/plugin/file_key_management_plugin/KeySingleton.h index d66ff44df27..37246c1b802 100644 --- a/plugin/file_key_management_plugin/KeySingleton.h +++ b/plugin/file_key_management_plugin/KeySingleton.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. +/* Copyright (C) 2014 eperi GmbH. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/plugin/file_key_management_plugin/file_key_management_plugin.cc b/plugin/file_key_management_plugin/file_key_management_plugin.cc index ce0d3aea208..11892d74385 100644 --- a/plugin/file_key_management_plugin/file_key_management_plugin.cc +++ b/plugin/file_key_management_plugin/file_key_management_plugin.cc @@ -18,23 +18,13 @@ #include #include #include -#include #include "sql_class.h" - -//#include "file_key_management_plugin.h" #include "KeySingleton.h" #include "EncKeys.h" - /* Encryption for tables and columns */ static char* filename = NULL; static char* filekey = NULL; -static char* encryption_method = NULL; - -static MYSQL_SYSVAR_STR(encryption_method, encryption_method, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Name of the encryption method. This can be aes_cbc (default) or aes_ctr (requires OpenSSL on Linux).", - NULL, NULL, "aes_cbc"); static MYSQL_SYSVAR_STR(filename, filename, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -47,7 +37,6 @@ static MYSQL_SYSVAR_STR(filekey, filekey, NULL, NULL, NULL); static struct st_mysql_sys_var* settings[] = { - MYSQL_SYSVAR(encryption_method), MYSQL_SYSVAR(filename), MYSQL_SYSVAR(filekey), NULL @@ -56,9 +45,11 @@ static struct st_mysql_sys_var* settings[] = { /** - * This method is using with the id 0 if exists. This method is used by innobase/xtradb for the key - * rotation feature of encrypting log files. - */ + This method is using with the id 0 if exists. + This method is used by innobase/xtradb for the key + rotation feature of encrypting log files. +*/ + static int get_highest_key_used_in_key_file() { if (KeySingleton::getInstance().hasKey(0)) @@ -93,7 +84,8 @@ static int get_key_size_from_key_file(unsigned int keyID) } } -static int get_key_from_key_file(unsigned int keyID, unsigned char* dstbuf, unsigned buflen) +static int get_key_from_key_file(unsigned int keyID, unsigned char* dstbuf, + unsigned buflen) { keyentry* entry = KeySingleton::getInstance().getKeys((int)keyID); @@ -117,7 +109,8 @@ static int get_key_from_key_file(unsigned int keyID, unsigned char* dstbuf, unsi } } -static int get_iv_from_key_file(unsigned int keyID, unsigned char* dstbuf, unsigned buflen) +static int get_iv_from_key_file(unsigned int keyID, unsigned char* dstbuf, + unsigned buflen) { keyentry* entry = KeySingleton::getInstance().getKeys((int)keyID); @@ -145,22 +138,14 @@ static int get_iv_from_key_file(unsigned int keyID, unsigned char* dstbuf, unsig static int file_key_management_plugin_init(void *p) { /* init */ - printf("@DE: Loading file_key_management_plugin_init \n"); - - /* Setting encryption method */ - if (encryption_method != NULL && strcmp("aes_ctr", encryption_method) == 0) + + if (current_aes_dynamic_method == MY_AES_ALGORITHM_NONE) { - my_aes_init_dynamic_encrypt(MY_AES_ALGORITHM_CTR); - } else if (encryption_method != NULL && strcmp("aes_ecb", encryption_method) == 0) - { - my_aes_init_dynamic_encrypt(MY_AES_ALGORITHM_ECB); - } - else - { - my_aes_init_dynamic_encrypt(MY_AES_ALGORITHM_CBC); + sql_print_error("No encryption method choosen with --encryption-algorithm. " + "file_key_management_plugin disabled"); + return 1; } - /* Initializing the key provider */ struct CryptoKeyFuncs_t func; func.getLatestCryptoKeyVersionFunc = get_highest_key_used_in_key_file; @@ -173,10 +158,10 @@ static int file_key_management_plugin_init(void *p) if (filename == NULL || strcmp("", filename) == 0) { - sql_print_error("Parameter file_key_management_plugin_filename is required"); + sql_print_error("Parameter file_key_management_plugin_filename is required"); - return 1; - } + return 1; + } KeySingleton::getInstance(filename, filekey); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 1d41d669979..da02b952f3c 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -629,7 +629,7 @@ char *mysqld_unix_port, *opt_mysql_tmpdir; ulong thread_handling; my_bool encrypt_tmp_disk_tables; -ulong encrypt_algorithm; +ulong encryption_algorithm; /** name of reference on left expression in rewritten IN subquery */ const char *in_left_expr_name= ""; @@ -4802,10 +4802,10 @@ static int init_server_components() setup_fpu(); init_thr_lock(); if (my_aes_init_dynamic_encrypt((enum_my_aes_encryption_algorithm) - encrypt_algorithm)) + encryption_algorithm)) { fprintf(stderr, "Can't initialize encryption algorithm to \"%s\".\nCheck that the program is linked with the right library (openssl?)\n", - encrypt_algorithm_names[encrypt_algorithm]); + encryption_algorithm_names[encryption_algorithm]); unireg_abort(1); } diff --git a/sql/mysqld.h b/sql/mysqld.h index 98eb5dbb2c1..71d586cc4c1 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -250,8 +250,8 @@ extern ulong connection_errors_max_connection; extern ulong connection_errors_peer_addr; extern ulong log_warnings; extern my_bool encrypt_tmp_disk_tables; -extern ulong encrypt_algorithm; -extern const char *encrypt_algorithm_names[]; +extern ulong encryption_algorithm; +extern const char *encryption_algorithm_names[]; /* THR_MALLOC is a key which will be used to set/get MEM_ROOT** for a thread, diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 6a5aa068c37..1ee443cfb83 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -2777,15 +2777,6 @@ bool change_password(THD *thd, LEX_USER *user) */ save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); - /* - This statement will be replicated as a statement, even when using - row-based replication. The flag will be reset at the end of the - statement. - This has to be handled here as it's called by set_var.cc, which is - not automaticly handled by sql_parse.cc - */ - save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); - if (mysql_bin_log.is_open() || (WSREP(thd) && !IF_WSREP(thd->wsrep_applier, 0))) { diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index d5dcbb5b1e1..8fce66ae19f 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -176,13 +176,9 @@ static struct - yet disable explicitly a component needed for the functionality to work, by using '--skip-performance-schema' (the plugin) */ - { "performance_schema", PLUGIN_FORCE }, + { "performance_schema", PLUGIN_FORCE } /* we disable few other plugins by default */ - /* TODO: Make it possible to default disable built-in - plugins wo/ editing this file. */ - { "example_key_management_plugin", PLUGIN_OFF} - //,{ "feedback", PLUGIN_OFF } }; @@ -1237,6 +1233,11 @@ static void reap_plugins(void) mysql_mutex_unlock(&LOCK_plugin); + /* + First free all normal plugins, last the key management plugin. + This is becasue the storage engines may need the key management plugin + during deinitialization. + */ list= reap; while ((plugin= *(--list))) { @@ -1247,7 +1248,7 @@ static void reap_plugins(void) list= reap; while ((plugin= *(--list))) { - if (plugin->plugin->type == MYSQL_KEY_MANAGEMENT_PLUGIN) + if (plugin->state != PLUGIN_IS_UNINITIALIZED) plugin_deinitialize(plugin, true); } @@ -1495,7 +1496,7 @@ static void init_plugin_psi_keys(void) */ int plugin_init(int *argc, char **argv, int flags) { - uint i; + uint i,j; bool is_myisam; struct st_maria_plugin **builtins; struct st_maria_plugin *plugin; @@ -1549,65 +1550,6 @@ int plugin_init(int *argc, char **argv, int flags) /* First we register builtin plugins */ - mandatory = false; - for (builtins= mysql_optional_plugins; *builtins; builtins++) - { - for (plugin= *builtins; plugin->info; plugin++) - { - if (opt_ignore_builtin_innodb && - !my_strnncoll(&my_charset_latin1, (const uchar*) plugin->name, - 6, (const uchar*) "InnoDB", 6)) - continue; - - if (plugin->type != MYSQL_KEY_MANAGEMENT_PLUGIN) - continue; - - bzero(&tmp, sizeof(tmp)); - tmp.plugin= plugin; - tmp.name.str= (char *)plugin->name; - tmp.name.length= strlen(plugin->name); - tmp.state= 0; - tmp.load_option= mandatory ? PLUGIN_FORCE : PLUGIN_ON; - - for (i=0; i < array_elements(override_plugin_load_policy); i++) - { - if (!my_strcasecmp(&my_charset_latin1, plugin->name, - override_plugin_load_policy[i].plugin_name)) - { - tmp.load_option= override_plugin_load_policy[i].override; - break; - } - } - - free_root(&tmp_root, MYF(MY_MARK_BLOCKS_FREE)); - tmp.state= PLUGIN_IS_UNINITIALIZED; - if (register_builtin(plugin, &tmp, &plugin_ptr)) - goto err_unlock; - - is_myisam= !my_strcasecmp(&my_charset_latin1, plugin->name, "MyISAM"); - - if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, !is_myisam && - (flags & PLUGIN_INIT_SKIP_INITIALIZATION))) - { - if (plugin_ptr->load_option == PLUGIN_FORCE) - goto err_unlock; - plugin_ptr->state= PLUGIN_IS_DISABLED; - } - - if (is_myisam) - { - DBUG_ASSERT(!global_system_variables.table_plugin); - global_system_variables.table_plugin= - intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); - DBUG_ASSERT(plugin_ptr->ref_count == 1); - } - } - } - - /* - First we register builtin plugins - */ - mandatory = true; for (builtins= mysql_mandatory_plugins; *builtins || mandatory; builtins++) { if (!*builtins) @@ -1624,9 +1566,6 @@ int plugin_init(int *argc, char **argv, int flags) 6, (const uchar*) "InnoDB", 6)) continue; - if (plugin->type == MYSQL_KEY_MANAGEMENT_PLUGIN) - continue; - bzero(&tmp, sizeof(tmp)); tmp.plugin= plugin; tmp.name.str= (char *)plugin->name; @@ -1707,16 +1646,22 @@ int plugin_init(int *argc, char **argv, int flags) reap= (st_plugin_int **) my_alloca((plugin_array.elements+1) * sizeof(void*)); *(reap++)= NULL; - for (i= 0; i < plugin_array.elements; i++) + /* first MYSQL_KEY_MANAGEMENT_PLUGIN, then the rest */ + for (j= 0 ; j <= 1; j++) { - plugin_ptr= *dynamic_element(&plugin_array, i, struct st_plugin_int **); - if (plugin_ptr->plugin_dl && plugin_ptr->state == PLUGIN_IS_UNINITIALIZED) + for (i= 0; i < plugin_array.elements; i++) { - if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, - (flags & PLUGIN_INIT_SKIP_INITIALIZATION))) + plugin_ptr= *dynamic_element(&plugin_array, i, struct st_plugin_int **); + if (((j == 0 && plugin->type == MYSQL_KEY_MANAGEMENT_PLUGIN) || j > 0) && + plugin_ptr->plugin_dl && + plugin_ptr->state == PLUGIN_IS_UNINITIALIZED) { - plugin_ptr->state= PLUGIN_IS_DYING; - *(reap++)= plugin_ptr; + if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, + (flags & PLUGIN_INIT_SKIP_INITIALIZATION))) + { + plugin_ptr->state= PLUGIN_IS_DYING; + *(reap++)= plugin_ptr; + } } } } diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h index 9cfdba1edcd..2bfdcb29dfc 100644 --- a/sql/sql_plugin.h +++ b/sql/sql_plugin.h @@ -78,8 +78,6 @@ typedef struct st_mysql_show_var SHOW_VAR; #define PLUGIN_IS_DYING 16 #define PLUGIN_IS_DISABLED 32 -/* A handle for the dynamic library containing a plugin or plugins. */ - struct st_ptr_backup { void **ptr; void *value; @@ -88,6 +86,8 @@ struct st_ptr_backup { void restore() { *ptr= value; } }; +/* A handle for the dynamic library containing a plugin or plugins. */ + struct st_plugin_dl { LEX_STRING dl; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index daa34433236..9d85c2646be 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -16661,8 +16661,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List &fields, { if (share->db_type() == TMP_ENGINE_HTON) { - if (create_internal_tmp_table(table, param->keyinfo, - param->start_recinfo, + if (create_internal_tmp_table(table, param->keyinfo, param->start_recinfo, ¶m->recinfo, select_options)) goto err; } @@ -17085,8 +17084,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, TMP_ENGINE_COLUMNDEF *start_recinfo, TMP_ENGINE_COLUMNDEF **recinfo, - ulonglong options, - my_bool encrypt) + ulonglong options) { int error; MI_KEYDEF keydef; @@ -17252,7 +17250,7 @@ create_internal_tmp_table_from_heap(THD *thd, TABLE *table, if (create_internal_tmp_table(&new_table, table->key_info, start_recinfo, recinfo, thd->lex->select_lex.options | - thd->variables.option_bits)) + thd->variables.option_bits)) goto err2; if (open_tmp_table(&new_table)) goto err1; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 616d2c60d41..e7092354606 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1127,21 +1127,22 @@ static Sys_var_mybool Sys_log_bin( #ifndef DBUG_OFF -static Sys_var_mybool Sys_danger_danger_use_dbug_keys( - "danger_danger_use_dbug_keys", - "Enable use of nonrandom keys for crypto", - READ_ONLY GLOBAL_VAR(opt_danger_danger_use_dbug_keys), +static Sys_var_mybool Sys_debug_use_static_keys( + "debug_use_static_encryption_keys", + "Enable use of nonrandom encryption keys. Only to be used in " + "internal testing", + READ_ONLY GLOBAL_VAR(debug_use_static_encryption_keys), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); -static PolyLock_rwlock PLock_sys_danger_danger_dbug_crypto_key_version( - &LOCK_dbug_crypto_key_version); +static PolyLock_rwlock PLock_sys_debug_encryption_key_version( + &LOCK_dbug_encryption_key_version); -static Sys_var_uint Sys_danger_danger_dbug_crypto_key_version( - "danger_danger_dbug_crypto_key_version", - "Crypto key version for debugging only", - GLOBAL_VAR(opt_danger_danger_dbug_crypto_key_version), +static Sys_var_uint Sys_debug_encryption_key_version( + "debug_encryption_key_version", + "Encryption key version. Only to be used in internal testing.", + GLOBAL_VAR(opt_debug_encryption_key_version), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0,UINT_MAX), DEFAULT(0), - BLOCK_SIZE(1), &PLock_sys_danger_danger_dbug_crypto_key_version); + BLOCK_SIZE(1), &PLock_sys_debug_encryption_key_version); #endif static Sys_var_mybool Sys_trust_function_creators( @@ -5159,13 +5160,13 @@ static Sys_var_mybool Sys_encrypt_tmp_disk_tables( GLOBAL_VAR(encrypt_tmp_disk_tables), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); -const char *encrypt_algorithm_names[]= +const char *encryption_algorithm_names[]= { "none", "aes_ecb", "aes_cbc", "aes_ctr", 0 }; -static Sys_var_enum Sys_encrypt_algorithm( - "encrypt_algorithm", - "Which encryption algorithm to use for table encryption", - READ_ONLY GLOBAL_VAR(encrypt_algorithm),CMD_LINE(REQUIRED_ARG), - encrypt_algorithm_names, DEFAULT(0)); +static Sys_var_enum Sys_encryption_algorithm( + "encryption_algorithm", + "Which encryption algorithm to use for table encryption. aes_cbc is the recommended one.", + READ_ONLY GLOBAL_VAR(encryption_algorithm),CMD_LINE(REQUIRED_ARG), + encryption_algorithm_names, DEFAULT(0)); static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var) { diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 3693620d394..e3e1e70feb7 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -30,7 +30,6 @@ MYSQL_CHECK_BZIP2() # OS tests IF(UNIX) - IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") CHECK_INCLUDE_FILES (libaio.h HAVE_LIBAIO_H) CHECK_LIBRARY_EXISTS(aio io_queue_init "" HAVE_LIBAIO) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ba7441f928c..8b9f5a49e7d 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -5659,7 +5659,6 @@ buf_page_encrypt_before_write( byte *dst_frame = bpage->crypt_buf; if (!fil_space_is_page_compressed(bpage->space)) { - fprintf(stderr, "JAN: problem:::\n"); // encrypt page content fil_space_encrypt(bpage->space, bpage->offset, bpage->newest_modification, @@ -5710,38 +5709,28 @@ Allocates memory to read in an encrypted page byte* buf_page_decrypt_before_read( /*=========================*/ - buf_page_t* bpage) /*!< in/out: buffer page to be read */ + buf_page_t* bpage, /*!< in/out: buffer page to be read */ + ulint zip_size) /*!< in: compressed page size, or 0 */ { - ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - if (bpage->offset == 0) { - /* File header pages are not encrypted */ -unencrypted: - if (zip_size) - return bpage->zip.data; - else - return ((buf_block_t*) bpage)->frame; - } - - if (fil_space_check_encryption_read(bpage->space) == false) { - goto unencrypted; - } - - if (srv_encrypt_tables) { - if (bpage->crypt_buf_free == NULL) { - // allocate buffer to read data into - bpage->crypt_buf_free = (byte*)malloc(size*2); - // TODO: Is 4K aligment enough ? - bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size); - } - - return bpage->crypt_buf; - } else { - // If database contains encrypted data it will be - // handled later. - goto unencrypted; - } + /* + Here we only need to allocate space for not header pages + in case of file space encryption. Table encryption is handled + later. + */ + if (!srv_encrypt_tables || bpage->offset == 0 || + fil_space_check_encryption_read(bpage->space) == false) + return zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; + + if (bpage->crypt_buf_free == NULL) + { + // allocate buffer to read data into + bpage->crypt_buf_free = (byte*)malloc(size*2); + // TODO: Is 4K aligment enough ? + bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size); + } + return bpage->crypt_buf; } /********************************************************************//** @@ -5760,7 +5749,6 @@ buf_page_decrypt_after_read( ((buf_block_t*) bpage)->frame; if (bpage->offset == 0) { - fprintf(stderr, "JAN: page type %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE)); /* File header pages are not encrypted */ ut_a(bpage->crypt_buf == NULL); return (TRUE); @@ -5775,12 +5763,7 @@ buf_page_decrypt_after_read( bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - fprintf(stderr, "JAN: key_version %u, page_type %lu\n", key_version, - mach_read_from_2(src_frame+FIL_PAGE_TYPE)); - if (key_version == 0) { - fprintf(stderr, "JAN: unencrypted, dst %p src %p\n", dst_frame, src_frame); - /* the page we read is unencrypted */ if (dst_frame != src_frame) { /* but we had allocated a crypt_buf */ @@ -5799,13 +5782,10 @@ buf_page_decrypt_after_read( memcpy(bpage->crypt_buf, dst_frame, size); } - fprintf(stderr, "JAN: buf0buf.cc: decrypt page 1 %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE)); - fprintf(stderr, "JAN: buf0buf.cc: decrypt page 2 %lu\n", mach_read_from_2(src_frame+FIL_PAGE_TYPE)); /* decrypt from src_frame to dst_frame */ fil_space_decrypt(bpage->space, src_frame, size, dst_frame); - fprintf(stderr, "JAN: buf0buf.cc: after decrypt %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE)); /* decompress from dst_frame to comp_buf and then copy to buffer pool */ if (page_compressed_encrypted) { @@ -5815,9 +5795,7 @@ buf_page_decrypt_after_read( bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); } - fprintf(stderr, "JAN: buf0buf.cc: decompress %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE)); fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); - fprintf(stderr, "JAN: buf0buf.cc: after decompress %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE)); } } bpage->key_version = key_version; diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 7a15545f517..19d18dcd870 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -176,7 +176,7 @@ buf_read_page_low( ut_ad(buf_page_in_file(bpage)); - byte* frame = buf_page_decrypt_before_read(bpage); + byte* frame = buf_page_decrypt_before_read(bpage, zip_size); if (sync) { thd_wait_begin(NULL, THD_WAIT_DISKIO); diff --git a/storage/innobase/enc/EncKeys.cc b/storage/innobase/enc/EncKeys.cc deleted file mode 100644 index e9cb134ef10..00000000000 --- a/storage/innobase/enc/EncKeys.cc +++ /dev/null @@ -1,433 +0,0 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** - @file EncKeys.cc - A class to keep keys for encryption/decryption. - -How it works... -The location and usage can be configured via the configuration file. -Example - -[mysqld] -... -innodb_data_encryption_providertype = 1 -innodb_data_encryption_providername = keys.enc -innodb_data_encryption_providerurl = /home/mdb/ -innodb_data_encryption_filekey = secret -... - -As provider type currently only value 1 is supported, which means, the keys are read from a file. -The filename is set up via the innodb_data_encryption_providername configuration value. -innodb_data_encryption_providerurl is used to configure the path to this file. This is usually -a folder name. -Examples: -innodb_data_encryption_providerurl = \\\\unc (windows share) -innodb_data_encryption_providerurl = e:/tmp/ (windows path) -innodb_data_encryption_providerurl = /tmp (linux path) - -The key file contains AES keys and initialization vectors as hex-encoded Strings. -Supported are keys of size 128, 192 or 256 bits. IV consists of 16 bytes. - -The key file should be encrypted and the key to decrypt the file can be given with the -innodb_data_encryption_filekey parameter. - -The file key can also be located if FILE: is prepended to the key. Then the following part is interpreted -as absolut to the file containing the file key. This file can optionally be encrypted, currently with a fix key. -Example: -innodb_data_encryption_filekey = FILE:y:/secret256.enc - -If the key file can not be read at server startup, for example if the file key is not present, -page_encryption feature is not availabe and access to page_encryption tables is not possible. - -Example files can be found inside the unittest/eperi folder. - -Open SSL command line utility can be used to create an encrypted key file. -Examples: -openssl enc –aes-256-cbc –md sha1 –k secret –in keys.txt –out keys.enc -openssl enc –aes-256-cbc –md sha1 –k –in secret –out secret.enc - - Created 09/15/2014 - ***********************************************************************/ -#ifdef __WIN__ -#define PCRE_STATIC 1 -#endif - -#include "EncKeys.h" -#include -#include -#include -#include -#include -#include -#include - - - - -const char* EncKeys::strMAGIC = "Salted__"; -const int EncKeys::magicSize = 8;//strlen(strMAGIC); // 8 byte -const char* EncKeys::newLine = "\n"; - -const char* EncKeys::errorNoKeyId = "KeyID = %u not found or with error. Check the key and the log file.\n"; -const char* EncKeys::errorInMatches = "Wrong match of the keyID in line %u, see the template.\n"; -const char* EncKeys::errorExceedKeyFileSize = "The size of the key file %s exceeds " - "the maximum allowed of %u bytes.\n"; -const char* EncKeys::errorExceedKeySize = "The key size exceeds the maximum allowed size of %u in line %u.\n"; -const char* EncKeys::errorEqualDoubleKey = "More than one identical key with keyID = %u found" - " in lines %u and %u.\nDelete one of them in the key file.\n"; -const char* EncKeys::errorUnequalDoubleKey = "More than one not identical key with keyID = %u found" - " in lines %u and %u.\nChoose the right one and delete the other in the key file.\n" - "I'll take the key from line %u\n"; -const char* EncKeys::errorNoInitializedKey = "The key could not be initialized.\n"; -const char* EncKeys::errorNotImplemented = "Initializing keys through key server is not" - " yet implemented.\nYou can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorOpenFile = "Could not open %s for reading. You can not read encrypted tables or columns.\n\n"; -const char* EncKeys::errorReadingFile = "Could not read from %s. You can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorFileSize = "Could not get the file size from %s. You can not read encrypted tables or columns\n\n"; -const char* EncKeys::errorFalseFileKey = "Wrong encryption / decryption key for keyfile '%s'.\n"; - -/* read this from a secret source in some later version */ -const char* EncKeys::initialPwd = "lg28s9ac5ffa537fd8798875c98e190df289da7e047c05"; - -EncKeys::EncKeys() { - countKeys = keyLineInKeyFile = 0; - for (int ii = 0; ii < MAX_KEYS; ii++) { - keys[ii].id = 0; - keys[ii].iv = keys[ii].key = NULL; - } - oneKey = NULL; -} - -EncKeys::~EncKeys() { - for (int ii = MAX_KEYS - 1; ii >= 0 ; ii--) { - delete[] keys[ii].iv; keys[ii].iv = NULL; - delete[] keys[ii].key; keys[ii].key = NULL; - - } -} - -bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { - if (KEYINITTYPE_FILE == initType) - { - int result = initKeysThroughFile(name, url, filekey); - return ERROR_FALSE_FILE_KEY != result && ERROR_OPEN_FILE != result && ERROR_READING_FILE != result; - } - else if (KEYINITTYPE_SERVER == initType) - { - return NO_ERROR_KEY_FILE_PARSE_OK == initKeysThroughServer(name, url, filekey); - } - return false; -} - -int EncKeys::initKeysThroughFile(const char *name, const char *path, const char *filekey) { - if (path==NULL || name==NULL) return ERROR_OPEN_FILE; - size_t len1 = strlen(path); - size_t len2 = strlen(name); - const char *MAGIC = "FILE:"; - const short MAGIC_LEN = 5; - int ret = NO_ERROR_KEY_FILE_PARSE_OK; - bool isUncPath= (len1>2) ? ((strncmp("\\\\", path, 2)==0) ? TRUE : FALSE) : FALSE; - bool isSlash = ((isUncPath? '\\':'/') == path[len1 - 1]); - char *secret = (char*) malloc(MAX_SECRET_SIZE +1 * sizeof(char)); - char *filename = (char*) malloc((len1 + len2 + (isSlash ? 1 : 2)) * sizeof(char)); - if(filekey != NULL) - { - //If secret starts with FILE: interpret the secret as filename. - if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) { - int fk_len = strlen(filekey); - char *secretfile = (char*)malloc( (1 + fk_len - MAGIC_LEN)* sizeof(char)); - memcpy(secretfile, filekey+MAGIC_LEN, fk_len - MAGIC_LEN); - secretfile[fk_len-MAGIC_LEN] = '\0'; - parseSecret(secretfile, secret); - free(secretfile); - } else - { - sprintf(secret, "%s", filekey); - } - } - sprintf(filename, "%s%s%s", path, isSlash ? "" : (isUncPath ? "\\":"/"), name); - ret = parseFile((const char *)filename, 254, secret); - free(filename); - free(secret); - return ret; -} - -int EncKeys::initKeysThroughServer( const char *name, const char *path, const char *filekey) -{ - //TODO -#ifdef UNIV_DEBUG - fprintf(stderr, errorNotImplemented); -#endif //UNIV_DEBUG - return ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; -} - -/* - * secret is limited to MAX_SECRET_SIZE characters - */ -void EncKeys::parseSecret( const char *secretfile, char *secret ) { - int maxSize = (MAX_SECRET_SIZE +16 + magicSize*2) ; - char* buf = (char*)malloc((maxSize) * sizeof(char)); - char* _initPwd = (char*)malloc((strlen(initialPwd)+1) * sizeof(char)); - - FILE *fp = fopen(secretfile, "rb"); - fseek(fp, 0L, SEEK_END); - long file_size = ftell(fp); - rewind(fp); - int bytes_to_read = (maxSize >= file_size)? file_size:(maxSize); - fread(buf, 1, bytes_to_read, fp); - if (memcmp(buf, strMAGIC, magicSize)) { - bytes_to_read = ((unsigned long) bytes_to_read>MAX_SECRET_SIZE) ? MAX_SECRET_SIZE : bytes_to_read; - memcpy(secret, buf, bytes_to_read); - secret[bytes_to_read] = '\0'; - } else { - unsigned char salt[magicSize]; - unsigned char *key = new unsigned char[keySize32]; - unsigned char *iv = new unsigned char[ivSize16]; - memcpy(&salt, buf + magicSize, magicSize); - memcpy(_initPwd, initialPwd, strlen(initialPwd)); - _initPwd[strlen(initialPwd)]= '\0'; - my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); - uint32 d_size = 0; - /* This should be checked! */ - (void) my_aes_decrypt_dynamic((const uchar*)buf + 2 * magicSize, - bytes_to_read - 2 * magicSize, - (uchar*)secret, &d_size, - (const uchar*)key, keySize32, - iv, ivSize16, 0); - if (d_size>EncKeys::MAX_SECRET_SIZE) { - d_size = EncKeys::MAX_SECRET_SIZE; - } - secret[d_size] = '\0'; - delete[] key; - delete[] iv; - } - free(buf); - free(_initPwd); - fclose(fp); -} - -/** - * Returns a struct keyentry with the asked 'id' or NULL. - */ -keyentry *EncKeys::getKeys(int id) { - if (KEY_MIN <= id && KEY_MAX >= id && (&keys[id - 1])->iv) - { - return &keys[id - 1]; - } -#ifdef UNIV_DEBUG - else { - - fprintf(stderr, errorNoKeyId, id); - return NULL; - } -#endif //UNIV_DEBUG -} - -/** - * Get the keys from the key file and decrypt it with the key . - * Store the keys with id smaller then in an array of structs keyentry. - * Returns NO_ERROR_PARSE_OK or an appropriate error code. - */ -int EncKeys::parseFile(const char* filename, const ulint maxKeyId, const char *secret) { - int errorCode = 0; - char *buffer = decryptFile(filename, secret, &errorCode); - ulint id = 0; - - if (NO_ERROR_PARSE_OK != errorCode) return errorCode; - else errorCode = NO_ERROR_KEY_FILE_PARSE_OK; - - char *line = strtok(buffer, newLine); - while ( NULL != line) { - keyLineInKeyFile++; - switch (parseLine(line, maxKeyId)) { - case NO_ERROR_PARSE_OK: - id = oneKey->id; - keys[oneKey->id - 1] = *oneKey; - delete(oneKey); - countKeys++; - fprintf(stderr, "Line: %lu --> ", keyLineInKeyFile); printKeyEntry(id); - break; - case ERROR_ID_TOO_BIG: - fprintf(stderr, errorExceedKeySize, KEY_MAX, keyLineInKeyFile); - fprintf(stderr, " --> %s\n", line); - errorCode = ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS; - break; - case ERROR_NOINITIALIZEDKEY: - fprintf(stderr, errorNoInitializedKey); - fprintf(stderr, " --> %s\n", line); - errorCode = ERROR_KEY_FILE_PARSE_NULL; - break; - case ERROR_WRONG_NUMBER_OF_MATCHES: - fprintf(stderr, errorInMatches, keyLineInKeyFile); - fprintf(stderr, " --> %s\n", line); - errorCode = ERROR_KEY_FILE_PARSE_NULL; - break; - case NO_ERROR_KEY_GREATER_THAN_ASKED: - fprintf(stderr, "No asked key in line %lu: %s\n", keyLineInKeyFile, line); - break; - case NO_ERROR_ISCOMMENT: - fprintf(stderr, "Is comment in line %lu: %s\n", keyLineInKeyFile, line); - default: - break; - } - line = strtok(NULL, newLine); - } - - free(line); line = NULL; - delete[] buffer; buffer = NULL; - return errorCode; -} - -int EncKeys::parseLine(const char *line, const ulint maxKeyId) { - int ret = NO_ERROR_PARSE_OK; - if (isComment(line)) - ret = NO_ERROR_ISCOMMENT; - else { - const char *error_p = NULL; - int offset; - pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", - 0, &error_p, &offset, NULL); - if ( NULL != error_p) - fprintf(stderr, "Error: %s\nOffset: %d\n", error_p, offset); - - int m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; - int rc = pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); - pcre_free(pattern); - if (4 == rc) { - char lin[MAX_KEY_LINE_SIZE + 1]; - strncpy( lin, line, MAX_KEY_LINE_SIZE); - lin[MAX_KEY_LINE_SIZE] = '\0'; - char *substring_start = lin + ovector[2]; - int substr_length = ovector[3] - ovector[2]; - if (3 < substr_length) - ret = ERROR_ID_TOO_BIG; - else { - char buffer[4]; - sprintf(buffer, "%.*s", substr_length, substring_start); - ulint id = atoi(buffer); - if (0 == id) ret = ERROR_NOINITIALIZEDKEY; - else if (KEY_MAX < id) ret = ERROR_ID_TOO_BIG; - else if (maxKeyId < id) ret = NO_ERROR_KEY_GREATER_THAN_ASKED; - else { - oneKey = new keyentry; - oneKey->id = id; - substring_start = lin + ovector[4]; - substr_length = ovector[5] - ovector[4]; - oneKey->iv = new char[substr_length + 1]; - sprintf(oneKey->iv, "%.*s", substr_length, substring_start); - substring_start = lin + ovector[6]; - substr_length = ovector[7] - ovector[6]; - oneKey->key = new char[substr_length + 1]; - sprintf(oneKey->key, "%.*s", substr_length, substring_start); - } - } - } - else - ret = ERROR_WRONG_NUMBER_OF_MATCHES; - } - return ret; -} - -/** - * Decrypt the key file 'filename' if it is encrypted with the key 'secret'. - * Store the content of the decrypted file in 'buffer'. The buffer has to be freed - * in the calling function. - */ -char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorCode) { - *errorCode = NO_ERROR_PARSE_OK; - fprintf(stderr, "Reading %s\n\n", filename); - FILE *fp = fopen(filename, "rb"); - if (NULL == fp) { - fprintf(stderr, errorOpenFile, filename); - *errorCode = ERROR_OPEN_FILE; - return NULL; - } - - if (fseek(fp, 0L, SEEK_END)) { - *errorCode = ERROR_READING_FILE; - return NULL; - } - long file_size = ftell(fp); // get the file size - if (MAX_KEY_FILE_SIZE < file_size) { - fprintf(stderr, errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); - *errorCode = ERROR_KEY_FILE_TOO_BIG; - fclose(fp); - return NULL; - } - else if (-1L == file_size) { - fprintf(stderr, errorFileSize, filename); - *errorCode = ERROR_READING_FILE; - return NULL; - } - - rewind(fp); - //Read file into buffer - uchar *buffer = new uchar[file_size + 1]; - file_size= fread(buffer, 1, file_size, fp); - buffer[file_size] = '\0'; - fclose(fp); - //Check for file encryption - if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. - unsigned char salt[magicSize]; - unsigned char *key = new unsigned char[keySize32]; - unsigned char *iv = new unsigned char[ivSize16]; - uchar *decrypted = new uchar[file_size]; - memcpy(&salt, buffer + magicSize, magicSize); - my_bytes_to_key((unsigned char *) salt, secret, key, iv); - uint32 d_size = 0; - int res = my_aes_encrypt_dynamic((const uchar*)buffer + 2 * magicSize, file_size - 2 * magicSize, - decrypted, &d_size, (const uchar*) key, keySize32, iv, ivSize16, 0); - if(0 != res) { - *errorCode = ERROR_FALSE_FILE_KEY; - delete[] buffer; buffer = NULL; - fprintf(stderr, errorFalseFileKey, filename); - } - else { - memcpy(buffer, decrypted, d_size); - buffer[d_size] = '\0'; - } - - delete[] decrypted; decrypted = NULL; - delete[] key; key = NULL; - delete[] iv; iv = NULL; - } - return (char*) buffer; -} - -bool EncKeys::isComment(const char *line) { - const char *error_p; - int offset, m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; - pcre *pattern = pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); - int rc = pcre_exec( pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); - pcre_free(pattern); - if (0 > rc) return false; - else return true; -} - - -void EncKeys::printKeyEntry( ulint id) -{ -#ifdef UNIV_DEBUG - keyentry *entry = getKeys(id); - if( NULL == entry) { - fprintf(stderr, "No such keyID=%lu\n",id); - } - else { - fprintf(stderr, "Key: id:%3lu \tiv:%lu bytes\tkey:%lu bytes\n", entry->id, strlen(entry->iv)/2, strlen(entry->key)/2); - } -#endif //UNIV_DEBUG -} diff --git a/storage/innobase/enc/KeySingleton.cc b/storage/innobase/enc/KeySingleton.cc deleted file mode 100644 index d75b55b4c07..00000000000 --- a/storage/innobase/enc/KeySingleton.cc +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** -@file KeySingleton.cc -Implementation of single pattern to keep keys for encrypting/decrypting pages. - -Created 09/13/2014 -***********************************************************************/ - - -#include "KeySingleton.h" -#include -#include - - -bool KeySingleton::instanceInited = false; -KeySingleton KeySingleton::theInstance; -EncKeys KeySingleton::encKeys; - - - -KeySingleton & KeySingleton::getInstance() { -#ifdef UNIV_DEBUG - if( !instanceInited) { - fprintf(stderr, "Encryption / decryption keys were not initialized. " - "You can not read encrypted tables or columns\n"); - } -#endif /* UNIV_DEBUG */ - return theInstance; -} - -KeySingleton & KeySingleton::getInstance(const char *name, const char *url, - const int initType, const char *filekey) { - - if(instanceInited) return theInstance; - instanceInited = encKeys.initKeys(name, url, initType, filekey); - if( !instanceInited) { - fprintf(stderr, "Could not initialize any of the encryption / decryption keys. " - "You can not read encrypted tables\n\n"); - fflush(stderr); - } else { - my_aes_init_dynamic_encrypt(MY_AES_ALGORITHM_CTR); - } - - return theInstance; -} - -keyentry *KeySingleton::getKeys(int id) { - return encKeys.getKeys(id); -} - -ibool KeySingleton::hasKey(int id) { - return encKeys.getKeys(id) != NULL; -} - diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 49eef1c5f7a..8ff2c490d35 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -304,9 +304,8 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } return fil_crypt_get_key(dst, key_length, crypt_data, *version, false); - } else { - return fil_crypt_get_key(dst, key_length, NULL, *version, true); } + return fil_crypt_get_key(dst, key_length, NULL, *version, true); } /****************************************************************** @@ -618,8 +617,6 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, fil_space_crypt_t* crypt_data; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - fprintf(stderr, "JAN: page_size %lu\n", page_size); - // get key (L) uint key_version; byte key[MY_AES_MAX_KEY_LENGTH]; @@ -712,7 +709,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, uint32 dstlen; if (page_compressed) { - srclen = page_size; + srclen = page_size - FIL_PAGE_DATA;; } @@ -727,7 +724,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, "Unable to encrypt data-block " " src: %p srclen: %ld buf: %p buflen: %d." " return-code: %d. Can't continue!\n", - src, (long)(page_size - unencrypted_bytes), + src, (long)srclen, dst, dstlen, rc); ut_error; } @@ -772,7 +769,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } else { /* Page compressed and encrypted tables have different FIL_HEADER */ - ulint page_len = log10((double)srclen)/log10((double)2); + ulint page_len = log10((double)page_size)/log10((double)2); /* Set up the correct page type */ mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); /* Set up the compression algorithm */ @@ -810,7 +807,6 @@ bool fil_space_decrypt(fil_space_crypt_t* crypt_data, const byte* src_frame, ulint page_size, byte* dst_frame) { - ulint space_id = mach_read_from_4(src_frame + FIL_PAGE_SPACE_ID); ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); // key version uint key_version; @@ -821,22 +817,17 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; - fprintf(stderr, "JAN: page type %lu\n", page_type); - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - fprintf(stderr, "JAN: key_version %lu\n", key_version); orig_page_type = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - fprintf(stderr, "JAN: decrypt: orig_page_type %lu\n", orig_page_type); } else { key_version = mach_read_from_4( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); } if (key_version == 0 && !page_encrypted) { - fprintf(stderr, "JAN: unencrypted\n"); //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return false; /* page not decrypted */ @@ -849,8 +840,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, src_frame + FIL_PAGE_OFFSET); ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - fprintf(stderr, "JAN: space %lu, offset %lu lsn %lu\n", space, offset, lsn); - // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -895,9 +884,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, } } - // decrypt page data - ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; - const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; @@ -905,18 +891,21 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, ulint compressed_len; ulint compression_method; + if (page_compressed) { orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); } + if (page_encrypted && !page_compressed) { orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); } if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)); + srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; } + int rc = (* my_aes_decrypt_dynamic)(src, srclen, dst, &dstlen, (unsigned char*)key, key_length, @@ -928,13 +917,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, "Unable to decrypt data-block " " src: %p srclen: %ld buf: %p buflen: %d." " return-code: %d. Can't continue!\n", - src, (long)(page_size - unencrypted_bytes), + src, (long)srclen, dst, dstlen, rc); ut_error; } if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - fprintf(stderr, "JAN: trailer...\n"); // copy page trailer memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, @@ -1221,7 +1209,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { !fil_tablespace_is_being_deleted(space)); /* try to reacquire pending op */ - if (fil_inc_pending_ops(space)) + if (fil_inc_pending_ops(space, true)) break; /* pending op reacquired! */ @@ -1268,7 +1256,7 @@ fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, if (fil_space_get_type(space) != FIL_TABLESPACE) return false; - if (fil_inc_pending_ops(space)) { + if (fil_inc_pending_ops(space, true)) { /* tablespace being dropped */ return false; } diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 85d9b3436e4..29b9580f8e2 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -269,10 +269,9 @@ fil_compress_page( int level = 0; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint write_size=0; - ulint comp_method = innodb_compression_algorithm; /* Cache to avoid - change during - function execution */ - ulint orig_page_type = 0; + /* Cache to avoid change during function execution */ + ulint comp_method = innodb_compression_algorithm; + ulint orig_page_type; ut_ad(buf); ut_ad(out_buf); @@ -489,9 +488,9 @@ fil_compress_page( srv_stats.page_compression_saved.add((len - write_size)); srv_stats.pages_page_compressed.inc(); + /* If we do not persistently trim rest of page, we need to write it + all */ if (!srv_use_trim) { - /* If persistent trims are not used we always write full - page */ write_size = len; } @@ -738,5 +737,3 @@ fil_decompress_page( ut_free(in_buf); } } - - diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 828badc232c..df5867ce43b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -104,7 +104,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "page0zip.h" #include "fil0pagecompress.h" - #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) #ifdef MYSQL_DYNAMIC_PLUGIN @@ -3193,7 +3192,7 @@ innobase_init( goto error; } } - + #ifndef HAVE_LZ4 if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" @@ -3261,7 +3260,6 @@ innobase_init( srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir : default_path); - /* Set default InnoDB data file size to 12 MB and let it be auto-extending. Thus users can use InnoDB in >= 4.0 without having to specify any startup options. */ @@ -19096,7 +19094,7 @@ static MYSQL_SYSVAR_ULONG(fatal_semaphore_wait_threshold, srv_fatal_semaphore_wa 0); static MYSQL_SYSVAR_BOOL(encrypt_tables, srv_encrypt_tables, 0, - "Encrypt tables", + "Encrypt all tables in the storage engine", 0, 0, 0); static MYSQL_SYSVAR_UINT(encryption_threads, srv_n_fil_crypt_threads, diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index b5727c627c1..7807c7ca7e6 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -26,8 +26,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "dict0stats.h" - - /* Structure defines translation table between mysql index and innodb index structures */ struct innodb_idx_translate_t { diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index b9e59913526..2863ab01ff9 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1463,7 +1463,8 @@ before getting decrypted */ byte* buf_page_decrypt_before_read( /*=========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ + buf_page_t* page, /*!< in/out: buffer page read from disk */ + ulint zip_size); /*!< in: compressed page size, or 0 */ /********************************************************************//** The hook that is called just after a page is read from disk. diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 07aab903024..1ed8cbf3293 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -345,9 +345,7 @@ struct fil_space_t { unflushed_spaces */ UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - - fil_space_crypt_t* crypt_data; - + fil_space_crypt_t* crypt_data; ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index d7dd6b38459..0647fd04e84 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -36,7 +36,7 @@ It is called when: Note: We should not use flags and conditions such as: (srv_encrypt_log && - opt_danger_danger_use_dbug_keys && + debug_use_static_keys && GetLatestCryptoKeyVersion() == UNENCRYPTED_KEY_VER) because they haven't been read and set yet in the situation of resetting redo logs. @@ -129,8 +129,8 @@ log_block_get_start_lsn( ulint log_block_no) /*!< in: log block number */ { lsn_t start_lsn = - (lsn & 0xffffffff00000000) | - (((log_block_no - 1) & 0x3fffffff) << 9); + (lsn & (lsn_t)0xffffffff00000000ULL) | + (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); return start_lsn; } diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index d5e8fe5bf2a..fbed6137cd7 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3568,10 +3568,10 @@ recv_reset_logs( latest key version. */ log_sys->redo_log_crypt_ver = UNENCRYPTED_KEY_VER; /* - Note: flags (srv_encrypt_log and opt_danger_danger_use_dbug_keys) + Note: flags (srv_encrypt_log and debug_use_static_keys) haven't been read and set yet! So don't use condition such as: - if (srv_encrypt_log && opt_danger_danger_use_dbug_keys) + if (srv_encrypt_log && debug_use_static_keys) */ log_init_crypt_msg_and_nonce(); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 0bcbe01c0f6..f41ddaf2b30 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -661,6 +661,8 @@ os_file_get_last_error_low( return(OS_FILE_OPERATION_ABORTED); } else if (err == ERROR_ACCESS_DENIED) { return(OS_FILE_ACCESS_VIOLATION); + } else if (err == ERROR_BUFFER_OVERFLOW) { + return(OS_FILE_NAME_TOO_LONG); } else { return(OS_FILE_ERROR_MAX + err); } @@ -735,6 +737,8 @@ os_file_get_last_error_low( return(OS_FILE_NOT_FOUND); case EEXIST: return(OS_FILE_ALREADY_EXISTS); + case ENAMETOOLONG: + return(OS_FILE_NAME_TOO_LONG); case EXDEV: case ENOTDIR: case EISDIR: @@ -1946,7 +1950,7 @@ os_file_create_func( try to set atomic writes and if that fails when creating a new table, produce a error. If atomic writes are used on existing file, ignore error and use traditional writes for that file */ - if (file != INVALID_HANDLE_VALUE + if (file != INVALID_HANDLE_VALUE && type == OS_DATA_FILE && (awrites == ATOMIC_WRITES_ON || (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) && !os_file_set_atomic_writes(name, file)) { @@ -2092,7 +2096,7 @@ os_file_create_func( try to set atomic writes and if that fails when creating a new table, produce a error. If atomic writes are used on existing file, ignore error and use traditional writes for that file */ - if (file != -1 + if (file != -1 && type == OS_DATA_FILE && (awrites == ATOMIC_WRITES_ON || (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) && !os_file_set_atomic_writes(name, file)) { @@ -2305,8 +2309,6 @@ os_file_close_func( #ifdef __WIN__ BOOL ret; - ut_a(file); - ret = CloseHandle(file); if (ret) { @@ -2344,8 +2346,6 @@ os_file_close_no_error_handling( #ifdef __WIN__ BOOL ret; - ut_a(file); - ret = CloseHandle(file); if (ret) { @@ -2574,8 +2574,6 @@ os_file_flush_func( #ifdef __WIN__ BOOL ret; - ut_a(file); - os_n_fsyncs++; ret = FlushFileBuffers(file); @@ -2909,7 +2907,6 @@ os_file_read_func( os_bytes_read_since_printout += n; try_again: - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3037,7 +3034,6 @@ os_file_read_no_error_handling_func( os_bytes_read_since_printout += n; try_again: - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3172,7 +3168,6 @@ os_file_write_func( os_n_file_writes++; - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3366,7 +3361,7 @@ os_file_status( struct _stat64 statinfo; ret = _stat64(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { + if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) { /* file does not exist */ *exists = FALSE; return(TRUE); @@ -3394,7 +3389,7 @@ os_file_status( struct stat statinfo; ret = stat(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { + if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) { /* file does not exist */ *exists = FALSE; return(TRUE); @@ -4674,10 +4669,10 @@ found: os_slot_alloc_lzo_mem(slot); } #endif - fprintf(stderr, "JAN: compress page\n"); + /* Call page compression */ tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), + fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, @@ -4717,7 +4712,6 @@ found: // if it is not yet allocated. os_slot_alloc_page_buf2(slot); - fprintf(stderr, "JAN: encrypt page\n"); fil_space_encrypt( fil_node_get_space_id(slot->message1), slot->offset, @@ -5039,7 +5033,6 @@ os_aio_func( #endif /* WIN_ASYNC_IO */ ulint wake_later; - ut_ad(file); ut_ad(buf); ut_ad(n > 0); ut_ad(n % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -5383,13 +5376,16 @@ os_aio_windows_handle( ut_a((slot->len & 0xFFFFFFFFUL) == slot->len); - switch (slot->type) { - case OS_FILE_WRITE: + switch (slot->type) { + case OS_FILE_WRITE: if (slot->message1 && slot->page_encryption && slot->page_encryption_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf2, - slot->offset, slot->len); + ret_val = os_file_write(slot->name, + slot->file, + slot->page_buf2, + slot->offset, + slot->len); } else { if (slot->message1 && slot->page_compression @@ -5402,8 +5398,7 @@ os_aio_windows_handle( (DWORD) slot->len, &len, &(slot->control)); } - } - + } break; case OS_FILE_READ: ret = ReadFile(slot->file, slot->buf, @@ -5441,7 +5436,6 @@ os_aio_windows_handle( os_slot_alloc_page_buf2(slot); os_slot_alloc_tmp_encryption_buf(slot); - fprintf(stderr, "JAN: decrypt data 1\n"); // Decrypt the data fil_space_decrypt( fil_node_get_space_id(slot->message1), @@ -5451,26 +5445,23 @@ os_aio_windows_handle( // Copy decrypted buffer back to buf memcpy(slot->buf, slot->page_buf2, slot->len); } - if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. + /* We allocate memory for page compressed buffer if + and only if it is not yet allocated. */ os_slot_alloc_page_buf(slot); - #ifdef HAVE_LZO if (fil_page_is_lzo_compressed(slot->buf)) { os_slot_alloc_lzo_mem(slot); } #endif - fprintf(stderr, "JAN: decompress data\n"); fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); + slot->page_buf, + slot->buf, + slot->len, + slot->write_size); } } else { - // OS_FILE_WRITE + /* OS_FILE_WRITE */ if (slot->page_compression_success && (fil_page_is_compressed(slot->page_buf) || fil_page_is_compressed_encrypted(slot->buf))) { @@ -5578,7 +5569,6 @@ retry: os_slot_alloc_tmp_encryption_buf(slot); ut_ad(slot->message1 != NULL); - fprintf(stderr, "JAN: decrypt data 2\n"); // Decrypt the data fil_space_decrypt( fil_node_get_space_id(slot->message1), @@ -5587,29 +5577,23 @@ retry: slot->page_buf2); // Copy decrypted buffer back to buf memcpy(slot->buf, slot->page_buf2, slot->len); - } + } - /* If the page is page compressed and this is read, - we decompress before we annouce the read is - complete. */ + /* If the table is page compressed and this + is read, we decompress before we announce + the read is complete. For writes, we free + the compressed page. */ if (fil_page_is_compressed(slot->buf)) { // We allocate memory for page compressed buffer if and only // if it is not yet allocated. os_slot_alloc_page_buf(slot); - #ifdef HAVE_LZO if (fil_page_is_lzo_compressed(slot->buf)) { os_slot_alloc_lzo_mem(slot); } #endif - fprintf(stderr, "JAN: decompress data\n"); - - fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); + fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); } } else { /* OS_FILE_WRITE */ @@ -5623,7 +5607,6 @@ retry: } } - /* Mark this request as completed. The error handling will be done in the calling function. */ os_mutex_enter(array->mutex); @@ -6698,7 +6681,7 @@ os_slot_alloc_lzo_mem( os_aio_slot_t* slot) /*!< in: slot structure */ { ut_a(slot != NULL); - if (slot->lzo_mem == NULL) { + if(slot->lzo_mem == NULL) { slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); ut_a(slot->lzo_mem != NULL); memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); @@ -6721,6 +6704,7 @@ os_slot_alloc_tmp_encryption_buf( } } + /***********************************************************************//** Try to get number of bytes per sector from file system. @return file block size */ diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 945ef2623cb..d9d4564fc99 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -238,6 +238,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, sizeof(MARIA_PINNED_PAGE), 1, 1, MYF(0))) return 1; + bitmap->share= share; bitmap->block_size= share->block_size; bitmap->file.file= file; _ma_bitmap_set_pagecache_callbacks(&bitmap->file, share); @@ -1233,8 +1234,7 @@ static void fill_block(MARIA_FILE_BITMAP *bitmap, */ -static my_bool allocate_head(MARIA_SHARE *share, - MARIA_FILE_BITMAP *bitmap, uint size, +static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, MARIA_BITMAP_BLOCK *block) { uint min_bits= size_to_head_pattern(bitmap, size); @@ -1242,11 +1242,11 @@ static my_bool allocate_head(MARIA_SHARE *share, uchar *best_data= 0; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); uint first_pattern= 0; /* if doing insert_order */ + MARIA_SHARE *share= bitmap->share; my_bool insert_order= MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER); DBUG_ENTER("allocate_head"); - DBUG_ASSERT(share != 0); DBUG_ASSERT(size <= FULL_PAGE_SIZE(share)); if (insert_order) @@ -1618,7 +1618,7 @@ static my_bool find_head(MARIA_HA *info, uint length, uint position) We need to have DIRENTRY_SIZE here to take into account that we may need an extra directory entry for the row */ - while (allocate_head(info->s, bitmap, length + DIR_ENTRY_SIZE, block)) + while (allocate_head(bitmap, length + DIR_ENTRY_SIZE, block)) if (move_to_next_bitmap(info, bitmap)) return 1; return 0; diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c index c59d6ecaa8f..bf66424e539 100644 --- a/storage/maria/ma_blockrec.c +++ b/storage/maria/ma_blockrec.c @@ -784,7 +784,7 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, @brief Extend a record area to fit a given size block @fn extend_area_on_page() - @param info Handler if head page and 0 if tail page + @param info Handler @param buff Page buffer @param dir Pointer to dir entry in buffer @param rownr Row number we working on @@ -793,6 +793,7 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, @param empty_space Total empty space in buffer This is updated with length after dir is allocated and current block freed + @param head_page 1 if head page, 0 for tail page @implementation The logic is as follows (same as in _ma_update_block_record()) @@ -815,16 +816,17 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, @retval 1 error (wrong info in block) */ -static my_bool extend_area_on_page(MARIA_SHARE *share, - MARIA_HA *info, +static my_bool extend_area_on_page(MARIA_HA *info, uchar *buff, uchar *dir, uint rownr, uint request_length, uint *empty_space, uint *ret_offset, - uint *ret_length) + uint *ret_length, + my_bool head_page) { uint rec_offset, length, org_rec_length; uint max_entry= (uint) buff[DIR_COUNT_OFFSET]; + MARIA_SHARE *share= info->s; uint block_size= share->block_size; DBUG_ENTER("extend_area_on_page"); @@ -906,8 +908,8 @@ static my_bool extend_area_on_page(MARIA_SHARE *share, int2store(dir+2, 0); _ma_compact_block_page(share, buff, rownr, 1, - info ? info->trn->min_read_from: 0, - info ? info->s->base.min_block_length : 0); + head_page ? info->trn->min_read_from: 0, + head_page ? share->base.min_block_length : 0); rec_offset= uint2korr(dir); length= uint2korr(dir+2); if (length < request_length) @@ -915,7 +917,7 @@ static my_bool extend_area_on_page(MARIA_SHARE *share, DBUG_PRINT("error", ("Not enough space: " "length: %u request_length: %u", length, request_length)); - _ma_set_fatal_error(info->s, HA_ERR_WRONG_IN_RECORD); + _ma_set_fatal_error(share, HA_ERR_WRONG_IN_RECORD); DBUG_RETURN(1); /* Error in block */ } *empty_space= length; /* All space is here */ @@ -928,7 +930,8 @@ static my_bool extend_area_on_page(MARIA_SHARE *share, *ret_length= length; check_directory(share, - buff, block_size, info ? info->s->base.min_block_length : 0, + buff, block_size, + head_page ? share->base.min_block_length : 0, *empty_space - length); DBUG_RETURN(0); } @@ -994,6 +997,7 @@ static uint empty_space_on_page(uchar *buff, uint block_size) @brief Ensure we have space for new directory entries @fn make_space_for_directory() + @param info Handler @param buff Page buffer @param max_entry Number of current entries in directory @param count Number of new entries to be added to directory @@ -1001,6 +1005,7 @@ static uint empty_space_on_page(uchar *buff, uint block_size) @param empty_space Total empty space in buffer. It's updated to reflect the new empty space @param first_pos Store position to last data byte on page here + @param head_page 1 if head page, 0 for tail page. @note This function is inline as the argument passing is the biggest @@ -1012,13 +1017,14 @@ static uint empty_space_on_page(uchar *buff, uint block_size) */ static inline my_bool -make_space_for_directory(MARIA_SHARE *share, - MARIA_HA *info, +make_space_for_directory(MARIA_HA *info, uchar *buff, uint max_entry, uint count, uchar *first_dir, uint *empty_space, - uint *first_pos) + uint *first_pos, + my_bool head_page) { uint length_needed= DIR_ENTRY_SIZE * count; + MARIA_SHARE *share= info->s; /* The following is not true only in the case and UNDO is used to reinsert @@ -1034,8 +1040,8 @@ make_space_for_directory(MARIA_SHARE *share, /* Create place for directory */ _ma_compact_block_page(share, buff, max_entry - 1, 0, - info ? info->trn->min_read_from : 0, - info ? info->s->base.min_block_length : 0); + head_page ? info->trn->min_read_from : 0, + head_page ? share->base.min_block_length : 0); *first_pos= (uint2korr(first_dir) + uint2korr(first_dir + 2)); *empty_space= uint2korr(buff + EMPTY_SPACE_OFFSET); if (*empty_space < length_needed) @@ -1065,13 +1071,14 @@ make_space_for_directory(MARIA_SHARE *share, SYNOPSIS find_free_position() - info Handler if head page and 0 otherwise + info Handler buff Page block_size Size of page res_rownr Store index to free position here res_length Store length of found segment here empty_space Store length of empty space on disk here. This is all empty space, including the found block. + @param head_page 1 if head page, 0 for tail page. NOTES If there is a free directory entry (entry with position == 0), @@ -1097,14 +1104,15 @@ make_space_for_directory(MARIA_SHARE *share, # Pointer to directory entry on page */ -static uchar *find_free_position(MARIA_SHARE *share, - MARIA_HA *info, +static uchar *find_free_position(MARIA_HA *info, uchar *buff, uint block_size, uint *res_rownr, - uint *res_length, uint *empty_space) + uint *res_length, uint *empty_space, + my_bool head_page) { uint max_entry, free_entry; uint length, first_pos; uchar *dir, *first_dir; + MARIA_SHARE *share= info->s; DBUG_ENTER("find_free_position"); max_entry= (uint) buff[DIR_COUNT_OFFSET]; @@ -1141,7 +1149,7 @@ static uchar *find_free_position(MARIA_SHARE *share, *res_length= length; check_directory(share, buff, block_size, - info ? info->s->base.min_block_length : 0, (uint) -1); + head_page ? share->base.min_block_length : 0, (uint) -1); DBUG_RETURN(dir); } /* No free places in dir; create a new one */ @@ -1150,9 +1158,8 @@ static uchar *find_free_position(MARIA_SHARE *share, if (max_entry == MAX_ROWS_PER_PAGE) DBUG_RETURN(0); - if (make_space_for_directory(share, - info, buff, max_entry, 1, - first_dir, empty_space, &first_pos)) + if (make_space_for_directory(info, buff, max_entry, 1, + first_dir, empty_space, &first_pos, head_page)) DBUG_RETURN(0); dir= first_dir - DIR_ENTRY_SIZE; @@ -1164,7 +1171,8 @@ static uchar *find_free_position(MARIA_SHARE *share, *res_length= length; check_directory(share, - buff, block_size, info ? info->s->base.min_block_length : 0, + buff, block_size, + head_page ? share->base.min_block_length : 0, *empty_space); DBUG_RETURN(dir); } @@ -1174,13 +1182,14 @@ static uchar *find_free_position(MARIA_SHARE *share, @brief Enlarge page directory to hold more entries @fn extend_directory() - @param info Handler if head page and 0 otherwise + @param info Handler @param buff Page buffer @param block_size Block size @param max_entry Number of directory entries on page @param new_entry Position for new entry @param empty_space Total empty space in buffer. It's updated to reflect the new empty space + @param head_page 1 if head page, 0 for tail page. @note This is only called on UNDO when we want to expand the directory @@ -1193,13 +1202,13 @@ static uchar *find_free_position(MARIA_SHARE *share, @retval 1 error (No data on page, fatal error) */ -static my_bool extend_directory(MARIA_SHARE *share, - MARIA_HA *info, uchar *buff, uint block_size, +static my_bool extend_directory(MARIA_HA *info, uchar *buff, uint block_size, uint max_entry, uint new_entry, - uint *empty_space) + uint *empty_space, my_bool head_page) { uint length, first_pos; uchar *dir, *first_dir; + MARIA_SHARE *share= info->s; DBUG_ENTER("extend_directory"); /* @@ -1209,10 +1218,9 @@ static my_bool extend_directory(MARIA_SHARE *share, */ first_dir= dir_entry_pos(buff, block_size, max_entry) + DIR_ENTRY_SIZE; - if (make_space_for_directory(share, - info, buff, max_entry, + if (make_space_for_directory(info, buff, max_entry, new_entry - max_entry + 1, - first_dir, empty_space, &first_pos)) + first_dir, empty_space, &first_pos, head_page)) DBUG_RETURN(1); /* Set the new directory entry to cover the max possible length */ @@ -1248,7 +1256,7 @@ static my_bool extend_directory(MARIA_SHARE *share, check_directory(share, buff, block_size, - info ? MY_MIN(info->s->base.min_block_length, length) : 0, + head_page ? MY_MIN(share->base.min_block_length, length) : 0, *empty_space); DBUG_RETURN(0); } @@ -1781,10 +1789,9 @@ static my_bool get_head_or_tail_page(MARIA_HA *info, DBUG_ASSERT((uint) (res->buff[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK) == page_type); - if (!(dir= find_free_position(share, - page_type == HEAD_PAGE ? info : 0, - res->buff, block_size, &res->rownr, - &res->length, &res->empty_space))) + if (!(dir= find_free_position(info, res->buff, block_size, &res->rownr, + &res->length, &res->empty_space, + page_type == HEAD_PAGE))) goto crashed; if (res->length < length) @@ -1885,9 +1892,9 @@ static my_bool get_rowpos_in_head_or_tail_page(MARIA_HA *info, max_entry= (uint) buff[DIR_COUNT_OFFSET]; if (max_entry <= rownr) { - if (extend_directory(share, - page_type == HEAD_PAGE ? info : 0, buff, block_size, - max_entry, rownr, &res->empty_space)) + if (extend_directory(info, buff, block_size, + max_entry, rownr, &res->empty_space, + page_type == HEAD_PAGE)) goto err; } @@ -1897,9 +1904,9 @@ static my_bool get_rowpos_in_head_or_tail_page(MARIA_HA *info, */ dir= dir_entry_pos(buff, block_size, rownr); - if (extend_area_on_page(share, page_type == HEAD_PAGE ? info : 0, buff, dir, - rownr, length, - &res->empty_space, &rec_offset, &max_length)) + if (extend_area_on_page(info, buff, dir, rownr, length, + &res->empty_space, &rec_offset, &max_length, + page_type == HEAD_PAGE)) goto err; res->buff= buff; @@ -3764,9 +3771,9 @@ static my_bool _ma_update_block_record2(MARIA_HA *info, */ block.org_bitmap_value= _ma_free_size_to_head_pattern(&share->bitmap, org_empty_size); - if (extend_area_on_page(share, info, buff, dir, rownr, + if (extend_area_on_page(info, buff, dir, rownr, new_row->total_length, &org_empty_size, - &rec_offset, &length)) + &rec_offset, &length, 1)) { errpos= 1; goto err; @@ -3945,9 +3952,9 @@ static my_bool _ma_update_at_original_place(MARIA_HA *info, of the row */ empty_size= org_empty_size; - if (extend_area_on_page(share, info, buff, dir, rownr, + if (extend_area_on_page(info, buff, dir, rownr, length_on_head_page, &empty_size, - &rec_offset, &length)) + &rec_offset, &length, 1)) goto err; row_pos.buff= buff; @@ -6411,8 +6418,8 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn, } make_empty_page(info, buff, page_type, 0); empty_space= block_size - PAGE_HEADER_SIZE(share) - PAGE_SUFFIX_SIZE; - (void) extend_directory(share, page_type == HEAD_PAGE ? info: 0, buff, - block_size, 0, rownr, &empty_space); + (void) extend_directory(info, buff, block_size, 0, rownr, &empty_space, + page_type == HEAD_PAGE); rec_offset= PAGE_HEADER_SIZE(share); dir= dir_entry_pos(buff, block_size, rownr); empty_space+= uint2korr(dir+2); @@ -6429,14 +6436,13 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn, if (max_entry <= rownr) { /* Add directory entry first in directory and data last on page */ - if (extend_directory(share, page_type == HEAD_PAGE ? info : 0, buff, - block_size, max_entry, rownr, &empty_space)) + if (extend_directory(info, buff, block_size, max_entry, rownr, + &empty_space, page_type == HEAD_PAGE)) goto crashed_file; } - if (extend_area_on_page(share, page_type == HEAD_PAGE ? info : 0, buff, - dir, rownr, + if (extend_area_on_page(info, buff, dir, rownr, (uint) data_length, &empty_space, - &rec_offset, &length)) + &rec_offset, &length, page_type == HEAD_PAGE)) goto crashed_file; } } diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index df1a69b112d..b7fcb6b7058 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -1043,7 +1043,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, if (encrypted) { if (ma_crypt_create(&share) || - ma_crypt_write(&share, &file)) + ma_crypt_write(&share, file)) goto err; } diff --git a/storage/maria/ma_crypt.c b/storage/maria/ma_crypt.c index 0871916b09f..89defe8cec9 100644 --- a/storage/maria/ma_crypt.c +++ b/storage/maria/ma_crypt.c @@ -83,7 +83,7 @@ ma_crypt_free(MARIA_SHARE* share) } int -ma_crypt_write(MARIA_SHARE* share, File* file) +ma_crypt_write(MARIA_SHARE* share, File file) { uchar buff[2]; MARIA_CRYPT_DATA *crypt_data= share->crypt_data; @@ -93,10 +93,10 @@ ma_crypt_write(MARIA_SHARE* share, File* file) buff[0] = crypt_data->type; buff[1] = crypt_data->iv_length; - if (mysql_file_write(*file, buff, 2, MYF(MY_NABP))) + if (mysql_file_write(file, buff, 2, MYF(MY_NABP))) return 1; - if (mysql_file_write(*file, crypt_data->iv, crypt_data->iv_length, + if (mysql_file_write(file, crypt_data->iv, crypt_data->iv_length, MYF(MY_NABP))) return 1; diff --git a/storage/maria/ma_crypt.h b/storage/maria/ma_crypt.h index 38fe18972e5..76752e19449 100644 --- a/storage/maria/ma_crypt.h +++ b/storage/maria/ma_crypt.h @@ -12,7 +12,7 @@ uint ma_crypt_get_data_page_header_space();/* bytes in data/index page header */ uint ma_crypt_get_index_page_header_space(struct st_maria_share *); uint ma_crypt_get_file_length(); /* bytes needed in file */ int ma_crypt_create(struct st_maria_share *); /* create encryption data */ -int ma_crypt_write(struct st_maria_share *, File *); /* write encryption data */ +int ma_crypt_write(struct st_maria_share *, File); /* write encryption data */ uchar* ma_crypt_read(struct st_maria_share *, uchar *buff); /* read crypt data*/ void ma_crypt_set_data_pagecache_callbacks(struct st_pagecache_file *file, diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c index 57e0471208d..bab07a48fde 100644 --- a/storage/maria/ma_write.c +++ b/storage/maria/ma_write.c @@ -392,7 +392,7 @@ err: else fatal_error= 1; - if (filepos != HA_POS_ERROR) + if (filepos != HA_OFFSET_ERROR) { if ((*share->write_record_abort)(info)) fatal_error= 1; diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index fc4ce1ee971..086b7a3775d 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -321,6 +321,7 @@ typedef struct st_maria_pack typedef struct st_maria_file_bitmap { + struct st_maria_share *share; uchar *map; pgcache_page_no_t page; /* Page number for current bitmap */ pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 82f7a8111be..1bb861cf820 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -222,7 +222,7 @@ not_to_recover: ut_ad(buf_page_in_file(bpage)); ut_ad(!mutex_own(&buf_pool_from_bpage(bpage)->LRU_list_mutex)); - byte* frame = buf_page_decrypt_before_read(bpage); + byte* frame = buf_page_decrypt_before_read(bpage, zip_size); if (sync) { thd_wait_begin(NULL, THD_WAIT_DISKIO); diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 0b61b31f7f5..e34297f4f86 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -209,8 +209,6 @@ fil_crypt_get_key(byte *dst, uint* key_length, else { // load iv - fprintf(stderr, "%d... %d\n",iv_len, version); - int rc = GetCryptoIV(version, (unsigned char*)iv, iv_len); fprintf(stderr, " %d\n",rc); @@ -620,8 +618,6 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, fil_space_crypt_t* crypt_data; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - fprintf(stderr, "JAN: page_size %lu\n", page_size); - // get key (L) uint key_version; byte key[MY_AES_MAX_KEY_LENGTH]; @@ -714,10 +710,9 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, uint32 dstlen; if (page_compressed) { - srclen = page_size; + srclen = page_size - FIL_PAGE_DATA; } - int rc = (* my_aes_encrypt_dynamic)(src, srclen, dst, &dstlen, (unsigned char*)key, key_length, @@ -729,7 +724,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, "Unable to encrypt data-block " " src: %p srclen: %ld buf: %p buflen: %d." " return-code: %d. Can't continue!\n", - src, (long)(page_size - unencrypted_bytes), + src, (long)srclen, dst, dstlen, rc); ut_error; } @@ -774,7 +769,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } else { /* Page compressed and encrypted tables have different FIL_HEADER */ - ulint page_len = log10((double)srclen)/log10((double)2); + ulint page_len = log10((double)page_size)/log10((double)2); /* Set up the correct page type */ mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); /* Set up the compression algorithm */ @@ -812,7 +807,6 @@ bool fil_space_decrypt(fil_space_crypt_t* crypt_data, const byte* src_frame, ulint page_size, byte* dst_frame) { - ulint space_id = mach_read_from_4(src_frame + FIL_PAGE_SPACE_ID); ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); // key version uint key_version; @@ -823,22 +817,18 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; - fprintf(stderr, "JAN: page type %lu\n", page_type); if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - fprintf(stderr, "JAN: key_version %lu\n", key_version); orig_page_type = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - fprintf(stderr, "JAN: decrypt: orig_page_type %lu\n", orig_page_type); } else { key_version = mach_read_from_4( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); } if (key_version == 0 && !page_encrypted) { - fprintf(stderr, "JAN: unencrypted\n"); //TODO: is this really needed ? memcpy(dst_frame, src_frame, page_size); return false; /* page not decrypted */ @@ -851,8 +841,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, src_frame + FIL_PAGE_OFFSET); ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - fprintf(stderr, "JAN: space %lu, offset %lu lsn %lu\n", space, offset, lsn); - // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -897,9 +885,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, } } - // decrypt page data - ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; - const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; @@ -907,18 +892,21 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, ulint compressed_len; ulint compression_method; + if (page_compressed) { orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); } + if (page_encrypted && !page_compressed) { orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); } if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)); + srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; } + int rc = (* my_aes_decrypt_dynamic)(src, srclen, dst, &dstlen, (unsigned char*)key, key_length, @@ -930,13 +918,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, "Unable to decrypt data-block " " src: %p srclen: %ld buf: %p buflen: %d." " return-code: %d. Can't continue!\n", - src, (long)(page_size - unencrypted_bytes), + src, (long)srclen, dst, dstlen, rc); ut_error; } if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - fprintf(stderr, "JAN: trailer...\n"); // copy page trailer memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, @@ -1223,7 +1210,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { !fil_tablespace_is_being_deleted(space)); /* try to reacquire pending op */ - if (fil_inc_pending_ops(space)) + if (fil_inc_pending_ops(space, true)) break; /* pending op reacquired! */ @@ -1270,7 +1257,7 @@ fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, if (fil_space_get_type(space) != FIL_TABLESPACE) return false; - if (fil_inc_pending_ops(space)) { + if (fil_inc_pending_ops(space, true)) { /* tablespace being dropped */ return false; } diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 544ace91ed3..fa25d8875ae 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -271,7 +271,7 @@ fil_compress_page( ulint write_size=0; /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - ulint orig_page_type = 0; + ulint orig_page_type; ut_ad(buf); ut_ad(out_buf); @@ -281,6 +281,8 @@ fil_compress_page( /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + /* Let's not compress file space header or + extent descriptor */ if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR) || (orig_page_type == FIL_PAGE_TYPE_XDES) ) { *out_len = len; return (buf); @@ -466,9 +468,6 @@ fil_compress_page( ut_a(block_size > 0); #endif write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); - /* Initialize rest of the written data to avoid - uninitialized bytes */ - memset(out_buf+tmp, 0, write_size-tmp); #ifdef UNIV_DEBUG ut_a(write_size > 0 && ((write_size % block_size) == 0)); ut_a(write_size >= tmp); @@ -481,14 +480,12 @@ fil_compress_page( space_id, fil_space_name(space), len, write_size); #endif /* UNIV_PAGECOMPRESS_DEBUG */ - srv_stats.page_compression_saved.add((len - write_size)); srv_stats.pages_page_compressed.inc(); + /* If we do not persistently trim rest of page, we need to write it + all */ if (!srv_use_trim) { - /* If persistent trims are not used we always write full - page and end of the page needs to be initialized.*/ - memset(out_buf+write_size, 0, len-write_size); write_size = len; } @@ -734,5 +731,3 @@ fil_decompress_page( ut_free(in_buf); } } - - diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index b86bae21172..d65e2e321f2 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -20276,7 +20276,7 @@ static MYSQL_SYSVAR_ULONG(fatal_semaphore_wait_threshold, srv_fatal_semaphore_wa 0); static MYSQL_SYSVAR_BOOL(encrypt_tables, srv_encrypt_tables, 0, - "Encrypt tables", + "Encrypt all tables in the storage engine", 0, 0, 0); static MYSQL_SYSVAR_UINT(encryption_threads, srv_n_fil_crypt_threads, @@ -20620,7 +20620,7 @@ maria_declare_plugin(xtradb) &innobase_storage_engine, innobase_hton_name, plugin_author, - "Percona-XtraDB, Supports transactions, row-level locking, foreign keys and encryption for tables and columns", + "Percona-XtraDB, Supports transactions, row-level locking, foreign keys and encryption for tables", PLUGIN_LICENSE_GPL, innobase_init, /* Plugin Init */ NULL, /* Plugin Deinit */ diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index e0f1a12fc4a..10b296f6cd4 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1491,7 +1491,8 @@ before getting decrypted */ byte* buf_page_decrypt_before_read( /*=========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ + buf_page_t* page, /*!< in/out: buffer page read from disk */ + ulint zip_size); /*!< in: compressed page size, or 0 */ /********************************************************************//** The hook that is called just after a page is read from disk. diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 42d020c84d9..f271522bbe9 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -339,9 +339,7 @@ struct fil_space_t { ibool is_corrupt; UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - - fil_space_crypt_t* crypt_data; - + fil_space_crypt_t* crypt_data; ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index ccb2f112f36..43da1f10c19 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -33,7 +33,7 @@ It is called when: Note: We should not use flags and conditions such as: (srv_encrypt_log && - opt_danger_danger_use_dbug_keys && + debug_use_static_keys && GetLatestCryptoKeyVersion() == UNENCRYPTED_KEY_VER) because they haven't been read and set yet in the situation of resetting redo logs. @@ -130,8 +130,8 @@ log_block_get_start_lsn( ulint log_block_no) /*!< in: log block number */ { lsn_t start_lsn = - (lsn & 0xffffffff00000000) | - (((log_block_no - 1) & 0x3fffffff) << 9); + (lsn & (lsn_t)0xffffffff00000000ULL) | + (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); return start_lsn; } diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 763f179f2d0..3491e246ba1 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -3659,10 +3659,10 @@ recv_reset_logs( latest key version. */ log_sys->redo_log_crypt_ver = UNENCRYPTED_KEY_VER; /* - Note: flags (srv_encrypt_log and opt_danger_danger_use_dbug_keys) + Note: flags (srv_encrypt_log and debug_use_static_keys) haven't been read and set yet! So don't use condition such as: - if (srv_encrypt_log && opt_danger_danger_use_dbug_keys) + if (srv_encrypt_log && debug_use_static_keys) */ log_init_crypt_msg_and_nonce(); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 9cb1bd8d7f9..13bafeb048b 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -728,6 +728,8 @@ os_file_get_last_error_low( return(OS_FILE_OPERATION_ABORTED); } else if (err == ERROR_ACCESS_DENIED) { return(OS_FILE_ACCESS_VIOLATION); + } else if (err == ERROR_BUFFER_OVERFLOW) { + return(OS_FILE_NAME_TOO_LONG); } else { return(OS_FILE_ERROR_MAX + err); } @@ -803,6 +805,8 @@ os_file_get_last_error_low( return(OS_FILE_NOT_FOUND); case EEXIST: return(OS_FILE_ALREADY_EXISTS); + case ENAMETOOLONG: + return(OS_FILE_NAME_TOO_LONG); case EXDEV: case ENOTDIR: case EISDIR: @@ -2062,7 +2066,7 @@ os_file_create_func( try to set atomic writes and if that fails when creating a new table, produce a error. If atomic writes are used on existing file, ignore error and use traditional writes for that file */ - if (file != INVALID_HANDLE_VALUE + if (file != INVALID_HANDLE_VALUE && type == OS_DATA_FILE && (awrites == ATOMIC_WRITES_ON || (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) && !os_file_set_atomic_writes(name, file)) { @@ -2211,7 +2215,7 @@ os_file_create_func( try to set atomic writes and if that fails when creating a new table, produce a error. If atomic writes are used on existing file, ignore error and use traditional writes for that file */ - if (file != -1 + if (file != -1 && type == OS_DATA_FILE && (awrites == ATOMIC_WRITES_ON || (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) && !os_file_set_atomic_writes(name, file)) { @@ -2426,8 +2430,6 @@ os_file_close_func( #ifdef __WIN__ BOOL ret; - ut_a(file); - ret = CloseHandle(file); if (ret) { @@ -2464,8 +2466,6 @@ os_file_close_no_error_handling( #ifdef __WIN__ BOOL ret; - ut_a(file); - ret = CloseHandle(file); if (ret) { @@ -2718,8 +2718,6 @@ os_file_flush_func( #ifdef __WIN__ BOOL ret; - ut_a(file); - os_n_fsyncs++; ret = FlushFileBuffers(file); @@ -3125,7 +3123,6 @@ os_file_read_func( os_bytes_read_since_printout += n; try_again: - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3152,30 +3149,15 @@ try_again: if (ret && len == n) { /* If page is encrypted we need to decrypt it first */ - /* - if (fil_page_is_encrypted((byte *)buf)) { - if (fil_decrypt_page( - NULL, - (byte *)buf, - n, - NULL, - &compressed, - NULL)) { - return FALSE; - } - } - */ - if (fil_page_is_compressed_encrypted((byte *)buf) || fil_page_is_encrypted((byte *)buf)) { - byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + byte *dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); // Decrypt the data - fil_space_decrypt( - (fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); + fil_space_decrypt((fil_space_crypt_t* ) NULL, + (byte *)buf, + n, + dst_frm); // Copy decrypted buffer back to buf memcpy(buf, dst_frm, n); ut_free(dst_frm); @@ -3202,32 +3184,18 @@ try_again: if ((ulint) ret == n) { /* If page is encrypted we need to decrypt it first */ - /* - if (fil_page_is_encrypted((byte *)buf)) { - if (fil_decrypt_page( - NULL, - (byte *)buf, - n, - NULL, - &compressed, - NULL)) { - return FALSE; - } - } - */ if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - - byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt( - (fil_space_crypt_t*) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); + fil_page_is_encrypted((byte *)buf)) { + + byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + // Decrypt the data + fil_space_decrypt((fil_space_crypt_t*) NULL, + (byte *)buf, + n, + dst_frm); + // Copy decrypted buffer back to buf + memcpy(buf, dst_frm, n); + ut_free(dst_frm); } /* Note that InnoDB writes files that are not formated @@ -3238,7 +3206,6 @@ try_again: fil_decompress_page(NULL, (byte *)buf, n, NULL); } - return(TRUE); } @@ -3302,7 +3269,6 @@ os_file_read_no_error_handling_func( os_bytes_read_since_printout += n; try_again: - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3329,31 +3295,17 @@ try_again: if (ret && len == n) { /* If page is encrypted we need to decrypt it first */ - /* - if (fil_page_is_encrypted((byte *)buf)) { - if (fil_decrypt_page( - NULL, - (byte *)buf, - n, - NULL, - &compressed, - NULL)) { - return (FALSE); - } - } - */ if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt( - (fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); + fil_page_is_encrypted((byte *)buf)) { + byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + // Decrypt the data + fil_space_decrypt((fil_space_crypt_t* ) NULL, + (byte *)buf, + n, + dst_frm); + // Copy decrypted buffer back to buf + memcpy(buf, dst_frm, n); + ut_free(dst_frm); } /* Note that InnoDB writes files that are not formated @@ -3376,37 +3328,19 @@ try_again: ret = os_file_pread(file, buf, n, offset, NULL); if ((ulint) ret == n) { - /* If the page is encrypted we need to decrypt it first */ - /* - if (fil_page_is_encrypted((byte *)buf)) { - if (fil_decrypt_page( - NULL, - (byte *)buf, - n, - NULL, - &compressed, - NULL)) { - return (FALSE); - } - } - */ - if (fil_page_is_compressed_encrypted((byte *)buf) || - fil_page_is_encrypted((byte *)buf)) { - - byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - // Decrypt the data - fil_space_decrypt( - (fil_space_crypt_t* ) NULL, - (byte *)buf, - n, - dst_frm); - // Copy decrypted buffer back to buf - memcpy(buf, dst_frm, n); - ut_free(dst_frm); - } - + fil_page_is_encrypted((byte *)buf)) { + byte * dst_frm = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + // Decrypt the data + fil_space_decrypt((fil_space_crypt_t* ) NULL, + (byte *)buf, + n, + dst_frm); + // Copy decrypted buffer back to buf + memcpy(buf, dst_frm, n); + ut_free(dst_frm); + } /* Note that InnoDB writes files that are not formated as file spaces and they do not have FIL_PAGE_TYPE @@ -3483,7 +3417,6 @@ os_file_write_func( os_n_file_writes++; - ut_ad(file); ut_ad(buf); ut_ad(n > 0); @@ -3646,7 +3579,7 @@ os_file_status( struct _stat64 statinfo; ret = _stat64(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { + if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) { /* file does not exist */ *exists = FALSE; return(TRUE); @@ -3674,7 +3607,7 @@ os_file_status( struct stat statinfo; ret = stat(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { + if (ret && (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG)) { /* file does not exist */ *exists = FALSE; return(TRUE); @@ -4814,7 +4747,7 @@ os_aio_array_reserve_slot( ulint page_encryption_key, /*!< page encryption key to be used */ ulint* write_size)/*!< in/out: Actual write size initialized - after fist successfull trim + after first successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ @@ -4942,8 +4875,7 @@ found: #endif /* Call page compression */ - tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), + tmp = fil_compress_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, @@ -4967,17 +4899,9 @@ found: os_mutex_enter(array->mutex); } -// if (srv_encrypt_tables) { -// page_encryption = TRUE; -// } - /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { - ulint real_len = len; - ulint ec = 0; - byte* tmp = NULL; - + if (message1 && type == OS_FILE_WRITE && page_encryption) { /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); @@ -4989,7 +4913,7 @@ found: fil_space_encrypt( fil_node_get_space_id(slot->message1), slot->offset, - NULL, + 0, /* QQ: Needs to be fixed to slot->lsn */ (byte *)buf, slot->len, slot->page_buf2, @@ -5302,7 +5226,6 @@ os_aio_func( #endif ulint wake_later; - ut_ad(file); ut_ad(buf); ut_ad(n > 0); ut_ad(n % OS_MIN_LOG_BLOCK_SIZE == 0); @@ -5396,6 +5319,7 @@ try_again: trx->io_reads++; trx->io_read += n; } + slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, page_compression, page_compression_level, @@ -5596,16 +5520,16 @@ os_aio_windows_handle( case OS_FILE_WRITE: if (slot->message1 && slot->page_encryption && slot->page_encryption_success) { ret_val = os_file_write(slot->name, slot->file, slot->page_buf2, - slot->offset, slot->len); - } else { - if (slot->message1 && slot->page_compression && slot->page_compression_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf, - slot->offset, slot->len); - } else { - ret_val = os_file_write(slot->name, slot->file, slot->buf, - slot->offset, slot->len); - } - } + slot->offset, slot->len); + } else { + if (slot->message1 && slot->page_compression && slot->page_compression_success) { + ret_val = os_file_write(slot->name, slot->file, slot->page_buf, + slot->offset, slot->len); + } else { + ret_val = os_file_write(slot->name, slot->file, slot->buf, + slot->offset, slot->len); + } + } break; case OS_FILE_READ: ret_val = os_file_read(slot->file, slot->buf, @@ -5659,16 +5583,13 @@ os_aio_windows_handle( os_slot_alloc_lzo_mem(slot); } #endif - fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); + fil_decompress_page(slot->page_buf, slot->buf, + slot->len, slot->write_size); } } else { /* OS_FILE_WRITE */ - if (slot->page_compression_success - && fil_page_is_compressed(slot->page_buf)) { + if (slot->page_compression_success && + fil_page_is_compressed(slot->page_buf)) { if (srv_use_trim && os_fallocate_failed == FALSE) { // Deallocate unused blocks from file system os_file_trim(slot); @@ -5774,34 +5695,28 @@ retry: ut_ad(slot->message1 != NULL); // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); + fil_space_decrypt(fil_node_get_space_id(slot->message1), + slot->buf, + slot->len, + slot->page_buf2); // Copy decrypted buffer back to buf memcpy(slot->buf, slot->page_buf2, slot->len); - } + } - /* If the page is page compressed and this is read, - we decompress before we annouce the read is - complete. For writes, we free the compressed page. */ + /* If the table is page compressed and this + is read, we decompress before we announce + the read is complete. For writes, we free + the compressed page. */ if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only // if it is not yet allocated. os_slot_alloc_page_buf(slot); - #ifdef HAVE_LZO if (fil_page_is_lzo_compressed(slot->buf)) { os_slot_alloc_lzo_mem(slot); } #endif - fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); + fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); } } else { /* OS_FILE_WRITE */ @@ -6906,7 +6821,8 @@ os_slot_alloc_page_buf( if (slot->page_buf == NULL) { byte* cbuf2; byte* cbuf; - /* We allocate extra to avoid memory overwrite on compression */ + /* We allocate extra to avoid memory overwrite on + compression */ cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); slot->page_compression_page = static_cast(cbuf2); @@ -6927,7 +6843,7 @@ os_slot_alloc_lzo_mem( os_aio_slot_t* slot) /*!< in: slot structure */ { ut_a(slot != NULL); - if (slot->lzo_mem == NULL) { + if(slot->lzo_mem == NULL) { slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); ut_a(slot->lzo_mem != NULL); memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); diff --git a/google/test_innodb_log_encryption.sh b/tests/test_innodb_log_encryption.sh similarity index 75% rename from google/test_innodb_log_encryption.sh rename to tests/test_innodb_log_encryption.sh index 413ed056448..774afa76d3b 100644 --- a/google/test_innodb_log_encryption.sh +++ b/tests/test_innodb_log_encryption.sh @@ -16,17 +16,17 @@ TEST_INSTANCE_DATA_DIR=${TEST_INSTANCE_PATH}/datadir google/instance restart ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys" google/instance restart ${TEST_INSTANCE_NAME} MYSQLD_EXTRA_ARGS="--innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=11" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=11" google/instance restart ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=12 --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=12 --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=123 --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=123 --innodb_encrypt_log=1" google/instance restart ${TEST_INSTANCE_NAME} # -- manually create a database sbtest # mysql> create database sbtest; @@ -37,12 +37,12 @@ sysbench --num-threads=10 --test=oltp --oltp-table-size=1000 --mysql-user=root - # -- change key version through mysql client # mysql -S ${TEST_INSTANCE_SOCK} k -u root -# mysql> set global variable danger_danger_dbug_crypto_key_version=7; +# mysql> set global variable debug_crypto_key_version=7; # ps aux | grep mysqld # -- simulate a fast shutdown # kill -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys" google/instance restart ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys" google/instance restart ${TEST_INSTANCE_NAME} google/instance restart ${TEST_INSTANCE_NAME} @@ -58,7 +58,7 @@ MYSQLD_EXTRA_ARGS="--innodb_fast_shutdown=0" google/instance restart ${TEST_INST google/instance stop ${TEST_INSTANCE_NAME} mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile0 ${TEST_INSTANCE_DATA_DIR}/ib_logfile0.1 mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile1 ${TEST_INSTANCE_DATA_DIR}/ib_logfile1.1 -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=777 --innodb_encrypt_log=1 --innodb_fast_shutdown=0" google/instance start ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=777 --innodb_encrypt_log=1 --innodb_fast_shutdown=0" google/instance start ${TEST_INSTANCE_NAME} grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 ################################################################## # - clean shutdown. @@ -83,7 +83,7 @@ grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 ################################################################## google/instance stop ${TEST_INSTANCE_NAME} mv ${TEST_INSTANCE_DIR} ${TEST_INSTANCE_DIR}.300 -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=888 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=888 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 ################################################################## # - fast shutdown. @@ -94,7 +94,7 @@ grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 google/instance stop ${TEST_INSTANCE_NAME} mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile0 ${TEST_INSTANCE_DATA_DIR}/ib_logfile0.3 mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile1 ${TEST_INSTANCE_DATA_DIR}/ib_logfile1.3 -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=999 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=999 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 ################################################################## # - fast shutdown while running workload. @@ -109,11 +109,11 @@ sysbench --num-threads=10 --test=oltp --oltp-table-size=1000 --mysql-user=root - google/instance stop ${TEST_INSTANCE_NAME} mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile0 ${TEST_INSTANCE_DATA_DIR}/ib_logfile0.4 mv ${TEST_INSTANCE_DATA_DIR}/ib_logfile1 ${TEST_INSTANCE_DATA_DIR}/ib_logfile1.4 -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys --danger_danger_dbug_crypto_key_version=333 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys --debug_crypto_key_version=333 --innodb_encrypt_log=1" google/instance start ${TEST_INSTANCE_NAME} grep -n corrupt ${TEST_INSTANCE_ERR_FILE} | tail -100 ################################################################## # - clean up ################################################################## google/instance stop ${TEST_INSTANCE_NAME} -MYSQLD_EXTRA_ARGS="--danger_danger_use_dbug_keys" google/instance start ${TEST_INSTANCE_NAME} +MYSQLD_EXTRA_ARGS="--debug_use_static_keys" google/instance start ${TEST_INSTANCE_NAME} google/instance stop ${TEST_INSTANCE_NAME}