From 2e7891080667c59ac80f788eef4d59d447595772 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 2 Jun 2021 08:40:30 -0700 Subject: [PATCH 01/23] MDEV-25635 Assertion failure when pushing from HAVING into WHERE of view This bug could manifest itself after pushing a where condition over a mergeable derived table / view / CTE DT into a grouping view / derived table / CTE V whose item list contained set functions with constant arguments such as MIN(2), SUM(1) etc. In such cases the field references used in the condition pushed into the view V that correspond set functions are wrapped into Item_direct_view_ref wrappers. Due to a wrong implementation of the virtual method const_item() for the class Item_direct_view_ref the wrapped set functions with constant arguments could be erroneously taken for constant items. This could lead to a wrong result set returned by the main select query in 10.2. In 10.4 where a possibility of pushing condition from HAVING into WHERE had been added this could cause a crash. Approved by Sergey Petrunya --- mysql-test/r/derived_cond_pushdown.result | 39 +++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 25 +++++++++++++++ sql/item.h | 5 ++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 25237aa11a9..28532ae88a4 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10634,4 +10634,43 @@ m 7 drop view v1; drop table t1; +# +# MDEV-25635: pushdown into grouping view using aggregate functions +# with constant arguments via a mergeable derived table +# +create table t1 (a int); +insert into t1 values (3), (7), (1), (3), (7), (7), (3); +create view v1 as select a, sum(1) as f, sum(1) as g from t1 group by a; +select * from v1; +a f g +1 1 1 +3 3 3 +7 3 3 +select * from (select * from v1) as dt where a=f and a=g; +a f g +1 1 1 +3 3 3 +explain extended select * from (select * from v1) as dt where a=f and a=g; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 100.00 Using where +3 DERIVED t1 ALL NULL NULL NULL NULL 7 100.00 Using temporary; Using filesort +Warnings: +Note 1003 select `v1`.`a` AS `a`,`v1`.`f` AS `f`,`v1`.`g` AS `g` from `test`.`v1` where `v1`.`a` = `v1`.`f` and `v1`.`a` = `v1`.`g` +create view v2 as select a, min(1) as f, min(1) as g from t1 group by a; +select * from v2; +a f g +1 1 1 +3 1 1 +7 1 1 +select * from (select * from v2) as dt where a=f and a=g; +a f g +1 1 1 +explain extended select * from (select * from v2) as dt where a=f and a=g; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 7 100.00 Using where +3 DERIVED t1 ALL NULL NULL NULL NULL 7 100.00 Using temporary; Using filesort +Warnings: +Note 1003 select `v2`.`a` AS `a`,`v2`.`f` AS `f`,`v2`.`g` AS `g` from `test`.`v2` where `v2`.`f` = `v2`.`a` and `v2`.`g` = `v2`.`a` +drop view v1,v2; +drop table t1; # End of 10.2 tests diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 31b49047bf1..58f38ac1e5a 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -2212,4 +2212,29 @@ select * from v1 where m > 0; drop view v1; drop table t1; +--echo # +--echo # MDEV-25635: pushdown into grouping view using aggregate functions +--echo # with constant arguments via a mergeable derived table +--echo # + +create table t1 (a int); +insert into t1 values (3), (7), (1), (3), (7), (7), (3); + +create view v1 as select a, sum(1) as f, sum(1) as g from t1 group by a; +select * from v1; +let $q1= +select * from (select * from v1) as dt where a=f and a=g; +eval $q1; +eval explain extended $q1; + +create view v2 as select a, min(1) as f, min(1) as g from t1 group by a; +select * from v2; +let $q2= +select * from (select * from v2) as dt where a=f and a=g; +eval $q2; +eval explain extended $q2; + +drop view v1,v2; +drop table t1; + --echo # End of 10.2 tests diff --git a/sql/item.h b/sql/item.h index c94709c733e..76be66d2a7c 100644 --- a/sql/item.h +++ b/sql/item.h @@ -4952,7 +4952,10 @@ public: table_map used_tables() const; void update_used_tables(); table_map not_null_tables() const; - bool const_item() const { return used_tables() == 0; } + bool const_item() const + { + return (*ref)->const_item() && (null_ref_table == NO_NULL_TABLE); + } TABLE *get_null_ref_table() const { return null_ref_table; } bool walk(Item_processor processor, bool walk_subquery, void *arg) { From fa0bbff032bc5717715fdf32ce4c8ebdfcf73944 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 2 Jun 2021 14:05:12 +0300 Subject: [PATCH 02/23] Fixed that compile-pentium64-valgrind-max works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removed Tokudb (no need to test this anymore with valgrind) - Added __attribute__(unused)) to a few places to be able to compile even if valgrind/memcheck.h is not installed. Reviewer: Marko Mäkelä --- BUILD/compile-pentium64-valgrind-max | 2 +- sql/field.cc | 2 +- storage/innobase/include/srv0mon.h | 2 +- storage/innobase/page/page0cur.cc | 26 +++++++++++++------------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/BUILD/compile-pentium64-valgrind-max b/BUILD/compile-pentium64-valgrind-max index 0653fb7fe75..84e78805246 100755 --- a/BUILD/compile-pentium64-valgrind-max +++ b/BUILD/compile-pentium64-valgrind-max @@ -33,6 +33,6 @@ path=`dirname $0` . "$path/SETUP.sh" extra_flags="$pentium64_cflags $debug_cflags $valgrind_flags" -extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs" +extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs --without-plugin-tokudb" . "$path/FINISH.sh" diff --git a/sql/field.cc b/sql/field.cc index 89c51288de8..cf4bb4ef4f2 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7806,7 +7806,7 @@ my_decimal *Field_varstring::val_decimal(my_decimal *decimal_value) #ifdef HAVE_valgrind void Field_varstring::mark_unused_memory_as_defined() { - uint used_length= get_length(); + uint used_length __attribute__((unused)) = get_length(); MEM_MAKE_DEFINED(get_data() + used_length, field_length - used_length); } #endif diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 58e36676398..934f88ac8ae 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -656,7 +656,7 @@ Use MONITOR_DEC if appropriate mutex protection exists. #ifdef HAVE_valgrind # define MONITOR_CHECK_DEFINED(value) do { \ - mon_type_t m = value; \ + mon_type_t m __attribute__((unused))= value; \ MEM_CHECK_DEFINED(&m, sizeof m); \ } while (0) #else /* HAVE_valgrind */ diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 9bf9fe66b33..14a0ab8aa8a 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -1299,12 +1299,12 @@ page_cur_insert_rec_low( #ifdef HAVE_valgrind { - const void* rec_start - = rec - rec_offs_extra_size(offsets); - ulint extra_size - = rec_offs_extra_size(offsets) - - (rec_offs_comp(offsets) - ? REC_N_NEW_EXTRA_BYTES + const void* rec_start __attribute__((unused)) + = rec - rec_offs_extra_size(offsets); + ulint extra_size __attribute__((unused)) + = rec_offs_extra_size(offsets) + - (rec_offs_comp(offsets) + ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES); /* All data bytes of the record must be valid. */ @@ -1530,13 +1530,13 @@ page_cur_insert_rec_zip( #ifdef HAVE_valgrind { - const void* rec_start - = rec - rec_offs_extra_size(offsets); - ulint extra_size - = rec_offs_extra_size(offsets) - - (rec_offs_comp(offsets) - ? REC_N_NEW_EXTRA_BYTES - : REC_N_OLD_EXTRA_BYTES); + const void* rec_start __attribute__((unused)) + = rec - rec_offs_extra_size(offsets); + ulint extra_size __attribute__((unused)) + = rec_offs_extra_size(offsets) + - (rec_offs_comp(offsets) + ? REC_N_NEW_EXTRA_BYTES + : REC_N_OLD_EXTRA_BYTES); /* All data bytes of the record must be valid. */ MEM_CHECK_DEFINED(rec, rec_offs_data_size(offsets)); From 5c896472b6cb315fc54091a342ec37a7c4b5421d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 2 Jun 2021 23:10:21 +0200 Subject: [PATCH 03/23] MDEV-25672 table alias from previous statement interferes later commands only perform the "correct table name" check for *new* generated columns, but not for already existing ones - they're guaranteed to be valid --- mysql-test/suite/vcol/r/vcol_syntax.result | 11 ++++++++++- mysql-test/suite/vcol/t/vcol_syntax.test | 17 +++++++++++++---- sql/item.h | 2 +- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_syntax.result b/mysql-test/suite/vcol/r/vcol_syntax.result index c8983f34c93..0063f38ea36 100644 --- a/mysql-test/suite/vcol/r/vcol_syntax.result +++ b/mysql-test/suite/vcol/r/vcol_syntax.result @@ -1,4 +1,3 @@ -drop table if exists t1; set @OLD_SQL_MODE=@@SESSION.SQL_MODE; create table t1 (a int, b int generated always as (a+1)); show create table t1; @@ -88,3 +87,13 @@ create table t1 (x int, y int default test2.t1.x); ERROR 42S22: Unknown column '`test2`.`t1`.`x`' in 'DEFAULT' create table t1 (x int, check (test2.t1.x > 0)); ERROR 42S22: Unknown column '`test2`.`t1`.`x`' in 'CHECK' +# +# MDEV-25672 table alias from previous statement interferes later commands +# +create table t1 (a int, v_a int generated always as (a)); +update t1 as x set a = 1; +alter table t1 force; +drop table t1; +# +# End of 10.2 tests +# diff --git a/mysql-test/suite/vcol/t/vcol_syntax.test b/mysql-test/suite/vcol/t/vcol_syntax.test index f425b52ab79..3c8a50a7f36 100644 --- a/mysql-test/suite/vcol/t/vcol_syntax.test +++ b/mysql-test/suite/vcol/t/vcol_syntax.test @@ -1,10 +1,6 @@ # # test syntax # ---disable_warnings -drop table if exists t1; ---enable_warnings - set @OLD_SQL_MODE=@@SESSION.SQL_MODE; create table t1 (a int, b int generated always as (a+1)); show create table t1; @@ -72,3 +68,16 @@ create table t1 (x int, y int check (y > test2.t1.x)); create table t1 (x int, y int default test2.t1.x); --error ER_BAD_FIELD_ERROR create table t1 (x int, check (test2.t1.x > 0)); + +--echo # +--echo # MDEV-25672 table alias from previous statement interferes later commands +--echo # +create table t1 (a int, v_a int generated always as (a)); +update t1 as x set a = 1; +alter table t1 force; +drop table t1; + + +--echo # +--echo # End of 10.2 tests +--echo # diff --git a/sql/item.h b/sql/item.h index 76be66d2a7c..cc1914a7ad4 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2841,7 +2841,7 @@ public: bool check_table_name_processor(void *arg) { Check_table_name_prm &p= *(Check_table_name_prm *) arg; - if (p.table_name.length && table_name) + if (!field && p.table_name.length && table_name) { DBUG_ASSERT(p.db.length); if ((db_name && From 663bc849b5a26a5325adf009a8e8fa9155c6b833 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 26 May 2021 23:41:59 -0700 Subject: [PATCH 04/23] MDEV-25714 Join using derived with aggregation returns incorrect results If a join query uses a derived table (view / CTE) with GROUP BY clause then the execution plan for such join may employ split optimization. When this optimization is employed the derived table is not materialized. Rather only some partitions of the derived table are subject to grouping. Split optimization can be applied only if: - there are some indexes over the tables used in the join specifying the derived table whose prefixes partially cover the field items used in the GROUP BY list (such indexes are called splitting indexes) - the WHERE condition of the join query contains conjunctive equalities between columns of the derived table that comprise major parts of splitting indexes and columns of the other join tables. When the optimizer evaluates extending of a partial join by the rows of the derived table it always considers a possibility of using split optimization. Different splitting indexes can be used depending on the extended partial join. At some rare conditions, for example, when there is a non-splitting covering index for a table joined in the join specifying the derived table usage of a splitting index to produce rows needed for grouping may be still less beneficial than usage of such covering index without any splitting technique. The function JOIN_TAB::choose_best_splitting() must take this into account. Approved by Oleksandr Byelkin --- mysql-test/main/derived_cond_pushdown.result | 2 +- mysql-test/main/derived_split_innodb.result | 61 ++++++++++++++++++++ mysql-test/main/derived_split_innodb.test | 37 ++++++++++++ sql/opt_split.cc | 27 +++++++-- 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index f3d63b5887f..5fc01112642 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -16712,7 +16712,7 @@ EXPLAIN EXTENDED SELECT * FROM v1 JOIN v2 ON v1.f = v2.f; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables -3 LATERAL DERIVED NULL NULL NULL NULL NULL NULL NULL NULL no matching row in const table +3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL no matching row in const table Warnings: Note 1003 /* select#1 */ select NULL AS `f`,`v2`.`f` AS `f` from `test`.`t1` `a` straight_join `test`.`t1` `b` join `test`.`v2` where 0 DROP VIEW v1,v2; diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index 0b57e72b821..7ea3b689f23 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -174,3 +174,64 @@ id select_type table type possible_keys key key_len ref rows Extra 2 LATERAL DERIVED t1 ref a,a_2 a 5 test.t1.a 1 Using where; Using temporary; Using filesort 2 LATERAL DERIVED t2 ref c c 5 test.t1.b 1 Using index DROP TABLE t1, t2; +# +# Bug mdev-25714: usage non-splitting covering index is cheaper than +# usage of the best splitting index for one group +# +create table t1 ( +id int not null, itemid int not null, index idx (itemid) +) engine=innodb; +insert into t1 values (1, 2), (2,2), (4,2), (4,2), (0,3), (3,3); +create table t2 (id int not null) engine=innodb; +insert into t2 values (2); +create table t3 ( +id int not null, itemid int not null, userid int not null, primary key (id), +index idx1 (userid, itemid), index idx2 (itemid) +) engine innodb; +insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +analyze table t1,t2,t3; +Table Op Msg_type Msg_text +test.t1 analyze status OK +test.t2 analyze status OK +test.t3 analyze status OK +set optimizer_switch='split_materialized=on'; +explain select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t1 ref idx idx 4 test.t2.id 1 +1 PRIMARY ref key0 key0 9 test.t2.id,test.t1.id 2 +2 DERIVED t3 ref idx1,idx2 idx1 4 const 5 Using where; Using index +select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id itemid id id +4 2 4 2 +4 2 4 2 +set optimizer_switch='split_materialized=off'; +explain select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t1 ref idx idx 4 test.t2.id 1 +1 PRIMARY ref key0 key0 9 test.t2.id,test.t1.id 2 +2 DERIVED t3 ref idx1 idx1 4 const 5 Using where; Using index +select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id itemid id id +4 2 4 2 +4 2 4 2 +drop table t1,t2,t3; +set optimizer_switch='split_materialized=default'; +# End of 10.3 tests diff --git a/mysql-test/main/derived_split_innodb.test b/mysql-test/main/derived_split_innodb.test index 19a6ecf216f..6f33c71eede 100644 --- a/mysql-test/main/derived_split_innodb.test +++ b/mysql-test/main/derived_split_innodb.test @@ -150,3 +150,40 @@ eval set statement optimizer_switch='split_materialized=on' for $query; DROP TABLE t1, t2; +--echo # +--echo # Bug mdev-25714: usage non-splitting covering index is cheaper than +--echo # usage of the best splitting index for one group +--echo # + +create table t1 ( + id int not null, itemid int not null, index idx (itemid) +) engine=innodb; +insert into t1 values (1, 2), (2,2), (4,2), (4,2), (0,3), (3,3); +create table t2 (id int not null) engine=innodb; +insert into t2 values (2); +create table t3 ( + id int not null, itemid int not null, userid int not null, primary key (id), + index idx1 (userid, itemid), index idx2 (itemid) +) engine innodb; +insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +analyze table t1,t2,t3; + +let $q= +select t1.id, t1.itemid, dt.id, t2.id + from t1, + (select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, + t2 + where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; + +set optimizer_switch='split_materialized=on'; +eval explain $q; +eval $q; + +set optimizer_switch='split_materialized=off'; +eval explain $q; +eval $q; + +drop table t1,t2,t3; +set optimizer_switch='split_materialized=default'; + +--echo # End of 10.3 tests diff --git a/sql/opt_split.cc b/sql/opt_split.cc index c3a2d03a93b..edf9ae3deff 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -960,11 +960,7 @@ SplM_plan_info * JOIN_TAB::choose_best_splitting(double record_count, in the cache */ spl_plan= spl_opt_info->find_plan(best_table, best_key, best_key_parts); - if (!spl_plan && - (spl_plan= (SplM_plan_info *) thd->alloc(sizeof(SplM_plan_info))) && - (spl_plan->best_positions= - (POSITION *) thd->alloc(sizeof(POSITION) * join->table_count)) && - !spl_opt_info->plan_cache.push_back(spl_plan)) + if (!spl_plan) { /* The plan for the chosen key has not been found in the cache. @@ -974,6 +970,27 @@ SplM_plan_info * JOIN_TAB::choose_best_splitting(double record_count, reset_validity_vars_for_keyuses(best_key_keyuse_ext_start, best_table, best_key, remaining_tables, true); choose_plan(join, all_table_map & ~join->const_table_map); + + /* + Check that the chosen plan is really a splitting plan. + If not or if there is not enough memory to save the plan in the cache + then just return with no splitting plan. + */ + POSITION *first_non_const_pos= join->best_positions + join->const_tables; + TABLE *table= first_non_const_pos->table->table; + key_map spl_keys= table->keys_usable_for_splitting; + if (!(first_non_const_pos->key && + spl_keys.is_set(first_non_const_pos->key->key)) || + !(spl_plan= (SplM_plan_info *) thd->alloc(sizeof(SplM_plan_info))) || + !(spl_plan->best_positions= + (POSITION *) thd->alloc(sizeof(POSITION) * join->table_count)) || + spl_opt_info->plan_cache.push_back(spl_plan)) + { + reset_validity_vars_for_keyuses(best_key_keyuse_ext_start, best_table, + best_key, remaining_tables, false); + return 0; + } + spl_plan->keyuse_ext_start= best_key_keyuse_ext_start; spl_plan->table= best_table; spl_plan->key= best_key; From 0b797130c674a4dd8b8dcf35d3ade38353e19284 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 26 May 2021 23:41:59 -0700 Subject: [PATCH 05/23] MDEV-25714 Join using derived with aggregation returns incorrect results If a join query uses a derived table (view / CTE) with GROUP BY clause then the execution plan for such join may employ split optimization. When this optimization is employed the derived table is not materialized. Rather only some partitions of the derived table are subject to grouping. Split optimization can be applied only if: - there are some indexes over the tables used in the join specifying the derived table whose prefixes partially cover the field items used in the GROUP BY list (such indexes are called splitting indexes) - the WHERE condition of the join query contains conjunctive equalities between columns of the derived table that comprise major parts of splitting indexes and columns of the other join tables. When the optimizer evaluates extending of a partial join by the rows of the derived table it always considers a possibility of using split optimization. Different splitting indexes can be used depending on the extended partial join. At some rare conditions, for example, when there is a non-splitting covering index for a table joined in the join specifying the derived table usage of a splitting index to produce rows needed for grouping may be still less beneficial than usage of such covering index without any splitting technique. The function JOIN_TAB::choose_best_splitting() must take this into account. Approved by Oleksandr Byelkin --- mysql-test/main/derived_cond_pushdown.result | 2 +- mysql-test/main/derived_split_innodb.result | 61 ++++++++++++++++++++ mysql-test/main/derived_split_innodb.test | 37 ++++++++++++ sql/opt_split.cc | 27 +++++++-- 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index 7644e65a868..52d8ccd1b80 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -16703,7 +16703,7 @@ EXPLAIN EXTENDED SELECT * FROM v1 JOIN v2 ON v1.f = v2.f; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables -3 LATERAL DERIVED NULL NULL NULL NULL NULL NULL NULL NULL no matching row in const table +3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL no matching row in const table Warnings: Note 1003 /* select#1 */ select NULL AS `f`,`v2`.`f` AS `f` from `test`.`t1` `a` straight_join `test`.`t1` `b` join `test`.`v2` where 0 DROP VIEW v1,v2; diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index ff71b7df097..6119145105b 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -175,3 +175,64 @@ id select_type table type possible_keys key key_len ref rows Extra 3 LATERAL DERIVED t1 ref a,a_2 a 5 test.t1.a 1 Using where; Using temporary; Using filesort 3 LATERAL DERIVED t2 ref c c 5 test.t1.b 1 Using index DROP TABLE t1, t2; +# +# Bug mdev-25714: usage non-splitting covering index is cheaper than +# usage of the best splitting index for one group +# +create table t1 ( +id int not null, itemid int not null, index idx (itemid) +) engine=innodb; +insert into t1 values (1, 2), (2,2), (4,2), (4,2), (0,3), (3,3); +create table t2 (id int not null) engine=innodb; +insert into t2 values (2); +create table t3 ( +id int not null, itemid int not null, userid int not null, primary key (id), +index idx1 (userid, itemid), index idx2 (itemid) +) engine innodb; +insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +analyze table t1,t2,t3; +Table Op Msg_type Msg_text +test.t1 analyze status OK +test.t2 analyze status OK +test.t3 analyze status OK +set optimizer_switch='split_materialized=on'; +explain select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t1 ref idx idx 4 test.t2.id 1 +1 PRIMARY ref key0 key0 9 test.t2.id,test.t1.id 2 +2 DERIVED t3 ref idx1,idx2 idx1 4 const 5 Using where; Using index +select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id itemid id id +4 2 4 2 +4 2 4 2 +set optimizer_switch='split_materialized=off'; +explain select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t1 ref idx idx 4 test.t2.id 1 +1 PRIMARY ref key0 key0 9 test.t2.id,test.t1.id 2 +2 DERIVED t3 ref idx1 idx1 4 const 5 Using where; Using index +select t1.id, t1.itemid, dt.id, t2.id +from t1, +(select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, +t2 +where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; +id itemid id id +4 2 4 2 +4 2 4 2 +drop table t1,t2,t3; +set optimizer_switch='split_materialized=default'; +# End of 10.3 tests diff --git a/mysql-test/main/derived_split_innodb.test b/mysql-test/main/derived_split_innodb.test index 19a6ecf216f..6f33c71eede 100644 --- a/mysql-test/main/derived_split_innodb.test +++ b/mysql-test/main/derived_split_innodb.test @@ -150,3 +150,40 @@ eval set statement optimizer_switch='split_materialized=on' for $query; DROP TABLE t1, t2; +--echo # +--echo # Bug mdev-25714: usage non-splitting covering index is cheaper than +--echo # usage of the best splitting index for one group +--echo # + +create table t1 ( + id int not null, itemid int not null, index idx (itemid) +) engine=innodb; +insert into t1 values (1, 2), (2,2), (4,2), (4,2), (0,3), (3,3); +create table t2 (id int not null) engine=innodb; +insert into t2 values (2); +create table t3 ( + id int not null, itemid int not null, userid int not null, primary key (id), + index idx1 (userid, itemid), index idx2 (itemid) +) engine innodb; +insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +analyze table t1,t2,t3; + +let $q= +select t1.id, t1.itemid, dt.id, t2.id + from t1, + (select itemid, max(id) as id from t3 where userid = 1 group by itemid) dt, + t2 + where t1.id = dt.id and t1.itemid = dt.itemid and t2.id=t1.itemid; + +set optimizer_switch='split_materialized=on'; +eval explain $q; +eval $q; + +set optimizer_switch='split_materialized=off'; +eval explain $q; +eval $q; + +drop table t1,t2,t3; +set optimizer_switch='split_materialized=default'; + +--echo # End of 10.3 tests diff --git a/sql/opt_split.cc b/sql/opt_split.cc index 395422de3c3..d272638f00c 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -960,11 +960,7 @@ SplM_plan_info * JOIN_TAB::choose_best_splitting(double record_count, in the cache */ spl_plan= spl_opt_info->find_plan(best_table, best_key, best_key_parts); - if (!spl_plan && - (spl_plan= (SplM_plan_info *) thd->alloc(sizeof(SplM_plan_info))) && - (spl_plan->best_positions= - (POSITION *) thd->alloc(sizeof(POSITION) * join->table_count)) && - !spl_opt_info->plan_cache.push_back(spl_plan)) + if (!spl_plan) { /* The plan for the chosen key has not been found in the cache. @@ -974,6 +970,27 @@ SplM_plan_info * JOIN_TAB::choose_best_splitting(double record_count, reset_validity_vars_for_keyuses(best_key_keyuse_ext_start, best_table, best_key, remaining_tables, true); choose_plan(join, all_table_map & ~join->const_table_map); + + /* + Check that the chosen plan is really a splitting plan. + If not or if there is not enough memory to save the plan in the cache + then just return with no splitting plan. + */ + POSITION *first_non_const_pos= join->best_positions + join->const_tables; + TABLE *table= first_non_const_pos->table->table; + key_map spl_keys= table->keys_usable_for_splitting; + if (!(first_non_const_pos->key && + spl_keys.is_set(first_non_const_pos->key->key)) || + !(spl_plan= (SplM_plan_info *) thd->alloc(sizeof(SplM_plan_info))) || + !(spl_plan->best_positions= + (POSITION *) thd->alloc(sizeof(POSITION) * join->table_count)) || + spl_opt_info->plan_cache.push_back(spl_plan)) + { + reset_validity_vars_for_keyuses(best_key_keyuse_ext_start, best_table, + best_key, remaining_tables, false); + return 0; + } + spl_plan->keyuse_ext_start= best_key_keyuse_ext_start; spl_plan->table= best_table; spl_plan->key= best_key; From 385f4316c3b8250943383145c115000ee153e3de Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 3 Jun 2021 20:43:04 -0700 Subject: [PATCH 06/23] Corrected the test case of MDEV-25714 in order to have the same EXPLAIN output as in 10.3 --- mysql-test/main/derived_split_innodb.result | 4 ++++ mysql-test/main/derived_split_innodb.test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index 6119145105b..63db8c94dde 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -190,6 +190,8 @@ id int not null, itemid int not null, userid int not null, primary key (id), index idx1 (userid, itemid), index idx2 (itemid) ) engine innodb; insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +set use_stat_tables='never'; +set optimizer_use_condition_selectivity=1; analyze table t1,t2,t3; Table Op Msg_type Msg_text test.t1 analyze status OK @@ -235,4 +237,6 @@ id itemid id id 4 2 4 2 drop table t1,t2,t3; set optimizer_switch='split_materialized=default'; +set use_stat_tables=default; +set optimizer_use_condition_selectivity=default; # End of 10.3 tests diff --git a/mysql-test/main/derived_split_innodb.test b/mysql-test/main/derived_split_innodb.test index 6f33c71eede..22793b448da 100644 --- a/mysql-test/main/derived_split_innodb.test +++ b/mysql-test/main/derived_split_innodb.test @@ -166,6 +166,8 @@ create table t3 ( index idx1 (userid, itemid), index idx2 (itemid) ) engine innodb; insert into t3 values (1,1,1), (2,1,1), (3,2,1), (4,2,1), (5,3,1); +set use_stat_tables='never'; +set optimizer_use_condition_selectivity=1; analyze table t1,t2,t3; let $q= @@ -185,5 +187,7 @@ eval $q; drop table t1,t2,t3; set optimizer_switch='split_materialized=default'; +set use_stat_tables=default; +set optimizer_use_condition_selectivity=default; --echo # End of 10.3 tests From 2d38c5e64edbfdc8368954775880c2d677fbd195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 4 Jun 2021 09:35:18 +0300 Subject: [PATCH 07/23] MDEV-17749 fixup: ./mtr --embedded main.lock_kill (again) --- mysql-test/main/lock_kill.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/lock_kill.test b/mysql-test/main/lock_kill.test index dfeebbc4f6a..2c1396e5ff3 100644 --- a/mysql-test/main/lock_kill.test +++ b/mysql-test/main/lock_kill.test @@ -17,7 +17,7 @@ LOCK TABLE t1 WRITE; eval KILL $conid; --enable_query_log --connection con1 ---error 0,2013,ER_CONNECTION_KILLED +--error 0,2006,2013,ER_CONNECTION_KILLED reap; --connection default --disconnect con1 @@ -35,7 +35,7 @@ LOCK TABLE t1 WRITE, t2 WRITE; eval KILL $conid; --enable_query_log --connection con1 ---error 0,2013,ER_CONNECTION_KILLED +--error 0,2006,2013,ER_CONNECTION_KILLED reap; --connection default --disconnect con1 From 7eed97ed9fe2b0c1e69167576be12759dffcd926 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Wed, 26 May 2021 14:15:26 +0200 Subject: [PATCH 08/23] MDEV-25777: JAVA_INCLUDE_PATH and JAVA_INCLUDE_PATH2 not found with out of source configuration and Ninja generator - As solution `PLUGIN_CONNECT=NO` use early check to disable plugin: Solution suggested by wlad@mariadb.com - `JNI_FOUND` is a internal result variable and should be set with cached library and header variables (like `JAVA_INCLUDE_PATH`) defined. * Note: wrapper cmake/FindJNI.cmake runs first time and cmake native Find returns only cached variable, like `JAVA_INCLUDE_PATH`, results variable are not cached). Reviewed by: serg@mariadb.com --- cmake/FindJNI.cmake | 2 +- storage/connect/CMakeLists.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/FindJNI.cmake b/cmake/FindJNI.cmake index 12305d7c86d..b2c6f849c87 100644 --- a/cmake/FindJNI.cmake +++ b/cmake/FindJNI.cmake @@ -1,4 +1,4 @@ -if(JAVA_AWT_LIBRARY) +if(JAVA_AWT_LIBRARY AND JAVA_INCLUDE_PATH) set(JNI_FOUND TRUE) return() endif() diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index e9533e8f3e5..d3632b689fe 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -13,6 +13,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +IF(WITHOUT_DYNAMIC_PLUGINS OR WITH_NONE OR ("${PLUGIN_CONNECT}" STREQUAL "NO")) + RETURN() +ENDIF() + SET(CONNECT_PLUGIN_STATIC "connect") SET(CONNECT_PLUGIN_DYNAMIC "connect") From ddddfc33c7825609a25ce9531183a0b0fb97f206 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 4 Jun 2021 14:57:11 +0200 Subject: [PATCH 09/23] Fix mtr tests with file_key_managment extension for Windows Commit b5615eff0d00 introduced comment in result file during shutdown. In case of Windows for the tests involving `file_key_managment.so` as plugin-load-add the tests will be overwritten with .dll extension. The same happens with environment variable `$FILE_KEY_MANAGMENT_SO`. So the patch is removing the extension to be extension agnostic. Reviewed by: wlad@mariadb.com --- .../encrypted_master_switch_to_unencrypted.test | 2 +- .../suite/encryption/r/innodb-bad-key-change2.result | 12 ++++++------ .../suite/encryption/r/innodb-bad-key-change4.result | 6 +++--- .../encryption/r/innodb-encryption-disable.result | 4 ++-- .../encryption/r/innodb-remove-encryption.result | 2 +- .../suite/encryption/t/innodb-bad-key-change2.test | 12 ++++++------ .../suite/encryption/t/innodb-bad-key-change3.test | 6 +++--- .../suite/encryption/t/innodb-bad-key-change4.test | 6 +++--- .../encryption/t/innodb-encryption-disable.test | 4 ++-- .../suite/encryption/t/innodb-remove-encryption.test | 2 +- mysql-test/suite/maria/encrypt-wrong-key.test | 6 +++--- 11 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test index eec72d64066..1e1b0cbd353 100644 --- a/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test +++ b/mysql-test/suite/binlog_encryption/encrypted_master_switch_to_unencrypted.test @@ -65,7 +65,7 @@ INSERT INTO table1_no_encryption SELECT NULL,NOW(),b FROM table1_no_encryption; --echo # Part 2: restart master, now with binlog encryption --echo ##################################################### ---let $rpl_server_parameters= --encrypt-binlog=1 --plugin-load-add=$FILE_KEY_MANAGEMENT_SO --file-key-management --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt +--let $rpl_server_parameters= --encrypt-binlog=1 --plugin-load-add=file_key_management --file-key-management --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt --let $rpl_server_number= 1 --source restart_server.inc diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index 543c3bc29b2..af1028f1331 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -7,12 +7,12 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt SELECT * FROM t1; ERROR 42S02: Table 'test.t1' doesn't exist in engine SHOW WARNINGS; @@ -35,11 +35,11 @@ test.t1 check Error Table 'test.t1' doesn't exist in engine test.t1 check status Operation failed SHOW WARNINGS; Level Code Message -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt FLUSH TABLES t1 FOR EXPORT; backup: t1 UNLOCK TABLES; -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt ALTER TABLE t1 DISCARD TABLESPACE; ERROR 42S02: Table 'test.t1' doesn't exist in engine DROP TABLE t1; @@ -47,7 +47,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; ALTER TABLE t1 DISCARD TABLESPACE; restore: t1 .ibd and .cfg files -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt ALTER TABLE t1 DISCARD TABLESPACE; Warnings: Warning 1814 Tablespace has been discarded for table `t1` @@ -61,7 +61,7 @@ t1 CREATE TABLE `t1` ( `f` varchar(8) DEFAULT NULL, PRIMARY KEY (`pk`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=4 -# restart: --innodb-encrypt-tables --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --innodb-encrypt-tables --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt RENAME TABLE t1 TO t1new; ERROR HY000: Error on rename of './test/t1' to './test/t1new' (errno: 155 "The table does not exist in the storage engine") ALTER TABLE t1 RENAME TO t1new; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index e37ee8eb8cd..ad218457068 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -4,12 +4,12 @@ call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9] call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys3.txt OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize Error Table 'test.t1' doesn't exist in engine @@ -22,5 +22,5 @@ test.t1 check Error Table 'test.t1' doesn't exist in engine test.t1 check status Operation failed SHOW WARNINGS; Level Code Message -# restart: --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt DROP TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index e49a6b759e9..bb4f02b9c39 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -4,7 +4,7 @@ call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9] call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` is corrupted"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -# restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt create table t5 ( `intcol1` int(32) DEFAULT NULL, `intcol2` int(32) DEFAULT NULL, @@ -27,6 +27,6 @@ select * from t1; ERROR 42S02: Table 'test.t1' doesn't exist in engine select * from t5; ERROR 42S02: Table 'test.t5' doesn't exist in engine -# restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt +# restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt drop table t1; drop table t5; diff --git a/mysql-test/suite/encryption/r/innodb-remove-encryption.result b/mysql-test/suite/encryption/r/innodb-remove-encryption.result index 08b31cb568d..9bce59dbbea 100644 --- a/mysql-test/suite/encryption/r/innodb-remove-encryption.result +++ b/mysql-test/suite/encryption/r/innodb-remove-encryption.result @@ -6,7 +6,7 @@ flush tables; create table t1(a int not null primary key, b char(200)) engine=innodb; # Restart server with encryption -# restart: --plugin-load-add=file_key_management.so --loose-file-key-management --loose-file-key-management-filename=MYSQL_TEST_DIR/std_data/keys.txt --file-key-management-encryption-algorithm=aes_cbc --innodb-encrypt-tables=ON --innodb-encryption-threads=4 --innodb-tablespaces-encryption --innodb-encryption-rotate-key-age=15 +# restart: --plugin-load-add=file_key_management --loose-file-key-management --loose-file-key-management-filename=MYSQL_TEST_DIR/std_data/keys.txt --file-key-management-encryption-algorithm=aes_cbc --innodb-encrypt-tables=ON --innodb-encryption-threads=4 --innodb-tablespaces-encryption --innodb-encryption-rotate-key-age=15 # Wait until encryption threads have encrypted all tablespaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; NAME diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index bdbf2327e5d..19399b1e891 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -20,7 +20,7 @@ call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE f # for innodb_checksum_algorithm=full_crc32 only call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc SET GLOBAL innodb_file_per_table = ON; @@ -29,7 +29,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc --error ER_NO_SUCH_TABLE_IN_ENGINE @@ -48,7 +48,7 @@ CHECK TABLE t1; --replace_regex /key_id [1-9][0-9]*/\1 / SHOW WARNINGS; ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc let MYSQLD_DATADIR =`SELECT @@datadir`; @@ -60,7 +60,7 @@ ib_backup_tablespaces("test", "t1"); EOF UNLOCK TABLES; ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc --error ER_NO_SUCH_TABLE_IN_ENGINE @@ -78,7 +78,7 @@ ib_discard_tablespaces("test", "t1"); ib_restore_tablespaces("test", "t1"); EOF ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc ALTER TABLE t1 DISCARD TABLESPACE; @@ -92,7 +92,7 @@ EOF ALTER TABLE t1 IMPORT TABLESPACE; SHOW CREATE TABLE t1; ---let $restart_parameters= --innodb-encrypt-tables --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters= --innodb-encrypt-tables --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc --error ER_ERROR_ON_RENAME diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test index dbd04748143..9c2918f3118 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test @@ -25,7 +25,7 @@ call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* becau 4;770A8A65DA156D24EE2A093277530143 EOF ---exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc @@ -62,7 +62,7 @@ ib_discard_tablespaces("test", "t1"); ib_restore_tablespaces("test", "t1"); EOF ---exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --source include/restart_mysqld.inc @@ -89,7 +89,7 @@ SELECT * FROM t1; 4;770A8A65DA156D24EE2A093277530143 EOF ---exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc DROP TABLE t1; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index b341fc81d39..58517f14978 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -16,7 +16,7 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); # for innodb_checksum_algorithm=full_crc32 only call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc SET GLOBAL innodb_file_per_table = ON; @@ -25,7 +25,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc --replace_regex /key_id [1-9][0-9]*/\1 / @@ -38,7 +38,7 @@ CHECK TABLE t1; --replace_regex /key_id [1-9][0-9]*/\1 / SHOW WARNINGS; ---let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc DROP TABLE t1; diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 4d0aa04bc56..2097a4ad184 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -16,7 +16,7 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` is corrupted"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); ---let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc create table t5 ( @@ -48,7 +48,7 @@ select * from t1; --error ER_NO_SUCH_TABLE_IN_ENGINE select * from t5; ---let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc drop table t1; diff --git a/mysql-test/suite/encryption/t/innodb-remove-encryption.test b/mysql-test/suite/encryption/t/innodb-remove-encryption.test index 24e00a00a02..3d719dbef74 100644 --- a/mysql-test/suite/encryption/t/innodb-remove-encryption.test +++ b/mysql-test/suite/encryption/t/innodb-remove-encryption.test @@ -18,7 +18,7 @@ create table t1(a int not null primary key, b char(200)) engine=innodb; --echo --echo # Restart server with encryption --- let $restart_parameters=--plugin-load-add=$FILE_KEY_MANAGEMENT_SO --loose-file-key-management --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt --file-key-management-encryption-algorithm=aes_cbc --innodb-encrypt-tables=ON --innodb-encryption-threads=4 --innodb-tablespaces-encryption --innodb-encryption-rotate-key-age=15 +-- let $restart_parameters=--plugin-load-add=file_key_management --loose-file-key-management --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt --file-key-management-encryption-algorithm=aes_cbc --innodb-encrypt-tables=ON --innodb-encryption-threads=4 --innodb-tablespaces-encryption --innodb-encryption-rotate-key-age=15 -- source include/restart_mysqld.inc --echo # Wait until encryption threads have encrypted all tablespaces diff --git a/mysql-test/suite/maria/encrypt-wrong-key.test b/mysql-test/suite/maria/encrypt-wrong-key.test index 2afa785dd0f..ca65e1018d0 100644 --- a/mysql-test/suite/maria/encrypt-wrong-key.test +++ b/mysql-test/suite/maria/encrypt-wrong-key.test @@ -17,7 +17,7 @@ call mtr.add_suppression("Failed to decrypt"); 1;770A8A65DA156D24EE2A093277530142 EOF ---exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc @@ -32,7 +32,7 @@ INSERT INTO t1 VALUES (1); 2;770A8A65DA156D24EE2A093277530143 EOF ---exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc @@ -44,7 +44,7 @@ INSERT INTO t1 VALUES (2); --shutdown_server --source include/wait_until_disconnected.inc ---exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--exec echo "restart:--aria-encrypt-tables=1 --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc From b1b4d67bcda32472f5b9c46465bff9db86904a00 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 4 Jun 2021 15:00:34 +0200 Subject: [PATCH 10/23] MDEV-21373 DBUG compilation - bad synchronization in ha_heap::external_lock() ha_heap::external_lock contains some consistency checks for the table,# in a debug compilation. This code suffers from lack of synchronization, in a rare case where mysql_lock_tables() fail, and unlock is forced, even if lock was not previously taken. To workaround, require EXTRA_DEBUG compile definition in order to activate the consistency checks.The code still might be useful in some cases - but the audience are developers looking for errors in single-threaded scenarios, rather than multiuser stress-tests. --- storage/heap/ha_heap.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 5af68f098a4..8a8e1f74e47 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -423,7 +423,7 @@ int ha_heap::reset_auto_increment(ulonglong value) int ha_heap::external_lock(THD *thd, int lock_type) { -#ifndef DBUG_OFF +#if !defined(DBUG_OFF) && defined(EXTRA_DEBUG) if (lock_type == F_UNLCK && file->s->changed && heap_check_heap(file, 0)) return HA_ERR_CRASHED; #endif From cebc435592043a83d5f430aec8bdaa79cd7c1d44 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 5 Jun 2021 16:57:10 +0200 Subject: [PATCH 11/23] MDEV-25859 - HeidiSQL 11.3 --- win/packaging/heidisql.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/win/packaging/heidisql.cmake b/win/packaging/heidisql.cmake index 6de033eacb2..c57282ce90a 100644 --- a/win/packaging/heidisql.cmake +++ b/win/packaging/heidisql.cmake @@ -1,4 +1,4 @@ -SET(HEIDISQL_BASE_NAME "HeidiSQL_11.2_32_Portable") +SET(HEIDISQL_BASE_NAME "HeidiSQL_11.3_32_Portable") SET(HEIDISQL_ZIP "${HEIDISQL_BASE_NAME}.zip") SET(HEIDISQL_URL "http://www.heidisql.com/downloads/releases/${HEIDISQL_ZIP}") SET(HEIDISQL_DOWNLOAD_DIR ${THIRD_PARTY_DOWNLOAD_LOCATION}/${HEIDISQL_BASE_NAME}) From d4a6e3a6988e47e826e4c7cf770c38a94cb58d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 6 May 2021 12:08:38 -0700 Subject: [PATCH 12/23] Deb: Misc cleanup and autobake-deb.sh and Salsa-CI fixes * Clean up autobake-deb.sh - No need to define any TokuDB rules, there is no such package - No need to define RocksDB arch, it already has "Architecture:" line - No need to dh-systemd backwards compat stanza, neither Debian Jessie nor Ubuntu Xenial has any new MariaDB 10.5 releases anymore - Minor spelling fixes * Ensure dch runs non-interactively so builds pass with new dch version A recent version of dch (available in Ubuntu Hirsute and Debian Bullseye) had a change in behaviour that it started prompting if the DEBEMAIL or EMAIL variable as unset, asking for confirmation. We can't have anything interactive in our build scripts, so prevent this prompt by giving --controlmaint to the command, so it always uses the name and email from the debian/control file and does not prompt anything. The command-line argument has been around for a long time, so it is safe to use on all Debian/Ubuntu builds we have. See https://manpages.debian.org/jessie/devscripts/dch.1.en.html Since MariaDB 10.5 is the oldest release we still release for Ubuntu Hisute and Debian Bullseye, merge this on 10.5 and from there merge up to latest. No need to consider 10.2, 10.3 and 10.4 as those will not be released for Ubuntu Bullseye or Ubuntu Hirsute. * Minor Salsa-CI cleanup - Fix spelling (synced from downstream Debian) * Many minor spelling fixes (synced from downstream Debian) --- debian/autobake-deb.sh | 34 +++++------------------- debian/mariadb-server-10.5.README.Debian | 16 +++++------ debian/mariadb-server-10.5.mysql.default | 4 +-- debian/mariadb-server-10.5.postinst | 2 +- debian/rules | 2 +- debian/salsa-ci.yml | 9 ++----- debian/tests/control | 2 +- debian/tests/smoke | 2 +- 8 files changed, 23 insertions(+), 48 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index f7be508c1bd..516b842ba57 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -6,12 +6,12 @@ # Always keep the actual packaging as up-to-date as possible following the latest # Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh # tests for backwards compatibility and strip away parts on older builders or -# specfic build environments. +# specific build environments. # Exit immediately on any error set -e -# This file is invocated from Buildbot and Travis-CI to build deb packages. +# This file is invoked from Buildbot and Travis-CI to build deb packages. # As both of those CI systems have many parallel jobs that include different # parts of the test suite, we don't need to run the mysql-test-run at all when # building the deb packages here. @@ -24,7 +24,7 @@ then echo >> debian/control cat storage/columnstore/columnstore/debian/control >> debian/control - # ColumnStore is explcitly disabled in the native build, so allow it now + # ColumnStore is explicitly disabled in the native build, so allow it now # when build it when triggered by autobake-deb.sh sed '/-DPLUGIN_COLUMNSTORE=NO/d' -i debian/rules fi @@ -42,7 +42,7 @@ then sed "/Package: mariadb-plugin-columnstore/,/^$/d" -i debian/control fi -# Don't build or try to put files in a package for selected plugins and compontents on Travis-CI +# Don't build or try to put files in a package for selected plugins and components on Travis-CI # in order to keep build small (in both duration and disk space) if [[ $TRAVIS ]] then @@ -62,27 +62,6 @@ then sed "/Package: libmariadbd-dev/,/^$/d" -i debian/control fi -## Skip TokuDB if arch is not amd64 -if [[ ! $(dpkg-architecture -q DEB_BUILD_ARCH) =~ amd64 ]] -then - sed '/Package: mariadb-plugin-tokudb/,/^$/d' -i debian/control -fi - - -if [[ $(arch) =~ i[346]86 ]] -then - sed "/Package: mariadb-plugin-rocksdb/,/^$/d" -i debian/control -fi - -# From Debian Stretch/Ubuntu Bionic onwards dh-systemd is just an empty -# transitional metapackage and the functionality was merged into debhelper. -# In Ubuntu Hirsute is was completely removed, so it can't be referenced anymore. -# Keep using it only on Debian Jessie and Ubuntu Xenial. -if apt-cache madison dh-systemd | grep 'dh-systemd' >/dev/null 2>&1 -then - sed 's/debhelper (>= 9.20160709~),/debhelper (>= 9), dh-systemd,/' -i debian/control -fi - # If rocksdb-tools is not available (before Debian Buster and Ubuntu Disco) # remove the dependency from the RocksDB plugin so it can install properly # and instead ship the one built from MariaDB sources @@ -93,7 +72,8 @@ then echo "usr/bin/sst_dump" >> debian/mariadb-plugin-rocksdb.install fi -# From Debian Buster/Ubuntu Bionic, libcurl4 replaces libcurl3 +# If libcurl4 is not available (before Debian Buster and Ubuntu Bionic) +# use older libcurl3 instead if ! apt-cache madison libcurl4 | grep 'libcurl4' >/dev/null 2>&1 then sed 's/libcurl4/libcurl3/g' -i debian/control @@ -111,7 +91,7 @@ CODENAME="$(lsb_release -sc)" EPOCH="1:" VERSION="${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" -dch -b -D "${CODENAME}" -v "${VERSION}" "Automatic build with ${LOGSTRING}." +dch -b -D "${CODENAME}" -v "${VERSION}" "Automatic build with ${LOGSTRING}." --controlmaint echo "Creating package version ${VERSION} ... " diff --git a/debian/mariadb-server-10.5.README.Debian b/debian/mariadb-server-10.5.README.Debian index f93484271fb..6042249a706 100644 --- a/debian/mariadb-server-10.5.README.Debian +++ b/debian/mariadb-server-10.5.README.Debian @@ -31,13 +31,13 @@ name has been kept as a symbolic link to the new name for backwards compatibilit * NATIVE SYSTEMD SERVICE INTRODUCED IN MARIADB 10.1 =================================================== -From MariaDB 10.1 onwards the upstream mariadb.service and mariadb@.service are +From MariaDB 10.1 onward the upstream mariadb.service and mariadb@.service are used to provide the full systemd experience. Some features available in traditional /etc/init.d/mysql have been changed. For details see https://mariadb.com/kb/en/mariadb/systemd/ -* MIXING PACKAGES FROM MARIAD.ORG AND OFFICIAL DEBIAN REPOSITORIES +* MIXING PACKAGES FROM MARIADB.ORG AND OFFICIAL DEBIAN REPOSITORIES ================================================================== Please note that the MariaDB packaging in official Debian repositories are of @@ -57,7 +57,7 @@ revision string. On new installs no root password is set and no debian-sys-maint user is created anymore. Instead the MariaDB root account is set to be authenticated -using the unix socket, e.g. any mysqld invocation by root or via sudo will +using the Unix socket, e.g. any mysqld invocation by root or via sudo will let the user see the mysqld prompt. You may never ever delete the mysql user "root". Although it has no password @@ -65,7 +65,7 @@ is set, the unix_auth plugin ensure that it can only be run locally as the root user. The credentials in /etc/mysql/debian.cnf specify the user which is used by the -init scripts to stop the server and perform logrotation. This used to be the +init scripts to stop the server and perform log rotation. This used to be the debian-sys-maint user which is no longer used as root can run directly. If you have start/stop problems make sure that the /etc/mysql/debian.cnf file @@ -79,7 +79,7 @@ file as is has been obsoleted. MariaDB in Debian is secure by default, because: - It only listens to the localhost socket and cannot be accessed remotely unless - the sysadmin changes the configurationin /etc/mysql to allow so. + the sysadmin changes the configuration in /etc/mysql to allow so. - There is no debian-sys-maint with password in /etc/mysql/debian.cnf anymore. - There is no root account with password anymore. The system admin needs to create one themselves if they need it. With no password, all issues related @@ -141,9 +141,9 @@ https://mariadb.com/kb It is recommended you create additional admin users for your database administration needs in addition to the default root user. -If your local unix account is the one you want to have local super user +If your local Unix account is the one you want to have local super user access on your database with you can create the following account that will -only work for the local unix user connecting to the database locally. +only work for the local Unix user connecting to the database locally. sudo /usr/bin/mysql -e "GRANT ALL ON *.* TO '$USER'@'localhost' IDENTIFIED VIA unix_socket WITH GRANT OPTION" @@ -176,7 +176,7 @@ https://mariadb.com/kb/en/configuring-mariadb-with-mycnf/. ============================== If the MySQL server is acting as a replication slave, you should not -set --tmpdir to point to a directory on a memory-based filesystem or to +set --tmpdir to point to a directory on a memory-based file system or to a directory that is cleared when the server host restarts. A replication slave needs some of its temporary files to survive a machine restart so that it can replicate temporary tables or LOAD DATA INFILE operations. If diff --git a/debian/mariadb-server-10.5.mysql.default b/debian/mariadb-server-10.5.mysql.default index 3810c73bf52..36079edecb2 100644 --- a/debian/mariadb-server-10.5.mysql.default +++ b/debian/mariadb-server-10.5.mysql.default @@ -1,9 +1,9 @@ # # NOTE: This file is read only by the traditional SysV init script. -# Debian 9 and Ubuntu 17.04 onwards do not normally read this file as they use +# Debian 9 and Ubuntu 17.04 onward do not normally read this file as they use # systemd by default. # -# For similar behaviour, systemd users should override ExecStart by dropping +# For similar behavior, systemd users should override ExecStart by dropping # files into /etc/systemd/system/mariadb.service.d/ # # See also: diff --git a/debian/mariadb-server-10.5.postinst b/debian/mariadb-server-10.5.postinst index 9b0120cb58f..97828ed15e8 100644 --- a/debian/mariadb-server-10.5.postinst +++ b/debian/mariadb-server-10.5.postinst @@ -36,7 +36,7 @@ case "$1" in # latest 'mariadb' file. This has also the added benefit that anything that # invokes traditional sysv init with either 'mysql' or 'mariadb' will end up # controlling this newly installed MariaDB, and thus we maintain better - # backwards compatiblity. + # backwards compatibility. # # Note that the 'Provides' line is also updated to avoid 'insserv' exiting # on failure (when it is run by update-rc.d) because of duplicate service diff --git a/debian/rules b/debian/rules index d029eead974..5e7da691b00 100755 --- a/debian/rules +++ b/debian/rules @@ -143,7 +143,7 @@ endif # If mariadb-test package is removed, also remove most of it's files grep --quiet "Package: mariadb-test" debian/control || rm -rf $(TMP)/usr/share/mysql/mysql-test - # Delete private files from libraries so they don't get shipped in the -dev pacakges + # Delete private files from libraries so they don't get shipped in the -dev packages rm -r $(TMP)/usr/include/mariadb/server/private # Don't ship sql-bench at all, just delete it completely even though it builds diff --git a/debian/salsa-ci.yml b/debian/salsa-ci.yml index af9e95b2bfa..29ea9afd9d0 100644 --- a/debian/salsa-ci.yml +++ b/debian/salsa-ci.yml @@ -69,9 +69,6 @@ build i386: image: $SALSA_CI_IMAGES_BASE_I386 variables: ARCH: 'i386' - except: - variables: - - $SALSA_CI_DISABLE_BUILD_PACKAGE_I386 =~ /^(1|yes|true)$/ build native deb: extends: .build-package @@ -545,8 +542,6 @@ mysql-8.0 Sid to mariadb-10.5 upgrade: except: variables: - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/ - # Installation often fails (not a MariaDB reason), so do not require this test to pass - allow_failure: true # Upgrading from MySQL 8.0 with datadir in place is not possible. Users need to do a data dump. # The Debian maintainer scripts detect this situation and simply moves old datadir aside and start fresh. @@ -638,12 +633,12 @@ mariadb.org-10.5 to mariadb-10.5 upgrade: - echo 'deb http://mirror.one.com/mariadb/repo/10.5/debian sid main' > /etc/apt/sources.list.d/mariadb.list - apt-get update - *test-install-readline-in-sid-for-backwards-compat - # The 10.5.9 relase is missing mariadb-plugin-columnstore, define all other packages but it to avoid hitting the error: + # The 10.5.9 release is missing mariadb-plugin-columnstore, define all other packages but it to avoid hitting the error: # The following packages have unmet dependencies: # mariadb-plugin-columnstore : Depends: mariadb-server-10.5 (= 1:10.5.8+maria~sid) but 1:10.5.9+maria~sid is to be installed - apt-get install -y libmariadb3 'libmariadb-*' 'libmariadbd*' 'mariadb-c*' 'mariadb-b*' 'mariadb-s*' 'mariadb-t*' 'mariadb-plugin-con*' 'mariadb-plugin-cr*' 'mariadb-plugin-g*' 'mariadb-plugin-m*' 'mariadb-plugin-o*' 'mariadb-plugin-s*' # Once 10.5.10 is out, revert back to: - # Package libmariadbclient-dev from mariadb.org conficts with libmariadb-dev in Sid, so cannot use wildcard that would include it + # Package libmariadbclient-dev from mariadb.org conflicts with libmariadb-dev in Sid, so cannot use wildcard that would include it #- apt-get install -y 'mariadb*' libmariadb3 'libmariadb-*' 'libmariadbd*' - *test-verify-initial # Install MariaDB built in this commit diff --git a/debian/tests/control b/debian/tests/control index 45097079629..3706829904f 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -1,7 +1,7 @@ Tests: smoke # RocksDB is not built for all archs. Rather than duplicating the condition # for its existence (see the list in debian/control), install it if available -# and check in the test if it's funcational when it should be. +# and check in the test if it's functional when it should be. # The plugin package also already depends on the other one. Depends: mariadb-plugin-rocksdb | mariadb-server-10.5 Restrictions: allow-stderr needs-root isolation-container diff --git a/debian/tests/smoke b/debian/tests/smoke index 5de9d425e16..4e06feae20c 100644 --- a/debian/tests/smoke +++ b/debian/tests/smoke @@ -31,7 +31,7 @@ if ! which systemctl then if ! /etc/init.d/mariadb status then - echo "Did not find systemctl and deamon was not running, starting it.." + echo "Did not find systemctl and daemon was not running, starting it.." /etc/init.d/mariadb start fi else From 9f9a925c399b9d960f095be0886f56f51396eb04 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 6 Jun 2021 08:46:59 +0200 Subject: [PATCH 13/23] MDEV-23815 Windows : mysql_upgrade_wizard fails, if service name has spaces The fix is to quote service name parameter, when it is passed to mysql_upgrade_service subprocess. --- win/upgrade_wizard/upgradeDlg.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/win/upgrade_wizard/upgradeDlg.cpp b/win/upgrade_wizard/upgradeDlg.cpp index d996c0ebe5d..d0dd6a3fa75 100644 --- a/win/upgrade_wizard/upgradeDlg.cpp +++ b/win/upgrade_wizard/upgradeDlg.cpp @@ -367,7 +367,10 @@ void CUpgradeDlg::UpgradeOneService(const string& servicename) ErrorExit("Stdout SetHandleInformation"); string commandline("mysql_upgrade_service.exe --service="); + commandline += "\""; commandline += servicename; + commandline += "\""; + si.cb = sizeof(si); si.hStdInput= GetStdHandle(STD_INPUT_HANDLE); si.hStdOutput= hPipeWrite; @@ -397,7 +400,7 @@ void CUpgradeDlg::UpgradeOneService(const string& servicename) else { /* - Creating a process with CREATE_BREAKAWAY_FROM_JOB, reset this flag + Creating a process with CREATE_BREAKAWAY_FROM_JOB failed, reset this flag and retry. */ if (!CreateProcess(NULL, (LPSTR)commandline.c_str(), NULL, NULL, TRUE, From 3c922d6defcfa6be819fe33794abcf507bb70c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 28 May 2021 22:04:17 -0700 Subject: [PATCH 14/23] Revert "CONNECT: move jar files to /usr/share and include them in DEBs" This partially reverts commit d7321893d8c50071632a102e17a7869da9cb03a5. The *.jar files are not being built and all Debian builds are failing as dh_install stops on missing files. To build them we would need to also add new Java build dependencies. In a stable release (10.2->10.5) we shouldn't add new files and certainly not any new build dependencies, so reverting commit. Also, the files are located in a different path, and already included in the mariadb-test-data package: /usr/share/mysql/mysql-test/plugin/connect/connect/std_data/JavaWrappers.jar /usr/share/mysql/mysql-test/plugin/connect/connect/std_data/JdbcMariaDB.jar /usr/share/mysql/mysql-test/plugin/connect/connect/std_data/Mongo2.jar /usr/share/mysql/mysql-test/plugin/connect/connect/std_data/Mongo3.jar This change needs to be redesigned and applies only on 10.6 or newer. --- debian/mariadb-plugin-connect.install | 4 ---- 1 file changed, 4 deletions(-) diff --git a/debian/mariadb-plugin-connect.install b/debian/mariadb-plugin-connect.install index 7b5a5f0633e..8a7aee412df 100644 --- a/debian/mariadb-plugin-connect.install +++ b/debian/mariadb-plugin-connect.install @@ -1,6 +1,2 @@ etc/mysql/conf.d/connect.cnf usr/lib/mysql/plugin/ha_connect.so -usr/share/mysql/Mongo2.jar -usr/share/mysql/Mongo3.jar -usr/share/mysql/JavaWrappers.jar -usr/share/mysql/JdbcInterface.jar From f456e716feac2fe9a2b1eb5247128c271e1a4e83 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 2 Jun 2021 18:29:30 +0300 Subject: [PATCH 15/23] Make maria.repair more resiliant for different failures This is because on different compilation and server configurations the memory usage is different and the query can get killed in different places with different error messages as a result. Reviewer: None (trival change) --- mysql-test/suite/maria/repair.result | 7 ------- mysql-test/suite/maria/repair.test | 2 ++ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/maria/repair.result b/mysql-test/suite/maria/repair.result index 296f251aa36..722d9f28712 100644 --- a/mysql-test/suite/maria/repair.result +++ b/mysql-test/suite/maria/repair.result @@ -29,12 +29,5 @@ CREATE TABLE t1 (i INT) ENGINE=Aria; INSERT INTO t1 VALUES (1); SET max_session_mem_used=50000; REPAIR LOCAL TABLE t1 USE_FRM; -Table Op Msg_type Msg_text -t1 repair error Failed to open partially repaired table -Warnings: -Error 1290 The MariaDB server is running with the --max-thread-mem-used=50000 option so it cannot execute this statement REPAIR LOCAL TABLE t1; -Table Op Msg_type Msg_text -test.t1 repair Error The MariaDB server is running with the --max-thread-mem-used=50000 option so it cannot execute this statement -test.t1 repair error Corrupt DROP TABLE t1; diff --git a/mysql-test/suite/maria/repair.test b/mysql-test/suite/maria/repair.test index 13165269b76..571f861c512 100644 --- a/mysql-test/suite/maria/repair.test +++ b/mysql-test/suite/maria/repair.test @@ -36,6 +36,8 @@ DROP TABLE t1; CREATE TABLE t1 (i INT) ENGINE=Aria; INSERT INTO t1 VALUES (1); SET max_session_mem_used=50000; +--disable_result_log REPAIR LOCAL TABLE t1 USE_FRM; REPAIR LOCAL TABLE t1; +--enable_result_log DROP TABLE t1; From 310dff5d847b3c117ab6bca8e6ccbcc8bca818d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 7 Jun 2021 19:07:45 +0300 Subject: [PATCH 16/23] MDEV-25783: Change buffer records are lost under heavy load ibuf_read_merge_pages(): Disable some code that was added in MDEV-20394 in order to avoid a server hang if the change buffer is corrupted, presumably due to the race condition in recovery that was later fixed in MDEV-24449. The code will still be available in debug builds when the command line option --debug=d,ibuf_merge_corruption is specified. Due to MDEV-19514, the impact of this code is much worse starting with the 10.5 series. In older versions, the code was only enabled during a shutdown with innodb_fast_shutdown=0, but in 10.5 it was active during the normal operation of the server. --- storage/innobase/ibuf/ibuf0ibuf.cc | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 3c7f2de2a67..9288a496735 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2296,9 +2296,11 @@ bool ibuf_delete_rec(const page_id_t page_id, btr_pcur_t* pcur, static void ibuf_read_merge_pages(const uint32_t* space_ids, const uint32_t* page_nos, ulint n_stored) { +#ifndef DBUG_OFF mem_heap_t* heap = mem_heap_create(512); ulint dops[IBUF_OP_COUNT]; memset(dops, 0, sizeof(dops)); +#endif for (ulint i = 0; i < n_stored; i++) { const ulint space_id = space_ids[i]; @@ -2331,6 +2333,28 @@ tablespace_deleted: goto tablespace_deleted; } } +#ifndef DBUG_OFF + DBUG_EXECUTE_IF("ibuf_merge_corruption", goto work_around;); + continue; + + /* The following code works around a hang when the + change buffer is corrupted, likely due to the race + condition in crash recovery that was fixed in + MDEV-24449. But, it also introduces corruption by + itself in the following scenario: + + (1) We merged buffered changes in buf_page_get_gen() + (2) We committed the mini-transaction + (3) Redo log and the page with the merged changes is written + (4) A write completion callback thread evicts the page. + (5) Other threads buffer changes for that page. + (6) We will wrongly discard those newly buffered changes below. + + This code will be available in debug builds, so that + users may try to fix a shutdown hang that occurs due + to a corrupted change buffer. */ + +work_around: /* Prevent an infinite loop, by removing entries from the change buffer also in the case the bitmap bits were wrongly clear even though buffered changes exist. */ @@ -2377,10 +2401,13 @@ done: ibuf_mtr_commit(&mtr); btr_pcur_close(&pcur); mem_heap_empty(heap); +#endif } +#ifndef DBUG_OFF ibuf_add_ops(ibuf.n_discarded_ops, dops); mem_heap_free(heap); +#endif } /*********************************************************************//** From eed419b487dd6f555cdfd92aa0756fdec1b9060f Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 7 Jun 2021 15:38:38 +0300 Subject: [PATCH 17/23] Fixed a DBUG_ASSERT when running zerofill() on aria tables This happended when an aria table was already used by the system before running zerofill, which could happen with Aria system tables. Fixed by using a proper page type when reading pages in zerofill --- storage/maria/ma_check.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index e1c85cd2d44..5b37cac74d5 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -3413,6 +3413,9 @@ static my_bool maria_zerofill_index(HA_CHECK *param, MARIA_HA *info, my_bool zero_lsn= (share->base.born_transactional && !(param->testflag & T_ZEROFILL_KEEP_LSN)); int error= 1; + enum pagecache_page_type page_type= (share->base.born_transactional ? + PAGECACHE_LSN_PAGE : + PAGECACHE_PLAIN_PAGE); DBUG_ENTER("maria_zerofill_index"); if (!(param->testflag & T_SILENT)) @@ -3427,7 +3430,7 @@ static my_bool maria_zerofill_index(HA_CHECK *param, MARIA_HA *info, if (!(buff= pagecache_read(share->pagecache, &share->kfile, page, DFLT_INIT_HITS, 0, - PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_WRITE, + page_type, PAGECACHE_LOCK_WRITE, &page_link.link))) { pagecache_unlock_by_link(share->pagecache, page_link.link, @@ -3504,6 +3507,9 @@ static my_bool maria_zerofill_data(HA_CHECK *param, MARIA_HA *info, uint block_size= share->block_size; MARIA_FILE_BITMAP *bitmap= &share->bitmap; my_bool zero_lsn= !(param->testflag & T_ZEROFILL_KEEP_LSN), error; + enum pagecache_page_type read_page_type= (share->base.born_transactional ? + PAGECACHE_LSN_PAGE : + PAGECACHE_PLAIN_PAGE); DBUG_ENTER("maria_zerofill_data"); /* This works only with BLOCK_RECORD files */ @@ -3527,7 +3533,7 @@ static my_bool maria_zerofill_data(HA_CHECK *param, MARIA_HA *info, if (!(buff= pagecache_read(share->pagecache, &info->dfile, page, 1, 0, - PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_WRITE, + read_page_type, PAGECACHE_LOCK_WRITE, &page_link.link))) { _ma_check_print_error(param, From bf5c050fd28f616d789a02b0fbd6fd8ff53c78d3 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 7 Jun 2021 17:40:30 +0300 Subject: [PATCH 18/23] MDEV-25866 Upgrade from pre-10.5.10 to 10.5.10 causes CHECK errors on encrypted Aria tables Hard to do a test case, but tested by hand and verified that mysql_upgrade will update the encrypted MariaDB tables. --- storage/maria/ha_maria.cc | 20 ++++++++++++++++++++ storage/maria/ha_maria.h | 1 + 2 files changed, 21 insertions(+) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 79d32886659..5bf6754ff9d 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -4198,6 +4198,26 @@ int ha_maria::find_unique_row(uchar *record, uint constrain_no) return rc; } + +/** + Check if a table needs to be repaired +*/ + +int ha_maria::check_for_upgrade(HA_CHECK_OPT *check) +{ + if (table->s->mysql_version && table->s->mysql_version <= 100509 && + (file->s->base.extra_options & MA_EXTRA_OPTIONS_ENCRYPTED)) + { + /* + Encrypted tables before 10.5.9 had a bug where LSN was not + stored on the pages. These must be repaired! + */ + return HA_ADMIN_NEEDS_ALTER; + } + return HA_ADMIN_OK; +} + + struct st_mysql_storage_engine maria_storage_engine= { MYSQL_HANDLERTON_INTERFACE_VERSION }; diff --git a/storage/maria/ha_maria.h b/storage/maria/ha_maria.h index b2c664a072d..6b4302145dd 100644 --- a/storage/maria/ha_maria.h +++ b/storage/maria/ha_maria.h @@ -144,6 +144,7 @@ public: int check(THD * thd, HA_CHECK_OPT * check_opt) override; int analyze(THD * thd, HA_CHECK_OPT * check_opt) override; int repair(THD * thd, HA_CHECK_OPT * check_opt) override; + int check_for_upgrade(HA_CHECK_OPT *check_opt) override; bool check_and_repair(THD * thd) override final; bool is_crashed() const override final; bool is_changed() const; From 8149e4d0a139b901c8902b5b9fae371cef47275f Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 7 Jun 2021 15:08:18 -0700 Subject: [PATCH 19/23] MDEV-25682 Explain shows an execution plan different from actually executed If a select query contained an ORDER BY clause that followed a LIMIT clause or an ORDER BY clause or ORDER BY with LIMIT the EXPLAIN output for the query showed an execution plan different from that was actually executed. Approved by Roman Nozdrin --- mysql-test/main/order_by.result | 25 +++++++++++++++++++++++++ mysql-test/main/order_by.test | 16 ++++++++++++++++ sql/item_subselect.cc | 2 ++ sql/sql_select.cc | 2 +- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result index f4e88d6e6e3..c8f63f881cc 100644 --- a/mysql-test/main/order_by.result +++ b/mysql-test/main/order_by.result @@ -3460,6 +3460,31 @@ SET max_length_for_sort_data=@save_max_length_for_sort_data; SET max_sort_length= @save_max_sort_length; SET sql_select_limit= @save_sql_select_limit; DROP TABLE t1; +# +# MDEV-25682: EXPLAIN for SELECT with ORDER BY after [ORDER BY] LIMIT +# +create table t1 (a int); +insert into t1 values (3), (7), (1); +explain (select a from t1 limit 2) order by a desc; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL Using filesort +(select a from t1 limit 2) order by a desc; +a +7 +3 +create table t2 (a int, b int); +insert into t2 values (3,70), (7,10), (1,40), (4,30); +explain (select b,a from t2 order by a limit 3) order by b desc; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 4 Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL Using filesort +(select b,a from t2 order by a limit 3) order by b desc; +b a +70 3 +40 1 +30 4 +drop table t1,t2; # End of 10.2 tests # # MDEV-16214: Incorrect plan taken by the optimizer , uses INDEX instead of ref access with ORDER BY diff --git a/mysql-test/main/order_by.test b/mysql-test/main/order_by.test index 74884144a98..08d5982b220 100644 --- a/mysql-test/main/order_by.test +++ b/mysql-test/main/order_by.test @@ -2293,6 +2293,22 @@ SET max_sort_length= @save_max_sort_length; SET sql_select_limit= @save_sql_select_limit; DROP TABLE t1; +--echo # +--echo # MDEV-25682: EXPLAIN for SELECT with ORDER BY after [ORDER BY] LIMIT +--echo # + +create table t1 (a int); +insert into t1 values (3), (7), (1); +explain (select a from t1 limit 2) order by a desc; +(select a from t1 limit 2) order by a desc; + +create table t2 (a int, b int); +insert into t2 values (3,70), (7,10), (1,40), (4,30); +explain (select b,a from t2 order by a limit 3) order by b desc; +(select b,a from t2 order by a limit 3) order by b desc; + +drop table t1,t2; + --echo # End of 10.2 tests --echo # diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 9f43561151d..352d80da92c 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -274,6 +274,8 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref) res= TRUE; goto end; } + if (sl == unit->first_select() && !sl->next_select()) + unit->fake_select_lex= 0; } } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 6c090ea5352..7f4c6d24b8d 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -26370,7 +26370,7 @@ bool mysql_explain_union(THD *thd, SELECT_LEX_UNIT *unit, select_result *result) sl->options|= SELECT_DESCRIBE; } - if (unit->is_unit_op()) + if (unit->is_unit_op() || unit->fake_select_lex) { if (unit->union_needs_tmp_table() && unit->fake_select_lex) { From bb28bffc3ed179a9912aced2b873e43999111887 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 7 Jun 2021 19:51:57 -0700 Subject: [PATCH 20/23] Ported the test case for MDEV-25682 from 10.2 No fix for this bug is needed starting from version 10.4. --- mysql-test/main/order_by.result | 25 +++++++++++++++++++++++++ mysql-test/main/order_by.test | 16 ++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result index 826daf0542f..129bd8928f2 100644 --- a/mysql-test/main/order_by.result +++ b/mysql-test/main/order_by.result @@ -3536,6 +3536,31 @@ SET max_length_for_sort_data=@save_max_length_for_sort_data; SET max_sort_length= @save_max_sort_length; SET sql_select_limit= @save_sql_select_limit; DROP TABLE t1; +# +# MDEV-25682: EXPLAIN for SELECT with ORDER BY after [ORDER BY] LIMIT +# +create table t1 (a int); +insert into t1 values (3), (7), (1); +explain (select a from t1 limit 2) order by a desc; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 2 Using filesort +2 DERIVED t1 ALL NULL NULL NULL NULL 3 +(select a from t1 limit 2) order by a desc; +a +7 +3 +create table t2 (a int, b int); +insert into t2 values (3,70), (7,10), (1,40), (4,30); +explain (select b,a from t2 order by a limit 3) order by b desc; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 Using filesort +2 DERIVED t2 ALL NULL NULL NULL NULL 4 Using filesort +(select b,a from t2 order by a limit 3) order by b desc; +b a +70 3 +40 1 +30 4 +drop table t1,t2; # End of 10.2 tests # # MDEV-16214: Incorrect plan taken by the optimizer , uses INDEX instead of ref access with ORDER BY diff --git a/mysql-test/main/order_by.test b/mysql-test/main/order_by.test index 1bf353fd69d..0bf0311a642 100644 --- a/mysql-test/main/order_by.test +++ b/mysql-test/main/order_by.test @@ -2294,6 +2294,22 @@ SET max_sort_length= @save_max_sort_length; SET sql_select_limit= @save_sql_select_limit; DROP TABLE t1; +--echo # +--echo # MDEV-25682: EXPLAIN for SELECT with ORDER BY after [ORDER BY] LIMIT +--echo # + +create table t1 (a int); +insert into t1 values (3), (7), (1); +explain (select a from t1 limit 2) order by a desc; +(select a from t1 limit 2) order by a desc; + +create table t2 (a int, b int); +insert into t2 values (3,70), (7,10), (1,40), (4,30); +explain (select b,a from t2 order by a limit 3) order by b desc; +(select b,a from t2 order by a limit 3) order by b desc; + +drop table t1,t2; + --echo # End of 10.2 tests --echo # From dfa2d0bc13362b949b1b1699955583f74e7db90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 7 Jun 2021 17:46:59 +0300 Subject: [PATCH 21/23] MDEV-25869 Change buffer entries are lost on InnoDB restart buf_read_ibuf_merge_pages(): If space->size is 0, invoke fil_space_get_size() to determine the size of the tablespace by reading the header page. Only after that proceed to delete any entries that are beyond the end of the tablespace. Otherwise, we could be deleting valid entries that actually need to be applied. This fixes a regression that had been introduced in commit b80df9eba23b4eb9694e770a41135127c6dbc5df (MDEV-21069), which aimed to avoid crashes during DROP TABLE of corrupted tables. --- storage/innobase/buf/buf0rea.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index ad583e577c4..6b68e9f8fa5 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2020, MariaDB Corporation. +Copyright (c) 2015, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -806,13 +806,18 @@ tablespace_deleted: continue; } - if (UNIV_UNLIKELY(page_nos[i] >= space->size)) { + ulint size = space->size; + if (!size) { + size = fil_space_get_size(space->id); + } + + if (UNIV_UNLIKELY(page_nos[i] >= size)) { do { ibuf_delete_recs(page_id_t(space_ids[i], page_nos[i])); } while (++i < n_stored && space_ids[i - 1] == space_ids[i] - && page_nos[i] >= space->size); + && page_nos[i] >= size); i--; next: fil_space_release(space); From b8d38c5e39d526e006f4e8e8977ac4c6166bc4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Jun 2021 15:03:50 +0300 Subject: [PATCH 22/23] dict_index_build_internal_clust(): Catch MDEV-20131 on CREATE TABLE --- storage/innobase/dict/dict0dict.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 59851acdaab..1ffeb8d1e16 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2318,8 +2318,8 @@ dict_index_build_internal_clust( ulint i; ibool* indexed; - ut_ad(dict_index_is_clust(index)); - ut_ad(!dict_index_is_ibuf(index)); + ut_ad(index->is_primary()); + ut_ad(!index->has_virtual()); ut_ad(mutex_own(&dict_sys.mutex)); From 75a65d3201a4486af96cf3277b6c5a4ba460eef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 9 Jun 2021 14:23:20 +0300 Subject: [PATCH 23/23] MDEV-25886 CHECK TABLE crash with DB_MISSING_HISTORY if innodb_read_only Occasionally, the test innodb.alter_copy would fail in MariaDB 10.6.1, reporting DB_MISSING_HISTORY during CHECK TABLE. It started to occur during the development of MDEV-25180, which introduced purge_sys.stop_SYS(). If we delay purge more during DDL operations, then the test would almost always fail. The reason is that during startup we will restore a purge view, and CHECK TABLE would still use REPEATABLE READ even though innodb_read_only is set and other isolation levels than READ UNCOMMITTED are not guaranteed to work. ha_innobase::check(): Use READ UNCOMMITTED isolation level if innodb_read_only is set or innodb_force_recovery exceeds 3. dict_set_corrupted(): Do not update the persistent data dictionary if innodb_force_recovery exceeds 3. --- storage/innobase/dict/dict0dict.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 10b878c0e49..1d0e2af3cd6 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -5425,7 +5425,7 @@ dict_set_corrupted( /* If this is read only mode, do not update SYS_INDEXES, just mark it as corrupted in memory */ - if (srv_read_only_mode) { + if (high_level_read_only) { index->type |= DICT_CORRUPT; goto func_exit; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4e63edc65c2..65039919793 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14796,10 +14796,9 @@ ha_innobase::check( /* We must run the index record counts at an isolation level >= READ COMMITTED, because a dirty read can see a wrong number - of records in some index; to play safe, we use always - REPEATABLE READ here (except when undo logs are unavailable) */ - m_prebuilt->trx->isolation_level = srv_force_recovery - >= SRV_FORCE_NO_UNDO_LOG_SCAN + of records in some index; to play safe, we normally use + REPEATABLE READ here */ + m_prebuilt->trx->isolation_level = high_level_read_only ? TRX_ISO_READ_UNCOMMITTED : TRX_ISO_REPEATABLE_READ;