Mirrors
/
mariadb
mirror of https://github.com/MariaDB/server
3
0
Fork 0
Code Releases Activity
Browse Source

Added some fixes that should make MyISAM & Aria REPAIR work with more than 4G records 

- If one specifies --force twice to myisamchk and aria_chk, then we will try to finnish the repair even if sort_buffer would be too small.
  This was done by dynamically allocate buffer handler objects as long as memory lasts.
- New option for myisamchk and aria_chk: create-missing-keys
- Changed default size of myisam_sort_buffer_size from 8M to 128M.
- Changed default size of sort_buffer_size in aria_chk from 128M to 256M.
- Increased information in error message about 'sort_buffer_size' beeing to small.
- Print also to 'show warnings' if repair was retried.
- Increased size of internal sort-buffer-readers from 16K to 128K
- Changed printing of 'number of records' to use %ll instead of casting to long
- Changed buffer sizes for myisam and aria to use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines.





include/my_global.h:
  Added MY_ALIGN_DOWN() to get previous alignment (for big memory areas)
include/myisam.h:
  Increased size of types to be able to handle more records
include/myisamchk.h:
  Increased size of types to be able to handle more records
  Added T_FORCE_SORT_MEMORY to force repair to work even if sort_buffer would not be big enough
mysql-test/r/myisam.result:
  Updated result
mysql-test/r/mysqld--help.result:
  Updated result
mysql-test/r/repair.result:
  Updated result
mysql-test/suite/maria/maria.result:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria.test:
  Added test cases for some fixed bugs in MyISAM to verify that Aria doesn't have them too.
mysql-test/suite/maria/maria3.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result:
  Updated result after sort buffer size increase
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test:
  Updated result after sort buffer size increase
mysql-test/t/myisam.test:
  Fixed error messages to not print system specific data
mysql-test/t/repair.test:
  Fixed error messages to not print system specific data
storage/maria/ha_maria.cc:
  Print also to 'show warnings' if repair was retried
  Changed default size of sort_buffer_size from 128M to 256M (same as in mysqld)
storage/maria/ma_check.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/ma_sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug where maria_update_key_parts() was called too early.
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
storage/maria/maria_chk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
storage/maria/maria_def.h:
  Increased size of types to be able to handle more records
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_ftdump.c:
  Renamed USE_BUFFER_INIT -> PAGE_BUFFER_INIT
storage/maria/maria_read_log.c:
  Use PAGE_BUFFER_INIT for page cache
storage/myisam/ha_myisam.cc:
  Changed default size of myisam_sort_buffer_size from 8M to 128M
storage/myisam/mi_check.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisam_ftdump.c:
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamchk.c:
  Added --create-missing-keys
  If one specifies --force twice then we will try to finnish the repair even if sort_buffer would be too small.
  check_param.sort_buffer_length varialble was used with wrong type.
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/myisamdef.h:
  Increased SORT_BUFFER_INIT to 64M
  (speeds up repair a lot and most machines have nowadays a lot of memory)
  Use MY_ALIGN_DOWN() to get same number of bytes allocated on different machines
  Renamed USE_BUFFER_INIT -> KEY_BUFFER_INIT
storage/myisam/sort.c:
  Increased size of internal sort-buffer-readers from 16K to 128K
  Increased size of types to be able to handle more records
  Added support for T_FORCE_SORT_MEMORY
  Don't allocate too many extra BUFFPEK at a time (they are probably not needed)
  Improved error message for "sort_buffer_size is too small"
  Changed printing of 'number of records' to use %ll instead of casting to long
  Fixed bug in detecting result from read_to_buffer().
  Added 'out of memory' checking when calling 'alloc_dynamic()'.
pull/57/head
Michael Widenius 13 years ago
parent
e5a0daae5a
commit
53d44ad18b
27 changed files with 651 additions and 326 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      include/my_global.h
  2. 12
      include/myisam.h
  3. 11
      include/myisamchk.h
  4. 4
      mysql-test/r/myisam.result
  5. 2
      mysql-test/r/mysqld--help.result
  6. 4
      mysql-test/r/repair.result
  7. 62
      mysql-test/suite/maria/maria.result
  8. 46
      mysql-test/suite/maria/maria.test
  9. 2
      mysql-test/suite/maria/maria3.result
  10. 22
      mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result
  11. 20
      mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result
  12. 4
      mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test
  13. 2
      mysql-test/t/myisam.test
  14. 2
      mysql-test/t/repair.test
  15. 8
      storage/maria/ha_maria.cc
  16. 2
      storage/maria/ma_check.c
  17. 344
      storage/maria/ma_sort.c
  18. 51
      storage/maria/maria_chk.c
  19. 18
      storage/maria/maria_def.h
  20. 2
      storage/maria/maria_ftdump.c
  21. 8
      storage/maria/maria_read_log.c
  22. 2
      storage/myisam/ha_myisam.cc
  23. 2
      storage/myisam/mi_check.c
  24. 2
      storage/myisam/myisam_ftdump.c
  25. 44
      storage/myisam/myisamchk.c
  26. 9
      storage/myisam/myisamdef.h
  27. 289
      storage/myisam/sort.c

3
include/my_global.h
View File

@ -847,7 +847,8 @@ typedef long my_ptrdiff_t;
typedef long long my_ptrdiff_t;
#endif
#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1))
#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1))
#define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1))
#define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double))
#define ALIGN_MAX_UNIT (sizeof(double))
/* Size to make adressable obj. */

12
include/myisam.h
View File

@ -356,8 +356,10 @@ typedef struct st_mi_sort_param
ulonglong notnull[HA_MAX_KEY_SEG+1];
my_off_t pos,max_pos,filepos,start_recpos;
uint key, key_length,real_key_length,sortbuff_size;
uint maxbuffers, keys, find_length, sort_keys_length;
uint key, key_length,real_key_length;
uint maxbuffers, find_length;
ulonglong sortbuff_size;
ha_rows keys;
my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */
@ -366,10 +368,10 @@ typedef struct st_mi_sort_param
int (*key_write)(struct st_mi_sort_param *, const void *);
void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_mi_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
ulonglong , struct st_buffpek *, IO_CACHE *);
my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
uint, uint);
uint, ulonglong);
} MI_SORT_PARAM;
/* functions in mi_check */

11
include/myisamchk.h
View File

@ -62,9 +62,10 @@
#define T_ZEROFILL ((ulonglong) 1L << 32)
#define T_ZEROFILL_KEEP_LSN ((ulonglong) 1L << 33)
/** If repair should not bump create_rename_lsn */
#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 33)
#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 34)
#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 35)
#define T_NO_CREATE_RENAME_LSN ((ulonglong) 1L << 34)
#define T_CREATE_UNIQUE_BY_SORT ((ulonglong) 1L << 35)
#define T_SUPPRESS_ERR_HANDLING ((ulonglong) 1L << 36)
#define T_FORCE_SORT_MEMORY ((ulonglong) 1L << 37)
#define T_REP_ANY (T_REP | T_REP_BY_SORT | T_REP_PARALLEL)
@ -180,8 +181,8 @@ typedef struct st_buffpek {
my_off_t file_pos; /* Where we are in the sort file */
uchar *base, *key; /* Key pointers */
ha_rows count; /* Number of rows in table */
ulong mem_count; /* numbers of keys in memory */
ulong max_keys; /* Max keys in buffert */
ha_rows mem_count; /* Numbers of keys in memory */
ha_rows max_keys; /* Max keys in buffert */
} BUFFPEK;
#endif /* _myisamchk_h */

4
mysql-test/r/myisam.result
View File

@ -2355,7 +2355,7 @@ Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '4'
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small
test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 7168
test.t1 repair status OK
SET myisam_repair_threads=2;
@ -2424,7 +2424,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings:
Error 1034 myisam_sort_buffer_size is too small
Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
INSERT INTO t1 VALUES('1');

2
mysql-test/r/mysqld--help.result
View File

@ -1045,7 +1045,7 @@ myisam-max-sort-file-size 9223372036853727232
myisam-mmap-size 18446744073709551615
myisam-recover-options DEFAULT
myisam-repair-threads 1
myisam-sort-buffer-size 8388608
myisam-sort-buffer-size 134216704
myisam-stats-method nulls_unequal
myisam-use-mmap FALSE
net-buffer-length 16384

4
mysql-test/r/repair.result
View File

@ -77,12 +77,12 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
Warnings:
Error 1034 myisam_sort_buffer_size is too small
Error 1034 myisam_sort_buffer_size is too small. X
Error 1034 Number of rows changed from 0 to 157
SET myisam_repair_threads=2;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error myisam_sort_buffer_size is too small
test.t1 repair error myisam_sort_buffer_size is too small. X
test.t1 repair warning Number of rows changed from 0 to 157
test.t1 repair status OK
SET myisam_repair_threads=@@global.myisam_repair_threads;

62
mysql-test/suite/maria/maria.result
View File

@ -2679,3 +2679,65 @@ select count(*) from t1;
count(*)
13
drop table t1;
#
# BUG#47444 - --myisam_repair_threads > 1 can result in all index
# cardinalities=1
#
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair status OK
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CARDINALITY
14
14
14
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
#
# BUG#47073 - valgrind errs, corruption,failed repair of partition,
# low myisam_sort_buffer_size
#
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=2;
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair error aria_sort_buffer_size is too small. X
test.t1 repair error Create index by sort failed
test.t1 repair info Retrying repair with keycache
test.t1 repair status OK
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;

46
mysql-test/suite/maria/maria.test
View File

@ -1963,6 +1963,52 @@ unlock tables;
select count(*) from t1;
drop table t1;
--echo #
--echo # BUG#47444 - --myisam_repair_threads > 1 can result in all index
--echo # cardinalities=1
--echo #
SET aria_repair_threads=2;
SET aria_sort_buffer_size=8192;
CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
SELECT CARDINALITY FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1';
CHECK TABLE t1;
DROP TABLE t1;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
SET aria_repair_threads=@@global.aria_repair_threads;
--echo #
--echo # BUG#47073 - valgrind errs, corruption,failed repair of partition,
--echo # low myisam_sort_buffer_size
--echo #
CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
(6,'0'),(7,'0');
INSERT INTO t1 SELECT a+10,b FROM t1;
INSERT INTO t1 SELECT a+20,b FROM t1;
INSERT INTO t1 SELECT a+40,b FROM t1;
INSERT INTO t1 SELECT a+80,b FROM t1;
INSERT INTO t1 SELECT a+160,b FROM t1;
INSERT INTO t1 SELECT a+320,b FROM t1;
INSERT INTO t1 SELECT a+640,b FROM t1;
INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET aria_sort_buffer_size=4096;
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=2;
# May report different values depending on threads activity.
--replace_regex /Current aria_sort_buffer_size.*/X/
REPAIR TABLE t1;
CHECK TABLE t1;
SET aria_repair_threads=@@global.aria_repair_threads;
SET aria_sort_buffer_size=@@global.aria_sort_buffer_size;
DROP TABLE t1;
#
# End of test
#

2
mysql-test/suite/maria/maria3.result
View File

@ -317,7 +317,7 @@ aria_pagecache_division_limit 100
aria_page_checksum OFF
aria_recover NORMAL
aria_repair_threads 1
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
aria_stats_method nulls_unequal
aria_sync_log_dir NEWFILE
show status like 'aria%';

22
mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result
View File

@ -1,30 +1,34 @@
SET @start_global_value = @@global.aria_sort_buffer_size;
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
134217728
268434432
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size
134217728
268434432
show global variables like 'aria_sort_buffer_size';
Variable_name Value
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
show session variables like 'aria_sort_buffer_size';
Variable_name Value
aria_sort_buffer_size 134217728
aria_sort_buffer_size 268434432
select * from information_schema.global_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728
ARIA_SORT_BUFFER_SIZE 268434432
select * from information_schema.session_variables where variable_name='aria_sort_buffer_size';
VARIABLE_NAME VARIABLE_VALUE
ARIA_SORT_BUFFER_SIZE 134217728
ARIA_SORT_BUFFER_SIZE 268434432
set global aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
10
4096
set session aria_sort_buffer_size=10;
Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '10'
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size
10
4096
set global aria_sort_buffer_size=1.1;
ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
set session aria_sort_buffer_size=1e1;
@ -36,7 +40,7 @@ Warnings:
Warning 1292 Truncated incorrect aria_sort_buffer_size value: '0'
select @@global.aria_sort_buffer_size;
@@global.aria_sort_buffer_size
4
4096
set session aria_sort_buffer_size=cast(-1 as unsigned int);
select @@session.aria_sort_buffer_size;
@@session.aria_sort_buffer_size

20
mysql-test/suite/sys_vars/r/myisam_sort_buffer_size_basic.result
View File

@ -1,11 +1,11 @@
SET @start_global_value = @@global.myisam_sort_buffer_size ;
SELECT @start_global_value;
@start_global_value
8388608
134216704
SET @start_session_value = @@session.myisam_sort_buffer_size ;
SELECT @start_session_value;
@start_session_value
8388608
134216704
'#--------------------FN_DYNVARS_005_01-------------------------#'
SET @@global.myisam_sort_buffer_size = 100;
Warnings:
@ -13,22 +13,22 @@ Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '100'
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size
8388608
134216704
SET @@session.myisam_sort_buffer_size = 200;
Warnings:
Warning 1292 Truncated incorrect myisam_sort_buffer_size value: '200'
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size
8388608
134216704
'#--------------------FN_DYNVARS_005_02-------------------------#'
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608;
@@global.myisam_sort_buffer_size = 8388608
SELECT @@global.myisam_sort_buffer_size = 134216704;
@@global.myisam_sort_buffer_size = 134216704
1
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608;
@@session.myisam_sort_buffer_size = 8388608
SELECT @@session.myisam_sort_buffer_size = 134216704;
@@session.myisam_sort_buffer_size = 134216704
1
'#--------------------FN_DYNVARS_005_03-------------------------#'
SET @@global.myisam_sort_buffer_size = 4;
@ -187,8 +187,8 @@ ERROR 42S22: Unknown column 'myisam_sort_buffer_size' in 'field list'
SET @@global.myisam_sort_buffer_size = @start_global_value;
SELECT @@global.myisam_sort_buffer_size ;
@@global.myisam_sort_buffer_size
8388608
134216704
SET @@session.myisam_sort_buffer_size = @start_session_value;
SELECT @@session.myisam_sort_buffer_size ;
@@session.myisam_sort_buffer_size
8388608
134216704

4
mysql-test/suite/sys_vars/t/myisam_sort_buffer_size_basic.test
View File

@ -60,10 +60,10 @@ SELECT @@session.myisam_sort_buffer_size ;
########################################################################
SET @@global.myisam_sort_buffer_size = DEFAULT;
SELECT @@global.myisam_sort_buffer_size = 8388608;
SELECT @@global.myisam_sort_buffer_size = 134216704;
SET @@session.myisam_sort_buffer_size = DEFAULT;
SELECT @@session.myisam_sort_buffer_size = 8388608;
SELECT @@session.myisam_sort_buffer_size = 134216704;
--echo '#--------------------FN_DYNVARS_005_03-------------------------#'

2
mysql-test/t/myisam.test
View File

@ -1600,6 +1600,7 @@ INSERT INTO t1 SELECT a+1280,b FROM t1;
INSERT INTO t1 SELECT a+2560,b FROM t1;
INSERT INTO t1 SELECT a+5120,b FROM t1;
SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1;
SET myisam_repair_threads=2;
@ -1648,6 +1649,7 @@ DROP TABLE t1, t2, t3;
CREATE TABLE t1(a CHAR(255), KEY(a));
SELECT * FROM t1, t1 AS a1;
SET myisam_sort_buffer_size=4;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),

2
mysql-test/t/repair.test
View File

@ -61,6 +61,7 @@ DROP TABLE t1;
#
CREATE TABLE t1(a CHAR(255), KEY(a));
SET myisam_sort_buffer_size=4096;
--replace_regex /Current myisam_sort_buffer_size.*/X/
INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
@ -79,6 +80,7 @@ INSERT INTO t1 VALUES
('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
('0'),('0'),('0'),('0'),('0'),('0'),('0');
SET myisam_repair_threads=2;
--replace_regex /Current myisam_sort_buffer_size.*/X/
REPAIR TABLE t1;
SET myisam_repair_threads=@@global.myisam_repair_threads;
SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;

8
storage/maria/ha_maria.cc
View File

@ -260,10 +260,10 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
"disables parallel repair.",
0, 0, 1, 1, 128, 1);
static MYSQL_THDVAR_ULONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing a "
"REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.",
0, 0, 128L*1024L*1024L, 4, UINT_MAX32, 1);
"REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
"Specifies how Aria index statistics collection code should treat "
@ -1455,6 +1455,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt)
if ((param.testflag & T_REP_BY_SORT))
{
param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP;
if (thd->vio_ok())
_ma_check_print_info(&param, "Retrying repair with keycache");
sql_print_information("Retrying repair of: '%s' with keycache",
table->s->path.str);
continue;

2
storage/maria/ma_check.c
View File

@ -114,7 +114,7 @@ void maria_chk_init(HA_CHECK *param)
param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR;
param->auto_increment_value= 0;
param->use_buffers=USE_BUFFER_INIT;
param->use_buffers= PAGE_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT;

344
storage/maria/ma_sort.c
View File

@ -36,8 +36,10 @@
#define MERGEBUFF2 31
#define MIN_SORT_MEMORY (4096-MALLOC_OVERHEAD)
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16)
#define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/*
Pointers of functions for store and read keys from temp file
@ -47,41 +49,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info,uint keys,
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer,
DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions);
static int write_keys(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_keys(MARIA_SORT_PARAM *info,uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MARIA_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile);
static int write_index(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count);
static int merge_many_buff(MARIA_SORT_PARAM *info,uint keys,
ha_keys count);
static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
BUFFPEK *buffpek,int *maxbuffer,
BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MARIA_SORT_PARAM *info,uint keys,
static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file,
uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MARIA_SORT_PARAM *,uint, uchar **,BUFFPEK *, int,
static int merge_index(MARIA_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *);
static int flush_maria_ft_buf(MARIA_SORT_PARAM *info);
static int write_keys_varlen(MARIA_SORT_PARAM *info, uchar **sort_keys,
uint count, BUFFPEK *buffpek,
static int write_keys_varlen(MARIA_SORT_PARAM *info,uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int write_merge_key(MARIA_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count);
uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count);
IO_CACHE *to_file,
uchar* key, uint sort_length,
ha_keys count);
static inline int
my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
@ -102,11 +105,11 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
size_t sortbuff_size)
{
int error,maxbuffer,skr;
size_t memavl,old_memavl;
uint keys,sort_length;
int error;
uint sort_length, maxbuffer;
size_t memavl, old_memavl;
DYNAMIC_ARRAY buffpek;
ha_rows records;
ha_rows records, keys;
uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_ma_create_index_by_sort");
@ -117,7 +120,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{
info->write_keys= write_keys_varlen;
info->read_to_buffer=read_to_buffer_varlen;
info->read_to_buffer= read_to_buffer_varlen;
info->write_key=write_merge_key_varlen;
}
else
@ -140,31 +143,53 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
while (memavl >= MIN_SORT_MEMORY)
{
if ((records < UINT_MAX32) &&
((my_off_t) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl))
keys= (uint)records+1;
/* Check if we can fit all keys into memory */
if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= memavl))
keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do
{
skr=maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
keys < maxbuffer)
{
_ma_check_print_error(info->sort_info->param,
"aria_sort_buffer_size is too small");
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM;
goto err;
}
}
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar**) my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0))))
{
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0)))
min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= 0;
@ -178,14 +203,20 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
}
if (memavl < MIN_SORT_MEMORY)
{
_ma_check_print_error(info->sort_info->param, "Aria sort buffer"
" too small"); /* purecov: tested */
goto err; /* purecov: tested */
/* purecov: begin inspected */
_ma_check_print_error(info->sort_info->param,
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
(ulonglong) sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys);
my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
@ -197,8 +228,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer == 0)
{
if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records);
if (write_index(info,sort_keys, (uint) records))
my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */
}
else
@ -207,7 +238,8 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages,
if (maxbuffer >= MERGEBUFF2)
{
if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */
my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */
@ -266,13 +298,13 @@ err:
/* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, uint keys,
static ha_rows find_all_keys(MARIA_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile,
uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions)
{
int error;
uint idx;
ha_rows idx;
DBUG_ENTER("find_all_keys");
idx=error=0;
@ -328,9 +360,11 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
MARIA_SORT_PARAM *sort_param= (MARIA_SORT_PARAM*) arg;
int error;
size_t memavl,old_memavl;
size_t memavl, old_memavl;
longlong sortbuff_size;
ha_keys keys, idx;
uint sort_length;
ulong idx, maxbuffer, keys;
uint maxbuffer;
uchar **sort_keys=0;
LINT_INIT(keys);
@ -364,8 +398,9 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
bzero((char*) &sort_param->buffpek,sizeof(sort_param->buffpek));
bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
memavl= max(sort_param->sortbuff_size, MIN_SORT_MEMORY);
idx= (uint)sort_param->sort_info->max_records;
sortbuff_size= sort_param->sortbuff_size;
memavl= max(sortbuff_size, MIN_SORT_MEMORY);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length;
maxbuffer= 1;
@ -373,23 +408,36 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <= (my_off_t) memavl)
keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else
{
ulong skr;
uint maxbuffer_org;
do
{
skr= maxbuffer;
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < maxbuffer)
{
_ma_check_print_error(sort_param->sort_info->param,
"aria_sort_buffer_size is too small");
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err;
}
}
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys= (uchar **)
my_malloc(keys*(sort_length+sizeof(char*))+
@ -397,7 +445,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0))))
{
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0)))
maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */
@ -412,14 +460,19 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
}
if (memavl < MIN_SORT_MEMORY)
{
/* purecov: begin inspected */
_ma_check_print_error(sort_param->sort_info->param,
"Aria sort buffer too small");
goto err; /* purecov: tested */
"aria_sort_buffer_size is too small. Current aria_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %lu keys\n",
sort_param->key+1, (ulong) keys);
my_fprintf(stdout,
"Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys;
idx= error= 0;
@ -431,7 +484,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
{
if (sort_param->real_key_length > sort_param->key_length)
{
if (write_key(sort_param,sort_keys[idx],
if (write_key(sort_param, sort_keys[idx],
&sort_param->tempfile_for_exceptions))
goto err;
continue;
@ -449,7 +502,7 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
(size_t) sort_param->key_length);
idx= 1;
}
sort_keys[idx]=sort_keys[idx - 1] + sort_param->key_length;
sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
}
if (error > 0)
goto err;
@ -465,14 +518,13 @@ pthread_handler_t _ma_thr_find_all_keys(void *arg)
else
sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok;
err:
DBUG_PRINT("error", ("got some error"));
sort_param->sort_info->got_error= 1; /* no need to protect with a mutex */
my_free(sort_keys);
sort_param->sort_keys=0;
sort_param->sort_keys= 0;
delete_dynamic(& sort_param->buffpek);
close_cached_file(&sort_param->tempfile);
close_cached_file(&sort_param->tempfile_for_exceptions);
@ -483,8 +535,12 @@ ok:
Detach from the share if the writer is involved. Avoid others to
be blocked. This includes a flush of the write buffer. This will
also indicate EOF to the readers.
That means that a writer always gets here first and readers -
only when they see EOF. But if a reader finishes prematurely
because of an error it may reach this earlier - don't allow it
to detach the writer thread.
*/
if (sort_param->sort_info->info->rec_cache.share)
if (sort_param->master && sort_param->sort_info->info->rec_cache.share)
remove_io_thread(&sort_param->sort_info->info->rec_cache);
/* Readers detach from the share if any. Avoid others to be blocked. */
@ -518,7 +574,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
for (i= 0, sinfo= sort_param ;
i < sort_info->total_keys ;
i++, rec_per_key_part+=sinfo->keyinfo->keysegs, sinfo++)
i++, sinfo++)
{
if (!sinfo->sort_keys)
{
@ -534,19 +590,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
{
if (param->testflag & T_VERBOSE)
{
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout);
}
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
flush_maria_ft_buf(sinfo) || _ma_flush_pending_blocks(sinfo))
got_error=1;
}
if (!got_error && param->testflag & T_STATISTICS)
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
}
my_free(sinfo->sort_keys);
my_free(sinfo->rec_buff);
@ -559,6 +611,7 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
delete_dynamic(&sinfo->buffpek),
close_cached_file(&sinfo->tempfile),
close_cached_file(&sinfo->tempfile_for_exceptions),
rec_per_key_part+= sinfo->keyinfo->keysegs,
sinfo++)
{
if (got_error)
@ -597,10 +650,12 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2)
{
if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **) mergebuf,
my_fprintf(stdout,
"Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile))
&maxbuffer, &sinfo->tempfile))
{
got_error=1;
continue;
@ -660,6 +715,13 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
}
}
}
if (!got_error && (param->testflag & T_STATISTICS))
maria_update_key_parts(sinfo->keyinfo, rec_per_key_part, sinfo->unique,
param->stats_method ==
MI_STATS_METHOD_IGNORE_NULLS ?
sinfo->notnull : NULL,
(ulonglong) share->state.state.records);
}
my_free(mergebuf);
DBUG_RETURN(got_error);
@ -669,12 +731,15 @@ int _ma_thr_write_keys(MARIA_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */
static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{
uchar **end;
uint sort_length=info->key_length;
DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -687,7 +752,7 @@ static int write_keys(MARIA_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{
if (my_b_write(tempfile, *sort_keys, (uint) sort_length))
if (my_b_write(tempfile, *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */
}
DBUG_RETURN(0);
@ -710,14 +775,17 @@ my_var_write(MARIA_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MARIA_SORT_PARAM *info,
register uchar **sort_keys,
uint count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
register uchar **sort_keys,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
{
uchar **end;
int err;
DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -756,9 +824,8 @@ static int write_key(MARIA_SORT_PARAM *info, uchar *key,
/* Write index */
static int write_index(MARIA_SORT_PARAM *info,
register uchar **sort_keys,
register uint count)
static int write_index(MARIA_SORT_PARAM *info, register uchar **sort_keys,
register ha_keys count)
{
DBUG_ENTER("write_index");
@ -777,11 +844,11 @@ static int write_index(MARIA_SORT_PARAM *info,
/* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file)
static int merge_many_buff(MARIA_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek,
uint *maxbuffer, IO_CACHE *t_file)
{
int tmp, merges, max_merges;
uint tmp, merges, max_merges;
IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff");
@ -807,11 +874,11 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
from_file= t_file ; to_file= &t_file2;
while (*maxbuffer >= MERGEBUFF2)
{
int i;
uint i;
reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1))
@ -825,14 +892,19 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file))
break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1;
*maxbuffer= (uint) (lastbuff-buffpek)-1;
if (info->sort_info->param->max_stage != 1) /* If not parallel */
_ma_report_progress(info->sort_info->param, merges++, max_merges);
}
cleanup:
close_cached_file(to_file); /* This holds old result */
if (to_file == t_file)
{
DBUG_ASSERT(t_file2.type == WRITE_CACHE);
*t_file=t_file2; /* Copy result file */
t_file->current_pos= &t_file->write_pos;
t_file->current_end= &t_file->write_end;
}
DBUG_RETURN(*maxbuffer >= MERGEBUFF2); /* Return 1 if interrupted */
} /* merge_many_buff */
@ -851,33 +923,35 @@ cleanup:
-1 Error
*/
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
uint length;
register ha_keys count;
my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
if (mysql_file_pread(fromfile->file, buffpek->base,
(length= sort_length*count),buffpek->file_pos,MYF_RW))
return((uint) -1); /* purecov: inspected */
if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length * count),
buffpek->file_pos, MYF_RW))
return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
register ha_keys count;
uint idx;
uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
buffp= buffpek->base;
@ -886,7 +960,7 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint16 length_of_key;
if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key),
buffpek->file_pos,MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, buffp, length_of_key,
buffpek->file_pos,MYF_RW))
@ -898,15 +972,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */
static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
uint idx;
ha_keys idx;
uchar *bufs = key;
for (idx=1;idx<=count;idx++)
@ -921,34 +995,40 @@ static int write_merge_key_varlen(MARIA_SORT_PARAM *info,
static int write_merge_key(MARIA_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count)
IO_CACHE *to_file, uchar *key,
uint sort_length, ha_keys count)
{
return my_b_write(to_file, key, (size_t) sort_length*count);
return my_b_write(to_file, key, ((size_t) sort_length) * count);
}
/*
Merge buffers to one buffer
If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/
static int
merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb)
{
int error;
uint sort_length,maxcount;
int error= 1;
uint sort_length;
ha_keys maxcount;
ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos);
my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos;
BUFFPEK *buffpek,**refpek;
QUEUE queue;
DBUG_ENTER("merge_buffers");
count=error=0;
maxcount=keys/((uint) (Tb-Fb) +1);
count= 0;
maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos);
if (to_file)
to_start_filepos=my_b_tell(to_file);
strpos= (uchar*) sort_keys;
@ -963,10 +1043,10 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
count+= buffpek->count;
buffpek->base= strpos;
buffpek->max_keys=maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
sort_length));
if (error == -1)
buffpek->max_keys= maxcount;
strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length));
if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek);
}
@ -980,27 +1060,20 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
if (info->write_key(info,to_file, buffpek->key,
(uint) sort_length,1))
{
error=1; goto err; /* purecov: inspected */
}
goto err; /* purecov: inspected */
}
else
{
if ((*info->key_write)(info,(void*) buffpek->key))
{
error=1; goto err; /* purecov: inspected */
}
goto err; /* purecov: inspected */
}
buffpek->key+=sort_length;
if (! --buffpek->mem_count)
{
/* It's enough to check for killedptr before a slow operation */
if (_ma_killed_ptr(info->sort_info->param))
{
error=1;
goto err;
}
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
{
uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys;
@ -1027,9 +1100,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
break; /* One buffer have been removed */
}
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
}
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */
}
}
@ -1061,8 +1134,9 @@ merge_buffers(MARIA_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
}
}
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) !=
-1 && error != 0);
while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
if (read_length == 0)
error= 0;
lastbuff->count=count;
if (to_file)
@ -1076,8 +1150,8 @@ err:
/* Do a merge to output-file (save only positions) */
static int
merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
merge_index(MARIA_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{
DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,

51
storage/maria/maria_chk.c
View File

@ -191,7 +191,7 @@ end:
enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_PAGE_BUFFER_SIZE,
OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_PAGE_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MARIA_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@ -229,6 +229,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
{"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.",
@ -362,8 +367,8 @@ static struct my_option my_long_options[] =
{ "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
"Size of page buffer. Used by --safe-repair",
&check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L,
(long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
GET_ULONG, REQUIRED_ARG, PAGE_BUFFER_INIT, 1024L*1024L,
SIZE_T_MAX, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
{ "read_buffer_size", OPT_READ_BUFFER_SIZE,
"Read buffer size for sequential reads during scanning",
&check_param.read_buffer_length,
@ -379,9 +384,8 @@ static struct my_option my_long_options[] =
{ "sort_buffer_size", OPT_SORT_BUFFER_SIZE,
"Size of sort buffer. Used by --recover",
&check_param.sort_buffer_length,
&check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
(long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
(long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
&check_param.sort_buffer_length, 0, GET_ULL, REQUIRED_ARG,
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, MALLOC_OVERHEAD, 1L, 0},
{ "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
"Internal buffer for sorting keys; Don't touch :)",
&check_param.sort_key_blocks,
@ -497,10 +501,18 @@ Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
--correct-checksum Correct checksum information for table.\n\
-D, --data-file-length=# Max length of data file (when recreating data\n\
file when it's full).\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick.\n\
-e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\
-f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell Aria to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\
@ -664,10 +676,13 @@ get_one_option(int optid,
if (argument == disabled_my_option)
{
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
}
else
{
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
}
@ -720,8 +735,26 @@ get_one_option(int optid,
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break;
case 'u':
if (argument == disabled_my_option)

18
storage/maria/maria_def.h
View File

@ -94,9 +94,10 @@ typedef struct st_maria_sort_param
*/
ulonglong unique[HA_MAX_KEY_SEG+1];
ulonglong notnull[HA_MAX_KEY_SEG+1];
ulonglong sortbuff_size;
MARIA_RECORD_POS pos,max_pos,filepos,start_recpos, current_filepos;
uint key, key_length,real_key_length,sortbuff_size;
uint key, key_length,real_key_length;
uint maxbuffers, keys, find_length, sort_keys_length;
my_bool fix_datafile, master;
my_bool calc_checksum; /* calculate table checksum */
@ -107,10 +108,10 @@ typedef struct st_maria_sort_param
int (*key_write)(struct st_maria_sort_param *, const uchar *);
void (*lock_in_memory)(HA_CHECK *);
int (*write_keys)(struct st_maria_sort_param *, register uchar **,
uint , struct st_buffpek *, IO_CACHE *);
uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
ulonglong , struct st_buffpek *, IO_CACHE *);
my_off_t (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
int (*write_key)(struct st_maria_sort_param *, IO_CACHE *,uchar *,
uint, uint);
uint, ulonglong);
} MARIA_SORT_PARAM;
int maria_write_data_suffix(MARIA_SORT_INFO *sort_info, my_bool fix_datafile);
@ -1223,10 +1224,11 @@ typedef struct st_maria_block_info
#define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*1024L*128-MALLOC_OVERHEAD)/8192)*8192)
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD)
#define SORT_BUFFER_INIT (1024L*1024L*256-MALLOC_OVERHEAD)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD)
/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define PAGE_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 8192)
#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*256L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
#define fast_ma_writeinfo(INFO) if (!(INFO)->s->tot_locks) (void) _ma_writeinfo((INFO),0)
#define fast_ma_readinfo(INFO) ((INFO)->lock_type == F_UNLCK) && _ma_readinfo((INFO),F_RDLCK,1)

2
storage/maria/maria_ftdump.c
View File

@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage();
}
init_pagecache(maria_pagecache, USE_BUFFER_INIT, 0, 0,
init_pagecache(maria_pagecache, PAGE_BUFFER_INIT, 0, 0,
MARIA_KEY_BLOCK_LENGTH, MY_WME);
if (!(info=maria_open(argv[0], O_RDONLY,

8
storage/maria/maria_read_log.c
View File

@ -32,7 +32,8 @@ const char *default_dbug_option= "d:t:o,/tmp/aria_read_log.trace";
static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
static my_bool opt_check;
static const char *opt_tmpdir;
static ulong opt_page_buffer_size, opt_translog_buffer_size;
static ulong opt_translog_buffer_size;
static ulonglong opt_page_buffer_size;
static ulonglong opt_start_from_lsn, opt_end_lsn, opt_start_from_checkpoint;
static MY_TMPDIR maria_chk_tmpdir;
@ -204,9 +205,8 @@ static struct my_option my_long_options[] =
{ "page-buffer-size", 'P',
"The size of the buffer used for index blocks for Aria tables",
&opt_page_buffer_size, &opt_page_buffer_size, 0,
GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT,
1024L*1024L, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD,
(long) IO_SIZE, 0},
GET_ULL, REQUIRED_ARG, PAGE_BUFFER_INIT,
PAGE_BUFFER_INIT, SIZE_T_MAX, MALLOC_OVERHEAD, (long) IO_SIZE, 0},
{ "start-from-lsn", 'o', "Start reading log from this lsn",
&opt_start_from_lsn, &opt_start_from_lsn,
0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },

2
storage/myisam/ha_myisam.cc
View File

@ -78,7 +78,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
"The buffer that is allocated when sorting the index when doing "
"a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE", NULL, NULL,
8192 * 1024, MIN_SORT_BUFFER + MALLOC_OVERHEAD, SIZE_T_MAX, 1);
SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
static MYSQL_SYSVAR_BOOL(use_mmap, opt_myisam_use_mmap, PLUGIN_VAR_NOCMDARG,
"Use memory mapping for reading and writing MyISAM tables", NULL, NULL, FALSE);

2
storage/myisam/mi_check.c
View File

@ -91,7 +91,7 @@ void myisamchk_init(HA_CHECK *param)
param->opt_follow_links=1;
param->keys_in_use= ~(ulonglong) 0;
param->search_after_block=HA_OFFSET_ERROR;
param->use_buffers=USE_BUFFER_INIT;
param->use_buffers= KEY_BUFFER_INIT;
param->read_buffer_length=READ_BUFFER_INIT;
param->write_buffer_length=READ_BUFFER_INIT;
param->sort_buffer_length=SORT_BUFFER_INIT;

2
storage/myisam/myisam_ftdump.c
View File

@ -84,7 +84,7 @@ int main(int argc,char *argv[])
usage();
}
init_key_cache(dflt_key_cache,MI_KEY_BLOCK_LENGTH,USE_BUFFER_INIT, 0, 0, 0);
init_key_cache(dflt_key_cache, MI_KEY_BLOCK_LENGTH, KEY_BUFFER_INIT, 0, 0, 0);
if (!(info=mi_open(argv[0], O_RDONLY,
HA_OPEN_ABORT_IF_LOCKED|HA_OPEN_FROM_SQL_LAYER)))

44
storage/myisam/myisamchk.c
View File

@ -134,7 +134,7 @@ int main(int argc, char **argv)
enum options_mc {
OPT_CHARSETS_DIR=256, OPT_SET_COLLATION,OPT_START_CHECK_POS,
OPT_CORRECT_CHECKSUM, OPT_KEY_BUFFER_SIZE,
OPT_CORRECT_CHECKSUM, OPT_CREATE_MISSING_KEYS, OPT_KEY_BUFFER_SIZE,
OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
@ -165,6 +165,11 @@ static struct my_option my_long_options[] =
{"correct-checksum", OPT_CORRECT_CHECKSUM,
"Correct checksum information for table.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"create-missing-keys", OPT_CREATE_MISSING_KEYS,
"Create missing keys. This assumes that the data file is correct and that "
"the the number of rows stored in the index file is correct. Enables "
"--quick",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
{"debug", '#',
"Output debug log. Often this is 'd:t:o,filename'.",
@ -270,7 +275,7 @@ static struct my_option my_long_options[] =
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{ "key_buffer_size", OPT_KEY_BUFFER_SIZE, "",
&check_param.use_buffers, &check_param.use_buffers, 0,
GET_ULL, REQUIRED_ARG, USE_BUFFER_INIT, MALLOC_OVERHEAD,
GET_ULL, REQUIRED_ARG, KEY_BUFFER_INIT, MALLOC_OVERHEAD,
SIZE_T_MAX, MALLOC_OVERHEAD, IO_SIZE, 0},
{ "key_cache_block_size", OPT_KEY_CACHE_BLOCK_SIZE, "",
&opt_key_cache_block_size,
@ -395,10 +400,18 @@ static void usage(void)
-e, --extend-check Try to recover every possible row from the data file\n\
Normally this will also find a lot of garbage rows;\n\
Don't use this option if you are not totally desperate.\n\
-f, --force Overwrite old temporary files.\n\
-f, --force Overwrite old temporary files. Add another --force to\n\
avoid 'myisam_sort_buffer_size is too small' errors.\n\
In this case we will attempt to do the repair with the\n\
given myisam_sort_buffer_size and dynamically allocate\n\
as many management buffers as needed.\n\
-k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
in the index file is correct. Enables --quick\n\
--max-record-length=#\n\
Skip rows bigger than this if myisamchk can't allocate\n\
memory to hold it.\n\
@ -541,10 +554,13 @@ get_one_option(int optid,
if (argument == disabled_my_option)
{
check_param.tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL;
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE);
check_param.testflag&= ~(T_FORCE_CREATE | T_UPDATE_STATE |
T_FORCE_SORT_MEMORY);
}
else
{
if (check_param.testflag & T_FORCE_CREATE)
check_param.testflag= T_FORCE_SORT_MEMORY;
check_param.tmpfile_createflag= O_RDWR | O_TRUNC;
check_param.testflag|= T_FORCE_CREATE | T_UPDATE_STATE;
}
@ -597,8 +613,26 @@ get_one_option(int optid,
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_FORCE_UNIQUENESS);
else
{
/*
If T_QUICK was specified before, but not OPT_CREATE_MISSING_KEYS,
then add T_FORCE_UNIQUENESS.
*/
check_param.testflag|=
(check_param.testflag & T_QUICK) ? T_FORCE_UNIQUENESS : T_QUICK;
((check_param.testflag & (T_QUICK | T_CREATE_MISSING_KEYS)) ==
T_QUICK ? T_FORCE_UNIQUENESS : T_QUICK);
}
break;
case OPT_CREATE_MISSING_KEYS:
if (argument == disabled_my_option)
check_param.testflag&= ~(T_QUICK | T_CREATE_MISSING_KEYS);
else
{
check_param.testflag|= T_QUICK | T_CREATE_MISSING_KEYS;
/* Use repair by sort by default */
if (!(check_param.testflag & T_REP_ANY))
check_param.testflag|= T_REP_BY_SORT;
}
break;
case 'u':
if (argument == disabled_my_option)

9
storage/myisam/myisamdef.h
View File

@ -629,10 +629,11 @@ typedef struct st_mi_block_info /* Parameter to _mi_get_block_info */
#define UPDATE_AUTO_INC 8
#define UPDATE_OPEN_COUNT 16
#define USE_BUFFER_INIT (((1024L*512L-MALLOC_OVERHEAD)/IO_SIZE)*IO_SIZE)
#define READ_BUFFER_INIT (1024L*256L-MALLOC_OVERHEAD)
#define SORT_BUFFER_INIT (2048L*1024L-MALLOC_OVERHEAD)
#define MIN_SORT_BUFFER (4096-MALLOC_OVERHEAD)
/* We use MY_ALIGN_DOWN here mainly to ensure that we get stable values for mysqld --help ) */
#define KEY_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L-MALLOC_OVERHEAD, IO_SIZE)
#define READ_BUFFER_INIT MY_ALIGN_DOWN(1024L*256L-MALLOC_OVERHEAD, 1024)
#define SORT_BUFFER_INIT MY_ALIGN_DOWN(1024L*1024L*128L-MALLOC_OVERHEAD, 1024)
#define MIN_SORT_BUFFER 4096
enum myisam_log_commands
{

289
storage/myisam/sort.c
View File

@ -34,8 +34,10 @@
#define MERGEBUFF 15
#define MERGEBUFF2 31
#define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
#define DISK_BUFFER_SIZE (IO_SIZE*16)
#define DISK_BUFFER_SIZE (IO_SIZE*128)
/* How many keys we can keep in memory */
typedef ulonglong ha_keys;
/*
Pointers of functions for store and read keys from temp file
@ -45,42 +47,42 @@ extern void print_error(const char *fmt,...);
/* Functions defined in this file */
static ha_rows find_all_keys(MI_SORT_PARAM *info,uint keys,
static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys,
DYNAMIC_ARRAY *buffpek,int *maxbuffer,
DYNAMIC_ARRAY *buffpek,uint *maxbuffer,
IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions);
static int write_keys(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,IO_CACHE *tempfile);
ha_keys count, BUFFPEK *buffpek,IO_CACHE *tempfile);
static int write_key(MI_SORT_PARAM *info, uchar *key,
IO_CACHE *tempfile);
static int write_index(MI_SORT_PARAM *info,uchar * *sort_keys,
uint count);
static int merge_many_buff(MI_SORT_PARAM *info,uint keys,
ha_keys count);
static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar * *sort_keys,
BUFFPEK *buffpek,int *maxbuffer,
BUFFPEK *buffpek, uint *maxbuffer,
IO_CACHE *t_file);
static uint read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MI_SORT_PARAM *info,uint keys,
static my_off_t read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int merge_buffers(MI_SORT_PARAM *info, ha_keys keys,
IO_CACHE *from_file, IO_CACHE *to_file,
uchar * *sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb);
static int merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int,
static int merge_index(MI_SORT_PARAM *,ha_keys,uchar **,BUFFPEK *, uint,
IO_CACHE *);
static int flush_ft_buf(MI_SORT_PARAM *info);
static int write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys,
uint count, BUFFPEK *buffpek,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile);
static uint read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek,
uint sort_length);
static int write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,
uchar *key, uint sort_length, uint count);
uchar *key, uint sort_length, ha_keys count);
static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file,
uchar* key, uint sort_length,
uint count);
ha_keys count);
static inline int
my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
@ -101,15 +103,15 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs);
int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
ulonglong sortbuff_size)
{
int error,maxbuffer,skr;
uint sort_length, keys;
int error;
uint sort_length, maxbuffer;
ulonglong memavl, old_memavl;
DYNAMIC_ARRAY buffpek;
ha_rows records;
ha_rows records, keys;
uchar **sort_keys;
IO_CACHE tempfile, tempfile_for_exceptions;
DBUG_ENTER("_create_index_by_sort");
DBUG_PRINT("enter",("sort_length: %d", info->key_length));
DBUG_PRINT("enter",("sort_length: %u", info->key_length));
if (info->keyinfo->flag & HA_VAR_LENGTH_KEY)
{
@ -135,37 +137,55 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
sort_length= info->key_length;
LINT_INIT(keys);
if ((memavl - sizeof(BUFFPEK)) / (sort_length + sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER)
{
if ((records < UINT_MAX32) &&
((my_off_t) (records + 1) *
(sort_length + sizeof(char*)) <= (my_off_t) memavl))
keys= (uint)records+1;
/* Check if we can fit all keys into memory */
if (((ulonglong) (records + 1) *
(sort_length + sizeof(char*)) <= memavl))
keys= records+1;
else if ((info->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (records / keys)+1);
}
else
{
/*
All keys can't fit in memory.
Calculate how many keys + buffers we can keep in memory
*/
uint maxbuffer_org;
do
{
skr=maxbuffer;
if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK) * maxbuffer ||
(keys= (memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
keys < maxbuffer)
{
mi_check_print_error(info->sort_info->param,
"myisam_sort_buffer_size is too small");
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records,
sort_length);
my_errno= ENOMEM;
goto err;
}
}
while ((maxbuffer= (int) (records/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (records/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys=(uchar **)my_malloc(keys*(sort_length+sizeof(char*))+
HA_FT_MAXBYTELEN, MYF(0))))
{
if (my_init_dynamic_array(&buffpek, sizeof(BUFFPEK), maxbuffer,
maxbuffer/2, MYF(0)))
min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= 0;
@ -179,24 +199,30 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
}
if (memavl < MIN_SORT_BUFFER)
{
mi_check_print_error(info->sort_info->param,"MyISAM sort buffer too small"); /* purecov: tested */
my_errno= ENOMEM; /* purecov: tested */
goto err; /* purecov: tested */
/* purecov: begin inspected */
mi_check_print_error(info->sort_info->param,
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) records, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
(*info->lock_in_memory)(info->sort_info->param);/* Everything is allocated */
if (!no_messages)
printf(" - Searching for keys, allocating buffer for %d keys\n",keys);
my_fprintf(stdout,
" - Searching for keys, allocating buffer for %llu keys\n",
(ulonglong) keys);
if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
if ((records= find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer,
&tempfile,&tempfile_for_exceptions))
== HA_POS_ERROR)
goto err; /* purecov: tested */
if (maxbuffer == 0)
{
if (!no_messages)
printf(" - Dumping %lu keys\n", (ulong) records);
if (write_index(info,sort_keys, (uint) records))
my_fprintf(stdout, " - Dumping %llu keys\n", (ulonglong) records);
if (write_index(info, sort_keys, (ha_keys) records))
goto err; /* purecov: inspected */
}
else
@ -205,7 +231,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
if (maxbuffer >= MERGEBUFF2)
{
if (!no_messages)
printf(" - Merging %lu keys\n", (ulong) records); /* purecov: tested */
my_fprintf(stdout, " - Merging %llu keys\n",
(ulonglong) records); /* purecov: tested */
if (merge_many_buff(info,keys,sort_keys,
dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile))
goto err; /* purecov: inspected */
@ -259,13 +286,13 @@ err:
/* Search after all keys and place them in a temp. file */
static ha_rows find_all_keys(MI_SORT_PARAM *info, uint keys,
static ha_rows find_all_keys(MI_SORT_PARAM *info, ha_rows keys,
uchar **sort_keys, DYNAMIC_ARRAY *buffpek,
int *maxbuffer, IO_CACHE *tempfile,
uint *maxbuffer, IO_CACHE *tempfile,
IO_CACHE *tempfile_for_exceptions)
{
int error;
uint idx;
ha_rows idx;
DBUG_ENTER("find_all_keys");
idx=error=0;
@ -314,9 +341,10 @@ pthread_handler_t thr_find_all_keys(void *arg)
{
MI_SORT_PARAM *sort_param= (MI_SORT_PARAM*) arg;
int error;
ulonglong memavl, old_memavl;
uint keys, sort_length;
uint idx, maxbuffer;
ulonglong memavl, old_memavl, sortbuff_size;
ha_keys keys, idx;
uint sort_length;
uint maxbuffer;
uchar **sort_keys=0;
LINT_INIT(keys);
@ -351,37 +379,47 @@ pthread_handler_t thr_find_all_keys(void *arg)
bzero((char*) &sort_param->unique, sizeof(sort_param->unique));
sort_keys= (uchar **) NULL;
memavl= max(sort_param->sortbuff_size, MIN_SORT_BUFFER);
idx= (uint)sort_param->sort_info->max_records;
sortbuff_size= sort_param->sortbuff_size;
memavl= max(sortbuff_size, MIN_SORT_BUFFER);
idx= (ha_keys) sort_param->sort_info->max_records;
sort_length= sort_param->key_length;
maxbuffer= 1;
if ((memavl - sizeof(BUFFPEK)) / (sort_length +
sizeof(char *)) > UINT_MAX32)
memavl= sizeof(BUFFPEK) + UINT_MAX32 * (sort_length + sizeof(char *));
while (memavl >= MIN_SORT_BUFFER)
{
if ((my_off_t) (idx+1)*(sort_length+sizeof(char*)) <=
(my_off_t) memavl)
keys= idx+1;
else if ((sort_param->sort_info->param->testflag &
(T_FORCE_SORT_MEMORY | T_CREATE_MISSING_KEYS)) ==
T_FORCE_SORT_MEMORY)
{
/*
Use all of the given sort buffer for key data.
Allocate 1000 buffers at a start for new data. More buffers
will be allocated when needed.
*/
keys= memavl / (sort_length+sizeof(char*));
maxbuffer= (uint) min((ulonglong) 1000, (idx / keys)+1);
}
else
{
uint skr;
uint maxbuffer_org;
do
{
skr= maxbuffer;
maxbuffer_org= maxbuffer;
if (memavl < sizeof(BUFFPEK)*maxbuffer ||
(keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
(sort_length+sizeof(char*))) <= 1 ||
keys < (uint) maxbuffer)
{
mi_check_print_error(sort_param->sort_info->param,
"myisam_sort_buffer_size is too small");
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
goto err;
}
}
while ((maxbuffer= (int) (idx/(keys-1)+1)) != skr);
while ((maxbuffer= (uint) (idx/(keys-1)+1)) != maxbuffer_org);
}
if ((sort_keys= (uchar**)
my_malloc(keys*(sort_length+sizeof(char*))+
@ -389,7 +427,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
HA_FT_MAXBYTELEN : 0), MYF(0))))
{
if (my_init_dynamic_array(&sort_param->buffpek, sizeof(BUFFPEK),
maxbuffer, maxbuffer/2, MYF(0)))
maxbuffer, min(maxbuffer/2, 1000), MYF(0)))
{
my_free(sort_keys);
sort_keys= (uchar **) NULL; /* for err: label */
@ -404,14 +442,19 @@ pthread_handler_t thr_find_all_keys(void *arg)
}
if (memavl < MIN_SORT_BUFFER)
{
/* purecov: begin inspected */
mi_check_print_error(sort_param->sort_info->param,
"MyISAM sort buffer too small");
goto err; /* purecov: tested */
"myisam_sort_buffer_size is too small. Current myisam_sort_buffer_size: %llu rows: %llu sort_length: %u",
sortbuff_size, (ulonglong) idx, sort_length);
my_errno= ENOMEM;
goto err;
/* purecov: end inspected */
}
if (sort_param->sort_info->param->testflag & T_VERBOSE)
printf("Key %d - Allocating buffer for %d keys\n",
sort_param->key + 1, keys);
my_fprintf(stdout,
"Key %d - Allocating buffer for %llu keys\n",
sort_param->key + 1, (ulonglong) keys);
sort_param->sort_keys= sort_keys;
idx= error= 0;
@ -436,7 +479,8 @@ pthread_handler_t thr_find_all_keys(void *arg)
&sort_param->tempfile))
goto err;
sort_keys[0]= (uchar*) (sort_keys+keys);
memcpy(sort_keys[0], sort_keys[idx - 1], (size_t) sort_param->key_length);
memcpy(sort_keys[0], sort_keys[idx - 1],
(size_t) sort_param->key_length);
idx= 1;
}
sort_keys[idx]= sort_keys[idx - 1] + sort_param->key_length;
@ -454,7 +498,6 @@ pthread_handler_t thr_find_all_keys(void *arg)
else
sort_param->keys= idx;
sort_param->sort_keys_length= keys;
goto ok;
err:
@ -527,7 +570,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
{
if (param->testflag & T_VERBOSE)
{
printf("Key %d - Dumping %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Dumping %llu keys\n", sinfo->key+1,
(ulonglong) sinfo->keys);
fflush(stdout);
}
if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
@ -584,10 +629,12 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
if (maxbuffer >= MERGEBUFF2)
{
if (param->testflag & T_VERBOSE)
printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys);
my_fprintf(stdout,
"Key %d - Merging %llu keys\n",
sinfo->key+1, (ulonglong) sinfo->keys);
if (merge_many_buff(sinfo, keys, (uchar **)mergebuf,
dynamic_element(&sinfo->buffpek, 0, BUFFPEK *),
(int*) &maxbuffer, &sinfo->tempfile))
&maxbuffer, &sinfo->tempfile))
{
got_error=1;
continue;
@ -651,12 +698,15 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
/* Write all keys in memory to file for later merge */
static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
uint count, BUFFPEK *buffpek, IO_CACHE *tempfile)
ha_keys count, BUFFPEK *buffpek, IO_CACHE *tempfile)
{
uchar **end;
uint sort_length=info->key_length;
DBUG_ENTER("write_keys");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -669,7 +719,7 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
{
if (my_b_write(tempfile,(uchar*) *sort_keys,(uint) sort_length))
if (my_b_write(tempfile,(uchar*) *sort_keys, sort_length))
DBUG_RETURN(1); /* purecov: inspected */
}
DBUG_RETURN(0);
@ -693,13 +743,16 @@ my_var_write(MI_SORT_PARAM *info, IO_CACHE *to_file, uchar *bufs)
static int write_keys_varlen(MI_SORT_PARAM *info,
register uchar **sort_keys,
uint count, BUFFPEK *buffpek,
ha_keys count, BUFFPEK *buffpek,
IO_CACHE *tempfile)
{
uchar **end;
int err;
DBUG_ENTER("write_keys_varlen");
if (!buffpek)
DBUG_RETURN(1); /* Out of memory */
my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
info);
if (!my_b_inited(tempfile) &&
@ -738,7 +791,7 @@ static int write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile)
/* Write index */
static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
register uint count)
register ha_keys count)
{
DBUG_ENTER("write_index");
@ -755,11 +808,11 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
/* Merge buffers to make < MERGEBUFF2 buffers */
static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
static int merge_many_buff(MI_SORT_PARAM *info, ha_keys keys,
uchar **sort_keys, BUFFPEK *buffpek,
int *maxbuffer, IO_CACHE *t_file)
uint *maxbuffer, IO_CACHE *t_file)
{
register int i;
register uint i;
IO_CACHE t_file2, *from_file, *to_file, *temp;
BUFFPEK *lastbuff;
DBUG_ENTER("merge_many_buff");
@ -777,7 +830,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
reinit_io_cache(from_file,READ_CACHE,0L,0,0);
reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
lastbuff=buffpek;
for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
for (i=0 ; i + MERGEBUFF*3/2 <= *maxbuffer ; i+=MERGEBUFF)
{
if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
buffpek+i,buffpek+i+MERGEBUFF-1))
@ -789,7 +842,7 @@ static int merge_many_buff(MI_SORT_PARAM *info, uint keys,
if (flush_io_cache(to_file))
break; /* purecov: inspected */
temp=from_file; from_file=to_file; to_file=temp;
*maxbuffer= (int) (lastbuff-buffpek)-1;
*maxbuffer= (uint) (lastbuff-buffpek)-1;
}
cleanup:
close_cached_file(to_file); /* This holds old result */
@ -818,35 +871,36 @@ cleanup:
-1 Error
*/
static uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
uint length;
register ha_keys count;
my_off_t length;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base,
(length= sort_length*count),
(length= sort_length * count),
buffpek->file_pos, MYF_RW))
return((uint) -1); /* purecov: inspected */
return(HA_OFFSET_ERROR); /* purecov: inspected */
buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */
buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer */
static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
{
register uint count;
register ha_keys count;
uint16 length_of_key = 0;
uint idx;
uchar *buffp;
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
if ((count= (ha_keys) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
buffp = buffpek->base;
@ -854,11 +908,11 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
{
if (mysql_file_pread(fromfile->file, (uchar*)&length_of_key,
sizeof(length_of_key), buffpek->file_pos, MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=sizeof(length_of_key);
if (mysql_file_pread(fromfile->file, (uchar*) buffp,
length_of_key, buffpek->file_pos, MYF_RW))
return((uint) -1);
return(HA_OFFSET_ERROR);
buffpek->file_pos+=length_of_key;
buffp = buffp + sort_length;
}
@ -866,15 +920,15 @@ static uint read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek,
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (((my_off_t) count) * sort_length);
} /* read_to_buffer_varlen */
static int write_merge_key_varlen(MI_SORT_PARAM *info,
IO_CACHE *to_file, uchar* key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
uint idx;
ha_keys idx;
uchar *bufs = key;
for (idx=1;idx<=count;idx++)
@ -890,32 +944,37 @@ static int write_merge_key_varlen(MI_SORT_PARAM *info,
static int write_merge_key(MI_SORT_PARAM *info __attribute__((unused)),
IO_CACHE *to_file, uchar *key,
uint sort_length, uint count)
uint sort_length, ha_keys count)
{
return my_b_write(to_file, key, (size_t) sort_length*count);
return my_b_write(to_file, key, ((size_t) sort_length) * count);
}
/*
Merge buffers to one buffer
If to_file == 0 then use info->key_write
Return:
0 ok
1 error
*/
static int
merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
merge_buffers(MI_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file,
IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff,
BUFFPEK *Fb, BUFFPEK *Tb)
{
int error;
uint sort_length,maxcount;
int error= 1;
uint sort_length;
ha_keys maxcount;
ha_rows count;
my_off_t UNINIT_VAR(to_start_filepos);
my_off_t UNINIT_VAR(to_start_filepos), read_length;
uchar *strpos;
BUFFPEK *buffpek,**refpek;
QUEUE queue;
DBUG_ENTER("merge_buffers");
count=error=0;
maxcount=keys/((uint) (Tb-Fb) +1);
maxcount= keys/((uint) (Tb-Fb) +1);
DBUG_ASSERT(maxcount > 0);
LINT_INIT(to_start_filepos);
if (to_file)
@ -932,10 +991,10 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
{
count+= buffpek->count;
buffpek->base= (uchar*) strpos;
buffpek->max_keys=maxcount;
strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,
sort_length));
if (error == -1)
buffpek->max_keys= maxcount;
strpos+= (read_length= info->read_to_buffer(from_file,buffpek,
sort_length));
if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
queue_insert(&queue,(uchar*) buffpek);
}
@ -966,10 +1025,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
/* It's enough to check for killedptr before a slow operation */
if (killed_ptr(info->sort_info->param))
{
error=1;
goto err;
}
if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length)))
if (!(read_length= info->read_to_buffer(from_file,buffpek,sort_length)))
{
uchar *base= buffpek->base;
uint max_keys=buffpek->max_keys;
@ -996,9 +1054,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
break; /* One buffer have been removed */
}
else if (read_length == HA_OFFSET_ERROR)
goto err; /* purecov: inspected */
}
else if (error == -1)
goto err; /* purecov: inspected */
queue_replace_top(&queue); /* Top element has been replaced */
}
}
@ -1030,8 +1088,9 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file,
}
}
}
while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 &&
error != 0);
while ((read_length= info->read_to_buffer(from_file,buffpek,sort_length)) != HA_OFFSET_ERROR && read_length != 0);
if (read_length == 0)
error= 0;
lastbuff->count=count;
if (to_file)
@ -1045,8 +1104,8 @@ err:
/* Do a merge to output-file (save only positions) */
static int
merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile)
merge_index(MI_SORT_PARAM *info, ha_keys keys, uchar **sort_keys,
BUFFPEK *buffpek, uint maxbuffer, IO_CACHE *tempfile)
{
DBUG_ENTER("merge_index");
if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek,
@ -1055,6 +1114,7 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
DBUG_RETURN(0);
} /* merge_index */
static int
flush_ft_buf(MI_SORT_PARAM *info)
{
@ -1067,4 +1127,3 @@ flush_ft_buf(MI_SORT_PARAM *info)
}
return err;
}
Write Preview
Loading…
Cancel
Save