Browse Source
Addresses #2249 [t:2249] Merge DB_ENV->put_multiple branch back onto main.
Addresses #2249 [t:2249] Merge DB_ENV->put_multiple branch back onto main.
Fixed some makefile issues, ported recovery tests to use toku_hard_crash_on_purpose(), fixed db-benchmark-test to use default name for 0th db git-svn-id: file:///svn/toku/tokudb@16936 c7de825b-a66e-492c-adef-691d508d4ae1pull/73/head
Yoni Fogel
13 years ago
38 changed files with 1668 additions and 108 deletions
-
9buildheader/db.h_4_1
-
9buildheader/db.h_4_3
-
9buildheader/db.h_4_4
-
54buildheader/db.h_4_5
-
56buildheader/db.h_4_6
-
7buildheader/make_db_h.c
-
7buildheader/tdb.h
-
174db-benchmark-test/db-benchmark-test.c
-
7include/db.h
-
52newbrt/brt.c
-
6newbrt/brt.h
-
10newbrt/brttypes.h
-
4newbrt/log-internal.h
-
1newbrt/log.h
-
12newbrt/logformat.c
-
42newbrt/logger.c
-
2newbrt/logger.h
-
165newbrt/recover.c
-
9newbrt/recover.h
-
2newbrt/tdb-recover.c
-
2newbrt/tests/recovery-cbegin-cend-hello.c
-
2newbrt/tests/recovery-cbegin-cend.c
-
2newbrt/tests/recovery-cbegin.c
-
2newbrt/tests/recovery-empty.c
-
2newbrt/tests/recovery-fopen-missing-file.c
-
2newbrt/tests/recovery-hello.c
-
2newbrt/tests/recovery-no-datadir.c
-
2newbrt/tests/recovery-no-log.c
-
2newbrt/tests/recovery-no-logdir.c
-
17newbrt/wbuf.h
-
2release/examples/db-insert.c
-
4src/tests/Makefile
-
270src/tests/env-put-multiple.c
-
214src/tests/recover-lsn-filter-multiple.c
-
198src/tests/recover-put-multiple-fdelete-all.c
-
212src/tests/recover-put-multiple-fdelete-some.c
-
4src/ydb-internal.h
-
200src/ydb.c
@ -0,0 +1,270 @@ |
|||
// this test makes sure the LSN filtering is used during recovery of put_multiple |
|||
|
|||
#include <sys/stat.h> |
|||
#include <fcntl.h> |
|||
#include "test.h" |
|||
|
|||
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE; |
|||
|
|||
enum {MAX_DBS = 64, MAX_KEY = 8, MAX_VAL = 8}; |
|||
DB *dbs_multiple[MAX_DBS]; |
|||
DB *dbs_single[MAX_DBS]; |
|||
char names_single[MAX_DBS][sizeof("dbs_0xFFF")]; |
|||
char names_multiple[MAX_DBS][sizeof("dbm_0xFFF")]; |
|||
uint32_t num_dbs; |
|||
uint32_t flags[MAX_DBS]; |
|||
uint32_t kbuf[MAX_DBS][MAX_KEY/4]; |
|||
uint32_t vbuf[MAX_DBS][MAX_VAL/4]; |
|||
|
|||
#define CKERRIFNOT0(r) do { if (num_dbs>0) { CKERR(r); } else { CKERR2(r, EINVAL); } } while (0) |
|||
#define CKERR2IFNOT0(r, rexpect) do { if (num_dbs>0) { CKERR2(r, rexpect); } else { CKERR2(r, EINVAL); } } while (0) |
|||
|
|||
static int |
|||
put_multiple_generate(DBT *row, uint32_t num_dbs_in, DB **UU(dbs_in), DBT *keys, DBT *vals, void *extra) { |
|||
assert(num_dbs_in > 0); |
|||
assert(num_dbs_in == num_dbs); |
|||
assert(extra==&num_dbs); //Verifying extra gets set right. |
|||
assert(row->size == 4); |
|||
uint32_t which; |
|||
for (which = 0; which < num_dbs_in; which++) { |
|||
kbuf[which][0] = *(uint32_t*)row->data; |
|||
kbuf[which][1] = which; |
|||
vbuf[which][0] = which; |
|||
vbuf[which][1] = *(uint32_t*)row->data; |
|||
keys[which].data = kbuf[which]; |
|||
keys[which].size = sizeof(kbuf[which]); |
|||
vals[which].data = vbuf[which]; |
|||
vals[which].size = sizeof(vbuf[which]); |
|||
} |
|||
return 0; |
|||
} |
|||
|
|||
static int |
|||
put_multiple_clean(DBT *UU(row), uint32_t UU(num_dbs_in), DB **UU(dbs_in), DBT *UU(keys), DBT *UU(vals), void *extra) { |
|||
assert(extra==&num_dbs); //Verifying extra gets set right. |
|||
return 0; |
|||
} |
|||
|
|||
static void run_test (void) { |
|||
int r; |
|||
if (verbose) |
|||
printf("env-put-multiple num_dbs[%u]\n", num_dbs); |
|||
|
|||
system("rm -rf " ENVDIR); |
|||
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); |
|||
|
|||
DB_ENV *env; |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
uint32_t which; |
|||
{ |
|||
//Create dbs. |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); |
|||
CKERR(r); |
|||
DB *db; |
|||
for (which = 0; which < num_dbs; which++) { |
|||
r = db_create(&dbs_multiple[which], env, 0); |
|||
CKERR(r); |
|||
db = dbs_multiple[which]; |
|||
r = db->open(db, txn, names_multiple[which], NULL, DB_BTREE, DB_CREATE, 0666); |
|||
CKERR(r); |
|||
r = db_create(&dbs_single[which], env, 0); |
|||
CKERR(r); |
|||
db = dbs_single[which]; |
|||
r = db->open(db, txn, names_single[which], NULL, DB_BTREE, DB_CREATE, 0666); |
|||
CKERR(r); |
|||
} |
|||
r = txn->commit(txn, 0); |
|||
} |
|||
|
|||
|
|||
uint32_t magic = 0xDEADBEEF; |
|||
// txn_begin; insert magic number |
|||
{ |
|||
for (which = 0; which < num_dbs; which++) { |
|||
flags[which] = 0; |
|||
} |
|||
memset(flags, 0, sizeof(flags)); //reset |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); |
|||
CKERR(r); |
|||
|
|||
DBT rowdbt={.data=&magic, .size=sizeof(magic)}; |
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERRIFNOT0(r); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txn, &key, &val, flags[which]); |
|||
CKERR(r); |
|||
} |
|||
r = txn->commit(txn, 0); |
|||
} |
|||
{ |
|||
//Insert again with DB_YESOVERWRITE, expect it to work. |
|||
for (which = 0; which < num_dbs; which++) { |
|||
flags[which] = DB_YESOVERWRITE; |
|||
} |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); |
|||
CKERR(r); |
|||
|
|||
DBT rowdbt={.data=&magic, .size=sizeof(magic)}; |
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERRIFNOT0(r); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txn, &key, &val, flags[which]); |
|||
CKERR(r); |
|||
} |
|||
r = txn->commit(txn, 0); |
|||
} |
|||
{ |
|||
//Insert again with DB_NOOVERWRITE, expect it to fail (unless 0 dbs). |
|||
for (which = 0; which < num_dbs; which++) { |
|||
flags[which] = DB_NOOVERWRITE; |
|||
} |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); |
|||
CKERR(r); |
|||
|
|||
DBT rowdbt={.data=&magic, .size=sizeof(magic)}; |
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERR2IFNOT0(r, DB_KEYEXIST); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txn, &key, &val, flags[which]); |
|||
CKERR2(r, DB_KEYEXIST); |
|||
} |
|||
r = txn->commit(txn, 0); |
|||
} |
|||
|
|||
{ |
|||
//Different number |
|||
magic = 0xFEEDADAD; |
|||
//Insert again with DB_YESOVERWRITE, using 2 transactions, expect it to fail (unless 0 dbs). |
|||
for (which = 0; which < num_dbs; which++) { |
|||
flags[which] = DB_YESOVERWRITE; |
|||
} |
|||
DB_TXN *txna; |
|||
r = env->txn_begin(env, NULL, &txna, 0); |
|||
CKERR(r); |
|||
|
|||
DBT rowdbt={.data=&magic, .size=sizeof(magic)}; |
|||
r = env->put_multiple(env, txna, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERRIFNOT0(r); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txna, &key, &val, flags[which]); |
|||
CKERR(r); |
|||
} |
|||
|
|||
DB_TXN *txnb; |
|||
r = env->txn_begin(env, NULL, &txnb, 0); |
|||
CKERR(r); |
|||
|
|||
//Lock should fail |
|||
r = env->put_multiple(env, txnb, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERR2IFNOT0(r, DB_LOCK_NOTGRANTED); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txnb, &key, &val, flags[which]); |
|||
CKERR2(r, DB_LOCK_NOTGRANTED); |
|||
} |
|||
r = txna->commit(txna, 0); |
|||
|
|||
//Should succeed this time. |
|||
r = env->put_multiple(env, txnb, &rowdbt, num_dbs, dbs_multiple, flags, &num_dbs); |
|||
CKERRIFNOT0(r); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
DBT key={.data = kbuf[which], .size = sizeof(kbuf[which])}; |
|||
DBT val={.data = vbuf[which], .size = sizeof(vbuf[which])}; |
|||
DB *db = dbs_single[which]; |
|||
r = db->put(db, txnb, &key, &val, flags[which]); |
|||
CKERR(r); |
|||
} |
|||
|
|||
r = txnb->commit(txnb, 0); |
|||
} |
|||
|
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); |
|||
CKERR(r); |
|||
DBC *c_single; |
|||
DBC *c_multiple; |
|||
|
|||
DBT k_single, v_single, k_multiple, v_multiple; |
|||
memset(&k_single, 0, sizeof(k_single)); |
|||
memset(&v_single, 0, sizeof(v_single)); |
|||
memset(&k_multiple, 0, sizeof(k_multiple)); |
|||
memset(&v_multiple, 0, sizeof(v_multiple)); |
|||
for (which = 0; which < num_dbs; which++) { |
|||
r = dbs_multiple[which]->cursor(dbs_multiple[which], txn, &c_multiple, 0); |
|||
CKERR(r); |
|||
r = dbs_single[which]->cursor(dbs_single[which], txn, &c_single, 0); |
|||
CKERR(r); |
|||
|
|||
int r1 = 0; |
|||
int r2; |
|||
while (r1 == 0) { |
|||
r1 = c_single->c_get(c_single, &k_single, &v_single, DB_NEXT); |
|||
r2 = c_multiple->c_get(c_multiple, &k_multiple, &v_multiple, DB_NEXT); |
|||
assert(r1==r2); |
|||
CKERR2s(r1, 0, DB_NOTFOUND); |
|||
if (r1 == 0) { |
|||
assert(k_single.size == k_multiple.size); |
|||
assert(v_single.size == v_multiple.size); |
|||
assert(memcmp(k_single.data, k_multiple.data, k_single.size) == 0); |
|||
assert(memcmp(v_single.data, v_multiple.data, v_single.size) == 0); |
|||
} |
|||
} |
|||
r = c_single->c_close(c_single); |
|||
CKERR(r); |
|||
r = c_multiple->c_close(c_multiple); |
|||
CKERR(r); |
|||
} |
|||
r = txn->commit(txn, 0); |
|||
} |
|||
{ |
|||
for (which = 0; which < num_dbs; which++) { |
|||
r = dbs_single[which]->close(dbs_single[which], 0); |
|||
CKERR(r); |
|||
r = dbs_multiple[which]->close(dbs_multiple[which], 0); |
|||
CKERR(r); |
|||
} |
|||
} |
|||
r = env->close(env, 0); |
|||
CKERR(r); |
|||
} |
|||
|
|||
int test_main (int argc, char *argv[]) { |
|||
parse_args(argc, argv); |
|||
uint32_t which; |
|||
for (which = 0; which < MAX_DBS; which++) { |
|||
sprintf(names_multiple[which], "dbm_0x%02X", which); |
|||
sprintf(names_single[which], "dbs_0x%02X", which); |
|||
} |
|||
for (num_dbs = 0; num_dbs < 4; num_dbs++) { |
|||
run_test(); |
|||
} |
|||
for (num_dbs = 4; num_dbs <= MAX_DBS; num_dbs *= 2) { |
|||
run_test(); |
|||
} |
|||
return 0; |
|||
} |
|||
@ -0,0 +1,214 @@ |
|||
// this test makes sure the LSN filtering is used during recovery of put_multiple |
|||
|
|||
#include <sys/stat.h> |
|||
#include <fcntl.h> |
|||
#include "test.h" |
|||
|
|||
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE; |
|||
|
|||
char *namea="a.db"; |
|||
char *nameb="b.db"; |
|||
enum {num_dbs = 2}; |
|||
|
|||
BOOL do_test=FALSE, do_recover=FALSE; |
|||
|
|||
static int |
|||
put_multiple_generate(DBT *row, uint32_t num_dbs_in, DB **UU(dbs_in), DBT *keys, DBT *vals, void *extra) { |
|||
assert((int)num_dbs_in == num_dbs); |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
assert(row->size >= 4); |
|||
int32_t keysize = *(int32_t*)row->data; |
|||
assert((int)row->size >= 4+keysize); |
|||
int32_t valsize = row->size - 4 - keysize; |
|||
void *key = row->data+4; |
|||
void *val = row->data+4 + keysize; |
|||
int which; |
|||
for (which = 0; which < num_dbs; which++) { |
|||
keys[which].size = keysize; |
|||
keys[which].data = key; |
|||
vals[which].size = valsize; |
|||
vals[which].data = val; |
|||
} |
|||
return 0; |
|||
} |
|||
|
|||
static int |
|||
put_multiple_clean(DBT *UU(row), uint32_t UU(num_dbs_in), DB **UU(dbs_in), DBT *UU(keys), DBT *UU(vals), void *extra) { |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
return 0; |
|||
} |
|||
|
|||
static void run_test (void) { |
|||
int r; |
|||
|
|||
system("rm -rf " ENVDIR); |
|||
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); |
|||
|
|||
DB_ENV *env; |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// create a txn that never closes, forcing recovery to run from the beginning of the log |
|||
{ |
|||
DB_TXN *oldest_living_txn; |
|||
r = env->txn_begin(env, NULL, &oldest_living_txn, 0); CKERR(r); |
|||
} |
|||
|
|||
DB *dba; |
|||
DB *dbb; |
|||
r = db_create(&dba, env, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
|
|||
DB *dbs[num_dbs] = {dba, dbb}; |
|||
uint32_t flags[num_dbs] = {DB_YESOVERWRITE, DB_YESOVERWRITE}; |
|||
// txn_begin; insert <a,a>; txn_abort |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="a", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
r = txn->abort(txn); CKERR(r); |
|||
} |
|||
r = dbb->close(dbb, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
dbs[1] = dbb; |
|||
|
|||
// txn_begin; insert <a,b>; |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="b", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
} |
|||
|
|||
// checkpoint |
|||
r = env->txn_checkpoint(env, 0, 0, 0); CKERR(r); |
|||
|
|||
// abort the process |
|||
toku_hard_crash_on_purpose(); |
|||
} |
|||
|
|||
|
|||
static void run_recover (void) { |
|||
DB_ENV *env; |
|||
int r; |
|||
|
|||
// Recovery starts from oldest_living_txn, which is older than any inserts done in run_test, |
|||
// so recovery always runs over the entire log. |
|||
|
|||
// run recovery |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// verify the data |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, namea, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR(r); |
|||
|
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBC *cursor; |
|||
r = db->cursor(db, txn, &cursor, 0); CKERR(r); |
|||
DBT k, v; |
|||
r = cursor->c_get(cursor, dbt_init_malloc(&k), dbt_init_malloc(&v), DB_FIRST); |
|||
assert(r == DB_NOTFOUND); |
|||
|
|||
r = cursor->c_close(cursor); CKERR(r); |
|||
|
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, nameb, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR(r); |
|||
|
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBC *cursor; |
|||
r = db->cursor(db, txn, &cursor, 0); CKERR(r); |
|||
DBT k, v; |
|||
r = cursor->c_get(cursor, dbt_init_malloc(&k), dbt_init_malloc(&v), DB_FIRST); |
|||
assert(r == DB_NOTFOUND); |
|||
|
|||
r = cursor->c_close(cursor); CKERR(r); |
|||
|
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
r = env->close(env, 0); CKERR(r); |
|||
exit(0); |
|||
} |
|||
|
|||
const char *cmd; |
|||
|
|||
static void test_parse_args (int argc, char *argv[]) { |
|||
int resultcode; |
|||
cmd = argv[0]; |
|||
argc--; argv++; |
|||
while (argc>0) { |
|||
if (strcmp(argv[0], "-v") == 0) { |
|||
verbose++; |
|||
} else if (strcmp(argv[0],"-q")==0) { |
|||
verbose--; |
|||
if (verbose<0) verbose=0; |
|||
} else if (strcmp(argv[0], "--test")==0) { |
|||
do_test=TRUE; |
|||
} else if (strcmp(argv[0], "--recover") == 0) { |
|||
do_recover=TRUE; |
|||
} else if (strcmp(argv[0], "-h")==0) { |
|||
resultcode=0; |
|||
do_usage: |
|||
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--test | --recover } \n", cmd); |
|||
exit(resultcode); |
|||
} else { |
|||
fprintf(stderr, "Unknown arg: %s\n", argv[0]); |
|||
resultcode=1; |
|||
goto do_usage; |
|||
} |
|||
argc--; |
|||
argv++; |
|||
} |
|||
} |
|||
|
|||
int test_main (int argc, char *argv[]) { |
|||
test_parse_args(argc, argv); |
|||
if (do_test) { |
|||
run_test(); |
|||
} else if (do_recover) { |
|||
run_recover(); |
|||
} |
|||
return 0; |
|||
} |
|||
@ -0,0 +1,198 @@ |
|||
// this test makes sure the LSN filtering is used during recovery of put_multiple |
|||
|
|||
#include <sys/stat.h> |
|||
#include <fcntl.h> |
|||
#include "test.h" |
|||
|
|||
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE; |
|||
|
|||
char *namea="a.db"; |
|||
char *nameb="b.db"; |
|||
enum {num_dbs = 2}; |
|||
|
|||
BOOL do_test=FALSE, do_recover=FALSE; |
|||
|
|||
static int |
|||
put_multiple_generate(DBT *row, uint32_t num_dbs_in, DB **UU(dbs_in), DBT *keys, DBT *vals, void *extra) { |
|||
assert(num_dbs_in > 0); |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
assert(row->size >= 4); |
|||
int32_t keysize = *(int32_t*)row->data; |
|||
assert((int)row->size >= 4+keysize); |
|||
int32_t valsize = row->size - 4 - keysize; |
|||
void *key = row->data+4; |
|||
void *val = row->data+4 + keysize; |
|||
uint32_t which; |
|||
for (which = 0; which < num_dbs_in; which++) { |
|||
keys[which].size = keysize; |
|||
keys[which].data = key; |
|||
vals[which].size = valsize; |
|||
vals[which].data = val; |
|||
} |
|||
return 0; |
|||
} |
|||
|
|||
static int |
|||
put_multiple_clean(DBT *UU(row), uint32_t UU(num_dbs_in), DB **UU(dbs_in), DBT *UU(keys), DBT *UU(vals), void *extra) { |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
return 0; |
|||
} |
|||
|
|||
static void run_test (void) { |
|||
int r; |
|||
|
|||
system("rm -rf " ENVDIR); |
|||
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); |
|||
|
|||
DB_ENV *env; |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// create a txn that never closes, forcing recovery to run from the beginning of the log |
|||
{ |
|||
DB_TXN *oldest_living_txn; |
|||
r = env->txn_begin(env, NULL, &oldest_living_txn, 0); CKERR(r); |
|||
} |
|||
|
|||
DB *dba; |
|||
DB *dbb; |
|||
r = db_create(&dba, env, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
|
|||
DB *dbs[num_dbs] = {dba, dbb}; |
|||
uint32_t flags[num_dbs] = {DB_YESOVERWRITE, DB_YESOVERWRITE}; |
|||
// txn_begin; insert <a,a>; txn_abort |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="a", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
r = txn->abort(txn); CKERR(r); |
|||
} |
|||
r = dbb->close(dbb, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
dbs[1] = dbb; |
|||
|
|||
// txn_begin; insert <a,b>; |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="b", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
} |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
r = dba->close(dba, 0); CKERR(r); |
|||
r = env->dbremove(env, txn, namea, NULL, 0); CKERR(r); |
|||
r = dba->close(dbb, 0); CKERR(r); |
|||
r = env->dbremove(env, txn, nameb, NULL, 0); CKERR(r); |
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
} |
|||
|
|||
r = env->log_flush(env, NULL); CKERR(r); |
|||
// abort the process |
|||
toku_hard_crash_on_purpose(); |
|||
} |
|||
|
|||
|
|||
static void run_recover (void) { |
|||
DB_ENV *env; |
|||
int r; |
|||
|
|||
// Recovery starts from oldest_living_txn, which is older than any inserts done in run_test, |
|||
// so recovery always runs over the entire log. |
|||
|
|||
// run recovery |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// verify the data |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, namea, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR2(r, ENOENT); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, nameb, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR2(r, ENOENT); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
r = env->close(env, 0); CKERR(r); |
|||
exit(0); |
|||
} |
|||
|
|||
const char *cmd; |
|||
|
|||
static void test_parse_args (int argc, char *argv[]) { |
|||
int resultcode; |
|||
cmd = argv[0]; |
|||
argc--; argv++; |
|||
while (argc>0) { |
|||
if (strcmp(argv[0], "-v") == 0) { |
|||
verbose++; |
|||
} else if (strcmp(argv[0],"-q")==0) { |
|||
verbose--; |
|||
if (verbose<0) verbose=0; |
|||
} else if (strcmp(argv[0], "--test")==0) { |
|||
do_test=TRUE; |
|||
} else if (strcmp(argv[0], "--recover") == 0) { |
|||
do_recover=TRUE; |
|||
} else if (strcmp(argv[0], "-h")==0) { |
|||
resultcode=0; |
|||
do_usage: |
|||
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--test | --recover } \n", cmd); |
|||
exit(resultcode); |
|||
} else { |
|||
fprintf(stderr, "Unknown arg: %s\n", argv[0]); |
|||
resultcode=1; |
|||
goto do_usage; |
|||
} |
|||
argc--; |
|||
argv++; |
|||
} |
|||
} |
|||
|
|||
int test_main (int argc, char *argv[]) { |
|||
test_parse_args(argc, argv); |
|||
if (do_test) { |
|||
run_test(); |
|||
} else if (do_recover) { |
|||
run_recover(); |
|||
} |
|||
return 0; |
|||
} |
|||
@ -0,0 +1,212 @@ |
|||
// this test makes sure the LSN filtering is used during recovery of put_multiple |
|||
|
|||
#include <sys/stat.h> |
|||
#include <fcntl.h> |
|||
#include "test.h" |
|||
|
|||
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE; |
|||
|
|||
char *namea="a.db"; |
|||
char *nameb="b.db"; |
|||
enum {num_dbs = 2}; |
|||
|
|||
BOOL do_test=FALSE, do_recover=FALSE; |
|||
|
|||
static int |
|||
put_multiple_generate(DBT *row, uint32_t num_dbs_in, DB **UU(dbs_in), DBT *keys, DBT *vals, void *extra) { |
|||
assert(num_dbs_in > 0); |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
assert(row->size >= 4); |
|||
int32_t keysize = *(int32_t*)row->data; |
|||
assert((int)row->size >= 4+keysize); |
|||
int32_t valsize = row->size - 4 - keysize; |
|||
void *key = row->data+4; |
|||
void *val = row->data+4 + keysize; |
|||
uint32_t which; |
|||
for (which = 0; which < num_dbs_in; which++) { |
|||
keys[which].size = keysize; |
|||
keys[which].data = key; |
|||
vals[which].size = valsize; |
|||
vals[which].data = val; |
|||
} |
|||
return 0; |
|||
} |
|||
|
|||
static int |
|||
put_multiple_clean(DBT *UU(row), uint32_t UU(num_dbs_in), DB **UU(dbs_in), DBT *UU(keys), DBT *UU(vals), void *extra) { |
|||
if (do_recover) |
|||
assert(extra==NULL); |
|||
else |
|||
assert(extra==&namea); //Verifying extra gets set right. |
|||
return 0; |
|||
} |
|||
|
|||
static void run_test (void) { |
|||
int r; |
|||
|
|||
system("rm -rf " ENVDIR); |
|||
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); |
|||
|
|||
DB_ENV *env; |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// create a txn that never closes, forcing recovery to run from the beginning of the log |
|||
{ |
|||
DB_TXN *oldest_living_txn; |
|||
r = env->txn_begin(env, NULL, &oldest_living_txn, 0); CKERR(r); |
|||
} |
|||
|
|||
DB *dba; |
|||
DB *dbb; |
|||
r = db_create(&dba, env, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
|
|||
DB *dbs[num_dbs] = {dba, dbb}; |
|||
uint32_t flags[num_dbs] = {DB_YESOVERWRITE, DB_YESOVERWRITE}; |
|||
// txn_begin; insert <a,a>; txn_abort |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="a", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
r = txn->abort(txn); CKERR(r); |
|||
} |
|||
r = dbb->close(dbb, 0); CKERR(r); |
|||
r = db_create(&dbb, env, 0); CKERR(r); |
|||
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r); |
|||
dbs[1] = dbb; |
|||
|
|||
// txn_begin; insert <a,b>; |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBT k={.data="a", .size=2}; |
|||
DBT v={.data="b", .size=2}; |
|||
uint8_t row[4+k.size+v.size]; |
|||
*(uint32_t*)&row[0] = k.size; |
|||
memcpy(row+4, k.data, k.size); |
|||
memcpy(row+4+k.size, v.data, v.size); |
|||
DBT rowdbt = {.data = row, .size = sizeof(row)}; |
|||
|
|||
r = env->put_multiple(env, txn, &rowdbt, num_dbs, dbs, flags, &namea); CKERR(r); |
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
} |
|||
{ |
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
r = dba->close(dba, 0); CKERR(r); |
|||
r = env->dbremove(env, txn, namea, NULL, 0); CKERR(r); |
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
} |
|||
|
|||
r = env->log_flush(env, NULL); CKERR(r); |
|||
// abort the process |
|||
toku_hard_crash_on_purpose(); |
|||
} |
|||
|
|||
|
|||
static void run_recover (void) { |
|||
DB_ENV *env; |
|||
int r; |
|||
|
|||
// Recovery starts from oldest_living_txn, which is older than any inserts done in run_test, |
|||
// so recovery always runs over the entire log. |
|||
|
|||
// run recovery |
|||
r = db_env_create(&env, 0); CKERR(r); |
|||
r = env->set_multiple_callbacks(env, |
|||
put_multiple_generate, put_multiple_clean, |
|||
NULL, NULL); |
|||
CKERR(r); |
|||
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); |
|||
|
|||
// verify the data |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, namea, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR2(r, ENOENT); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
{ |
|||
DB *db; |
|||
r = db_create(&db, env, 0); CKERR(r); |
|||
r = db->open(db, NULL, nameb, NULL, DB_UNKNOWN, DB_AUTO_COMMIT, 0666); CKERR(r); |
|||
|
|||
DB_TXN *txn; |
|||
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); |
|||
DBC *cursor; |
|||
r = db->cursor(db, txn, &cursor, 0); CKERR(r); |
|||
DBT k, v; |
|||
r = cursor->c_get(cursor, dbt_init_malloc(&k), dbt_init_malloc(&v), DB_FIRST); |
|||
CKERR(r); |
|||
assert(k.size == 2); |
|||
assert(v.size == 2); |
|||
assert(memcmp(k.data, "a", 2) == 0); |
|||
assert(memcmp(v.data, "b", 2) == 0); |
|||
|
|||
r = cursor->c_close(cursor); CKERR(r); |
|||
|
|||
r = txn->commit(txn, 0); CKERR(r); |
|||
r = db->close(db, 0); CKERR(r); |
|||
} |
|||
r = env->close(env, 0); CKERR(r); |
|||
exit(0); |
|||
} |
|||
|
|||
const char *cmd; |
|||
|
|||
static void test_parse_args (int argc, char *argv[]) { |
|||
int resultcode; |
|||
cmd = argv[0]; |
|||
argc--; argv++; |
|||
while (argc>0) { |
|||
if (strcmp(argv[0], "-v") == 0) { |
|||
verbose++; |
|||
} else if (strcmp(argv[0],"-q")==0) { |
|||
verbose--; |
|||
if (verbose<0) verbose=0; |
|||
} else if (strcmp(argv[0], "--test")==0) { |
|||
do_test=TRUE; |
|||
} else if (strcmp(argv[0], "--recover") == 0) { |
|||
do_recover=TRUE; |
|||
} else if (strcmp(argv[0], "-h")==0) { |
|||
resultcode=0; |
|||
do_usage: |
|||
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--test | --recover } \n", cmd); |
|||
exit(resultcode); |
|||
} else { |
|||
fprintf(stderr, "Unknown arg: %s\n", argv[0]); |
|||
resultcode=1; |
|||
goto do_usage; |
|||
} |
|||
argc--; |
|||
argv++; |
|||
} |
|||
} |
|||
|
|||
int test_main (int argc, char *argv[]) { |
|||
test_parse_args(argc, argv); |
|||
if (do_test) { |
|||
run_test(); |
|||
} else if (do_recover) { |
|||
run_recover(); |
|||
} |
|||
return 0; |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue