Mirrors
/
mariadb
mirror of https://github.com/MariaDB/server
3
0
Fork 0
Code Releases Activity
Browse Source

Create a test for #1389. Addresses #1389. 

git-svn-id: file:///svn/toku/tokudb@8802 c7de825b-a66e-492c-adef-691d508d4ae1
pull/73/head
Bradley C. Kuszmaul 13 years ago
committed by Yoni Fogel
parent
7d34170edf
commit
3c6cbdbf42
14 changed files with 87 additions and 11 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      buildheader/db.h_4_1
  2. 6
      buildheader/db.h_4_3
  3. 6
      buildheader/db.h_4_4
  4. 6
      buildheader/db.h_4_5
  5. 6
      buildheader/db.h_4_6
  6. 6
      buildheader/make_db_h.c
  7. 5
      db-benchmark-test/Makefile
  8. 23
      db-benchmark-test/db-benchmark-test.c
  9. 6
      include/db.h
  10. 3
      newbrt/log-internal.h
  11. 11
      newbrt/log.c
  12. 3
      newbrt/log.h
  13. 1
      newbrt/logformat.c
  14. 10
      src/ydb.c

6
buildheader/db.h_4_1
View File

@ -218,11 +218,15 @@ struct __toku_db_txn_active {
char __toku_dummy0[4];
DB_LSN lsn; /* 32-bit offset=8 size=8, 64=bit offset=8 size=8 */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[8];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[7];
char __toku_dummy1[24];
void *api_internal; /* 32-bit offset=68 size=4, 64=bit offset=112 size=8 */
void* __toku_dummy2[1];

6
buildheader/db.h_4_3
View File

@ -234,11 +234,15 @@ struct __toku_db_txn_active {
DB_LSN lsn; /* 32-bit offset=8 size=8, 64=bit offset=8 size=8 */
char __toku_dummy1[132]; /* Padding at the end */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[13];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[12];
char __toku_dummy1[24];
void *api_internal; /* 32-bit offset=88 size=4, 64=bit offset=152 size=8 */
void* __toku_dummy2[2];

6
buildheader/db.h_4_4
View File

@ -240,11 +240,15 @@ struct __toku_db_txn_active {
DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
char __toku_dummy2[184]; /* Padding at the end */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[16];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[15];
char __toku_dummy1[8];
void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
void* __toku_dummy2[2];

6
buildheader/db.h_4_5
View File

@ -239,11 +239,15 @@ struct __toku_db_txn_active {
DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
char __toku_dummy2[200]; /* Padding at the end */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[16];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[15];
char __toku_dummy1[8];
void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
void* __toku_dummy2[2];

6
buildheader/db.h_4_6
View File

@ -243,11 +243,15 @@ struct __toku_db_txn_active {
DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
char __toku_dummy2[200]; /* Padding at the end */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[17];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[16];
char __toku_dummy1[8];
void *api_internal; /* 32-bit offset=88 size=4, 64=bit offset=168 size=8 */
void* __toku_dummy2[2];

6
buildheader/make_db_h.c
View File

@ -319,7 +319,11 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
assert(sizeof(db_txn_active_fields32)==sizeof(db_txn_active_fields64));
print_struct("db_txn_active", 0, db_txn_active_fields32, db_txn_active_fields64, sizeof(db_txn_active_fields32)/sizeof(db_txn_active_fields32[0]), 0);
assert(sizeof(db_txn_fields32)==sizeof(db_txn_fields64));
print_struct("db_txn", 1, db_txn_fields32, db_txn_fields64, sizeof(db_txn_fields32)/sizeof(db_txn_fields32[0]), 0);
{
printf("struct txn_stat {\n u_int64_t rolltmp_raw_count;\n};\n");
const char *extra[] = {"int (*txn_stat)(DB_TXN *, struct txn_stat **)"};
print_struct("db_txn", 1, db_txn_fields32, db_txn_fields64, sizeof(db_txn_fields32)/sizeof(db_txn_fields32[0]), extra);
}
assert(sizeof(db_txn_stat_fields32)==sizeof(db_txn_stat_fields64));
print_struct("db_txn_stat", 0, db_txn_stat_fields32, db_txn_stat_fields64, sizeof(db_txn_stat_fields32)/sizeof(db_txn_stat_fields32[0]), 0);

5
db-benchmark-test/Makefile
View File

@ -44,7 +44,7 @@ build: $(TARGETS)
build.bdb: $(TARGET_BDB) $(SCANSCAN_BDB)
build.tdb: $(TARGET_TDB) $(SCANSCAN_TDB)
check: check-default check-xfast check-x
check: check-default check-xfast check-x check-no-rolltmp
check-default: $(TARGET_TDB)
$(VGRIND) ./$(TARGET_TDB) $(QUIET) $(SUMMARIZE_CMD)
@ -57,6 +57,9 @@ check-x: $(TARGET_TDB)
check-xfast: $(TARGET_TDB)
./$(TARGET_TDB) $(QUIET) --noserial -x --valsize 1000 --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1 $(SUMMARIZE_CMD)
# A relatively fast test that detects #853 (don't log changes to a dictionary created in the same txn)
check-no-rolltmp: $(TARGET_TDB)
./$(TARGET_TDB) --env no-rolltmp.dir --singlex --nolog --check_small_rolltmp $(SUMMARIZE_CMD)
clean:
rm -f $(TARGETS)

23
db-benchmark-test/db-benchmark-test.c
View File

@ -46,6 +46,7 @@ int prelockflag = 0;
int items_per_transaction = DEFAULT_ITEMS_PER_TRANSACTION;
int items_per_iteration = DEFAULT_ITEMS_TO_INSERT_PER_ITERATION;
int singlex = 0; // Do a single transaction
int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
int do_transactions = 0;
int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
int do_abort = 0;
@ -146,7 +147,14 @@ static void benchmark_shutdown (void) {
int r;
if (do_transactions && singlex) {
system("ls -l bench.tokudb");
#if defined(TOKUDB)
struct txn_stat *s;
r = tid->txn_stat(tid, &s);
assert(r==0);
assert(s->rolltmp_raw_count < 100);
os_free(s);
//system("ls -l bench.tokudb");
#endif
r = (do_abort ? tid->abort(tid) : tid->commit(tid, 0)); assert(r==0);
}
@ -285,6 +293,7 @@ static int print_usage (const char *argv0) {
fprintf(stderr, " --compressibility C creates data that should compress by about a factor C. Default C is large. C is an float.\n");
fprintf(stderr, " --xcount N how many insertions per transaction (default=%d)\n", DEFAULT_ITEMS_PER_TRANSACTION);
fprintf(stderr, " --singlex Run the whole job as a single transaction. (Default don't run as a single transaction.)\n");
fprintf(stderr, " --check_small_rolltmp (Only valid in --singlex mode) Verify that very little data was saved in the rollback logs.\n");
fprintf(stderr, " --prelock Prelock the database.\n");
fprintf(stderr, " --prelockflag Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
fprintf(stderr, " --abort Abort the singlex after the transaction is over. (Requires --singlex.)\n");
@ -338,6 +347,8 @@ int main (int argc, const char *argv[]) {
} else if (strcmp(arg, "--singlex") == 0) {
do_transactions = 1;
singlex = 1;
} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
check_small_rolltmp = 1;
} else if (strcmp(arg, "--xcount") == 0) {
if (i+1 >= argc) return print_usage(argv[0]);
items_per_transaction = strtoll(argv[++i], 0, 10);
@ -396,6 +407,16 @@ int main (int argc, const char *argv[]) {
if (!norandom) printf("random ");
printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
}
#if !defined TOKUDB
if (check_small_rolltmp) {
fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
return print_usage(argv[0]);
}
#endif
if (check_small_rolltmp && !singlex) {
fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
return print_usage(argv[0]);
}
benchmark_setup();
gettimeofday(&t1,0);
biginsert(total_n_items, &t1);

6
include/db.h
View File

@ -240,11 +240,15 @@ struct __toku_db_txn_active {
DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
char __toku_dummy2[184]; /* Padding at the end */
};
struct txn_stat {
u_int64_t rolltmp_raw_count;
};
struct __toku_db_txn {
DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
struct __toku_db_txn_internal *i;
void* __toku_dummy0[16];
int (*txn_stat)(DB_TXN *, struct txn_stat **);
void* __toku_dummy0[15];
char __toku_dummy1[8];
void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
void* __toku_dummy2[2];

3
newbrt/log-internal.h
View File

@ -99,7 +99,8 @@ struct tokutxn {
size_t rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
char *rollentry_filename;
int rollentry_fd; // If we spill the roll_entries, we write them into this fd.
toku_off_t rollentry_filesize; // How many bytes are in the rollentry.
toku_off_t rollentry_filesize; // How many bytes are in the rollentry file (compressed)
u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
OMT open_brts; // a collection of the brts that we touched. Indexed by filenum.
};

11
newbrt/log.c
View File

@ -456,6 +456,7 @@ int toku_logger_commit (TOKUTXN txn, int nosync, void(*yield)(void*yieldv), void
}
txn->parent->newest_logentry = txn->newest_logentry;
txn->parent->rollentry_resident_bytecount += txn->rollentry_resident_bytecount;
txn->parent->rollentry_raw_count += txn->rollentry_raw_count;
txn->rollentry_resident_bytecount = 0;
}
if (txn->parent->oldest_logentry==0) {
@ -526,6 +527,7 @@ int toku_logger_txn_begin (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER
list_push(&logger->live_txns, &result->live_txns_link);
result->rollentry_resident_bytecount=0;
result->rollentry_raw_count = 0;
result->rollentry_filename = 0;
result->rollentry_fd = -1;
result->rollentry_filesize = 0;
@ -987,7 +989,7 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
}
int toku_maybe_spill_rollbacks (TOKUTXN txn) {
if (txn->rollentry_resident_bytecount>1<<20) {
if (txn->rollentry_resident_bytecount>(1<<20)) {
struct roll_entry *item;
ssize_t bufsize = txn->rollentry_resident_bytecount;
char *MALLOC_N(bufsize, buf);
@ -1120,3 +1122,10 @@ int toku_txn_find_by_xid (BRT brt, TXNID xid, TOKUTXN *txnptr) {
return r;
}
// Return the number of bytes that went into the rollback data structure (the uncompressed count if there is compression)
int
toku_logger_txn_rolltmp_raw_count(TOKUTXN txn, u_int64_t *raw_count)
{
*raw_count = txn->rollentry_raw_count;
return 0;
}

3
newbrt/log.h
View File

@ -48,6 +48,9 @@ int toku_logger_abort(TOKUTXN, void(*/*yield*/)(void*), void*/*yield_v*/);
// Closes a txn. Call after commiting or aborting.
void toku_logger_txn_close (TOKUTXN);
// Return the number of bytes that went into the rollback data structure (the uncompressed count if there is compression)
int toku_logger_txn_rolltmp_raw_count(TOKUTXN, u_int64_t *count);
int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, int /*mode*/);

1
newbrt/logformat.c
View File

@ -500,6 +500,7 @@ generate_rollbacks (void) {
fprintf(cf, " else txn->newest_logentry->next = v;\n");
fprintf(cf, " txn->newest_logentry = v;\n");
fprintf(cf, " txn->rollentry_resident_bytecount += rollback_fsize;\n");
fprintf(cf, " txn->rollentry_raw_count += rollback_fsize;\n");
fprintf(cf, " return toku_maybe_spill_rollbacks(txn);\n}\n");
});

10
src/ydb.c
View File

@ -1097,6 +1097,15 @@ static u_int32_t locked_txn_id(DB_TXN *txn) {
toku_ydb_lock(); u_int32_t r = toku_txn_id(txn); toku_ydb_unlock(); return r;
}
static int toku_txn_stat (DB_TXN *txn, struct txn_stat **txn_stat) {
XMALLOC(*txn_stat);
return toku_logger_txn_rolltmp_raw_count(txn->i->tokutxn, &(*txn_stat)->rolltmp_raw_count);
}
static int locked_txn_stat (DB_TXN *txn, struct txn_stat **txn_stat) {
toku_ydb_lock(); u_int32_t r = toku_txn_stat(txn, txn_stat); toku_ydb_unlock(); return r;
}
static int locked_txn_commit(DB_TXN *txn, u_int32_t flags) {
toku_ydb_lock(); int r = toku_txn_commit(txn, flags); toku_ydb_unlock(); return r;
}
@ -1135,6 +1144,7 @@ static int toku_txn_begin(DB_ENV *env, DB_TXN * stxn, DB_TXN ** txn, u_int32_t f
result->commit = locked_txn_commit;
result->id = locked_txn_id;
result->parent = stxn;
result->txn_stat = locked_txn_stat;
MALLOC(result->i);
if (!result->i) {
toku_free(result);

Write Preview
Loading…
Cancel
Save