Mirrors
/
mariadb
mirror of https://github.com/MariaDB/server
3
0
Fork 0
Code Releases Activity
Browse Source

db_load/gen/dump project done until tokudb.bugs issues are worked on. 

git-svn-id: file:///svn/tokudb@479 c7de825b-a66e-492c-adef-691d508d4ae1
pull/56/head
Yoni Fogel 18 years ago
parent
077516261a
commit
807296a40b
5 changed files with 459 additions and 686 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 25
      utils/Makefile
  2. 141
      utils/tokudb_common_funcs.h
  3. 477
      utils/tokudb_dump.c
  4. 124
      utils/tokudb_gen.c
  5. 378
      utils/tokudb_load.c

25
utils/Makefile
View File

@ -1,4 +1,4 @@
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -I /usr/local/Berkeleydb.4.1/include/ -ldb
CFLAGS = -std=gnu89 -W -Wall -Wno-unused -g -fPIC -ldb
LFLAGS = -l CPPFLAGS =
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
@ -10,18 +10,17 @@ UTILS= \
tokudb_dump \
#End
.PHONY: all clean test test_gen test_gen_hex test_load
.PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS)
test: test_gen test_load
test: test_gen test_load test_dump
test_gen: test_gen_hex
# SHELL=/bin/bash
BDB_LOAD=/usr/local/Berkeleydb.4.1/bin/db_load
BDB_DUMP=/usr/local/Berkeleydb.4.1/bin/db_dump
BDB=/usr/local/BerkeleyDB.4.6
BDB_DUMP=$(BDB)/bin/db_dump
BDB_LOAD=$(BDB)/bin/db_load
TEST_GEN_HEX_NUMKEYS=10000
TEST_GEN_HEX_LENGTHMIN=0
@ -41,7 +40,7 @@ test_gen_hex:
if ! diff -q $@.load_dump.temp $@.gen_sorted.temp; then echo "Test Failed!"; exit 1; fi
rm $@.*.temp
test_load: test_gen
test_load:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
@ -52,6 +51,16 @@ test_load: test_gen
$(BDB_DUMP) $@.tokudb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
test_dump:
#Generating $(TEST_GEN_HEX_NUMKEYS) keys. [$(TEST_GEN_HEX_LENGTHMIN),$(TEST_GEN_HEX_LENGTHLIMIT)) bytes + identifier overhead
echo "Generating text input > db > text"
rm -f $@.*.temp
./tokudb_gen $(TEST_GEN_HEX_FLAGS) > $@.gen.temp
$(BDB_LOAD) $@.bdb.temp < $@.gen.temp
$(BDB_DUMP) $@.bdb.temp > $@.dump.bdb.temp
./tokudb_dump $@.bdb.temp > $@.dump.tokudb.temp
if ! diff -q $@.dump.bdb.temp $@.dump.tokudb.temp; then echo "Test Failed!"; exit 1; fi
#if diff -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
clean:
rm -rf *.so *.o $(UTILS) *.temp

141
utils/tokudb_common_funcs.h
View File

@ -2,10 +2,27 @@
#define TOKUDB_COMMON_FUNCS_H
#include "tokudb_common.h"
int strtoint32 (DB_ENV* dbenv, char* progname, char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (DB_ENV* dbenv, char* progname, char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (DB_ENV* dbenv, char* progname, char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
//DB_ENV->err disabled since it does not use db_strerror
#define ERROR(retval, ...) \
if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
fprintf(stderr, __VA_ARGS__); \
}
//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
#define ERRORX(...) \
if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
else { \
fprintf(stderr, "%s: ", g.progname); \
fprintf(stderr, __VA_ARGS__); \
}
int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
/*
* Convert a string to an integer of type "type".
@ -20,7 +37,7 @@ int strtouint64 (DB_ENV* dbenv, char* progname, char* str, uint64_t* num, uint
*
*/
#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max, int base) \
int name(char* str, type* num, type min, type max, int base) \
{ \
char* test; \
bigtype value; \
@ -28,31 +45,26 @@ int name(DB_ENV* dbenv, char* progname, char* str, type* num, type min, type max
assert(str); \
assert(num); \
assert(min <= max); \
assert(dbenv || progname); \
assert(g.dbenv || g.progname); \
assert(base == 0 || (base >= 2 && base <= 36)); \
\
errno = 0; \
while (isspace(*str)) str++; \
value = strtofunc(str, &test, base); \
if ((*test != '\0' && *test != '\n') || test == str) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Invalid numeric argument\n", progname, str); \
else dbenv->err(dbenv, 0, "%s: Invalid numeric argument", str); \
ERRORX("%s: Invalid numeric argument\n", str); \
errno = EINVAL; \
goto error; \
} \
if (errno != 0) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: %s\n", progname, str, strerror(errno)); \
else dbenv->err(dbenv, errno, "%s", str); \
goto error; \
ERROR(errno, "%s\n", str); \
} \
if (value < min) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Less than minimum value (%" frmt ")\n", progname, str, min); \
else dbenv->err(dbenv, 0, "%s: Less than minimum value (%" frmt ")", str, min); \
ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
goto error; \
} \
if (value > max) { \
if (dbenv == NULL) fprintf(stderr, "%s: %s: Greater than maximum value (%" frmt ")\n", progname, str, max); \
else dbenv->err(dbenv, 0, "%s: Greater than maximum value (%" frmt ")", str, max); \
ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
goto error; \
} \
*num = value; \
@ -76,4 +88,103 @@ void outputbyte(uint8_t ch)
else printf("%02x", ch);
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void outputplaintextstring(char* str)
{
bool old_plaintext = g.plaintext;
g.plaintext = true;
outputstring(str);
g.plaintext = old_plaintext;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
ERROR(errno, "printabletocstring");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
ERROR(0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
ERROR(0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
ERROR(0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
ERROR(0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int verify_library_version()
{
int major;
int minor;
db_version(&major, &minor, NULL);
if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
ERRORX("version %d.%d doesn't match library version %d.%d\n",
DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
#endif /* #if !defined(TOKUDB_COMMON_H) */

477
utils/tokudb_dump.c
View File

@ -9,69 +9,39 @@
#include <db.h>
#include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct {
bool leadingspace;
bool plaintext;
bool header;
bool keys;
bool footer;
bool is_private;
bool file_error;
char* progname;
char* homedir;
char* database;
char* subdatabase;
char** config_options;
int32_t version;
int exitcode;
int recover_flags;
DBTYPE dbtype;
DBTYPE opened_dbtype;
DB* db;
DB_ENV* dbenv;
struct {
char* data[2];
} dump_dbt;
struct {
char* data;
} dump_header;
} dump_globals;
dump_globals g;
#include "tokudb_common_funcs.h"
int usage ();
int longusage ();
int dump_database ();
int create_init_env();
int dump_header ();
int dump_footer ();
int dump_database ();
int open_database ();
int dump_pairs ();
int apply_commandline_options();
int dump_pairs ();
int dump_footer ();
int dump_header ();
int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) {
int ch;
int retval;
struct option options[] = {
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option;
/* Set up the globals. */
memset(&g, 0, sizeof(g));
@ -79,32 +49,57 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN;
g.progname = argv[0];
g.header = true;
g.footer = true;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*));
if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno));
goto error;
}
if (verify_library_version() != 0) goto error;
while ((ch = getopt_long_only(argc, argv, "HTVc:f:h:p:t:", options, NULL)) != EOF) {
while ((ch = getopt(argc, argv, "klNpRrVd:f:h:P:s:")) != EOF) {
switch (ch) {
case ('H'): {
g.exitcode = longusage();
goto cleanup;
case ('k'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('l'): {
//TODO: Implement (Requires master database support)
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('N'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('T'): {
g.plaintext = true;
g.leadingspace = false;
g.header = false;
g.footer = false;
break;
}
case ('V'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
case ('p'): {
g.plaintext = true;
break;
}
case ('R'): {
g.recover_flags |= DB_SALVAGE | DB_AGGRESSIVE;
//TODO: Implement aggressive recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('c'): {
*next_config_option++ = optarg;
break;
case ('r'): {
g.recover_flags |= DB_SALVAGE;
//TODO: Implement recovery (requires db->verify())
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
goto cleanup;
}
case ('d'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('f'): {
if (freopen(optarg, "w", stdout) == NULL) {
@ -119,10 +114,14 @@ int main(int argc, char *argv[]) {
g.homedir = optarg;
break;
}
case ('p'): {
case ('P'): {
/* Clear password. */
memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('s'): {
g.subdatabase = optarg;
goto error;
}
case ('?'):
@ -134,6 +133,18 @@ int main(int argc, char *argv[]) {
}
argc -= optind;
argv += optind;
if (g.plaintext) g.recover_flags |= DB_PRINTABLE;
if (g.subdatabase != NULL && IS_SET_ALL(g.recover_flags, DB_SALVAGE)) {
if (IS_SET_ALL(g.recover_flags, DB_AGGRESSIVE)) {
ERRORX("The -s and -R options may not both be specified.\n");
goto error;
}
ERRORX("The -s and -r options may not both be specified.\n");
goto error;
}
if (argc != 1) {
g.exitcode = usage();
@ -143,7 +154,6 @@ int main(int argc, char *argv[]) {
g.database = argv[0];
if (create_init_env() != 0) goto error;
//TODO: Figure out how to dump multiple databases at once.
if (dump_database() != 0) goto error;
if (false) {
error:
@ -153,52 +163,36 @@ error:
cleanup:
if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) {
g.exitcode = EXIT_FAILURE;
fprintf(stderr, "%s: dbenv->close: %s\n", g.progname, db_strerror(retval));
fprintf(stderr, "%s: %s: dbenv->close\n", g.progname, db_strerror(retval));
}
//TODO: /* Resend any caught signal. */
if (g.config_options) free(g.config_options);
if (g.subdatabase) free(g.subdatabase);
// if (g.read_header.data) free(g.read_header.data);
// if (g.get_dbt.data[0]) free(g.get_dbt.data[0]);
// if (g.get_dbt.data[1]) free(g.get_dbt.data[1]);
return g.exitcode;
}
int dump_database()
{
DB_ENV* dbenv = g.dbenv;
int retval;
/* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) {
dbenv->err(dbenv, retval, "db_create");
ERROR(retval, "db_create");
return EXIT_FAILURE;
}
if (apply_commandline_options() != 0) goto error;
if (g.file_error) goto cleanup;
/*
//TODO: Uncomment.. implement get_type, and verify the internal type is btree.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
*/
/*
TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT");
ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error;
}
*/
//TODO: dump_header()
if (open_database() != 0) goto error;
if (g.header && dump_header() != 0) goto error;
if (dump_pairs() != 0) goto error;
//TODO: dump_footer()
if (g.file_error) goto cleanup;
if (g.footer && dump_footer() != 0) goto error;
if (false) {
error:
@ -213,20 +207,10 @@ cleanup:
int usage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
printf
(
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
fprintf(stderr,
"usage: %s [-klNprRV] [-f output_file] [-h home]\n"
" [-s subdatabase] db_file\n",
g.progname);
return EXIT_FAILURE;
}
@ -264,20 +248,20 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize");
ERROR(retval, "DB_ENV->set_cachesize");
goto error;
}
*/
g.is_private = true;
//TODO: Do we want to support transactions/logging even in single-process mode?
//Maybe if the db already exists.
//If db does not exist.. makes sense not to log or have transactions
//TODO: Do we want to support transactions even in single-process mode?
//Logging is not necessary.. this is read-only.
//However, do we need to use DB_INIT_LOG to join a logging environment?
REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN);
SET_BITS(flags, DB_CREATE | DB_PRIVATE);
retval = dbenv->open(dbenv, g.homedir, flags, 0);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open");
ERROR(retval, "DB_ENV->open");
goto error;
}
success:
@ -288,219 +272,52 @@ error:
return EXIT_FAILURE;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
#define DUMP_FLAG(bit, dump) if (IS_SET_ALL(flags, bit)) printf(dump);
#define DUMP_IGNORED_FLAG(bit, dump)
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
dbenv->err(dbenv, ENOMEM, "");
int dump_header()
{
u_int32_t flags;
int retval;
DB* db = g.db;
assert(g.header);
printf("VERSION=3\n");
printf("format=%s\n", g.plaintext ? "print" : "bytevalue");
assert(g.dbtype == DB_BTREE || (g.dbtype == DB_UNKNOWN && g.opened_dbtype == DB_BTREE));
printf("type=btree\n");
//TODO: Get page size from db. Currently tokudb does not support db->get_pagesize.
printf("db_pagesize=4096\n");
if (g.subdatabase) {
printf("subdatabase=");
outputplaintextstring(g.subdatabase);
printf("\n");
}
if ((retval = db->get_flags(db, &flags)) != 0) {
ERROR(retval, "DB->get_flags");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
DUMP_IGNORED_FLAG(DB_CHKSUM, "chksum=1");
DUMP_FLAG( DB_DUP, "duplicates=1");
DUMP_IGNORED_FLAG(DB_DUPSORT, "dupsort=1");
DUMP_IGNORED_FLAG(DB_RECNUM, "recnum=1");
printf("HEADER=END\n");
if (ferror(stdout)) goto error;
return EXIT_SUCCESS;
error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \
continue; \
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \
"set_flags: %s", \
field); \
goto error; \
} \
continue; \
}
#define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
goto error; \
}
#define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \
"%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
if ((retval = dbfunction(db, value[0])) != 0) { \
goto printerror; \
} \
continue; \
}
int dump_header()
{
//TODO: All of this.. have not started.
return 0;
}
int apply_commandline_options()
int dump_footer()
{
char** next_config_option = g.config_options;
unsigned index;
char* field;
char* value = NULL;
bool first;
int ch;
int32_t num;
int retval;
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
assert(g.header);
for (index = 0; g.config_options[index]; index++) {
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
field = g.config_options[index];
printf("DATA=END\n");
if (ferror(stdout)) goto error;
if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
value[0] = '\0';
value++;
if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
goto error;
}
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) {
free(g.subdatabase);
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
goto error;
}
continue;
}
/*
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
if (value) {
/* Restore the field=value format. */
value[-1] = '=';
value = NULL;
}
return EXIT_SUCCESS;
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
}
error:
return EXIT_FAILURE;
}
@ -508,77 +325,33 @@ error:
int open_database()
{
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
int open_flags = DB_RDONLY;
//TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database);
ERROR(retval, "DB->open: %s", g.database);
goto error;
}
//TODO: Ensure we have enough cache to store some min number of btree pages.
//NOTE: This may require closing db, environment, and creating new ones.
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
retval = db->get_type(db, &g.opened_dbtype);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database);
ERROR(retval, "DB->get_type");
goto error;
}
assert(g.dbtype == DB_BTREE);
if (existingtype != g.dbtype) {
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
if (g.opened_dbtype != DB_BTREE) {
ERRORX("Unsupported db type %d\n", g.opened_dbtype);
goto error;
}
*/
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size)
{
DB_ENV* dbenv = g.dbenv;
assert(pmem);
assert(size);
assert(IS_POWER_OF_2(*size));
*size <<= 1;
if (*size == 0) {
/* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "doublechararray\n");
goto error;
}
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, "");
if (g.dbtype != DB_UNKNOWN && g.opened_dbtype != g.dbtype) {
ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, g.opened_dbtype);
goto error;
}
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
@ -615,15 +388,14 @@ int dump_pairs()
DBT key;
DBT data;
int spacech;
DB* db = g.db;///////////////
DB_ENV* dbenv = g.dbenv;
DB* db = g.db;
DBC* dbc = NULL;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
if ((retval = db->cursor(db, (DB_TXN*)NULL, &dbc, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->cursor");
ERROR(retval, "DB->cursor");
goto error;
}
while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) {
@ -631,7 +403,7 @@ int dump_pairs()
if (dump_dbt(&data) != 0) goto error;
}
if (retval != DB_NOTFOUND) {
dbenv->err(dbenv, retval, "DBC->c_get");
ERROR(retval, "DBC->c_get");
goto error;
}
@ -642,7 +414,7 @@ error:
}
cleanup:
if (dbc && (retval = dbc->c_close(dbc)) != 0) {
dbenv->err(dbenv, retval, "DBC->c_close");
ERROR(retval, "DBC->c_close");
g.exitcode = EXIT_FAILURE;
}
success:
@ -652,12 +424,11 @@ success:
int close_database()
{
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
assert(db);
if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close");
ERROR(retval, "DB->close");
goto error;
}
return EXIT_SUCCESS;

124
utils/tokudb_gen.c
View File

@ -5,17 +5,15 @@
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <db.h>
#include "tokudb_common.h"
extern char* optarg;
extern int optind;
extern int optopt;
extern int opterr;
extern int optreset;
typedef struct {
DB_ENV* dbenv;
bool plaintext;
char* progname;
} gen_globals;
gen_globals g;
@ -29,8 +27,6 @@ int get_delimiter(char* str);
char dbt_delimiter = '\n';
char sort_delimiter[2];
char* progname;
uint32_t lengthmin = 0;
bool set_lengthmin = false;
uint32_t lengthlimit = 0;
@ -54,10 +50,13 @@ int main (int argc, char *argv[]) {
/* Set up the globals. */
memset(&g, 0, sizeof(g));
progname = argv[0];
g.progname = argv[0];
if (verify_library_version() != 0) goto error;
strcpy(sort_delimiter, "");
while ((ch = getopt(argc, argv, "PfFhHTpr:s:d:p:m:M:n:o:")) != EOF) {
while ((ch = getopt(argc, argv, "PfFhHTpVr:s:d:m:M:n:o:")) != EOF) {
switch (ch) {
case ('P'): {
printableonly = true;
@ -80,22 +79,20 @@ int main (int argc, char *argv[]) {
break;
}
case ('T'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = false;
header = false;
footer = false;
break;
}
case ('p'): {
g.plaintext = true;
g.plaintext = true;
leadingspace = true;
break;
}
case ('o'): {
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr,
"%s: %s: reopen: %s\n",
progname, optarg, strerror(errno));
ERROR(errno, "%s: reopen\n", optarg);
goto error;
}
break;
@ -103,15 +100,13 @@ int main (int argc, char *argv[]) {
case ('d'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-d) Key (or value) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-d) Key (or value) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-d) Key (or value) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-d) Key (or value) delimiter cannot be a hex digit.",
temp);
goto error;
}
dbt_delimiter = (char)temp;
@ -120,15 +115,13 @@ int main (int argc, char *argv[]) {
case ('s'): {
int temp = get_delimiter(optarg);
if (temp == EOF) {
fprintf(stderr,
"%s: %s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
progname, optarg);
ERRORX("%s: (-s) Sorting (Between key/value pairs) delimiter must be one character.",
optarg);
goto error;
}
if (isxdigit(temp)) {
fprintf(stderr,
"%s: %c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
progname, temp);
ERRORX("%c: (-s) Sorting (Between key/value pairs) delimiter cannot be a hex digit.",
temp);
goto error;
}
sort_delimiter[0] = (char)temp;
@ -136,45 +129,41 @@ int main (int argc, char *argv[]) {
break;
}
case ('r'): {
if (strtouint32(NULL, progname, optarg, &seed, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-r) Random seed invalid.",
progname, optarg);
if (strtouint32(optarg, &seed, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-r) Random seed invalid.", optarg);
goto error;
}
set_seed = true;
break;
}
case ('m'): {
if (strtouint32(NULL, progname, optarg, &lengthmin, 0, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-m) Min length of keys/values invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthmin, 0, UINT32_MAX, 10)) {
ERRORX("%s: (-m) Min length of keys/values invalid.", optarg);
goto error;
}
set_lengthmin = true;
break;
}
case ('M'): {
if (strtouint32(NULL, progname, optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-M) Limit of key/value length invalid.",
progname, optarg);
if (strtouint32(optarg, &lengthlimit, 1, UINT32_MAX, 10)) {
ERRORX("%s: (-M) Limit of key/value length invalid.", optarg);
goto error;
}
set_lengthlimit = true;
break;
}
case ('n'): {
if (strtouint64(NULL, progname, optarg, &numkeys, 0, UINT64_MAX, 10)) {
fprintf(stderr,
"%s: %s: (-n) Number of keys to generate invalid.",
progname, optarg);
if (strtouint64(optarg, &numkeys, 0, UINT64_MAX, 10)) {
ERRORX("%s: (-n) Number of keys to generate invalid.", optarg);
goto error;
}
set_numkeys = true;
break;
}
case ('V'): {
printf("%s\n", db_version(NULL, NULL, NULL));
return EXIT_SUCCESS;
}
case ('?'):
default: {
return (usage());
@ -185,61 +174,55 @@ int main (int argc, char *argv[]) {
argv += optind;
if (justheader && !header) {
fprintf(stderr,
"%s: The -h and -H options may not both be specified.\n",
progname);
ERRORX("The -h and -H options may not both be specified.\n");
goto error;
}
if (justfooter && !footer) {
fprintf(stderr,
"%s: The -f and -F options may not both be specified.\n",
progname);
ERRORX("The -f and -F options may not both be specified.\n");
goto error;
}
if (justfooter && justheader) {
fprintf(stderr,
"%s: The -H and -F options may not both be specified.\n",
progname);
ERRORX("The -H and -F options may not both be specified.\n");
goto error;
}
if (justfooter && header) {
fprintf(stderr, "%s: -F implies -h\n", progname);
ERRORX("-F implies -h\n");
header = false;
}
if (justheader && footer) {
fprintf(stderr, "%s: -H implies -f\n", progname);
ERRORX("-H implies -f\n");
footer = false;
}
if (!leadingspace) {
if (footer) {
fprintf(stderr, "%s: -p implies -f\n", progname);
ERRORX("-p implies -f\n");
footer = false;
}
if (header) {
fprintf(stderr, "%s: -p implies -h\n", progname);
ERRORX("-p implies -h\n");
header = false;
}
}
if (justfooter || justheader) outputkeys = false;
else if (!set_numkeys)
{
fprintf(stderr, "%s: The -n option is required.\n", progname);
ERRORX("The -n option is required.\n");
goto error;
}
if (outputkeys && !set_seed) {
fprintf(stderr, "%s: Using default seed. (-r 1).\n", progname);
ERRORX("Using default seed. (-r 1).\n");
seed = 1;
}
if (outputkeys && !set_lengthmin) {
fprintf(stderr, "%s: Using default lengthmin. (-m 0).\n", progname);
ERRORX("Using default lengthmin. (-m 0).\n");
lengthmin = 0;
}
if (outputkeys && !set_lengthlimit) {
fprintf(stderr, "%s: Using default lengthlimit. (-M 1024).\n", progname);
ERRORX("Using default lengthlimit. (-M 1024).\n");
lengthlimit = 1024;
}
if (outputkeys && lengthmin >= lengthlimit) {
fprintf(stderr, "%s: Max key size must be greater than min key size.\n", progname);
ERRORX("Max key size must be greater than min key size.\n");
goto error;
}
@ -256,7 +239,7 @@ int main (int argc, char *argv[]) {
}
if (outputkeys) generate_keys();
if (footer) printf("DATA=END\n");
return 0;
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
@ -266,10 +249,10 @@ error:
int usage()
{
fprintf(stderr,
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" [-m lengthmin] [-M lengthlimit] [-r random seed]\n"
" [-o filename] -n numkeys\n",
progname);
"usage: %s [-PfFhHTpV] [-r random seed] [-s delimiter] \n"
" [-d delimiter] [-m lengthmin] [-M lengthlimit] \n"
" -n numkeys [-o output_file] \n",
g.progname);
return EXIT_FAILURE;
}
@ -296,15 +279,6 @@ int32_t random_below(int32_t limit)
return random() % limit;
}
void outputstring(char* str)
{
char* p;
for (p = str; *p != '\0'; p++) {
outputbyte((uint8_t)*p);
}
}
void generate_keys()
{
bool usedemptykey = false;

378
utils/tokudb_load.c
View File

@ -9,14 +9,6 @@
#include <db.h>
#include "tokudb_common.h"
typedef struct {
char* data[2];
} gdbt;
typedef struct {
char* data;
} rhdr;
typedef struct {
bool leadingspace;
bool plaintext;
@ -56,24 +48,11 @@ int open_database ();
int read_keys ();
int apply_commandline_options();
int close_database ();
int hextoint (int ch);
int doublechararray(char** pmem, uint64_t* size);
int main(int argc, char *argv[]) {
int ch;
int retval;
struct option options[] = {
{ "no_overwrite", no_argument, NULL, 'n' },
{ "help", no_argument, NULL, 'H' },
{ "plain_text", no_argument, NULL, 'T' },
{ "Version", no_argument, NULL, 'V' },
{ "config", required_argument, NULL, 'c' },
{ "input_file", required_argument, NULL, 'f' },
{ "home", required_argument, NULL, 'h' },
{ "password", required_argument, NULL, 'p' },
{ "type", required_argument, NULL, 't' },
{ NULL, 0, NULL, 0 }
};
char** next_config_option;
/* Set up the globals. */
@ -83,24 +62,21 @@ int main(int argc, char *argv[]) {
g.dbtype = DB_UNKNOWN;
g.progname = argv[0];
g.header = true;
if (verify_library_version() != 0) goto error;
next_config_option = g.config_options = (char**) calloc(argc, sizeof(char*));
if (next_config_option == NULL) {
fprintf(stderr, "%s: %s\n", g.progname, strerror(errno));
ERROR(errno, "main: calloc\n");
goto error;
}
while ((ch = getopt_long_only(argc, argv, "nHTVc:f:h:p:t:", options, NULL)) != EOF) {
while ((ch = getopt(argc, argv, "nTVc:f:h:p:t:r:")) != EOF) {
switch (ch) {
case ('n'): {
/* g.overwritekeys = false; */
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('H'): {
g.exitcode = longusage();
goto cleanup;
}
case ('T'): {
g.plaintext = true;
g.leadingspace = false;
@ -108,8 +84,8 @@ int main(int argc, char *argv[]) {
break;
}
case ('V'): {
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
goto error;
printf("%s\n", db_version(NULL, NULL, NULL));
goto cleanup;
}
case ('c'): {
*next_config_option++ = optarg;
@ -131,7 +107,7 @@ int main(int argc, char *argv[]) {
case ('p'): {
/* Clear password. */
memset(optarg, 0, strlen(optarg));
fprintf(stderr, "%s: -%c option not supported.\n", g.progname, ch);
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('t'): {
@ -146,6 +122,10 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "%s: Unrecognized db type %s.\n", g.progname, optarg);
goto error;
}
case ('r'): {
ERRORX("-%c option not supported.\n", ch);
goto error;
}
case ('?'):
default: {
g.exitcode = usage();
@ -195,7 +175,7 @@ int load_database()
/* Create a database handle. */
retval = db_create(&g.db, g.dbenv, 0);
if (retval != 0) {
dbenv->err(dbenv, retval, "db_create");
ERROR(retval, "db_create");
return EXIT_FAILURE;
}
@ -204,15 +184,10 @@ int load_database()
if (apply_commandline_options() != 0) goto error;
if (g.eof) goto cleanup;
//TODO: Only quit out if DB does NOT EXIST.
if (g.dbtype == DB_UNKNOWN) {
dbenv->err(dbenv, 0, "no database type specified");
goto error;
}
/*
TODO: If/when supporting encryption
if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
dbenv->err(dbenv, ret, "DB->set_flags: DB_ENCRYPT");
ERROR(ret, "DB->set_flags: DB_ENCRYPT");
goto error;
}
*/
@ -234,20 +209,10 @@ cleanup:
int usage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
printf
(
"usage: %s [-ThHfF] [-d delimiter] [-s delimiter]\n"
" -m minsize -M maxsize [-r random seed]\n"
" (-n maxnumkeys | -N maxkibibytes) [-o filename]\n",
g.progname
);
return EXIT_FAILURE;
}
int longusage()
{
printf("TODO: Implement %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);
fprintf(stderr,
"usage: %s [-nTV] [-c name=value] [-f input_file]\n"
" [-h home] [-t btree] db_file\n",
g.progname);
return EXIT_FAILURE;
}
@ -269,14 +234,13 @@ int create_init_env()
/*
TODO: If/when supporting encryption
if (g.password && (retval = dbenv->set_encrypt(dbenv, g.password, DB_ENCRYPT_AES))) {
dbenv->err(dbenv, retval, "set_passwd");
ERROR(retval, "set_passwd");
goto error;
}
*/
/* Open the dbenvironment. */
g.is_private = false;
// flags = DB_JOINENV | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON;
flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON;
//TODO: Transactions.. SET_BITS(flags, DB_INIT_TXN);
@ -289,7 +253,7 @@ int create_init_env()
///TODO: UNCOMMENT/IMPLEMENT
retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->set_cachesize");
ERROR(retval, "DB_ENV->set_cachesize");
goto error;
}
*/
@ -302,7 +266,7 @@ int create_init_env()
retval = dbenv->open(dbenv, g.homedir ? g.homedir : ".", flags, 0);
if (retval) {
dbenv->err(dbenv, retval, "DB_ENV->open");
ERROR(retval, "DB_ENV->open");
goto error;
}
success:
@ -313,90 +277,34 @@ error:
return EXIT_FAILURE;
}
int printabletocstring(char* inputstr, char** poutputstr)
{
char highch;
char lowch;
char nextch;
char* cstring;
DB_ENV* dbenv = g.dbenv;
assert(inputstr);
assert(poutputstr);
assert(*poutputstr == NULL);
cstring = (char*)malloc((strlen(inputstr) + 1) * sizeof(char));
if (cstring == NULL) {
dbenv->err(dbenv, ENOMEM, "");
goto error;
}
for (*poutputstr = cstring; *inputstr != '\0'; inputstr++) {
if (*inputstr == '\\') {
if ((highch = *++inputstr) == '\\') {
*cstring++ = '\\';
continue;
}
if (highch == '\0' || (lowch = *++inputstr) == '\0') {
dbenv->err(dbenv, 0, "unexpected end of input data or key/data pair");
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Unexpected '%c' (non-hex) input.\n", lowch);
goto error;
}
nextch = (hextoint(highch) << 4) | hextoint(lowch);
if (nextch == '\0') {
/* Database names are c strings, and cannot have extra NULL terminators. */
dbenv->err(dbenv, 0, "Unexpected '\\00' in input.\n");
goto error;
}
*cstring++ = nextch;
}
else *cstring++ = *inputstr;
}
/* Terminate the string. */
*cstring = '\0';
return EXIT_SUCCESS;
error:
dbenv->err(dbenv, 0, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_NUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
/*if ((retval = dbfunction(db, num)) != 0) goto printerror;*/ \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
if ((retval = dbfunction(db, num)) != 0) goto printerror; \
continue; \
}
///TODO: IMPLEMENT/Replace original line.
#define PARSE_UNSUPPORTEDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 1, INT32_MAX, 10)) goto error; \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_IGNOREDNUMBER(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 1, INT32_MAX, 10)) goto error; \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_FLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
if ((retval = db->set_flags(db, flag)) != 0) { \
dbenv->err(dbenv, retval, \
"set_flags: %s", \
field); \
ERROR(retval, "set_flags: %s", field); \
goto error; \
} \
continue; \
@ -404,22 +312,31 @@ if (!strcmp(field, match)) { \
#define PARSE_UNSUPPORTEDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(dbenv, NULL, value, &num, 0, 1, 10)) { \
dbenv->err(dbenv, 0, \
"%s: boolean name=value pairs require a value of 0 or 1", \
field); \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
dbenv->err(dbenv, 0, "%s option not supported.\n", field); \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_IGNOREDFLAG(match, flag) \
if (!strcmp(field, match)) { \
if (strtoint32(value, &num, 0, 1, 10)) { \
ERRORX("%s: boolean name=value pairs require a value of 0 or 1", \
field); \
goto error; \
} \
ERRORX("%s option not supported yet (ignored).\n", field); \
continue; \
}
#define PARSE_CHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
dbenv->err(dbenv, 0, \
"%s=%s: Expected 1-byte value", \
field, value); \
ERRORX("%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
if ((retval = dbfunction(db, value[0])) != 0) { \
@ -428,6 +345,34 @@ if (!strcmp(field, match)) { \
continue; \
}
#define PARSE_UNSUPPORTEDCHAR(match, dbfunction) \
if (!strcmp(field, match)) { \
if (strlen(value) != 1) { \
ERRORX("%s=%s: Expected 1-byte value", \
field, value); \
goto error; \
} \
ERRORX("%s option not supported.\n", field); \
goto error; \
}
#define PARSE_COMMON_CONFIGURATIONS() \
PARSE_IGNOREDNUMBER( "bt_minkey", db->set_bt_minkey); \
PARSE_IGNOREDFLAG( "chksum", DB_CHKSUM); \
PARSE_IGNOREDNUMBER( "db_lorder", db->set_lorder); \
PARSE_IGNOREDNUMBER( "db_pagesize", db->set_pagesize); \
PARSE_FLAG( "duplicates", DB_DUP); \
PARSE_IGNOREDFLAG( "dupsort", DB_DUPSORT); \
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize); \
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor); \
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem); \
PARSE_UNSUPPORTEDNUMBER("re_len", db->set_re_len); \
PARSE_UNSUPPORTEDCHAR( "re_pad", db->set_re_pad); \
PARSE_IGNOREDFLAG( "recnum", DB_RECNUM); \
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
int read_header()
{
static uint64_t datasize = 1 << 10;
@ -443,7 +388,7 @@ int read_header()
assert(g.header);
if (g.read_header.data == NULL && (g.read_header.data = (char*)malloc(datasize * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "read_header: malloc");
goto error;
}
while (!g.eof) {
@ -453,7 +398,6 @@ int read_header()
while (true) {
if ((ch = getchar()) == EOF) {
g.eof = true;
//TODO: Check ferror for every EOF test.
if (ferror(stdin)) goto formaterror;
break;
}
@ -477,9 +421,9 @@ int read_header()
if (!strcmp(field, "HEADER")) break;
if (!strcmp(field, "VERSION")) {
if (strtoint32(dbenv, NULL, value, &g.version, 1, INT32_MAX, 10)) goto error;
if (strtoint32(value, &g.version, 1, INT32_MAX, 10)) goto error;
if (g.version != 3) {
dbenv->err(dbenv, 0, "line %lu: VERSION %d is unsupported", g.linenumber, g.version);
ERRORX("line %llu: VERSION %d is unsupported", g.linenumber, g.version);
goto error;
}
continue;
@ -501,54 +445,41 @@ int read_header()
continue;
}
if (!strcmp(value, "hash") || strcmp(value, "recno") || strcmp(value, "queue")) {
dbenv->err(dbenv, 0, "db type %s not supported.\n", value);
ERRORX("db type %s not supported.\n", value);
goto error;
}
dbenv->err(dbenv, 0, "line %lu: unknown type %s", g.linenumber, value);
ERRORX("line %llu: unknown type %s", g.linenumber, value);
goto error;
}
if (!strcmp(field, "database") || !strcmp(field, "subdatabase")) {
if (g.subdatabase != NULL) {
//TODO: Free before quitting main., clear at start too.. could be a new db without a name?
free(g.subdatabase);
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
ERROR(retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
if (strtoint32(value, &temp, 0, 1, 10)) {
ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
ERRORX("keys=0 not supported");
goto error;
}
continue;
}
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
PARSE_COMMON_CONFIGURATIONS();
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
success:
@ -556,11 +487,11 @@ success:
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
ERROR(retval, "%s=%s", field, value);
}
if (false) {
formaterror:
dbenv->err(dbenv, 0, "line %lu: unexpected format", g.linenumber);
ERRORX("line %llu: unexpected format", g.linenumber);
}
error:
return EXIT_FAILURE;
@ -590,14 +521,14 @@ int apply_commandline_options()
field = g.config_options[index];
if ((value = strchr(field, '=')) == NULL) {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
ERRORX("command-line configuration uses name=value format");
goto error;
}
value[0] = '\0';
value++;
if (field[0] == '\0' || value[0] == '\0') {
dbenv->err(dbenv, 0, "command-line configuration uses name=value format");
ERRORX("command-line configuration uses name=value format");
goto error;
}
@ -607,44 +538,29 @@ int apply_commandline_options()
g.subdatabase = NULL;
}
if ((retval = printabletocstring(value, &g.subdatabase))) {
dbenv->err(dbenv, retval, "error reading db name");
ERROR(retval, "error reading db name");
goto error;
}
continue;
}
if (!strcmp(field, "keys")) {
int32_t temp;
if (strtoint32(dbenv, NULL, value, &temp, 0, 1, 10)) {
dbenv->err(dbenv, 0,
if (strtoint32(value, &temp, 0, 1, 10)) {
ERROR(0,
"%s: boolean name=value pairs require a value of 0 or 1",
field);
goto error;
}
g.keys = temp;
if (!g.keys) {
dbenv->err(dbenv, 0, "keys=0 not supported", field);
ERRORX("keys=0 not supported");
goto error;
}
continue;
}
/*
///TODO: UNCOMMENT/IMPLEMENT
PARSE_NUMBER( "bt_minkey", db->set_bt_minkey);
PARSE_NUMBER( "db_lorder", db->set_lorder);
PARSE_NUMBER( "db_pagesize", db->set_pagesize);
PARSE_NUMBER( "re_len", db->set_re_len);
PARSE_UNSUPPORTEDNUMBER("extentsize", db->set_q_extentsize);
PARSE_UNSUPPORTEDNUMBER("h_ffactor", db->set_h_ffactor);
PARSE_UNSUPPORTEDNUMBER("h_nelem", db->set_h_nelem);
PARSE_CHAR( "re_pad", db->set_re_pad);
PARSE_FLAG( "chksum", DB_CHKSUM_SHA1);
PARSE_FLAG( "duplicates", DB_DUP);
PARSE_FLAG( "dupsort", DB_DUPSORT);
PARSE_FLAG( "recnum", DB_RECNUM);
PARSE_UNSUPPORTEDFLAG( "renumber", DB_RENUMBER);
*/
PARSE_COMMON_CONFIGURATIONS();
dbenv->err(dbenv, 0, "unknown input-file header configuration keyword \"%s\"", field);
ERRORX("unknown input-file header configuration keyword \"%s\"", field);
goto error;
}
if (value) {
@ -656,7 +572,7 @@ int apply_commandline_options()
if (false) {
printerror:
dbenv->err(dbenv, retval, "%s=%s", field, value);
ERROR(retval, "%s=%s", field, value);
}
error:
return EXIT_FAILURE;
@ -667,54 +583,46 @@ int open_database()
DB* db = g.db;
DB_ENV* dbenv = g.dbenv;
int retval;
DBTYPE opened_type;
int open_flags = DB_CREATE;
int open_flags = 0;
//TODO: Transaction auto commit stuff
//if (TXN_ON(dbenv)) SET_BITS(open_flags, DB_AUTO_COMMIT);
//TODO: First see if it exists.. THEN create it?
//Try to see if it exists first.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->open: %s", g.database);
//Does not exist and we did not specify a type.
if (g.dbtype == DB_UNKNOWN) {
ERRORX("no database type specified");
goto error;
}
SET_BITS(open_flags, DB_CREATE);
//Try creating it.
retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
if (retval != 0) {
ERROR(retval, "DB->open: %s", g.database);
goto error;
}
}
if ((retval = db->get_type(db, &opened_type)) != 0) {
ERROR(retval, "DB->get_type");
goto error;
}
//TODO: Ensure we have enough cache to store some min number of btree pages.
//NOTE: This may require closing db, environment, and creating new ones.
/*
///TODO: UNCOMMENT/IMPLEMENT
DBTYPE existingtype;
retval = db->get_type(db, &existingtype);
if (retval != 0) {
dbenv->err(dbenv, retval, "DB->get_type: %s", g.database);
if (opened_type != DB_BTREE) {
ERRORX("Unsupported db type %d\n", opened_type);
goto error;
}
assert(g.dbtype == DB_BTREE);
if (existingtype != g.dbtype) {
fprintf(stderr, "Existing database is not a dictionary (DB_BTREE).\n");
if (g.dbtype != DB_UNKNOWN && opened_type != g.dbtype) {
ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, opened_type);
goto error;
}
*/
return EXIT_SUCCESS;
error:
fprintf(stderr, "Quitting out due to errors.\n");
return EXIT_FAILURE;
}
int hextoint(int ch)
{
if (ch >= '0' && ch <= '9') {
return ch - '0';
}
if (ch >= 'a' && ch <= 'z') {
return ch - 'a' + 10;
}
if (ch >= 'A' && ch <= 'Z') {
return ch - 'A' + 10;
}
return EOF;
}
int doublechararray(char** pmem, uint64_t* size)
{
DB_ENV* dbenv = g.dbenv;
@ -726,11 +634,11 @@ int doublechararray(char** pmem, uint64_t* size)
*size <<= 1;
if (*size == 0) {
/* Overflowed uint64_t. */
dbenv->err(dbenv, 0, "Line %llu: Line too long.\n", g.linenumber);
ERRORX("Line %llu: Line too long.\n", g.linenumber);
goto error;
}
if ((*pmem = (char*)realloc(*pmem, *size)) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "doublechararray: realloc");
goto error;
}
return EXIT_SUCCESS;
@ -755,7 +663,7 @@ int get_dbt(DBT* pdbt)
which = 1 - which;
if (g.get_dbt.data[which] == NULL &&
(g.get_dbt.data[which] = (char*)malloc(datasize[which] * sizeof(char))) == NULL) {
dbenv->err(dbenv, errno, "");
ERROR(errno, "get_dbt: malloc");
goto error;
}
@ -781,22 +689,22 @@ int get_dbt(DBT* pdbt)
}
else if (highch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
else if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error;
}
lowch = getchar();
if (lowch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
else if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error;
}
@ -808,7 +716,7 @@ int get_dbt(DBT* pdbt)
nextch = firstch;
break;
}
dbenv->err(dbenv, 0, "Line %llu: Nonprintable character found.", g.linenumber);
ERRORX("Line %llu: Nonprintable character found.", g.linenumber);
goto error;
}
}
@ -835,15 +743,15 @@ int get_dbt(DBT* pdbt)
lowch = getchar();
if (lowch == EOF) {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file (2 hex digits per byte).\n", g.linenumber);
goto error;
}
if (!isxdigit(highch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, highch);
goto error;
}
if (!isxdigit(lowch)) {
dbenv->err(dbenv, 0, "Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
ERRORX("Line %llu: Unexpected '%c' (non-hex) input.\n", g.linenumber, lowch);
goto error;
}
if (index == datasize[which]) {
@ -873,7 +781,7 @@ int insert_pair(DBT* key, DBT* data)
int retval = db->put(db, NULL, key, data, g.overwritekeys ? 0 : DB_NOOVERWRITE);
if (retval != 0) {
//TODO: Check for transaction failures/etc.. retry if necessary.
dbenv->err(dbenv, retval, "DB->put");
ERROR(retval, "DB->put");
goto error;
}
return EXIT_SUCCESS;
@ -905,7 +813,7 @@ int read_keys()
g.linenumber++;
if (get_dbt(&key) != 0) goto error;
if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber);
ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error;
}
g.linenumber++;
@ -937,13 +845,13 @@ int read_keys()
}
default: {
unexpectedinput:
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading key.\n", g.linenumber);
ERRORX("Line %llu: Unexpected input while reading key.\n", g.linenumber);
goto error;
}
}
if (g.eof) {
dbenv->err(dbenv, 0, "Line %llu: Key exists but value missing.", g.linenumber);
ERRORX("Line %llu: Key exists but value missing.", g.linenumber);
goto error;
}
g.linenumber++;
@ -951,7 +859,7 @@ unexpectedinput:
switch (spacech) {
case (EOF): {
g.eof = true;
dbenv->err(dbenv, 0, "Line %llu: Unexpected end of file while reading value.\n", g.linenumber);
ERRORX("Line %llu: Unexpected end of file while reading value.\n", g.linenumber);
goto error;
}
case (' '): {
@ -960,7 +868,7 @@ unexpectedinput:
break;
}
default: {
dbenv->err(dbenv, 0, "Line %llu: Unexpected input while reading value.\n", g.linenumber);
ERRORX("Line %llu: Unexpected input while reading value.\n", g.linenumber);
goto error;
}
}
@ -980,7 +888,7 @@ int close_database()
assert(db);
if ((retval = db->close(db, 0)) != 0) {
dbenv->err(dbenv, retval, "DB->close");
ERROR(retval, "DB->close");
goto error;
}
return EXIT_SUCCESS;

Write Preview
Loading…
Cancel
Save