From 0d44815bf5a41dd47147974ade5d652de5fd77fe Mon Sep 17 00:00:00 2001 From: Rich Prohaska Date: Tue, 16 Apr 2013 23:57:27 -0400 Subject: [PATCH] merge some tokudb.1032 into 1032b. addresses #1032 git-svn-id: file:///svn/toku/tokudb.1032b@7778 c7de825b-a66e-492c-adef-691d508d4ae1 --- include/Makefile.include | 385 ++++++ include/os.h | 61 + include/portability.h | 86 ++ linux/Makefile | 21 + linux/linux.c | 165 +++ linux/os-types.h | 16 + linux/tests/Makefile | 13 + linux/tests/test-pagesize.c | 11 + linux/tests/test-wss.c | 30 + linux/tests/test-xid.c | 19 + linux/toku_pthread.c | 6 + linux/toku_pthread.h | 107 ++ newbrt/Makefile | 237 +--- newbrt/brt-serialize.c | 6 +- newbrt/cachetable-rwlock.h | 26 +- newbrt/cachetable-writequeue.h | 28 +- newbrt/cachetable.c | 19 +- newbrt/includes.h | 2 +- newbrt/leafentry.c | 92 +- newbrt/leafentry.h | 52 + newbrt/log-internal.h | 12 +- newbrt/portability.h | 122 -- newbrt/rwlock.c | 2 +- newbrt/tests/Makefile | 203 +-- newbrt/tests/bread-test.c | 2 +- newbrt/tests/cachetable-rename-test.c | 12 +- newbrt/tests/cachetable-rwlock-test.c | 58 +- newbrt/tests/cachetable-test.c | 18 +- newbrt/tests/cachetable-test2.c | 12 +- newbrt/tests/cachetable-writequeue-test.c | 44 +- newbrt/tests/threadpool-test.c | 36 +- newbrt/threadpool.c | 8 +- windows/Makefile | 24 + windows/dirent.h | 18 + windows/inttypes.h | 13 + windows/misc.h | 72 ++ windows/os-types.h | 32 + windows/rss.c | 54 + windows/stdint.h | 37 + windows/tests/Makefile | 17 + windows/tests/test-dirs.c | 81 ++ windows/tests/test-file-truncate | 0 windows/tests/test-fileid.c | 43 + windows/tests/test-ftruncate.c | 60 + windows/tests/test-open-dir.c | 16 + windows/tests/test-open-rename.c | 50 + windows/tests/test-open-unlink.c | 49 + windows/tests/test-pread.c | 39 + windows/tests/test-pthread-cond.c | 85 ++ windows/tests/test-pthread-create.c | 37 + windows/tests/test-rss.c | 30 + windows/tests/test-sleep.c | 34 + windows/tests/test-stat.c | 30 + windows/tests/test-strtoll.c | 42 + windows/tests/test-unique-fileid.c | 21 + windows/tests/test-usleep-ws.c | 44 + windows/tests/test-usleep.c | 29 + windows/toku_pthread.c | 181 +++ windows/toku_pthread.h | 87 ++ windows/unistd.h | 22 + windows/windows.c | 420 +++++++ windows/zconf.h | 332 +++++ windows/zlib.h | 1357 +++++++++++++++++++++ 63 files changed, 4614 insertions(+), 653 deletions(-) create mode 100644 include/Makefile.include create mode 100644 include/os.h create mode 100644 include/portability.h create mode 100644 linux/Makefile create mode 100644 linux/linux.c create mode 100644 linux/os-types.h create mode 100644 linux/tests/Makefile create mode 100644 linux/tests/test-pagesize.c create mode 100644 linux/tests/test-wss.c create mode 100644 linux/tests/test-xid.c create mode 100644 linux/toku_pthread.c create mode 100644 linux/toku_pthread.h delete mode 100644 newbrt/portability.h create mode 100644 windows/Makefile create mode 100644 windows/dirent.h create mode 100644 windows/inttypes.h create mode 100644 windows/misc.h create mode 100644 windows/os-types.h create mode 100644 windows/rss.c create mode 100644 windows/stdint.h create mode 100644 windows/tests/Makefile create mode 100644 windows/tests/test-dirs.c create mode 100644 windows/tests/test-file-truncate create mode 100644 windows/tests/test-fileid.c create mode 100644 windows/tests/test-ftruncate.c create mode 100644 windows/tests/test-open-dir.c create mode 100644 windows/tests/test-open-rename.c create mode 100644 windows/tests/test-open-unlink.c create mode 100644 windows/tests/test-pread.c create mode 100644 windows/tests/test-pthread-cond.c create mode 100644 windows/tests/test-pthread-create.c create mode 100644 windows/tests/test-rss.c create mode 100644 windows/tests/test-sleep.c create mode 100644 windows/tests/test-stat.c create mode 100644 windows/tests/test-strtoll.c create mode 100644 windows/tests/test-unique-fileid.c create mode 100644 windows/tests/test-usleep-ws.c create mode 100644 windows/tests/test-usleep.c create mode 100644 windows/toku_pthread.c create mode 100644 windows/toku_pthread.h create mode 100644 windows/unistd.h create mode 100644 windows/windows.c create mode 100644 windows/zconf.h create mode 100644 windows/zlib.h diff --git a/include/Makefile.include b/include/Makefile.include new file mode 100644 index 00000000000..cd4a3f857a0 --- /dev/null +++ b/include/Makefile.include @@ -0,0 +1,385 @@ +# -*- Mode: Makefile -*- + +# TODO: Find a way to point out directories (i.e. current dir, or root dir) +# TODO: Find the names of the icc windows library includes.. i.e. dlls/etc. +# TODO: Find icc equivalents for W64 (warn about 32 vs 64 bit incompatibility +# TODO: Add (linux) icc warning about windows porting issues. +# TODO: There is an icc option for short 1-liner diagnostics.. see if there is +# an opposite! (i.e. long diagnostics which maybe will give us a stack trace +# for warnings) +# TODO: Learn how the hell to use idb. +# +# Default include makefile. +# Before (or after?) including this makefile, define the following variables +# INCLUDES = -I $(TOKUROOT)/.. -I . -I $(TOKUROOT)/... #Whatever include directories +# List of directories to include for search paths. +# TOKUROOT = absolute path of the root of this branch. +# TODO: find alternative way to do this. +# BINS = base names for bins +# SUBDIRS = Subdirs to make/clean +# +# On cygwin do: +# make CYGWIN=cygwin check +# +# For default=debug (no optimization) do +# make DEBUG=1 + +# For verbose output do +# make VERBOSE=1 +# For very verbose output do +# make VERBOSE=2 + +# For CIL do +# make CIL=1 + +# For coverage do +# make GCOV=1 + +# For profiling do +# make PROF=1 + +# To make warnings be warnngs instead of errors do +# make WERROR= + +.DELETE_ON_ERROR: + +ifneq ($(GCOV),) + GCOV_FLAGS = -fprofile-arcs -ftest-coverage +endif + +ifneq ($(PROF),) + PROF_FLAGS = -pg +endif + +OPT_OPTFLAGS = -O3 -finline-functions +DBG_OPTFLAGS = -O0 + +ifeq ($(VERBOSE),2) +VERBVERBOSE=-v +MAYBEATSIGN= +else ifeq ($(VERBOSE),1) + VERBVERBOSE=-q + MAYBEATSIGN= + else + VERBVERBOSE=-q + MAYBEATSIGN=@ +endif + +CPPFLAGS = $(INCLUDEDIRS) -I$(PORTABILITY_HEADERS) -I$(TOKUROOT)include + +#CFLAG default options +# Add -Wconversion #DISABLED for now. + +WERROR = -Werror +WALL = -Wall -Wextra -Wcast-align -Wbad-function-cast +ifeq ($(SKIP_NORETURN),) +WALL += -Wmissing-noreturn +endif + +FORMAT = -Wmissing-format-attribute +VISIBILITY= -fvisibility=hidden +FPICFLAGS = -fPIC +SHADOW = -Wshadow +SYMBOLS = -g3 -ggdb3 +PORTABILITY= +SKIP_WARNING= +COMBINE_C = -combine -c +LIBPORTABILITY = $(TOKUROOT)lib/libportability.$(AEXT) + +PORTABILITY_HEADERS= $(TOKUROOT)linux +ALWAYS_LINK= $(LIBPORTABILITY) -lz -lpthread +C99 = -std=c99 +W64 = #-Wshorten-64-to-32 +BINOUTPUT = -o +OOUTPUT = -o +AROUTPUT =#empty +BINSUF=#none +WRONGBINSUF=.exe +OEXT=o +WRONGOEXT=obj +AEXT=a +WRONGAEXT=lib +SOEXT=so +WRONGSOEXT=dll +AR=ar +DBG_ARFLAGS=r +OPT_ARFLAGS=r +LINK=-l +DEPEND_COMPILE += \ + $(TOKUROOT)include/db.h \ + $(TOKUROOT)include/os.h \ + $(TOKUROOT)include/portability.h \ + $(TOKUROOT)include/rdtsc.h \ +# keep this line so I can have a \ on the previous line +CCQUIET= + +comma:=, +empty:=# +space:=$(empty) $(empty) +LINK_LPATH=$(patsubst %,-Wl$(comma)-rpath$(comma)%,$(RPATH_DIRS)) +LINK_RPATH=$(patsubst %,-L%$(space),$(RPATH_DIRS)) +DLINK_FILES_PREPROCESS_1=$(patsubst %.$(SOEXT),%,$(DLINK_FILES)) +DLINK_FILES_PREPROCESS_2=$(patsubst lib%,%,$(notdir $(DLINK_FILES_PREPROCESS_1))) +LINK_DLINK_FILES=$(patsubst %,$(LINK)%,$(notdir $(DLINK_FILES_PREPROCESS_2))) + +CRUNTIME= +DEPEND_LINK += \ + $(LIBPORTABILITY) \ + $(LINK_FILES) \ +# keep this line so I can have a \ on the previous line + + +ifeq ($(CYGWIN),) + OS_CHOICE=linux +else ifneq ($(CC),icc) + OS_CHOICE=linux +else + OS_CHOICE=windows +endif + +DEPEND_COMPILE += $(wildcard $(TOKUROOT)$(OS_CHOICE)/*.h) + +#Tools +VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=full --show-reachable=yes \ + --suppressions=$(TOKUROOT)newbrt/valgrind.suppressions \ + --suppressions=$(TOKUROOT)src/tests/bdb.suppressions +HGRIND=valgrind --quiet --tool=helgrind + +# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test +# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test + +ifeq ($(CC),icc) + #icc only: + OPT_OPTFLAGS = -O3 -ip -ipo1 + DBG_OPTFLAGS = -O0 + COMBINE_C = -ipo-c + FORMAT= #No argument for extra format warnings. + WALL = -Wall -Wcheck # '-Wextra' becomes '-Wcheck' in icc + SYMBOLS= -g -debug all -inline-debug-info + PORTABILITY=-diag-enable port-win + AR=xiar + ifneq ($(CYGWIN),) + #Cygwin + ICC_NOWARN=-Qdiag-disable: + SKIP_WARNING += $(ICC_NOWARN)869 # Don't complain about unused variables (since we defined __attribute__ to be nothing.) + else + #Linux + ICC_NOWARN=-diag-disable #Need the space + endif + SKIP_WARNING += $(ICC_NOWARN)177 # Don't complain about static variables that are not used. + #SKIP_WARNING += $(ICC_NOWARN)188 # Don't complain about enumerated type mixed with another type. + SKIP_WARNING += $(ICC_NOWARN)589 # Don't complain about goto into a block that skips initializing variables. GCC catches the actual uninitialized variables. + SKIP_WARNING += $(ICC_NOWARN)981 # Don't complain about "operands are evaluated in unspecified order". This seems to be generated whenever more than one argument to a function or operand is computed by function call. + SKIP_WARNING += $(ICC_NOWARN)1324 # Don't complain about rdtsc clobbering its registers more than once. +endif + +ifneq ($(CYGWIN),) + #Cygwin (Windows) Must override some settings + BINSUF=.exe + WRONGBINSUF=#empty + ALWAYS_LINK=$(LIBPORTABILITY) /usr/lib/libz.a + VGRIND =#No Valgrind in cygwin + HGRIND =#No Hgrind in cygwin + FPICFLAGS=#FPIC is default and not allowed as an option. + VISIBILITY=#Not supported + SHADOW=#Not supported + ifeq ($(CC),icc) + #Cygwin icc only + ifeq ($(CRUNTIME),) + ifneq ($(DEBUG),) + CRUNTIME=MDd + else + CRUNTIME=MD + endif + endif + ALWAYS_LINK=$(LIBPORTABILITY) $(TOKUROOT)windows/lib/$(CRUNTIME)/zlib.lib Ws2_32.lib psapi.lib + LINK=#Empty + BINOUTPUT=-Fe + OOUTPUT=-Fo + AROUTPUT=/out: + OEXT=obj + WRONGOEXT=o + AEXT=lib + WRONGAEXT=a + SOEXT=dll + WRONGSOEXT=so + C99 = -Qstd=c99 + OPT_OPTFLAGS = -Ox -Qip -Qipo1 + DBG_OPTFLAGS = -Od + COMBINE_C = -Qipo-c + WERROR = -WX # Windows icc version of -Werror + SYMBOLS= -Zi -debug:all -Qinline-debug-info + PORTABILITY= + SKIP_WARNING += $(ICC_NOWARN)1786 # Don't complain about 'read/write/other standards' being deprecated + PORTABILITY_HEADERS = $(TOKUROOT)windows + AR=xilib + DBG_AROPT=-qnoipo + OPT_AROPT=-qipo + DBG_ARFLAGS=$(DBG_AROPT) -lib /VERBOSE /WX /NOLOGO + OPT_ARFLAGS=$(OPT_AROPT) -lib /VERBOSE /WX /NOLOGO + CCQUIET=-nologo + LINK_DLINK_FILES=$(patsubst %.$(SOEXT),%.$(AEXT),$(DLINK_FILES)) + LINK_RPATH= + LINK_LPATH= + DEPEND_LINK += $(LINK_DLINK_FILES) + W64 = -Wport #-Wp64 + else + #Cygwin gcc only + FORMAT = -Wno-format + endif +endif + +ifneq ($(DEBUG),) + OPTFLAGS = $(DBG_OPTFLAGS) + ARFLAGS = $(DBG_ARFLAGS) +else + OPTFLAGS = $(OPT_OPTFLAGS) + ARFLAGS = $(OPT_ARFLAGS) +endif + +CFLAGS = $(WALL) $(W64) $(WERROR) $(FORMAT) $(VISIBILITY) $(FPICFLAGS) $(SHADOW) +CFLAGS += $(OPTFLAGS) $(GCOV_FLAGS) $(PROF_FLAGS) +CFLAGS += $(SYMBOLS) $(SKIP_WARNING) $(C99) $(CCQUIET) + +LDFLAGS_NOLIB = $(OPTFLAGS) $(SYMBOLS) $(GCOV_FLAGS) $(PROF_FLAGS) +LDFLAGS = $(LDFLAGS_NOLIB) $(LINK_FILES) $(ALWAYS_LINK) $(LINK_DLINK_FILES) $(LINK_RPATH) $(LINK_LPATH) +ifneq ($(CRUNTIME),) +LDFLAGS += -$(CRUNTIME) +CFLAGS += -$(CRUNTIME) +endif +# Need XOPEN_SOURCE=600 to get strtoll() +CPPFLAGS += -D_SVID_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=600 + +#TODO: Fix this up if we want to keep using CIL. +#ifeq ($(CIL),1) +# CC=../../cil/cil-1.3.6/bin/cilly --merge --keepmerged +#endif + +# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test + +ifneq ($(CYGWIN),) + CSCOPE=mlcscope +else + CSCOPE=cscope +endif + +.PHONY: tags +tags: cscope.out TAGS + +TAGS: $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] + $(MAYBEATSIGN)etags $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] + +cscope.files: $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] + $(MAYBEATSIGN)(echo $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] | tr " " "\n") > $@ # Very long command line quieted. + +cscope.out: cscope.files $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] + $(MAYBEATSIGN)$(CSCOPE) -b + +.PHONY: clean clean-default +clean: clean-default + +clean-default: + $(MAYBEATSIGN)rm -f $(BINS) *.$(AEXT) *.$(SOEXT) *.$(OEXT) + $(MAYBEATSIGN)rm -f *.bb *.bbg *.da *.gcov *.gcno *.gcda + $(MAYBEATSIGN)rm -f *.exe *.obj *.pdb *.ilk TAGS cscope.out cscope.files + +#Prevent using the wrong extensions/target types (Otherwise prereqs get confused). +%$(WRONGBINSUF): %.c + $(MAYBEATSIGN)echo "Wrong target type: $@ should be $*$(BINSUF)" && false +%$(WRONGBINSUF): %.$(WRONGOEXT) + $(MAYBEATSIGN)echo "Wrong target type: $@ should be $*$(BINSUF)" && false +%.$(WRONGOEXT): %.c + $(MAYBEATSIGN)echo "Wrong target type: $@ should be $*.$(OEXT)" && false +%.$(WRONGAEXT): + $(MAYBEATSIGN)echo "Wrong target type: $@ should be $*.$(AEXT)" && false +%.$(WRONGSOEXT): + $(MAYBEATSIGN)echo "Wrong target type: $@ should be $*.$(SOEXT)" && false + +ifeq ($(SKIP_LIBPORTABILITYRULE),) +ifeq ($(CYGWIN),) +$(LIBPORTABILITY): $(TOKUROOT)linux/*.[ch] + $(MAYBEATSIGN)cd $(TOKUROOT)linux && $(MAKE) -s install +else ifneq ($(CC),icc) +$(LIBPORTABILITY): $(TOKUROOT)linux/*.[ch] + $(MAYBEATSIGN)cd $(TOKUROOT)linux && $(MAKE) -s install +else +$(LIBPORTABILITY): $(TOKUROOT)windows/*.[ch] + $(MAYBEATSIGN)cd $(TOKUROOT)windows && $(MAKE) -s install +endif +endif + +ifeq ($(SKIP_LOCKTREERULE),) +LOCKTREE = $(TOKUROOT)src/lock_tree/locktree.$(AEXT) +LOCKTREE_LINEAR = $(TOKUROOT)src/lock_tree/locktree_linear.$(AEXT) +LOCKTREE_TLOG = $(TOKUROOT)src/lock_tree/locktree_tlog.$(AEXT) +LOCKTREE_LOG = $(TOKUROOT)src/lock_tree/locktree_log.$(AEXT) +$(LOCKTREE) $(LOCKTREE_LINEAR) $(LOCKTREE_TLOG) $(LOCKTREE_LOG): $(@D)*.[ch] + $(MAYBEATSIGN)cd $(@D) && $(MAKE) -s $(@F) +endif + +ifeq ($(SKIP_RANGETREERULE),) +RANGETREE = $(TOKUROOT)src/range_tree/rangetree.$(AEXT) +RANGETREE_LINEAR = $(TOKUROOT)src/range_tree/rangetree_linear.$(AEXT) +RANGETREE_TLOG = $(TOKUROOT)src/range_tree/rangetree_tlog.$(AEXT) +RANGETREE_LOG = $(TOKUROOT)src/range_tree/rangetree_log.$(AEXT) +$(RANGETREE) $(RANGETREE_LINEAR) $(RANGETREE_TLOG) $(RANGETREE_LOG): $(@D)*.[ch] + $(MAYBEATSIGN)cd $(@D) && $(MAKE) -s $(@F) +endif + +ifeq ($(SKIP_NEWBRTRULE),) +NEWBRT = $(TOKUROOT)newbrt/newbrt.$(AEXT) +$(NEWBRT): $(@D)*.[ch] + $(MAYBEATSIGN)cd $(@D) && $(MAKE) -s $(@F) +endif + +BIN_FROM_C_FLAGS =$(CFLAGS) $(CPPFLAGS) $(BINOUTPUT)$@ $(LDFLAGS) +BIN_FROM_C_FLAGS_NOLIB=$(CFLAGS) $(CPPFLAGS) $(BINOUTPUT)$@ $(LDFLAGS_NOLIB) +%$(BINSUF):%.c $(DEPEND_COMPILE) $(DEPEND_LINK) + $(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS) + +BIN_FROM_O_FLAGS =$(CFLAGS) $(CPPFLAGS) $(BINOUTPUT)$@ $(LDFLAGS) +BIN_FROM_O_FLAGS_NOLIB=$(CFLAGS) $(CPPFLAGS) $(BINOUTPUT)$@ $(LDFLAGS_NOLIB) +%$(BINSUF):%.$(OEXT) $(DEPEND_COMPILE) $(DEPEND_LINK) + $(MAYBEATSIGN)$(CC) $< $(BIN_FROM_O_FLAGS) + +O_FROM_C_FLAGS= $(CFLAGS) $(CPPFLAGS) -c $(OOUTPUT)$@ +%.$(OEXT):%.c $(DEPEND_COMPILE) + $(MAYBEATSIGN)$(CC) $< $(O_FROM_C_FLAGS) + +%.$(AEXT): + $(MAYBEATSIGN)$(AR) $(ARFLAGS) $(AROUTPUT)$@ $^ + + +ifeq ($(SOEXT),so) +EXPORTMAPFILE=export.map +EXPORTMAP = -Wl,--version-script=$(EXPORTMAPFILE) +SHARED=-shared +endif +ifeq ($(SOEXT),dll) + ifeq ($(DEBUG),) + SHARED=/LD + else + SHARED=/LDd + endif +EXPORTMAPFILE=export.def +EXPORTMAP=/link /def:$(EXPORTMAPFILE) +endif + +SO_FLAGS=$(SHARED) $(BIN_FROM_O_FLAGS) $(EXPORTMAP) +%.$(SOEXT): $(EXPORTMAPFILE) + $(MAYBEATSIGN)$(CC) $(SO_FLAGS) + +#Testing tools +ifeq ($(SUMMARIZE),1) +SUMMARIZE_CMD = ;if test $$? = 0; then printf "%-60sPASS\n" $(HERE)/$@; else printf "%-60sFAIL\n" $(HERE)/$@ ; test 0 = 1; fi +SUMMARIZE_SHOULD_FAIL= ;if test $$? = 0; then printf "%-60sXFAIL\n" $(HERE)/$@; else printf "%-60sXPASS\n" $(HERE)/$@ ; test 0 = 1; fi +INVERTER=;test $$? -ne 0 +else +SUMMARIZE_CMD = +endif + +#Auto change variables from raw to include .exe if necessary. +BINS = $(patsubst %,%$(BINSUF),$(BINS_RAW)) +OBJS = $(patsubst %,%.$(OEXT),$(OBJS_RAW)) +REGRESSION_TESTS = $(patsubst %,%$(BINSUF),$(REGRESSION_TESTS_RAW)) + diff --git a/include/os.h b/include/os.h new file mode 100644 index 00000000000..2f5e156b75a --- /dev/null +++ b/include/os.h @@ -0,0 +1,61 @@ +#if !defined(TOKU_OS_INTERFACE_H) +#define TOKU_OS_INTERFACE_H + +#if defined __cplusplus +extern "C" { +#endif + +#include "os-types.h" + +// Returns: the current process id +int os_getpid(void); + +// Returns: the current thread id +int os_gettid(void); + +// Returns: the number of processors in the system +int os_get_number_processors(void); + +// Returns: the number of active processors in the system +int os_get_number_active_processors(void); + +// Returns: the system page size +int os_get_pagesize(void); + +// Returns: the total number of bytes of physical memory +uint64_t os_get_phys_memory_size(void); + +// Returns: 0 on success +// sets fsize to the number of bytes in a file +int os_get_file_size(int fildes, int64_t *fsize); + +// Returns: 0 on success +// Initializes id as a unique fileid for fildes on success. +int os_get_unique_file_id(int fildes, struct fileid *id); + +//Locks a file (should not be open to begin with). +//Returns: file descriptor (or -1 on error) +int os_lock_file(char *name); + +//Unlocks and closes a file locked by os_lock_on_file +int os_unlock_file(int fildes); + +int os_mkdir(const char *pathname, mode_t mode); + +// Get the current process user and kernel use times +int os_get_process_times(struct timeval *usertime, struct timeval *kerneltime); + +// Get the current in memory size (in bytes) of the current process +int os_get_rss(int64_t *rss); + +// Get the maximum in memory size (in bytes) of the current process +int os_get_max_rss(int64_t *maxrss); + +int os_initialize_settings(int verbosity); + +#if defined __cplusplus +}; +#endif + +#endif + diff --git a/include/portability.h b/include/portability.h new file mode 100644 index 00000000000..0c4c7c43ba8 --- /dev/null +++ b/include/portability.h @@ -0,0 +1,86 @@ +#ifndef TOKU_PORTABILITY_H +#define TOKU_PORTABILITY_H + +#if defined __cplusplus +extern "C" { +#endif + +// Portability layer +#define DEV_NULL_FILE "/dev/null" + +#if defined(_MSC_VER) +// Microsoft compiler +#define TOKU_WINDOWS 1 +#endif + +#if defined(__INTEL_COMPILER) +// Intel compiler + +#if defined(__ICL) +#define TOKU_WINDOWS 1 +#endif + +#undef DEV_NULL_FILE +#define DEV_NULL_FILE "NUL" + +#endif + +#if defined(TOKU_WINDOWS) +// Windows + +// ntohl and htonl are defined in winsock.h +#include +#include +#include +#include "stdint.h" +#include "inttypes.h" +#include "toku_pthread.h" +#include "unistd.h" +#include "misc.h" + +#define UNUSED_WARNING(a) a=a /* To make up for missing attributes */ + +#if defined(__ICL) +#define __attribute__(x) /* Nothing */ +#endif + +#elif defined(__INTEL_COMPILER) + +#if defined(__ICC) +// Intel linux + +#include +#include +#include +#include +#include +#include + +#endif + +#elif defined(__GNUC__) +// GCC linux + +#include +#include +#include +#include +// Define ntohl using arpa/inet.h +#include +#include + +#else + +#error Not ICC and not GNUC. What compiler? + +#endif + +#include "os.h" + +#define UU(x) x __attribute__((__unused__)) + +#if defined __cplusplus +}; +#endif + +#endif diff --git a/linux/Makefile b/linux/Makefile new file mode 100644 index 00000000000..5aad2cffe3e --- /dev/null +++ b/linux/Makefile @@ -0,0 +1,21 @@ +# -*- Mode: Makefile -*- + +.DEFAULT_GOAL=install +TOKUROOT=../ +INCLUDEDIRS=-I$(TOKUROOT)include/linux -I$(TOKUROOT)newbrt +include $(TOKUROOT)include/Makefile.include + +OPT_AROPT=-qnoipo #Disable ipo for lib creation even when optimization is on. + +SRCS = $(wildcard *.c) +OBJS = $(patsubst %.c,%.$(OEXT),$(SRCS)) +TARGET = tokulinux.$(AEXT) + +install: $(TARGET) + $(MAYBEATSIGN)cp $(TARGET) $(LIBPORTABILITY) + +$(TARGET): $(OBJS) + +clean: + $(MAYBEATSIGN)rm -rf $(TARGET) $(LIBPORTABILITY) + diff --git a/linux/linux.c b/linux/linux.c new file mode 100644 index 00000000000..e4e2c7f192a --- /dev/null +++ b/linux/linux.c @@ -0,0 +1,165 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "portability.h" +#include "os.h" + +int +os_getpid(void) { + return getpid(); +} + +int +os_gettid(void) { + return syscall(__NR_gettid); +} + +int +os_get_number_processors(void) { + return sysconf(_SC_NPROCESSORS_CONF); +} + +int +os_get_number_active_processors(void) { + return sysconf(_SC_NPROCESSORS_ONLN); +} + +int +os_get_pagesize(void) { + return sysconf(_SC_PAGESIZE); +} + +uint64_t +os_get_phys_memory_size(void) { + uint64_t npages = sysconf(_SC_PHYS_PAGES); + uint64_t pagesize = sysconf(_SC_PAGESIZE); + return npages*pagesize; +} + +int +os_get_file_size(int fildes, int64_t *fsize) { + struct stat sbuf; + int r = fstat(fildes, &sbuf); + if (r==0) { + *fsize = sbuf.st_size; + } + return r; +} + +int +os_get_unique_file_id(int fildes, struct fileid *id) { + struct stat statbuf; + memset(id, 0, sizeof(*id)); + int r=fstat(fildes, &statbuf); + if (r==0) { + memset(id, 0, sizeof(*id)); + id->st_dev = statbuf.st_dev; + id->st_ino = statbuf.st_ino; + } + return r; +} + +int +os_lock_file(char *name) { + int r; + int fd = open(name, O_RDWR|O_CREAT, S_IRUSR | S_IWUSR); + if (fd>=0) { + r = flock(fd, LOCK_EX | LOCK_NB); + if (r!=0) { + r = errno; //Save errno from flock. + close(fd); + fd = -1; //Disable fd. + errno = r; + } + } + return fd; +} + +int +os_unlock_file(int fildes) { + int r = flock(fildes, LOCK_UN); + if (r==0) r = close(fildes); + return r; +} + +int +os_mkdir(const char *pathname, mode_t mode) { + int r = mkdir(pathname, mode); + return r; +} + +int +os_get_process_times(struct timeval *usertime, struct timeval *kerneltime) { + int r; + struct rusage rusage; + r = getrusage(RUSAGE_SELF, &rusage); + if (r == -1) + return errno; + if (usertime) + *usertime = rusage.ru_utime; + if (kerneltime) + *kerneltime = rusage.ru_stime; + return 0; +} + +int +os_initialize_settings(int UU(verbosity)) { + int r = 0; + static int initialized = 0; + assert(initialized==0); + initialized=1; + return r; +} + +int +os_get_max_rss(int64_t *maxrss) { + char statusname[100]; + sprintf(statusname, "/proc/%d/status", getpid()); + FILE *f = fopen(statusname, "r"); + if (f == NULL) + return errno; + int r = ENOENT; + char line[100]; + while (fgets(line, sizeof line, f)) { + r = sscanf(line, "VmHWM:\t%lld kB\n", (long long *) maxrss); + if (r == 1) { + *maxrss *= 1<<10; + r = 0; + break; + } + } + fclose(f); + return r; +} + +int +os_get_rss(int64_t *rss) { + char statusname[100]; + sprintf(statusname, "/proc/%d/status", getpid()); + FILE *f = fopen(statusname, "r"); + if (f == NULL) + return errno; + int r = ENOENT; + char line[100]; + while (fgets(line, sizeof line, f)) { + r = sscanf(line, "VmRSS:\t%lld kB\n", (long long *) rss); + if (r == 1) { + *rss *= 1<<10; + r = 0; + break; + } + } + fclose(f); + return r; +} diff --git a/linux/os-types.h b/linux/os-types.h new file mode 100644 index 00000000000..4d907c52346 --- /dev/null +++ b/linux/os-types.h @@ -0,0 +1,16 @@ +#if !defined(OS_INTERFACE_LINUX_H) +#define OS_INTERFACE_LINUX_H +#include + +typedef int os_handle_t; + +struct fileid { + dev_t st_dev; /* device and inode are enough to uniquely identify a file in unix. */ + ino_t st_ino; +}; + +#if !defined(O_BINARY) +#define O_BINARY 0 +#endif + +#endif diff --git a/linux/tests/Makefile b/linux/tests/Makefile new file mode 100644 index 00000000000..df7f67777a9 --- /dev/null +++ b/linux/tests/Makefile @@ -0,0 +1,13 @@ +CPPFLAGS = -I../../include +CFLAGS = -Wall -Werror -g -O0 +LDFLAGS = ../tokulinux.a +SRCS = $(wildcard test-*.c) +TARGETS = $(patsubst %.c,%,$(SRCS)) + +all: $(TARGETS) + +%: %.c + $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(LDFLAGS) + +clean: + rm -rf $(TARGETS) \ No newline at end of file diff --git a/linux/tests/test-pagesize.c b/linux/tests/test-pagesize.c new file mode 100644 index 00000000000..bc2c8ce761a --- /dev/null +++ b/linux/tests/test-pagesize.c @@ -0,0 +1,11 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include "os.h" + +int main(void) { + assert(os_get_pagesize() == getpagesize()); + return 0; +} diff --git a/linux/tests/test-wss.c b/linux/tests/test-wss.c new file mode 100644 index 00000000000..61c766a3b6b --- /dev/null +++ b/linux/tests/test-wss.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include +#include +#include + +static void do_mallocs(void) { + int i; + for (i=0; i<1000; i++) { + int nbytes = 1024*1024; + void *vp = malloc(nbytes); + memset(vp, 0, nbytes); + } +} + +int main(void) { + int64_t rss; + + os_get_max_rss(&rss); + printf("%"PRId64"\n", rss); + do_mallocs(); + os_get_max_rss(&rss); + printf("%"PRId64"\n", rss); + + return 0; +} + + diff --git a/linux/tests/test-xid.c b/linux/tests/test-xid.c new file mode 100644 index 00000000000..d9c63cb6c63 --- /dev/null +++ b/linux/tests/test-xid.c @@ -0,0 +1,19 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include "os.h" +#include + +int os_getpid(void); + +static int gettid(void) { + return syscall(__NR_gettid); +} + +int main(void) { + assert(os_getpid() == getpid()); + assert(os_gettid() == gettid()); + return 0; +} diff --git a/linux/toku_pthread.c b/linux/toku_pthread.c new file mode 100644 index 00000000000..ad495fb948e --- /dev/null +++ b/linux/toku_pthread.c @@ -0,0 +1,6 @@ +#define _GNU_SOURCE 1 +#include + +int toku_pthread_yield(void) { + return pthread_yield(); +} diff --git a/linux/toku_pthread.h b/linux/toku_pthread.h new file mode 100644 index 00000000000..f246dcc7189 --- /dev/null +++ b/linux/toku_pthread.h @@ -0,0 +1,107 @@ +#ifndef _TOKU_PTHREAD_H +#define _TOKU_PTHREAD_H + +#if defined __cplusplus +extern "C" { +#endif + +#include + +typedef pthread_attr_t toku_pthread_attr_t; +typedef pthread_t toku_pthread_t; +typedef pthread_mutexattr_t toku_pthread_mutexattr_t; +typedef pthread_mutex_t toku_pthread_mutex_t; +typedef pthread_condattr_t toku_pthread_condattr_t; +typedef pthread_cond_t toku_pthread_cond_t; + +int toku_pthread_yield(void); + +static inline +int toku_pthread_attr_init(toku_pthread_attr_t *attr) { + return pthread_attr_init(attr); +} + +static inline +int toku_pthread_attr_destroy(toku_pthread_attr_t *attr) { + return pthread_attr_destroy(attr); +} + +static inline +int toku_pthread_attr_getstacksize(toku_pthread_attr_t *attr, size_t *stacksize) { + return pthread_attr_getstacksize(attr, stacksize); +} + +static inline +int toku_pthread_attr_setstacksize(toku_pthread_attr_t *attr, size_t stacksize) { + return pthread_attr_setstacksize(attr, stacksize); +} + +static inline +int toku_pthread_create(toku_pthread_t *thread, const toku_pthread_attr_t *attr, void *(*start_function)(void *), void *arg) { + return pthread_create(thread, attr, start_function, arg); +} + +static inline +int toku_pthread_join(toku_pthread_t thread, void **value_ptr) { + return pthread_join(thread, value_ptr); +} + +static inline +toku_pthread_t toku_pthread_self(void) { + return pthread_self(); +} + +#define TOKU_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER + +static inline +int toku_pthread_mutex_init(toku_pthread_mutex_t *mutex, const toku_pthread_mutexattr_t *attr) { + return pthread_mutex_init(mutex, attr); +} + +static inline +int toku_pthread_mutex_destroy(toku_pthread_mutex_t *mutex) { + return pthread_mutex_destroy(mutex); +} + +static inline +int toku_pthread_mutex_lock(toku_pthread_mutex_t *mutex) { + return pthread_mutex_lock(mutex); +} + +int toku_pthread_mutex_trylock(toku_pthread_mutex_t *mutex); + +static inline +int toku_pthread_mutex_unlock(toku_pthread_mutex_t *mutex) { + return pthread_mutex_unlock(mutex); +} + +static inline +int toku_pthread_cond_init(toku_pthread_cond_t *cond, const toku_pthread_condattr_t *attr) { + return pthread_cond_init(cond, attr); +} + +static inline +int toku_pthread_cond_destroy(toku_pthread_cond_t *cond) { + return pthread_cond_destroy(cond); +} + +static inline +int toku_pthread_cond_wait(toku_pthread_cond_t *cond, toku_pthread_mutex_t *mutex) { + return pthread_cond_wait(cond, mutex); +} + +static inline +int toku_pthread_cond_signal(toku_pthread_cond_t *cond) { + return pthread_cond_signal(cond); +} + +static inline +int toku_pthread_cond_broadcast(toku_pthread_cond_t *cond) { + return pthread_cond_broadcast(cond); +} + +#if defined __cplusplus +}; +#endif + +#endif diff --git a/newbrt/Makefile b/newbrt/Makefile index 22b9b5aaf0f..5b5f8ec5bc0 100644 --- a/newbrt/Makefile +++ b/newbrt/Makefile @@ -1,120 +1,38 @@ -# On cygwin do: -# make CYGWIN=cygwin check +# -*- Mode: Makefile -*- -# For verbose output do -# make VERBOSE=1 -# For very verbose output do -# make VERBOSE=2 +.DEFAULT_GOAL= build +TOKUROOT=../ +INCLUDEDIRS=-I. +COMBINE=1 +#TODO: Replace DEPEND_COMPILE with auto-dependancy generation. +DEPEND_COMPILE += \ + $(wildcard *.h) \ + log_header.h \ +# keep this line so I can have a \ on the previous line -# For CIL do -# make CIL=1 +NEWBRT = newbrt.$(AEXT) +#All executables need to statically link to newbrt +LINK_FILES += $(NEWBRT) -# GCOV_FLAGS = -fprofile-arcs -ftest-coverage -# PROF_FLAGS = -pg -OPTFLAGS = -O3 +SKIP_NEWBRTRULE=1 +include $(TOKUROOT)include/Makefile.include -ifeq ($(VERBOSE),2) -VERBVERBOSE=-v -MAYBEATSIGN= -else - ifeq ($(VERBOSE),1) - VERBVERBOSE=-q - MAYBEATSIGN= - else - VERBVERBOSE=-q - MAYBEATSIGN=@ - endif -endif - -#CFLAG default options -WERROR = -Werror -WALL = -Wall -Wextra -Wcast-align -Wbad-function-cast -FORMAT = -Wmissing-format-attribute -VISIBILITY= -fvisibility=hidden -FPICFLAGS = -fPIC -SHADOW = -Wshadow -SYMBOLS = -g3 -ggdb3 -PORTABILITY= -SKIP_WARNING= -COMBINE_C = -combine -c -LINK_FILES= -lz -lpthread -C99 = -std=c99 - -#Tools -VGRIND = valgrind --quiet --error-exitcode=1 --leak-check=yes - -ifeq ($(CC),icc) - #icc only: - OPTFLAGS = -O3 -ip - COMBINE_C = -ipo-c - FORMAT= #No argument for extra format warnings. - WALL = -Wall -Wcheck # '-Wextra' becomes '-Wcheck' in icc - SYMBOLS= -g -debug all -inline-debug-info - PORTABILITY=-diag-enable port-win - ifneq ($(CYGWIN),) - #Cygwin - ICC_NOWARN=-Qdiag-disable: - else - #Linux - ICC_NOWARN=-diag-disable #Need the space - endif - SKIP_WARNING += $(ICC_NOWARN)177 # Don't complain about static variables that are not used. - #SKIP_WARNING += $(ICC_NOWARN)188 # Don't complain about enumerated type mixed with another type. - SKIP_WARNING += $(ICC_NOWARN)589 # Don't complain about goto into a block that skips initializing variables. GCC catches the actual uninitialized variables. - SKIP_WARNING += $(ICC_NOWARN)869 # Don't complain about unused variables (since we defined __attribute__ to be nothing.) - SKIP_WARNING += $(ICC_NOWARN)981 # Don't complain about "operands are evaluated in unspecified order". This seems to be generated whenever more than one argument to a function or operand is computed by function call. - SKIP_WARNING += $(ICC_NOWARN)1324 # Don't complain about rdtsc clobbering its registers more than once. -endif - -ifneq ($(CYGWIN),) - #Cygwin (Windows) Must override some settings - CYG_ADD_LIBZ=/usr/lib/libz.a - VGRIND =#No Valgrind in cygwin - FPICFLAGS=#FPIC is default and not allowed as an option. - VISIBILITY=#Not supported - SHADOW=#Not supported - ifeq ($(CC),icc) - #Cygwin icc only - C99 = -Qstd=c99 - OPTFLAGS = -Ox -Qip -Qipo2 - COMBINE_C = -Qipo-c - WERROR = -WX # Windows icc version of -Werror - SYMBOLS= -Zi -debug:all -Qinline-debug-info - PORTABILITY= - LINK_FILES=#Not supported - SKIP_WARNING += $(ICC_NOWARN)1786 # Don't complain about 'read/write/other standards' being deprecated - else - #Cygwin gcc only - FORMAT = -Wno-format - endif -endif - -CFLAGS = $(WALL) $(WERROR) $(FORMAT) $(VISIBILITY) $(FPICFLAGS) $(SHADOW) -CFLAGS += $(OPTFLAGS) $(GCOV_FLAGS) $(PROF_FLAGS) -CFLAGS += $(SYMBOLS) $(SKIP_WARNING) $(C99) - -LDFLAGS = $(OPTFLAGS) $(SYMBOLS) $(GCOV_FLAGS) $(PROF_FLAGS) $(LINK_FILES) -# Need XOPEN_SOURCE=600 to get strtoll() -CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=600 - -# Add -Wconversion +# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test -ifdef BRT_FANOUT -CPPFLAGS += -DBRT_FANOUT=$(BRT_FANOUT) -endif +BINS_RAW= \ + brtdump \ + tdb_logprint \ + tdb-recover \ +# Intentionally left blank +# BINS will be defined automatically. -ifeq ($(CIL),1) - CC=../../cil/cil-1.3.6/bin/cilly --merge --keepmerged -endif +BINS_O = $(patsubst %,%.$(OEXT),$(BINS_RAW)) -# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test -BINS= brtdump \ - tdb_logprint \ -# Intentionally left blank +.PHONY: build default bins libs -build default: bins libs tdb-recover tdb_logprint $(TEST_OFILES) - cd tests;$(MAKE) build +build default: bins libs $(TEST_NEWBRT) + $(MAYBEATSIGN)cd tests;$(MAKE) build BRT_SOURCES = \ block_allocator \ @@ -122,6 +40,7 @@ BRT_SOURCES = \ brt-serialize \ brt-verify \ brt \ + brt-test-helpers \ cachetable \ fifo \ fingerprint \ @@ -135,82 +54,52 @@ BRT_SOURCES = \ omt \ recover \ roll \ + threadpool \ toku_assert \ - ybt \ - x1764 \ trace_mem \ - threadpool \ -# keep this line so I can ha vea \ on the previous line - -ifeq ($(CIL),1) - OFILES = $(patsubst %,%.o,$(BRT_SOURCES)) -else - OFILES = newbrt.o -endif -TEST_OFILES = brt-test-helpers.o + x1764 \ + ybt \ +# keep this line so I can have a \ on the previous line -HFILES = $(wildcard *.h) +TEST_NEWBRT = brt-test-helpers.$(OEXT) BRT_C_FILES = $(patsubst %,%.c,$(BRT_SOURCES)) +BRT_O_FILES = $(patsubst %,%.$(OEXT),$(BRT_SOURCES)) -ifeq ($(CIL),1) -else -newbrt.o: $(BRT_C_FILES) $(HFILES) - $(CC) $(COMBINE_C) $(CFLAGS) $(CPPFLAGS) $(BRT_C_FILES) -o $@ -endif - -tdb_logprint.o: log-internal.h brttypes.h log.h kv-pair.h log_header.h -tdb_logprint: $(OFILES) $(CYG_ADD_LIBZ) +newbrt.o: $(BRT_C_FILES) $(DEPEND_COMPILE) + $(MAYBEATSIGN)$(CC) $(BRT_C_FILES) $(COMBINE_C) $(O_FROM_C_FLAGS) -recover.o: log_header.h log-internal.h log.h brttypes.h kv-pair.h memory.h key.h cachetable.h -tdb-recover: $(OFILES) $(CYG_ADD_LIBZ) -roll.o: log_header.h log-internal.h log.h brttypes.h kv-pair.h memory.h key.h cachetable.h omt.h bread.h +ifneq ($(CYGWIN),) +$(NEWBRT): $(BRT_O_FILES) +else ifeq ($(CC),icc) +$(NEWBRT): $(BRT_O_FILES) +else ifeq ($(COMBINE),0) +$(NEWBRT): $(BRT_O_FILES) +else +$(NEWBRT): newbrt.o +endif -log_code.o: log_header.h wbuf.h log-internal.h rbuf.h +log_code.$(OEXT): log_header.h wbuf.h log-internal.h rbuf.h log_header.h: log_code.c @echo generated log_code.c so log_header.c was also generated -log_code.c: logformat - ./logformat +log_code.c: logformat$(BINSUF) + $(MAYBEATSIGN)./logformat +#Needs to be done manually since it does not include newbrt. +logformat$(BINSUF): logformat.c $(LIBPORTABILITY) + $(MAYBEATSIGN)$(CC) $< $(BIN_FROM_O_FLAGS_NOLIB) $(LIBPORTABILITY) -libs: $(OFILES) $(CYG_ADD_LIBZ) + +libs: $(NEWBRT) bins: $(BINS) + # Put the benchmarktest_256 first since it takes the longest (and we want to use parallelism in the make) # Put check_benchmarktest_256 first because it is long-running (and therefore on the critical path, so get it started) check: bins - cd tests;$(MAKE) check - -check-fanout: - let BRT_FANOUT=4; \ - while [ $$BRT_FANOUT -le 16 ] ;do \ - $(MAKE) clean; $(MAKE) check BRT_FANOUT=$$BRT_FANOUT; \ - let BRT_FANOUT=BRT_FANOUT+1; \ - done - -BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h omt.h brt.h brt-search.h brttypes.h ybt.h log.h ../include/db.h kv-pair.h memory.h mempool.h leafentry.h log_header.h -brt-test-helpers.o: $(BRT_INTERNAL_H_INCLUDES) toku_assert.h -logformat: logformat.c -brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES) - -$(OFILES): $(BRT_INTERNAL_H_INCLUDES) - - -test_toku_malloc_plain_free: newbrt.o - - -cachetable-test.o: cachetable.h memory.h -cachetable-test: $(OFILES) $(CYG_ADD_LIBZ) - -cachetable-test2.o: cachetable.h memory.h -cachetable-test2: $(OFILES) $(CYG_ADD_LIBZ) - -test-assert: newbrt.o - -brtdump: $(OFILES) $(CYG_ADD_LIBZ) - -test_oexcl: test_oexcl.o newbrt.o + $(MAYBEATSIGN)cd tests;$(MAKE) check +%$(BINSUF): $(NEWBRT) $(LIBPORTABILITY) checko2: ifeq ($(OPTFLAGS),-O3) @@ -221,17 +110,13 @@ endif clean: clean-local clean-tests clean-tests: - cd tests;$(MAKE) clean + $(MAYBEATSIGN)cd tests;$(MAKE) clean clean-local: - rm -rf $(BINS) *.o *.bb *.bbg *.da *.gcov *.gcno *.gcda - rm -rf test_oexcl.c.tmp *.brt - rm -rf *.obj *.pdb *.ilk *.exe - rm -rf log_code.c log_header.h logformat + $(MAYBEATSIGN)rm -rf $(NEWBRT) + $(MAYBEATSIGN)rm -rf test_oexcl.c.tmp *.brt + $(MAYBEATSIGN)rm -rf log_code.c log_header.h logformat # After doing (cd ../src/tests;make test_log5.recover), run these. The files should have no differences. -testdump: brtdump - ./brtdump ../src/tests/dir.test_log5.c.tdb.recover/foo.db > dump.r && ./brtdump ../src/tests/dir.test_log5.c.tdb/foo.db > dump.o && diff dump.o dump.r +testdump: brtdump$(BINSUF) + $(MAYBEATSIGN)./brtdump ../src/tests/dir.test_log5.c.tdb.recover/foo.db > dump.r && ./brtdump ../src/tests/dir.test_log5.c.tdb/foo.db > dump.$(OEXT) && diff dump.$(OEXT) dump.r -TAGS: ../*/*.c ../*/*.h - etags ../*/*.c ../*/*.h -ofiles: $(OFILES) diff --git a/newbrt/brt-serialize.c b/newbrt/brt-serialize.c index a65d7c38037..7cd6f485cda 100644 --- a/newbrt/brt-serialize.c +++ b/newbrt/brt-serialize.c @@ -43,13 +43,13 @@ static void maybe_preallocate_in_file (int fd, u_int64_t size) { } // This mutex protects pwrite from running in parallel, and also protects modifications to the block allocator. -static pthread_mutex_t pwrite_mutex = PTHREAD_MUTEX_INITIALIZER; +static toku_pthread_mutex_t pwrite_mutex = TOKU_PTHREAD_MUTEX_INITIALIZER; static int pwrite_is_locked=0; static inline void lock_for_pwrite (void) { // Locks the pwrite_mutex. - int r = pthread_mutex_lock(&pwrite_mutex); + int r = toku_pthread_mutex_lock(&pwrite_mutex); assert(r==0); pwrite_is_locked = 1; } @@ -57,7 +57,7 @@ lock_for_pwrite (void) { static inline void unlock_for_pwrite (void) { pwrite_is_locked = 0; - int r = pthread_mutex_unlock(&pwrite_mutex); + int r = toku_pthread_mutex_unlock(&pwrite_mutex); assert(r==0); } diff --git a/newbrt/cachetable-rwlock.h b/newbrt/cachetable-rwlock.h index fd21406fb87..e29618c5370 100644 --- a/newbrt/cachetable-rwlock.h +++ b/newbrt/cachetable-rwlock.h @@ -16,10 +16,10 @@ typedef struct ctpair_rwlock *CTPAIR_RWLOCK; struct ctpair_rwlock { int pinned; // the number of readers int want_pin; // the number of blocked readers - pthread_cond_t wait_pin; + toku_pthread_cond_t wait_pin; int writer; // the number of writers int want_write; // the number of blocked writers - pthread_cond_t wait_write; + toku_pthread_cond_t wait_write; }; // initialize a read write lock @@ -29,9 +29,9 @@ void ctpair_rwlock_init(CTPAIR_RWLOCK rwlock) { int r; rwlock->pinned = rwlock->want_pin = 0; - r = pthread_cond_init(&rwlock->wait_pin, 0); assert(r == 0); + r = toku_pthread_cond_init(&rwlock->wait_pin, 0); assert(r == 0); rwlock->writer = rwlock->want_write = 0; - r = pthread_cond_init(&rwlock->wait_write, 0); assert(r == 0); + r = toku_pthread_cond_init(&rwlock->wait_write, 0); assert(r == 0); } // destroy a read write lock @@ -42,18 +42,18 @@ ctpair_rwlock_destroy(CTPAIR_RWLOCK rwlock) { int r; assert(rwlock->pinned == 0 && rwlock->want_pin == 0); assert(rwlock->writer == 0 && rwlock->want_write == 0); - r = pthread_cond_destroy(&rwlock->wait_pin); assert(r == 0); - r = pthread_cond_destroy(&rwlock->wait_write); assert(r == 0); + r = toku_pthread_cond_destroy(&rwlock->wait_pin); assert(r == 0); + r = toku_pthread_cond_destroy(&rwlock->wait_write); assert(r == 0); } // obtain a read lock // expects: mutex is locked -static inline void ctpair_read_lock(CTPAIR_RWLOCK rwlock, pthread_mutex_t *mutex) { +static inline void ctpair_read_lock(CTPAIR_RWLOCK rwlock, toku_pthread_mutex_t *mutex) { if (rwlock->writer || rwlock->want_write) { rwlock->want_pin++; while (rwlock->writer || rwlock->want_write) { - int r = pthread_cond_wait(&rwlock->wait_pin, mutex); assert(r == 0); + int r = toku_pthread_cond_wait(&rwlock->wait_pin, mutex); assert(r == 0); } rwlock->want_pin--; } @@ -66,18 +66,18 @@ static inline void ctpair_read_lock(CTPAIR_RWLOCK rwlock, pthread_mutex_t *mutex static inline void ctpair_read_unlock(CTPAIR_RWLOCK rwlock) { rwlock->pinned--; if (rwlock->pinned == 0 && rwlock->want_write) { - int r = pthread_cond_signal(&rwlock->wait_write); assert(r == 0); + int r = toku_pthread_cond_signal(&rwlock->wait_write); assert(r == 0); } } // obtain a write lock // expects: mutex is locked -static inline void ctpair_write_lock(CTPAIR_RWLOCK rwlock, pthread_mutex_t *mutex) { +static inline void ctpair_write_lock(CTPAIR_RWLOCK rwlock, toku_pthread_mutex_t *mutex) { if (rwlock->pinned || rwlock->writer) { rwlock->want_write++; while (rwlock->pinned || rwlock->writer) { - int r = pthread_cond_wait(&rwlock->wait_write, mutex); assert(r == 0); + int r = toku_pthread_cond_wait(&rwlock->wait_write, mutex); assert(r == 0); } rwlock->want_write--; } @@ -91,9 +91,9 @@ static inline void ctpair_write_unlock(CTPAIR_RWLOCK rwlock) { rwlock->writer--; if (rwlock->writer == 0) { if (rwlock->want_write) { - int r = pthread_cond_signal(&rwlock->wait_write); assert(r == 0); + int r = toku_pthread_cond_signal(&rwlock->wait_write); assert(r == 0); } else if (rwlock->want_pin) { - int r = pthread_cond_broadcast(&rwlock->wait_pin); assert(r == 0); + int r = toku_pthread_cond_broadcast(&rwlock->wait_pin); assert(r == 0); } } } diff --git a/newbrt/cachetable-writequeue.h b/newbrt/cachetable-writequeue.h index b9f5ab8d1b4..9231a51f2f3 100644 --- a/newbrt/cachetable-writequeue.h +++ b/newbrt/cachetable-writequeue.h @@ -6,9 +6,9 @@ typedef struct writequeue *WRITEQUEUE; struct writequeue { PAIR head, tail; // head and tail of the linked list of pair's - pthread_cond_t wait_read; // wait for read + toku_pthread_cond_t wait_read; // wait for read int want_read; // number of threads waiting to read - pthread_cond_t wait_write; // wait for write + toku_pthread_cond_t wait_write; // wait for write int want_write; // number of threads waiting to write int ninq; // number of pairs in the queue char closed; // kicks waiting threads off of the write queue @@ -21,9 +21,9 @@ struct writequeue { static void writequeue_init(WRITEQUEUE wq) { wq->head = wq->tail = 0; int r; - r = pthread_cond_init(&wq->wait_read, 0); assert(r == 0); + r = toku_pthread_cond_init(&wq->wait_read, 0); assert(r == 0); wq->want_read = 0; - r = pthread_cond_init(&wq->wait_write, 0); assert(r == 0); + r = toku_pthread_cond_init(&wq->wait_write, 0); assert(r == 0); wq->want_write = 0; wq->ninq = 0; wq->closed = 0; @@ -35,8 +35,8 @@ static void writequeue_init(WRITEQUEUE wq) { static void writequeue_destroy(WRITEQUEUE wq) { assert(wq->head == 0 && wq->tail == 0); int r; - r = pthread_cond_destroy(&wq->wait_read); assert(r == 0); - r = pthread_cond_destroy(&wq->wait_write); assert(r == 0); + r = toku_pthread_cond_destroy(&wq->wait_read); assert(r == 0); + r = toku_pthread_cond_destroy(&wq->wait_write); assert(r == 0); } // close the writequeue @@ -45,8 +45,8 @@ static void writequeue_destroy(WRITEQUEUE wq) { static void writequeue_set_closed(WRITEQUEUE wq) { wq->closed = 1; int r; - r = pthread_cond_broadcast(&wq->wait_read); assert(r == 0); - r = pthread_cond_broadcast(&wq->wait_write); assert(r == 0); + r = toku_pthread_cond_broadcast(&wq->wait_read); assert(r == 0); + r = toku_pthread_cond_broadcast(&wq->wait_write); assert(r == 0); } // determine whether or not the write queue is empty @@ -70,7 +70,7 @@ static void writequeue_enq(WRITEQUEUE wq, PAIR pair) { wq->tail = pair; wq->ninq++; if (wq->want_read) { - int r = pthread_cond_signal(&wq->wait_read); assert(r == 0); + int r = toku_pthread_cond_signal(&wq->wait_read); assert(r == 0); } } @@ -80,12 +80,12 @@ static void writequeue_enq(WRITEQUEUE wq, PAIR pair) { // write queue and return it // returns: 0 if success, otherwise an error -static int writequeue_deq(WRITEQUEUE wq, pthread_mutex_t *mutex, PAIR *pairptr) { +static int writequeue_deq(WRITEQUEUE wq, toku_pthread_mutex_t *mutex, PAIR *pairptr) { while (writequeue_empty(wq)) { if (wq->closed) return EINVAL; wq->want_read++; - int r = pthread_cond_wait(&wq->wait_read, mutex); assert(r == 0); + int r = toku_pthread_cond_wait(&wq->wait_read, mutex); assert(r == 0); wq->want_read--; } PAIR pair = wq->head; @@ -101,9 +101,9 @@ static int writequeue_deq(WRITEQUEUE wq, pthread_mutex_t *mutex, PAIR *pairptr) // suspend the writer thread // expects: the mutex is locked -static void writequeue_wait_write(WRITEQUEUE wq, pthread_mutex_t *mutex) { +static void writequeue_wait_write(WRITEQUEUE wq, toku_pthread_mutex_t *mutex) { wq->want_write++; - int r = pthread_cond_wait(&wq->wait_write, mutex); assert(r == 0); + int r = toku_pthread_cond_wait(&wq->wait_write, mutex); assert(r == 0); wq->want_write--; } @@ -112,7 +112,7 @@ static void writequeue_wait_write(WRITEQUEUE wq, pthread_mutex_t *mutex) { static void writequeue_wakeup_write(WRITEQUEUE wq) { if (wq->want_write) { - int r = pthread_cond_broadcast(&wq->wait_write); assert(r == 0); + int r = toku_pthread_cond_broadcast(&wq->wait_write); assert(r == 0); } } diff --git a/newbrt/cachetable.c b/newbrt/cachetable.c index ff9d4626930..7eff965e832 100644 --- a/newbrt/cachetable.c +++ b/newbrt/cachetable.c @@ -74,7 +74,7 @@ struct cachetable { long size_writing; // the sum of the sizes of the pairs being written LSN lsn_of_checkpoint; // the most recent checkpoint in the log. TOKULOGGER logger; - pthread_mutex_t mutex; // coarse lock that protects the cachetable, the cachefiles, and the pair's + toku_pthread_mutex_t mutex; // coarse lock that protects the cachetable, the cachefiles, and the pair's struct writequeue wq; // write queue for the writer threads THREADPOOL threadpool; // pool of writer threads char checkpointing; // checkpoint in progress @@ -84,7 +84,7 @@ struct cachetable { static inline void cachetable_lock(CACHETABLE ct __attribute__((unused))) { #if DO_CACHETABLE_LOCK - int r = pthread_mutex_lock(&ct->mutex); assert(r == 0); + int r = toku_pthread_mutex_lock(&ct->mutex); assert(r == 0); #endif } @@ -92,7 +92,7 @@ static inline void cachetable_lock(CACHETABLE ct __attribute__((unused))) { static inline void cachetable_unlock(CACHETABLE ct __attribute__((unused))) { #if DO_CACHETABLE_LOCK - int r = pthread_mutex_unlock(&ct->mutex); assert(r == 0); + int r = toku_pthread_mutex_unlock(&ct->mutex); assert(r == 0); #endif } @@ -105,11 +105,6 @@ static inline void cachetable_wait_write(CACHETABLE ct) { } } -struct fileid { - dev_t st_dev; /* device and inode are enough to uniquely identify a file in unix. */ - ino_t st_ino; -}; - struct cachefile { CACHEFILE next; u_int64_t refcount; /* CACHEFILEs are shared. Use a refcount to decide when to really close it. @@ -156,7 +151,7 @@ int toku_create_cachetable(CACHETABLE *result, long size_limit, LSN initial_lsn, t->checkpointing = 0; int r; writequeue_init(&t->wq); - r = pthread_mutex_init(&t->mutex, 0); assert(r == 0); + r = toku_pthread_mutex_init(&t->mutex, 0); assert(r == 0); // set the max number of writeback threads to min(MAX_WRITER_THREADS,nprocs_online) int nprocs = sysconf(_SC_NPROCESSORS_ONLN); @@ -969,7 +964,7 @@ int toku_cachetable_close (CACHETABLE *tp) { cachetable_unlock(t); threadpool_destroy(&t->threadpool); writequeue_destroy(&t->wq); - r = pthread_mutex_destroy(&t->mutex); assert(r == 0); + r = toku_pthread_mutex_destroy(&t->mutex); assert(r == 0); toku_free(t->table); toku_free(t); *tp = 0; @@ -1118,7 +1113,7 @@ FILENUM toku_cachefile_filenum (CACHEFILE cf) { // The writer thread waits for work in the write queue and writes the pair static void *cachetable_writer(void *arg) { - // printf("%lu:%s:start %p\n", pthread_self(), __FUNCTION__, arg); + // printf("%lu:%s:start %p\n", toku_pthread_self(), __FUNCTION__, arg); CACHETABLE ct = arg; int r; cachetable_lock(ct); @@ -1132,7 +1127,7 @@ static void *cachetable_writer(void *arg) { cachetable_write_pair(ct, p); } cachetable_unlock(ct); - // printf("%lu:%s:exit %p\n", pthread_self(), __FUNCTION__, arg); + // printf("%lu:%s:exit %p\n", toku_pthread_self(), __FUNCTION__, arg); return arg; } diff --git a/newbrt/includes.h b/newbrt/includes.h index 5af1810b43b..ab7ff99b19c 100644 --- a/newbrt/includes.h +++ b/newbrt/includes.h @@ -12,7 +12,7 @@ #if !defined(TOKU_WINDOWS) #include #include -#include +#include #include #include #include diff --git a/newbrt/leafentry.c b/newbrt/leafentry.c index e22b87d4e5c..dedd79a8afc 100644 --- a/newbrt/leafentry.c +++ b/newbrt/leafentry.c @@ -271,164 +271,164 @@ void toku_free_LEAFENTRY(LEAFENTRY le) { } -inline int le_is_provdel(LEAFENTRY le) { +int le_is_provdel(LEAFENTRY le) { return get_le_state(le)==LE_PROVDEL; } -inline void* latest_key_le_committed (u_int32_t UU(keylen), void *key, u_int32_t UU(vallen), void *UU(val)) { +void* latest_key_le_committed (u_int32_t UU(keylen), void *key, u_int32_t UU(vallen), void *UU(val)) { return key; } -inline void* latest_key_le_both (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { +void* latest_key_le_both (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { return kval; } -inline void* latest_key_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +void* latest_key_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return 0; // for provisional delete, there is no *latest* key, so return NULL } -inline void* latest_key_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(plen), void *UU(pval)) { +void* latest_key_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(plen), void *UU(pval)) { return kval; } -inline void* le_latest_key (LEAFENTRY le) { +void* le_latest_key (LEAFENTRY le) { LESWITCHCALL(le, latest_key); abort(); return 0; // make certain compilers happy } -inline u_int32_t latest_keylen_le_committed (u_int32_t keylen, void *UU(key), u_int32_t UU(vallen), void *UU(val)) { +u_int32_t latest_keylen_le_committed (u_int32_t keylen, void *UU(key), u_int32_t UU(vallen), void *UU(val)) { return keylen; } -inline u_int32_t latest_keylen_le_both (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { +u_int32_t latest_keylen_le_both (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { return klen; } -inline u_int32_t latest_keylen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +u_int32_t latest_keylen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return 0; // for provisional delete, there is no *latest* key, so return 0. What else can we do? } -inline u_int32_t latest_keylen_le_provpair (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { +u_int32_t latest_keylen_le_provpair (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { return klen; } -inline u_int32_t le_latest_keylen (LEAFENTRY le) { +u_int32_t le_latest_keylen (LEAFENTRY le) { LESWITCHCALL(le, latest_keylen); abort(); return 0; // make certain compilers happy } -inline void* latest_val_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { +void* latest_val_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { return val; } -inline void* latest_val_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *pval) { +void* latest_val_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *pval) { return pval; } -inline void* latest_val_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +void* latest_val_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return 0; // for provisional delete, there is no *latest* key, so return NULL } -inline void* latest_val_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *pval) { +void* latest_val_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *pval) { return pval; } -inline void* le_latest_val (LEAFENTRY le) { +void* le_latest_val (LEAFENTRY le) { LESWITCHCALL(le, latest_val); abort(); return 0; // make certain compilers happy } -inline u_int32_t latest_vallen_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t vallen, void *UU(val)) { +u_int32_t latest_vallen_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t vallen, void *UU(val)) { return vallen; } -inline u_int32_t latest_vallen_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t plen, void *UU(pval)) { +u_int32_t latest_vallen_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t plen, void *UU(pval)) { return plen; } -inline u_int32_t latest_vallen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +u_int32_t latest_vallen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return 0; // for provisional delete, there is no *latest* key, so return 0. What else can we do? } -inline u_int32_t latest_vallen_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t plen, void *UU(pval)) { +u_int32_t latest_vallen_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t plen, void *UU(pval)) { return plen; } -inline u_int32_t le_latest_vallen (LEAFENTRY le) { +u_int32_t le_latest_vallen (LEAFENTRY le) { LESWITCHCALL(le, latest_vallen); abort(); return 0; // make certain compilers happy } -inline void* any_key_le_committed (u_int32_t UU(keylen), void *key, u_int32_t UU(vallen), void *UU(val)) { +void* any_key_le_committed (u_int32_t UU(keylen), void *key, u_int32_t UU(vallen), void *UU(val)) { return key; } -inline void* any_key_le_both (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { +void* any_key_le_both (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { return kval; } -inline void* any_key_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval)) { +void* any_key_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(clen), void *UU(cval)) { return kval; } -inline void* any_key_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(plen), void *UU(pval)) { +void* any_key_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *kval, u_int32_t UU(plen), void *UU(pval)) { return kval; } -inline void* le_any_key (LEAFENTRY le) { +void* le_any_key (LEAFENTRY le) { LESWITCHCALL(le, any_key); abort(); return 0; // make certain compilers happy } -inline u_int32_t any_keylen_le_committed (u_int32_t keylen, void *UU(key), u_int32_t UU(vallen), void *UU(val)) { +u_int32_t any_keylen_le_committed (u_int32_t keylen, void *UU(key), u_int32_t UU(vallen), void *UU(val)) { return keylen; } -inline u_int32_t any_keylen_le_both (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { +u_int32_t any_keylen_le_both (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { return klen; } -inline u_int32_t any_keylen_le_provdel (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +u_int32_t any_keylen_le_provdel (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return klen; } -inline u_int32_t any_keylen_le_provpair (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { +u_int32_t any_keylen_le_provpair (TXNID UU(xid), u_int32_t klen, void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { return klen; } -inline u_int32_t le_any_keylen (LEAFENTRY le) { +u_int32_t le_any_keylen (LEAFENTRY le) { LESWITCHCALL(le, any_keylen); abort(); return 0; // make certain compilers happy } -inline void* any_val_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { +void* any_val_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { return val; } -inline void* any_val_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *cval, u_int32_t UU(plen), void *UU(pval)) { +void* any_val_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *cval, u_int32_t UU(plen), void *UU(pval)) { return cval; } -inline void* any_val_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *cval) { +void* any_val_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *cval) { return cval; } -inline void* any_val_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *pval) { +void* any_val_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *pval) { return pval; } -inline void* le_any_val (LEAFENTRY le) { +void* le_any_val (LEAFENTRY le) { LESWITCHCALL(le, any_val); abort(); return 0; // make certain compilers happy } -inline u_int32_t any_vallen_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t vallen, void *UU(val)) { +u_int32_t any_vallen_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t vallen, void *UU(val)) { return vallen; } -inline u_int32_t any_vallen_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t plen, void *UU(pval)) { +u_int32_t any_vallen_le_both (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t plen, void *UU(pval)) { return plen; } -inline u_int32_t any_vallen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t clen, void *UU(cval)) { +u_int32_t any_vallen_le_provdel (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t clen, void *UU(cval)) { return clen; // for provisional delete, there is no *any* key, so return 0. What else can we do? } -inline u_int32_t any_vallen_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t plen, void *UU(pval)) { +u_int32_t any_vallen_le_provpair (TXNID UU(xid), u_int32_t UU(klen), void *UU(kval), u_int32_t plen, void *UU(pval)) { return plen; } -inline u_int32_t le_any_vallen (LEAFENTRY le) { +u_int32_t le_any_vallen (LEAFENTRY le) { LESWITCHCALL(le, any_vallen); abort(); return 0; // make certain compilers happy } -inline u_int64_t any_xid_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { +u_int64_t any_xid_le_committed (u_int32_t UU(keylen), void *UU(key), u_int32_t UU(vallen), void *UU(val)) { return 0; } -inline u_int64_t any_xid_le_both (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { +u_int64_t any_xid_le_both (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval), u_int32_t UU(plen), void *UU(pval)) { return xid; } -inline u_int64_t any_xid_le_provdel (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { +u_int64_t any_xid_le_provdel (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(clen), void *UU(cval)) { return xid; } -inline u_int64_t any_xid_le_provpair (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { +u_int64_t any_xid_le_provpair (TXNID xid, u_int32_t UU(klen), void *UU(kval), u_int32_t UU(plen), void *UU(pval)) { return xid; } -inline u_int64_t le_any_xid (LEAFENTRY le) { +u_int64_t le_any_xid (LEAFENTRY le) { LESWITCHCALL(le, any_xid); abort(); return 0; // make certain compilers happy } diff --git a/newbrt/leafentry.h b/newbrt/leafentry.h index a626bd74cc2..337a8e28a91 100644 --- a/newbrt/leafentry.h +++ b/newbrt/leafentry.h @@ -156,5 +156,57 @@ void* le_any_val (LEAFENTRY le); u_int32_t le_any_vallen (LEAFENTRY le); u_int64_t le_any_xid (LEAFENTRY le); +void *latest_key_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +void *latest_key_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +void *latest_key_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +void *latest_key_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int32_t latest_keylen_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int32_t latest_keylen_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int32_t latest_keylen_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int32_t latest_keylen_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +void *latest_val_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +void *latest_val_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +void *latest_val_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +void *latest_val_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int32_t latest_vallen_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int32_t latest_vallen_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int32_t latest_vallen_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int32_t latest_vallen_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int64_t latest_xid_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int64_t latest_xid_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int64_t latest_xid_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int64_t latest_xid_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +// + +void *any_key_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +void *any_key_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +void *any_key_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +void *any_key_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int32_t any_keylen_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int32_t any_keylen_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int32_t any_keylen_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int32_t any_keylen_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +void *any_val_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +void *any_val_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +void *any_val_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +void *any_val_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int32_t any_vallen_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int32_t any_vallen_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int32_t any_vallen_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int32_t any_vallen_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + +u_int64_t any_xid_le_committed(u_int32_t klen, void *kval, u_int32_t vallen, void *val); +u_int64_t any_xid_le_both(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval, u_int32_t plen, void *pval); +u_int64_t any_xid_le_provdel(TXNID xid, u_int32_t klen, void *kval, u_int32_t clen, void *cval); +u_int64_t any_xid_le_provpair(TXNID xid, u_int32_t klen, void *kval, u_int32_t plen, void *pval); + #endif diff --git a/newbrt/log-internal.h b/newbrt/log-internal.h index 0af1608c5c0..da74d64096d 100644 --- a/newbrt/log-internal.h +++ b/newbrt/log-internal.h @@ -10,7 +10,7 @@ #include "memarena.h" #include #if !defined(TOKU_WINDOWS) -#include +#include #endif #include #include @@ -24,16 +24,16 @@ #define LOGGER_BUF_SIZE (1<<24) struct mylock { - pthread_mutex_t lock; + toku_pthread_mutex_t lock; int is_locked; }; static inline int ml_init(struct mylock *l) { l->is_locked=0; - return pthread_mutex_init(&l->lock, 0); + return toku_pthread_mutex_init(&l->lock, 0); } static inline int ml_lock(struct mylock *l) { - int r = pthread_mutex_lock(&l->lock); + int r = toku_pthread_mutex_lock(&l->lock); assert(l->is_locked==0); l->is_locked=1; return r; @@ -41,11 +41,11 @@ static inline int ml_lock(struct mylock *l) { static inline int ml_unlock(struct mylock *l) { assert(l->is_locked==1); l->is_locked=0; - return pthread_mutex_unlock(&l->lock); + return toku_pthread_mutex_unlock(&l->lock); } static inline int ml_destroy(struct mylock *l) { assert(l->is_locked==0); - return pthread_mutex_destroy(&l->lock); + return toku_pthread_mutex_destroy(&l->lock); } diff --git a/newbrt/portability.h b/newbrt/portability.h deleted file mode 100644 index c9af907e2ee..00000000000 --- a/newbrt/portability.h +++ /dev/null @@ -1,122 +0,0 @@ -#ifndef PORTABILITY_H -#define PORTABILITY_H - -// Portability layer - -#if defined(__INTEL_COMPILER) -#if !defined(__ICL) && !defined(__ICC) -#error Which intel compiler? -#endif -#if defined(__ICL) && defined(__ICC) -#error Cannot distinguish between windows and linux intel compiler -#endif - -#if defined(__ICL) -//Windows Intel Compiler -#define TOKU_WINDOWS - -//Define standard integer types. -typedef __int8 int8_t; -typedef unsigned __int8 u_int8_t; -typedef __int16 int16_t; -typedef unsigned __int16 u_int16_t; -typedef __int32 int32_t; -typedef unsigned __int32 u_int32_t; -typedef __int64 int64_t; -typedef unsigned __int64 u_int64_t; - -//Define printf types. -#define PRId64 "I64d" -#define PRIu64 "I64u" -#define PRId32 "d" -#define PRIu32 "u" - -//Limits -#define INT8_MIN _I8_MIN -#define INT8_MAX _I8_MAX -#define UINT8_MAX _UI8_MAX -#define INT16_MIN _I16_MIN -#define INT16_MAX _I16_MAX -#define UINT16_MAX _UI16_MAX -#define INT32_MIN _I32_MIN -#define INT32_MAX _I32_MAX -#define UINT32_MAX _UI32_MAX -#define INT64_MIN _I64_MIN -#define INT64_MAX _I64_MAX -#define UINT64_MAX _UI64_MAX - -#define srandom srand -#define random rand - -#define FAKE_WINDOWS_STUBS 0 // Disable these fakes. -#if FAKE_WINDOWS_STUBS == 1 - -//#define chmod(a,b) (void)0 /* Remove temporarily till compatibility layer exists */ -//Define chmod -typedef int mode_t; -static inline -int -chmod(const char *path, mode_t mode) { - //TODO: Write API to support what we really need. - //Linux version supports WRITE/EXECUTE/READ bits separately for user/group/world - //windows _chmod supports WRITE/READ bits separately for ?? one type (user? world?) - //See _chmod in sys/stat.h - //Supports setting read/write mode (not separately for user/group/world) - return 0; -} -#define S_IRUSR 0 -#define S_IRGRP 0 -#define S_IROTH 0 - -//Fake typedefs to skip warnings. -typedef size_t ssize_t; -typedef int pthread_cond_t; -typedef void* pthread_mutex_t; -struct timeval { - int tv_sec; - int tv_usec; -}; - -#endif //FAKE_WINDOWS_STUBS - -#endif //Windows Intel Compiler - -// Intel compiler. -// Define ntohl using bswap. -// Define __attribute__ to be null - -#include - -static inline -u_int32_t -ntohl(u_int32_t x) { - return _bswap(x); -} -static inline -u_int32_t -htonl(u_int32_t x) { - return _bswap(x); -} - -#define __attribute__(x) - - -#elif defined __GNUC__ - -// Gcc: -// Define ntohl using arpa/inet.h -#include - -#else -#error Not ICC and not GNUC. What compiler? -#endif - -// This hack is to avoid initializing variables that gcc can figure out are not used in an undefined way. -// But some compiler cannot figure it out, so we initialize the value to zero. -#if defined __INTEL_COMPILER -#define MAYBE_INIT(n) = n -#elif defined __GNUC__ -#define MAYBE_INIT(n) -#endif - -#endif diff --git a/newbrt/rwlock.c b/newbrt/rwlock.c index 77974945d69..61f7860fd1f 100644 --- a/newbrt/rwlock.c +++ b/newbrt/rwlock.c @@ -8,7 +8,7 @@ ***************************************** * * TokuDB employs readers/writers locks for the ephemeral locks (e.g., - * on BRT nodes) Why not just use the pthread_rwlock API? + * on BRT nodes) Why not just use the toku_pthread_rwlock API? * * 1) we need multiprocess rwlocks (not just multithreaded) * diff --git a/newbrt/tests/Makefile b/newbrt/tests/Makefile index 6f3090e5fae..44b6a78ef46 100644 --- a/newbrt/tests/Makefile +++ b/newbrt/tests/Makefile @@ -1,3 +1,13 @@ +.DEFAULT_GOAL=build +TOKUROOT=../../ +INCLUDEDIRS=-I. -I.. +include $(TOKUROOT)include/Makefile.include +CPPFLAGS += -D_GNU_SOURCE + +ifeq ($(CC),icc) +SKIP_WARNING += $(ICC_NOWARN)1418 #Non static functions do not need prototypes. +endif + # On cygwin do: # make CYGWIN=cygwin check @@ -6,108 +16,10 @@ # For very verbose output do # make VERBOSE=2 -# GCOV_FLAGS = -fprofile-arcs -ftest-coverage -# PROF_FLAGS = -pg -OPTFLAGS = -O3 - -ifeq ($(VERBOSE),2) -VERBVERBOSE=-v -else - ifeq ($(VERBOSE),1) - VERBVERBOSE=-q - else - VERBVERBOSE=-q - endif -endif - -#CFLAG default options -WERROR = -Werror -WALL = -Wall -Wextra -Wcast-align -Wbad-function-cast -Wmissing-noreturn -FORMAT = -Wmissing-format-attribute -VISIBILITY= -fvisibility=hidden -FPICFLAGS = -fPIC -SHADOW = -Wshadow -SYMBOLS = -g3 -ggdb3 -PORTABILITY= -SKIP_WARNING= -COMBINE_C = -combine -c -LINK_FILES= -lz -lpthread -C99 = -std=c99 - -#Tools -VGRIND = valgrind --quiet --error-exitcode=1 --leak-check=yes --suppressions=../valgrind.suppressions - -ifeq ($(CC),icc) - #icc only: - OPTFLAGS = -O3 -ip -ipo2 - COMBINE_C = -ipo-c - FORMAT= #No argument for extra format warnings. - WALL = -Wall -Wcheck # '-Wextra' becomes '-Wcheck' in icc - SYMBOLS= -g -debug all -inline-debug-info - PORTABILITY=-diag-enable port-win - ifneq ($(CYGWIN),) - #Cygwin - ICC_NOWARN=-Qdiag-disable: - else - #Linux - ICC_NOWARN=-diag-disable #Need the space - endif - SKIP_WARNING += $(ICC_NOWARN)177 # Don't complain about static variables that are not used. - #SKIP_WARNING += $(ICC_NOWARN)188 # Don't complain about enumerated type mixed with another type. - SKIP_WARNING += $(ICC_NOWARN)589 # Don't complain about goto into a block that skips initializing variables. GCC catches the actual uninitialized variables. - SKIP_WARNING += $(ICC_NOWARN)869 # Don't complain about unused variables (since we defined __attribute__ to be nothing.) - SKIP_WARNING += $(ICC_NOWARN)981 # Don't complain about "operands are evaluated in unspecified order". This seems to be generated whenever more than one argument to a function or operand is computed by function call. - SKIP_WARNING += $(ICC_NOWARN)1324 # Don't complain about rdtsc clobbering its registers more than once. -endif - -ifneq ($(CYGWIN),) - #Cygwin (Windows) Must override some settings - CYG_ADD_LIBZ=/usr/lib/libz.a - VGRIND =#No Valgrind in cygwin - FPICFLAGS=#FPIC is default and not allowed as an option. - VISIBILITY=#Not supported - SHADOW=#Not supported - ifeq ($(CC),icc) - #Cygwin icc only - C99 = -Qstd=c99 - OPTFLAGS = -Ox -Qip -Qipo2 - COMBINE_C = -Qipo-c - WERROR = -WX # Windows icc version of -Werror - SYMBOLS= -Zi -debug:all -Qinline-debug-info - PORTABILITY= - LINK_FILES=#Not supported - SKIP_WARNING += $(ICC_NOWARN)1786 # Don't complain about 'read/write/other standards' being deprecated - else - #Cygwin gcc only - FORMAT = -Wno-format - endif -endif - -CFLAGS = $(WALL) $(WERROR) $(FORMAT) $(VISIBILITY) $(FPICFLAGS) $(SHADOW) -CFLAGS += $(OPTFLAGS) $(GCOV_FLAGS) $(PROF_FLAGS) -CFLAGS += $(SYMBOLS) $(SKIP_WARNING) $(C99) - -LDFLAGS = $(OPTFLAGS) $(SYMBOLS) $(GCOV_FLAGS) $(PROF_FLAGS) $(LINK_FILES) -# Need XOPEN_SOURCE=600 to get strtoll() -CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=600 - - - -CPPFLAGS += -D_GNU_SOURCE -I.. - -# Add -Wconversion - -HERE=newbrt/tests - -ifeq ($(SUMMARIZE),1) -SUMMARIZE_CMD = ;if test $$? = 0; then printf "%-60sPASS\n" $(HERE)/$@; else printf "%-60sFAIL\n" $(HERE)/$@ ; test 0 = 1; fi -else -SUMMARIZE_CMD = -endif # Put these one-per-line so that if we insert a new one the svn diff can understand it better. # Also keep them sorted. -REGRESSION_TESTS = \ +REGRESSION_TESTS_RAW = \ block_allocator_test \ bread-test \ brt-serialize-test \ @@ -121,7 +33,6 @@ REGRESSION_TESTS = \ brt-test3 \ brt-test4 \ brt-test5 \ - test-dump-brt \ cachetable-rwlock-test \ cachetable-writequeue-test \ threadpool-test \ @@ -136,8 +47,6 @@ REGRESSION_TESTS = \ cachetable-count-pinned-test \ cachetable-debug-test \ cachetable-debug-test \ - cachetable-unpin-and-remove-test \ - dup-delete-all \ fifo-test \ list-test \ keyrange \ @@ -167,89 +76,55 @@ REGRESSION_TESTS = \ # This line intentially kept commented so I can have a \ on the end of the previous line # Add in the binaries that must be run in various ways. -BINS = $(REGRESSION_TESTS) \ +BINS_RAW = $(REGRESSION_TESTS_RAW) \ benchmark-test \ cachetable-scan \ # This line intentially kept commented so I can have a \ on the end of the previous line +# BINS will be defined by adding .exe if appropriate. CHECKS = \ - benchmarktest_256 \ - test-assertA test-assertB \ - $(REGRESSION_TESTS) \ + benchmarktest_256 \ + test-assertA \ + test-assertB \ + $(REGRESSION_TESTS) \ # This line intentially kept commented so I can have a \ on the previous line - -ifeq ($(CIL),1) - CC=../../../cil/cil-1.3.6/bin/cilly --merge --keepmerged -endif +#CHECKS will be defined automatically. build: $(BINS) check: $(patsubst %,check_%,$(CHECKS)) check_fail: - test 0 = 1 $(SUMMARIZE_CMD) + $(MAYBEATSIGN)test 0 = 1 $(SUMMARIZE_CMD) check_ok: - test 0 = 0 $(SUMMARIZE_CMD) + $(MAYBEATSIGN)test 0 = 0 $(SUMMARIZE_CMD) -check_benchmarktest_256: benchmark-test - $(VGRIND) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1 $(SUMMARIZE_CMD) +check_benchmarktest_256: benchmark-test$(BINSUF) + $(MAYBEATSIGN)$(VGRIND) ./$< $(VERBVERBOSE) --valsize 256 --verify 1 $(SUMMARIZE_CMD) -check_test-assertA: test-assert +check_test-assertA: test-assert$(BINSUF) @# no arguments, should err - $(VGRIND) ./test-assert > /dev/null 2>&1 ; test $$? = 1 $(SUMMARIZE_CMD) -check_test-assertB: test-assert + $(MAYBEATSIGN)$(VGRIND) ./$< > /dev/null 2>&1 ; test $$? = 1 $(SUMMARIZE_CMD) +check_test-assertB: test-assert$(BINSUF) @# one argument, not "ok" should err @rm -f test-assert.out - ($(VGRIND) ./test-assert notok) > test-assert.out 2>&1 ; test $$? = 1 && fgrep failed test-assert.out > /dev/null $(SUMMARIZE_CMD) -check_test-assertC: tst-assert + $(MAYBEATSIGN)($(VGRIND) ./$< notok) > test-assert.out 2>&1 ; test $$? = 1 && fgrep failed test-assert.out > /dev/null $(SUMMARIZE_CMD) +check_test-assertC: test-assert$(BINSUF) +check_test-assert$(BINSUF): test-assert$(BINSUF) + @# one argument, "ok" should not error + $(MAYBEATSIGN)$(VGRIND) ./$< ok $(SUMMARIZE_CMD) +check_%$(BINSUF): %$(BINSUF) + $(MAYBEATSIGN)$(VGRIND) ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD) -check_test-assert: test-assert - @# one argument, "ok" should not error - $(DVGRIND) ./test-assert ok $(SUMMARIZE_CMD) - -check_%: % - $(VGRIND) ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD) - -benchmark-test.o: ../brt.h ../brt-search.h ../../include/db.h -ifeq ($(CIL),1) - -BRT_SOURCES = \ - block_allocator \ - bread \ - brt-serialize \ - brt-verify \ - brt \ - cachetable \ - fifo \ - fingerprint \ - key \ - leafentry \ - log \ - log_code \ - memory \ - memarena \ - mempool \ - omt \ - recover \ - roll \ - toku_assert \ - ybt \ - x1764 \ - trace_mem \ - threadpool \ -# keep this line so I can ha vea \ on the previous line -$(BINS): $(patsubst %,../%.o,$(BRT_SOURCES)) $(CYG_ADD_LIBZ) -else -$(BINS): ../newbrt.o $(CYG_ADD_LIBZ) -endif +benchmark-test.$(OEXT): ../brt.h ../brt-search.h ../../include/db.h -test-inc-split test-del-inorder: ../brt-test-helpers.o +LINK_FILES += ../newbrt.$(AEXT) clean: - rm -rf $(BINS) *.o *.bb *.bbg *.da *.gcov *.gcno *.gcda - rm -rf test_oexcl.c.tmp *.brt *.dir *.tdb *.dat *.out - rm -rf *.exe *.pdb *.ilk *.obj - rm -f cachetable-fd-test.ctest2.data + $(MAYBEATSIGN)rm -rf *.dir + $(MAYBEATSIGN)rm -f cachetable-fd-test.ctest2.data test_oexcl.c.tmp + $(MAYBEATSIGN)rm -f *.brt *.tdb *.dat *.data *.out + +foo: ../cachetable.$(OEXT) ../fifo.$(OEXT) ../fingerprint.$(OEXT) ../key.$(OEXT) ../memory.$(OEXT) ../memarena.$(OEXT) ../mempool.$(OEXT) ../omt.$(OEXT) ../toku_assert.$(OEXT) ../ybt.$(OEXT) ../x1764.$(OEXT) ../trace_mem.$(OEXT) ../threadpool.$(OEXT) -foo: ../cachetable.o ../fifo.o ../fingerprint.o ../key.o ../memory.o ../memarena.o ../mempool.o ../omt.o ../toku_assert.o ../ybt.o ../x1764.o ../trace_mem.o ../threadpool.o diff --git a/newbrt/tests/bread-test.c b/newbrt/tests/bread-test.c index 34cc621679a..78006c690c4 100644 --- a/newbrt/tests/bread-test.c +++ b/newbrt/tests/bread-test.c @@ -1,4 +1,4 @@ -#include "../portability.h" +#include "portability.h" #include #include diff --git a/newbrt/tests/cachetable-rename-test.c b/newbrt/tests/cachetable-rename-test.c index b404aae8851..b4e942234fb 100644 --- a/newbrt/tests/cachetable-rename-test.c +++ b/newbrt/tests/cachetable-rename-test.c @@ -8,22 +8,22 @@ // global data, especially between the test thread and the cachetable // writeback threads -pthread_mutex_t test_mutex; +toku_pthread_mutex_t test_mutex; static inline void test_mutex_init() { - int r = pthread_mutex_init(&test_mutex, 0); assert(r == 0); + int r = toku_pthread_mutex_init(&test_mutex, 0); assert(r == 0); } static inline void test_mutex_destroy() { - int r = pthread_mutex_destroy(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_destroy(&test_mutex); assert(r == 0); } static inline void test_mutex_lock() { - int r = pthread_mutex_lock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_lock(&test_mutex); assert(r == 0); } static inline void test_mutex_unlock() { - int r = pthread_mutex_unlock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_unlock(&test_mutex); assert(r == 0); } enum { KEYLIMIT = 4, TRIALLIMIT=256000 }; @@ -100,7 +100,7 @@ static void test_rename (void) { test_mutex_lock(); while (n_keys >= KEYLIMIT) { test_mutex_unlock(); - pthread_yield(); + toku_pthread_yield(); test_mutex_lock(); } assert(n_keyse = 0; ctpair_rwlock_init(&rwe->the_rwlock); - int r = pthread_mutex_init(&rwe->mutex, 0); assert(r == 0); + int r = toku_pthread_mutex_init(&rwe->mutex, 0); assert(r == 0); } static void rw_event_destroy (struct rw_event *rwe) { ctpair_rwlock_destroy(&rwe->the_rwlock); - int r = pthread_mutex_destroy(&rwe->mutex); assert(r == 0); + int r = toku_pthread_mutex_destroy(&rwe->mutex); assert(r == 0); } static void * @@ -74,15 +74,15 @@ test_writer_priority_thread (void *arg) { struct rw_event *rwe = arg; int r; - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_write_lock(&rwe->the_rwlock, &rwe->mutex); rwe->e++; assert(rwe->e == 3); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); rwe->e++; assert(rwe->e == 4); ctpair_write_unlock(&rwe->the_rwlock); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); return arg; } @@ -95,35 +95,35 @@ test_writer_priority (void) { int r; rw_event_init(rwe); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_read_lock(&rwe->the_rwlock, &rwe->mutex); sleep(1); rwe->e++; assert(rwe->e == 1); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); - pthread_t tid; - r = pthread_create(&tid, 0, test_writer_priority_thread, rwe); + toku_pthread_t tid; + r = toku_pthread_create(&tid, 0, test_writer_priority_thread, rwe); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); rwe->e++; assert(rwe->e == 2); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_read_unlock(&rwe->the_rwlock); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_read_lock(&rwe->the_rwlock, &rwe->mutex); rwe->e++; assert(rwe->e == 5); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_read_unlock(&rwe->the_rwlock); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); void *ret; - r = pthread_join(tid, &ret); assert(r == 0); + r = toku_pthread_join(tid, &ret); assert(r == 0); rw_event_destroy(rwe); } @@ -135,12 +135,12 @@ test_single_writer_thread (void *arg) { struct rw_event *rwe = arg; int r; - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_write_lock(&rwe->the_rwlock, &rwe->mutex); rwe->e++; assert(rwe->e == 3); assert(ctpair_writers(&rwe->the_rwlock) == 1); ctpair_write_unlock(&rwe->the_rwlock); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); return arg; } @@ -152,25 +152,25 @@ test_single_writer (void) { rw_event_init(rwe); assert(ctpair_writers(&rwe->the_rwlock) == 0); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); ctpair_write_lock(&rwe->the_rwlock, &rwe->mutex); assert(ctpair_writers(&rwe->the_rwlock) == 1); sleep(1); rwe->e++; assert(rwe->e == 1); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); - pthread_t tid; - r = pthread_create(&tid, 0, test_single_writer_thread, rwe); + toku_pthread_t tid; + r = toku_pthread_create(&tid, 0, test_single_writer_thread, rwe); sleep(1); - r = pthread_mutex_lock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwe->mutex); assert(r == 0); rwe->e++; assert(rwe->e == 2); assert(ctpair_writers(&rwe->the_rwlock) == 1); assert(ctpair_users(&rwe->the_rwlock) == 2); ctpair_write_unlock(&rwe->the_rwlock); - r = pthread_mutex_unlock(&rwe->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwe->mutex); assert(r == 0); void *ret; - r = pthread_join(tid, &ret); assert(r == 0); + r = toku_pthread_join(tid, &ret); assert(r == 0); assert(ctpair_writers(&rwe->the_rwlock) == 0); rw_event_destroy(rwe); diff --git a/newbrt/tests/cachetable-test.c b/newbrt/tests/cachetable-test.c index 7a3492dcbc0..a973eaec941 100644 --- a/newbrt/tests/cachetable-test.c +++ b/newbrt/tests/cachetable-test.c @@ -9,22 +9,22 @@ // global data, especially between the test thread and the cachetable // writeback threads -pthread_mutex_t test_mutex; +toku_pthread_mutex_t test_mutex; static inline void test_mutex_init() { - int r = pthread_mutex_init(&test_mutex, 0); assert(r == 0); + int r = toku_pthread_mutex_init(&test_mutex, 0); assert(r == 0); } static inline void test_mutex_destroy() { - int r = pthread_mutex_destroy(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_destroy(&test_mutex); assert(r == 0); } static inline void test_mutex_lock() { - int r = pthread_mutex_lock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_lock(&test_mutex); assert(r == 0); } static inline void test_mutex_unlock() { - int r = pthread_mutex_unlock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_unlock(&test_mutex); assert(r == 0); } #endif @@ -198,14 +198,14 @@ static void test0 (void) { expect1(2); /* 2 is the oldest unpinned item. */ r=toku_cachetable_put(f, make_blocknum(6), h6, make_item(6), test_object_size, flush, fetch, t3); /* 6P 5U 4P 3U 1P */ assert(r==0); - while (expect_n_flushes != 0) pthread_yield(); + while (expect_n_flushes != 0) toku_pthread_yield(); assert(expect_n_flushes==0); expect1(3); r=toku_cachetable_put(f, make_blocknum(7), h7, make_item(7), test_object_size, flush, fetch, t3); assert(r==0); - while (expect_n_flushes != 0) pthread_yield(); + while (expect_n_flushes != 0) toku_pthread_yield(); assert(expect_n_flushes==0); r=toku_cachetable_unpin(f, make_blocknum(7), h7, CACHETABLE_DIRTY, test_object_size); /* 7U 6P 5U 4P 1P */ assert(r==0); @@ -231,7 +231,7 @@ static void test0 (void) { assert(did_fetch.b==2); /* Expect that 2 is fetched in. */ assert(((struct item *)item_v)->key.b==2); assert(strcmp(((struct item *)item_v)->something,"something")==0); - while (expect_n_flushes != 0) pthread_yield(); + while (expect_n_flushes != 0) toku_pthread_yield(); assert(expect_n_flushes==0); } @@ -624,7 +624,7 @@ static void test_size_flush() { int n_entries, hash_size; long size_current, size_limit; toku_cachetable_get_state(t, &n_entries, &hash_size, &size_current, &size_limit); while (n_entries != min2(i+1, n)) { - pthread_yield(); + toku_pthread_yield(); toku_cachetable_get_state(t, &n_entries, 0, 0, 0); } assert(n_entries == min2(i+1, n)); diff --git a/newbrt/tests/cachetable-test2.c b/newbrt/tests/cachetable-test2.c index b07b8845974..797d0c03978 100644 --- a/newbrt/tests/cachetable-test2.c +++ b/newbrt/tests/cachetable-test2.c @@ -8,22 +8,22 @@ // global data, especially between the test thread and the cachetable // writeback threads -pthread_mutex_t test_mutex; +toku_pthread_mutex_t test_mutex; static inline void test_mutex_init() { - int r = pthread_mutex_init(&test_mutex, 0); assert(r == 0); + int r = toku_pthread_mutex_init(&test_mutex, 0); assert(r == 0); } static inline void test_mutex_destroy() { - int r = pthread_mutex_destroy(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_destroy(&test_mutex); assert(r == 0); } static inline void test_mutex_lock() { - int r = pthread_mutex_lock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_lock(&test_mutex); assert(r == 0); } static inline void test_mutex_unlock() { - int r = pthread_mutex_unlock(&test_mutex); assert(r == 0); + int r = toku_pthread_mutex_unlock(&test_mutex); assert(r == 0); } static const int test_object_size = 1; @@ -48,7 +48,7 @@ static void print_ints(void) { } static void item_becomes_present(CACHEFILE cf, CACHEKEY key) { - while (n_present >= N_PRESENT_LIMIT) pthread_yield(); + while (n_present >= N_PRESENT_LIMIT) toku_pthread_yield(); test_mutex_lock(); assert(n_presentwritequeue); - int r = pthread_mutex_init(&wqm->mutex, 0); assert(r == 0); + int r = toku_pthread_mutex_init(&wqm->mutex, 0); assert(r == 0); } static void writequeue_with_mutex_destroy (struct writequeue_with_mutex *wqm) { writequeue_destroy(&wqm->writequeue); - int r = pthread_mutex_destroy(&wqm->mutex); assert(r == 0); + int r = toku_pthread_mutex_destroy(&wqm->mutex); assert(r == 0); } static void * @@ -95,11 +95,11 @@ test_set_closed_waiter(void *arg) { struct writequeue_with_mutex *wqm = arg; int r; - r = pthread_mutex_lock(&wqm->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&wqm->mutex); assert(r == 0); PAIR p; r = writequeue_deq(&wqm->writequeue, &wqm->mutex, &p); assert(r == EINVAL); - r = pthread_mutex_unlock(&wqm->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&wqm->mutex); assert(r == 0); return arg; } @@ -109,12 +109,12 @@ test_set_closed_thread (void) { int r; writequeue_with_mutex_init(wqm); - pthread_t tid; - r = pthread_create(&tid, 0, test_set_closed_waiter, wqm); assert(r == 0); + toku_pthread_t tid; + r = toku_pthread_create(&tid, 0, test_set_closed_waiter, wqm); assert(r == 0); sleep(1); writequeue_set_closed(&wqm->writequeue); void *ret; - r = pthread_join(tid, &ret); + r = toku_pthread_join(tid, &ret); assert(r == 0 && ret == wqm); writequeue_with_mutex_destroy(wqm); } @@ -126,14 +126,14 @@ test_set_closed_thread (void) { // writers when the wq size <= 1/2 of the wq limit struct rwfc { - pthread_mutex_t mutex; + toku_pthread_mutex_t mutex; struct writequeue writequeue; int current, limit; }; static void rwfc_init (struct rwfc *rwfc, int limit) { int r; - r = pthread_mutex_init(&rwfc->mutex, 0); assert(r == 0); + r = toku_pthread_mutex_init(&rwfc->mutex, 0); assert(r == 0); writequeue_init(&rwfc->writequeue); rwfc->current = 0; rwfc->limit = limit; } @@ -142,7 +142,7 @@ static void rwfc_destroy (struct rwfc *rwfc) { int r; writequeue_destroy(&rwfc->writequeue); - r = pthread_mutex_destroy(&rwfc->mutex); assert(r == 0); + r = toku_pthread_mutex_destroy(&rwfc->mutex); assert(r == 0); } static void * @@ -151,16 +151,16 @@ rwfc_reader (void *arg) { int r; while (1) { PAIR ctpair; - r = pthread_mutex_lock(&rwfc->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwfc->mutex); assert(r == 0); r = writequeue_deq(&rwfc->writequeue, &rwfc->mutex, &ctpair); if (r == EINVAL) { - r = pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); break; } if (2*rwfc->current-- > rwfc->limit && 2*rwfc->current <= rwfc->limit) { writequeue_wakeup_write(&rwfc->writequeue); } - r = pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); destroy_pair(ctpair); usleep(random() % 100); } @@ -172,25 +172,25 @@ test_flow_control (int limit, int n) { struct rwfc my_rwfc, *rwfc = &my_rwfc; int r; rwfc_init(rwfc, limit); - pthread_t tid; - r = pthread_create(&tid, 0, rwfc_reader, rwfc); assert(r == 0); + toku_pthread_t tid; + r = toku_pthread_create(&tid, 0, rwfc_reader, rwfc); assert(r == 0); sleep(1); // this is here to block the reader on the first deq int i; for (i=0; imutex); assert(r == 0); + r = toku_pthread_mutex_lock(&rwfc->mutex); assert(r == 0); writequeue_enq(&rwfc->writequeue, ctpair); rwfc->current++; while (rwfc->current >= rwfc->limit) { // printf("%d - %d %d\n", i, rwfc->current, rwfc->limit); writequeue_wait_write(&rwfc->writequeue, &rwfc->mutex); } - r = pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); + r = toku_pthread_mutex_unlock(&rwfc->mutex); assert(r == 0); // usleep(random() % 1); } writequeue_set_closed(&rwfc->writequeue); void *ret; - r = pthread_join(tid, &ret); assert(r == 0); + r = toku_pthread_join(tid, &ret); assert(r == 0); rwfc_destroy(rwfc); } diff --git a/newbrt/tests/threadpool-test.c b/newbrt/tests/threadpool-test.c index 60ebfcbd9cd..e6b479245f3 100644 --- a/newbrt/tests/threadpool-test.c +++ b/newbrt/tests/threadpool-test.c @@ -5,15 +5,15 @@ #include #include #include -#include +#include #include "threadpool.h" int verbose = 0; struct my_threadpool { THREADPOOL threadpool; - pthread_mutex_t mutex; - pthread_cond_t wait; + toku_pthread_mutex_t mutex; + toku_pthread_cond_t wait; int closed; }; @@ -22,23 +22,23 @@ my_threadpool_init (struct my_threadpool *my_threadpool, int max_threads) { int r; r = threadpool_create(&my_threadpool->threadpool, max_threads); assert(r == 0); assert(my_threadpool != 0); - r = pthread_mutex_init(&my_threadpool->mutex, 0); assert(r == 0); - r = pthread_cond_init(&my_threadpool->wait, 0); assert(r == 0); + r = toku_pthread_mutex_init(&my_threadpool->mutex, 0); assert(r == 0); + r = toku_pthread_cond_init(&my_threadpool->wait, 0); assert(r == 0); my_threadpool->closed = 0; } static void my_threadpool_destroy (struct my_threadpool *my_threadpool) { int r; - r = pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); my_threadpool->closed = 1; - r = pthread_cond_broadcast(&my_threadpool->wait); assert(r == 0); - r = pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); + r = toku_pthread_cond_broadcast(&my_threadpool->wait); assert(r == 0); + r = toku_pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); if (verbose) printf("current %d\n", threadpool_get_current_threads(my_threadpool->threadpool)); threadpool_destroy(&my_threadpool->threadpool); assert(my_threadpool->threadpool == 0); - r = pthread_mutex_destroy(&my_threadpool->mutex); assert(r == 0); - r = pthread_cond_destroy(&my_threadpool->wait); assert(r == 0); + r = toku_pthread_mutex_destroy(&my_threadpool->mutex); assert(r == 0); + r = toku_pthread_cond_destroy(&my_threadpool->wait); assert(r == 0); } static void * @@ -46,12 +46,12 @@ fbusy (void *arg) { struct my_threadpool *my_threadpool = arg; int r; - r = pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); while (!my_threadpool->closed) { - r = pthread_cond_wait(&my_threadpool->wait, &my_threadpool->mutex); assert(r == 0); + r = toku_pthread_cond_wait(&my_threadpool->wait, &my_threadpool->mutex); assert(r == 0); } - r = pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); - if (verbose) printf("%lu:%s:exit\n", pthread_self(), __FUNCTION__); + r = toku_pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); + if (verbose) printf("%lu:%s:exit\n", toku_pthread_self(), __FUNCTION__); return arg; } @@ -60,13 +60,13 @@ fidle (void *arg) { struct my_threadpool *my_threadpool = arg; int r; - r = pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); + r = toku_pthread_mutex_lock(&my_threadpool->mutex); assert(r == 0); threadpool_set_thread_idle(my_threadpool->threadpool); while (!my_threadpool->closed) { - r = pthread_cond_wait(&my_threadpool->wait, &my_threadpool->mutex); assert(r == 0); + r = toku_pthread_cond_wait(&my_threadpool->wait, &my_threadpool->mutex); assert(r == 0); } - r = pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); - if (verbose) printf("%lu:%s:exit\n", pthread_self(), __FUNCTION__); + r = toku_pthread_mutex_unlock(&my_threadpool->mutex); assert(r == 0); + if (verbose) printf("%lu:%s:exit\n", toku_pthread_self(), __FUNCTION__); return arg; } diff --git a/newbrt/threadpool.c b/newbrt/threadpool.c index aa52a59e0bf..7bf5f98bf76 100644 --- a/newbrt/threadpool.c +++ b/newbrt/threadpool.c @@ -7,11 +7,11 @@ struct threadpool { int max_threads; int current_threads; int busy_threads; - pthread_t pids[]; + toku_pthread_t pids[]; }; int threadpool_create(THREADPOOL *threadpoolptr, int max_threads) { - size_t size = sizeof (struct threadpool) + max_threads*sizeof (pthread_t); + size_t size = sizeof (struct threadpool) + max_threads*sizeof (toku_pthread_t); struct threadpool *threadpool = malloc(size); if (threadpool == 0) return ENOMEM; @@ -30,7 +30,7 @@ void threadpool_destroy(THREADPOOL *threadpoolptr) { int i; for (i=0; icurrent_threads; i++) { int r; void *ret; - r = pthread_join(threadpool->pids[i], &ret); + r = toku_pthread_join(threadpool->pids[i], &ret); assert(r == 0); } *threadpoolptr = 0; @@ -39,7 +39,7 @@ void threadpool_destroy(THREADPOOL *threadpoolptr) { void threadpool_maybe_add(THREADPOOL threadpool, void *(*f)(void *), void *arg) { if ((threadpool->current_threads == 0 || threadpool->busy_threads < threadpool->current_threads) && threadpool->current_threads < threadpool->max_threads) { - int r = pthread_create(&threadpool->pids[threadpool->current_threads], 0, f, arg); + int r = toku_pthread_create(&threadpool->pids[threadpool->current_threads], 0, f, arg); if (r == 0) { threadpool->current_threads++; threadpool_set_thread_busy(threadpool); diff --git a/windows/Makefile b/windows/Makefile new file mode 100644 index 00000000000..70064ae98c2 --- /dev/null +++ b/windows/Makefile @@ -0,0 +1,24 @@ +# -*- Mode: Makefile -*- + +.DEFAULT_GOAL=install +TOKUROOT=../ +INCLUDEDIRS=-I. +SKIP_LIBPORTABILITYRULE=1 +include $(TOKUROOT)include/Makefile.include + +OPT_AROPT=-qnoipo #Disable ipo for lib creation even when optimization is on. + +SRCS = $(wildcard *.c) +OBJS = $(patsubst %.c,%.$(OEXT),$(SRCS)) +TARGET = tokuwindows.$(AEXT) + +install: $(LIBPORTABILITY) + +$(LIBPORTABILITY): $(TARGET) + $(MAYBEATSIGN)cp $< $@ + +$(TARGET): $(OBJS) + +clean: + $(MAYBEATSIGN)rm -rf $(TARGET) $(LIBPORTABILITY) + diff --git a/windows/dirent.h b/windows/dirent.h new file mode 100644 index 00000000000..4a01d1dc65e --- /dev/null +++ b/windows/dirent.h @@ -0,0 +1,18 @@ +#ifndef _TOKU_DIRENT_H +#define _TOKU_DIRENT_H + +//The DIR functions do not exist in windows, but the Linux API ends up +//just using a wrapper. We might convert these into an os_* type api. + +DIR *opendir(const char *name); + +struct dirent *readdir(DIR *dir); + +int closedir(DIR *dir); + +#ifndef NAME_MAX +#define NAME_MAX 255 +#endif + +#endif + diff --git a/windows/inttypes.h b/windows/inttypes.h new file mode 100644 index 00000000000..7026b42a16a --- /dev/null +++ b/windows/inttypes.h @@ -0,0 +1,13 @@ +#ifndef _INTTYPES_H +#define _INTTYPES_H + +#include + +//Define printf types. +#define PRId64 "I64d" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRId32 "d" +#define PRIu32 "u" + +#endif diff --git a/windows/misc.h b/windows/misc.h new file mode 100644 index 00000000000..79fb689e831 --- /dev/null +++ b/windows/misc.h @@ -0,0 +1,72 @@ +#ifndef _MISC_H +#define _MISC_H +#include "os.h" +#include + +//These are functions that really exist in windows but are named +//something else. +//TODO: Sort these into some .h file that makes sense. + +int fsync(int fildes); + +int gettimeofday(struct timeval *tv, struct timezone *tz); + +long long int strtoll(const char *nptr, char **endptr, int base); + + +//TODO: Enforce use of these macros. Otherwise, open, creat, and chmod may fail +//os_mkdir actually ignores the permissions, so it won't fail. + +//Permissions +//User permissions translate to global +//Execute bit does not exist +//TODO: Determine if we need to use BINARY mode for opening. +#define S_IRWXU S_IRUSR | S_IWUSR | S_IXUSR +#define S_IRUSR S_IREAD +#define S_IWUSR S_IWRITE +//Execute bit does not exist +#define S_IXUSR (0) + +//Group permissions thrown away. +#define S_IRWXG S_IRGRP | S_IWGRP | S_IXGRP +#define S_IRGRP (0) +#define S_IWGRP (0) +#define S_IXGRP (0) + +//Other permissions thrown away. (Except for read) +//MySQL defines S_IROTH as S_IREAD. Avoid the warning. +#if defined(S_IROTH) +#undef S_IROTH +#endif +#define S_IRWXO S_IROTH | S_IWOTH | S_IXOTH +#define S_IROTH S_IREAD +#define S_IWOTH (0) +#define S_IXOTH (0) + +long int random(void); +void srandom(unsigned int seed); + +//snprintf +//TODO: Put in its own file, or define a snprintf function based on _vsnprintf +//in its own file +#define snprintf(str, size, fmt, ...) _snprintf(str, size, fmt, __VA_ARGS__) + +//strtoll has a different name in windows. +#define strtoll _strtoi64 +#define strtoull _strtoui64 + +//rmdir has a different name in windows. +#define rmdir _rmdir + +#ifndef PATH_MAX +#define PATH_MAX 1 +#endif + +char *realpath(const char *path, char *resolved_path); + + +int unsetenv(const char *name); +int setenv(const char *name, const char *value, int overwrite); + +#endif + diff --git a/windows/os-types.h b/windows/os-types.h new file mode 100644 index 00000000000..bfa325d7871 --- /dev/null +++ b/windows/os-types.h @@ -0,0 +1,32 @@ +#if !defined(OS_INTERFACE_WINDOWS_H) +#define OS_INTERFACE_WINDOWS_H +#include +#include + +// define an OS handle +typedef void *os_handle_t; +typedef int pid_t; +typedef int mode_t; + +struct fileid { + uint32_t st_dev; + uint64_t st_ino; + uint64_t st_creat; +}; + +enum { + DT_UNKNOWN = 0, + DT_DIR = 4, + DT_REG = 8 +}; + +struct dirent { + char d_name[_MAX_PATH]; + unsigned char d_type; +}; + +struct __toku_windir; +typedef struct __toku_windir DIR; + +#endif + diff --git a/windows/rss.c b/windows/rss.c new file mode 100644 index 00000000000..12b6201a792 --- /dev/null +++ b/windows/rss.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +#define DO_MEMORY_INFO 1 + +#if DO_MEMORY_INFO +#include + +static int +get_memory_info(PROCESS_MEMORY_COUNTERS *meminfo) { + int r; + + r = GetProcessMemoryInfo(GetCurrentProcess(), meminfo, sizeof *meminfo); + if (r == 0) + return GetLastError(); + return 0; +} + +#endif + +int +os_get_rss(int64_t *rss) { + int r; +#if DO_MEMORY_INFO + PROCESS_MEMORY_COUNTERS meminfo; + + r = get_memory_info(&meminfo); + if (r == 0) + *rss = meminfo.WorkingSetSize; +#else + r = 0; + *rss = 0; +#endif + return r; +} + +int +os_get_max_rss(int64_t *maxrss) { + int r; +#if DO_MEMORY_INFO + PROCESS_MEMORY_COUNTERS meminfo; + + r = get_memory_info(&meminfo); + if (r == 0) + *maxrss = meminfo.PeakWorkingSetSize; +#else + r = 0; + *maxrss = 0; +#endif + return r; +} + diff --git a/windows/stdint.h b/windows/stdint.h new file mode 100644 index 00000000000..01f9a39fc75 --- /dev/null +++ b/windows/stdint.h @@ -0,0 +1,37 @@ +#ifndef _STDINT_H +#define _STDINT_H +#include +#include + +//Define standard integer types. +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef unsigned __int8 u_int8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int16 u_int16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int32 u_int32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +typedef unsigned __int64 u_int64_t; + +typedef SSIZE_T ssize_t; + +//Limits +#define INT8_MIN _I8_MIN +#define INT8_MAX _I8_MAX +#define UINT8_MAX _UI8_MAX +#define INT16_MIN _I16_MIN +#define INT16_MAX _I16_MAX +#define UINT16_MAX _UI16_MAX +#define INT32_MIN _I32_MIN +#define INT32_MAX _I32_MAX +#define UINT32_MAX _UI32_MAX +#define INT64_MIN _I64_MIN +#define INT64_MAX _I64_MAX +#define UINT64_MAX _UI64_MAX + +#endif + diff --git a/windows/tests/Makefile b/windows/tests/Makefile new file mode 100644 index 00000000000..c5b55b90b7d --- /dev/null +++ b/windows/tests/Makefile @@ -0,0 +1,17 @@ +# -*- Mode: Makefile -*- + +.DEFAULT_GOAL=all +TOKUROOT=../../ +INCLUDEDIRS=-I$(TOKUROOT)include/windows -I$(TOKUROOT)newbrt +include $(TOKUROOT)include/Makefile.include + +SKIP_WARNING += $(ICC_NOWARN)1418 #Non static functions do not need prototypes. + +SRCS = $(wildcard test-*.c) +BINS_RAW = $(patsubst %.c,%,$(SRCS)) +#Bins will be generated. +$(BINS): $(LIBPORTABILITY) +$(BINS): CFLAGS+=-DTESTDIR=\"dir.$<.dir\" + +all: $(BINS) + diff --git a/windows/tests/test-dirs.c b/windows/tests/test-dirs.c new file mode 100644 index 00000000000..730e41c9f50 --- /dev/null +++ b/windows/tests/test-dirs.c @@ -0,0 +1,81 @@ +#define _CRT_SECURE_NO_DEPRECATE +#include +#include +#include +#include +#include +#include "portability.h" +#include "os.h" +#include + +int verbose; + +static int walk(const char *dirname) { + DIR *d; + struct dirent *dirent; + int dotfound = 0, dotdotfound = 0, otherfound = 0; + + d = opendir(dirname); + if (d == NULL) + return -1; + while ((dirent = readdir(d))) { + if (verbose) + printf("%p %s\n", dirent, dirent->d_name); + if (strcmp(dirent->d_name, ".") == 0) + dotfound++; + else if (strcmp(dirent->d_name, "..") == 0) + dotdotfound++; + else + otherfound++; + } + closedir(d); + assert(dotfound == 1 && dotdotfound == 1); + return otherfound; +} + +int main(int argc, char *argv[]) { + int i; + int found; + int fd; + int r; + + for (i=1; i= 0); + close(fd); + } + found = walk(TESTDIR); + assert(found == N); + + // walk and remove files + + return 0; +} + diff --git a/windows/tests/test-file-truncate b/windows/tests/test-file-truncate new file mode 100644 index 00000000000..e69de29bb2d diff --git a/windows/tests/test-fileid.c b/windows/tests/test-fileid.c new file mode 100644 index 00000000000..06829d7c9a7 --- /dev/null +++ b/windows/tests/test-fileid.c @@ -0,0 +1,43 @@ + +#include +#include +#include +#include +#include "portability.h" +#include "os.h" + +int verbose=0; + +//TODO: Test that different files are different, +// other stuff +static void test_handles(const char *fname) { + unlink(fname); + int fd = open(fname, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); + assert(fd!=-1); + int i; + struct fileid id_base; + struct fileid id; + int r = os_get_unique_file_id(fd, &id_base); + assert(r==0); + for (i=0; i < 1<<16; i++) { + r = os_get_unique_file_id(fd, &id); + assert(r==0); + assert(memcmp(&id, &id_base, sizeof(id))==0); + } + r = close(fd); + assert(r==0); +} + +int main(int argc, char *argv[]) { + int i; + + for (i=1; i +#include +#include +#include +#include +#include +#include +#include + +#if 0 && defined _WIN32 +#include +static int ftruncate(int fd, uint64_t offset) { + HANDLE h = (HANDLE) _get_osfhandle(fd); + printf("%s:%d %p\n", __FILE__, __LINE__, h); fflush(stdout); + if (h == INVALID_HANDLE_VALUE) + return -1; + int r = _lseeki64(fd, 0, SEEK_SET); + printf("%s:%d %d\n", __FILE__, __LINE__, r); fflush(stdout); + if (r != 0) + return -2; + BOOL b = SetEndOfFile(h); + printf("%s:%d %d\n", __FILE__, __LINE__, b); fflush(stdout); + if (!b) + return -3; + return 0; +} +#endif + +int main(void) { + int r; + int fd; + + fd = open("test-file-truncate", O_CREAT+O_RDWR+O_TRUNC, S_IREAD+S_IWRITE); + assert(fd != -1); + + int i; + for (i=0; i<32; i++) { + char junk[4096]; + memset(junk, 0, sizeof junk); + r = write(fd, junk, sizeof junk); + assert(r == sizeof junk); + } + + struct stat filestat; + r = fstat(fd, &filestat); + assert(r == 0); + + printf("orig size %lu\n", (unsigned long) filestat.st_size); fflush(stdout); + + r = ftruncate(fd, 0); + assert(r == 0); + + r = fstat(fd, &filestat); + assert(r == 0); + + printf("truncated size %lu\n", (unsigned long) filestat.st_size); fflush(stdout); + assert(filestat.st_size == 0); + + return 0; +} diff --git a/windows/tests/test-open-dir.c b/windows/tests/test-open-dir.c new file mode 100644 index 00000000000..b18dd27dbef --- /dev/null +++ b/windows/tests/test-open-dir.c @@ -0,0 +1,16 @@ +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + int i; + for (i=1; i= 0) close(fd); + } + return 0; +} diff --git a/windows/tests/test-open-rename.c b/windows/tests/test-open-rename.c new file mode 100644 index 00000000000..353a6c3f987 --- /dev/null +++ b/windows/tests/test-open-rename.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include + +#ifndef S_IRUSR +#define S_IRUSR S_IREAD +#endif +#ifndef S_IWUSR +#define S_IWUSR S_IWRITE +#endif + +#define TESTFILE "test-open-unlink-file" +#define NEWNAME TESTFILE ".junk" + +int main(void) { + int r; + int fd; + + system("rm -rf test-open-unlink-file"); + + fd = open(TESTFILE, O_CREAT+O_RDWR, S_IRUSR+S_IWUSR); + assert(fd != -1); + + r = rename(TESTFILE, NEWNAME); + printf("%s:%d rename %d %d\n", __FILE__, __LINE__, r, errno); fflush(stdout); +#if defined(__linux__) + assert(r == 0); + + r = close(fd); + assert(r == 0); +#endif +#if defined(_WIN32) + assert(r == -1); + + r = close(fd); + assert(r == 0); + + r = rename(TESTFILE, NEWNAME); + printf("%s:%d rename %d %d\n", __FILE__, __LINE__, r, errno); fflush(stdout); + assert(r == 0); +#endif + + return 0; +} diff --git a/windows/tests/test-open-unlink.c b/windows/tests/test-open-unlink.c new file mode 100644 index 00000000000..7f6ee952fea --- /dev/null +++ b/windows/tests/test-open-unlink.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include + +#ifndef S_IRUSR +#define S_IRUSR S_IREAD +#endif +#ifndef S_IWUSR +#define S_IWUSR S_IWRITE +#endif + +const char TESTFILE[] = "test-open-unlink-file"; + +int main(void) { + int r; + int fd; + + system("rm -rf test-open-unlink-file"); + + fd = open(TESTFILE, O_CREAT+O_RDWR, S_IRUSR+S_IWUSR); + assert(fd != -1); + + r = unlink(TESTFILE); + printf("%s:%d unlink %d %d\n", __FILE__, __LINE__, r, errno); fflush(stdout); +#if defined(__linux__) + assert(r == 0); + + r = close(fd); + assert(r == 0); +#endif +#if defined(_WIN32) + assert(r == -1); + + r = close(fd); + assert(r == 0); + + r = unlink(TESTFILE); + printf("%s:%d unlink %d %d\n", __FILE__, __LINE__, r, errno); fflush(stdout); + assert(r == 0); +#endif + + return 0; +} diff --git a/windows/tests/test-pread.c b/windows/tests/test-pread.c new file mode 100644 index 00000000000..acabf4fc2a1 --- /dev/null +++ b/windows/tests/test-pread.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include "portability.h" +#include "os.h" + +int verbose; + +static void test_pread_empty(const char *fname) { + int fd; + char c[12]; + uint64_t r; + + unlink(fname); + fd = open(fname, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO); + if (verbose) + printf("open %s fd %d\n", fname, fd); + assert(fd != -1); + r = pread(fd, c, sizeof c, 0); + assert(r == 0); + r = close(fd); + if (verbose) + printf("close %s %"PRIu64"\n", fname, r); +} + +int main(int argc, char *argv[]) { + int i; + + for (i=1; i +#include +#include +#include +#include +#include "toku_pthread.h" + +struct q { + toku_pthread_mutex_t m; + toku_pthread_cond_t r; + toku_pthread_cond_t w; + void *item; +}; + +static void q_init(struct q *q) { + toku_pthread_mutex_init(&q->m, NULL); + toku_pthread_cond_init(&q->r, NULL); + toku_pthread_cond_init(&q->w, NULL); + q->item = NULL; +} + +static void q_destroy(struct q *q) { + toku_pthread_cond_destroy(&q->w); + toku_pthread_cond_destroy(&q->r); + toku_pthread_mutex_destroy(&q->m); +} + +static void *q_get(struct q *q) { + void *item; + toku_pthread_mutex_lock(&q->m); + while (q->item == NULL) + toku_pthread_cond_wait(&q->r, &q->m); + item = q->item; q->item = NULL; + toku_pthread_mutex_unlock(&q->m); + toku_pthread_cond_signal(&q->w); + return item; +} + +static void q_put(struct q *q, void *item) { + toku_pthread_mutex_lock(&q->m); + while (q->item != NULL) + toku_pthread_cond_wait(&q->w, &q->m); + q->item = item; + toku_pthread_mutex_unlock(&q->m); + toku_pthread_cond_signal(&q->r); +} + +static void *writer(void *arg) { + struct q *q = arg; + int i; + + printf("%s %p %lu\n", __FUNCTION__, arg, GetCurrentThreadId()); + for (i=0; i<100; i++) + q_put(q, (void*)(i+1)); + return arg; +} + +static void *reader(void *arg) { + struct q *q = arg; + int i; + + printf("%s %p %lu\n", __FUNCTION__, arg, GetCurrentThreadId()); + for (i=0; i<100; i++) { + void *item = q_get(q); + printf("%p\n", item); fflush(stdout); + Sleep(i); + } + return arg; +} + +int main(void) { + int i; + void *ret; + toku_pthread_t t[2]; + struct q q; + + q_init(&q); + toku_pthread_create(&t[0], NULL, reader, &q); + toku_pthread_create(&t[1], NULL, writer, &q); + for (i=0; i<2; i++) + toku_pthread_join(t[i], &ret); + q_destroy(&q); + return 0; +} + diff --git a/windows/tests/test-pthread-create.c b/windows/tests/test-pthread-create.c new file mode 100644 index 00000000000..551c10490d1 --- /dev/null +++ b/windows/tests/test-pthread-create.c @@ -0,0 +1,37 @@ +#include +#include +#include +#include +#include +#include "toku_pthread.h" + +static void *myfunc1(void *arg) { + printf("%s %p %lu\n", __FUNCTION__, arg, GetCurrentThreadId()); + fflush(stdout); + Sleep(10*1000); + return arg; +} + +static void *myfunc2(void *arg) { + printf("%s %p %lu\n", __FUNCTION__, arg, GetCurrentThreadId()); + fflush(stdout); + Sleep(10*1000); + return arg; +} + +int main(void) { +#define N 10 + toku_pthread_t t[N]; + int i; + + for (i=0; i +#include +#include +#include +#include +#include +#include + +static void do_mallocs(void) { + int i; + for (i=0; i<1000; i++) { + int nbytes = 1024*1024; + void *vp = malloc(nbytes); + memset(vp, 0, nbytes); + } +} + +int main(void) { + int64_t rss; + + os_get_max_rss(&rss); + printf("%I64d\n", rss); + do_mallocs(); + os_get_max_rss(&rss); + printf("%I64d\n", rss); + + return 0; +} + + diff --git a/windows/tests/test-sleep.c b/windows/tests/test-sleep.c new file mode 100644 index 00000000000..f18608520dc --- /dev/null +++ b/windows/tests/test-sleep.c @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include "portability.h" +#include "os.h" + +int verbose; + +int main(int argc, char *argv[]) { + int i; + + for (i=1; i +#include +#include +#include +#include +#include + +void test_stat(char *dirname) { + int r; + struct stat s; + r = stat(dirname, &s); + printf("stat %s %d\n", dirname, r); +} + +int main(void) { + int r; + + test_stat("."); + + r = os_mkdir("testdir", S_IRWXU); + assert(r == 0); + + test_stat("testdir"); + + test_stat("./testdir"); + + test_stat("./testdir/"); + + return 0; +} diff --git a/windows/tests/test-strtoll.c b/windows/tests/test-strtoll.c new file mode 100644 index 00000000000..ea3941ce3eb --- /dev/null +++ b/windows/tests/test-strtoll.c @@ -0,0 +1,42 @@ +#include +#include +#include +#include "portability.h" +#include "os.h" + +int verbose; + +void testit(int64_t i, int base) { + int64_t o; +#define SN 32 + char s[SN]; + sprintf(s, "%I64d", i); + o = strtoll(s, NULL, base); + if (verbose) + printf("%s: %I64d %I64d %s\n", __FUNCTION__, i, o, s); + assert(i == o); +} + +int main(int argc, char *argv[]) { + int i; + int64_t n; + int64_t o; +#define SN 32 + char s[SN]; + + for (i=1; i +#include +#include +#include +#include "portability.h" + +int main(void) { + int r; + int fd; + struct fileid fid; + + fd = open(DEV_NULL_FILE, O_RDWR); + assert(fd != -1); + + r = os_get_unique_file_id(fd, &fid); + printf("%s:%d %d\n", __FILE__, __LINE__, r); + + r = close(fd); + assert(r != -1); + return 0; +} diff --git a/windows/tests/test-usleep-ws.c b/windows/tests/test-usleep-ws.c new file mode 100644 index 00000000000..a93d29d4e26 --- /dev/null +++ b/windows/tests/test-usleep-ws.c @@ -0,0 +1,44 @@ +#include +#include +#include +#include +#include +#include +int verbose; + +int usleep(SOCKET s, unsigned int useconds) { + fd_set dummy; + struct timeval tv; + FD_ZERO(&dummy); + FD_SET(s, &dummy); + tv.tv_sec = useconds / 1000000; + tv.tv_usec = useconds % 1000000; + return select(0, 0, 0, &dummy, &tv); +} + +int main(int argc, char *argv[]) { + int i; + int n = 1; + WSADATA wsadata; + SOCKET s; + + for (i=1; i +#include +#include +#include +#include "portability.h" +#include "os.h" + +int verbose; + +int main(int argc, char *argv[]) { + int i; + int n = 1; + + for (i=1; i +#include +#include +#include + +int +toku_pthread_mutex_init(toku_pthread_mutex_t *mutex, const toku_pthread_mutexattr_t *attr) { + assert(attr == NULL); + // assert(!mutex->initialized); + InitializeCriticalSection(&mutex->section); + mutex->initialized = TRUE; + return 0; +} + +int +toku_pthread_mutex_destroy(toku_pthread_mutex_t *mutex) { + assert(mutex->initialized); + DeleteCriticalSection(&mutex->section); + mutex->initialized = FALSE; + return 0; +} + +int +toku_pthread_mutex_lock(toku_pthread_mutex_t *mutex) { + if (!mutex->initialized) { + int r = toku_pthread_mutex_init(mutex, NULL); + assert(r==0); + } + assert(mutex->initialized); + EnterCriticalSection(&mutex->section); + return 0; +} + +#if 0 +int +toku_pthread_mutex_trylock(toku_pthread_mutex_t *mutex) { + int r = 0; + if (!TryEnterCriticalSection(&mutex->section)) + r = EBUSY; + return r; +} +#endif + +int +toku_pthread_mutex_unlock(toku_pthread_mutex_t *mutex) { + assert(mutex->initialized); + LeaveCriticalSection(&mutex->section); + return 0; +} + +int +toku_pthread_attr_init(toku_pthread_attr_t *attr) { + attr->stacksize = 0; + return 0; +} + +int +toku_pthread_attr_destroy(toku_pthread_attr_t *attr) { + attr = attr; + return 0; +} + +int +toku_pthread_attr_setstacksize(toku_pthread_attr_t *attr, size_t stacksize) { + attr->stacksize = stacksize; + return 0; +} + +int +toku_pthread_attr_getstacksize(toku_pthread_attr_t *attr, size_t *stacksize) { + *stacksize = attr->stacksize; + return 0; +} + + +static DWORD WINAPI +toku_pthread_start(LPVOID a) { + toku_pthread_t thread = (toku_pthread_t) a; + thread->ret = thread->f(thread->arg); + return 0; +} + +int +toku_pthread_create(toku_pthread_t *threadptr, const toku_pthread_attr_t *attr, void *(*f)(void *), void *arg) { + size_t stacksize = 0; + toku_pthread_t thread = malloc(sizeof (struct toku_pthread)); + if (thread == 0) + return ENOMEM; + if (attr) + stacksize = attr->stacksize; + thread->f = f; + thread->arg = arg; + // _beginthread or CreateThread + thread->handle = CreateThread(NULL, stacksize, toku_pthread_start, thread, 0, &thread->id); + *threadptr = thread; + return 0; +} + +int +toku_pthread_join(toku_pthread_t thread, void **ret) { + WaitForSingleObject(thread->handle, INFINITE); + if (ret) + *ret = thread->ret; + free(thread); + return 0; +} + +toku_pthread_t +toku_pthread_self(void) { + // lookup a pthread by thread id and return it + return 0; +} + +#if 0 +void +toku_pthread_exit(void *ret) { + toku_pthread_t self = toku_pthread_self(); + thread->ret = ret; + // _endthread or ExitThread + ExitThread(0); +} +#endif + +int +toku_pthread_cond_init(toku_pthread_cond_t *cond, const toku_pthread_condattr_t *attr) { + int r; + assert(attr == 0); + cond->events[0] = CreateEvent(NULL, FALSE, FALSE, NULL); + if (cond->events[0] == NULL) + return GetLastError(); + cond->events[1] = CreateEvent(NULL, TRUE, FALSE, NULL); + if (cond->events[1] == NULL) { + r = GetLastError(); + CloseHandle(cond->events[0]); + return r; + } + cond->waiters = 0; + return 0; +} + +int +toku_pthread_cond_destroy(toku_pthread_cond_t *cond) { + int i; + for (i=0; ievents[i]); + return 0; +} + +int +toku_pthread_cond_wait(toku_pthread_cond_t *cond, toku_pthread_mutex_t *mutex) { + DWORD r; + cond->waiters++; + toku_pthread_mutex_unlock(mutex); + r = WaitForMultipleObjects(TOKU_PTHREAD_COND_NEVENTS, cond->events, FALSE, INFINITE); + toku_pthread_mutex_lock(mutex); + cond->waiters--; + if (cond->waiters == 0 && r == WAIT_OBJECT_0 + 1) + ResetEvent(cond->events[1]); + return 0; +} + +int +toku_pthread_cond_broadcast(toku_pthread_cond_t *cond) { + if (cond->waiters > 0) + SetEvent(cond->events[1]); + return 0; +} + +int +toku_pthread_cond_signal(toku_pthread_cond_t *cond) { + if (cond->waiters > 0) + SetEvent(cond->events[0]); + return 0; +} + +int +toku_pthread_yield(void) { + Sleep(0); + return 0; +} + diff --git a/windows/toku_pthread.h b/windows/toku_pthread.h new file mode 100644 index 00000000000..cc3d91490ce --- /dev/null +++ b/windows/toku_pthread.h @@ -0,0 +1,87 @@ +#ifndef _TOKU_PTHREAD_H +#define _TOKU_PTHREAD_H + +#if defined __cplusplus +extern "C" { +#endif + +// pthread types + +typedef struct toku_pthread_attr { + SIZE_T stacksize; +} toku_pthread_attr_t; + +typedef struct toku_pthread { + HANDLE handle; + DWORD id; + void *(*f)(void *); + void *arg; + void *ret; +} *toku_pthread_t; + +typedef struct toku_pthread_mutexattr *toku_pthread_mutexattr_t; + +typedef struct toku_pthread_mutex { + CRITICAL_SECTION section; + BOOL initialized; +} toku_pthread_mutex_t; + +#define TOKU_PTHREAD_MUTEX_INITIALIZER { 0, 0 } + +typedef struct toku_pthread_condattr *toku_pthread_condattr_t; + +// WINNT >= 6 supports condition variables. For now, we use a couple of events. +#define TOKU_PTHREAD_COND_NEVENTS 2 + +typedef struct toku_pthread_cond { +#if 0 + CONDITION_VARIABLE wcv; +#else + HANDLE events[TOKU_PTHREAD_COND_NEVENTS]; + int waiters; +#endif +} toku_pthread_cond_t; + +// pthread interface + +int toku_pthread_yield(void); + +int toku_pthread_attr_init(toku_pthread_attr_t *); +int toku_pthread_attr_destroy(toku_pthread_attr_t *); +int toku_pthread_attr_getstacksize(toku_pthread_attr_t *, size_t *stacksize); +int toku_pthread_attr_setstacksize(toku_pthread_attr_t *, size_t stacksize); + +int toku_pthread_create(toku_pthread_t *thread, const toku_pthread_attr_t *attr, void *(*start_function)(void *), void *arg); + +int toku_pthread_join(toku_pthread_t thread, void **value_ptr); + +toku_pthread_t toku_pthread_self(void); + + +int toku_pthread_mutex_init(toku_pthread_mutex_t *mutex, const toku_pthread_mutexattr_t *attr); + +int toku_pthread_mutex_destroy(toku_pthread_mutex_t *mutex); + +int toku_pthread_mutex_lock(toku_pthread_mutex_t *mutex); + +int toku_pthread_mutex_trylock(toku_pthread_mutex_t *mutex); + +int toku_pthread_mutex_unlock(toku_pthread_mutex_t *mutex); + + + +int toku_pthread_cond_init(toku_pthread_cond_t *cond, const toku_pthread_condattr_t *attr); + +int toku_pthread_cond_destroy(toku_pthread_cond_t *cond); + +int toku_pthread_cond_wait(toku_pthread_cond_t *cond, toku_pthread_mutex_t *mutex); + +int toku_pthread_cond_signal(toku_pthread_cond_t *cond); + +int toku_pthread_cond_broadcast(toku_pthread_cond_t *cond); + +#if defined __cplusplus +}; +#endif + +#endif diff --git a/windows/unistd.h b/windows/unistd.h new file mode 100644 index 00000000000..fe88250dd5c --- /dev/null +++ b/windows/unistd.h @@ -0,0 +1,22 @@ +#ifndef _TOKUWIN_UNISTD_H +#define _TOKUWIN_UNISTD_H +#include +#include + +int +ftruncate(int fildes, int64_t offset); + +int64_t +pwrite(int fildes, const void *buf, size_t nbyte, int64_t offset); + +int64_t +pread(int fildes, void *buf, size_t nbyte, int64_t offset); + +unsigned int +sleep(unsigned int); + +int +usleep(unsigned int); + +#endif + diff --git a/windows/windows.c b/windows/windows.c new file mode 100644 index 00000000000..385a7a3beec --- /dev/null +++ b/windows/windows.c @@ -0,0 +1,420 @@ +#define _CRT_SECURE_NO_DEPRECATE + +//rand_s requires _CRT_RAND_S be defined before including stdlib +#define _CRT_RAND_S +#include + +#include +#include "portability.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct __toku_windir { + struct dirent ent; + struct _finddatai64_t data; + intptr_t handle; + BOOL finished; +}; + +DIR* +opendir(const char *name) { + char *format = NULL; + DIR *result = malloc(sizeof(*result)); + int r; + if (!result) { + r = ENOMEM; + goto cleanup; + } + format = malloc(strlen(name)+2+1); //2 for /*, 1 for '\0' + if (!format) { + r = ENOMEM; + goto cleanup; + } + strcpy(format, name); + if (format[strlen(format)-1]=='/') format[strlen(format)-1]='\0'; + strcat(format, "/*"); + result->handle = _findfirsti64(format, &result->data); + // printf("%s:%d %p %d\n", __FILE__, __LINE__, result->handle, errno); fflush(stdout); + if (result->handle==-1L) { + if (errno==ENOENT) { + int64_t r_stat; + //ENOENT can mean a good directory with no files, OR + //a directory that does not exist. + struct _stat64 buffer; + format[strlen(format)-3] = '\0'; //Strip the "/*" + r_stat = _stati64(format, &buffer); + if (r_stat==0) { + //Empty directory. + result->finished = TRUE; + r = 0; + goto cleanup; + } + } + r = errno; + assert(r!=0); + goto cleanup; + } + result->finished = FALSE; + r = 0; +cleanup: + if (r!=0) { + if (result) free(result); + result = NULL; + } + if (format) free(format); + return result; +} + +struct dirent* +readdir(DIR *dir) { + struct dirent *result; + int r; + if (dir->finished) { + errno = ENOENT; + result = NULL; + goto cleanup; + } + assert(dir->handle!=-1L); + strcpy(dir->ent.d_name, dir->data.name); + if (dir->data.attrib&_A_SUBDIR) dir->ent.d_type=DT_DIR; + else dir->ent.d_type=DT_REG; + r = _findnexti64(dir->handle, &dir->data); + if (r==-1L) dir->finished = TRUE; + result = &dir->ent; +cleanup: + return result; +} + +int +closedir(DIR *dir) { + int r; + if (dir->handle==-1L) r = 0; + else r = _findclose(dir->handle); + free(dir); + return r; +} + +int +fsync(int fildes) { + int r = _commit(fildes); + return r; +} + +int +os_get_file_size(int fildes, int64_t *size) { + struct _stat64 sbuf; + int r = _fstati64(fildes, &sbuf); + if (r==0) { + *size = sbuf.st_size; + } + return r; +} + +uint64_t +os_get_phys_memory_size(void) { + MEMORYSTATUS memory_status; + GlobalMemoryStatus(&memory_status); + return memory_status.dwTotalPhys; +} + + +int +os_get_number_processors(void) { + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + return system_info.dwNumberOfProcessors; +} + +int +os_get_number_active_processors(void) { + SYSTEM_INFO system_info; + DWORD mask, n; + GetSystemInfo(&system_info); + mask = system_info.dwActiveProcessorMask; + for (n=0; mask; mask >>= 1) + n += mask & 1; + return n; +} + +int +os_get_pagesize(void) { + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + return system_info.dwPageSize; +} + +int +os_get_unique_file_id(int fildes, struct fileid *id) { + int r; + BY_HANDLE_FILE_INFORMATION info; + HANDLE filehandle; + + memset(id, 0, sizeof(*id)); + filehandle = (HANDLE)_get_osfhandle(fildes); + if (filehandle==INVALID_HANDLE_VALUE) { + r = errno; assert(r!=0); + goto cleanup; + } + r = GetFileInformationByHandle(filehandle, &info); + if (r==0) { //0 is error here. + r = GetLastError(); assert(r!=0); + goto cleanup; + } + id->st_dev = info.dwVolumeSerialNumber; + id->st_ino = info.nFileIndexHigh; + id->st_ino <<= 32; + id->st_ino |= info.nFileIndexLow; + id->st_creat = info.ftCreationTime.dwHighDateTime; + id->st_creat <<= 32; + id->st_creat |= info.ftCreationTime.dwLowDateTime; + r = 0; +cleanup: + return r; +} + +static void +convert_filetime_timeval(FILETIME ft, struct timeval *tv) { + ULARGE_INTEGER t; + + t.u.HighPart = ft.dwHighDateTime; + t.u.LowPart = ft.dwLowDateTime; + t.QuadPart /= 10; + if (tv) { + tv->tv_sec = t.QuadPart / 1000000; + tv->tv_usec = t.QuadPart % 1000000; + } +} + +int +os_get_process_times(struct timeval *usertime, struct timeval *kerneltime) { + FILETIME w_createtime, w_exittime, w_usertime, w_kerneltime; + + if (GetProcessTimes(GetCurrentProcess(), &w_createtime, &w_exittime, &w_kerneltime, &w_usertime)) { + convert_filetime_timeval(w_usertime, usertime); + convert_filetime_timeval(w_kerneltime, kerneltime); + return 0; + } + return GetLastError(); +} + +int +os_getpid(void) { +#if 0 + return _getpid(); +#else + return GetCurrentProcessId(); +#endif +} + +int +os_gettid(void) { + return GetCurrentThreadId(); +} + +int +gettimeofday(struct timeval *tv, struct timezone *tz) { + FILETIME ft; + ULARGE_INTEGER t; + + GetSystemTimeAsFileTime(&ft); + t.u.LowPart = ft.dwLowDateTime; + t.u.HighPart = ft.dwHighDateTime; + t.QuadPart -= 116444736000000000i64; + t.QuadPart /= 10; + if (tv) { + tv->tv_sec = t.QuadPart / 1000000; + tv->tv_usec = t.QuadPart % 1000000; + } + if (tz) { + assert(0); + } + return 0; +} + +int +os_lock_file(char *name) { + int fd = _sopen(name, O_CREAT, _SH_DENYRW, S_IREAD|S_IWRITE); + return fd; +} + +int +os_unlock_file(int fildes) { + int r = close(fildes); + return r; +} + +int64_t +pread(int fildes, void *buf, size_t nbyte, int64_t offset) { + int64_t r = _lseeki64(fildes, offset, SEEK_SET); + if (r>=0) { + assert(r==offset); + r = read(fildes, buf, nbyte); + } + // printf("%s: %d %p %u %I64d %I64d\n", __FUNCTION__, fildes, buf, nbyte, offset, r); fflush(stdout); + return r; +} + +int64_t +pwrite(int fildes, const void *buf, size_t nbyte, int64_t offset) { + int64_t r = _lseeki64(fildes, offset, SEEK_SET); + if (r>=0) { + assert(r==offset); + r = write(fildes, buf, nbyte); + } + // printf("%s: %d %p %u %I64d %I64d\n", __FUNCTION__, fildes, buf, nbyte, offset, r); fflush(stdout); + return r; +} + +int +os_mkdir(const char *pathname, mode_t mode) { + int r = mkdir(pathname); + UNUSED_WARNING(mode); + if (r!=0) r = errno; + return r; +} + +unsigned int +sleep(unsigned int seconds) { + unsigned int m = seconds / 1000000; + unsigned int n = seconds % 1000000; + unsigned int i; + for (i=0; i 0) + m = 1; + Sleep(m); + return 0; +} + +static void printfParameterHandler(const wchar_t* expression, + const wchar_t* function, const wchar_t* file, + unsigned int line, uintptr_t pReserved) { + fwprintf(stderr, L"Invalid parameter detected in function %s." + L" File: %s Line: %d\n" + L"Expression: %s\n", function, file, line, expression); +} + +static void ignoreParameterHandler(const wchar_t* expression, + const wchar_t* function, const wchar_t* file, + unsigned int line, uintptr_t pReserved) { + UNUSED_WARNING(expression); + UNUSED_WARNING(function); + UNUSED_WARNING(file); + UNUSED_WARNING(line); + UNUSED_WARNING(pReserved); +} + +int +os_initialize_settings(int verbosity) { + int r; + static int initialized = 0; + assert(initialized==0); + initialized=1; + if (verbosity>0) + _set_invalid_parameter_handler(printfParameterHandler); + else + _set_invalid_parameter_handler(ignoreParameterHandler); +#if defined(_DEBUG) + _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); + _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); +#endif + r = 0; + return r; +} + +long int +random(void) { + u_int32_t r; + errno_t r_error = rand_s(&r); + assert(r_error==0); + //Should return 0 to 2**31-1 instead of 2**32-1 + r >>= 1; + return r; +} + +//TODO: Implement srandom to modify the way rand_s works (IF POSSIBLE).. or +//reimplement random. +void +srandom(unsigned int seed) { + UNUSED_WARNING(seed); +} + +int +setenv(const char *name, const char *value, int overwrite) { + char buf[2]; //Need a dummy buffer + BOOL exists = TRUE; + int r = GetEnvironmentVariable(name, buf, sizeof(buf)); + if (r==0) { + r = GetLastError(); + if (r==ERROR_ENVVAR_NOT_FOUND) exists = FALSE; + else { + errno = r; + r = -1; + goto cleanup; + } + } + if (overwrite || !exists) { + r = SetEnvironmentVariable(name, value); + if (r==0) { + errno = GetLastError(); + r = -1; + goto cleanup; + } + } + r = 0; +cleanup: + return r; +} + +int +unsetenv(const char *name) { + int r = SetEnvironmentVariable(name, NULL); + if (r==0) { //0 is failure + r = -1; + errno = GetLastError(); + } + else r = 0; + return r; +} + +int +ftruncate(int fd, int64_t offset) { + HANDLE h; + BOOL b; + int r; + + h = (HANDLE) _get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) + return -1; + r = _lseeki64(fd, 0, SEEK_SET); + if (r != 0) + return -2; + b = SetEndOfFile(h); + if (!b) + return -3; + return 0; +} + diff --git a/windows/zconf.h b/windows/zconf.h new file mode 100644 index 00000000000..03a9431c8be --- /dev/null +++ b/windows/zconf.h @@ -0,0 +1,332 @@ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2005 Jean-loup Gailly. + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#ifndef ZCONF_H +#define ZCONF_H + +/* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + */ +#ifdef Z_PREFIX +# define deflateInit_ z_deflateInit_ +# define deflate z_deflate +# define deflateEnd z_deflateEnd +# define inflateInit_ z_inflateInit_ +# define inflate z_inflate +# define inflateEnd z_inflateEnd +# define deflateInit2_ z_deflateInit2_ +# define deflateSetDictionary z_deflateSetDictionary +# define deflateCopy z_deflateCopy +# define deflateReset z_deflateReset +# define deflateParams z_deflateParams +# define deflateBound z_deflateBound +# define deflatePrime z_deflatePrime +# define inflateInit2_ z_inflateInit2_ +# define inflateSetDictionary z_inflateSetDictionary +# define inflateSync z_inflateSync +# define inflateSyncPoint z_inflateSyncPoint +# define inflateCopy z_inflateCopy +# define inflateReset z_inflateReset +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# define uncompress z_uncompress +# define adler32 z_adler32 +# define crc32 z_crc32 +# define get_crc_table z_get_crc_table +# define zError z_zError + +# define alloc_func z_alloc_func +# define free_func z_free_func +# define in_func z_in_func +# define out_func z_out_func +# define Byte z_Byte +# define uInt z_uInt +# define uLong z_uLong +# define Bytef z_Bytef +# define charf z_charf +# define intf z_intf +# define uIntf z_uIntf +# define uLongf z_uLongf +# define voidpf z_voidpf +# define voidp z_voidp +#endif + +#if defined(__MSDOS__) && !defined(MSDOS) +# define MSDOS +#endif +#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) +# define OS2 +#endif +#if defined(_WINDOWS) && !defined(WINDOWS) +# define WINDOWS +#endif +#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) +# ifndef WIN32 +# define WIN32 +# endif +#endif +#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) +# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) +# ifndef SYS16BIT +# define SYS16BIT +# endif +# endif +#endif + +/* + * Compile with -DMAXSEG_64K if the alloc function cannot allocate more + * than 64k bytes at a time (needed on systems with 16-bit int). + */ +#ifdef SYS16BIT +# define MAXSEG_64K +#endif +#ifdef MSDOS +# define UNALIGNED_OK +#endif + +#ifdef __STDC_VERSION__ +# ifndef STDC +# define STDC +# endif +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +#endif +#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) +# define STDC +#endif +#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) +# define STDC +#endif +#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) +# define STDC +#endif +#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) +# define STDC +#endif + +#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ +# define STDC +#endif + +#ifndef STDC +# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ +# define const /* note: need a more gentle solution here */ +# endif +#endif + +/* Some Mac compilers merge all .h files incorrectly: */ +#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) +# define NO_DUMMY_DECL +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# ifdef MAXSEG_64K +# define MAX_MEM_LEVEL 8 +# else +# define MAX_MEM_LEVEL 9 +# endif +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects. +*/ + + /* Type declarations */ + +#ifndef OF /* function prototypes */ +# ifdef STDC +# define OF(args) args +# else +# define OF(args) () +# endif +#endif + +/* The following definitions for FAR are needed only for MSDOS mixed + * model programming (small or medium model with some far allocations). + * This was tested only with MSC; for other MSDOS compilers you may have + * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, + * just define FAR to be empty. + */ +#ifdef SYS16BIT +# if defined(M_I86SM) || defined(M_I86MM) + /* MSC small or medium model */ +# define SMALL_MEDIUM +# ifdef _MSC_VER +# define FAR _far +# else +# define FAR far +# endif +# endif +# if (defined(__SMALL__) || defined(__MEDIUM__)) + /* Turbo C small or medium model */ +# define SMALL_MEDIUM +# ifdef __BORLANDC__ +# define FAR _far +# else +# define FAR far +# endif +# endif +#endif + +#if defined(WINDOWS) || defined(WIN32) + /* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +# ifdef ZLIB_DLL +# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) +# ifdef ZLIB_INTERNAL +# define ZEXTERN extern __declspec(dllexport) +# else +# define ZEXTERN extern __declspec(dllimport) +# endif +# endif +# endif /* ZLIB_DLL */ + /* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +# ifdef ZLIB_WINAPI +# ifdef FAR +# undef FAR +# endif +# include + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define ZEXPORT WINAPI +# ifdef WIN32 +# define ZEXPORTVA WINAPIV +# else +# define ZEXPORTVA FAR CDECL +# endif +# endif +#endif + +#if defined (__BEOS__) +# ifdef ZLIB_DLL +# ifdef ZLIB_INTERNAL +# define ZEXPORT __declspec(dllexport) +# define ZEXPORTVA __declspec(dllexport) +# else +# define ZEXPORT __declspec(dllimport) +# define ZEXPORTVA __declspec(dllimport) +# endif +# endif +#endif + +#ifndef ZEXTERN +# define ZEXTERN extern +#endif +#ifndef ZEXPORT +# define ZEXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA +#endif + +#ifndef FAR +# define FAR +#endif + +#if !defined(__MACTYPES__) +typedef unsigned char Byte; /* 8 bits */ +#endif +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +#ifdef SMALL_MEDIUM + /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ +# define Bytef Byte FAR +#else + typedef Byte FAR Bytef; +#endif +typedef char FAR charf; +typedef int FAR intf; +typedef uInt FAR uIntf; +typedef uLong FAR uLongf; + +#ifdef STDC + typedef void const *voidpc; + typedef void FAR *voidpf; + typedef void *voidp; +#else + typedef Byte const *voidpc; + typedef Byte FAR *voidpf; + typedef Byte *voidp; +#endif + +#if 0 /* HAVE_UNISTD_H -- this line is updated by ./configure */ +# include /* for off_t */ +# include /* for SEEK_* and off_t */ +# ifdef VMS +# include /* for off_t */ +# endif +# define z_off_t off_t +#endif +#ifndef SEEK_SET +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif +#ifndef z_off_t +# define z_off_t long +#endif + +#if defined(__OS400__) +# define NO_vsnprintf +#endif + +#if defined(__MVS__) +# define NO_vsnprintf +# ifdef FAR +# undef FAR +# endif +#endif + +/* MVS linker does not support external names larger than 8 bytes */ +#if defined(__MVS__) +# pragma map(deflateInit_,"DEIN") +# pragma map(deflateInit2_,"DEIN2") +# pragma map(deflateEnd,"DEEND") +# pragma map(deflateBound,"DEBND") +# pragma map(inflateInit_,"ININ") +# pragma map(inflateInit2_,"ININ2") +# pragma map(inflateEnd,"INEND") +# pragma map(inflateSync,"INSY") +# pragma map(inflateSetDictionary,"INSEDI") +# pragma map(compressBound,"CMBND") +# pragma map(inflate_table,"INTABL") +# pragma map(inflate_fast,"INFA") +# pragma map(inflate_copyright,"INCOPY") +#endif + +#endif /* ZCONF_H */ diff --git a/windows/zlib.h b/windows/zlib.h new file mode 100644 index 00000000000..022817927ce --- /dev/null +++ b/windows/zlib.h @@ -0,0 +1,1357 @@ +/* zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.3, July 18th, 2005 + + Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + The data format used by the zlib library is described by RFCs (Request for + Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt + (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). +*/ + +#ifndef ZLIB_H +#define ZLIB_H + +#include "zconf.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZLIB_VERSION "1.2.3" +#define ZLIB_VERNUM 0x1230 + +/* + The 'zlib' compression library provides in-memory compression and + decompression functions, including integrity checks of the uncompressed + data. This version of the library supports only one compression method + (deflation) but other algorithms will be added later and will have the same + stream interface. + + Compression can be done in a single step if the buffers are large + enough (for example if an input file is mmap'ed), or can be done by + repeated calls of the compression function. In the latter case, the + application must provide more input and/or consume the output + (providing more output space) before each call. + + The compressed data format used by default by the in-memory functions is + the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped + around a deflate stream, which is itself documented in RFC 1951. + + The library also supports reading and writing files in gzip (.gz) format + with an interface similar to that of stdio using the functions that start + with "gz". The gzip format is different from the zlib format. gzip is a + gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. + + This library can optionally read and write gzip streams in memory as well. + + The zlib format was designed to be compact and fast for use in memory + and on communications channels. The gzip format was designed for single- + file compression on file systems, has a larger header than zlib to maintain + directory information, and uses a different, slower check method than zlib. + + The library does not install any signal handler. The decoder checks + the consistency of the compressed data, so the library should never + crash even in case of corrupted input. +*/ + +typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); +typedef void (*free_func) OF((voidpf opaque, voidpf address)); + +struct internal_state; + +typedef struct z_stream_s { + Bytef *next_in; /* next input byte */ + uInt avail_in; /* number of bytes available at next_in */ + uLong total_in; /* total nb of input bytes read so far */ + + Bytef *next_out; /* next output byte should be put there */ + uInt avail_out; /* remaining free space at next_out */ + uLong total_out; /* total nb of bytes output so far */ + + char *msg; /* last error message, NULL if no error */ + struct internal_state FAR *state; /* not visible by applications */ + + alloc_func zalloc; /* used to allocate the internal state */ + free_func zfree; /* used to free the internal state */ + voidpf opaque; /* private data object passed to zalloc and zfree */ + + int data_type; /* best guess about the data type: binary or text */ + uLong adler; /* adler32 value of the uncompressed data */ + uLong reserved; /* reserved for future use */ +} z_stream; + +typedef z_stream FAR *z_streamp; + +/* + gzip header information passed to and from zlib routines. See RFC 1952 + for more details on the meanings of these fields. +*/ +typedef struct gz_header_s { + int text; /* true if compressed data believed to be text */ + uLong time; /* modification time */ + int xflags; /* extra flags (not used when writing a gzip file) */ + int os; /* operating system */ + Bytef *extra; /* pointer to extra field or Z_NULL if none */ + uInt extra_len; /* extra field length (valid if extra != Z_NULL) */ + uInt extra_max; /* space at extra (only when reading header) */ + Bytef *name; /* pointer to zero-terminated file name or Z_NULL */ + uInt name_max; /* space at name (only when reading header) */ + Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */ + uInt comm_max; /* space at comment (only when reading header) */ + int hcrc; /* true if there was or will be a header crc */ + int done; /* true when done reading gzip header (not used + when writing a gzip file) */ +} gz_header; + +typedef gz_header FAR *gz_headerp; + +/* + The application must update next_in and avail_in when avail_in has + dropped to zero. It must update next_out and avail_out when avail_out + has dropped to zero. The application must initialize zalloc, zfree and + opaque before calling the init function. All other fields are set by the + compression library and must not be updated by the application. + + The opaque value provided by the application will be passed as the first + parameter for calls of zalloc and zfree. This can be useful for custom + memory management. The compression library attaches no meaning to the + opaque value. + + zalloc must return Z_NULL if there is not enough memory for the object. + If zlib is used in a multi-threaded application, zalloc and zfree must be + thread safe. + + On 16-bit systems, the functions zalloc and zfree must be able to allocate + exactly 65536 bytes, but will not be required to allocate more than this + if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, + pointers returned by zalloc for objects of exactly 65536 bytes *must* + have their offset normalized to zero. The default allocation function + provided by this library ensures this (see zutil.c). To reduce memory + requirements and avoid any allocation of 64K objects, at the expense of + compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h). + + The fields total_in and total_out can be used for statistics or + progress reports. After compression, total_in holds the total size of + the uncompressed data and may be saved for use in the decompressor + (particularly if the decompressor wants to decompress everything in + a single step). +*/ + + /* constants */ + +#define Z_NO_FLUSH 0 +#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */ +#define Z_SYNC_FLUSH 2 +#define Z_FULL_FLUSH 3 +#define Z_FINISH 4 +#define Z_BLOCK 5 +/* Allowed flush values; see deflate() and inflate() below for details */ + +#define Z_OK 0 +#define Z_STREAM_END 1 +#define Z_NEED_DICT 2 +#define Z_ERRNO (-1) +#define Z_STREAM_ERROR (-2) +#define Z_DATA_ERROR (-3) +#define Z_MEM_ERROR (-4) +#define Z_BUF_ERROR (-5) +#define Z_VERSION_ERROR (-6) +/* Return codes for the compression/decompression functions. Negative + * values are errors, positive values are used for special but normal events. + */ + +#define Z_NO_COMPRESSION 0 +#define Z_BEST_SPEED 1 +#define Z_BEST_COMPRESSION 9 +#define Z_DEFAULT_COMPRESSION (-1) +/* compression levels */ + +#define Z_FILTERED 1 +#define Z_HUFFMAN_ONLY 2 +#define Z_RLE 3 +#define Z_FIXED 4 +#define Z_DEFAULT_STRATEGY 0 +/* compression strategy; see deflateInit2() below for details */ + +#define Z_BINARY 0 +#define Z_TEXT 1 +#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ +#define Z_UNKNOWN 2 +/* Possible values of the data_type field (though see inflate()) */ + +#define Z_DEFLATED 8 +/* The deflate compression method (the only one supported in this version) */ + +#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ + +#define zlib_version zlibVersion() +/* for compatibility with versions < 1.0.2 */ + + /* basic functions */ + +ZEXTERN const char * ZEXPORT zlibVersion OF((void)); +/* The application can compare zlibVersion and ZLIB_VERSION for consistency. + If the first character differs, the library code actually used is + not compatible with the zlib.h header file used by the application. + This check is automatically made by deflateInit and inflateInit. + */ + +/* +ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); + + Initializes the internal stream state for compression. The fields + zalloc, zfree and opaque must be initialized before by the caller. + If zalloc and zfree are set to Z_NULL, deflateInit updates them to + use default allocation functions. + + The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: + 1 gives best speed, 9 gives best compression, 0 gives no compression at + all (the input data is simply copied a block at a time). + Z_DEFAULT_COMPRESSION requests a default compromise between speed and + compression (currently equivalent to level 6). + + deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if level is not a valid compression level, + Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible + with the version assumed by the caller (ZLIB_VERSION). + msg is set to null if there is no error message. deflateInit does not + perform any compression: this will be done by deflate(). +*/ + + +ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); +/* + deflate compresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce some + output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. deflate performs one or both of the + following actions: + + - Compress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), next_in and avail_in are updated and + processing will resume at this point for the next call of deflate(). + + - Provide more output starting at next_out and update next_out and avail_out + accordingly. This action is forced if the parameter flush is non zero. + Forcing flush frequently degrades the compression ratio, so this parameter + should be set only when necessary (in interactive applications). + Some output may be provided even if flush is not set. + + Before the call of deflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming + more output, and updating avail_in or avail_out accordingly; avail_out + should never be zero before the call. The application can consume the + compressed output when it wants, for example when the output buffer is full + (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK + and with zero avail_out, it must be called again after making room in the + output buffer because there might be more output pending. + + Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to + decide how much data to accumualte before producing output, in order to + maximize compression. + + If the parameter flush is set to Z_SYNC_FLUSH, all pending output is + flushed to the output buffer and the output is aligned on a byte boundary, so + that the decompressor can get all input data available so far. (In particular + avail_in is zero after the call if enough output space has been provided + before the call.) Flushing may degrade compression for some compression + algorithms and so it should be used only when necessary. + + If flush is set to Z_FULL_FLUSH, all output is flushed as with + Z_SYNC_FLUSH, and the compression state is reset so that decompression can + restart from this point if previous compressed data has been damaged or if + random access is desired. Using Z_FULL_FLUSH too often can seriously degrade + compression. + + If deflate returns with avail_out == 0, this function must be called again + with the same value of the flush parameter and more output space (updated + avail_out), until the flush is complete (deflate returns with non-zero + avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that + avail_out is greater than six to avoid repeated flush markers due to + avail_out == 0 on return. + + If the parameter flush is set to Z_FINISH, pending input is processed, + pending output is flushed and deflate returns with Z_STREAM_END if there + was enough output space; if deflate returns with Z_OK, this function must be + called again with Z_FINISH and more output space (updated avail_out) but no + more input data, until it returns with Z_STREAM_END or an error. After + deflate has returned Z_STREAM_END, the only possible operations on the + stream are deflateReset or deflateEnd. + + Z_FINISH can be used immediately after deflateInit if all the compression + is to be done in a single step. In this case, avail_out must be at least + the value returned by deflateBound (see below). If deflate does not return + Z_STREAM_END, then it must be called again as described above. + + deflate() sets strm->adler to the adler32 checksum of all input read + so far (that is, total_in bytes). + + deflate() may update strm->data_type if it can make a good guess about + the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered + binary. This field is only for information purposes and does not affect + the compression algorithm in any manner. + + deflate() returns Z_OK if some progress has been made (more input + processed or more output produced), Z_STREAM_END if all input has been + consumed and all output has been produced (only when flush is set to + Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example + if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible + (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not + fatal, and deflate() can be called again with more input and more output + space to continue compressing. +*/ + + +ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any + pending output. + + deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the + stream state was inconsistent, Z_DATA_ERROR if the stream was freed + prematurely (some input or output was discarded). In the error case, + msg may be set but then points to a static string (which must not be + deallocated). +*/ + + +/* +ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); + + Initializes the internal stream state for decompression. The fields + next_in, avail_in, zalloc, zfree and opaque must be initialized before by + the caller. If next_in is not Z_NULL and avail_in is large enough (the exact + value depends on the compression method), inflateInit determines the + compression method from the zlib header and allocates all data structures + accordingly; otherwise the allocation will be deferred to the first call of + inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to + use default allocation functions. + + inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller. msg is set to null if there is no error + message. inflateInit does not perform any decompression apart from reading + the zlib header if present: this will be done by inflate(). (So next_in and + avail_in may be modified, but next_out and avail_out are unchanged.) +*/ + + +ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); +/* + inflate decompresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. inflate performs one or both of the + following actions: + + - Decompress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), next_in is updated and processing + will resume at this point for the next call of inflate(). + + - Provide more output starting at next_out and update next_out and avail_out + accordingly. inflate() provides as much output as possible, until there + is no more input data or no more space in the output buffer (see below + about the flush parameter). + + Before the call of inflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming + more output, and updating the next_* and avail_* values accordingly. + The application can consume the uncompressed output when it wants, for + example when the output buffer is full (avail_out == 0), or after each + call of inflate(). If inflate returns Z_OK and with zero avail_out, it + must be called again after making room in the output buffer because there + might be more output pending. + + The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, + Z_FINISH, or Z_BLOCK. Z_SYNC_FLUSH requests that inflate() flush as much + output as possible to the output buffer. Z_BLOCK requests that inflate() stop + if and when it gets to the next deflate block boundary. When decoding the + zlib or gzip format, this will cause inflate() to return immediately after + the header and before the first block. When doing a raw inflate, inflate() + will go ahead and process the first block, and will return when it gets to + the end of that block, or when it runs out of data. + + The Z_BLOCK option assists in appending to or combining deflate streams. + Also to assist in this, on return inflate() will set strm->data_type to the + number of unused bits in the last byte taken from strm->next_in, plus 64 + if inflate() is currently decoding the last block in the deflate stream, + plus 128 if inflate() returned immediately after decoding an end-of-block + code or decoding the complete header up to just before the first byte of the + deflate stream. The end-of-block will not be indicated until all of the + uncompressed data from that block has been written to strm->next_out. The + number of unused bits may in general be greater than seven, except when + bit 7 of data_type is set, in which case the number of unused bits will be + less than eight. + + inflate() should normally be called until it returns Z_STREAM_END or an + error. However if all decompression is to be performed in a single step + (a single call of inflate), the parameter flush should be set to + Z_FINISH. In this case all pending input is processed and all pending + output is flushed; avail_out must be large enough to hold all the + uncompressed data. (The size of the uncompressed data may have been saved + by the compressor for this purpose.) The next operation on this stream must + be inflateEnd to deallocate the decompression state. The use of Z_FINISH + is never required, but can be used to inform inflate that a faster approach + may be used for the single inflate() call. + + In this implementation, inflate() always flushes as much output as + possible to the output buffer, and always uses the faster approach on the + first call. So the only effect of the flush parameter in this implementation + is on the return value of inflate(), as noted below, or when it returns early + because Z_BLOCK is used. + + If a preset dictionary is needed after this call (see inflateSetDictionary + below), inflate sets strm->adler to the adler32 checksum of the dictionary + chosen by the compressor and returns Z_NEED_DICT; otherwise it sets + strm->adler to the adler32 checksum of all output produced so far (that is, + total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described + below. At the end of the stream, inflate() checks that its computed adler32 + checksum is equal to that saved by the compressor and returns Z_STREAM_END + only if the checksum is correct. + + inflate() will decompress and check either zlib-wrapped or gzip-wrapped + deflate data. The header type is detected automatically. Any information + contained in the gzip header is not retained, so applications that need that + information should instead use raw inflate, see inflateInit2() below, or + inflateBack() and perform their own processing of the gzip header and + trailer. + + inflate() returns Z_OK if some progress has been made (more input processed + or more output produced), Z_STREAM_END if the end of the compressed data has + been reached and all uncompressed output has been produced, Z_NEED_DICT if a + preset dictionary is needed at this point, Z_DATA_ERROR if the input data was + corrupted (input stream not conforming to the zlib format or incorrect check + value), Z_STREAM_ERROR if the stream structure was inconsistent (for example + if next_in or next_out was NULL), Z_MEM_ERROR if there was not enough memory, + Z_BUF_ERROR if no progress is possible or if there was not enough room in the + output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + inflate() can be called again with more input and more output space to + continue decompressing. If Z_DATA_ERROR is returned, the application may then + call inflateSync() to look for a good compression block if a partial recovery + of the data is desired. +*/ + + +ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any + pending output. + + inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state + was inconsistent. In the error case, msg may be set but then points to a + static string (which must not be deallocated). +*/ + + /* Advanced functions */ + +/* + The following functions are needed only in some special applications. +*/ + +/* +ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy)); + + This is another version of deflateInit with more compression options. The + fields next_in, zalloc, zfree and opaque must be initialized before by + the caller. + + The method parameter is the compression method. It must be Z_DEFLATED in + this version of the library. + + The windowBits parameter is the base two logarithm of the window size + (the size of the history buffer). It should be in the range 8..15 for this + version of the library. Larger values of this parameter result in better + compression at the expense of memory usage. The default value is 15 if + deflateInit is used instead. + + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits + determines the window size. deflate() will then generate raw deflate data + with no zlib header or trailer, and will not compute an adler32 check value. + + windowBits can also be greater than 15 for optional gzip encoding. Add + 16 to windowBits to write a simple gzip header and trailer around the + compressed data instead of a zlib wrapper. The gzip header will have no + file name, no extra data, no comment, no modification time (set to zero), + no header crc, and the operating system will be set to 255 (unknown). If a + gzip stream is being written, strm->adler is a crc32 instead of an adler32. + + The memLevel parameter specifies how much memory should be allocated + for the internal compression state. memLevel=1 uses minimum memory but + is slow and reduces compression ratio; memLevel=9 uses maximum memory + for optimal speed. The default value is 8. See zconf.h for total memory + usage as a function of windowBits and memLevel. + + The strategy parameter is used to tune the compression algorithm. Use the + value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a + filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no + string match), or Z_RLE to limit match distances to one (run-length + encoding). Filtered data consists mostly of small values with a somewhat + random distribution. In this case, the compression algorithm is tuned to + compress them better. The effect of Z_FILTERED is to force more Huffman + coding and less string matching; it is somewhat intermediate between + Z_DEFAULT and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as + Z_HUFFMAN_ONLY, but give better compression for PNG image data. The strategy + parameter only affects the compression ratio but not the correctness of the + compressed output even if it is not set appropriately. Z_FIXED prevents the + use of dynamic Huffman codes, allowing for a simpler decoder for special + applications. + + deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid + method). msg is set to null if there is no error message. deflateInit2 does + not perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the compression dictionary from the given byte sequence + without producing any compressed output. This function must be called + immediately after deflateInit, deflateInit2 or deflateReset, before any + call of deflate. The compressor and decompressor must use exactly the same + dictionary (see inflateSetDictionary). + + The dictionary should consist of strings (byte sequences) that are likely + to be encountered later in the data to be compressed, with the most commonly + used strings preferably put towards the end of the dictionary. Using a + dictionary is most useful when the data to be compressed is short and can be + predicted with good accuracy; the data can then be compressed better than + with the default empty dictionary. + + Depending on the size of the compression data structures selected by + deflateInit or deflateInit2, a part of the dictionary may in effect be + discarded, for example if the dictionary is larger than the window size in + deflate or deflate2. Thus the strings most likely to be useful should be + put at the end of the dictionary, not at the front. In addition, the + current implementation of deflate will use at most the window size minus + 262 bytes of the provided dictionary. + + Upon return of this function, strm->adler is set to the adler32 value + of the dictionary; the decompressor may later use this value to determine + which dictionary has been used by the compressor. (The adler32 value + applies to the whole dictionary even if only a subset of the dictionary is + actually used by the compressor.) If a raw deflate was requested, then the + adler32 value is not computed and strm->adler is not set. + + deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a + parameter is invalid (such as NULL dictionary) or the stream state is + inconsistent (for example if deflate has already been called for this stream + or if the compression method is bsort). deflateSetDictionary does not + perform any compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when several compression strategies will be + tried, for example when there are several ways of pre-processing the input + data with a filter. The streams that will be discarded should then be freed + by calling deflateEnd. Note that deflateCopy duplicates the internal + compression state which can be quite large, so this strategy is slow and + can consume lots of memory. + + deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); +/* + This function is equivalent to deflateEnd followed by deflateInit, + but does not free and reallocate all the internal compression state. + The stream will keep the same compression level and any other attributes + that may have been set by deflateInit2. + + deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being NULL). +*/ + +ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, + int level, + int strategy)); +/* + Dynamically update the compression level and compression strategy. The + interpretation of level and strategy is as in deflateInit2. This can be + used to switch between compression and straight copy of the input data, or + to switch to a different kind of input data requiring a different + strategy. If the compression level is changed, the input available so far + is compressed with the old level (and may be flushed); the new level will + take effect only at the next call of deflate(). + + Before the call of deflateParams, the stream state must be set as for + a call of deflate(), since the currently available input may have to + be compressed and flushed. In particular, strm->avail_out must be non-zero. + + deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source + stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR + if strm->avail_out was zero. +*/ + +ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain)); +/* + Fine tune deflate's internal compression parameters. This should only be + used by someone who understands the algorithm used by zlib's deflate for + searching for the best matching string, and even then only by the most + fanatic optimizer trying to squeeze out the last compressed bit for their + specific input data. Read the deflate.c source code for the meaning of the + max_lazy, good_length, nice_length, and max_chain parameters. + + deflateTune() can be called after deflateInit() or deflateInit2(), and + returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. + */ + +ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, + uLong sourceLen)); +/* + deflateBound() returns an upper bound on the compressed size after + deflation of sourceLen bytes. It must be called after deflateInit() + or deflateInit2(). This would be used to allocate an output buffer + for deflation in a single pass, and so would be called before deflate(). +*/ + +ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + deflatePrime() inserts bits in the deflate output stream. The intent + is that this function is used to start off the deflate output with the + bits leftover from a previous deflate stream when appending to it. As such, + this function can only be used for raw deflate, and must be used before the + first deflate() call after a deflateInit2() or deflateReset(). bits must be + less than or equal to 16, and that many of the least significant bits of + value will be inserted in the output. + + deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, + gz_headerp head)); +/* + deflateSetHeader() provides gzip header information for when a gzip + stream is requested by deflateInit2(). deflateSetHeader() may be called + after deflateInit2() or deflateReset() and before the first call of + deflate(). The text, time, os, extra field, name, and comment information + in the provided gz_header structure are written to the gzip header (xflag is + ignored -- the extra flags are set according to the compression level). The + caller must assure that, if not Z_NULL, name and comment are terminated with + a zero byte, and that if extra is not Z_NULL, that extra_len bytes are + available there. If hcrc is true, a gzip header crc is included. Note that + the current versions of the command-line version of gzip (up through version + 1.3.x) do not support header crc's, and will report that it is a "multi-part + gzip file" and give up. + + If deflateSetHeader is not used, the default gzip header has text false, + the time set to zero, and os set to 255, with no extra, name, or comment + fields. The gzip header is returned to the default state by deflateReset(). + + deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, + int windowBits)); + + This is another version of inflateInit with an extra parameter. The + fields next_in, avail_in, zalloc, zfree and opaque must be initialized + before by the caller. + + The windowBits parameter is the base two logarithm of the maximum window + size (the size of the history buffer). It should be in the range 8..15 for + this version of the library. The default value is 15 if inflateInit is used + instead. windowBits must be greater than or equal to the windowBits value + provided to deflateInit2() while compressing, or it must be equal to 15 if + deflateInit2() was not used. If a compressed stream with a larger window + size is given as input, inflate() will return with the error code + Z_DATA_ERROR instead of trying to allocate a larger window. + + windowBits can also be -8..-15 for raw inflate. In this case, -windowBits + determines the window size. inflate() will then process raw deflate data, + not looking for a zlib or gzip header, not generating a check value, and not + looking for any check values for comparison at the end of the stream. This + is for use with other formats that use the deflate compressed data format + such as zip. Those formats provide their own check values. If a custom + format is developed using the raw deflate format for compressed data, it is + recommended that a check value such as an adler32 or a crc32 be applied to + the uncompressed data as is done in the zlib, gzip, and zip formats. For + most applications, the zlib format should be used as is. Note that comments + above on the use in deflateInit2() applies to the magnitude of windowBits. + + windowBits can also be greater than 15 for optional gzip decoding. Add + 32 to windowBits to enable zlib and gzip decoding with automatic header + detection, or add 16 to decode only the gzip format (the zlib format will + return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is + a crc32 instead of an adler32. + + inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if a parameter is invalid (such as a null strm). msg + is set to null if there is no error message. inflateInit2 does not perform + any decompression apart from reading the zlib header if present: this will + be done by inflate(). (So next_in and avail_in may be modified, but next_out + and avail_out are unchanged.) +*/ + +ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the decompression dictionary from the given uncompressed byte + sequence. This function must be called immediately after a call of inflate, + if that call returned Z_NEED_DICT. The dictionary chosen by the compressor + can be determined from the adler32 value returned by that call of inflate. + The compressor and decompressor must use exactly the same dictionary (see + deflateSetDictionary). For raw inflate, this function can be called + immediately after inflateInit2() or inflateReset() and before any call of + inflate() to set the dictionary. The application must insure that the + dictionary that was used for compression is provided. + + inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a + parameter is invalid (such as NULL dictionary) or the stream state is + inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the + expected one (incorrect adler32 value). inflateSetDictionary does not + perform any decompression: this will be done by subsequent calls of + inflate(). +*/ + +ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); +/* + Skips invalid compressed data until a full flush point (see above the + description of deflate with Z_FULL_FLUSH) can be found, or until all + available input is skipped. No output is provided. + + inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR + if no more input was provided, Z_DATA_ERROR if no flush point has been found, + or Z_STREAM_ERROR if the stream structure was inconsistent. In the success + case, the application may save the current current value of total_in which + indicates where valid compressed data was found. In the error case, the + application may repeatedly call inflateSync, providing more input each time, + until success or end of the input data. +*/ + +ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when randomly accessing a large stream. The + first pass through the stream can periodically record the inflate state, + allowing restarting inflate at those points when randomly accessing the + stream. + + inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); +/* + This function is equivalent to inflateEnd followed by inflateInit, + but does not free and reallocate all the internal decompression state. + The stream will keep attributes that may have been set by inflateInit2. + + inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being NULL). +*/ + +ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + This function inserts bits in the inflate input stream. The intent is + that this function is used to start inflating at a bit position in the + middle of a byte. The provided bits will be used before any bytes are used + from next_in. This function should only be used with raw inflate, and + should be used before the first inflate() call after inflateInit2() or + inflateReset(). bits must be less than or equal to 16, and that many of the + least significant bits of value will be inserted in the input. + + inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, + gz_headerp head)); +/* + inflateGetHeader() requests that gzip header information be stored in the + provided gz_header structure. inflateGetHeader() may be called after + inflateInit2() or inflateReset(), and before the first call of inflate(). + As inflate() processes the gzip stream, head->done is zero until the header + is completed, at which time head->done is set to one. If a zlib stream is + being decoded, then head->done is set to -1 to indicate that there will be + no gzip header information forthcoming. Note that Z_BLOCK can be used to + force inflate() to return immediately after header processing is complete + and before any actual data is decompressed. + + The text, time, xflags, and os fields are filled in with the gzip header + contents. hcrc is set to true if there is a header CRC. (The header CRC + was valid if done is set to one.) If extra is not Z_NULL, then extra_max + contains the maximum number of bytes to write to extra. Once done is true, + extra_len contains the actual extra field length, and extra contains the + extra field, or that field truncated if extra_max is less than extra_len. + If name is not Z_NULL, then up to name_max characters are written there, + terminated with a zero unless the length is greater than name_max. If + comment is not Z_NULL, then up to comm_max characters are written there, + terminated with a zero unless the length is greater than comm_max. When + any of extra, name, or comment are not Z_NULL and the respective field is + not present in the header, then that field is set to Z_NULL to signal its + absence. This allows the use of deflateSetHeader() with the returned + structure to duplicate the header. However if those fields are set to + allocated memory, then the application will need to save those pointers + elsewhere so that they can be eventually freed. + + If inflateGetHeader is not used, then the header information is simply + discarded. The header is always checked for validity, including the header + CRC if present. inflateReset() will reset the process to discard the header + information. The application would need to call inflateGetHeader() again to + retrieve the header from the next gzip stream. + + inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, + unsigned char FAR *window)); + + Initialize the internal stream state for decompression using inflateBack() + calls. The fields zalloc, zfree and opaque in strm must be initialized + before the call. If zalloc and zfree are Z_NULL, then the default library- + derived memory allocation routines are used. windowBits is the base two + logarithm of the window size, in the range 8..15. window is a caller + supplied buffer of that size. Except for special applications where it is + assured that deflate was used with small window sizes, windowBits must be 15 + and a 32K byte window must be supplied to be able to decompress general + deflate streams. + + See inflateBack() for the usage of these routines. + + inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of + the paramaters are invalid, Z_MEM_ERROR if the internal state could not + be allocated, or Z_VERSION_ERROR if the version of the library does not + match the version of the header file. +*/ + +typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *)); +typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); + +ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc)); +/* + inflateBack() does a raw inflate with a single call using a call-back + interface for input and output. This is more efficient than inflate() for + file i/o applications in that it avoids copying between the output and the + sliding window by simply making the window itself the output buffer. This + function trusts the application to not change the output buffer passed by + the output function, at least until inflateBack() returns. + + inflateBackInit() must be called first to allocate the internal state + and to initialize the state with the user-provided window buffer. + inflateBack() may then be used multiple times to inflate a complete, raw + deflate stream with each call. inflateBackEnd() is then called to free + the allocated state. + + A raw deflate stream is one with no zlib or gzip header or trailer. + This routine would normally be used in a utility that reads zip or gzip + files and writes out uncompressed files. The utility would decode the + header and process the trailer on its own, hence this routine expects + only the raw deflate stream to decompress. This is different from the + normal behavior of inflate(), which expects either a zlib or gzip header and + trailer around the deflate stream. + + inflateBack() uses two subroutines supplied by the caller that are then + called by inflateBack() for input and output. inflateBack() calls those + routines until it reads a complete deflate stream and writes out all of the + uncompressed data, or until it encounters an error. The function's + parameters and return types are defined above in the in_func and out_func + typedefs. inflateBack() will call in(in_desc, &buf) which should return the + number of bytes of provided input, and a pointer to that input in buf. If + there is no input available, in() must return zero--buf is ignored in that + case--and inflateBack() will return a buffer error. inflateBack() will call + out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() + should return zero on success, or non-zero on failure. If out() returns + non-zero, inflateBack() will return with an error. Neither in() nor out() + are permitted to change the contents of the window provided to + inflateBackInit(), which is also the buffer that out() uses to write from. + The length written by out() will be at most the window size. Any non-zero + amount of input may be provided by in(). + + For convenience, inflateBack() can be provided input on the first call by + setting strm->next_in and strm->avail_in. If that input is exhausted, then + in() will be called. Therefore strm->next_in must be initialized before + calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called + immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in + must also be initialized, and then if strm->avail_in is not zero, input will + initially be taken from strm->next_in[0 .. strm->avail_in - 1]. + + The in_desc and out_desc parameters of inflateBack() is passed as the + first parameter of in() and out() respectively when they are called. These + descriptors can be optionally used to pass any information that the caller- + supplied in() and out() functions need to do their job. + + On return, inflateBack() will set strm->next_in and strm->avail_in to + pass back any unused input that was provided by the last in() call. The + return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR + if in() or out() returned an error, Z_DATA_ERROR if there was a format + error in the deflate stream (in which case strm->msg is set to indicate the + nature of the error), or Z_STREAM_ERROR if the stream was not properly + initialized. In the case of Z_BUF_ERROR, an input or output error can be + distinguished using strm->next_in which will be Z_NULL only if in() returned + an error. If strm->next is not Z_NULL, then the Z_BUF_ERROR was due to + out() returning non-zero. (in() will always be called before out(), so + strm->next_in is assured to be defined if out() returns non-zero.) Note + that inflateBack() cannot return Z_OK. +*/ + +ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); +/* + All memory allocated by inflateBackInit() is freed. + + inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream + state was inconsistent. +*/ + +ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); +/* Return flags indicating compile-time options. + + Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: + 1.0: size of uInt + 3.2: size of uLong + 5.4: size of voidpf (pointer) + 7.6: size of z_off_t + + Compiler, assembler, and debug options: + 8: DEBUG + 9: ASMV or ASMINF -- use ASM code + 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention + 11: 0 (reserved) + + One-time table building (smaller code, but not thread-safe if true): + 12: BUILDFIXED -- build static block decoding tables when needed + 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed + 14,15: 0 (reserved) + + Library content (indicates missing functionality): + 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking + deflate code when not needed) + 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect + and decode gzip streams (to avoid linking crc code) + 18-19: 0 (reserved) + + Operation variations (changes in library functionality): + 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate + 21: FASTEST -- deflate algorithm with only one, lowest compression level + 22,23: 0 (reserved) + + The sprintf variant used by gzprintf (zero is best): + 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format + 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! + 26: 0 = returns value, 1 = void -- 1 means inferred string length returned + + Remainder: + 27-31: 0 (reserved) + */ + + + /* utility functions */ + +/* + The following utility functions are implemented on top of the + basic stream-oriented functions. To simplify the interface, some + default options are assumed (compression level and memory usage, + standard memory allocation functions). The source code of these + utility functions can easily be modified if you need special options. +*/ + +ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Compresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total + size of the destination buffer, which must be at least the value returned + by compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed buffer. + This function can be used to compress a whole file at once if the + input file is mmap'ed. + compress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer. +*/ + +ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)); +/* + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed buffer. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. +*/ + +ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); +/* + compressBound() returns an upper bound on the compressed size after + compress() or compress2() on sourceLen bytes. It would be used before + a compress() or compress2() call to allocate the destination buffer. +*/ + +ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Decompresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total + size of the destination buffer, which must be large enough to hold the + entire uncompressed data. (The size of the uncompressed data must have + been saved previously by the compressor and transmitted to the decompressor + by some mechanism outside the scope of this compression library.) + Upon exit, destLen is the actual size of the compressed buffer. + This function can be used to decompress a whole file at once if the + input file is mmap'ed. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. +*/ + + +typedef voidp gzFile; + +ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); +/* + Opens a gzip (.gz) file for reading or writing. The mode parameter + is as in fopen ("rb" or "wb") but can also include a compression level + ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for + Huffman only compression as in "wb1h", or 'R' for run-length encoding + as in "wb1R". (See the description of deflateInit2 for more information + about the strategy parameter.) + + gzopen can be used to read a file which is not in gzip format; in this + case gzread will directly read from the file without decompression. + + gzopen returns NULL if the file could not be opened or if there was + insufficient memory to allocate the (de)compression state; errno + can be checked to distinguish the two cases (if errno is zero, the + zlib error is Z_MEM_ERROR). */ + +ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); +/* + gzdopen() associates a gzFile with the file descriptor fd. File + descriptors are obtained from calls like open, dup, creat, pipe or + fileno (in the file has been previously opened with fopen). + The mode parameter is as in gzopen. + The next call of gzclose on the returned gzFile will also close the + file descriptor fd, just like fclose(fdopen(fd), mode) closes the file + descriptor fd. If you want to keep fd open, use gzdopen(dup(fd), mode). + gzdopen returns NULL if there was insufficient memory to allocate + the (de)compression state. +*/ + +ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); +/* + Dynamically update the compression level or strategy. See the description + of deflateInit2 for the meaning of these parameters. + gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not + opened for writing. +*/ + +ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); +/* + Reads the given number of uncompressed bytes from the compressed file. + If the input file was not in gzip format, gzread copies the given number + of bytes into the buffer. + gzread returns the number of uncompressed bytes actually read (0 for + end of file, -1 for error). */ + +ZEXTERN int ZEXPORT gzwrite OF((gzFile file, + voidpc buf, unsigned len)); +/* + Writes the given number of uncompressed bytes into the compressed file. + gzwrite returns the number of uncompressed bytes actually written + (0 in case of error). +*/ + +ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...)); +/* + Converts, formats, and writes the args to the compressed file under + control of the format string, as in fprintf. gzprintf returns the number of + uncompressed bytes actually written (0 in case of error). The number of + uncompressed bytes written is limited to 4095. The caller should assure that + this limit is not exceeded. If it is exceeded, then gzprintf() will return + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. +*/ + +ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); +/* + Writes the given null-terminated string to the compressed file, excluding + the terminating null character. + gzputs returns the number of characters written, or -1 in case of error. +*/ + +ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); +/* + Reads bytes from the compressed file until len-1 characters are read, or + a newline character is read and transferred to buf, or an end-of-file + condition is encountered. The string is then terminated with a null + character. + gzgets returns buf, or Z_NULL in case of error. +*/ + +ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); +/* + Writes c, converted to an unsigned char, into the compressed file. + gzputc returns the value that was written, or -1 in case of error. +*/ + +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +/* + Reads one byte from the compressed file. gzgetc returns this byte + or -1 in case of end of file or error. +*/ + +ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); +/* + Push one character back onto the stream to be read again later. + Only one character of push-back is allowed. gzungetc() returns the + character pushed, or -1 on failure. gzungetc() will fail if a + character has been pushed but not read yet, or if c is -1. The pushed + character will be discarded if the stream is repositioned with gzseek() + or gzrewind(). +*/ + +ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); +/* + Flushes all pending output into the compressed file. The parameter + flush is as in the deflate() function. The return value is the zlib + error number (see function gzerror below). gzflush returns Z_OK if + the flush parameter is Z_FINISH and all output could be flushed. + gzflush should be called only when strictly necessary because it can + degrade compression. +*/ + +ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, + z_off_t offset, int whence)); +/* + Sets the starting position for the next gzread or gzwrite on the + given compressed file. The offset represents a number of bytes in the + uncompressed data stream. The whence parameter is defined as in lseek(2); + the value SEEK_END is not supported. + If the file is opened for reading, this function is emulated but can be + extremely slow. If the file is opened for writing, only forward seeks are + supported; gzseek then compresses a sequence of zeroes up to the new + starting position. + + gzseek returns the resulting offset location as measured in bytes from + the beginning of the uncompressed stream, or -1 in case of error, in + particular if the file is opened for writing and the new starting position + would be before the current position. +*/ + +ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); +/* + Rewinds the given file. This function is supported only for reading. + + gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET) +*/ + +ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); +/* + Returns the starting position for the next gzread or gzwrite on the + given compressed file. This position represents a number of bytes in the + uncompressed data stream. + + gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) +*/ + +ZEXTERN int ZEXPORT gzeof OF((gzFile file)); +/* + Returns 1 when EOF has previously been detected reading the given + input stream, otherwise zero. +*/ + +ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); +/* + Returns 1 if file is being read directly without decompression, otherwise + zero. +*/ + +ZEXTERN int ZEXPORT gzclose OF((gzFile file)); +/* + Flushes all pending output if necessary, closes the compressed file + and deallocates all the (de)compression state. The return value is the zlib + error number (see function gzerror below). +*/ + +ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); +/* + Returns the error message for the last error which occurred on the + given compressed file. errnum is set to zlib error number. If an + error occurred in the file system and not in the compression library, + errnum is set to Z_ERRNO and the application may consult errno + to get the exact error code. +*/ + +ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); +/* + Clears the error and end-of-file flags for file. This is analogous to the + clearerr() function in stdio. This is useful for continuing to read a gzip + file that is being written concurrently. +*/ + + /* checksum functions */ + +/* + These functions are not related to compression but are exported + anyway because they might be useful in applications using the + compression library. +*/ + +ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); +/* + Update a running Adler-32 checksum with the bytes buf[0..len-1] and + return the updated checksum. If buf is NULL, this function returns + the required initial value for the checksum. + An Adler-32 checksum is almost as reliable as a CRC32 but can be computed + much faster. Usage example: + + uLong adler = adler32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + adler = adler32(adler, buffer, length); + } + if (adler != original_adler) error(); +*/ + +ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, + z_off_t len2)); +/* + Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 + and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for + each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. +*/ + +ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); +/* + Update a running CRC-32 with the bytes buf[0..len-1] and return the + updated CRC-32. If buf is NULL, this function returns the required initial + value for the for the crc. Pre- and post-conditioning (one's complement) is + performed within this function so it shouldn't be done by the application. + Usage example: + + uLong crc = crc32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + crc = crc32(crc, buffer, length); + } + if (crc != original_crc) error(); +*/ + +ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); + +/* + Combine two CRC-32 check values into one. For two sequences of bytes, + seq1 and seq2 with lengths len1 and len2, CRC-32 check values were + calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 + check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and + len2. +*/ + + + /* various hacks, don't look :) */ + +/* deflateInit and inflateInit are macros to allow checking the zlib version + * and the compiler's view of z_stream: + */ +ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size)); +ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size)); +#define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, sizeof(z_stream)) +#define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, sizeof(z_stream)) +#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, sizeof(z_stream)) +#define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream)) +#define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, sizeof(z_stream)) + + +#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) + struct internal_state {int dummy;}; /* hack for buggy compilers */ +#endif + +ZEXTERN const char * ZEXPORT zError OF((int)); +ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z)); +ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void)); + +#ifdef __cplusplus +} +#endif + +#endif /* ZLIB_H */