Merge branch 'merge-tokudb-5.6' into 10.0

storage/tokudb/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(TOKUDB_VERSION 5.6.38-83.0)
+SET(TOKUDB_VERSION 5.6.39-83.1)
 # PerconaFT only supports x86-64 and cmake-2.8.9+
 IF(CMAKE_VERSION VERSION_LESS "2.8.9")
   MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB")

storage/tokudb/PerconaFT/.clang-format

@@ -0,0 +1,36 @@
+Language:        Cpp
+BasedOnStyle:  Google
+
+# The following parameters are default for Google style,
+# but as they are important for our project they
+# are set explicitly here
+AlignAfterOpenBracket: Align
+BreakBeforeBinaryOperators: None
+ColumnLimit:     80
+PointerAlignment: Left
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+UseTab:          Never
+
+# Non-default parametes
+NamespaceIndentation: All
+IndentWidth:     4
+TabWidth:        4
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+BinPackParameters: false
+BinPackArguments: false
+ExperimentalAutoDetectBinPacking: false
+AllowAllParametersOfDeclarationOnNextLine: false
+#AlignConsecutiveAssignments: yes
+#AlignConsecutiveDeclarations: yes
+BreakStringLiterals: false
+ReflowComments: true

storage/tokudb/PerconaFT/CMakeLists.txt

@@ -9,6 +9,12 @@ project(TokuDB)
 set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
 set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 
+# See: https://jira.percona.com/browse/TDB-93
+IF(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-address-of-packed-member")
+  SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-address-of-packed-member")
+ENDIF()
+
 # detect when we are being built as a subproject
 if (DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
   add_definitions( -DMYSQL_TOKUDB_ENGINE=1)

storage/tokudb/PerconaFT/README.md

@@ -9,20 +9,18 @@ PerconaFT is provided as a shared library with an interface similar to
 Berkeley DB.
 
 To build the full MySQL product, see the instructions for
-[Percona/tokudb-engine][tokudb-engine].  To build TokuMX, see the instructions
-for [Percona/percona-server-mongodb][mongo].  This document covers PerconaFT only.
+[Percona/percona-server][percona-server].  This document covers PerconaFT only.
 
-[tokudb-engine]: https://github.com/Percona/tokudb-engine
-[mongo]: https://github.com/Percona/percona-server-mongodb
+[percona-server]: https://github.com/Percona/percona-server
 
 
 Building
 --------
 
 PerconaFT is built using CMake >= 2.8.9.  Out-of-source builds are
-recommended.  You need a C++11 compiler, though only GCC >= 4.7 and
-Apple's Clang are tested.  You also need zlib development packages
-(`yum install zlib-devel` or `apt-get install zlib1g-dev`).
+recommended.  You need a C++11 compiler, though only some versions
+of GCC >= 4.7 and Clang are tested.  You also need zlib development
+packages (`yum install zlib-devel` or `apt-get install zlib1g-dev`).
 
 You will also need the source code for jemalloc, checked out in
 `third_party/`.
@@ -42,16 +40,16 @@ CC=gcc47 CXX=g++47 cmake \
 cmake --build . --target install
 ```
 
-This will build `libtokudb.so` and `libtokuportability.so` and install it,
+This will build `libft.so` and `libtokuportability.so` and install it,
 some header files, and some examples to `percona-ft/prefix/`.  It will also
 build jemalloc and install it alongside these libraries, you should link
 to that if you are planning to run benchmarks or in production.
 
 ### Platforms
 
-PerconaFT is supported on 64-bit Centos, should work on other 64-bit linux
-distributions, and may work on OSX 10.8 and FreeBSD.  PerconaFT is not
-supported on 32-bit systems.
+PerconaFT is supported on 64-bit Centos, Debian, and Ubuntu and should work
+on other 64-bit linux distributions, and may work on OSX 10.8 and FreeBSD.
+PerconaFT is not supported on 32-bit systems.
 
 [Transparent hugepages][transparent-hugepages] is a feature in newer linux
 kernel versions that causes problems for the memory usage tracking
@@ -97,16 +95,9 @@ We have two publicly accessible mailing lists for TokuDB:
  - tokudb-dev@googlegroups.com is for discussion of the development of
    TokuDB.
 
-and two for TokuMX:
-
- - tokumx-user@googlegroups.com is for general and support related
-   questions about the use of TokuMX.
- - tokumx-dev@googlegroups.com is for discussion of the development of
-   TokuMX.
-
 All source code and test contributions must be provided under a [BSD 2-Clause][bsd-2] license. For any small change set, the license text may be contained within the commit comment and the pull request. For larger contributions, the license must be presented in a COPYING.<feature_name> file in the root of the PerconaFT project. Please see the [BSD 2-Clause license template][bsd-2] for the content of the license text.
 
-[jira]: https://tokutek.atlassian.net/browse/FT/
+[jira]: https://jira.percona.com/projects/TDB
 [bsd-2]: http://opensource.org/licenses/BSD-2-Clause/
 
 

storage/tokudb/PerconaFT/ft/ft-ops.cc

@@ -4880,6 +4880,94 @@ static void toku_pfs_keys_init(const char *toku_instr_group_name) {
     toku_instr_probe_1 = new toku_instr_probe(*fti_probe_1_key);
 }
 
+static void toku_pfs_keys_destroy(void) {
+    delete kibbutz_mutex_key;
+    delete minicron_p_mutex_key;
+    delete queue_result_mutex_key;
+    delete tpool_lock_mutex_key;
+    delete workset_lock_mutex_key;
+    delete bjm_jobs_lock_mutex_key;
+    delete log_internal_lock_mutex_key;
+    delete cachetable_ev_thread_lock_mutex_key;
+    delete cachetable_disk_nb_mutex_key;
+    delete safe_file_size_lock_mutex_key;
+    delete cachetable_m_mutex_key;
+    delete checkpoint_safe_mutex_key;
+    delete ft_ref_lock_mutex_key;
+    delete ft_open_close_lock_mutex_key;
+    delete loader_error_mutex_key;
+    delete bfs_mutex_key;
+    delete loader_bl_mutex_key;
+    delete loader_fi_lock_mutex_key;
+    delete loader_out_mutex_key;
+    delete result_output_condition_lock_mutex_key;
+    delete block_table_mutex_key;
+    delete rollback_log_node_cache_mutex_key;
+    delete txn_lock_mutex_key;
+    delete txn_state_lock_mutex_key;
+    delete txn_child_manager_mutex_key;
+    delete txn_manager_lock_mutex_key;
+    delete treenode_mutex_key;
+    delete locktree_request_info_mutex_key;
+    delete locktree_request_info_retry_mutex_key;
+    delete manager_mutex_key;
+    delete manager_escalation_mutex_key;
+    delete db_txn_struct_i_txn_mutex_key;
+    delete manager_escalator_mutex_key;
+    delete indexer_i_indexer_lock_mutex_key;
+    delete indexer_i_indexer_estimate_lock_mutex_key;
+
+    delete tokudb_file_data_key;
+    delete tokudb_file_load_key;
+    delete tokudb_file_tmp_key;
+    delete tokudb_file_log_key;
+
+    delete fti_probe_1_key;
+
+    delete extractor_thread_key;
+    delete fractal_thread_key;
+    delete io_thread_key;
+    delete eviction_thread_key;
+    delete kibbutz_thread_key;
+    delete minicron_thread_key;
+    delete tp_internal_thread_key;
+
+    delete result_state_cond_key;
+    delete bjm_jobs_wait_key;
+    delete cachetable_p_refcount_wait_key;
+    delete cachetable_m_flow_control_cond_key;
+    delete cachetable_m_ev_thread_cond_key;
+    delete bfs_cond_key;
+    delete result_output_condition_key;
+    delete manager_m_escalator_done_key;
+    delete lock_request_m_wait_cond_key;
+    delete queue_result_cond_key;
+    delete ws_worker_wait_key;
+    delete rwlock_wait_read_key;
+    delete rwlock_wait_write_key;
+    delete rwlock_cond_key;
+    delete tp_thread_wait_key;
+    delete tp_pool_wait_free_key;
+    delete frwlock_m_wait_read_key;
+    delete kibbutz_k_cond_key;
+    delete minicron_p_condvar_key;
+    delete locktree_request_info_retry_cv_key;
+
+    delete multi_operation_lock_key;
+    delete low_priority_multi_operation_lock_key;
+    delete cachetable_m_list_lock_key;
+    delete cachetable_m_pending_lock_expensive_key;
+    delete cachetable_m_pending_lock_cheap_key;
+    delete cachetable_m_lock_key;
+    delete result_i_open_dbs_rwlock_key;
+    delete checkpoint_safe_rwlock_key;
+    delete cachetable_value_key;
+    delete safe_file_size_lock_rwlock_key;
+
+    delete cachetable_disk_nb_rwlock_key;
+    delete toku_instr_probe_1;
+}
+
 int toku_ft_layer_init(void) {
     int r = 0;
 
@@ -4916,8 +5004,7 @@ void toku_ft_layer_destroy(void) {
     toku_status_destroy();
     partitioned_counters_destroy();
     toku_scoped_malloc_destroy();
-
-    delete toku_instr_probe_1;
+    toku_pfs_keys_destroy();
 
     // Portability must be cleaned up last
     toku_portability_destroy();

storage/tokudb/PerconaFT/ft/tests/ft-clock-test.cc

@@ -184,11 +184,11 @@ static void test2(int fd, FT ft_h, FTNODE *dn) {
     PAIR_ATTR attr;
     memset(&attr, 0, sizeof(attr));
     toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
-    invariant(BP_STATE(*dn, 0) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
+    invariant(BP_STATE(*dn, 0) == ((is_leaf) ? PT_ON_DISK : PT_COMPRESSED));
     invariant(BP_STATE(*dn, 1) == PT_AVAIL);
     invariant(BP_SHOULD_EVICT(*dn, 1));
     toku_ftnode_pe_callback(*dn, attr, ft_h, def_pe_finalize_impl, nullptr);
-    invariant(BP_STATE(*dn, 1) == (is_leaf) ? PT_ON_DISK : PT_COMPRESSED);
+    invariant(BP_STATE(*dn, 1) == ((is_leaf) ? PT_ON_DISK : PT_COMPRESSED));
 
     bool req = toku_ftnode_pf_req_callback(*dn, &bfe_subset);
     invariant(req);

storage/tokudb/PerconaFT/ft/tests/log-test4.cc

@@ -54,7 +54,7 @@ test_main (int argc __attribute__((__unused__)),
     {
 	ml_lock(&logger->input_lock);
 	toku_logger_make_space_in_inbuf(logger, 5);
-	snprintf(logger->inbuf.buf+logger->inbuf.n_in_buf, 5, "a1234");
+	memcpy(logger->inbuf.buf+logger->inbuf.n_in_buf, "a1234", 5);
 	logger->inbuf.n_in_buf+=5;
 	logger->lsn.lsn++;
 	logger->inbuf.max_lsn_in_buf = logger->lsn;

storage/tokudb/PerconaFT/portability/tests/test-max-data.cc

@@ -64,7 +64,7 @@ int main(int argc, char *const argv[]) {
     if (verbose) printf("maxdata=%" PRIu64 " 0x%" PRIx64 "\n", maxdata, maxdata);
 
     // check the data size
-#if defined(__x86_64__) || defined(__aarch64__)
+#if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__)
     assert(maxdata > (1ULL << 32));
 #elif __i386__
     assert(maxdata < (1ULL << 32));

storage/tokudb/PerconaFT/portability/toku_instrumentation.h

@@ -52,6 +52,8 @@ class toku_instr_key {
                    UU(const char *name)) {}
 
     explicit toku_instr_key(UU(pfs_key_t key_id)) {}
+
+    ~toku_instr_key() {}
 };
 
 typedef toku_instr_probe_empty toku_instr_probe;

storage/tokudb/PerconaFT/portability/toku_portability.h

@@ -157,7 +157,7 @@ extern "C" {
 #endif
 
 // Deprecated functions.
-#if !defined(TOKU_ALLOW_DEPRECATED)
+#if !defined(TOKU_ALLOW_DEPRECATED) && !defined(__clang__)
 int      creat(const char *pathname, mode_t mode)   __attribute__((__deprecated__));
 int      fstat(int fd, struct stat *buf)            __attribute__((__deprecated__));
 int      stat(const char *path, struct stat *buf)   __attribute__((__deprecated__));

storage/tokudb/PerconaFT/portability/toku_pthread.h

@@ -168,11 +168,7 @@ typedef struct toku_mutex_aligned {
     }
 #else  // __linux__, at least
 #define ZERO_COND_INITIALIZER \
-    {                         \
-        {                     \
-            { 0 }             \
-        }                     \
-    }
+    {}
 #endif
 
 static inline void toku_mutexattr_init(toku_pthread_mutexattr_t *attr) {

storage/tokudb/PerconaFT/portability/toku_time.h

@@ -43,6 +43,9 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include <time.h>
 #include <sys/time.h>
 #include <stdint.h>
+#if defined(__powerpc__)
+# include <sys/platform/ppc.h>
+#endif
 
 static inline float toku_tdiff (struct timeval *a, struct timeval *b) {
     return (float)((a->tv_sec - b->tv_sec) + 1e-6 * (a->tv_usec - b->tv_usec));
@@ -106,6 +109,8 @@ static inline tokutime_t toku_time_now(void) {
     uint64_t result;
     __asm __volatile__ ("mrs %[rt], cntvct_el0" : [rt] "=r" (result));
     return result;
+#elif defined(__powerpc__)
+    return __ppc_get_timebase();
 #else
 #error No timer implementation for this platform
 #endif

storage/tokudb/PerconaFT/src/tests/checkpoint_stress.cc

@@ -351,7 +351,7 @@ test_main (int argc, char * const argv[]) {
                 // arg that suppresses valgrind on this child process
                 break;
             }
-            // otherwise, fall through to an error
+            /* fall through */ // otherwise, fall through to an error
 	case 'h':
         case '?':
             usage(argv[0]);

storage/tokudb/PerconaFT/src/tests/directory_lock.cc

@@ -69,7 +69,7 @@ static void verify_shared_ops_fail(DB_ENV* env, DB* db) {
     uint32_t flags = 0;
     DBT key,val;
     DBT in_key,in_val;
-    uint32_t in_key_data, in_val_data = 0;
+    uint32_t in_key_data = 0, in_val_data = 0;
     memset(&in_key, 0, sizeof(in_key));
     memset(&in_val, 0, sizeof(in_val));
     in_key.size = sizeof(in_key_data);

storage/tokudb/PerconaFT/src/tests/loader-cleanup-test.cc

@@ -172,12 +172,12 @@ err_type_str (enum test_type t) {
     case einval_o:       return "open";
     case enospc_fc:      return "fclose";
     case abort_via_poll: return "abort_via_poll";
-    case commit:         assert(0);
-    case abort_txn:      assert(0);
-    case abort_loader:   assert(0);
+    case commit:         abort();
+    case abort_txn:      abort();
+    case abort_loader:   abort();
     }
     // I know that Barry prefers the single-return case, but writing the code this way means that the compiler will complain if I forget something in the enum. -Bradley
-    assert(0);
+    abort();
     return NULL;
 }
 
@@ -193,12 +193,12 @@ err_msg_type_str (enum test_type t) {
     case einval_o:       return "EINVAL";
     case enospc_fc:      return "ENOSPC";
     case abort_via_poll: return "non-zero";
-    case commit:         assert(0);
-    case abort_txn:      assert(0);
-    case abort_loader:   assert(0);
+    case commit:         abort();
+    case abort_txn:      abort();
+    case abort_loader:   abort();
     }
     // I know that Barry prefers the single-return case, but writing the code this way means that the compiler will complain if I forget something in the enum. -Bradley
-    assert(0);
+    abort();
     return NULL;
 }
 
@@ -873,7 +873,7 @@ static void run_test(enum test_type t, int trigger)
     case abort_via_poll:
 	poll_count_trigger  = trigger;       break;
     default:
-	assert(0);
+	abort();
     }
 
 

storage/tokudb/PerconaFT/src/tests/recover-del-multiple-abort.cc

@@ -81,7 +81,7 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
         memcpy(dest_key->data, &pri_data[dbnum], dest_key->size);
         break;
     default:
-        assert(0);
+        abort();
     }
 
     if (dest_val) {
@@ -95,9 +95,9 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
             }
             break;
         case DB_DBT_REALLOC:
-            assert(0);
+            abort();
         default:
-            assert(0);
+            abort();
         }
     }
     

storage/tokudb/PerconaFT/src/tests/recover-del-multiple-srcdb-fdelete-all.cc

@@ -85,7 +85,7 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
         memcpy(dest_key->data, &pri_data[dbnum], dest_key->size);
         break;
     default:
-        assert(0);
+        abort();
     }
 
     if (dest_val) {
@@ -99,9 +99,9 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
             }
             break;
         case DB_DBT_REALLOC:
-            assert(0);
+            abort();
         default:
-            assert(0);
+            abort();
         }
     }
     

storage/tokudb/PerconaFT/src/tests/recover-del-multiple.cc

@@ -84,7 +84,7 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
         memcpy(dest_key->data, &pri_data[dbnum], dest_key->size);
         break;
     default:
-        assert(0);
+        abort();
     }
 
     if (dest_val) {
@@ -98,9 +98,9 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
             }
             break;
         case DB_DBT_REALLOC:
-            assert(0);
+            abort();
         default:
-            assert(0);
+            abort();
         }
     }
     

storage/tokudb/PerconaFT/src/tests/recover-put-multiple-abort.cc

@@ -81,7 +81,7 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
         memcpy(dest_key->data, &pri_data[dbnum], dest_key->size);
         break;
     default:
-        assert(0);
+        abort();
     }
 
     if (dest_val) {
@@ -95,9 +95,9 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals
             }
             break;
         case DB_DBT_REALLOC:
-            assert(0);
+            abort();
         default:
-            assert(0);
+            abort();
         }
     }
     

storage/tokudb/PerconaFT/src/tests/recovery_fileops_unit.cc

@@ -158,7 +158,7 @@ do_args(int argc, char * const argv[]) {
         choices[i] = -1;
     }
 
-    char c;
+    signed char c;
     while ((c = getopt(argc, argv, "vqhcrO:A:B:C:D:E:F:G:H:I:J:X:")) != -1) {
         switch (c) {
             case 'v':
@@ -217,7 +217,7 @@ do_args(int argc, char * const argv[]) {
                     // arg that suppresses valgrind on this child process
                     break;
                 }
-            // otherwise, fall through to an error
+                /* fall through */ // otherwise, fall through to an error
 	default:
             usage();
             break;

storage/tokudb/PerconaFT/src/tests/test-prepare3.cc

@@ -128,6 +128,7 @@ static void check_prepared_list (enum prepared_state ps[NTXNS], long count, DB_P
 	    goto next;
 	case PREPARED:
 	    count_prepared++;
+            /* fall through */
 	case MAYBE_COMMITTED:
 	case MAYBE_ABORTED:
 	    count_maybe_prepared++;

storage/tokudb/hatoku_hton.cc

@@ -972,7 +972,7 @@ static bool tokudb_sync_on_prepare(void) {
 }   
 
 static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) {
-    TOKUDB_DBUG_ENTER("");
+    TOKUDB_DBUG_ENTER("%u", all);
     TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "enter");
     int r = 0;
 
@@ -1000,6 +1000,22 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) {
         r = txn->xa_prepare(txn, &thd_xid, syncflag);
         // test hook to induce a crash on a debug build
         DBUG_EXECUTE_IF("tokudb_crash_prepare_after", DBUG_SUICIDE(););
+
+        // XA log entries can be interleaved in the binlog since XA prepare on the master
+        // flushes to the binlog.  There can be log entries from different clients pushed
+        // into the binlog before XA commit is executed on the master.  Therefore, the slave
+        // thread must be able to juggle multiple XA transactions.  Tokudb does this by
+        // zapping the client transaction context on the slave when executing the XA prepare
+        // and expecting to process XA commit with commit_by_xid (which supplies the XID so
+        // that the transaction can be looked up and committed).
+        if (r == 0 && all && thd->slave_thread) {
+            TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "zap txn context %u", thd_sql_command(thd));
+            if (thd_sql_command(thd) == SQLCOM_XA_PREPARE) {
+                trx->all = NULL;
+                trx->sub_sp_level = NULL;
+                trx->sp_level = NULL;
+            }
+        }
     } else {
         TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "nothing to prepare %d", all);
     }
@@ -1030,6 +1046,7 @@ static int tokudb_xa_recover(handlerton* hton, XID* xid_list, uint len) {
 static int tokudb_commit_by_xid(handlerton* hton, XID* xid) {
     TOKUDB_DBUG_ENTER("");
     TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "enter");
+    TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "xid %p", xid);
     int r = 0;
     DB_TXN* txn = NULL;
     TOKU_XA_XID* toku_xid = (TOKU_XA_XID*)xid;
@@ -1049,6 +1066,7 @@ cleanup:
 static int tokudb_rollback_by_xid(handlerton* hton, XID*  xid) {
     TOKUDB_DBUG_ENTER("");
     TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "enter");
+    TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_XA, "xid %p", xid);
     int r = 0;
     DB_TXN* txn = NULL;
     TOKU_XA_XID* toku_xid = (TOKU_XA_XID*)xid;