FT-278 Put ft_msg in its own class and file, organize deserialization

and upgrade code so there's less complexity in ft/ft_node-serialize.cc

ft/CMakeLists.txt

@@ -35,7 +35,6 @@ set(FT_SOURCES
   ft-cachetable-wrappers
   ft-flusher
   ft-hot-flusher
-  ft_msg
   ft_node-serialize
   ft-node-deserialize
   ft-ops
@@ -52,6 +51,7 @@ set(FT_SOURCES
   logfilemgr
   logger
   log_upgrade
+  msg
   msg_buffer
   node
   pivotkeys

ft/ft-flusher.cc

@@ -1682,7 +1682,7 @@ void toku_bnc_flush_to_child(FT ft, NONLEAF_CHILDINFO bnc, FTNODE child, TXNID p
             ft(t), child(n), bnc(nl), gc_info(g), remaining_memsize(bnc->msg_buffer.buffer_size_in_use()) {
             stats_delta = { 0, 0 };
         }
-        int operator()(FT_MSG msg, bool is_fresh) {
+        int operator()(const ft_msg &msg, bool is_fresh) {
             size_t flow_deltas[] = { 0, 0 };
             size_t memsize_in_buffer = message_buffer::msg_memsize_in_buffer(msg);
             if (remaining_memsize <= bnc->flow[0]) {

ft/ft-internal.h

@@ -119,8 +119,6 @@ PATENT RIGHTS GRANT:
 struct block_table;
 struct ft_search;
 
-enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
-enum { FT_MSG_OVERHEAD = (2 + sizeof(MSN)) };   // the type plus freshness plus MSN
 enum { FT_DEFAULT_FANOUT = 16 };
 enum { FT_DEFAULT_NODE_SIZE = 4 * 1024 * 1024 };
 enum { FT_DEFAULT_BASEMENT_NODE_SIZE = 128 * 1024 };
@@ -493,7 +491,7 @@ toku_bfe_rightmost_child_wanted(struct ftnode_fetch_extra *bfe, FTNODE node);
 // allocate a block number
 // allocate and initialize a ftnode
 // put the ftnode into the cache table
-void toku_create_new_ftnode (FT_HANDLE t, FTNODE *result, int height, int n_children);
+void toku_create_new_ftnode(FT_HANDLE ft_handle, FTNODE *result, int height, int n_children);
 
 /* Stuff for testing */
 // toku_testsetup_initialize() must be called before any other test_setup_xxx() functions are called.
@@ -506,14 +504,10 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_h, BLOCKNUM, const char *key, in
 int toku_testsetup_insert_to_nonleaf (FT_HANDLE ft_h, BLOCKNUM, enum ft_msg_type, const char *key, int keylen, const char *val, int vallen);
 void toku_pin_node_with_min_bfe(FTNODE* node, BLOCKNUM b, FT_HANDLE t);
 
-void toku_ft_root_put_msg(FT ft, FT_MSG msg, txn_gc_info *gc_info);
+void toku_ft_root_put_msg(FT ft, const ft_msg &msg, txn_gc_info *gc_info);
 
-void
-toku_get_node_for_verify(
-    BLOCKNUM blocknum,
-    FT_HANDLE ft_h,
-    FTNODE* nodep
-    );
+// TODO: Rename
+void toku_get_node_for_verify(BLOCKNUM blocknum, FT_HANDLE ft_h, FTNODE* nodep);
 
 int
 toku_verify_ftnode (FT_HANDLE ft_h,

ft/ft-ops.cc

@@ -208,7 +208,7 @@ basement nodes, bulk fetch,  and partial fetch:
 #include "ft/ft-flusher.h"
 #include "ft/ft-internal.h"
 #include "ft/ft_layout_version.h"
-#include "ft/ft_msg.h"
+#include "ft/msg.h"
 #include "ft/leafentry.h"
 #include "ft/log-internal.h"
 #include "ft/node.h"
@@ -1578,20 +1578,11 @@ ft_init_new_root(FT ft, FTNODE oldroot, FTNODE *newrootp)
         );
 }
 
-// TODO Use this function to clean up other places where bits of messages are passed around
-//      such as toku_bnc_insert_msg() and the call stack above it.
-static uint64_t
-ft_msg_size(FT_MSG msg) {
-    size_t keyval_size = msg->u.id.key->size + msg->u.id.val->size;
-    size_t xids_size = xids_get_serialize_size(msg->xids);
-    return keyval_size + KEY_VALUE_OVERHEAD + FT_MSG_OVERHEAD + xids_size;
-}
-
 static void inject_message_in_locked_node(
     FT ft, 
     FTNODE node, 
     int childnum, 
-    FT_MSG_S *msg, 
+    const ft_msg &msg, 
     size_t flow_deltas[],
     txn_gc_info *gc_info
     ) 
@@ -1616,15 +1607,17 @@ static void inject_message_in_locked_node(
     // Get the MSN from the header.  Now that we have a write lock on the
     // node we're injecting into, we know no other thread will get an MSN
     // after us and get that message into our subtree before us.
-    msg->msn.msn = toku_sync_add_and_fetch(&ft->h->max_msn_in_ft.msn, 1);
-    paranoid_invariant(msg->msn.msn > node->max_msn_applied_to_node_on_disk.msn);
+    MSN msg_msn = { .msn = toku_sync_add_and_fetch(&ft->h->max_msn_in_ft.msn, 1) };
+    ft_msg msg_with_msn(msg.kdbt(), msg.vdbt(), msg.type(), msg_msn, msg.xids());
+    paranoid_invariant(msg_with_msn.msn().msn > node->max_msn_applied_to_node_on_disk.msn);
+
     STAT64INFO_S stats_delta = {0,0};
     toku_ftnode_put_msg(
         ft->cmp,
         ft->update_fun,
         node,
         childnum,
-        msg,
+        msg_with_msn,
         true,
         gc_info,
         flow_deltas,
@@ -1642,17 +1635,17 @@ static void inject_message_in_locked_node(
 
     // update some status variables
     if (node->height != 0) {
-        uint64_t msgsize = ft_msg_size(msg);
+        size_t msgsize = msg.total_size();
         STATUS_INC(FT_MSG_BYTES_IN, msgsize);
         STATUS_INC(FT_MSG_BYTES_CURR, msgsize);
         STATUS_INC(FT_MSG_NUM, 1);
-        if (ft_msg_type_applies_all(msg->type)) {
+        if (ft_msg_type_applies_all(msg.type())) {
             STATUS_INC(FT_MSG_NUM_BROADCAST, 1);
         }
     }
 
     // verify that msn of latest message was captured in root node
-    paranoid_invariant(msg->msn.msn == node->max_msn_applied_to_node_on_disk.msn);
+    paranoid_invariant(msg_with_msn.msn().msn == node->max_msn_applied_to_node_on_disk.msn);
 
     if (node->blocknum.b == ft->rightmost_blocknum.b) {
         if (ft->seqinsert_score < FT_SEQINSERT_SCORE_THRESHOLD) {
@@ -1794,7 +1787,7 @@ static bool process_maybe_reactive_child(FT ft, FTNODE parent, FTNODE child, int
     abort();
 }
 
-static void inject_message_at_this_blocknum(FT ft, CACHEKEY cachekey, uint32_t fullhash, FT_MSG_S *msg, size_t flow_deltas[], txn_gc_info *gc_info)
+static void inject_message_at_this_blocknum(FT ft, CACHEKEY cachekey, uint32_t fullhash, const ft_msg &msg, size_t flow_deltas[], txn_gc_info *gc_info)
 // Effect:
 //  Inject message into the node at this blocknum (cachekey).
 //  Gets a write lock on the node for you.
@@ -1845,7 +1838,7 @@ static void push_something_in_subtree(
     FT ft, 
     FTNODE subtree_root, 
     int target_childnum, 
-    FT_MSG_S *msg, 
+    const ft_msg &msg, 
     size_t flow_deltas[], 
     txn_gc_info *gc_info,
     int depth, 
@@ -1903,10 +1896,10 @@ static void push_something_in_subtree(
         NONLEAF_CHILDINFO bnc;
 
         // toku_ft_root_put_msg should not have called us otherwise.
-        paranoid_invariant(ft_msg_type_applies_once(msg->type));
+        paranoid_invariant(ft_msg_type_applies_once(msg.type()));
 
         childnum = (target_childnum >= 0 ? target_childnum
-                    : toku_ftnode_which_child(subtree_root, msg->u.id.key, ft->cmp));
+                    : toku_ftnode_which_child(subtree_root, msg.kdbt(), ft->cmp));
         bnc = BNC(subtree_root, childnum);
 
         if (toku_bnc_n_entries(bnc) > 0) {
@@ -2042,7 +2035,7 @@ static void push_something_in_subtree(
 
 void toku_ft_root_put_msg(
     FT ft, 
-    FT_MSG_S *msg, 
+    const ft_msg &msg, 
     txn_gc_info *gc_info
     )
 // Effect:
@@ -2142,7 +2135,7 @@ void toku_ft_root_put_msg(
     // anyway.
 
     // Now, either inject here or promote.  We decide based on a heuristic:
-    if (node->height == 0 || !ft_msg_type_applies_once(msg->type)) {
+    if (node->height == 0 || !ft_msg_type_applies_once(msg.type())) {
         // If the root's a leaf or we're injecting a broadcast, drop the read lock and inject here.
         toku_unpin_ftnode_read_only(ft, node);
         STATUS_INC(FT_PRO_NUM_ROOT_H0_INJECT, 1);
@@ -2153,7 +2146,7 @@ void toku_ft_root_put_msg(
     } else {
         // The root's height 1.  We may be eligible for promotion here.
         // On the extremes, we want to promote, in the middle, we don't.
-        int childnum = toku_ftnode_which_child(node, msg->u.id.key, ft->cmp);
+        int childnum = toku_ftnode_which_child(node, msg.kdbt(), ft->cmp);
         if (childnum == 0 || childnum == node->n_children - 1) {
             // On the extremes, promote.  We know which childnum we're going to, so pass that down too.
             push_something_in_subtree(ft, node, childnum, msg, flow_deltas, gc_info, 0, LEFT_EXTREME | RIGHT_EXTREME, false);
@@ -2476,7 +2469,7 @@ void toku_ft_optimize (FT_HANDLE ft_h) {
         DBT val;
         toku_init_dbt(&key);
         toku_init_dbt(&val);
-        FT_MSG_S ftmsg = { FT_OPTIMIZE, ZERO_MSN, message_xids, .u = { .id = {&key,&val} } };
+        ft_msg msg(&key, &val, FT_OPTIMIZE, ZERO_MSN, message_xids);
 
         TXN_MANAGER txn_manager = toku_ft_get_txn_manager(ft_h);
         txn_manager_state txn_state_for_gc(txn_manager);
@@ -2487,7 +2480,7 @@ void toku_ft_optimize (FT_HANDLE ft_h) {
                             // no messages above us, we can implicitly promote uxrs based on this xid
                             oldest_referenced_xid_estimate,
                             true);
-        toku_ft_root_put_msg(ft_h->ft, &ftmsg, &gc_info);
+        toku_ft_root_put_msg(ft_h->ft, msg, &gc_info);
         xids_destroy(&message_xids);
     }
 }
@@ -2601,17 +2594,13 @@ static void ft_insert_directly_into_leaf(FT ft, FTNODE leaf, int target_childnum
 //           algorithm would have selected the given leaf node as the point of injection.
 //           That means this function relies on the current implementation of promotion.
 {
-    FT_MSG_S ftcmd = { type, ZERO_MSN, message_xids, .u = { .id = { key, val } } }; 
+    ft_msg msg(key, val, type, ZERO_MSN, message_xids);
     size_t flow_deltas[] = { 0, 0 }; 
-    inject_message_in_locked_node(ft, leaf, target_childnum, &ftcmd, flow_deltas, gc_info);
+    inject_message_in_locked_node(ft, leaf, target_childnum, msg, flow_deltas, gc_info);
 }
 
 static void
-ft_send_update_msg(FT_HANDLE ft_h, FT_MSG_S *msg, TOKUTXN txn) {
-    msg->xids = (txn
-                 ? toku_txn_get_xids(txn)
-                 : xids_get_root_xids());
-    
+ft_send_update_msg(FT_HANDLE ft_h, const ft_msg &msg, TOKUTXN txn) {
     TXN_MANAGER txn_manager = toku_ft_get_txn_manager(ft_h);
     txn_manager_state txn_state_for_gc(txn_manager);
 
@@ -2650,9 +2639,9 @@ void toku_ft_maybe_update(FT_HANDLE ft_h, const DBT *key, const DBT *update_func
     if (oplsn_valid && oplsn.lsn <= (treelsn = toku_ft_checkpoint_lsn(ft_h->ft)).lsn) {
         // do nothing
     } else {
-        FT_MSG_S msg = { FT_UPDATE, ZERO_MSN, NULL,
-                         .u = { .id = { key, update_function_extra } } };
-        ft_send_update_msg(ft_h, &msg, txn);
+        XIDS message_xids = txn ? toku_txn_get_xids(txn) : xids_get_root_xids();
+        ft_msg msg(key, update_function_extra, FT_UPDATE, ZERO_MSN, message_xids);
+        ft_send_update_msg(ft_h, msg, txn);
     }
 }
 
@@ -2682,23 +2671,22 @@ void toku_ft_maybe_update_broadcast(FT_HANDLE ft_h, const DBT *update_function_e
         oplsn.lsn <= (treelsn = toku_ft_checkpoint_lsn(ft_h->ft)).lsn) {
 
     } else {
-        DBT nullkey;
-        const DBT *nullkeyp = toku_init_dbt(&nullkey);
-        FT_MSG_S msg = { FT_UPDATE_BROADCAST_ALL, ZERO_MSN, NULL,
-                         .u = { .id = { nullkeyp, update_function_extra } } };
-        ft_send_update_msg(ft_h, &msg, txn);
+        DBT empty_dbt;
+        XIDS message_xids = txn ? toku_txn_get_xids(txn) : xids_get_root_xids();
+        ft_msg msg(toku_init_dbt(&empty_dbt), update_function_extra, FT_UPDATE_BROADCAST_ALL, ZERO_MSN, message_xids);
+        ft_send_update_msg(ft_h, msg, txn);
     }
 }
 
 void toku_ft_send_insert(FT_HANDLE ft_handle, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info) {
-    FT_MSG_S ftmsg = { type, ZERO_MSN, xids, .u = { .id = { key, val } } };
-    toku_ft_root_put_msg(ft_handle->ft, &ftmsg, gc_info);
+    ft_msg msg(key, val, type, ZERO_MSN, xids);
+    toku_ft_root_put_msg(ft_handle->ft, msg, gc_info);
 }
 
 void toku_ft_send_commit_any(FT_HANDLE ft_handle, DBT *key, XIDS xids, txn_gc_info *gc_info) {
     DBT val;
-    FT_MSG_S ftmsg = { FT_COMMIT_ANY, ZERO_MSN, xids, .u = { .id = { key, toku_init_dbt(&val) } } };
-    toku_ft_root_put_msg(ft_handle->ft, &ftmsg, gc_info);
+    ft_msg msg(key, toku_init_dbt(&val), FT_COMMIT_ANY, ZERO_MSN, xids);
+    toku_ft_root_put_msg(ft_handle->ft, msg, gc_info);
 }
 
 void toku_ft_delete(FT_HANDLE ft_handle, DBT *key, TOKUTXN txn) {
@@ -2769,8 +2757,8 @@ void toku_ft_maybe_delete(FT_HANDLE ft_h, DBT *key, TOKUTXN txn, bool oplsn_vali
 
 void toku_ft_send_delete(FT_HANDLE ft_handle, DBT *key, XIDS xids, txn_gc_info *gc_info) {
     DBT val; toku_init_dbt(&val);
-    FT_MSG_S ftmsg = { FT_DELETE_ANY, ZERO_MSN, xids, .u = { .id = { key, &val } } };
-    toku_ft_root_put_msg(ft_handle->ft, &ftmsg, gc_info);
+    ft_msg msg(key, toku_init_dbt(&val), FT_DELETE_ANY, ZERO_MSN, xids);
+    toku_ft_root_put_msg(ft_handle->ft, msg, gc_info);
 }
 
 /* ******************** open,close and create  ********************** */
@@ -4480,12 +4468,12 @@ toku_dump_ftnode (FILE *file, FT_HANDLE ft_handle, BLOCKNUM blocknum, int depth,
                     FILE *file;
                     int depth;
                     print_msg_fn(FILE *f, int d) : file(f), depth(d) { }
-                    int operator()(FT_MSG msg, bool UU(is_fresh)) {
+                    int operator()(const ft_msg &msg, bool UU(is_fresh)) {
                         fprintf(file, "%*s xid=%" PRIu64 " %u (type=%d) msn=0x%" PRIu64 "\n",
                                       depth+2, "",
-                                      xids_get_innermost_xid(ft_msg_get_xids(msg)),
-                                      (unsigned)toku_dtoh32(*(int*)ft_msg_get_key(msg)),
-                                      ft_msg_get_type(msg), msg->msn.msn);
+                                      xids_get_innermost_xid(msg.xids()),
+                                      static_cast<unsigned>(toku_dtoh32(*(int*)msg.kdbt()->data)),
+                                      msg.type(), msg.msn().msn);
                         return 0;
                     }
                 } print_fn(file, depth);

ft/ft-ops.h

@@ -99,7 +99,7 @@ PATENT RIGHTS GRANT:
 #include "cachetable.h"
 #include "log.h"
 #include "compress.h"
-#include "ft_msg.h"
+#include "ft/msg.h"
 
 int toku_open_ft_handle (const char *fname, int is_create, FT_HANDLE *, int nodesize, int basementnodesize, enum toku_compression_method compression_method, CACHETABLE, TOKUTXN, int(*)(DB *,const DBT*,const DBT*)) __attribute__ ((warn_unused_result));
 

ft/ft-test-helpers.cc

@@ -224,20 +224,17 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const
     toku_verify_or_set_counts(node);
     assert(node->height==0);
 
-    DBT keydbt,valdbt;
-    MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(),
-                     .u = { .id = { toku_fill_dbt(&keydbt, key, keylen),
-                                    toku_fill_dbt(&valdbt, val, vallen) } } };
+    DBT kdbt, vdbt;
+    ft_msg msg(toku_fill_dbt(&kdbt, key, keylen), toku_fill_dbt(&vdbt, val, vallen),
+               FT_INSERT, next_dummymsn(), xids_get_root_xids());
 
     static size_t zero_flow_deltas[] = { 0, 0 };
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
-    toku_ftnode_put_msg(
-        ft_handle->ft->cmp,
+    toku_ftnode_put_msg(ft_handle->ft->cmp,
                         ft_handle->ft->update_fun,
                         node,
                         -1,
-        &msg,
+                        msg,
                         true,
                         &gc_info,
                         zero_flow_deltas,

ft/ft-verify.cc

@@ -204,13 +204,13 @@ int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct ve
     int keep_going_on_failure = e->keep_going_on_failure;
     int result = 0;
     DBT k, v;
-    FT_MSG_S msg = e->msg_buffer->get_message(offset, &k, &v);
+    ft_msg msg = e->msg_buffer->get_message(offset, &k, &v);
     bool is_fresh = e->msg_buffer->get_freshness(offset);
     if (e->broadcast) {
-        VERIFY_ASSERTION(ft_msg_type_applies_all((enum ft_msg_type) msg.type) || ft_msg_type_does_nothing((enum ft_msg_type) msg.type),
+        VERIFY_ASSERTION(ft_msg_type_applies_all((enum ft_msg_type) msg.type()) || ft_msg_type_does_nothing((enum ft_msg_type) msg.type()),
                          e->i, "message found in broadcast list that is not a broadcast");
     } else {
-        VERIFY_ASSERTION(ft_msg_type_applies_once((enum ft_msg_type) msg.type),
+        VERIFY_ASSERTION(ft_msg_type_applies_once((enum ft_msg_type) msg.type()),
                          e->i, "message found in fresh or stale message tree that does not apply once");
         if (e->is_fresh) {
             if (e->messages_have_been_moved) {
@@ -322,14 +322,14 @@ struct verify_msg_fn {
         blocknum(b), this_msn(tmsn), verbose(v), keep_going_on_failure(k), messages_have_been_moved(m), last_msn(ZERO_MSN), msg_i(0) {
     }
 
-    int operator()(FT_MSG msg, bool is_fresh) {
-        enum ft_msg_type type = (enum ft_msg_type) msg->type;
-        MSN msn = msg->msn;
-        XIDS xid = msg->xids;
-        const void *key = ft_msg_get_key(msg);
-        const void *data = ft_msg_get_val(msg);
-        ITEMLEN keylen = ft_msg_get_keylen(msg);
-        ITEMLEN datalen = ft_msg_get_vallen(msg);
+    int operator()(const ft_msg &msg, bool is_fresh) {
+        enum ft_msg_type type = (enum ft_msg_type) msg.type();
+        MSN msn = msg.msn();
+        XIDS xid = msg.xids();
+        const void *key = msg.kdbt()->data;
+        const void *data = msg.vdbt()->data;
+        ITEMLEN keylen = msg.kdbt()->size;
+        ITEMLEN datalen = msg.vdbt()->size;
 
         int r = verify_msg_in_child_buffer(ft_handle, type, msn, key, keylen, data, datalen, xid,
                                            curr_less_pivot,

ft/ft_node-serialize.cc

@@ -859,7 +859,62 @@ toku_serialize_ftnode_to (int fd, BLOCKNUM blocknum, FTNODE node, FTNODE_DISK_DA
 }
 
 static void
-deserialize_child_buffer_v26(NONLEAF_CHILDINFO bnc, struct rbuf *rbuf, const toku::comparator &cmp) {
+sort_and_steal_offset_arrays(NONLEAF_CHILDINFO bnc,
+                             const toku::comparator &cmp,
+                             int32_t **fresh_offsets, int32_t nfresh,
+                             int32_t **stale_offsets, int32_t nstale,
+                             int32_t **broadcast_offsets, int32_t nbroadcast) {
+    // We always have fresh / broadcast offsets (even if they are empty)
+    // but we may not have stale offsets, in the case of v13 upgrade.
+    invariant(fresh_offsets != nullptr);
+    invariant(broadcast_offsets != nullptr);
+    invariant(cmp.valid());
+
+    typedef toku::sort<int32_t, const struct toku_msg_buffer_key_msn_cmp_extra, toku_msg_buffer_key_msn_cmp> msn_sort;
+
+    const int32_t n_in_this_buffer = nfresh + nstale + nbroadcast;
+    struct toku_msg_buffer_key_msn_cmp_extra extra(cmp, &bnc->msg_buffer);
+    msn_sort::mergesort_r(*fresh_offsets, nfresh, extra);
+    bnc->fresh_message_tree.destroy();
+    bnc->fresh_message_tree.create_steal_sorted_array(fresh_offsets, nfresh, n_in_this_buffer);
+    if (stale_offsets) {
+        msn_sort::mergesort_r(*stale_offsets, nstale, extra);
+        bnc->stale_message_tree.destroy();
+        bnc->stale_message_tree.create_steal_sorted_array(stale_offsets, nstale, n_in_this_buffer);
+    }
+    bnc->broadcast_list.destroy();
+    bnc->broadcast_list.create_steal_sorted_array(broadcast_offsets, nbroadcast, n_in_this_buffer);
+}
+
+static MSN
+deserialize_child_buffer_v13(FT ft, NONLEAF_CHILDINFO bnc, struct rbuf *rb) {
+    // We skip 'stale' offsets for upgraded nodes.
+    int32_t nfresh = 0, nbroadcast = 0;
+    int32_t *fresh_offsets = nullptr, *broadcast_offsets = nullptr;
+
+    // Only sort buffers if we have a valid comparison function. In certain scenarios,
+    // like deserialie_ft_versioned() or tokuftdump, we'll need to deserialize ftnodes
+    // for simple inspection and don't actually require that the message buffers are
+    // properly sorted. This is very ugly, but correct.
+    const bool sort = ft->cmp.valid();
+
+    MSN highest_msn_in_this_buffer =
+        bnc->msg_buffer.deserialize_from_rbuf_v13(rb, &ft->h->highest_unused_msn_for_upgrade,
+                                                  sort ? &fresh_offsets : nullptr, &nfresh,
+                                                  sort ? &broadcast_offsets : nullptr, &nbroadcast);
+
+    if (sort) {
+        sort_and_steal_offset_arrays(bnc, ft->cmp,
+                                     &fresh_offsets, nfresh,
+                                     nullptr, 0, // no stale offsets
+                                     &broadcast_offsets, nbroadcast);
+    }
+
+    return highest_msn_in_this_buffer;
+}
+
+static void
+deserialize_child_buffer_v26(NONLEAF_CHILDINFO bnc, struct rbuf *rb, const toku::comparator &cmp) {
     int32_t nfresh = 0, nstale = 0, nbroadcast = 0;
     int32_t *fresh_offsets, *stale_offsets, *broadcast_offsets;
 
@@ -870,50 +925,44 @@ deserialize_child_buffer_v26(NONLEAF_CHILDINFO bnc, struct rbuf *rbuf, const tok
     const bool sort = cmp.valid();
 
     // read in the message buffer
-    bnc->msg_buffer.deserialize_from_rbuf(rbuf,
+    bnc->msg_buffer.deserialize_from_rbuf(rb,
                                           sort ? &fresh_offsets : nullptr, &nfresh,
                                           sort ? &stale_offsets : nullptr, &nstale,
                                           sort ? &broadcast_offsets : nullptr, &nbroadcast);
 
     if (sort) {
-        int n_in_this_buffer = nfresh + nstale + nbroadcast;
-        struct toku_msg_buffer_key_msn_cmp_extra extra(cmp, &bnc->msg_buffer);
-        toku::sort<int32_t, const struct toku_msg_buffer_key_msn_cmp_extra, toku_msg_buffer_key_msn_cmp>::mergesort_r(fresh_offsets, nfresh, extra);
-        bnc->fresh_message_tree.destroy();
-        bnc->fresh_message_tree.create_steal_sorted_array(&fresh_offsets, nfresh, n_in_this_buffer);
-        toku::sort<int32_t, const struct toku_msg_buffer_key_msn_cmp_extra, toku_msg_buffer_key_msn_cmp>::mergesort_r(stale_offsets, nstale, extra);
-        bnc->stale_message_tree.destroy();
-        bnc->stale_message_tree.create_steal_sorted_array(&stale_offsets, nstale, n_in_this_buffer);
-        bnc->broadcast_list.destroy();
-        bnc->broadcast_list.create_steal_sorted_array(&broadcast_offsets, nbroadcast, n_in_this_buffer);
+        sort_and_steal_offset_arrays(bnc, cmp,
+                                     &fresh_offsets, nfresh,
+                                     &stale_offsets, nstale,
+                                     &broadcast_offsets, nbroadcast);
     }
 }
 
 static void
-deserialize_child_buffer(NONLEAF_CHILDINFO bnc, struct rbuf *rbuf) {
+deserialize_child_buffer(NONLEAF_CHILDINFO bnc, struct rbuf *rb) {
     // read in the message buffer
-    bnc->msg_buffer.deserialize_from_rbuf(rbuf,
+    bnc->msg_buffer.deserialize_from_rbuf(rb,
                                           nullptr, nullptr,  // fresh_offsets, nfresh,
                                           nullptr, nullptr,  // stale_offsets, nstale,
                                           nullptr, nullptr); // broadcast_offsets, nbroadcast
 
     // read in each message tree (fresh, stale, broadcast)
-    int32_t nfresh = rbuf_int(rbuf);
+    int32_t nfresh = rbuf_int(rb);
     int32_t *XMALLOC_N(nfresh, fresh_offsets);
     for (int i = 0; i < nfresh; i++) {
-        fresh_offsets[i] = rbuf_int(rbuf);
+        fresh_offsets[i] = rbuf_int(rb);
     }
 
-    int32_t nstale = rbuf_int(rbuf);
+    int32_t nstale = rbuf_int(rb);
     int32_t *XMALLOC_N(nstale, stale_offsets);
     for (int i = 0; i < nstale; i++) {
-        stale_offsets[i] = rbuf_int(rbuf);
+        stale_offsets[i] = rbuf_int(rb);
     }
 
-    int32_t nbroadcast = rbuf_int(rbuf);
+    int32_t nbroadcast = rbuf_int(rb);
     int32_t *XMALLOC_N(nbroadcast, broadcast_offsets);
     for (int i = 0; i < nbroadcast; i++) {
-        broadcast_offsets[i] = rbuf_int(rbuf);
+        broadcast_offsets[i] = rbuf_int(rb);
     }
 
     // build OMTs out of each offset array
@@ -1681,83 +1730,12 @@ deserialize_and_upgrade_internal_node(FTNODE node,
     MSN highest_msn;
     highest_msn.msn = 0;
 
-    // Only sort buffers if we have a valid comparison function. In certain scenarios,
-    // like deserialie_ft_versioned() or tokuftdump, we'll need to deserialize ftnodes
-    // for simple inspection and don't actually require that the message buffers are
-    // properly sorted. This is very ugly, but correct.
-    const bool sort_buffers = bfe->ft->cmp.valid();
-
     // Deserialize de-compressed buffers.
     for (int i = 0; i < node->n_children; ++i) {
         NONLEAF_CHILDINFO bnc = BNC(node, i);
-        int n_in_this_buffer = rbuf_int(rb);          // 22. node count
-
-        int32_t *fresh_offsets = nullptr;
-        int32_t *broadcast_offsets = nullptr;
-        int nfresh = 0;
-        int nbroadcast = 0;
-
-        // We skip 'stale' offsets for upgraded nodes.
-        if (sort_buffers) {
-            XMALLOC_N(n_in_this_buffer, fresh_offsets);
-            XMALLOC_N(n_in_this_buffer, broadcast_offsets);
-        }
-
-        // Atomically decrement the header's MSN count by the number
-        // of messages in the buffer.
-        MSN lowest;
-        uint64_t amount = n_in_this_buffer;
-        lowest.msn = toku_sync_sub_and_fetch(&bfe->ft->h->highest_unused_msn_for_upgrade.msn, amount);
+        MSN highest_msn_in_this_buffer = deserialize_child_buffer_v13(bfe->ft, bnc, rb);
         if (highest_msn.msn == 0) {
-            highest_msn.msn = lowest.msn + n_in_this_buffer;
-        }
-
-        // Create the message buffers from the deserialized buffer.
-        for (int j = 0; j < n_in_this_buffer; ++j) {
-            bytevec key; ITEMLEN keylen;
-            bytevec val; ITEMLEN vallen;
-            unsigned char ctype = rbuf_char(rb);       // 23. message type
-            enum ft_msg_type type = (enum ft_msg_type) ctype;
-            XIDS xids;
-            xids_create_from_buffer(rb, &xids);        // 24. XID
-            rbuf_bytes(rb, &key, &keylen);             // 25. key
-            rbuf_bytes(rb, &val, &vallen);             // 26. value
-
-            // <CER> can we factor this out?
-            int32_t *dest = nullptr;
-            if (sort_buffers) {
-                if (ft_msg_type_applies_once(type)) {
-                    dest = &fresh_offsets[nfresh];
-                    nfresh++;
-                } else if (ft_msg_type_applies_all(type) || ft_msg_type_does_nothing(type)) {
-                    dest = &broadcast_offsets[nbroadcast];
-                    nbroadcast++;
-                } else {
-                    abort();
-                }
-            }
-
-            // Increment our MSN, the last message should have the
-            // newest/highest MSN.  See above for a full explanation.
-            lowest.msn++;
-            // TODO: Function to parse stuff out of an rbuf into an FT_MSG
-            DBT k, v;
-            FT_MSG_S msg = {
-                type, lowest, xids,
-                .u = { .id = { toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, val, vallen) } }
-            };
-            bnc->msg_buffer.enqueue(&msg, true, dest);
-            xids_destroy(&xids);
-        }
-
-        if (sort_buffers) {
-            struct toku_msg_buffer_key_msn_cmp_extra extra(bfe->ft->cmp, &bnc->msg_buffer);
-            typedef toku::sort<int32_t, const struct toku_msg_buffer_key_msn_cmp_extra, toku_msg_buffer_key_msn_cmp> key_msn_sort;
-            key_msn_sort::mergesort_r(fresh_offsets, nfresh, extra);
-            bnc->fresh_message_tree.destroy();
-            bnc->fresh_message_tree.create_steal_sorted_array(&fresh_offsets, nfresh, n_in_this_buffer);
-            bnc->broadcast_list.destroy();
-            bnc->broadcast_list.create_steal_sorted_array(&broadcast_offsets, nbroadcast, n_in_this_buffer);
+            highest_msn.msn = highest_msn_in_this_buffer.msn;
         }
     }
 

ft/leafentry.h

@@ -99,7 +99,7 @@ PATENT RIGHTS GRANT:
 
 #include "ft/txn_manager.h"
 #include "ft/rbuf.h"
-#include "ft/ft_msg.h"
+#include "ft/msg.h"
 
 /*
     Memory format of packed leaf entry
@@ -248,7 +248,7 @@ toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, // NULL if there was no stored
 class bn_data;
 
 void
-toku_le_apply_msg(FT_MSG   msg,
+toku_le_apply_msg(const ft_msg &msg,
                   LEAFENTRY old_leafentry, // NULL if there was no stored data.
                   bn_data* data_buffer, // bn_data storing leafentry, if NULL, means there is no bn_data
                   uint32_t idx, // index in data_buffer where leafentry is stored (and should be replaced

ft/loader/loader.cc

@@ -2941,16 +2941,12 @@ static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int
     // #3588 TODO can do the rebalancing here and avoid a lot of work later
     FTNODE leafnode = lbuf->node;
     uint32_t idx = BLB_DATA(leafnode, 0)->num_klpairs();
-    DBT thekey = { .data = key, .size = (uint32_t) keylen }; 
-    DBT theval = { .data = val, .size = (uint32_t) vallen };
-    FT_MSG_S msg = { .type = FT_INSERT,
-                     .msn = ZERO_MSN,
-                     .xids = lbuf->xids,
-                     .u = { .id = { &thekey, &theval } } };
+    DBT kdbt, vdbt;
+    ft_msg msg(toku_fill_dbt(&kdbt, key, keylen), toku_fill_dbt(&vdbt, val, vallen), FT_INSERT, ZERO_MSN, lbuf->xids);
     uint64_t workdone = 0;
     // there's no mvcc garbage in a bulk-loaded FT, so there's no need to pass useful gc info
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true);
-    toku_ft_bn_apply_msg_once(BLB(leafnode,0), &msg, idx, keylen, NULL, &gc_info, &workdone, stats_to_update);
+    toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, &workdone, stats_to_update);
 }
 
 static int write_literal(struct dbout *out, void*data,  size_t len) {

ft/ft_msg.cc → ft/msg.cc

@@ -88,46 +88,84 @@ PATENT RIGHTS GRANT:
 
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 
+#include "portability/toku_portability.h"
 
-#include <toku_portability.h>
-#include "fttypes.h"
-#include "xids.h"
-#include "ft_msg.h"
+#include "ft/fttypes.h"
+#include "ft/msg.h"
+#include "ft/xids.h"
+#include "ft/ybt.h"
 
+ft_msg::ft_msg(const DBT *key, const DBT *val, enum ft_msg_type t, MSN m, XIDS x) :
+    _key(key ? *key : toku_empty_dbt()),
+    _val(val ? *val : toku_empty_dbt()),
+    _type(t), _msn(m), _xids(x) {
+}
+
+ft_msg ft_msg::deserialize_from_rbuf(struct rbuf *rb, XIDS *x, bool *is_fresh) {
+    bytevec keyp, valp;
+    ITEMLEN keylen, vallen;
+    enum ft_msg_type t = (enum ft_msg_type) rbuf_char(rb);
+    *is_fresh = rbuf_char(rb);
+    MSN m = rbuf_msn(rb);
+    xids_create_from_buffer(rb, x);
+    rbuf_bytes(rb, &keyp, &keylen);
+    rbuf_bytes(rb, &valp, &vallen);
+
+    DBT k, v;
+    return ft_msg(toku_fill_dbt(&k, keyp, keylen), toku_fill_dbt(&v, valp, vallen), t, m, *x);
+}
+
+ft_msg ft_msg::deserialize_from_rbuf_v13(struct rbuf *rb, MSN m, XIDS *x) {
+    bytevec keyp, valp;
+    ITEMLEN keylen, vallen;
+    enum ft_msg_type t = (enum ft_msg_type) rbuf_char(rb);
+    xids_create_from_buffer(rb, x);
+    rbuf_bytes(rb, &keyp, &keylen);
+    rbuf_bytes(rb, &valp, &vallen);
+
+    DBT k, v;
+    return ft_msg(toku_fill_dbt(&k, keyp, keylen), toku_fill_dbt(&v, valp, vallen), t, m, *x);
+}
 
-uint32_t 
-ft_msg_get_keylen(FT_MSG ft_msg) {
-    uint32_t rval = ft_msg->u.id.key->size;
-    return rval;
+const DBT *ft_msg::kdbt() const {
+    return &_key;
 }
 
-uint32_t 
-ft_msg_get_vallen(FT_MSG ft_msg) {
-    uint32_t rval = ft_msg->u.id.val->size;
-    return rval;
+const DBT *ft_msg::vdbt() const {
+    return &_val;
 }
 
-XIDS
-ft_msg_get_xids(FT_MSG ft_msg) {
-    XIDS rval = ft_msg->xids;
-    return rval;
+enum ft_msg_type ft_msg::type() const {
+    return _type;
 }
 
-void *
-ft_msg_get_key(FT_MSG ft_msg) {
-    void * rval = ft_msg->u.id.key->data;
-    return rval;
+MSN ft_msg::msn() const {
+    return _msn;
 }
 
-void *
-ft_msg_get_val(FT_MSG ft_msg) {
-    void * rval = ft_msg->u.id.val->data;
-    return rval;
+XIDS ft_msg::xids() const {
+    return _xids;
 }
 
-enum ft_msg_type
-ft_msg_get_type(FT_MSG ft_msg) {
-    enum ft_msg_type rval = ft_msg->type;
-    return rval;
+size_t ft_msg::total_size() const {
+    // Must store two 4-byte lengths
+    static const size_t key_val_overhead = 8;
+
+    // 1 byte type, 1 byte freshness, then 8 byte MSN
+    static const size_t msg_overhead = 2 + sizeof(MSN);
+
+    static const size_t total_overhead = key_val_overhead + msg_overhead;
+
+    const size_t keyval_size = _key.size + _val.size;
+    const size_t xids_size = xids_get_serialize_size(xids());
+    return total_overhead + keyval_size + xids_size;
 }
 
+void ft_msg::serialize_to_wbuf(struct wbuf *wb, bool is_fresh) const {
+    wbuf_nocrc_char(wb, (unsigned char) _type);
+    wbuf_nocrc_char(wb, (unsigned char) is_fresh);
+    wbuf_MSN(wb, _msn);
+    wbuf_nocrc_xids(wb, _xids);
+    wbuf_nocrc_bytes(wb, _key.data, _key.size);
+    wbuf_nocrc_bytes(wb, _val.data, _val.size);
+}

ft/ft_msg.h → ft/msg.h

@@ -181,32 +181,36 @@ ft_msg_type_does_nothing(enum ft_msg_type type)
 
 typedef struct xids_t *XIDS;
 
-/* tree commands */
-struct ft_msg {
-    enum ft_msg_type type;
-    MSN          msn;          // message sequence number
-    XIDS         xids;
-    union {
-        /* insert or delete */
-        struct ft_msg_insert_delete {
-            const DBT *key;   // for insert, delete, upsertdel
-            const DBT *val;   // for insert, delete, (and it is the "extra" for upsertdel, upsertdel_broadcast_all)
-        } id;
-    } u;
-};
+class ft_msg {
+public:
+    ft_msg(const DBT *key, const DBT *val, enum ft_msg_type t, MSN m, XIDS x);
+
+    enum ft_msg_type type() const;
+
+    MSN msn() const;
 
-// Message sent into the ft to implement insert, delete, update, etc
-typedef struct ft_msg FT_MSG_S;
-typedef struct ft_msg *FT_MSG;
+    XIDS xids() const;
 
-uint32_t ft_msg_get_keylen(FT_MSG ft_msg);
+    const DBT *kdbt() const;
 
-uint32_t ft_msg_get_vallen(FT_MSG ft_msg);
+    const DBT *vdbt() const;
 
-XIDS ft_msg_get_xids(FT_MSG ft_msg);
+    size_t total_size() const;
 
-void *ft_msg_get_key(FT_MSG ft_msg);
+    void serialize_to_wbuf(struct wbuf *wb, bool is_fresh) const;
 
-void *ft_msg_get_val(FT_MSG ft_msg);
+    // deserialization goes through a static factory function so the ft msg
+    // API stays completely const and there's no default constructor
+    static ft_msg deserialize_from_rbuf(struct rbuf *rb, XIDS *xids, bool *is_fresh);
 
-enum ft_msg_type ft_msg_get_type(FT_MSG ft_msg);
+    // Version 13/14 messages did not have an msn - so `m' is the MSN
+    // that will be assigned to the message that gets deserialized.
+    static ft_msg deserialize_from_rbuf_v13(struct rbuf *rb, MSN m, XIDS *xids);
+
+private:
+    const DBT _key;
+    const DBT _val;
+    enum ft_msg_type _type;
+    MSN _msn;
+    XIDS _xids;
+};

ft/msg_buffer.cc

@@ -128,42 +128,75 @@ void message_buffer::deserialize_from_rbuf(struct rbuf *rb,
 
     _resize(rb->size + 64); // rb->size is a good hint for how big the buffer will be
 
-    // read in each message individually
+    // deserialize each message individually, noting whether it was fresh
+    // and putting its buffer offset in the appropriate offsets array
     for (int i = 0; i < n_in_this_buffer; i++) {
-        bytevec key; ITEMLEN keylen;
-        bytevec val; ITEMLEN vallen;
-        // this is weird but it's necessary to pass icc and gcc together
-        unsigned char ctype = rbuf_char(rb);
-        enum ft_msg_type type = (enum ft_msg_type) ctype;
-        bool is_fresh = rbuf_char(rb);
-        MSN msn = rbuf_msn(rb);
         XIDS xids;
-        xids_create_from_buffer(rb, &xids);
-        rbuf_bytes(rb, &key, &keylen); /* Returns a pointer into the rbuf. */
-        rbuf_bytes(rb, &val, &vallen);
-        int32_t *dest = nullptr;
-        if (ft_msg_type_applies_once(type)) {
+        bool is_fresh;
+        const ft_msg msg = ft_msg::deserialize_from_rbuf(rb, &xids, &is_fresh);
+
+        int32_t *dest;
+        if (ft_msg_type_applies_once(msg.type())) {
             if (is_fresh) {
                 dest = fresh_offsets ? *fresh_offsets + (*nfresh)++ : nullptr;
             } else {
                 dest = stale_offsets ? *stale_offsets + (*nstale)++ : nullptr;
             }
         } else {
-            invariant(ft_msg_type_applies_all(type) || ft_msg_type_does_nothing(type));
+            invariant(ft_msg_type_applies_all(msg.type()) || ft_msg_type_does_nothing(msg.type()));
             dest = broadcast_offsets ? *broadcast_offsets + (*nbroadcast)++ : nullptr;
         }
 
-        // TODO: Function to parse stuff out of an rbuf into an FT_MSG
-        DBT k, v;
-        FT_MSG_S msg = {
-            type, msn, xids,
-            .u = { .id = { toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, val, vallen) } }
+        enqueue(msg, is_fresh, dest);
+        xids_destroy(&xids);
+    }
+
+    invariant(_num_entries == n_in_this_buffer);
+}
+
+MSN message_buffer::deserialize_from_rbuf_v13(struct rbuf *rb,
+                                              MSN *highest_unused_msn_for_upgrade,
+                                              int32_t **fresh_offsets, int32_t *nfresh,
+                                              int32_t **broadcast_offsets, int32_t *nbroadcast) {
+    // read the number of messages in this buffer
+    int n_in_this_buffer = rbuf_int(rb);
+    if (fresh_offsets != nullptr) {
+        XMALLOC_N(n_in_this_buffer, *fresh_offsets);
+    }
+    if (broadcast_offsets != nullptr) {
+        XMALLOC_N(n_in_this_buffer, *broadcast_offsets);
+    }
+
+    // Atomically decrement the header's MSN count by the number
+    // of messages in the buffer.
+    MSN highest_msn_in_this_buffer = {
+        .msn = toku_sync_sub_and_fetch(&highest_unused_msn_for_upgrade->msn, n_in_this_buffer)
     };
-        enqueue(&msg, is_fresh, dest);
+
+    // Create the message buffers from the deserialized buffer.
+    for (int i = 0; i < n_in_this_buffer; i++) {
+        XIDS xids;
+        // There were no stale messages at this version, so call it fresh.
+        const bool is_fresh = true;
+
+        // Increment our MSN, the last message should have the
+        // newest/highest MSN.  See above for a full explanation.
+        highest_msn_in_this_buffer.msn++;
+        const ft_msg msg = ft_msg::deserialize_from_rbuf_v13(rb, highest_msn_in_this_buffer, &xids);
+
+        int32_t *dest;
+        if (ft_msg_type_applies_once(msg.type())) {
+            dest = fresh_offsets ? *fresh_offsets + (*nfresh)++ : nullptr;
+        } else {
+            invariant(ft_msg_type_applies_all(msg.type()) || ft_msg_type_does_nothing(msg.type()));
+            dest = broadcast_offsets ? *broadcast_offsets + (*nbroadcast)++ : nullptr;
+        }
+
+        enqueue(msg, is_fresh, dest);
         xids_destroy(&xids);
     }
 
-    invariant(num_entries() == n_in_this_buffer);
+    return highest_msn_in_this_buffer;
 }
 
 void message_buffer::_resize(size_t new_size) {
@@ -184,7 +217,7 @@ struct message_buffer::buffer_entry *message_buffer::get_buffer_entry(int32_t of
     return (struct buffer_entry *) (_memory + offset);
 }
 
-void message_buffer::enqueue(FT_MSG msg, bool is_fresh, int32_t *offset) {
+void message_buffer::enqueue(const ft_msg &msg, bool is_fresh, int32_t *offset) {
     int need_space_here = msg_memsize_in_buffer(msg);
     int need_space_total = _memory_used + need_space_here;
     if (_memory == nullptr || need_space_total > _memory_size) {
@@ -192,18 +225,18 @@ void message_buffer::enqueue(FT_MSG msg, bool is_fresh, int32_t *offset) {
         int next_2 = next_power_of_two(need_space_total);
         _resize(next_2);
     }
-    ITEMLEN keylen = ft_msg_get_keylen(msg);
-    ITEMLEN datalen = ft_msg_get_vallen(msg);
+    ITEMLEN keylen = msg.kdbt()->size;
+    ITEMLEN datalen = msg.vdbt()->size;
     struct buffer_entry *entry = get_buffer_entry(_memory_used);
-    entry->type = (unsigned char) ft_msg_get_type(msg);
-    entry->msn = msg->msn;
-    xids_cpy(&entry->xids_s, ft_msg_get_xids(msg));
+    entry->type = (unsigned char) msg.type();
+    entry->msn = msg.msn();
+    xids_cpy(&entry->xids_s, msg.xids());
     entry->is_fresh = is_fresh;
     unsigned char *e_key = xids_get_end_of_array(&entry->xids_s);
     entry->keylen = keylen;
-    memcpy(e_key, ft_msg_get_key(msg), keylen);
+    memcpy(e_key, msg.kdbt()->data, keylen);
     entry->vallen = datalen;
-    memcpy(e_key + keylen, ft_msg_get_val(msg), datalen);
+    memcpy(e_key + keylen, msg.vdbt()->data, datalen);
     if (offset) {
         *offset = _memory_used;
     }
@@ -221,7 +254,7 @@ bool message_buffer::get_freshness(int32_t offset) const {
     return entry->is_fresh;
 }
 
-FT_MSG_S message_buffer::get_message(int32_t offset, DBT *keydbt, DBT *valdbt) const {
+ft_msg message_buffer::get_message(int32_t offset, DBT *keydbt, DBT *valdbt) const {
     struct buffer_entry *entry = get_buffer_entry(offset);
     ITEMLEN keylen = entry->keylen;
     ITEMLEN vallen = entry->vallen;
@@ -230,11 +263,7 @@ FT_MSG_S message_buffer::get_message(int32_t offset, DBT *keydbt, DBT *valdbt) c
     const XIDS xids = (XIDS) &entry->xids_s;
     bytevec key = xids_get_end_of_array(xids);
     bytevec val = (uint8_t *) key + entry->keylen;
-    FT_MSG_S msg = {
-        type, msn, xids,
-        .u = { .id = { toku_fill_dbt(keydbt, key, keylen), toku_fill_dbt(valdbt, val, vallen) } }
-    };
-    return msg;
+    return ft_msg(toku_fill_dbt(keydbt, key, keylen), toku_fill_dbt(valdbt, val, vallen), type, msn, xids);
 }
 
 void message_buffer::get_message_key_msn(int32_t offset, DBT *key, MSN *msn) const {
@@ -269,28 +298,21 @@ bool message_buffer::equals(message_buffer *other) const {
 }
 
 void message_buffer::serialize_to_wbuf(struct wbuf *wb) const {
-    wbuf_nocrc_int(wb, num_entries());
+    wbuf_nocrc_int(wb, _num_entries);
     struct msg_serialize_fn {
         struct wbuf *wb;
         msg_serialize_fn(struct wbuf *w) : wb(w) { }
-        int operator()(FT_MSG msg, bool is_fresh) {
-            enum ft_msg_type type = (enum ft_msg_type) msg->type;
-            paranoid_invariant((int) type >= 0 && (int) type < 256);
-            wbuf_nocrc_char(wb, (unsigned char) type);
-            wbuf_nocrc_char(wb, (unsigned char) is_fresh);
-            wbuf_MSN(wb, msg->msn);
-            wbuf_nocrc_xids(wb, ft_msg_get_xids(msg));
-            wbuf_nocrc_bytes(wb, ft_msg_get_key(msg), ft_msg_get_keylen(msg));
-            wbuf_nocrc_bytes(wb, ft_msg_get_val(msg), ft_msg_get_vallen(msg));
+        int operator()(const ft_msg &msg, bool is_fresh) {
+            msg.serialize_to_wbuf(wb, is_fresh);
             return 0;
         }
     } serialize_fn(wb);
     iterate(serialize_fn);
 }
 
-size_t message_buffer::msg_memsize_in_buffer(FT_MSG msg) {
-    const uint32_t keylen = ft_msg_get_keylen(msg);
-    const uint32_t datalen = ft_msg_get_vallen(msg);
-    const size_t xidslen = xids_get_size(msg->xids);
+size_t message_buffer::msg_memsize_in_buffer(const ft_msg &msg) {
+    const uint32_t keylen = msg.kdbt()->size;
+    const uint32_t datalen = msg.vdbt()->size;
+    const size_t xidslen = xids_get_size(msg.xids());
     return sizeof(struct buffer_entry) + keylen + datalen + xidslen - sizeof(XIDS_S);
 }

ft/msg_buffer.h

@@ -91,7 +91,7 @@ PATENT RIGHTS GRANT:
 #include "ft/fttypes.h"
 #include "ft/xids-internal.h"
 #include "ft/xids.h"
-#include "ft/ft_msg.h"
+#include "ft/msg.h"
 #include "ft/ybt.h"
 
 class message_buffer {
@@ -111,13 +111,24 @@ public:
                                int32_t **stale_offsets, int32_t *nstale,
                                int32_t **broadcast_offsets, int32_t *nbroadcast);
 
-    void enqueue(FT_MSG msg, bool is_fresh, int32_t *offset);
+    // effect: deserializes a message buffer whose messages are at version 13/14
+    // returns: similar to deserialize_from_rbuf(), excpet there are no stale messages
+    //          and each message is assigned a sequential value from *highest_unused_msn_for_upgrade,
+    //          which is modified as needed using toku_sync_fech_and_sub()
+    // returns: the highest MSN assigned to any message in this buffer
+    // requires: similar to deserialize_from_rbuf(), and highest_unused_msn_for_upgrade != nullptr
+    MSN deserialize_from_rbuf_v13(struct rbuf *rb,
+                                  MSN *highest_unused_msn_for_upgrade,
+                                  int32_t **fresh_offsets, int32_t *nfresh,
+                                  int32_t **broadcast_offsets, int32_t *nbroadcast);
+
+    void enqueue(const ft_msg &msg, bool is_fresh, int32_t *offset);
 
     void set_freshness(int32_t offset, bool is_fresh);
 
     bool get_freshness(int32_t offset) const;
 
-    FT_MSG_S get_message(int32_t offset, DBT *keydbt, DBT *valdbt) const;
+    ft_msg get_message(int32_t offset, DBT *keydbt, DBT *valdbt) const;
 
     void get_message_key_msn(int32_t offset, DBT *key, MSN *msn) const;
 
@@ -133,13 +144,13 @@ public:
     int iterate(F &fn) const {
         for (int32_t offset = 0; offset < _memory_used; ) {
             DBT k, v;
-            FT_MSG_S msg = get_message(offset, &k, &v);
+            const ft_msg msg = get_message(offset, &k, &v);
             bool is_fresh = get_freshness(offset);
-            int r = fn(&msg, is_fresh);
+            int r = fn(msg, is_fresh);
             if (r != 0) {
                 return r;
             }
-            offset += msg_memsize_in_buffer(&msg);
+            offset += msg_memsize_in_buffer(msg);
         }
         return 0;
     }
@@ -148,7 +159,7 @@ public:
 
     void serialize_to_wbuf(struct wbuf *wb) const;
 
-    static size_t msg_memsize_in_buffer(FT_MSG msg);
+    static size_t msg_memsize_in_buffer(const ft_msg &msg);
 
 private:
     void _resize(size_t new_size);

ft/node.cc

@@ -266,18 +266,18 @@ static void
 do_bn_apply_msg(FT_HANDLE ft_handle, BASEMENTNODE bn, message_buffer *msg_buffer, int32_t offset,
                 txn_gc_info *gc_info, uint64_t *workdone, STAT64INFO stats_to_update) {
     DBT k, v;
-    FT_MSG_S msg = msg_buffer->get_message(offset, &k, &v);
+    ft_msg msg = msg_buffer->get_message(offset, &k, &v);
 
     // The messages are being iterated over in (key,msn) order or just in
     // msn order, so all the messages for one key, from one buffer, are in
     // ascending msn order.  So it's ok that we don't update the basement
     // node's msn until the end.
-    if (msg.msn.msn > bn->max_msn_applied.msn) {
+    if (msg.msn().msn > bn->max_msn_applied.msn) {
         toku_ft_bn_apply_msg(
             ft_handle->ft->cmp,
             ft_handle->ft->update_fun,
             bn,
-            &msg,
+            msg,
             gc_info,
             workdone,
             stats_to_update
@@ -1120,7 +1120,7 @@ void toku_ft_nonleaf_append_child(FTNODE node, FTNODE child, const DBT *pivotkey
 void
 toku_ft_bn_apply_msg_once (
     BASEMENTNODE bn,
-    const FT_MSG msg,
+    const ft_msg &msg,
     uint32_t idx,
     uint32_t le_keylen,
     LEAFENTRY le,
@@ -1137,7 +1137,7 @@ toku_ft_bn_apply_msg_once (
     LEAFENTRY new_le=0;
     int64_t numbytes_delta = 0;  // how many bytes of user data (not including overhead) were added or deleted from this row
     int64_t numrows_delta = 0;   // will be +1 or -1 or 0 (if row was added or deleted or not)
-    uint32_t key_storage_size = ft_msg_get_keylen(msg) + sizeof(uint32_t);
+    uint32_t key_storage_size = msg.kdbt()->size + sizeof(uint32_t);
     if (le) {
         oldsize = leafentry_memsize(le) + key_storage_size;
     }
@@ -1223,17 +1223,11 @@ static void setval_fun (const DBT *new_val, void *svextra_v) {
         // can't leave scope until toku_ft_bn_apply_msg_once if
         // this is a delete
         DBT val;
-        FT_MSG_S msg = { FT_NONE, svextra->msn, svextra->xids,
-                         .u = { .id = {svextra->key, NULL} } };
-        if (new_val) {
-            msg.type = FT_INSERT;
-            msg.u.id.val = new_val;
-        } else {
-            msg.type = FT_DELETE_ANY;
-            toku_init_dbt(&val);
-            msg.u.id.val = &val;
-        }
-        toku_ft_bn_apply_msg_once(svextra->bn, &msg,
+        ft_msg msg(svextra->key,
+                   new_val ? new_val : toku_init_dbt(&val),
+                   new_val ? FT_INSERT : FT_DELETE_ANY,
+                   svextra->msn, svextra->xids);
+        toku_ft_bn_apply_msg_once(svextra->bn, msg,
                                   svextra->idx, svextra->le_keylen, svextra->le,
                                   svextra->gc_info,
                                   svextra->workdone, svextra->stats_to_update);
@@ -1245,7 +1239,7 @@ static void setval_fun (const DBT *new_val, void *svextra_v) {
 // so capturing the msn in the setval_extra_s is not strictly required.         The alternative
 // would be to put a dummy msn in the messages created by setval_fun(), but preserving
 // the original msn seems cleaner and it preserves accountability at a lower layer.
-static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEMENTNODE bn, FT_MSG msg, uint32_t idx,
+static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEMENTNODE bn, const ft_msg &msg, uint32_t idx,
                      LEAFENTRY le,
                      void* keydata,
                      uint32_t keylen,
@@ -1261,24 +1255,24 @@ static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEME
 
     // the location of data depends whether this is a regular or
     // broadcast update
-    if (msg->type == FT_UPDATE) {
+    if (msg.type() == FT_UPDATE) {
         // key is passed in with command (should be same as from le)
         // update function extra is passed in with command
-        keyp = msg->u.id.key;
-        update_function_extra = msg->u.id.val;
+        keyp = msg.kdbt();
+        update_function_extra = msg.vdbt();
     } else {
-        invariant(msg->type == FT_UPDATE_BROADCAST_ALL);
+        invariant(msg.type() == FT_UPDATE_BROADCAST_ALL);
         // key is not passed in with broadcast, it comes from le
         // update function extra is passed in with command
         paranoid_invariant(le);  // for broadcast updates, we just hit all leafentries
                      // so this cannot be null
         paranoid_invariant(keydata);
         paranoid_invariant(keylen);
-        paranoid_invariant(msg->u.id.key->size == 0);
+        paranoid_invariant(msg.kdbt()->size == 0);
         keyp = toku_fill_dbt(&key, keydata, keylen);
-        update_function_extra = msg->u.id.val;
+        update_function_extra = msg.vdbt();
     }
-    toku_ft_status_note_update(msg->type == FT_UPDATE_BROADCAST_ALL);
+    toku_ft_status_note_update(msg.type() == FT_UPDATE_BROADCAST_ALL);
 
     if (le && !le_latest_is_del(le)) {
         // if the latest val exists, use it, and we'll use the leafentry later
@@ -1291,7 +1285,7 @@ static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEME
     }
     le_for_update = le;
 
-    struct setval_extra_s setval_extra = {setval_tag, false, 0, bn, msg->msn, msg->xids,
+    struct setval_extra_s setval_extra = {setval_tag, false, 0, bn, msg.msn(), msg.xids(),
                                           keyp, idx, keylen, le_for_update, gc_info,
                                           workdone, stats_to_update};
     // call handlerton's ft->update_fun(), which passes setval_extra to setval_fun()
@@ -1314,7 +1308,7 @@ toku_ft_bn_apply_msg (
     const toku::comparator &cmp,
     ft_update_func update_fun,
     BASEMENTNODE bn,
-    FT_MSG msg,
+    const ft_msg &msg,
     txn_gc_info *gc_info, 
     uint64_t *workdone,
     STAT64INFO stats_to_update
@@ -1330,12 +1324,12 @@ toku_ft_bn_apply_msg (
 
     uint32_t num_klpairs;
     int r;
-    struct toku_msg_leafval_heaviside_extra be(cmp, msg->u.id.key);
+    struct toku_msg_leafval_heaviside_extra be(cmp, msg.kdbt());
 
     unsigned int doing_seqinsert = bn->seqinsert;
     bn->seqinsert = 0;
 
-    switch (msg->type) {
+    switch (msg.type()) {
     case FT_INSERT_NO_OVERWRITE:
     case FT_INSERT: {
         uint32_t idx;
@@ -1415,10 +1409,10 @@ toku_ft_bn_apply_msg (
                 // work was done by this message. since this is a broadcast message,
                 // we have to create a new message whose key is the current le's key.
                 DBT curr_keydbt;
-                FT_MSG_S curr_msg = *msg;
-                curr_msg.u.id.key = toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen);
-                toku_ft_bn_apply_msg_once(bn, &curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update);
-                // at this point, we cannot trust msg->u.id.key to be valid.
+                ft_msg curr_msg(toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen),
+                                msg.vdbt(), msg.type(), msg.msn(), msg.xids());
+                toku_ft_bn_apply_msg_once(bn, curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update);
+                // at this point, we cannot trust msg.kdbt to be valid.
                 uint32_t new_dmt_size = bn->data_buffer.num_klpairs();
                 if (new_dmt_size != num_klpairs) {
                     paranoid_invariant(new_dmt_size + 1 == num_klpairs);
@@ -1444,14 +1438,14 @@ toku_ft_bn_apply_msg (
             r = bn->data_buffer.fetch_klpair(idx, &storeddata, &curr_keylen, &curr_keyp);
             assert_zero(r);
             int deleted = 0;
-            if (le_has_xids(storeddata, msg->xids)) {
+            if (le_has_xids(storeddata, msg.xids())) {
                 // message application code needs a key in order to determine how much
                 // work was done by this message. since this is a broadcast message,
                 // we have to create a new message whose key is the current le's key.
                 DBT curr_keydbt;
-                FT_MSG_S curr_msg = *msg;
-                curr_msg.u.id.key = toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen);
-                toku_ft_bn_apply_msg_once(bn, &curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update);
+                ft_msg curr_msg(toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen),
+                                msg.vdbt(), msg.type(), msg.msn(), msg.xids());
+                toku_ft_bn_apply_msg_once(bn, curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update);
                 uint32_t new_dmt_size = bn->data_buffer.num_klpairs();
                 if (new_dmt_size != num_klpairs) {
                     paranoid_invariant(new_dmt_size + 1 == num_klpairs);
@@ -1480,8 +1474,8 @@ toku_ft_bn_apply_msg (
             {
                 //Point to msg's copy of the key so we don't worry about le being freed
                 //TODO: 46 MAYBE Get rid of this when le_apply message memory is better handled
-                key = msg->u.id.key->data;
-                keylen = msg->u.id.key->size;
+                key = msg.kdbt()->data;
+                keylen = msg.kdbt()->size;
             }
             r = do_update(update_fun, cmp.get_descriptor(), bn, msg, idx, NULL, NULL, 0, gc_info, workdone, stats_to_update);
         } else if (r==0) {
@@ -1557,15 +1551,15 @@ int toku_msg_buffer_key_msn_cmp(const struct toku_msg_buffer_key_msn_cmp_extra &
 // Effect: Enqueue the message represented by the parameters into the
 //   bnc's buffer, and put it in either the fresh or stale message tree,
 //   or the broadcast list.
-static void bnc_insert_msg(NONLEAF_CHILDINFO bnc, FT_MSG msg, bool is_fresh, const toku::comparator &cmp) {
+static void bnc_insert_msg(NONLEAF_CHILDINFO bnc, const ft_msg &msg, bool is_fresh, const toku::comparator &cmp) {
     int r = 0;
     int32_t offset;
     bnc->msg_buffer.enqueue(msg, is_fresh, &offset);
-    enum ft_msg_type type = ft_msg_get_type(msg);
+    enum ft_msg_type type = msg.type();
     if (ft_msg_type_applies_once(type)) {
         DBT key;
-        toku_fill_dbt(&key, ft_msg_get_key(msg), ft_msg_get_keylen(msg));
-        struct toku_msg_buffer_key_msn_heaviside_extra extra(cmp, &bnc->msg_buffer, &key, msg->msn);
+        toku_fill_dbt(&key, msg.kdbt()->data, msg.kdbt()->size);
+        struct toku_msg_buffer_key_msn_heaviside_extra extra(cmp, &bnc->msg_buffer, &key, msg.msn());
         if (is_fresh) {
             r = bnc->fresh_message_tree.insert<struct toku_msg_buffer_key_msn_heaviside_extra, toku_msg_buffer_key_msn_heaviside>(offset, extra, nullptr);
             assert_zero(r);
@@ -1585,15 +1579,13 @@ static void bnc_insert_msg(NONLEAF_CHILDINFO bnc, FT_MSG msg, bool is_fresh, con
 void toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const toku::comparator &cmp)
 {
     DBT k, v;
-    FT_MSG_S msg = {
-        type, msn, xids, .u = { .id = { toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, data, datalen) } }
-    };
-    bnc_insert_msg(bnc, &msg, is_fresh, cmp);
+    ft_msg msg(toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, data, datalen), type, msn, xids);
+    bnc_insert_msg(bnc, msg, is_fresh, cmp);
 }
 
 // append a msg to a nonleaf node's child buffer
 static void ft_append_msg_to_child_buffer(const toku::comparator &cmp, FTNODE node,
-                                          int childnum, FT_MSG msg, bool is_fresh) {
+                                          int childnum, const ft_msg &msg, bool is_fresh) {
     paranoid_invariant(BP_STATE(node,childnum) == PT_AVAIL);
     bnc_insert_msg(BNC(node, childnum), msg, is_fresh, cmp);
     node->dirty = 1;
@@ -1601,19 +1593,17 @@ static void ft_append_msg_to_child_buffer(const toku::comparator &cmp, FTNODE no
 
 // This is only exported for tests.
 void toku_ft_append_to_child_buffer(const toku::comparator &cmp, FTNODE node, int childnum, enum ft_msg_type type, MSN msn, XIDS xids, bool is_fresh, const DBT *key, const DBT *val) {
-    FT_MSG_S msg = {
-        type, msn, xids, .u = { .id = { key, val } }
-    };
-    ft_append_msg_to_child_buffer(cmp, node, childnum, &msg, is_fresh);
+    ft_msg msg(key, val, type, msn, xids);
+    ft_append_msg_to_child_buffer(cmp, node, childnum, msg, is_fresh);
 }
 
-static void ft_nonleaf_msg_once_to_child(const toku::comparator &cmp, FTNODE node, int target_childnum, FT_MSG msg, bool is_fresh, size_t flow_deltas[])
+static void ft_nonleaf_msg_once_to_child(const toku::comparator &cmp, FTNODE node, int target_childnum, const ft_msg &msg, bool is_fresh, size_t flow_deltas[])
 // Previously we had passive aggressive promotion, but that causes a lot of I/O a the checkpoint.  So now we are just putting it in the buffer here.
 // Also we don't worry about the node getting overfull here.  It's the caller's problem.
 {
     unsigned int childnum = (target_childnum >= 0
                              ? target_childnum
-                             : toku_ftnode_which_child(node, msg->u.id.key, cmp));
+                             : toku_ftnode_which_child(node, msg.kdbt(), cmp));
     ft_append_msg_to_child_buffer(cmp, node, childnum, msg, is_fresh);
     NONLEAF_CHILDINFO bnc = BNC(node, childnum);
     bnc->flow[0] += flow_deltas[0];
@@ -1684,7 +1674,7 @@ int toku_ftnode_hot_next_child(FTNODE node, const DBT *k, const toku::comparator
 }
 
 static void
-ft_nonleaf_msg_all(const toku::comparator &cmp, FTNODE node, FT_MSG msg, bool is_fresh, size_t flow_deltas[])
+ft_nonleaf_msg_all(const toku::comparator &cmp, FTNODE node, const ft_msg &msg, bool is_fresh, size_t flow_deltas[])
 // Effect: Put the message into a nonleaf node.  We put it into all children, possibly causing the children to become reactive.
 //  We don't do the splitting and merging.  That's up to the caller after doing all the puts it wants to do.
 //  The re_array[i] gets set to the reactivity of any modified child i.         (And there may be several such children.)
@@ -1695,7 +1685,7 @@ ft_nonleaf_msg_all(const toku::comparator &cmp, FTNODE node, FT_MSG msg, bool is
 }
 
 static void
-ft_nonleaf_put_msg(const toku::comparator &cmp, FTNODE node, int target_childnum, FT_MSG msg, bool is_fresh, size_t flow_deltas[])
+ft_nonleaf_put_msg(const toku::comparator &cmp, FTNODE node, int target_childnum, const ft_msg &msg, bool is_fresh, size_t flow_deltas[])
 // Effect: Put the message into a nonleaf node.  We may put it into a child, possibly causing the child to become reactive.
 //  We don't do the splitting and merging.  That's up to the caller after doing all the puts it wants to do.
 //  The re_array[i] gets set to the reactivity of any modified child i.         (And there may be several such children.)
@@ -1708,16 +1698,16 @@ ft_nonleaf_put_msg(const toku::comparator &cmp, FTNODE node, int target_childnum
     // node->max_msn_applied_to_node_on_disk here,
     // and don't do it in toku_ftnode_put_msg
     //
-    MSN msg_msn = msg->msn;
+    MSN msg_msn = msg.msn();
     invariant(msg_msn.msn > node->max_msn_applied_to_node_on_disk.msn);
     node->max_msn_applied_to_node_on_disk = msg_msn;
 
-    if (ft_msg_type_applies_once(msg->type)) {
+    if (ft_msg_type_applies_once(msg.type())) {
         ft_nonleaf_msg_once_to_child(cmp, node, target_childnum, msg, is_fresh, flow_deltas);
-    } else if (ft_msg_type_applies_all(msg->type)) {
+    } else if (ft_msg_type_applies_all(msg.type())) {
         ft_nonleaf_msg_all(cmp, node, msg, is_fresh, flow_deltas);
     } else {
-        paranoid_invariant(ft_msg_type_does_nothing(msg->type));
+        paranoid_invariant(ft_msg_type_does_nothing(msg.type()));
     }
 }
 
@@ -1874,7 +1864,7 @@ toku_ftnode_put_msg (
     ft_update_func update_fun,
     FTNODE node,
     int target_childnum,
-    FT_MSG msg,
+    const ft_msg &msg,
     bool is_fresh,
     txn_gc_info *gc_info,
     size_t flow_deltas[],
@@ -1908,7 +1898,7 @@ void toku_ft_leaf_apply_msg(
     ft_update_func update_fun,
     FTNODE node,
     int target_childnum,  // which child to inject to, or -1 if unknown
-    FT_MSG msg,
+    const ft_msg &msg,
     txn_gc_info *gc_info,
     uint64_t *workdone,
     STAT64INFO stats_to_update
@@ -1937,18 +1927,18 @@ void toku_ft_leaf_apply_msg(
     // This is why we handle node->max_msn_applied_to_node_on_disk both here
     // and in ft_nonleaf_put_msg, as opposed to in one location, toku_ftnode_put_msg.
     //
-    MSN msg_msn = msg->msn;
+    MSN msg_msn = msg.msn();
     if (msg_msn.msn > node->max_msn_applied_to_node_on_disk.msn) {
         node->max_msn_applied_to_node_on_disk = msg_msn;
     }
 
-    if (ft_msg_type_applies_once(msg->type)) {
+    if (ft_msg_type_applies_once(msg.type())) {
         unsigned int childnum = (target_childnum >= 0
                                  ? target_childnum
-                                 : toku_ftnode_which_child(node, msg->u.id.key, cmp));
+                                 : toku_ftnode_which_child(node, msg.kdbt(), cmp));
         BASEMENTNODE bn = BLB(node, childnum);
-        if (msg->msn.msn > bn->max_msn_applied.msn) {
-            bn->max_msn_applied = msg->msn;
+        if (msg.msn().msn > bn->max_msn_applied.msn) {
+            bn->max_msn_applied = msg.msn();
             toku_ft_bn_apply_msg(cmp,
                                  update_fun,
                                  bn,
@@ -1960,10 +1950,10 @@ void toku_ft_leaf_apply_msg(
             toku_ft_status_note_msn_discard();
         }
     }
-    else if (ft_msg_type_applies_all(msg->type)) {
+    else if (ft_msg_type_applies_all(msg.type())) {
         for (int childnum=0; childnum<node->n_children; childnum++) {
-            if (msg->msn.msn > BLB(node, childnum)->max_msn_applied.msn) {
-                BLB(node, childnum)->max_msn_applied = msg->msn;
+            if (msg.msn().msn > BLB(node, childnum)->max_msn_applied.msn) {
+                BLB(node, childnum)->max_msn_applied = msg.msn();
                 toku_ft_bn_apply_msg(cmp,
                                      update_fun,
                                      BLB(node, childnum),
@@ -1976,8 +1966,8 @@ void toku_ft_leaf_apply_msg(
             }
         }
     }
-    else if (!ft_msg_type_does_nothing(msg->type)) {
-        invariant(ft_msg_type_does_nothing(msg->type));
+    else if (!ft_msg_type_does_nothing(msg.type())) {
+        invariant(ft_msg_type_does_nothing(msg.type()));
     }
     VERIFY_NODE(t, node);
 }

ft/node.h

@@ -495,20 +495,20 @@ int toku_ftnode_hot_next_child(FTNODE node, const DBT *k, const toku::comparator
 
 void toku_ftnode_put_msg(const toku::comparator &cmp, ft_update_func update_fun,
                          FTNODE node, int target_childnum,
-                         FT_MSG msg, bool is_fresh, txn_gc_info *gc_info,
+                         const ft_msg &msg, bool is_fresh, txn_gc_info *gc_info,
                          size_t flow_deltas[], STAT64INFO stats_to_update);
 
-void toku_ft_bn_apply_msg_once(BASEMENTNODE bn, const FT_MSG msg, uint32_t idx,
+void toku_ft_bn_apply_msg_once(BASEMENTNODE bn, const ft_msg &msg, uint32_t idx,
                                uint32_t le_keylen, LEAFENTRY le, txn_gc_info *gc_info,
                                uint64_t *workdonep, STAT64INFO stats_to_update);
 
 void toku_ft_bn_apply_msg(const toku::comparator &cmp, ft_update_func update_fun,
-                          BASEMENTNODE bn, FT_MSG msg, txn_gc_info *gc_info,
+                          BASEMENTNODE bn, const ft_msg &msg, txn_gc_info *gc_info,
                           uint64_t *workdone, STAT64INFO stats_to_update);
 
 void toku_ft_leaf_apply_msg(const toku::comparator &cmp, ft_update_func update_fun,
                             FTNODE node, int target_childnum,
-                            FT_MSG msg, txn_gc_info *gc_info,
+                            const ft_msg &msg, txn_gc_info *gc_info,
                             uint64_t *workdone, STAT64INFO stats_to_update);
 
 CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT ft);

ft/roll.cc

@@ -257,13 +257,11 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
     XIDS xids;
     xids = toku_txn_get_xids(txn);
     {
-        FT_MSG_S ftmsg = { type, ZERO_MSN, xids,
-                           .u = { .id = { (key.len > 0)
-                                          ? toku_fill_dbt(&key_dbt,  key.data,  key.len)
-                                          : toku_init_dbt(&key_dbt),
-                                          data
-                                          ? toku_fill_dbt(&data_dbt, data->data, data->len)
-                                          : toku_init_dbt(&data_dbt) } } };
+        const DBT *kdbt = key.len > 0 ? toku_fill_dbt(&key_dbt, key.data, key.len) :
+                                        toku_init_dbt(&key_dbt);
+        const DBT *vdbt = data ? toku_fill_dbt(&data_dbt, data->data, data->len) :
+                                 toku_init_dbt(&data_dbt);
+        ft_msg msg(kdbt, vdbt, type, ZERO_MSN, xids);
 
         TXN_MANAGER txn_manager = toku_logger_get_txn_manager(txn->logger);
         txn_manager_state txn_state_for_gc(txn_manager);
@@ -274,7 +272,7 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
                             // no messages above us, we can implicitly promote uxrs based on this xid
                             oldest_referenced_xid_estimate,
                             !txn->for_recovery);
-        toku_ft_root_put_msg(ft, &ftmsg, &gc_info);
+        toku_ft_root_put_msg(ft, msg, &gc_info);
         if (reset_root_xid_that_created) {
             TXNID new_root_xid_that_created = xids_get_outermost_xid(xids);
             toku_reset_root_xid_that_created(ft, new_root_xid_that_created);

ft/tests/fifo-test.cc

@@ -136,10 +136,8 @@ test_enqueue(int n) {
             startmsn = msn;
         enum ft_msg_type type = (enum ft_msg_type) i;
         DBT k, v;
-        FT_MSG_S msg = {
-            type, msn, xids, .u = { .id = { toku_fill_dbt(&k, thekey, thekeylen), toku_fill_dbt(&v, theval, thevallen) } }
-        };
-        msg_buffer.enqueue(&msg, true, nullptr);
+        ft_msg msg(toku_fill_dbt(&k, thekey, thekeylen), toku_fill_dbt(&v, theval, thevallen), type, msn, xids);
+        msg_buffer.enqueue(msg, true, nullptr);
         xids_destroy(&xids);
         toku_free(thekey);
         toku_free(theval);
@@ -152,20 +150,20 @@ test_enqueue(int n) {
         checkit_fn(MSN smsn, bool v)
             : startmsn(smsn), verbose(v), i(0) {
         }
-        int operator()(FT_MSG msg, bool UU(is_fresh)) {
+        int operator()(const ft_msg &msg, bool UU(is_fresh)) {
             int thekeylen = i + 1;
             int thevallen = i + 2;
             char *thekey = buildkey(thekeylen);
             char *theval = buildval(thevallen);
 
-            MSN msn = msg->msn;
-            enum ft_msg_type type = ft_msg_get_type(msg);
+            MSN msn = msg.msn();
+            enum ft_msg_type type = msg.type();
             if (verbose) printf("checkit %d %d %" PRIu64 "\n", i, type, msn.msn);
             assert(msn.msn == startmsn.msn + i);
-            assert((int) ft_msg_get_keylen(msg) == thekeylen); assert(memcmp(ft_msg_get_key(msg), thekey, ft_msg_get_keylen(msg)) == 0);
-            assert((int) ft_msg_get_vallen(msg) == thevallen); assert(memcmp(ft_msg_get_val(msg), theval, ft_msg_get_vallen(msg)) == 0);
+            assert((int) msg.kdbt()->size == thekeylen); assert(memcmp(msg.kdbt()->data, thekey, msg.kdbt()->size) == 0);
+            assert((int) msg.vdbt()->size == thevallen); assert(memcmp(msg.vdbt()->data, theval, msg.vdbt()->size) == 0);
             assert(i % 256 == (int)type);
-            assert((TXNID)i==xids_get_innermost_xid(ft_msg_get_xids(msg)));
+            assert((TXNID)i==xids_get_innermost_xid(msg.xids()));
             i += 1;
             toku_free(thekey);
             toku_free(theval);

ft/tests/make-tree.cc

@@ -125,8 +125,8 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u = {.id = { &thekey, &theval }} };
-    toku_ft_bn_apply_msg_once(BLB(leafnode,0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
+    toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     leafnode->max_msn_applied_to_node_on_disk = msn;
 

ft/tests/msnfilter.cc

@@ -131,18 +131,18 @@ append_leaf(FT_HANDLE ft, FTNODE leafnode, void *key, uint32_t keylen, void *val
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     ft->ft->h->max_msn_in_ft = msn;
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
 
-    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, &msg, &gc_info, nullptr, nullptr);
+    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg, &gc_info, nullptr, nullptr);
     {
 	int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair);
 	assert(r==0);
 	assert(pair.call_count==1);
     }
 
-    FT_MSG_S badmsg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &badval }} };
-    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, &badmsg, &gc_info, nullptr, nullptr);
+    ft_msg badmsg(&thekey, &badval, FT_INSERT, msn, xids_get_root_xids());
+    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, badmsg, &gc_info, nullptr, nullptr);
 
     // message should be rejected for duplicate msn, row should still have original val
     {
@@ -154,8 +154,8 @@ append_leaf(FT_HANDLE ft, FTNODE leafnode, void *key, uint32_t keylen, void *val
     // now verify that message with proper msn gets through
     msn = next_dummymsn();
     ft->ft->h->max_msn_in_ft = msn;
-    FT_MSG_S msg2 = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &val2 }} };
-    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, &msg2, &gc_info, nullptr, nullptr);
+    ft_msg msg2(&thekey, &val2,  FT_INSERT, msn, xids_get_root_xids());
+    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg2, &gc_info, nullptr, nullptr);
 
     // message should be accepted, val should have new value
     {
@@ -166,8 +166,8 @@ append_leaf(FT_HANDLE ft, FTNODE leafnode, void *key, uint32_t keylen, void *val
 
     // now verify that message with lesser (older) msn is rejected
     msn.msn = msn.msn - 10;
-    FT_MSG_S msg3 = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &badval } }};
-    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, &msg3, &gc_info, nullptr, nullptr);
+    ft_msg msg3(&thekey, &badval, FT_INSERT, msn, xids_get_root_xids());
+    toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg3, &gc_info, nullptr, nullptr);
 
     // message should be rejected, val should still have value in pair2
     {

ft/tests/orthopush-flush.cc

@@ -132,7 +132,7 @@ rand_bytes_limited(void *dest, int size)
 // generate a random message with xids and a key starting with pfx, insert
 // it in bnc, and save it in output params save and is_fresh_out
 static void
-insert_random_message(NONLEAF_CHILDINFO bnc, FT_MSG_S **save, bool *is_fresh_out, XIDS xids, int pfx)
+insert_random_message(NONLEAF_CHILDINFO bnc, ft_msg **save, bool *is_fresh_out, XIDS xids, int pfx)
 {
     int keylen = (random() % 128) + 16;
     int vallen = (random() % 128) + 16;
@@ -144,18 +144,10 @@ insert_random_message(NONLEAF_CHILDINFO bnc, FT_MSG_S **save, bool *is_fresh_out
     MSN msn = next_dummymsn();
     bool is_fresh = (random() & 0x100) == 0;
 
-    DBT *keydbt, *valdbt;
-    XMALLOC(keydbt);
-    XMALLOC(valdbt);
-    toku_fill_dbt(keydbt, key, keylen + (sizeof pfx));
-    toku_fill_dbt(valdbt, val, vallen);
-    FT_MSG_S *XMALLOC(result);
-    result->type = FT_INSERT;
-    result->msn = msn;
-    result->xids = xids;
-    result->u.id.key = keydbt;
-    result->u.id.val = valdbt;
-    *save = result;
+    DBT keydbt, valdbt;
+    toku_fill_dbt(&keydbt, key, keylen + (sizeof pfx));
+    toku_fill_dbt(&valdbt, val, vallen);
+    *save = new ft_msg(&keydbt, &valdbt, FT_INSERT, msn, xids);
     *is_fresh_out = is_fresh;
 
     toku_bnc_insert_msg(bnc, key, keylen + (sizeof pfx), val, vallen,
@@ -193,17 +185,12 @@ insert_random_message_to_bn(
     valdbt = &valdbt_s;
     toku_fill_dbt(keydbt, key, (sizeof *pfxp) + keylen);
     toku_fill_dbt(valdbt, val, vallen);
-    FT_MSG_S msg;
-    msg.type = FT_INSERT;
-    msg.msn = msn;
-    msg.xids = xids;
-    msg.u.id.key = keydbt;
-    msg.u.id.val = valdbt;
     *keylenp = keydbt->size;
     *keyp = toku_xmemdup(keydbt->data, keydbt->size);
+    ft_msg msg(keydbt, valdbt, FT_INSERT, msn, xids);
     int64_t numbytes;
-    toku_le_apply_msg(&msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes);
-    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb, &msg, &non_mvcc_gc_info, NULL, NULL);
+    toku_le_apply_msg(msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes);
+    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb, msg, &non_mvcc_gc_info, NULL, NULL);
     if (msn.msn > blb->max_msn_applied.msn) {
         blb->max_msn_applied = msn;
     }
@@ -243,21 +230,16 @@ insert_same_message_to_bns(
     valdbt = &valdbt_s;
     toku_fill_dbt(keydbt, key, (sizeof *pfxp) + keylen);
     toku_fill_dbt(valdbt, val, vallen);
-    FT_MSG_S msg;
-    msg.type = FT_INSERT;
-    msg.msn = msn;
-    msg.xids = xids;
-    msg.u.id.key = keydbt;
-    msg.u.id.val = valdbt;
     *keylenp = keydbt->size;
     *keyp = toku_xmemdup(keydbt->data, keydbt->size);
+    ft_msg msg(keydbt, valdbt, FT_INSERT, msn, xids);
     int64_t numbytes;
-    toku_le_apply_msg(&msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes);
-    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb1, &msg, &non_mvcc_gc_info, NULL, NULL);
+    toku_le_apply_msg(msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes);
+    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb1, msg, &non_mvcc_gc_info, NULL, NULL);
     if (msn.msn > blb1->max_msn_applied.msn) {
         blb1->max_msn_applied = msn;
     }
-    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb2, &msg, &non_mvcc_gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb2, msg, &non_mvcc_gc_info, NULL, NULL);
     if (msn.msn > blb2->max_msn_applied.msn) {
         blb2->max_msn_applied = msn;
     }
@@ -284,7 +266,7 @@ orthopush_flush_update_fun(DB * UU(db), const DBT *UU(key), const DBT *UU(old_va
 // the update message will overwrite the value with something generated
 // here, and add one to the int pointed to by applied
 static void
-insert_random_update_message(NONLEAF_CHILDINFO bnc, FT_MSG_S **save, bool is_fresh, XIDS xids, int pfx, int *applied, MSN *max_msn)
+insert_random_update_message(NONLEAF_CHILDINFO bnc, ft_msg **save, bool is_fresh, XIDS xids, int pfx, int *applied, MSN *max_msn)
 {
     int keylen = (random() % 16) + 16;
     int vallen = (random() % 16) + 16;
@@ -297,18 +279,10 @@ insert_random_update_message(NONLEAF_CHILDINFO bnc, FT_MSG_S **save, bool is_fre
     update_extra->num_applications = applied;
     MSN msn = next_dummymsn();
 
-    DBT *keydbt, *valdbt;
-    XMALLOC(keydbt);
-    XMALLOC(valdbt);
-    toku_fill_dbt(keydbt, key, keylen + (sizeof pfx));
-    toku_fill_dbt(valdbt, update_extra, sizeof *update_extra);
-    FT_MSG_S *XMALLOC(result);
-    result->type = FT_UPDATE;
-    result->msn = msn;
-    result->xids = xids;
-    result->u.id.key = keydbt;
-    result->u.id.val = valdbt;
-    *save = result;
+    DBT keydbt, valdbt;
+    toku_fill_dbt(&keydbt, key, keylen + (sizeof pfx));
+    toku_fill_dbt(&valdbt, update_extra, sizeof *update_extra);
+    *save = new ft_msg(&keydbt, &valdbt, FT_UPDATE, msn, xids);
 
     toku_bnc_insert_msg(bnc, key, keylen + (sizeof pfx),
                         update_extra, sizeof *update_extra,
@@ -325,8 +299,8 @@ static void
 flush_to_internal(FT_HANDLE t) {
     int r;
 
-    FT_MSG_S **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
-    FT_MSG_S **MALLOC_N(4096,child_messages);
+    ft_msg **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
+    ft_msg **MALLOC_N(4096,child_messages);
     bool *MALLOC_N(4096,parent_messages_is_fresh);
     bool *MALLOC_N(4096,child_messages_is_fresh);
     memset(parent_messages_is_fresh, 0, 4096*(sizeof parent_messages_is_fresh[0]));
@@ -368,47 +342,47 @@ flush_to_internal(FT_HANDLE t) {
 
     struct checkit_fn {
         int num_parent_messages;
-        FT_MSG *parent_messages;
+        ft_msg **parent_messages;
         int *parent_messages_present;
         bool *parent_messages_is_fresh;
         int num_child_messages;
-        FT_MSG *child_messages;
+        ft_msg **child_messages;
         int *child_messages_present;
         bool *child_messages_is_fresh;
-        checkit_fn(int np, FT_MSG *pm, int *npp, bool *pmf, int nc, FT_MSG *cm, int *ncp, bool *cmf) :
+        checkit_fn(int np, ft_msg **pm, int *npp, bool *pmf, int nc, ft_msg **cm, int *ncp, bool *cmf) :
             num_parent_messages(np), parent_messages(pm), parent_messages_present(npp), parent_messages_is_fresh(pmf),
             num_child_messages(nc), child_messages(cm), child_messages_present(ncp), child_messages_is_fresh(cmf) {
         }
-        int operator()(FT_MSG msg, bool is_fresh) {
+        int operator()(const ft_msg &msg, bool is_fresh) {
             DBT keydbt;
             DBT valdbt;
-            toku_fill_dbt(&keydbt, ft_msg_get_key(msg), ft_msg_get_keylen(msg));
-            toku_fill_dbt(&valdbt, ft_msg_get_val(msg), ft_msg_get_vallen(msg));
+            toku_fill_dbt(&keydbt, msg.kdbt()->data, msg.kdbt()->size);
+            toku_fill_dbt(&valdbt, msg.vdbt()->data, msg.vdbt()->size);
             int found = 0;
-            MSN msn = msg->msn;
-            enum ft_msg_type type = ft_msg_get_type(msg);
-            XIDS xids = ft_msg_get_xids(msg);
+            MSN msn = msg.msn();
+            enum ft_msg_type type = msg.type();
+            XIDS xids = msg.xids();
             for (int k = 0; k < num_parent_messages; ++k) {
-                if (dummy_cmp(&keydbt, parent_messages[k]->u.id.key) == 0 &&
-                        msn.msn == parent_messages[k]->msn.msn) {
+                if (dummy_cmp(&keydbt, parent_messages[k]->kdbt()) == 0 &&
+                        msn.msn == parent_messages[k]->msn().msn) {
                     assert(parent_messages_present[k] == 0);
                     assert(found == 0);
-                    assert(dummy_cmp(&valdbt, parent_messages[k]->u.id.val) == 0);
-                    assert(type == parent_messages[k]->type);
-                    assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(parent_messages[k]->xids));
+                    assert(dummy_cmp(&valdbt, parent_messages[k]->vdbt()) == 0);
+                    assert(type == parent_messages[k]->type());
+                    assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(parent_messages[k]->xids()));
                     assert(parent_messages_is_fresh[k] == is_fresh);
                     parent_messages_present[k]++;
                     found++;
                 }
             }
             for (int k = 0; k < num_child_messages; ++k) {
-                if (dummy_cmp(&keydbt, child_messages[k]->u.id.key) == 0 &&
-                        msn.msn == child_messages[k]->msn.msn) {
+                if (dummy_cmp(&keydbt, child_messages[k]->kdbt()) == 0 &&
+                        msn.msn == child_messages[k]->msn().msn) {
                     assert(child_messages_present[k] == 0);
                     assert(found == 0);
-                    assert(dummy_cmp(&valdbt, child_messages[k]->u.id.val) == 0);
-                    assert(type == child_messages[k]->type);
-                    assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(child_messages[k]->xids));
+                    assert(dummy_cmp(&valdbt, child_messages[k]->vdbt()) == 0);
+                    assert(type == child_messages[k]->type());
+                    assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(child_messages[k]->xids()));
                     assert(child_messages_is_fresh[k] == is_fresh);
                     child_messages_present[k]++;
                     found++;
@@ -433,18 +407,14 @@ flush_to_internal(FT_HANDLE t) {
     xids_destroy(&xids_234);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        toku_free(parent_messages[i]->u.id.key->data);
-        toku_free((DBT *) parent_messages[i]->u.id.key);
-        toku_free(parent_messages[i]->u.id.val->data);
-        toku_free((DBT *) parent_messages[i]->u.id.val);
-        toku_free(parent_messages[i]);
+        toku_free(parent_messages[i]->kdbt()->data);
+        toku_free(parent_messages[i]->vdbt()->data);
+        delete parent_messages[i];
     }
     for (i = 0; i < num_child_messages; ++i) {
-        toku_free(child_messages[i]->u.id.key->data);
-        toku_free((DBT *) child_messages[i]->u.id.key);
-        toku_free(child_messages[i]->u.id.val->data);
-        toku_free((DBT *) child_messages[i]->u.id.val);
-        toku_free(child_messages[i]);
+        toku_free(child_messages[i]->kdbt()->data);
+        toku_free(child_messages[i]->vdbt()->data);
+        delete child_messages[i];
     }
     destroy_nonleaf_childinfo(parent_bnc);
     toku_ftnode_free(&child);
@@ -459,8 +429,8 @@ static void
 flush_to_internal_multiple(FT_HANDLE t) {
     int r;
 
-    FT_MSG_S **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
-    FT_MSG_S **MALLOC_N(4096,child_messages);
+    ft_msg **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
+    ft_msg **MALLOC_N(4096,child_messages);
     bool *MALLOC_N(4096,parent_messages_is_fresh);
     bool *MALLOC_N(4096,child_messages_is_fresh);
     memset(parent_messages_is_fresh, 0, 4096*(sizeof parent_messages_is_fresh[0]));
@@ -474,7 +444,7 @@ flush_to_internal_multiple(FT_HANDLE t) {
     CKERR(r);
 
     NONLEAF_CHILDINFO child_bncs[8];
-    FT_MSG childkeys[7];
+    ft_msg *childkeys[7];
     int i;
     for (i = 0; i < 8; ++i) {
         child_bncs[i] = toku_create_empty_nl();
@@ -488,7 +458,7 @@ flush_to_internal_multiple(FT_HANDLE t) {
         insert_random_message(child_bncs[i%8], &child_messages[i], &child_messages_is_fresh[i], xids_123, i%8);
         total_size += toku_bnc_memory_used(child_bncs[i%8]);
         if (i % 8 < 7) {
-            if (childkeys[i%8] == NULL || dummy_cmp(child_messages[i]->u.id.key, childkeys[i%8]->u.id.key) > 0) {
+            if (childkeys[i%8] == NULL || dummy_cmp(child_messages[i]->kdbt(), childkeys[i%8]->kdbt()) > 0) {
                 childkeys[i%8] = child_messages[i];
             }
         }
@@ -509,7 +479,7 @@ flush_to_internal_multiple(FT_HANDLE t) {
         set_BNC(child, i, child_bncs[i]);
         BP_STATE(child, i) = PT_AVAIL;
         if (i < 7) {
-            child->pivotkeys.insert_at(childkeys[i]->u.id.key, i);
+            child->pivotkeys.insert_at(childkeys[i]->kdbt(), i);
         }
     }
 
@@ -528,47 +498,47 @@ flush_to_internal_multiple(FT_HANDLE t) {
     for (int j = 0; j < 8; ++j) {
         struct checkit_fn {
             int num_parent_messages;
-            FT_MSG *parent_messages;
+            ft_msg **parent_messages;
             int *parent_messages_present;
             bool *parent_messages_is_fresh;
             int num_child_messages;
-            FT_MSG *child_messages;
+            ft_msg **child_messages;
             int *child_messages_present;
             bool *child_messages_is_fresh;
-            checkit_fn(int np, FT_MSG *pm, int *npp, bool *pmf, int nc, FT_MSG *cm, int *ncp, bool *cmf) :
+            checkit_fn(int np, ft_msg **pm, int *npp, bool *pmf, int nc, ft_msg **cm, int *ncp, bool *cmf) :
                 num_parent_messages(np), parent_messages(pm), parent_messages_present(npp), parent_messages_is_fresh(pmf),
                 num_child_messages(nc), child_messages(cm), child_messages_present(ncp), child_messages_is_fresh(cmf) {
             }
-            int operator()(FT_MSG msg, bool is_fresh) {
+            int operator()(const ft_msg &msg, bool is_fresh) {
                 DBT keydbt;
                 DBT valdbt;
-                toku_fill_dbt(&keydbt, ft_msg_get_key(msg), ft_msg_get_keylen(msg));
-                toku_fill_dbt(&valdbt, ft_msg_get_val(msg), ft_msg_get_vallen(msg));
+                toku_fill_dbt(&keydbt, msg.kdbt()->data, msg.kdbt()->size);
+                toku_fill_dbt(&valdbt, msg.vdbt()->data, msg.vdbt()->size);
                 int found = 0;
-                MSN msn = msg->msn;
-                enum ft_msg_type type = ft_msg_get_type(msg);
-                XIDS xids = ft_msg_get_xids(msg);
+                MSN msn = msg.msn();
+                enum ft_msg_type type = msg.type();
+                XIDS xids = msg.xids();
                 for (int _i = 0; _i < num_parent_messages; ++_i) {
-                    if (dummy_cmp(&keydbt, parent_messages[_i]->u.id.key) == 0 &&
-                            msn.msn == parent_messages[_i]->msn.msn) {
+                    if (dummy_cmp(&keydbt, parent_messages[_i]->kdbt()) == 0 &&
+                            msn.msn == parent_messages[_i]->msn().msn) {
                         assert(parent_messages_present[_i] == 0);
                         assert(found == 0);
-                        assert(dummy_cmp(&valdbt, parent_messages[_i]->u.id.val) == 0);
-                        assert(type == parent_messages[_i]->type);
-                        assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(parent_messages[_i]->xids));
+                        assert(dummy_cmp(&valdbt, parent_messages[_i]->vdbt()) == 0);
+                        assert(type == parent_messages[_i]->type());
+                        assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(parent_messages[_i]->xids()));
                         assert(parent_messages_is_fresh[_i] == is_fresh);
                         parent_messages_present[_i]++;
                         found++;
                     }
                 }
                 for (int _i = 0; _i < num_child_messages; ++_i) {
-                    if (dummy_cmp(&keydbt, child_messages[_i]->u.id.key) == 0 &&
-                            msn.msn == child_messages[_i]->msn.msn) {
+                    if (dummy_cmp(&keydbt, child_messages[_i]->kdbt()) == 0 &&
+                            msn.msn == child_messages[_i]->msn().msn) {
                         assert(child_messages_present[_i] == 0);
                         assert(found == 0);
-                        assert(dummy_cmp(&valdbt, child_messages[_i]->u.id.val) == 0);
-                        assert(type == child_messages[_i]->type);
-                        assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(child_messages[_i]->xids));
+                        assert(dummy_cmp(&valdbt, child_messages[_i]->vdbt()) == 0);
+                        assert(type == child_messages[_i]->type());
+                        assert(xids_get_innermost_xid(xids) == xids_get_innermost_xid(child_messages[_i]->xids()));
                         assert(child_messages_is_fresh[_i] == is_fresh);
                         child_messages_present[_i]++;
                         found++;
@@ -594,18 +564,14 @@ flush_to_internal_multiple(FT_HANDLE t) {
     xids_destroy(&xids_234);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        toku_free(parent_messages[i]->u.id.key->data);
-        toku_free((DBT *) parent_messages[i]->u.id.key);
-        toku_free(parent_messages[i]->u.id.val->data);
-        toku_free((DBT *) parent_messages[i]->u.id.val);
-        toku_free(parent_messages[i]);
+        toku_free(parent_messages[i]->kdbt()->data);
+        toku_free(parent_messages[i]->vdbt()->data);
+        delete parent_messages[i];
     }
     for (i = 0; i < num_child_messages; ++i) {
-        toku_free(child_messages[i]->u.id.key->data);
-        toku_free((DBT *) child_messages[i]->u.id.key);
-        toku_free(child_messages[i]->u.id.val->data);
-        toku_free((DBT *) child_messages[i]->u.id.val);
-        toku_free(child_messages[i]);
+        toku_free(child_messages[i]->kdbt()->data);
+        toku_free(child_messages[i]->vdbt()->data);
+        delete child_messages[i];
     }
     destroy_nonleaf_childinfo(parent_bnc);
     toku_ftnode_free(&child);
@@ -627,7 +593,7 @@ static void
 flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) {
     int r;
 
-    FT_MSG_S **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
+    ft_msg **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4096
     LEAFENTRY* child_messages = NULL;
     XMALLOC_N(4096,child_messages);
     void** key_pointers = NULL;
@@ -705,7 +671,7 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) {
     if (make_leaf_up_to_date) {
         for (i = 0; i < num_parent_messages; ++i) {
             if (!parent_messages_is_fresh[i]) {
-                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
+                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {
@@ -741,7 +707,7 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) {
         toku_apply_ancestors_messages_to_node(t, child, &ancestors, pivot_bounds::infinite_bounds(), &msgs_applied, -1);
 
         struct checkit_fn {
-            int operator()(FT_MSG UU(msg), bool is_fresh) {
+            int operator()(const ft_msg &UU(msg), bool is_fresh) {
                  assert(!is_fresh);
                 return 0;
             }
@@ -784,9 +750,9 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) {
             }
             int found = 0;
             for (i = num_parent_messages - 1; i >= 0; --i) {
-                if (dummy_cmp(&keydbt, parent_messages[i]->u.id.key) == 0) {
+                if (dummy_cmp(&keydbt, parent_messages[i]->kdbt()) == 0) {
                     if (found == 0) {
-                        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(e, parent_messages[i]->u.id.val->data);
+                        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(e, parent_messages[i]->vdbt()->data);
                         assert(dummy_cmp(&valdbt, &e->new_val) == 0);
                         found++;
                     }
@@ -827,13 +793,11 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) {
     xids_destroy(&xids_234);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        toku_free(parent_messages[i]->u.id.key->data);
-        toku_free((DBT *) parent_messages[i]->u.id.key);
-        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->u.id.val->data);
+        toku_free(parent_messages[i]->kdbt()->data);
+        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->vdbt()->data);
         toku_free(extra->new_val.data);
-        toku_free(parent_messages[i]->u.id.val->data);
-        toku_free((DBT *) parent_messages[i]->u.id.val);
-        toku_free(parent_messages[i]);
+        toku_free(parent_messages[i]->vdbt()->data);
+        delete parent_messages[i];
     }
     for (i = 0; i < num_child_messages; ++i) {
         toku_free(child_messages[i]);
@@ -858,7 +822,7 @@ static void
 flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
     int r;
 
-    FT_MSG_S **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4k
+    ft_msg **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4k
     LEAFENTRY* child_messages = NULL;
     XMALLOC_N(4096,child_messages);
     void** key_pointers = NULL;
@@ -928,9 +892,9 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
 
     if (make_leaf_up_to_date) {
         for (i = 0; i < num_parent_messages; ++i) {
-            if (dummy_cmp(parent_messages[i]->u.id.key, &childkeys[7]) <= 0 &&
+            if (dummy_cmp(parent_messages[i]->kdbt(), &childkeys[7]) <= 0 &&
                 !parent_messages_is_fresh[i]) {
-                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
+                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {
@@ -944,7 +908,7 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
 
     for (i = 0; i < num_parent_messages; ++i) {
         if (make_leaf_up_to_date &&
-            dummy_cmp(parent_messages[i]->u.id.key, &childkeys[7]) <= 0 &&
+            dummy_cmp(parent_messages[i]->kdbt(), &childkeys[7]) <= 0 &&
             !parent_messages_is_fresh[i]) {
             assert(parent_messages_applied[i] == 1);
         } else {
@@ -970,19 +934,19 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
     struct checkit_fn {
         DBT *childkeys;
         int num_parent_messages;
-        FT_MSG *parent_messages;
+        ft_msg **parent_messages;
         bool *parent_messages_is_fresh;
-        checkit_fn(DBT *ck, int np, FT_MSG *pm, bool *pmf) :
+        checkit_fn(DBT *ck, int np, ft_msg **pm, bool *pmf) :
             childkeys(ck), num_parent_messages(np), parent_messages(pm), parent_messages_is_fresh(pmf) {
         }
-        int operator()(FT_MSG msg, bool is_fresh) {
+        int operator()(const ft_msg &msg, bool is_fresh) {
             DBT keydbt;
-            toku_fill_dbt(&keydbt, ft_msg_get_key(msg), ft_msg_get_keylen(msg));
-            MSN msn = msg->msn;
+            toku_fill_dbt(&keydbt, msg.kdbt()->data, msg.kdbt()->size);
+            MSN msn = msg.msn();
             if (dummy_cmp(&keydbt, &childkeys[7]) > 0) {
                 for (int _i = 0; _i < num_parent_messages; ++_i) {
-                    if (dummy_cmp(&keydbt, parent_messages[_i]->u.id.key) == 0 &&
-                            msn.msn == parent_messages[_i]->msn.msn) {
+                    if (dummy_cmp(&keydbt, parent_messages[_i]->kdbt()) == 0 &&
+                            msn.msn == parent_messages[_i]->msn().msn) {
                         assert(is_fresh == parent_messages_is_fresh[_i]);
                         break;
                     }
@@ -1004,7 +968,7 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
     assert(total_messages <= num_parent_messages + num_child_messages);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        if (dummy_cmp(parent_messages[i]->u.id.key, &childkeys[7]) <= 0) {
+        if (dummy_cmp(parent_messages[i]->kdbt(), &childkeys[7]) <= 0) {
             assert(parent_messages_applied[i] == 1);
         } else {
             assert(parent_messages_applied[i] == 0);
@@ -1016,13 +980,11 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) {
     xids_destroy(&xids_234);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        toku_free(parent_messages[i]->u.id.key->data);
-        toku_free((DBT *) parent_messages[i]->u.id.key);
-        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->u.id.val->data);
+        toku_free(parent_messages[i]->kdbt()->data);
+        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->vdbt()->data);
         toku_free(extra->new_val.data);
-        toku_free(parent_messages[i]->u.id.val->data);
-        toku_free((DBT *) parent_messages[i]->u.id.val);
-        toku_free(parent_messages[i]);
+        toku_free(parent_messages[i]->vdbt()->data);
+        delete parent_messages[i];
     }
     for (i = 0; i < num_child_messages; ++i) {
         toku_free(child_messages[i]);
@@ -1049,7 +1011,7 @@ static void
 compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) {
     int r;
 
-    FT_MSG_S **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4k
+    ft_msg **MALLOC_N(4096,parent_messages);  // 128k / 32 = 4k
     LEAFENTRY* child_messages = NULL;
     XMALLOC_N(4096,child_messages);
     void** key_pointers = NULL;
@@ -1135,8 +1097,8 @@ compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) {
     if (make_leaf_up_to_date) {
         for (i = 0; i < num_parent_messages; ++i) {
             if (!parent_messages_is_fresh[i]) {
-                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child1, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
-                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child2, -1, parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
+                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child1, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
+                toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child2, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {
@@ -1164,7 +1126,7 @@ compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) {
     toku_apply_ancestors_messages_to_node(t, child2, &ancestors, pivot_bounds::infinite_bounds(), &msgs_applied, -1);
 
     struct checkit_fn {
-        int operator()(FT_MSG UU(msg), bool is_fresh) {
+        int operator()(const ft_msg &UU(msg), bool is_fresh) {
             assert(!is_fresh);
             return 0;
         }
@@ -1211,13 +1173,11 @@ compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) {
     xids_destroy(&xids_234);
 
     for (i = 0; i < num_parent_messages; ++i) {
-        toku_free(parent_messages[i]->u.id.key->data);
-        toku_free((DBT *) parent_messages[i]->u.id.key);
-        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->u.id.val->data);
+        toku_free(parent_messages[i]->kdbt()->data);
+        struct orthopush_flush_update_fun_extra *CAST_FROM_VOIDP(extra, parent_messages[i]->vdbt()->data);
         toku_free(extra->new_val.data);
-        toku_free(parent_messages[i]->u.id.val->data);
-        toku_free((DBT *) parent_messages[i]->u.id.val);
-        toku_free(parent_messages[i]);
+        toku_free(parent_messages[i]->vdbt()->data);
+        delete parent_messages[i];
     }
     for (i = 0; i < num_child_messages; ++i) {
         toku_free(key_pointers[i]);

ft/tests/test-leafentry-child-txn.cc

@@ -111,17 +111,6 @@ static void add_committed_entry(ULE ule, DBT *val, TXNID xid) {
     ule->uxrs[index].xid    = xid;
 }
 
-static FT_MSG_S
-msg_init(enum ft_msg_type type, XIDS xids,
-         DBT *key, DBT *val) {
-    FT_MSG_S msg;
-    msg.type = type;
-    msg.xids = xids;
-    msg.u.id.key = key;
-    msg.u.id.val = val;
-    return msg;
-}
-
 //Test all the different things that can happen to a
 //committed leafentry (logical equivalent of a committed insert).
 static void
@@ -161,29 +150,33 @@ run_test(void) {
     add_committed_entry(&ule_initial, &val, 10);
 
     // now do the application of xids to the ule    
-    FT_MSG_S msg;
     // do a commit
-    msg = msg_init(FT_COMMIT_ANY, msg_xids_2, &key, &val);
-    test_msg_modify_ule(&ule_initial, &msg);
+    {
+        ft_msg msg(&key, &val, FT_COMMIT_ANY, ZERO_MSN, msg_xids_2);
+        test_msg_modify_ule(&ule_initial, msg);
         assert(ule->num_cuxrs == 2);
         assert(ule->uxrs[0].xid == TXNID_NONE);
         assert(ule->uxrs[1].xid == 10);
         assert(ule->uxrs[0].valp == &val_data_one);
         assert(ule->uxrs[1].valp == &val_data_two);
+    }
 
     // do an abort
-    msg = msg_init(FT_ABORT_ANY, msg_xids_2, &key, &val);
-    test_msg_modify_ule(&ule_initial, &msg);
+    {
+        ft_msg msg(&key, &val, FT_ABORT_ANY, ZERO_MSN, msg_xids_2);
+        test_msg_modify_ule(&ule_initial, msg);
         assert(ule->num_cuxrs == 2);
         assert(ule->uxrs[0].xid == TXNID_NONE);
         assert(ule->uxrs[1].xid == 10);
         assert(ule->uxrs[0].valp == &val_data_one);
         assert(ule->uxrs[1].valp == &val_data_two);
+    }
 
     // do an insert
     val.data = &val_data_three;
-    msg = msg_init(FT_INSERT, msg_xids_2, &key, &val);
-    test_msg_modify_ule(&ule_initial, &msg);
+    {
+        ft_msg msg(&key, &val, FT_INSERT, ZERO_MSN, msg_xids_2);
+        test_msg_modify_ule(&ule_initial, msg);
         // now that message applied, verify that things are good
         assert(ule->num_cuxrs == 2);
         assert(ule->num_puxrs == 2);
@@ -195,7 +188,7 @@ run_test(void) {
         assert(ule->uxrs[1].valp == &val_data_two);
         assert(ule->uxrs[2].type == XR_PLACEHOLDER);
         assert(ule->uxrs[3].valp == &val_data_three);
-    
+    } 
 
     xids_destroy(&msg_xids_2);
     xids_destroy(&msg_xids_1);

ft/tests/test-leafentry-nested.cc

@@ -442,7 +442,7 @@ test_le_pack (void) {
 }
 
 static void
-test_le_apply(ULE ule_initial, FT_MSG msg, ULE ule_expected) {
+test_le_apply(ULE ule_initial, const ft_msg &msg, ULE ule_expected) {
     int r;
     LEAFENTRY le_initial;
     LEAFENTRY le_expected;
@@ -496,17 +496,6 @@ static const ULE_S ule_committed_delete = {
     .uxrs = (UXR_S *)ule_committed_delete.uxrs_static
 };
 
-static FT_MSG_S
-msg_init(enum ft_msg_type type, XIDS xids,
-         DBT *key, DBT *val) {
-    FT_MSG_S msg;
-    msg.type = type;
-    msg.xids = xids;
-    msg.u.id.key = key;
-    msg.u.id.val = val;
-    return msg;
-}
-
 static uint32_t
 next_nesting_level(uint32_t current) {
     uint32_t rval = current + 1;
@@ -531,9 +520,9 @@ generate_committed_for(ULE ule, DBT *val) {
 }
 
 static void
-generate_provpair_for(ULE ule, FT_MSG msg) {
+generate_provpair_for(ULE ule, const ft_msg &msg) {
     uint32_t level;
-    XIDS xids = msg->xids;
+    XIDS xids = msg.xids();
     ule->uxrs = ule->uxrs_static;
 
     ule->num_cuxrs = 1;
@@ -550,8 +539,8 @@ generate_provpair_for(ULE ule, FT_MSG msg) {
         ule->uxrs[level].xid    = xids_get_xid(xids, level-1);
     }
     ule->uxrs[num_uxrs - 1].type   = XR_INSERT;
-    ule->uxrs[num_uxrs - 1].vallen = msg->u.id.val->size;
-    ule->uxrs[num_uxrs - 1].valp   = msg->u.id.val->data;
+    ule->uxrs[num_uxrs - 1].vallen = msg.vdbt()->size;
+    ule->uxrs[num_uxrs - 1].valp   = msg.vdbt()->data;
     ule->uxrs[num_uxrs - 1].xid    = xids_get_innermost_xid(xids);
 }
 
@@ -560,7 +549,6 @@ generate_provpair_for(ULE ule, FT_MSG msg) {
 static void
 test_le_empty_apply(void) {
     ULE_S ule_initial        = ule_committed_delete;
-    FT_MSG_S msg;
 
     DBT key;
     DBT val;
@@ -585,34 +573,41 @@ test_le_empty_apply(void) {
                     //Abort/commit of an empty le is an empty le
                     ULE_S ule_expected = ule_committed_delete;
 
-                    msg = msg_init(FT_COMMIT_ANY, msg_xids,  &key, &val);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
-                    msg = msg_init(FT_COMMIT_BROADCAST_TXN, msg_xids,  &key, &val);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
-
-                    msg = msg_init(FT_ABORT_ANY, msg_xids, &key, &val);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
-                    msg = msg_init(FT_ABORT_BROADCAST_TXN, msg_xids, &key, &val);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
+                    {
+                        ft_msg msg(&key, &val, FT_COMMIT_ANY, ZERO_MSN, msg_xids);
+                        test_le_apply(&ule_initial, msg, &ule_expected);
+                    }
+                    {
+                        ft_msg msg(&key, &val, FT_COMMIT_BROADCAST_TXN, ZERO_MSN, msg_xids);
+                        test_le_apply(&ule_initial, msg, &ule_expected);
+                    }
+                    {
+                        ft_msg msg(&key, &val, FT_ABORT_ANY, ZERO_MSN, msg_xids);
+                        test_le_apply(&ule_initial, msg, &ule_expected);
+                    }
+                    {
+                        ft_msg msg(&key, &val, FT_ABORT_BROADCAST_TXN, ZERO_MSN, msg_xids);
+                        test_le_apply(&ule_initial, msg, &ule_expected);
+                    }
                 }
                 {
                     //delete of an empty le is an empty le
                     ULE_S ule_expected = ule_committed_delete;
 
-                    msg = msg_init(FT_DELETE_ANY, msg_xids, &key, &val);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
+                    ft_msg msg(&key, &val, FT_DELETE_ANY, ZERO_MSN, msg_xids);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
                 }
                 {
-                    msg = msg_init(FT_INSERT, msg_xids, &key, &val);
+                    ft_msg msg(&key, &val, FT_INSERT, ZERO_MSN, msg_xids);
                     ULE_S ule_expected;
-                    generate_provpair_for(&ule_expected, &msg);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
+                    generate_provpair_for(&ule_expected, msg);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
                 }
                 {
-                    msg = msg_init(FT_INSERT_NO_OVERWRITE, msg_xids, &key, &val);
+                    ft_msg msg(&key, &val, FT_INSERT_NO_OVERWRITE, ZERO_MSN, msg_xids);
                     ULE_S ule_expected;
-                    generate_provpair_for(&ule_expected, &msg);
-                    test_le_apply(&ule_initial, &msg, &ule_expected);
+                    generate_provpair_for(&ule_expected, msg);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
                 }
             }
         }
@@ -620,16 +615,16 @@ test_le_empty_apply(void) {
 }
 
 static void
-generate_provdel_for(ULE ule, FT_MSG msg) {
+generate_provdel_for(ULE ule, const ft_msg &msg) {
     uint32_t level;
-    XIDS xids = msg->xids;
+    XIDS xids = msg.xids();
 
     ule->num_cuxrs = 1;
     ule->num_puxrs = xids_get_num_xids(xids);
     uint32_t num_uxrs = ule->num_cuxrs + ule->num_puxrs;
     ule->uxrs[0].type   = XR_INSERT;
-    ule->uxrs[0].vallen = msg->u.id.val->size;
-    ule->uxrs[0].valp   = msg->u.id.val->data;
+    ule->uxrs[0].vallen = msg.vdbt()->size;
+    ule->uxrs[0].valp   = msg.vdbt()->data;
     ule->uxrs[0].xid    = TXNID_NONE;
     for (level = ule->num_cuxrs; level < ule->num_cuxrs + ule->num_puxrs - 1; level++) {
         ule->uxrs[level].type   = XR_PLACEHOLDER;
@@ -644,9 +639,9 @@ generate_provdel_for(ULE ule, FT_MSG msg) {
 }
 
 static void
-generate_both_for(ULE ule, DBT *oldval, FT_MSG msg) {
+generate_both_for(ULE ule, DBT *oldval, const ft_msg &msg) {
     uint32_t level;
-    XIDS xids = msg->xids;
+    XIDS xids = msg.xids();
 
     ule->num_cuxrs = 1;
     ule->num_puxrs = xids_get_num_xids(xids);
@@ -662,8 +657,8 @@ generate_both_for(ULE ule, DBT *oldval, FT_MSG msg) {
         ule->uxrs[level].xid    = xids_get_xid(xids, level-1);
     }
     ule->uxrs[num_uxrs - 1].type   = XR_INSERT;
-    ule->uxrs[num_uxrs - 1].vallen = msg->u.id.val->size;
-    ule->uxrs[num_uxrs - 1].valp   = msg->u.id.val->data;
+    ule->uxrs[num_uxrs - 1].vallen = msg.vdbt()->size;
+    ule->uxrs[num_uxrs - 1].valp   = msg.vdbt()->data;
     ule->uxrs[num_uxrs - 1].xid    = xids_get_innermost_xid(xids);
 }
 
@@ -673,7 +668,6 @@ static void
 test_le_committed_apply(void) {
     ULE_S ule_initial;
     ule_initial.uxrs = ule_initial.uxrs_static;
-    FT_MSG_S msg;
 
     DBT key;
     DBT val;
@@ -696,23 +690,30 @@ test_le_committed_apply(void) {
             if (nesting_level > 0) {
                 //Commit/abort will not change a committed le
                 ULE_S ule_expected = ule_initial;
-                msg = msg_init(FT_COMMIT_ANY, msg_xids,  &key, &val);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
-                msg = msg_init(FT_COMMIT_BROADCAST_TXN, msg_xids,  &key, &val);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
-
-                msg = msg_init(FT_ABORT_ANY, msg_xids, &key, &val);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
-                msg = msg_init(FT_ABORT_BROADCAST_TXN, msg_xids, &key, &val);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
+                {
+                    ft_msg msg(&key, &val, FT_COMMIT_ANY, ZERO_MSN, msg_xids);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
+                }
+                {
+                    ft_msg msg(&key, &val, FT_COMMIT_BROADCAST_TXN, ZERO_MSN, msg_xids);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
+                }
+                {
+                    ft_msg msg(&key, &val, FT_ABORT_ANY, ZERO_MSN, msg_xids);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
+                }
+                {
+                    ft_msg msg(&key, &val, FT_ABORT_BROADCAST_TXN, ZERO_MSN, msg_xids);
+                    test_le_apply(&ule_initial, msg, &ule_expected);
+                }
             }
 
             {
-                msg = msg_init(FT_DELETE_ANY, msg_xids, &key, &val);
+                ft_msg msg(&key, &val, FT_DELETE_ANY, ZERO_MSN, msg_xids);
                 ULE_S ule_expected;
                 ule_expected.uxrs = ule_expected.uxrs_static;
-                generate_provdel_for(&ule_expected, &msg);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
+                generate_provdel_for(&ule_expected, msg);
+                test_le_apply(&ule_initial, msg, &ule_expected);
             }
 
             {
@@ -721,11 +722,11 @@ test_le_committed_apply(void) {
                 fillrandom(valbuf2, valsize2);
                 DBT val2;
                 toku_fill_dbt(&val2, valbuf2, valsize2);
-                msg = msg_init(FT_INSERT, msg_xids, &key, &val2);
+                ft_msg msg(&key, &val2, FT_INSERT, ZERO_MSN, msg_xids);
                 ULE_S ule_expected;
                 ule_expected.uxrs = ule_expected.uxrs_static;
-                generate_both_for(&ule_expected, &val, &msg);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
+                generate_both_for(&ule_expected, &val, msg);
+                test_le_apply(&ule_initial, msg, &ule_expected);
             }
             {
                 //INSERT_NO_OVERWRITE will not change a committed insert
@@ -735,8 +736,8 @@ test_le_committed_apply(void) {
                 fillrandom(valbuf2, valsize2);
                 DBT val2;
                 toku_fill_dbt(&val2, valbuf2, valsize2);
-                msg = msg_init(FT_INSERT_NO_OVERWRITE, msg_xids, &key, &val2);
-                test_le_apply(&ule_initial, &msg, &ule_expected);
+                ft_msg msg(&key, &val2, FT_INSERT_NO_OVERWRITE, ZERO_MSN, msg_xids);
+                test_le_apply(&ule_initial, msg, &ule_expected);
             }
         }
     }
@@ -855,7 +856,6 @@ static void test_le_garbage_collection_birdie(void) {
 }
 
 static void test_le_optimize(void) {
-    FT_MSG_S msg;
     DBT key;
     DBT val;
     ULE_S ule_initial;
@@ -873,7 +873,7 @@ static void test_le_optimize(void) {
     XIDS msg_xids; 
     int r = xids_create_child(root_xids, &msg_xids, optimize_txnid);
     assert(r==0);
-    msg = msg_init(FT_OPTIMIZE, msg_xids, &key, &val);
+    ft_msg msg(&key, &val, FT_OPTIMIZE, ZERO_MSN, msg_xids);
 
     //
     // create the key
@@ -898,7 +898,7 @@ static void test_le_optimize(void) {
     ule_expected.uxrs[0].vallen = valsize;
     ule_expected.uxrs[0].valp = valbuf;
 
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     //
@@ -916,7 +916,7 @@ static void test_le_optimize(void) {
     ule_expected.uxrs[1].vallen = 0;
     ule_expected.uxrs[1].valp = NULL;
     
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     //
@@ -929,19 +929,19 @@ static void test_le_optimize(void) {
     ule_expected.num_cuxrs = 1;
     ule_expected.num_puxrs = 1;
     ule_expected.uxrs[1].xid = 1500;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     ule_initial.uxrs[1].xid = 1000;
     ule_expected.uxrs[1].xid = 1000;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     ule_initial.uxrs[1].xid = 500;
     ule_expected.uxrs[1].xid = 500;
     ule_expected.num_cuxrs = 2;
     ule_expected.num_puxrs = 0;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     //
@@ -963,12 +963,12 @@ static void test_le_optimize(void) {
     ule_expected.uxrs[2].vallen = valsize;
     ule_expected.uxrs[2].valp = valbuf;
     ule_expected.uxrs[1].xid = 1200;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     ule_initial.uxrs[1].xid = 1000;
     ule_expected.uxrs[1].xid = 1000;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     ule_initial.uxrs[1].xid = 800;
@@ -978,7 +978,7 @@ static void test_le_optimize(void) {
     ule_expected.uxrs[1].type = ule_initial.uxrs[2].type;
     ule_expected.uxrs[1].valp = ule_initial.uxrs[2].valp;
     ule_expected.uxrs[1].vallen = ule_initial.uxrs[2].vallen;
-    test_msg_modify_ule(&ule_initial,&msg);
+    test_msg_modify_ule(&ule_initial, msg);
     verify_ule_equal(&ule_initial, &ule_expected);
 
     

ft/tests/verify-bad-msn.cc

@@ -127,9 +127,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
     MSN msn = next_dummymsn();
 
     // apply an insert to the leaf node
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // Create bad tree (don't do following):
     // leafnode->max_msn_applied_to_node = msn;

ft/tests/verify-bad-pivots.cc

@@ -115,9 +115,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tests/verify-dup-in-leaf.cc

@@ -116,9 +116,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tests/verify-dup-pivots.cc

@@ -115,9 +115,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tests/verify-misrouted-msgs.cc

@@ -116,9 +116,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode,0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tests/verify-unsorted-leaf.cc

@@ -118,9 +118,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tests/verify-unsorted-pivots.cc

@@ -115,9 +115,9 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen)
 
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
-    FT_MSG_S msg = { FT_INSERT, msn, xids_get_root_xids(), .u={.id = { &thekey, &theval }} };
+    ft_msg msg(&thekey, &theval, FT_INSERT, msn, xids_get_root_xids());
     txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false);
-    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), &msg, idx, keylen, NULL, &gc_info, NULL, NULL);
+    toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

ft/tokuftdump.cc

@@ -280,14 +280,14 @@ static void dump_node(int fd, BLOCKNUM blocknum, FT ft) {
             }
             if (do_dump_data) {
                 struct dump_data_fn {
-                    int operator()(FT_MSG msg, bool UU(is_fresh)) {
-                        enum ft_msg_type type = (enum ft_msg_type) msg->type;
-                        MSN msn = msg->msn;
-                        XIDS xids = msg->xids;
-                        const void *key = ft_msg_get_key(msg);
-                        const void *data = ft_msg_get_val(msg);
-                        ITEMLEN keylen = ft_msg_get_keylen(msg);
-                        ITEMLEN datalen = ft_msg_get_vallen(msg);
+                    int operator()(const ft_msg &msg, bool UU(is_fresh)) {
+                        enum ft_msg_type type = (enum ft_msg_type) msg.type();
+                        MSN msn = msg.msn();
+                        XIDS xids = msg.xids();
+                        const void *key = msg.kdbt()->data;
+                        const void *data = msg.vdbt()->data;
+                        ITEMLEN keylen = msg.kdbt()->size;
+                        ITEMLEN datalen = msg.vdbt()->size;
                         printf("    msn=%" PRIu64 " (0x%" PRIx64 ") ", msn.msn, msn.msn);
                         printf("    TYPE=");
                         switch (type) {

ft/ule-internal.h

@@ -135,7 +135,7 @@ typedef struct ule {     // unpacked leaf entry
 
 
 
-void test_msg_modify_ule(ULE ule, FT_MSG msg);
+void test_msg_modify_ule(ULE ule, const ft_msg &msg);
 
 
 //////////////////////////////////////////////////////////////////////////////////////

ft/ule.cc

@@ -105,7 +105,7 @@ PATENT RIGHTS GRANT:
 #include <toku_portability.h>
 #include "ft/fttypes.h"
 #include "ft/ft-internal.h"
-#include "ft/ft_msg.h"
+#include "ft/msg.h"
 #include "ft/leafentry.h"
 #include "ft/logger.h"
 #include "ft/txn.h"
@@ -216,7 +216,7 @@ const UXR_S committed_delete = {
 // Local functions:
 
 static void msg_init_empty_ule(ULE ule);
-static void msg_modify_ule(ULE ule, FT_MSG msg);
+static void msg_modify_ule(ULE ule, const ft_msg &msg);
 static void ule_init_empty_ule(ULE ule);
 static void ule_do_implicit_promotions(ULE ule, XIDS xids);
 static void ule_try_promote_provisional_outermost(ULE ule, TXNID oldest_possible_live_xid);
@@ -496,7 +496,7 @@ enum {
 //   Otehrwise the new_leafentry_p points at the new leaf entry.
 // As of October 2011, this function always returns 0.
 void
-toku_le_apply_msg(FT_MSG   msg,
+toku_le_apply_msg(const ft_msg &msg,
                   LEAFENTRY old_leafentry, // NULL if there was no stored data.
                   bn_data* data_buffer, // bn_data storing leafentry, if NULL, means there is no bn_data
                   uint32_t idx, // index in data_buffer where leafentry is stored (and should be replaced
@@ -510,7 +510,7 @@ toku_le_apply_msg(FT_MSG   msg,
     int64_t oldnumbytes = 0;
     int64_t newnumbytes = 0;
     uint64_t oldmemsize = 0;
-    uint32_t keylen = ft_msg_get_keylen(msg);
+    uint32_t keylen = msg.kdbt()->size;
 
     if (old_leafentry == NULL) {
         msg_init_empty_ule(&ule);
@@ -555,7 +555,7 @@ toku_le_apply_msg(FT_MSG   msg,
         &ule, // create packed leafentry
         data_buffer,
         idx,
-        ft_msg_get_key(msg), // contract of this function is caller has this set, always
+        msg.kdbt()->data, // contract of this function is caller has this set, always
         keylen, // contract of this function is caller has this set, always
         old_keylen,
         oldmemsize,
@@ -693,10 +693,10 @@ msg_init_empty_ule(ULE ule) {
 // Purpose is to modify the unpacked leafentry in our private workspace.
 //
 static void 
-msg_modify_ule(ULE ule, FT_MSG msg) {
-    XIDS xids = ft_msg_get_xids(msg);
+msg_modify_ule(ULE ule, const ft_msg &msg) {
+    XIDS xids = msg.xids();
     invariant(xids_get_num_xids(xids) < MAX_TRANSACTION_RECORDS);
-    enum ft_msg_type type = ft_msg_get_type(msg);
+    enum ft_msg_type type = msg.type();
     if (type != FT_OPTIMIZE && type != FT_OPTIMIZE_FOR_UPGRADE) {
         ule_do_implicit_promotions(ule, xids);
     }
@@ -709,9 +709,9 @@ msg_modify_ule(ULE ule, FT_MSG msg) {
         //fall through to FT_INSERT on purpose.
     }
     case FT_INSERT: {
-        uint32_t vallen = ft_msg_get_vallen(msg);
+        uint32_t vallen = msg.vdbt()->size;
         invariant(IS_VALID_LEN(vallen));
-        void * valp      = ft_msg_get_val(msg);
+        void * valp      = msg.vdbt()->data;
         ule_apply_insert(ule, xids, vallen, valp);
         break;
     }
@@ -738,17 +738,15 @@ msg_modify_ule(ULE ule, FT_MSG msg) {
         assert(false); // These messages don't get this far.  Instead they get translated (in setval_fun in do_update) into FT_INSERT messages.
         break;
     default:
-        assert(false /* illegal FT_MSG.type */);
+        assert(false); /* illegal ft msg type */
         break;
     }
 }
 
-void 
-test_msg_modify_ule(ULE ule, FT_MSG msg){
+void test_msg_modify_ule(ULE ule, const ft_msg &msg){
     msg_modify_ule(ule,msg);
 }
 
-
 static void ule_optimize(ULE ule, XIDS xids) {
     if (ule->num_puxrs) {
         TXNID uncommitted = ule->uxrs[ule->num_cuxrs].xid;      // outermost uncommitted

ft/ybt.cc

@@ -90,23 +90,30 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include <db.h>
-#include <memory.h>
 #include <string.h>
-#include <fttypes.h>
 
-#include "ybt.h"
+#include "portability/memory.h"
+
+#include "ft/fttypes.h"
+#include "ft/ybt.h"
 
 DBT *
-toku_init_dbt(DBT *ybt) {
-    memset(ybt, 0, sizeof(*ybt));
-    return ybt;
+toku_init_dbt(DBT *dbt) {
+    memset(dbt, 0, sizeof(*dbt));
+    return dbt;
+}
+
+DBT
+toku_empty_dbt(void) {
+    static const DBT empty_dbt = { .data = 0, .size = 0, .ulen = 0, .flags = 0 };
+    return empty_dbt;
 }
 
 DBT *
-toku_init_dbt_flags(DBT *ybt, uint32_t flags) {
-    toku_init_dbt(ybt);
-    ybt->flags = flags;
-    return ybt;
+toku_init_dbt_flags(DBT *dbt, uint32_t flags) {
+    toku_init_dbt(dbt);
+    dbt->flags = flags;
+    return dbt;
 }
 
 DBT_ARRAY *

ft/ybt.h

@@ -102,6 +102,9 @@ PATENT RIGHTS GRANT:
 
 DBT *toku_init_dbt(DBT *);
 
+// returns: an initialized but empty dbt (for which toku_dbt_is_empty() is true)
+DBT toku_empty_dbt(void);
+
 DBT *toku_init_dbt_flags(DBT *, uint32_t flags);
 
 void toku_destroy_dbt(DBT *);