Browse Source

Start integraring encryption and compression so that we have

only one implementation of encryption using CRYPT_SCHEME_1
where

L = AES_ECB(KEY, IV)
CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE)

and two different implementations how keys are stored.

For AIO this means that compression and encryption is called
at os0file as earlier. For all other we do these at buf0buf.cc.
replace/d7d589dc01f6d70d1518b74d46fd3b75e76267f5
Jan Lindström 11 years ago
committed by Sergei Golubchik
parent
commit
831bf6eada
  1. 6
      include/my_crypt.h
  2. 6
      mysys/my_crypt.cc
  3. 6
      mysys/my_crypt_key_management.cc
  4. 1
      storage/innobase/CMakeLists.txt
  5. 109
      storage/innobase/buf/buf0buf.cc
  6. 22
      storage/innobase/buf/buf0dblwr.cc
  7. 24
      storage/innobase/buf/buf0flu.cc
  8. 4
      storage/innobase/buf/buf0rea.cc
  9. 335
      storage/innobase/fil/fil0crypt.cc
  10. 17
      storage/innobase/fil/fil0fil.cc
  11. 6
      storage/innobase/fil/fil0pagecompress.cc
  12. 638
      storage/innobase/fil/fil0pageencryption.cc
  13. 7
      storage/innobase/handler/ha_innodb.cc
  14. 15
      storage/innobase/include/buf0buf.h
  15. 9
      storage/innobase/include/fil0fil.h
  16. 44
      storage/innobase/include/fil0pageencryption.h
  17. 33
      storage/innobase/include/fsp0pageencryption.ic
  18. 2
      storage/innobase/include/log0log.h
  19. 13
      storage/innobase/include/os0file.h
  20. 3
      storage/innobase/include/os0file.ic
  21. 3
      storage/innobase/include/univ.i
  22. 10
      storage/innobase/log/log0log.cc
  23. 4
      storage/innobase/log/log0recv.cc
  24. 177
      storage/innobase/os/os0file.cc

6
include/my_crypt.h

@ -1,8 +1,12 @@
// TODO: Add Windows support
#ifdef __linux__
#ifndef MYSYS_MY_CRYPT_H_
#define MYSYS_MY_CRYPT_H_
#include <assert.h>
#include <openssl/evp.h>
#include "my_global.h"
@ -282,3 +286,5 @@ enum CryptResult RandomBytes(unsigned char* buf, int num);
C_MODE_END
#endif // MYSYS_MY_CRYPT_H_
#endif // __linux__

6
mysys/my_crypt.cc

@ -1,3 +1,7 @@
// TODO: add Windows support
#ifdef __linux__
#include "my_crypt.h"
#include <openssl/aes.h>
@ -300,3 +304,5 @@ CryptResult RandomBytes(unsigned char* buf, int num) {
}
} // extern "C"
#endif // __linux__

6
mysys/my_crypt_key_management.cc

@ -1,9 +1,10 @@
#include "my_crypt_key_management.h"
#include <arpa/inet.h>
#include <cstring>
#ifdef __linux__
#include <arpa/inet.h>
#endif
#include "my_pthread.h"
#ifndef DBUG_OFF
@ -57,6 +58,7 @@ int GetCryptoKey(unsigned int version, unsigned char* key, unsigned int size) {
if (size < sizeof(version)) {
return 1;
}
version = htonl(version);
memcpy(key, &version, sizeof(version));
return 0;

1
storage/innobase/CMakeLists.txt

@ -365,7 +365,6 @@ SET(INNOBASE_SOURCES
eval/eval0proc.cc
fil/fil0fil.cc
fil/fil0pagecompress.cc
fil/fil0pageencryption.cc
fil/fil0crypt.cc
fsp/fsp0fsp.cc
fut/fut0fut.cc

109
storage/innobase/buf/buf0buf.cc

@ -55,6 +55,7 @@ Created 11/5/1995 Heikki Tuuri
#include "srv0mon.h"
#include "buf0checksum.h"
#include "fil0pageencryption.h"
#include "fil0pagecompress.h"
#include "ut0byte.h"
#include <new>
@ -503,7 +504,7 @@ buf_page_is_corrupted(
ulint zip_size) /*!< in: size of compressed page;
0 for uncompressed pages */
{
ulint page_encrypted = fil_page_is_encrypted(read_buf);
ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf);
ulint checksum_field1;
ulint checksum_field2;
ibool crc32_inited = FALSE;
@ -1002,6 +1003,8 @@ buf_block_init(
block->page.io_fix = BUF_IO_NONE;
block->page.crypt_buf = NULL;
block->page.crypt_buf_free = NULL;
block->page.comp_buf = NULL;
block->page.comp_buf_free = NULL;
block->page.key_version = 0;
block->modify_clock = 0;
@ -3492,6 +3495,8 @@ buf_page_init_low(
bpage->write_size = 0;
bpage->crypt_buf = NULL;
bpage->crypt_buf_free = NULL;
bpage->comp_buf = NULL;
bpage->comp_buf_free = NULL;
bpage->key_version = 0;
HASH_INVALIDATE(bpage, hash);
@ -5604,6 +5609,11 @@ buf_page_encrypt_before_write(
buf_page_t* bpage, /*!< in/out: buffer page to be flushed */
const byte* src_frame) /*!< in: src frame */
{
if (srv_encrypt_tables == FALSE) {
/* Encryption is disabled */
return const_cast<byte*>(src_frame);
}
if (bpage->offset == 0) {
/* Page 0 of a tablespace is not encrypted */
ut_ad(bpage->key_version == 0);
@ -5640,18 +5650,32 @@ buf_page_encrypt_before_write(
* considered a lot.
*/
bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2);
byte *dst_frame = bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
if (bpage->crypt_buf_free == NULL) {
bpage->crypt_buf_free = (byte*)malloc(page_size*2);
// TODO: Is 4k aligment enough ?
bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, page_size);
}
// encrypt page content
fil_space_encrypt(bpage->space, bpage->offset,
bpage->newest_modification,
src_frame, zip_size, dst_frame);
byte *dst_frame = bpage->crypt_buf;
unsigned key_version =
mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
ut_ad(key_version == 0 || key_version >= bpage->key_version);
bpage->key_version = key_version;
if (!fil_space_is_page_compressed(bpage->space)) {
fprintf(stderr, "JAN: problem:::\n");
// encrypt page content
fil_space_encrypt(bpage->space, bpage->offset,
bpage->newest_modification,
src_frame, zip_size, dst_frame, 0);
unsigned key_version =
mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
ut_ad(key_version == 0 || key_version >= bpage->key_version);
bpage->key_version = key_version;
// return dst_frame which will be written
return dst_frame;
} else {
// We do compression and encryption later on os0file.cc
dst_frame = (byte *)src_frame;
}
// return dst_frame which will be written
return dst_frame;
@ -5670,6 +5694,13 @@ buf_page_encrypt_after_write(
bpage->crypt_buf_free = NULL;
bpage->crypt_buf = NULL;
}
if (bpage->comp_buf_free != NULL) {
free(bpage->comp_buf_free);
bpage->comp_buf_free = NULL;
bpage->comp_buf = NULL;
}
return (TRUE);
}
@ -5697,10 +5728,20 @@ unencrypted:
goto unencrypted;
}
// allocate buffer to read data into
bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2);
bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
return bpage->crypt_buf;
if (srv_encrypt_tables) {
if (bpage->crypt_buf_free == NULL) {
// allocate buffer to read data into
bpage->crypt_buf_free = (byte*)malloc(size*2);
// TODO: Is 4K aligment enough ?
bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size);
}
return bpage->crypt_buf;
} else {
// If database contains encrypted data it will be
// handled later.
goto unencrypted;
}
}
/********************************************************************//**
@ -5712,27 +5753,38 @@ buf_page_decrypt_after_read(
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{
ut_ad(bpage->key_version == 0);
ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
if (bpage->offset == 0) {
fprintf(stderr, "JAN: page type %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE));
/* File header pages are not encrypted */
ut_a(bpage->crypt_buf == NULL);
return (TRUE);
}
ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
const byte* src_frame = bpage->crypt_buf != NULL ?
bpage->crypt_buf : dst_frame;
unsigned key_version =
mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
fprintf(stderr, "JAN: key_version %lu, page_type %lu\n", key_version,
mach_read_from_2(src_frame+FIL_PAGE_TYPE));
if (key_version == 0) {
fprintf(stderr, "JAN: unencrypted, dst %p src %p\n", dst_frame, src_frame);
/* the page we read is unencrypted */
if (dst_frame != src_frame) {
/* but we had allocated a crypt_buf */
// TODO: Can this be avoided ?
memcpy(dst_frame, src_frame, size);
}
} else {
@ -5742,12 +5794,31 @@ buf_page_decrypt_after_read(
* malloc a buffer, copy page to it
* and then decrypt from that into real page*/
bpage->crypt_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
// TODO: is 4k aligment enough ?
src_frame = bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
memcpy(bpage->crypt_buf, dst_frame, size);
}
fprintf(stderr, "JAN: buf0buf.cc: decrypt page 1 %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE));
fprintf(stderr, "JAN: buf0buf.cc: decrypt page 2 %lu\n", mach_read_from_2(src_frame+FIL_PAGE_TYPE));
/* decrypt from src_frame to dst_frame */
fil_space_decrypt(bpage->space,
src_frame, size, dst_frame);
fprintf(stderr, "JAN: buf0buf.cc: after decrypt %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE));
/* decompress from dst_frame to comp_buf and then copy to
buffer pool */
if (page_compressed_encrypted) {
if (bpage->comp_buf_free == NULL) {
bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
// TODO: is 4k aligment enough ?
bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE);
}
fprintf(stderr, "JAN: buf0buf.cc: decompress %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE));
fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL);
fprintf(stderr, "JAN: buf0buf.cc: after decompress %lu\n", mach_read_from_2(dst_frame+FIL_PAGE_TYPE));
}
}
bpage->key_version = key_version;

22
storage/innobase/buf/buf0dblwr.cc

@ -524,7 +524,7 @@ buf_dblwr_process()
fil_io(OS_FILE_READ, true, space_id, zip_size,
page_no, 0,
zip_size ? zip_size : UNIV_PAGE_SIZE,
read_buf, NULL, 0);
read_buf, NULL, 0, 0);
if (fil_space_verify_crypt_checksum(read_buf, zip_size)) {
/* page is encrypted and checksum is OK */
@ -579,7 +579,7 @@ buf_dblwr_process()
fil_io(OS_FILE_WRITE, true, space_id,
zip_size, page_no, 0,
zip_size ? zip_size : UNIV_PAGE_SIZE,
page, NULL, 0);
page, NULL, 0, 0);
ib_logf(IB_LOG_LEVEL_INFO,
"Recovered the page from"
@ -599,7 +599,7 @@ buf_dblwr_process()
zip_size, page_no, 0,
zip_size ? zip_size
: UNIV_PAGE_SIZE,
page, NULL, 0);
page, NULL, 0, 0);
}
}
}
@ -621,9 +621,9 @@ buf_dblwr_process()
memset(buf, 0, bytes);
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
buf_dblwr->block1, 0, bytes, buf, NULL, NULL);
buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0);
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
buf_dblwr->block2, 0, bytes, buf, NULL, NULL);
buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0);
ut_free(unaligned_buf);
}
@ -833,7 +833,7 @@ buf_dblwr_write_block_to_datafile(
buf_page_get_page_no(bpage), 0,
buf_page_get_zip_size(bpage),
frame,
(void*) bpage, 0);
(void*) bpage, 0, bpage->newest_modification);
return;
}
@ -845,7 +845,7 @@ buf_dblwr_write_block_to_datafile(
fil_io(flags, sync, buf_block_get_space(block), 0,
buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE,
frame, (void*) block, (ulint *)&bpage->write_size);
frame, (void*) block, (ulint *)&bpage->write_size, bpage->newest_modification );
}
/********************************************************************//**
@ -939,7 +939,7 @@ try_again:
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
buf_dblwr->block1, 0, len,
(void*) write_buf, NULL, 0);
(void*) write_buf, NULL, 0, 0);
if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) {
/* No unwritten pages in the second block. */
@ -955,7 +955,7 @@ try_again:
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
buf_dblwr->block2, 0, len,
(void*) write_buf, NULL, 0);
(void*) write_buf, NULL, 0, 0);
flush:
/* increment the doublewrite flushed pages counter */
@ -1187,14 +1187,14 @@ retry:
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
offset, 0, UNIV_PAGE_SIZE,
(void*) (buf_dblwr->write_buf
+ UNIV_PAGE_SIZE * i), NULL, 0);
+ UNIV_PAGE_SIZE * i), NULL, 0, bpage->newest_modification);
} else {
/* It is a regular page. Write it directly to the
doublewrite buffer */
fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0,
offset, 0, UNIV_PAGE_SIZE,
frame,
NULL, 0);
NULL, 0, bpage->newest_modification);
}
/* Now flush the doublewrite buffer data to disk */

24
storage/innobase/buf/buf0flu.cc

@ -913,10 +913,16 @@ buf_flush_write_block_low(
if (!srv_use_doublewrite_buf || !buf_dblwr) {
fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER,
sync, buf_page_get_space(bpage), zip_size,
buf_page_get_page_no(bpage), 0,
sync,
buf_page_get_space(bpage),
zip_size,
buf_page_get_page_no(bpage),
0,
zip_size ? zip_size : UNIV_PAGE_SIZE,
frame, bpage, &bpage->write_size);
frame,
bpage,
&bpage->write_size,
bpage->newest_modification);
} else {
/* InnoDB uses doublewrite buffer and doublewrite buffer
@ -928,10 +934,16 @@ buf_flush_write_block_low(
if (awrites == ATOMIC_WRITES_ON) {
fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER,
FALSE, buf_page_get_space(bpage), zip_size,
buf_page_get_page_no(bpage), 0,
FALSE,
buf_page_get_space(bpage),
zip_size,
buf_page_get_page_no(bpage),
0,
zip_size ? zip_size : UNIV_PAGE_SIZE,
frame, bpage, &bpage->write_size);
frame,
bpage,
&bpage->write_size,
bpage->newest_modification);
} else if (flush_type == BUF_FLUSH_SINGLE_PAGE) {
buf_dblwr_write_single_page(bpage, sync);
} else {

4
storage/innobase/buf/buf0rea.cc

@ -186,7 +186,7 @@ buf_read_page_low(
*err = fil_io(OS_FILE_READ | wake_later
| ignore_nonexistent_pages,
sync, space, zip_size, offset, 0, zip_size,
frame, bpage, &bpage->write_size);
frame, bpage, &bpage->write_size, 0);
} else {
ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE);
@ -194,7 +194,7 @@ buf_read_page_low(
| ignore_nonexistent_pages,
sync, space, 0, offset, 0, UNIV_PAGE_SIZE,
frame, bpage,
&bpage->write_size);
&bpage->write_size, 0);
}
if (sync) {

335
storage/innobase/fil/fil0crypt.cc

@ -9,10 +9,17 @@
#include "ut0ut.h"
#include "btr0scrub.h"
#include "fsp0fsp.h"
#include "fil0pagecompress.h"
#include "fil0pageencryption.h"
#include <my_crypt.h>
#include <my_crypt_key_management.h>
#include <my_aes.h>
#include <KeySingleton.h>
#include <math.h>
/** Mutex for keys */
UNIV_INTERN ib_mutex_t fil_crypt_key_mutex;
@ -174,49 +181,86 @@ Get key bytes for a space/key-version */
static
void
fil_crypt_get_key(byte *dst, uint dstlen,
fil_space_crypt_t* crypt_data, uint version)
fil_space_crypt_t* crypt_data, uint version, bool page_encrypted)
{
mutex_enter(&crypt_data->mutex);
/* TODO: Find a better way to do this */
unsigned char keybuf[CRYPT_SCHEME_1_IV_LEN] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9,
0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23};
byte key[CRYPT_SCHEME_1_IV_LEN];
uint8 key_len = sizeof(keybuf);
// Check if we already have key
for (uint i = 0; i < crypt_data->key_count; i++) {
if (crypt_data->keys[i].key_version == version) {
memcpy(dst, crypt_data->keys[i].key,
sizeof(crypt_data->keys[i].key));
mutex_exit(&crypt_data->mutex);
return;
unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed,
0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e};
ulint iv_len = 16;
bool key_error = false;
if (!page_encrypted) {
mutex_enter(&crypt_data->mutex);
// Check if we already have key
for (uint i = 0; i < crypt_data->key_count; i++) {
if (crypt_data->keys[i].key_version == version) {
memcpy(dst, crypt_data->keys[i].key,
sizeof(crypt_data->keys[i].key));
mutex_exit(&crypt_data->mutex);
return;
}
}
}
// Not found!
crypt_data->keyserver_requests++;
// Not found!
crypt_data->keyserver_requests++;
// Rotate keys to make room for a new
for (uint i = 1; i < array_elements(crypt_data->keys); i++) {
crypt_data->keys[i] = crypt_data->keys[i - 1];
}
// Rotate keys to make room for a new
for (uint i = 1; i < array_elements(crypt_data->keys); i++) {
crypt_data->keys[i] = crypt_data->keys[i - 1];
}
// TODO(jonaso): Integrate with real key server
int rc = GetCryptoKey(version, keybuf, key_len);
if (rc != 0) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to retrieve key with"
" version %u return-code: %d. Can't continue!\n",
version, rc);
ut_error;
}
} else {
/* For page encrypted tables we need to get the L */
// TODO(jonaso): Integrate with real key server
/* Get key and IV */
KeySingleton& keys = KeySingleton::getInstance();
// Get new key from key server
unsigned char keybuf[CRYPT_SCHEME_1_IV_LEN];
unsigned keylen = sizeof(keybuf);
int rc = GetCryptoKey(version, keybuf, keylen);
if (rc != 0) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to retrieve key with"
" version %u return-code: %d. Can't continue!\n",
version, rc);
ut_error;
if (keys.isAvailable() && keys.getKeys(version) != NULL) {
char* keyString = keys.getKeys(version)->key;
char* ivString = keys.getKeys(version)->iv;
if (keyString == NULL || ivString == NULL) {
key_error=true;
} else {
my_aes_hexToUint(keyString, (unsigned char*)&keybuf, key_len);
my_aes_hexToUint(ivString, (unsigned char*)&iv, iv_len);
}
} else {
key_error = true;
}
if (key_error == true) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Key file not found");
ut_error;
}
}
// Now compute L by encrypting IV using this key
const unsigned char* src = crypt_data->iv;
const int srclen = crypt_data->iv_length;
unsigned char* buf = crypt_data->keys[0].key;
int buflen = sizeof(crypt_data->keys[0].key);
rc = EncryptAes128Ecb(keybuf,
const unsigned char* src = page_encrypted ? iv : crypt_data->iv;
const int srclen = page_encrypted ? iv_len : crypt_data->iv_length;
unsigned char* buf = page_encrypted ? key : crypt_data->keys[0].key;
int buflen = page_encrypted ? key_len : sizeof(crypt_data->keys[0].key);
int rc = EncryptAes128Ecb(keybuf,
src, srclen,
buf, &buflen);
if (rc != CRYPT_OK) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt key-block "
@ -226,16 +270,20 @@ fil_crypt_get_key(byte *dst, uint dstlen,
ut_error;
}
crypt_data->keys[0].key_version = version;
crypt_data->key_count++;
if (!page_encrypted) {
crypt_data->keys[0].key_version = version;
crypt_data->key_count++;
if (crypt_data->key_count > array_elements(crypt_data->keys)) {
crypt_data->key_count = array_elements(crypt_data->keys);
if (crypt_data->key_count > array_elements(crypt_data->keys)) {
crypt_data->key_count = array_elements(crypt_data->keys);
}
}
memcpy(dst, crypt_data->keys[0].key,
sizeof(crypt_data->keys[0].key));
mutex_exit(&crypt_data->mutex);
memcpy(dst, buf, buflen);
if (!page_encrypted) {
mutex_exit(&crypt_data->mutex);
}
}
/******************************************************************
@ -245,8 +293,12 @@ void
fil_crypt_get_latest_key(byte *dst, uint dstlen,
fil_space_crypt_t* crypt_data, uint *version)
{
*version = GetLatestCryptoKeyVersion();
return fil_crypt_get_key(dst, dstlen, crypt_data, *version);
if (srv_encrypt_tables) {
*version = GetLatestCryptoKeyVersion();
return fil_crypt_get_key(dst, dstlen, crypt_data, *version, false);
} else {
return fil_crypt_get_key(dst, dstlen, NULL, *version, true);
}
}
/******************************************************************
@ -553,26 +605,46 @@ Encrypt a page */
UNIV_INTERN
void
fil_space_encrypt(ulint space, ulint offset, lsn_t lsn,
const byte* src_frame, ulint zip_size, byte* dst_frame)
const byte* src_frame, ulint zip_size, byte* dst_frame, ulint encryption_key)
{
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space);
fil_space_crypt_t* crypt_data;
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
if (crypt_data == NULL || srv_encrypt_tables == FALSE) {
memcpy(dst_frame, src_frame, page_size);
return;
}
// get key (L)
uint key_version;
byte key[CRYPT_SCHEME_1_IV_LEN];
fil_crypt_get_latest_key(key, sizeof(key), crypt_data, &key_version);
if (srv_encrypt_tables) {
crypt_data = fil_space_get_crypt_data(space);
fil_crypt_get_latest_key(key, sizeof(key), crypt_data, &key_version);
} else {
key_version = encryption_key;
fil_crypt_get_latest_key(key, sizeof(key), NULL, (uint*)&key_version);
}
ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED);
ibool page_encrypted = fil_space_is_page_encrypted(space);
ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
// copy page header
memcpy(dst_frame, src_frame, FIL_PAGE_DATA);
// store key version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
ulint orig_page_type = mach_read_from_2(dst_frame+FIL_PAGE_TYPE);
if (page_encrypted && !page_compressed) {
// key id
mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
key_version);
// original page type
mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2,
orig_page_type);
// new page type
mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED);
} else {
// store key version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
key_version);
}
// create counter block (C)
unsigned char counter[AES_128_BLOCK_SIZE];
@ -582,15 +654,20 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn,
// encrypt page data
ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END;
ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END);
const byte* src = src_frame + FIL_PAGE_DATA;
byte* dst = dst_frame + FIL_PAGE_DATA;
int dstlen;
if (page_compressed) {
srclen = page_size;
}
int rc = EncryptAes128Ctr(key, counter, sizeof(counter),
src, (page_size - unencrypted_bytes),
src, srclen,
dst, &dstlen);
if (! ((rc == CRYPT_OK) &&
((ulint) dstlen == (page_size - unencrypted_bytes)))) {
if (! ((rc == CRYPT_OK) && ((ulint) dstlen == srclen))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to encrypt data-block "
" src: %p srclen: %ld buf: %p buflen: %d."
@ -600,42 +677,56 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn,
ut_error;
}
// copy page trailer
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END);
if (!page_compressed) {
// copy page trailer
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END);
/* handle post encryption checksum */
ib_uint32_t checksum = 0;
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
/* handle post encryption checksum */
ib_uint32_t checksum = 0;
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
if (zip_size == 0) {
switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(dst_frame);
break;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t) buf_calc_page_new_checksum(
dst_frame);
break;
case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC;
break;
/* no default so the compiler will emit a warning
* if new enum is added and not handled here */
if (zip_size == 0) {
switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(dst_frame);
break;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t) buf_calc_page_new_checksum(
dst_frame);
break;
case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC;
break;
/* no default so the compiler will emit a warning
* if new enum is added and not handled here */
}
} else {
checksum = page_zip_calc_checksum(dst_frame, zip_size,
algorithm);
}
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4,
checksum);
} else {
checksum = page_zip_calc_checksum(dst_frame, zip_size,
algorithm);
/* Page compressed and encrypted tables have different
FIL_HEADER */
ulint page_len = log10((double)srclen)/log10((double)2);
/* Set up the correct page type */
mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
/* Set up the compression algorithm */
mach_write_to_2(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4, orig_page_type);
/* Set up the compressed size */
mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6, page_len);
/* Set up the compression method */
mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7, compression_alg);
}
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4,
checksum);
}
/*********************************************************************
@ -663,12 +754,30 @@ bool
fil_space_decrypt(fil_space_crypt_t* crypt_data,
const byte* src_frame, ulint page_size, byte* dst_frame)
{
ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
// key version
uint key_version = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
uint key_version;
bool page_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
|| page_type == FIL_PAGE_PAGE_ENCRYPTED);
if (key_version == 0) {
memcpy(dst_frame, src_frame, page_size);
ulint orig_page_type=0;
fprintf(stderr, "JAN: page type %lu\n", page_type);
if (page_type == FIL_PAGE_PAGE_ENCRYPTED) {
key_version = mach_read_from_2(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
fprintf(stderr, "JAN: key_version %lu\n", key_version);
orig_page_type = mach_read_from_2(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2);
fprintf(stderr, "JAN: decrypt: orig_page_type %lu\n", orig_page_type);
} else {
key_version = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
}
if (key_version == 0 && !page_encrypted) {
fprintf(stderr, "JAN: unencrypted\n");
return false; /* page not decrypted */
}
@ -679,15 +788,20 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data,
src_frame + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN);
ut_a(crypt_data != NULL);
fprintf(stderr, "JAN: space %lu, offset %lu lsn %lu\n", space, offset, lsn);
// get key (L)
byte key[CRYPT_SCHEME_1_IV_LEN];
fil_crypt_get_key(key, sizeof(key), crypt_data, key_version);
fil_crypt_get_key(key, sizeof(key), crypt_data, key_version, page_encrypted);
// copy page header
memcpy(dst_frame, src_frame, FIL_PAGE_DATA);
if (page_type == FIL_PAGE_PAGE_ENCRYPTED) {
// orig page type
mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type);
}
// create counter block
unsigned char counter[AES_128_BLOCK_SIZE];
mach_write_to_4(counter + 0, space);
@ -700,11 +814,21 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data,
const byte* src = src_frame + FIL_PAGE_DATA;
byte* dst = dst_frame + FIL_PAGE_DATA;
int dstlen;
ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END);
orig_page_type = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4);
ulint compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6);
ulint compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7);
if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
srclen = pow((double)2, (double)((int)compressed_len));
}
int rc = DecryptAes128Ctr(key, counter, sizeof(counter),
src, (page_size - unencrypted_bytes),
src, srclen,
dst, &dstlen);
if (! ((rc == CRYPT_OK) &&
((ulint) dstlen == (page_size - unencrypted_bytes)))) {
if (! ((rc == CRYPT_OK) && ((ulint) dstlen == srclen))) {
ib_logf(IB_LOG_LEVEL_FATAL,
"Unable to decrypt data-block "
" src: %p srclen: %ld buf: %p buflen: %d."
@ -714,13 +838,24 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data,
ut_error;
}
// copy page trailer
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END);
if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
fprintf(stderr, "JAN: trailer...\n");
// copy page trailer
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END);
// clear key-version & crypt-checksum from dst
memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8);
// clear key-version & crypt-checksum from dst
memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8);
} else {
/* For page compressed tables we set up the FIL_HEADER again */
/* setting original page type */
mach_write_to_2(dst_frame + FIL_PAGE_TYPE, orig_page_type);
/* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */
mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC);
/* Set up the flush lsn to be compression algorithm */
mach_write_to_8(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, compression_method);
}
return true; /* page was decrypted */
}

17
storage/innobase/fil/fil0fil.cc

@ -285,7 +285,7 @@ fil_read(
actual page size does not decrease. */
{
return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset,
byte_offset, len, buf, message, write_size));
byte_offset, len, buf, message, write_size, 0));
}
/********************************************************************//**
@ -312,16 +312,17 @@ fil_write(
this must be appropriately aligned */
void* message, /*!< in: message for aio handler if non-sync
aio used, else ignored */
ulint* write_size) /*!< in/out: Actual write size initialized
ulint* write_size, /*!< in/out: Actual write size initialized
after fist successfull trim
operation for this page and if
initialized we do not trim again if
actual page size does not decrease. */
lsn_t lsn) /* lsn of the newest modification */
{
ut_ad(!srv_read_only_mode);
return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset,
byte_offset, len, buf, message, write_size));
byte_offset, len, buf, message, write_size, lsn));
}
/*******************************************************************//**
@ -1831,7 +1832,7 @@ fil_write_lsn_and_arch_no_to_file(
lsn);
err = fil_write(TRUE, space, 0, sum_of_sizes, 0,
UNIV_PAGE_SIZE, buf, NULL, 0);
UNIV_PAGE_SIZE, buf, NULL, 0, lsn);
}
mem_free(buf1);
@ -5212,7 +5213,7 @@ retry:
success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC,
node->name, node->handle, buf,
offset, page_size * n_pages,
node, NULL, 0, FALSE, 0, 0, 0);
node, NULL, 0, FALSE, 0, 0, 0, 0);
#endif /* UNIV_HOTBACKUP */
if (success) {
os_has_said_disk_full = FALSE;
@ -5588,11 +5589,12 @@ fil_io(
appropriately aligned */
void* message, /*!< in: message for aio handler if non-sync
aio used, else ignored */
ulint* write_size) /*!< in/out: Actual write size initialized
ulint* write_size, /*!< in/out: Actual write size initialized
after fist successfull trim
operation for this page and if
initialized we do not trim again if
actual page size does not decrease. */
lsn_t lsn) /* lsn of the newest modification */
{
ulint mode;
fil_space_t* space;
@ -5819,7 +5821,8 @@ fil_io(
page_compressed,
page_compression_level,
page_encrypted,
page_encryption_key);
page_encryption_key,
lsn);
#endif /* UNIV_HOTBACKUP */

6
storage/innobase/fil/fil0pagecompress.cc

@ -473,9 +473,6 @@ fil_compress_page(
ut_a(block_size > 0);
#endif
write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size);
/* Initialize rest of the written data to avoid
uninitialized bytes */
memset(out_buf+tmp, 0, write_size-tmp);
#ifdef UNIV_DEBUG
ut_a(write_size > 0 && ((write_size % block_size) == 0));
ut_a(write_size >= tmp);
@ -494,8 +491,7 @@ fil_compress_page(
if (!srv_use_trim) {
/* If persistent trims are not used we always write full
page and end of the page needs to be initialized.*/
memset(out_buf+write_size, 0, len-write_size);
page */
write_size = len;
}

638
storage/innobase/fil/fil0pageencryption.cc

@ -1,638 +0,0 @@
/*****************************************************************************
Copyright (C) 2014 eperi GmbH. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*****************************************************************************/
/*****************************************************************
@file fil/fil0pageencryption.cc
Implementation for page encryption file spaces.
Created 08/25/2014 Ludger Göckel eperi-GmbH
Modified 11/26/2014 Jan Lindström MariaDB Corporation
***********************************************************************/
#include "fil0fil.h"
#include "fil0pageencryption.h"
#include "fsp0pageencryption.h"
#include "my_dbug.h"
#include "page0zip.h"
#include "buf0checksum.h"
#include <my_aes.h>
#include <KeySingleton.h>
#include <math.h>
/*
* derived from libFLAC, which is gpl v2
*/
byte crc_table[] = {
0x00,0x07,0x0E,0x09,0x1C,0x1B,0x12,0x15,0x38,0x3F,0x36,0x31,0x24,0x23,0x2A,0x2D,0x70,0x77,0x7E,0x79,
0x6C,0x6B,0x62,0x65,0x48,0x4F,0x46,0x41,0x54,0x53,0x5A,0x5D,0xE0,0xE7,0xEE,0xE9,0xFC,0xFB,0xF2,0xF5,
0xD8,0xDF,0xD6,0xD1,0xC4,0xC3,0xCA,0xCD,0x90,0x97,0x9E,0x99,0x8C,0x8B,0x82,0x85,0xA8,0xAF,0xA6,0xA1,
0xB4,0xB3,0xBA,0xBD,0xC7,0xC0,0xC9,0xCE,0xDB,0xDC,0xD5,0xD2,0xFF,0xF8,0xF1,0xF6,0xE3,0xE4,0xED,0xEA,
0xB7,0xB0,0xB9,0xBE,0xAB,0xAC,0xA5,0xA2,0x8F,0x88,0x81,0x86,0x93,0x94,0x9D,0x9A,0x27,0x20,0x29,0x2E,
0x3B,0x3C,0x35,0x32,0x1F,0x18,0x11,0x16,0x03,0x04,0x0D,0x0A,0x57,0x50,0x59,0x5E,0x4B,0x4C,0x45,0x42,
0x6F,0x68,0x61,0x66,0x73,0x74,0x7D,0x7A,0x89,0x8E,0x87,0x80,0x95,0x92,0x9B,0x9C,0xB1,0xB6,0xBF,0xB8,
0xAD,0xAA,0xA3,0xA4,0xF9,0xFE,0xF7,0xF0,0xE5,0xE2,0xEB,0xEC,0xC1,0xC6,0xCF,0xC8,0xDD,0xDA,0xD3,0xD4,
0x69,0x6E,0x67,0x60,0x75,0x72,0x7B,0x7C,0x51,0x56,0x5F,0x58,0x4D,0x4A,0x43,0x44,0x19,0x1E,0x17,0x10,
0x05,0x02,0x0B,0x0C,0x21,0x26,0x2F,0x28,0x3D,0x3A,0x33,0x34,0x4E,0x49,0x40,0x47,0x52,0x55,0x5C,0x5B,
0x76,0x71,0x78,0x7F,0x6A,0x6D,0x64,0x63,0x3E,0x39,0x30,0x37,0x22,0x25,0x2C,0x2B,0x06,0x01,0x08,0x0F,
0x1A,0x1D,0x14,0x13,0xAE,0xA9,0xA0,0xA7,0xB2,0xB5,0xBC,0xBB,0x96,0x91,0x98,0x9F,0x8A,0x8D,0x84,0x83,
0xDE,0xD9,0xD0,0xD7,0xC2,0xC5,0xCC,0xCB,0xE6,0xE1,0xE8,0xEF,0xFA,0xFD,0xF4,0xF3
};
/****************************************************************//**
Calculate checksum for encrypted pages
@return checksum */
static
byte
fil_page_encryption_calc_checksum(
/*==============================*/
unsigned char* buf, /*!<in: buffer where to calculate checksum */
ulint len) /*!<in: buffer length */
{
byte crc = 0;
for (ulint i = 0; i < len; i++) {
crc = crc_table[(crc ^ buf[i]) & 0xff];
}
return crc;
}
/****************************************************************//**
Recalculate checksum for encrypted pages */
static
void
do_check_sum(
/*=========*/
ulint page_size, /*!< in: page size */
ulint zip_size, /*!< in: compressed page size */
byte* buf) /*!< in: buffer */
{
ib_uint32_t checksum = 0;
if (zip_size) {
checksum = page_zip_calc_checksum(buf,zip_size,
static_cast<srv_checksum_algorithm_t>(
srv_checksum_algorithm));
mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
return;
}
switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) {
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(buf);
break;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t) buf_calc_page_new_checksum(buf);
break;
case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC;
break;
/* no default so the compiler will emit a warning if new enum
is added and not handled here */
}
mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
/* old style checksum is omitted */
}
/****************************************************************//**
For page encrypted pages encrypt the page before actual write
operation.
Note, that FIL_PAGE_TYPE_FSP_HDR and FIL_PAGE_TYPE_XDES type pages
are not encrypted!
Pages are encrypted with AES/CBC/NoPadding algorithm.
"No padding" is used to ensure, that the encrypted page does not
exceed the page size. If "no padding" is used, the input for encryption
must be of size (multiple * AES blocksize). AES Blocksize is usually 16
(bytes).
Everything in the page is encrypted except for the 38 byte FIL header.
Since the length of the payload is not a multiple of the AES blocksize,
and to ensure that every byte of the payload is encrypted, two encryption
operations are done. Each time with a block of adequate size as input.
1st block contains everything from beginning of payload bytes except for
the remainder. 2nd block is of size 64 and contains the remainder and
the last (64 - sizeof(remainder)) bytes of the encrypted 1st block.
Each encrypted page receives a new page type for PAGE_ENCRYPTION.
The original page type (2 bytes) is stored in the Checksum header of the
page (position FIL_PAGE_SPACE_OR_CHKSUM). Additionally the encryption
key identifier is stored in the Checksum Header. This uses 1 byte.
Checksum verification for encrypted pages is disabled. This checksum
should be restored after decryption.
To be able to verify decryption in a later stage, a 1-byte checksum at
position 4 of the FIL_PAGE_SPACE_OR_CHKSUM header is stored.
For page compressed table pages the log base 2 of the length of the
encrypted data is stored.
@return encrypted page or original page if encryption failed to be
written*/
UNIV_INTERN
byte*
fil_encrypt_page(
/*==============*/
ulint space_id, /*!< in: tablespace id of the table. */
byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */
byte* out_buf, /*!< out: encrypted buffer */
ulint len, /*!< in: length of input buffer.*/
ulint encryption_key, /*!< in: encryption key */
ulint* out_len, /*!< out: actual length of encrypted page */
ulint* errorCode, /*!< out: an error code. set,
if page is intentionally not encrypted */
byte* tmp_encryption_buf) /*!< in: temporary buffer or NULL */
{
int err = AES_OK;
int key = 0;
uint32 data_size = 0;
ulint orig_page_type = 0;
uint32 write_size = 0;
fil_space_t* space = NULL;
byte* tmp_buf = NULL;
ulint page_len = 0;
ulint offset = 0;
ut_ad(buf);ut_ad(out_buf);
key = encryption_key;
*errorCode = AES_OK;
ut_ad(fil_space_is_page_encrypted(space_id));
fil_system_enter();
space = fil_space_get_by_id(space_id);
fil_system_exit();
#ifdef UNIV_DEBUG_PAGEENCRYPTION
ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET);
fprintf(stderr,
"InnoDB: Note: Preparing for encryption for space %lu name %s len %lu, page no %lu\n",
space_id, fil_space_name(space), len, pageno);
#endif /* UNIV_DEBUG_PAGEENCRYPTION */
/* read original page type */
orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE);
/* Do not encrypt file space header or extend descriptor */
if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR)
|| (orig_page_type == FIL_PAGE_TYPE_XDES) ) {
*errorCode = PAGE_ENCRYPTION_WILL_NOT_ENCRYPT;
*out_len = len;
return (buf);
}
if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) {
page_len = log10((double)len)/log10((double)2);
}
byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA);
/* data_size bytes will be encrypted at first.
* data_size will be the length of the cipher text since no padding is used.*/
data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE;
const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9,
0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
uint8 key_len = 16;
const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed,
0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e};
ulint iv_len = 16;
KeySingleton& keys = KeySingleton::getInstance();
if (!keys.isAvailable()) {
err = AES_KEY_CREATION_FAILED;
} else if (keys.getKeys(encryption_key) == NULL) {
err = PAGE_ENCRYPTION_KEY_MISSING;
} else {
char* keyString = keys.getKeys(encryption_key)->key;
char* ivString = keys.getKeys(encryption_key)->iv;
if (keyString == NULL || ivString == NULL) {
*errorCode = PAGE_ENCRYPTION_WILL_NOT_ENCRYPT;
*out_len = len;
return (buf);
}
key_len = strlen(keyString)/2;
my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len);
my_aes_hexToUint(ivString, (unsigned char*)&iv, 16);
}
/* 1st encryption: data_size bytes starting from FIL_PAGE_DATA */
if (err == AES_OK) {
err = my_aes_encrypt_cbc(
(char*) buf + FIL_PAGE_DATA,
data_size,
(char *) out_buf + FIL_PAGE_DATA,
&write_size,
(const unsigned char *) &rkey,
key_len,
(const unsigned char *) &iv,
iv_len,
1);
ut_ad(write_size == data_size);
if (err == AES_OK) {
/* copy remaining bytes from input buffer to output buffer.
* Note, that this copies the final 8 bytes of a
* page, which consists of the
* Old-style checksum and the "Low 32 bits of LSN */
memcpy(out_buf + FIL_PAGE_DATA + data_size,
buf + FIL_PAGE_DATA + data_size ,
len - FIL_PAGE_DATA -data_size);
if (tmp_encryption_buf == NULL) {
//create temporary buffer for 2nd encryption
tmp_buf = static_cast<byte *>(ut_malloc(64));
} else {
tmp_buf = tmp_encryption_buf;
}
/* 2nd encryption: 64 bytes from out_buf,
result length is 64 bytes */
err = my_aes_encrypt_cbc((char*)out_buf + len -offset -64,
64,
(char*)tmp_buf,
&write_size,
(const unsigned char *)&rkey,
key_len,
(const unsigned char *)&iv,
iv_len, 1);
ut_ad(write_size == 64);
/* copy 64 bytes from 2nd encryption to out_buf*/
memcpy(out_buf + len - offset -64, tmp_buf, 64);
}
}
/* error handling */
if (err != AES_OK) {
/* If an error occurred we leave the actual page as it was */
fprintf(stderr,
"InnoDB: Warning: Encryption failed for space %lu "
"name %s len %lu rt %d write %lu, error: %d\n",
space_id, fil_space_name(space), len, err, (ulint)data_size, err);
fflush(stderr);
srv_stats.pages_page_encryption_error.inc();
*out_len = len;
/* free temporary buffer */
if (tmp_buf!=NULL && tmp_encryption_buf == NULL) {
ut_free(tmp_buf);
}
*errorCode = err;
return (buf);
}
/* Set up the page header. Copied from input buffer*/
memcpy(out_buf, buf, FIL_PAGE_DATA);
/* Set up the correct page type */
mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED);
/* The 1st checksum field is used to store original page type, etc.
* checksum check for page encrypted pages is omitted.
*/
/* Set up the encryption key. Written to the 1st byte of
the checksum header field. This header is currently used to store data. */
mach_write_to_1(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, key);
/* store original page type. Written to 2nd and 3rd byte
of the checksum header field */
mach_write_to_2(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 1, orig_page_type);
if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) {
/* set byte 4 of checksum field to page length (ln(len)) */
memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, page_len, 1);
} else {
/* set byte 4 of checksum field to checksum byte */
memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, checksum_byte, 1);
}
#ifdef UNIV_DEBUG
/* Verify */
ut_ad(fil_page_is_encrypted(out_buf));
#endif /* UNIV_DEBUG */
srv_stats.pages_page_encrypted.inc();
*out_len = len;
/* free temporary buffer */
if (tmp_buf!=NULL && tmp_encryption_buf == NULL) {
ut_free(tmp_buf);
}
return (out_buf);
}
/****************************************************************//**
For page encrypted pages decrypt the page after actual read
operation.
See fil_encrypt_page for details, how the encryption works.
If the decryption can be verified, original page should be completely restored.
This includes original page type, 4-byte checksum field at page start.
If it is not a page compressed table's page, decryption is verified against
a 1-byte checksum built over the plain data bytes. If this verification
fails, an error state is returned.
@return decrypted page */
ulint
fil_decrypt_page(
/*=============*/
byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< in/out: buffer from which to read; in aio
this must be appropriately aligned */
ulint len, /*!< in: length buffer, which should be decrypted.*/
ulint* write_size, /*!< out: size of the decrypted
data. If no error occurred equal to len */
ibool* page_compressed,/*!<out: is page compressed.*/
byte* tmp_encryption_buf) /*!< in: temporary buffer or NULL */
{
int err = AES_OK;
ulint page_decryption_key;
uint32 data_size = 0;
ulint orig_page_type = 0;
uint32 tmp_write_size = 0;
ulint offset = 0;
byte *in_buf = NULL;
byte *tmp_buf = NULL;
fil_space_t* space = NULL;
ulint page_compression_flag = 0;
ut_ad(buf);
ut_ad(len);
/* Before actual decrypt, make sure that page type is correct */
ulint current_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE);
if ((current_page_type == FIL_PAGE_TYPE_FSP_HDR)
|| (current_page_type == FIL_PAGE_TYPE_XDES)) {
/* assumed as unencrypted */
if (write_size!=NULL) {
*write_size = len;
}
return AES_OK;
}
if (current_page_type != FIL_PAGE_PAGE_ENCRYPTED) {
fprintf(stderr, "InnoDB: Corruption: We try to decrypt corrupted page\n"
"InnoDB: CRC %lu type %lu.\n"
"InnoDB: len %lu\n",
mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM),
mach_read_from_2(buf + FIL_PAGE_TYPE), len);
fflush(stderr);
return PAGE_ENCRYPTION_WRONG_PAGE_TYPE;
}
/* 1st checksum field is used to store original page type, etc.
* checksum check for page encrypted pages is omitted.
*/
/* read page encryption key */
page_decryption_key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM);
/* Get the page type */
orig_page_type = mach_read_from_2(buf + FIL_PAGE_SPACE_OR_CHKSUM + 1);
/* read checksum byte */
byte stored_checksum_byte = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM + 3);
if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) {
if (page_compressed != NULL) {
*page_compressed = 1L;
}
page_compression_flag = 1;
len = pow((double)2, (double)((int)stored_checksum_byte));
offset = 0;
}
data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE;
const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9,
0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23,
0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
uint8 key_len = 16;
const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed,
0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e};
uint8 iv_len = 16;
KeySingleton& keys = KeySingleton::getInstance();
if (!keys.isAvailable()) {
err = PAGE_ENCRYPTION_ERROR;
} else if (keys.getKeys(page_decryption_key) == NULL) {
err = PAGE_ENCRYPTION_KEY_MISSING;
} else {
char* keyString = keys.getKeys(page_decryption_key)->key;
char* ivString = keys.getKeys(page_decryption_key)->iv;
key_len = strlen(keyString)/2;
if (keyString == NULL || ivString == NULL) {
return err;
}
my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len);
my_aes_hexToUint(ivString, (unsigned char*)&iv, 16);
}
if (err != AES_OK) {
/* surely key could not be determined. */
fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n"
"InnoDB: but decrypt failed with error %d, encryption key %d.\n",
err, (int)page_decryption_key);
fflush(stderr);
return err;
}
if (tmp_encryption_buf == NULL) {
tmp_buf= static_cast<byte *>(ut_malloc(64));
} else {
tmp_buf = tmp_encryption_buf;
}
// If no buffer was given, we need to allocate temporal buffer
if (page_buf == NULL) {
#ifdef UNIV_PAGECOMPRESS_DEBUG
fprintf(stderr,
"InnoDB: Note: FIL: Encryption buffer not given, allocating...\n");
#endif /* UNIV_PAGECOMPRESS_DEBUG */
in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2));
} else {
in_buf = page_buf;
}
/* 1st decryption: 64 bytes */
/* 64 bytes from data area are copied to temporary buffer.
* These are the last 64 of the (encrypted) payload */
memcpy(tmp_buf, buf + len - offset - 64, 64);
err = my_aes_decrypt_cbc(
(const char*) tmp_buf,
64,
(char *) in_buf + len - offset - 64,
&tmp_write_size,
(const unsigned char *) &rkey,
key_len,
(const unsigned char *) &iv,
iv_len,
1);
ut_ad(tmp_write_size == 64);
/* If decrypt fails it means that page is corrupted or has an unknown key */
if (err != AES_OK) {
fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n"
"InnoDB: but decrypt failed with error %d.\n"
"InnoDB: size %lu len %lu, key %d\n", err, (ulint)data_size,
len, (int)page_decryption_key);
fflush(stderr);
if (tmp_encryption_buf == NULL) {
ut_free(tmp_buf);
}
if (page_buf == NULL) {
ut_free(in_buf);
}
return err;
}
ut_ad(tmp_write_size == 64);
/* copy 1st part of payload from buf to in_buf */
/* do not override result of 1st decryption */
memcpy(in_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA);
/* Decrypt rest of the page */
err = my_aes_decrypt_cbc((char*) in_buf + FIL_PAGE_DATA,
data_size,
(char *) buf + FIL_PAGE_DATA,
&tmp_write_size,
(const unsigned char *)&rkey,
key_len,
(const unsigned char *)&iv,
iv_len,
1);
ut_ad(tmp_write_size = data_size);
/* copy remaining bytes from in_buf to buf.
*/
ulint bytes_to_copy = len - FIL_PAGE_DATA - data_size - offset;
memcpy(buf + FIL_PAGE_DATA + data_size, in_buf + FIL_PAGE_DATA + data_size, bytes_to_copy);
/* apart from header data everything is now in in_buf */
if (tmp_encryption_buf == NULL) {
ut_free(tmp_buf);
}
#ifdef UNIV_PAGEENCRIPTION_DEBUG
fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len);
fflush(stderr);
#endif
if (page_buf == NULL) {
ut_free(in_buf);
}
/* setting original page type */
mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type);
ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET);
ulint flags = 0;
ulint zip_size = 0;
/* please note, that page with number 0 is not encrypted */
if (pageno == 0 ) {
flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
} else {
ulint space_id = mach_read_from_4(buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_system_enter();
space = fil_space_get_by_id(space_id);
flags = fil_space_flags(space);
fil_system_exit();
}
if (!(page_compression_flag)) {
zip_size = fsp_flags_get_zip_size(flags);
}
if (write_size!=NULL) {
*write_size = len;
}
if (!(page_compression_flag)) {
byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA);
if (checksum_byte != stored_checksum_byte) {
err = PAGE_ENCRYPTION_WRONG_KEY;
fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n"
"InnoDB: but decryption verification failed with error %d,"
" encryption key %d.\n",
err, (int)page_decryption_key);
fflush(stderr);
return err;
}
/* calc check sums and write to the buffer, if page is not of type PAGE_COMPRESSED.
* if the decryption is verified, it is assumed that the
* original page was restored, re-calculating the original
* checksums should be ok
*/
do_check_sum(len, zip_size, buf);
} else {
/* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */
mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC);
}
srv_stats.pages_page_decrypted.inc();
return err;
}

7
storage/innobase/handler/ha_innodb.cc

@ -11282,6 +11282,13 @@ ha_innobase::check_table_options(
atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes;
if (options->page_encryption) {
if (srv_encrypt_tables) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,
HA_WRONG_CREATE_OPTION,
"InnoDB: PAGE_ENCRYPTION not available if innodb_encrypt_tables=ON");
return "INNODB_ENCRYPT_TABLES";
}
if (!use_tablespace) {
push_warning(
thd, Sql_condition::WARN_LEVEL_WARN,

15
storage/innobase/include/buf0buf.h

@ -1548,18 +1548,23 @@ struct buf_page_t{
zip.data == NULL means an active
buf_pool->watch */
ulint write_size; /* Write size is set when this
ulint write_size; /* Write size is set when this
page is first time written and then
if written again we check is TRIM
operation needed. */
unsigned key_version; /*!< key version for this block */
byte* crypt_buf; /*!< for encryption the data needs to be
unsigned key_version; /*!< key version for this block */
byte* crypt_buf; /*!< for encryption the data needs to be
copied to a separate buffer before it's
encrypted&written. this as a page can be
read while it's being flushed */
byte* crypt_buf_free; /*!< for encryption, allocated buffer
that is then alligned */
byte* crypt_buf_free; /*!< for encryption, allocated buffer
that is then alligned */
byte* comp_buf; /*!< for compression we need
temporal buffer because page
can be read while it's being flushed */
byte* comp_buf_free; /*!< for compression, allocated
buffer that is then alligned */
#ifndef UNIV_HOTBACKUP
buf_page_t* hash; /*!< node used in chaining to

9
storage/innobase/include/fil0fil.h

@ -173,6 +173,8 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2;
/* @} */
/** File page types (values of FIL_PAGE_TYPE) @{ */
#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 35631 /* page compressed +
encrypted page */
#define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< page compressed page */
#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */
#define FIL_PAGE_INDEX 17855 /*!< B-tree node */
@ -993,11 +995,12 @@ fil_io(
appropriately aligned */
void* message, /*!< in: message for aio handler if non-sync
aio used, else ignored */
ulint* write_size) /*!< in/out: Actual write size initialized
ulint* write_size, /*!< in/out: Actual write size initialized
after fist successfull trim
operation for this page and if
initialized we do not trim again if
actual page size does not decrease. */
lsn_t lsn) /* lsn of the newest modification */
__attribute__((nonnull(8)));
/**********************************************************************//**
Waits for an aio operation to complete. This function is used to write the
@ -1404,7 +1407,9 @@ fil_space_encrypt(
lsn_t lsn, /*!< in: page lsn */
const byte* src_frame,/*!< in: page frame */
ulint size, /*!< in: size of data to encrypt */
byte* dst_frame); /*!< in: where to encrypt to */
byte* dst_frame, /*!< in: where to encrypt to */
ulint page_encryption_key); /*!< in: page encryption key id if page
encrypted */
/*********************************************************************
Decrypt buffer page */

44
storage/innobase/include/fil0pageencryption.h

@ -46,16 +46,23 @@ fil_space_is_page_encrypted(
/*=========================*/
ulint id); /*!< in: space id */
/*******************************************************************//**
Find out whether the page is page encrypted
@return true if page is page encrypted, false if not */
UNIV_INLINE
ibool
fil_page_is_encrypted(
/*===================*/
/*==================*/
const byte *buf); /*!< in: page */
/*******************************************************************//**
Find out whether the page is page compressed and then encrypted
@return true if page is page compressed+encrypted, false if not */
UNIV_INLINE
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
const byte *buf); /*!< in: page */
/*******************************************************************//**
Find out whether the page can be decrypted
@ -66,37 +73,4 @@ fil_page_can_not_decrypt(
/*===================*/
const byte *buf); /*!< in: page */
/****************************************************************//**
For page encrypted pages encrypt the page before actual write
operation.
@return encrypted page to be written*/
byte*
fil_encrypt_page(
/*==============*/
ulint space_id, /*!< in: tablespace id of the table. */
byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */
byte* out_buf, /*!< out: encrypted buffer */
ulint len, /*!< in: length of input buffer.*/
ulint encryption_key, /*!< in: encryption key */
ulint* out_len, /*!< out: actual length of encrypted page */
ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */
byte* tmp_encryption_buf); /*!< in: temporary buffer or NULL */
/****************************************************************//**
For page encrypted pages decrypt the page after actual read
operation.
@return decrypted page */
ulint
fil_decrypt_page(
/*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< in/out: buffer from which to read; in aio
this must be appropriately aligned */
ulint len, /*!< in: length buffer, which should be decrypted.*/
ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len */
ibool* page_compressed,/*!<out: is page compressed.*/
byte* tmp_encryption_buf); /*!< in: temporary buffer or NULL */
#endif // fil0pageencryption_h

33
storage/innobase/include/fsp0pageencryption.ic

@ -28,8 +28,6 @@ Created 08/28/2014
#include "fil0pageencryption.h"
/********************************************************************//**
Determine if the tablespace is page encrypted from dict_table_t::flags.
@return TRUE if page encrypted, FALSE if not page encrypted */
@ -102,20 +100,29 @@ fil_space_get_page_encryption_key(
return(flags);
}
/*******************************************************************//**
Find out whether the page is page encrypted
Find out whether the page is page is encrypted
@return true if page is page encrypted, false if not */
UNIV_INLINE
ibool
fil_page_is_encrypted(
/*===================*/
const byte *buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED);
/*==================*/
const byte *buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED);
}
/*******************************************************************//**
Find out whether the page is page is first compressed and then encrypted
@return true if page is page compressed+encrypted, false if not */
UNIV_INLINE
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
const byte *buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
}
/*******************************************************************//**
Find out whether the page can be decrypted.
@ -132,10 +139,11 @@ The function for decrypting the page should already be executed before this.
UNIV_INLINE
ulint
fil_page_can_not_decrypt(
/*===================*/
/*=====================*/
const byte *buf) /*!< in: page */
{
ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
if (page_type == FIL_PAGE_TYPE_FSP_HDR) {
ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
if (fsp_flags_is_page_encrypted(flags)) {
@ -150,8 +158,9 @@ fil_page_can_not_decrypt(
}
}
}
if(page_type == FIL_PAGE_PAGE_ENCRYPTED) {
ulint key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM);
if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
if (KeySingleton::getInstance().isAvailable() &&
!KeySingleton::getInstance().hasKey(key)) {
return PAGE_ENCRYPTION_KEY_MISSING;

2
storage/innobase/include/log0log.h

@ -42,8 +42,6 @@ Created 12/9/1995 Heikki Tuuri
#endif /* !UNIV_HOTBACKUP */
#include "log0crypt.h"
/* Type used for all log sequence number storage and arithmetics */
typedef ib_uint64_t lsn_t;
#define LSN_MAX IB_UINT64_MAX
#define LSN_PF UINT64PF

13
storage/innobase/include/os0file.h

@ -314,11 +314,12 @@ The wrapper functions have the prefix of "innodb_". */
# define os_aio(type, mode, name, file, buf, offset, \
n, message1, message2, write_size, \
page_compression, page_compression_level, \
page_encryption, page_encryption_key) \
page_encryption, page_encryption_key, lsn) \
pfs_os_aio_func(type, mode, name, file, buf, offset, \
n, message1, message2, write_size, \
page_compression, page_compression_level, \
page_encryption, page_encryption_key, __FILE__, __LINE__)
page_encryption, page_encryption_key, \
lsn, __FILE__, __LINE__)
# define os_file_read(file, buf, offset, n, compressed) \
@ -362,11 +363,11 @@ to original un-instrumented file I/O APIs */
# define os_aio(type, mode, name, file, buf, offset, n, message1, \
message2, write_size, page_compression, page_compression_level, \
page_encryption, page_encryption_key) \
page_encryption, page_encryption_key, lsn) \
os_aio_func(type, mode, name, file, buf, offset, n, \
message1, message2, write_size, \
page_compression, page_compression_level, \
page_encryption, page_encryption_key)
page_encryption, page_encryption_key, lsn)
# define os_file_read(file, buf, offset, n, compressed) \
os_file_read_func(file, buf, offset, n, compressed)
@ -788,6 +789,7 @@ pfs_os_aio_func(
on this file space */
ulint page_encryption_key, /*!< page encryption
key to be used */
lsn_t lsn, /* lsn of the newest modification */
const char* src_file,/*!< in: file name where func invoked */
ulint src_line);/*!< in: line where the func invoked */
/*******************************************************************//**
@ -1168,8 +1170,9 @@ os_aio_func(
level to be used */
ibool page_encryption, /*!< in: is page encryption used
on this file space */
ulint page_encryption_key); /*!< page encryption key
ulint page_encryption_key, /*!< page encryption key
to be used */
lsn_t lsn); /* lsn of the newest modification */
/************************************************************************//**
Wakes up all async i/o threads so that they know to exit themselves in
shutdown. */

3
storage/innobase/include/os0file.ic

@ -228,6 +228,7 @@ pfs_os_aio_func(
on this file space */
ulint page_encryption_key, /*!< page encryption
key to be used */
lsn_t lsn, /* lsn of the newest modification */
const char* src_file,/*!< in: file name where func invoked */
ulint src_line)/*!< in: line where the func invoked */
{
@ -245,7 +246,7 @@ pfs_os_aio_func(
result = os_aio_func(type, mode, name, file, buf, offset,
n, message1, message2, write_size,
page_compression, page_compression_level,
page_encryption, page_encryption_key);
page_encryption, page_encryption_key, lsn);
register_pfs_file_io_end(locker, n);

3
storage/innobase/include/univ.i

@ -478,6 +478,9 @@ typedef uint32_t ib_uint32_t;
# define IB_ID_FMT UINT64PF
/* Type used for all log sequence number storage and arithmetics */
typedef ib_uint64_t lsn_t;
#ifdef _WIN64
typedef unsigned __int64 ulint;
typedef __int64 lint;

10
storage/innobase/log/log0log.cc

@ -1276,7 +1276,7 @@ log_group_file_header_flush(
(ulint) (dest_offset / UNIV_PAGE_SIZE),
(ulint) (dest_offset % UNIV_PAGE_SIZE),
OS_FILE_LOG_BLOCK_SIZE,
buf, group, 0);
buf, group, 0, 0);
srv_stats.os_log_pending_writes.dec();
}
@ -1443,7 +1443,7 @@ loop:
fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0,
(ulint) (next_offset / UNIV_PAGE_SIZE),
(ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf,
group, 0);
group, 0, 0);
srv_stats.os_log_pending_writes.dec();
@ -2011,7 +2011,7 @@ log_group_checkpoint(
write_offset / UNIV_PAGE_SIZE,
write_offset % UNIV_PAGE_SIZE,
OS_FILE_LOG_BLOCK_SIZE,
buf, ((byte*) group + 1), 0);
buf, ((byte*) group + 1), 0, 0);
ut_ad(((ulint) group & 0x1UL) == 0);
}
@ -2093,7 +2093,7 @@ log_group_read_checkpoint_info(
fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0,
field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE,
OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0);
OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0);
}
/******************************************************//**
@ -2417,7 +2417,7 @@ loop:
fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0,
(ulint) (source_offset / UNIV_PAGE_SIZE),
(ulint) (source_offset % UNIV_PAGE_SIZE),
len, buf, NULL, 0);
len, buf, NULL, 0, 0);
if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER &&
!log_group_decrypt_after_read(group, buf, len))

4
storage/innobase/log/log0recv.cc

@ -3101,7 +3101,7 @@ recv_recovery_from_checkpoint_start_func(
fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0,
0, 0, LOG_FILE_HDR_SIZE,
log_hdr_buf, max_cp_group, 0);
log_hdr_buf, max_cp_group, 0, 0);
if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP,
(byte*)"ibbackup", (sizeof "ibbackup") - 1)) {
@ -3132,7 +3132,7 @@ recv_recovery_from_checkpoint_start_func(
fil_io(OS_FILE_WRITE | OS_FILE_LOG, true,
max_cp_group->space_id, 0,
0, 0, OS_FILE_LOG_BLOCK_SIZE,
log_hdr_buf, max_cp_group, 0);
log_hdr_buf, max_cp_group, 0, 0);
}
#ifdef UNIV_LOG_ARCHIVE

177
storage/innobase/os/os0file.cc

@ -256,6 +256,8 @@ struct os_aio_slot_t{
/*!< TRUE if page compression was
successfull, false if not */
lsn_t lsn; /* lsn of the newest modification */
ulint file_block_size;/*!< file block size */
#ifdef WIN_ASYNC_IO
@ -2955,27 +2957,6 @@ try_again:
os_mutex_exit(os_file_count_mutex);
if (ret && len == n) {
/* If page is encrypted we need to decrypt it first */
if (fil_page_is_encrypted((byte *)buf)) {
if (fil_decrypt_page(
NULL,
(byte *)buf,
n,
NULL,
&compressed,
NULL)) {
return FALSE;
}
}
/* Note that InnoDB writes files that are not formated
as file spaces and they do not have FIL_PAGE_TYPE
field, thus we must use here information is the actual
file space compressed. */
if (fil_page_is_compressed((byte *)buf)) {
fil_decompress_page(NULL, (byte *)buf, len, NULL);
}
return(TRUE);
}
#else /* __WIN__ */
@ -2988,28 +2969,6 @@ try_again:
ret = os_file_pread(file, buf, n, offset);
if ((ulint) ret == n) {
/* If page is encrypted we need to decrypt it first */
if (fil_page_is_encrypted((byte *)buf)) {
if (fil_decrypt_page(
NULL,
(byte *)buf,
n,
NULL,
&compressed,
NULL)) {
return FALSE;
}
}
/* Note that InnoDB writes files that are not formated
as file spaces and they do not have FIL_PAGE_TYPE
field, thus we must use here information is the actual
file space compressed. */
if (fil_page_is_compressed((byte *)buf)) {
fil_decompress_page(NULL, (byte *)buf, n, NULL);
}
return(TRUE);
}
@ -3126,27 +3085,6 @@ try_again:
os_mutex_exit(os_file_count_mutex);
if (ret && len == n) {
/* If page is encrypted we need to decrypt it first */
if (fil_page_is_encrypted((byte *)buf)) {
if (fil_decrypt_page(
NULL,
(byte *)buf,
n,
NULL,
&compressed,
NULL)) {
return (FALSE);
}
}
/* Note that InnoDB writes files that are not formated
as file spaces and they do not have FIL_PAGE_TYPE
field, thus we must use here information is the actual
file space compressed. */
if (fil_page_is_compressed((byte *)buf)) {
fil_decompress_page(NULL, (byte *)buf, n, NULL);
}
return(TRUE);
}
#else /* __WIN__ */
@ -3159,27 +3097,6 @@ try_again:
ret = os_file_pread(file, buf, n, offset);
if ((ulint) ret == n) {
/* If the page is encrypted we need to decrypt it first */
if (fil_page_is_encrypted((byte *)buf)) {
if (fil_decrypt_page(
NULL,
(byte *)buf,
n,
NULL,
&compressed,
NULL)) {
return (FALSE);
}
}
/* Note that InnoDB writes files that are not formated
as file spaces and they do not have FIL_PAGE_TYPE
field, thus we must use here information is the actual
file space compressed. */
if (fil_page_is_compressed((byte *)buf)) {
fil_decompress_page(NULL, (byte *)buf, n, NULL);
}
return(TRUE);
}
#endif /* __WIN__ */
@ -4634,8 +4551,9 @@ os_aio_array_reserve_slot(
level to be used */
ibool page_encryption, /*!< in: is page encryption used
on this file space */
ulint page_encryption_key) /*!< page encryption key
ulint page_encryption_key, /*!< page encryption key
to be used */
lsn_t lsn) /* lsn of the newest modification */
{
os_aio_slot_t* slot = NULL;
#ifdef WIN_ASYNC_IO
@ -4724,6 +4642,7 @@ found:
slot->type = type;
slot->buf = static_cast<byte*>(buf);
slot->offset = offset;
slot->lsn = lsn;
slot->io_already_done = FALSE;
slot->page_compression_success = FALSE;
slot->page_encryption_success = FALSE;
@ -4755,7 +4674,7 @@ found:
os_slot_alloc_lzo_mem(slot);
}
#endif
fprintf(stderr, "JAN: compress page\n");
/* Call page compression */
tmp = fil_compress_page(
fil_node_get_space_id(slot->message1),
@ -4784,42 +4703,32 @@ found:
}
if (srv_encrypt_tables) {
page_encryption = TRUE;
}
/* If the space is page encryption and this is write operation
then we encrypt the page */
if (message1 && type == OS_FILE_WRITE && page_encryption ) {
ulint real_len = len;
ulint ec = 0;
byte* tmp = NULL;
/* Release the array mutex while encrypting */
os_mutex_exit(array->mutex);
// We allocate memory for page encrypted buffer if and only
// if it is not yet allocated.
os_slot_alloc_page_buf2(slot);
os_slot_alloc_tmp_encryption_buf(slot);
tmp = fil_encrypt_page(
fprintf(stderr, "JAN: encrypt page\n");
fil_space_encrypt(
fil_node_get_space_id(slot->message1),
slot->offset,
slot->lsn,
(byte *)buf,
slot->len,
slot->page_buf2,
len,
page_encryption_key,
&real_len,
&ec,
slot->tmp_encryption_buf);
slot->page_encryption_key);
/* If encryption succeeded, set up the length and buffer */
if (tmp != buf) {
len = real_len;
buf = slot->page_buf2;
slot->len = real_len;
slot->page_encryption_success = TRUE;
} else {
/* Use original not encrypted page */
slot->page_encryption_success = FALSE;
buf = slot->buf;
}
slot->page_encryption_success = TRUE;
buf = slot->page_buf2;
/* Take array mutex back */
os_mutex_enter(array->mutex);
@ -5113,8 +5022,9 @@ os_aio_func(
level to be used */
ibool page_encryption, /*!< in: is page encryption used
on this file space */
ulint page_encryption_key) /*!< page encryption key
ulint page_encryption_key, /*!< page encryption key
to be used */
lsn_t lsn) /* lsn of the newest modification */
{
os_aio_array_t* array;
os_aio_slot_t* slot;
@ -5232,7 +5142,7 @@ try_again:
slot = os_aio_array_reserve_slot(type, array, message1, message2, file,
name, buf, offset, n, write_size,
page_compression, page_compression_level,
page_encryption, page_encryption_key);
page_encryption, page_encryption_key, lsn);
if (type == OS_FILE_READ) {
if (srv_use_native_aio) {
@ -5525,16 +5435,21 @@ os_aio_windows_handle(
}
if (slot->type == OS_FILE_READ) {
if (fil_page_is_encrypted(slot->buf)) {
if (fil_page_is_compressed_encrypted(slot->buf) ||
fil_page_is_encrypted(slot->buf)) {
ut_ad(slot->message1 != NULL);
os_slot_alloc_page_buf2(slot);
os_slot_alloc_tmp_encryption_buf(slot);
fil_decrypt_page(
slot->page_buf2,
fprintf(stderr, "JAN: decrypt data 1\n");
// Decrypt the data
fil_space_decrypt(
fil_node_get_space_id(slot->message1),
slot->buf,
slot->len,
slot->write_size,
NULL,
slot->tmp_encryption_buf);
slot->page_buf2);
// Copy decrypted buffer back to buf
memcpy(slot->buf, slot->page_buf2, slot->len);
}
if (fil_page_is_compressed(slot->buf)) {
@ -5547,6 +5462,7 @@ os_aio_windows_handle(
os_slot_alloc_lzo_mem(slot);
}
#endif
fprintf(stderr, "JAN: decompress data\n");
fil_decompress_page(
slot->page_buf,
slot->buf,
@ -5555,7 +5471,9 @@ os_aio_windows_handle(
}
} else {
// OS_FILE_WRITE
if (slot->page_compression_success && fil_page_is_compressed(slot->page_buf)) {
if (slot->page_compression_success &&
(fil_page_is_compressed(slot->page_buf) ||
fil_page_is_compressed_encrypted(slot->buf))) {
if (srv_use_trim && os_fallocate_failed == FALSE) {
// Deallocate unused blocks from file system
os_file_trim(slot);
@ -5654,16 +5572,21 @@ retry:
if (slot->type == OS_FILE_READ) {
/* If the page is page encrypted we encrypt */
if (fil_page_is_encrypted(slot->buf)) {
if (fil_page_is_compressed_encrypted(slot->buf) ||
fil_page_is_encrypted(slot->buf)) {
os_slot_alloc_page_buf2(slot);
os_slot_alloc_tmp_encryption_buf(slot);
fil_decrypt_page(
slot->page_buf2,
ut_ad(slot->message1 != NULL);
fprintf(stderr, "JAN: decrypt data 2\n");
// Decrypt the data
fil_space_decrypt(
fil_node_get_space_id(slot->message1),
slot->buf,
slot->len,
slot->write_size,
NULL,
slot->tmp_encryption_buf);
slot->page_buf2);
// Copy decrypted buffer back to buf
memcpy(slot->buf, slot->page_buf2, slot->len);
}
/* If the page is page compressed and this is read,
@ -5680,6 +5603,8 @@ retry:
}
#endif
fprintf(stderr, "JAN: decompress data\n");
fil_decompress_page(
slot->page_buf,
slot->buf,
@ -5689,8 +5614,8 @@ retry:
} else {
/* OS_FILE_WRITE */
if (slot->page_compression_success &&
fil_page_is_compressed(slot->page_buf)) {
ut_ad(slot->page_compression_page);
(fil_page_is_compressed(slot->page_buf) ||
fil_page_is_compressed_encrypted(slot->buf))) {
if (srv_use_trim && os_fallocate_failed == FALSE) {
// Deallocate unused blocks from file system
os_file_trim(slot);

Loading…
Cancel
Save