Merge baker@bk-internal.mysql.com:/home/bk/mysql-5.1-arch

into  zim.(none):/home/brian/mysql/archive-format-5.1

mysql-test/r/archive.result

@@ -1,4 +1,4 @@
-drop table if exists t1,t2,t3;
+drop table if exists t1,t2,t3,t4,t5;
 CREATE TABLE t1 (
 Period smallint(4) unsigned zerofill DEFAULT '0000' NOT NULL,
 Varor_period smallint(4) unsigned DEFAULT '0' NOT NULL

mysql-test/t/archive.test

@@ -6,7 +6,7 @@
 -- source include/have_binlog_format_mixed_or_statement.inc
 
 --disable_warnings
-drop table if exists t1,t2,t3;
+drop table if exists t1,t2,t3,t4,t5;
 --enable_warnings
 
 CREATE TABLE t1 (

storage/archive/Makefile.am

@@ -30,7 +30,7 @@ LDADD =
 DEFS =                  @DEFS@
 
 noinst_HEADERS =	ha_archive.h azlib.h
-noinst_PROGRAMS	=	archive_test
+noinst_PROGRAMS	=	archive_test archive_reader
 
 EXTRA_LTLIBRARIES =	ha_archive.la
 pkglib_LTLIBRARIES =	@plugin_archive_shared_target@
@@ -55,6 +55,14 @@ archive_test_LDADD =	$(top_builddir)/mysys/libmysys.a \
 			@ZLIB_LIBS@
 archive_test_LDFLAGS = @NOINST_LDFLAGS@
 
+archive_reader_SOURCES = archive_reader.c azio.c
+archive_reader_CFLAGS =	$(AM_CFLAGS)
+archive_reader_LDADD =	$(top_builddir)/mysys/libmysys.a \
+			$(top_builddir)/dbug/libdbug.a \
+			$(top_builddir)/strings/libmystrings.a \
+			@ZLIB_LIBS@
+archive_reader_LDFLAGS = @NOINST_LDFLAGS@
+
 
 EXTRA_DIST =		CMakeLists.txt plug.in
 # Don't update the files from bitkeeper

storage/archive/archive_reader.c

@@ -0,0 +1,39 @@
+#include "azlib.h"
+#include <string.h>
+#include <assert.h>
+#include <stdio.h>
+
+#define BUFFER_LEN 1024
+
+int main(int argc, char *argv[])
+{
+  unsigned int ret;
+  azio_stream reader_handle;
+
+  MY_INIT(argv[0]);
+
+  if (argc < 2)
+  {
+    printf("No file specified. \n");
+    return 0;
+  }
+
+  if (!(ret= azopen(&reader_handle, argv[1], O_RDONLY|O_BINARY)))
+  {
+    printf("Could not create test file\n");
+    return 0;
+  }
+
+  printf("Version :%u\n", reader_handle.version);
+  printf("Start position :%llu\n", (unsigned long long)reader_handle.start);
+  printf("Block size :%u\n", reader_handle.block_size);
+  printf("Rows: %llu\n", reader_handle.rows);
+  printf("Autoincrement: %llu\n", reader_handle.auto_increment);
+  printf("Check Point: %llu\n", reader_handle.check_point);
+  printf("Forced Flushes: %llu\n", reader_handle.forced_flushes);
+  printf("State: %s\n", ( reader_handle.dirty ? "dirty" : "clean"));
+
+  azclose(&reader_handle);
+
+  return 0;
+}

storage/archive/archive_test.c

@@ -14,50 +14,223 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA */
 
 #include "azlib.h"
+#include <string.h>
+#include <assert.h>
 #include <stdio.h>
 
-#define TEST_STRING "This is a test"
+#define TEST_FILENAME "test.az"
+#define TEST_STRING "YOU don't know about me without you have read a book by the name of The Adventures of Tom Sawyer; but that ain't no matter.  That book was made by Mr. Mark Twain, and he told the truth, mainly.  There was things which he stretched, but mainly he told the truth.  That is nothing.  I never seen anybody but lied one time or another, without it was Aunt Polly, or the widow, or maybe Mary.  Aunt Polly--Tom's Aunt Polly, she is--and Mary, and the Widow Douglas is all told about in that book, which is mostly a true book, with some stretchers, as I said before.  Now the way that the book winds up is this:  Tom and me found the money that the robbers hid in the cave, and it made us rich.  We got six thousand dollars apiece--all gold.  It was an awful sight of money when it was piled up.  Well, Judge Thatcher he took it and put it out at interest, and it fetched us a dollar a day apiece all the year round --more than a body could tell what to do with.  The Widow Douglas she took me for her son, and allowed she would..."
+#define TEST_LOOP_NUM 100
 #define BUFFER_LEN 1024
+#define TWOGIG 2147483648
+#define FOURGIG 4294967296
+#define EIGHTGIG 8589934592
 
-int main(int argc __attribute__((unused)), char *argv[])
+/* prototypes */
+int size_test(unsigned long long length, unsigned long long rows_to_test_for);
+
+
+int main(int argc, char *argv[])
 {
-  int ret;
-  azio_stream foo, foo1;
+  unsigned int ret;
+
+  int error;
+  unsigned int x;
+  int written_rows= 0;
+  azio_stream writer_handle, reader_handle;
   char buffer[BUFFER_LEN];
 
+  unlink(TEST_FILENAME);
+
+  if (argc > 1)
+    return 0;
+
   MY_INIT(argv[0]);
 
-  if (!(ret= azopen(&foo, "test", O_CREAT|O_WRONLY|O_TRUNC|O_BINARY)))
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_CREAT|O_RDWR|O_BINARY)))
   {
     printf("Could not create test file\n");
     return 0;
   }
-  azwrite(&foo, TEST_STRING, sizeof(TEST_STRING));
-  azflush(&foo, Z_FINISH);
 
-  if (!(ret= azopen(&foo1, "test", O_RDONLY|O_BINARY)))
+  if (!(ret= azopen(&reader_handle, TEST_FILENAME, O_RDONLY|O_BINARY)))
+  {
+    printf("Could not open test file\n");
+    return 0;
+  }
+
+  assert(reader_handle.rows == 0);
+  assert(reader_handle.auto_increment == 0);
+  assert(reader_handle.check_point == 0);
+  assert(reader_handle.forced_flushes == 0);
+  assert(reader_handle.dirty == 1);
+
+  for (x= 0; x < TEST_LOOP_NUM; x++)
+  {
+    ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+    assert(ret == BUFFER_LEN);
+    written_rows++;
+  }
+  azflush(&writer_handle,  Z_SYNC_FLUSH);
+
+  /* Lets test that our internal stats are good */
+  assert(writer_handle.rows == TEST_LOOP_NUM);
+
+  /* Reader needs to be flushed to make sure it is up to date */
+  azflush(&reader_handle,  Z_SYNC_FLUSH);
+  assert(reader_handle.rows == TEST_LOOP_NUM);
+  assert(reader_handle.auto_increment == 0);
+  assert(reader_handle.check_point == 0);
+  assert(reader_handle.forced_flushes == 1);
+  assert(reader_handle.dirty == 1);
+
+  writer_handle.auto_increment= 4;
+  azflush(&writer_handle, Z_SYNC_FLUSH);
+  assert(writer_handle.rows == TEST_LOOP_NUM);
+  assert(writer_handle.auto_increment == 4);
+  assert(writer_handle.check_point == 0);
+  assert(writer_handle.forced_flushes == 2);
+  assert(writer_handle.dirty == 1);
+
+  if (!(ret= azopen(&reader_handle, TEST_FILENAME, O_RDONLY|O_BINARY)))
   {
     printf("Could not open test file\n");
     return 0;
   }
-  ret= azread(&foo1, buffer, BUFFER_LEN);
-  printf("Read %d bytes\n", ret);
-  printf("%s\n", buffer);
-  azrewind(&foo1);
-  azclose(&foo);
-  if (!(ret= azopen(&foo, "test", O_APPEND|O_WRONLY|O_BINARY)))
+
+  /* Read the original data */
+  for (x= 0; x < writer_handle.rows; x++)
+  {
+    ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+    assert(!error);
+    assert(ret == BUFFER_LEN);
+    assert(!memcmp(buffer, TEST_STRING, ret));
+  }
+  assert(writer_handle.rows == TEST_LOOP_NUM);
+
+  /* Test here for falling off the planet */
+
+  /* Final Write before closing */
+  ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+
+  /* We don't use FINISH, but I want to have it tested */
+  azflush(&writer_handle,  Z_FINISH);
+
+  assert(writer_handle.rows == TEST_LOOP_NUM+1);
+
+  /* Read final write */
+  azrewind(&reader_handle);
+  for (x= 0; x < writer_handle.rows; x++)
+  {
+    ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+    assert(ret == BUFFER_LEN);
+    assert(!error);
+    assert(!memcmp(buffer, TEST_STRING, ret));
+  }
+
+
+  azclose(&writer_handle);
+
+  /* Rewind and full test */
+  azrewind(&reader_handle);
+  for (x= 0; x < writer_handle.rows; x++)
+  {
+    ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+    assert(ret == BUFFER_LEN);
+    assert(!error);
+    assert(!memcmp(buffer, TEST_STRING, ret));
+  }
+
+  printf("Finished reading\n");
+
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_RDWR|O_BINARY)))
+  {
+    printf("Could not open file (%s) for appending\n", TEST_FILENAME);
+    return 0;
+  }
+  ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+  assert(ret == BUFFER_LEN);
+  azflush(&writer_handle,  Z_SYNC_FLUSH);
+
+  /* Rewind and full test */
+  azrewind(&reader_handle);
+  for (x= 0; x < writer_handle.rows; x++)
+  {
+    ret= azread(&reader_handle, buffer, BUFFER_LEN, &error);
+    assert(!error);
+    assert(ret == BUFFER_LEN);
+    assert(!memcmp(buffer, TEST_STRING, ret));
+  }
+
+  azclose(&writer_handle);
+  azclose(&reader_handle);
+  unlink(TEST_FILENAME);
+
+  /* Start size tests */
+  printf("About to run 2/4/8 gig tests now, you may want to hit CTRL-C\n");
+  size_test(TWOGIG, 2097152);
+  size_test(FOURGIG, 4194304);
+  size_test(EIGHTGIG, 8388608);
+
+  return 0;
+}
+
+int size_test(unsigned long long length, unsigned long long rows_to_test_for)
+{
+  azio_stream writer_handle, reader_handle;
+  unsigned long long write_length;
+  unsigned long long read_length= 0;
+  unsigned int ret;
+  char buffer[BUFFER_LEN];
+  int error;
+
+  if (!(ret= azopen(&writer_handle, TEST_FILENAME, O_CREAT|O_RDWR|O_TRUNC|O_BINARY)))
   {
     printf("Could not create test file\n");
     return 0;
   }
-  azwrite(&foo, TEST_STRING, sizeof(TEST_STRING));
-  azflush(&foo, Z_FINISH);
-  ret= azread(&foo1, buffer, BUFFER_LEN);
-  printf("Read %d bytes\n", ret);
-  printf("%s\n", buffer);
-  azclose(&foo);
-  azclose(&foo1);
 
-  /* unlink("test"); */
+  for (write_length= 0; write_length < length ; write_length+= ret)
+  {
+    ret= azwrite(&writer_handle, TEST_STRING, BUFFER_LEN);
+    if (ret != BUFFER_LEN)
+    {
+      printf("Size %u\n", ret);
+      assert(ret != BUFFER_LEN);
+    }
+    if ((write_length % 14031) == 0)
+    {
+      azflush(&writer_handle,  Z_SYNC_FLUSH);
+    }
+  }
+  assert(write_length == length);
+  azflush(&writer_handle,  Z_SYNC_FLUSH);
+
+  printf("Reading back data\n");
+
+  if (!(ret= azopen(&reader_handle, TEST_FILENAME, O_RDONLY|O_BINARY)))
+  {
+    printf("Could not open test file\n");
+    return 0;
+  }
+
+  while ((ret= azread(&reader_handle, buffer, BUFFER_LEN, &error)))
+  {
+    read_length+= ret;
+    assert(!memcmp(buffer, TEST_STRING, ret));
+    if (ret != BUFFER_LEN)
+    {
+      printf("Size %u\n", ret);
+      assert(ret != BUFFER_LEN);
+    }
+  }
+
+  assert(read_length == length);
+  assert(writer_handle.rows == rows_to_test_for);
+  azclose(&writer_handle);
+  azclose(&reader_handle);
+  unlink(TEST_FILENAME);
+
   return 0;
 }

storage/archive/azio.c

@@ -1,12 +1,12 @@
 /*
   azio is a modified version of gzio. It  makes use of mysys and removes mallocs.
+    -Brian Aker
 */
 
 /* gzio.c -- IO on .gz files
  * Copyright (C) 1995-2005 Jean-loup Gailly.
  * For conditions of distribution and use, see copyright notice in zlib.h
  *
- * Compile this file with -DNO_GZCOMPRESS to avoid the compression code.
  */
 
 /* @(#) $Id$ */
@@ -17,6 +17,7 @@
 #include <string.h>
 
 static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
+static int const az_magic[2] = {0xfe, 0x03}; /* az magic header */
 
 /* gzip flag byte */
 #define ASCII_FLAG   0x01 /* bit 0 set: file probably ascii text */
@@ -30,9 +31,11 @@ int az_open(azio_stream *s, const char *path, int Flags, File  fd);
 int do_flush(azio_stream *file, int flush);
 int    get_byte(azio_stream *s);
 void   check_header(azio_stream *s);
+void write_header(azio_stream *s);
 int    destroy(azio_stream *s);
 void putLong(File file, uLong x);
 uLong  getLong(azio_stream *s);
+void read_header(azio_stream *s, unsigned char *buffer);
 
 /* ===========================================================================
   Opens a gzip (.gz) file for reading or writing. The mode parameter
@@ -46,14 +49,14 @@ uLong  getLong(azio_stream *s);
 int az_open (azio_stream *s, const char *path, int Flags, File fd)
 {
   int err;
-  int level = Z_DEFAULT_COMPRESSION; /* compression level */
+  int level = Z_NO_COMPRESSION; /*         Z_DEFAULT_COMPRESSION;*/ /* compression level */
   int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */
 
   s->stream.zalloc = (alloc_func)0;
   s->stream.zfree = (free_func)0;
   s->stream.opaque = (voidpf)0;
-  memset(s->inbuf, 0, Z_BUFSIZE);
-  memset(s->outbuf, 0, Z_BUFSIZE);
+  memset(s->inbuf, 0, AZ_BUFSIZE);
+  memset(s->outbuf, 0, AZ_BUFSIZE);
   s->stream.next_in = s->inbuf;
   s->stream.next_out = s->outbuf;
   s->stream.avail_in = s->stream.avail_out = 0;
@@ -65,20 +68,25 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd)
   s->crc = crc32(0L, Z_NULL, 0);
   s->transparent = 0;
   s->mode = 'r';
+  s->version = (unsigned char)az_magic[1]; /* this needs to be a define to version */
 
-  if (Flags & O_WRONLY || Flags & O_APPEND) 
+  /*
+    We do our own version of append by nature. 
+    We must always have write access to take card of the header.
+  */
+  DBUG_ASSERT(Flags | O_APPEND);
+  DBUG_ASSERT(Flags | O_WRONLY);
+
+  if (Flags & O_RDWR) 
     s->mode = 'w';
 
-  if (s->mode == 'w') {
-#ifdef NO_GZCOMPRESS
-    err = Z_STREAM_ERROR;
-#else
+  if (s->mode == 'w') 
+  {
     err = deflateInit2(&(s->stream), level,
                        Z_DEFLATED, -MAX_WBITS, 8, strategy);
     /* windowBits is passed < 0 to suppress zlib header */
 
     s->stream.next_out = s->outbuf;
-#endif
     if (err != Z_OK)
     {
       destroy(s);
@@ -100,7 +108,7 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd)
       return Z_NULL;
     }
   }
-  s->stream.avail_out = Z_BUFSIZE;
+  s->stream.avail_out = AZ_BUFSIZE;
 
   errno = 0;
   s->file = fd < 0 ? my_open(path, Flags, MYF(0)) : fd;
@@ -110,35 +118,64 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd)
     destroy(s);
     return Z_NULL;
   }
-  if (s->mode == 'w') {
-    char buffer[10];
-    /* Write a very simple .gz header:
-  */
-    buffer[0] = gz_magic[0];
-    buffer[1] = gz_magic[1];
-    buffer[2] = Z_DEFLATED;
-    buffer[3] = 0 /*flags*/;
-    buffer[4] = 0;
-    buffer[5] = 0;
-    buffer[6] = 0;
-    buffer[7] = 0 /*time*/;
-    buffer[8] = 0 /*xflags*/;
-    buffer[9] = 0x03;
-    s->start = 10L;
-    my_write(s->file, buffer, (uint)s->start, MYF(0));
-    /* We use 10L instead of ftell(s->file) to because ftell causes an
-     * fflush on some systems. This version of the library doesn't use
-     * start anyway in write mode, so this initialization is not
-     * necessary.
-   */
-  } else {
-    check_header(s); /* skip the .gz header */
-    s->start = my_tell(s->file, MYF(0)) - s->stream.avail_in;
+
+  if (Flags & O_CREAT || Flags & O_TRUNC) 
+  {
+    s->rows= 0;
+    s->forced_flushes= 0;
+    s->auto_increment= 0;
+    s->check_point= 0;
+    s->dirty= 1; /* We create the file dirty */
+    write_header(s);
+    my_seek(s->file, 0, MY_SEEK_END, MYF(0));
+  }
+  else if (s->mode == 'w') 
+  {
+    unsigned char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE];
+    my_pread(s->file, buffer, AZHEADER_SIZE + AZMETA_BUFFER_SIZE, 0, MYF(0));
+    read_header(s, buffer); /* skip the .az header */
+    my_seek(s->file, 0, MY_SEEK_END, MYF(0));
+  }
+  else
+  {
+    check_header(s); /* skip the .az header */
   }
 
   return 1;
 }
 
+
+void write_header(azio_stream *s)
+{
+  char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE];
+  char *ptr= buffer;
+
+  s->start = AZHEADER_SIZE + AZMETA_BUFFER_SIZE;
+  s->block_size= AZ_BUFSIZE;
+  s->version = (unsigned char)az_magic[1];
+
+
+  /* Write a very simple .az header: */
+  bzero(buffer, AZHEADER_SIZE + AZMETA_BUFFER_SIZE);
+  *(ptr + AZ_MAGIC_POS)= az_magic[0];
+  *(ptr + AZ_VERSION_POS)= (unsigned char)s->version;
+  *(ptr + AZ_BLOCK_POS)= (unsigned char)(s->block_size/1024); /* Reserved for block size */
+  *(ptr + AZ_STRATEGY_POS)= (unsigned char)Z_DEFAULT_STRATEGY; /* Compression Type */
+
+  int4store(ptr + AZ_FRM_POS, 0); /* FRM Block */
+  int4store(ptr + AZ_META_POS, 0); /* Meta Block */
+  int8store(ptr + AZ_START_POS, (unsigned long long)s->start); /* Start of Data Block Index Block */
+  printf("ROWS %llu\n", s->rows);
+  int8store(ptr + AZ_ROW_POS, (unsigned long long)s->rows); /* Start of Data Block Index Block */
+  int8store(ptr + AZ_FLUSH_POS, (unsigned long long)s->forced_flushes); /* Start of Data Block Index Block */
+  int8store(ptr + AZ_CHECK_POS, (unsigned long long)s->check_point); /* Start of Data Block Index Block */
+  int8store(ptr + AZ_AUTOINCREMENT_POS, (unsigned long long)s->auto_increment); /* Start of Data Block Index Block */
+  *(ptr + AZ_DIRTY_POS)= (unsigned char)s->dirty; /* Start of Data Block Index Block */
+
+  /* Always begin at the begining, and end there as well */
+  my_pwrite(s->file, buffer, (uint)s->start, 0, MYF(0));
+}
+
 /* ===========================================================================
   Opens a gzip (.gz) file for reading or writing.
 */
@@ -170,7 +207,7 @@ int get_byte(s)
   if (s->stream.avail_in == 0) 
   {
     errno = 0;
-    s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, Z_BUFSIZE, MYF(0));
+    s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0));
     if (s->stream.avail_in == 0) 
     {
       s->z_eof = 1;
@@ -206,7 +243,7 @@ void check_header(azio_stream *s)
   if (len < 2) {
     if (len) s->inbuf[0] = s->stream.next_in[0];
     errno = 0;
-    len = (uInt)my_read(s->file, (byte *)s->inbuf + len, Z_BUFSIZE >> len, MYF(0));
+    len = (uInt)my_read(s->file, (byte *)s->inbuf + len, AZ_BUFSIZE >> len, MYF(0));
     if (len == 0) s->z_err = Z_ERRNO;
     s->stream.avail_in += len;
     s->stream.next_in = s->inbuf;
@@ -217,41 +254,76 @@ void check_header(azio_stream *s)
   }
 
   /* Peek ahead to check the gzip magic header */
-  if (s->stream.next_in[0] != gz_magic[0] ||
-      s->stream.next_in[1] != gz_magic[1]) {
-    s->transparent = 1;
-    return;
-  }
-  s->stream.avail_in -= 2;
-  s->stream.next_in += 2;
+  if ( s->stream.next_in[0] == gz_magic[0]  && s->stream.next_in[1] == gz_magic[1])
+  {
+    s->stream.avail_in -= 2;
+    s->stream.next_in += 2;
+    s->version= (unsigned char)2;
+
+    /* Check the rest of the gzip header */
+    method = get_byte(s);
+    flags = get_byte(s);
+    if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
+      s->z_err = Z_DATA_ERROR;
+      return;
+    }
 
-  /* Check the rest of the gzip header */
-  method = get_byte(s);
-  flags = get_byte(s);
-  if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
-    s->z_err = Z_DATA_ERROR;
-    return;
-  }
+    /* Discard time, xflags and OS code: */
+    for (len = 0; len < 6; len++) (void)get_byte(s);
 
-  /* Discard time, xflags and OS code: */
-  for (len = 0; len < 6; len++) (void)get_byte(s);
+    if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
+      len  =  (uInt)get_byte(s);
+      len += ((uInt)get_byte(s))<<8;
+      /* len is garbage if EOF but the loop below will quit anyway */
+      while (len-- != 0 && get_byte(s) != EOF) ;
+    }
+    if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
+      while ((c = get_byte(s)) != 0 && c != EOF) ;
+    }
+    if ((flags & COMMENT) != 0) {   /* skip the .gz file comment */
+      while ((c = get_byte(s)) != 0 && c != EOF) ;
+    }
+    if ((flags & HEAD_CRC) != 0) {  /* skip the header crc */
+      for (len = 0; len < 2; len++) (void)get_byte(s);
+    }
+    s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK;
+    s->start = my_tell(s->file, MYF(0)) - s->stream.avail_in;
+  }
+  else if ( s->stream.next_in[0] == az_magic[0]  && s->stream.next_in[1] == az_magic[1])
+  {
+    unsigned char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE];
 
-  if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
-    len  =  (uInt)get_byte(s);
-    len += ((uInt)get_byte(s))<<8;
-    /* len is garbage if EOF but the loop below will quit anyway */
-    while (len-- != 0 && get_byte(s) != EOF) ;
+    for (len = 0; len < (AZHEADER_SIZE + AZMETA_BUFFER_SIZE); len++) 
+      buffer[len]= get_byte(s);
+    s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK;
+    read_header(s, buffer);
   }
-  if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
-    while ((c = get_byte(s)) != 0 && c != EOF) ;
+  else
+  {
+    s->z_err = Z_OK;
+
+    return;
   }
-  if ((flags & COMMENT) != 0) {   /* skip the .gz file comment */
-    while ((c = get_byte(s)) != 0 && c != EOF) ;
+}
+
+void read_header(azio_stream *s, unsigned char *buffer)
+{
+  if (buffer[0] == az_magic[0]  && buffer[1] == az_magic[1])
+  {
+    s->version= (unsigned int)buffer[AZ_VERSION_POS];
+    s->block_size= 1024 * buffer[AZ_BLOCK_POS];
+    s->start= (unsigned long long)uint8korr(buffer + AZ_START_POS);
+    s->rows= (unsigned long long)uint8korr(buffer + AZ_ROW_POS);
+    s->check_point= (unsigned long long)uint8korr(buffer + AZ_CHECK_POS);
+    s->forced_flushes= (unsigned long long)uint8korr(buffer + AZ_FLUSH_POS);
+    s->auto_increment= (unsigned long long)uint8korr(buffer + AZ_AUTOINCREMENT_POS);
+    s->dirty= (unsigned int)buffer[AZ_DIRTY_POS];
   }
-  if ((flags & HEAD_CRC) != 0) {  /* skip the header crc */
-    for (len = 0; len < 2; len++) (void)get_byte(s);
+  else
+  {
+    DBUG_ASSERT(buffer[0] == az_magic[0]  && buffer[1] == az_magic[1]);
+    return;
   }
-  s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK;
 }
 
 /* ===========================================================================
@@ -265,11 +337,7 @@ int destroy (s)
 
   if (s->stream.state != NULL) {
     if (s->mode == 'w') {
-#ifdef NO_GZCOMPRESS
-      err = Z_STREAM_ERROR;
-#else
       err = deflateEnd(&(s->stream));
-#endif
     } 
     else if (s->mode == 'r') 
     {
@@ -292,15 +360,28 @@ int destroy (s)
   Reads the given number of uncompressed bytes from the compressed file.
   azread returns the number of bytes actually read (0 for end of file).
 */
-int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
+unsigned int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned int len, int *error)
 {
   Bytef *start = (Bytef*)buf; /* starting point for crc computation */
   Byte  *next_out; /* == stream.next_out but not forced far (for MSDOS) */
+  *error= 0;
+
+  if (s->mode != 'r')
+  { 
+    *error= Z_STREAM_ERROR;
+    return 0;
+  }
 
-  if (s->mode != 'r') return Z_STREAM_ERROR;
+  if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)
+  { 
+    *error= s->z_err;
+    return 0;
+  }
 
-  if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1;
-  if (s->z_err == Z_STREAM_END) return 0;  /* EOF */
+  if (s->z_err == Z_STREAM_END)  /* EOF */
+  { 
+    return 0;
+  }
 
   next_out = (Byte*)buf;
   s->stream.next_out = (Bytef*)buf;
@@ -315,7 +396,9 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
     start++;
     if (s->last) {
       s->z_err = Z_STREAM_END;
-      return 1;
+      { 
+        return 1;
+      }
     }
   }
 
@@ -342,12 +425,14 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
       s->in  += len;
       s->out += len;
       if (len == 0) s->z_eof = 1;
-      return (int)len;
+      { 
+        return len;
+      }
     }
     if (s->stream.avail_in == 0 && !s->z_eof) {
 
       errno = 0;
-      s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, Z_BUFSIZE, MYF(0));
+      s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0));
       if (s->stream.avail_in == 0) 
       {
         s->z_eof = 1;
@@ -374,7 +459,8 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
          * Check for such files:
        */
         check_header(s);
-        if (s->z_err == Z_OK) {
+        if (s->z_err == Z_OK) 
+        {
           inflateReset(&(s->stream));
           s->crc = crc32(0L, Z_NULL, 0);
         }
@@ -386,34 +472,40 @@ int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned len)
 
   if (len == s->stream.avail_out &&
       (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO))
-    return -1;
-  return (int)(len - s->stream.avail_out);
+  {
+    *error= s->z_err;
+
+    return 0;
+  }
+
+  return (len - s->stream.avail_out);
 }
 
 
-#ifndef NO_GZCOMPRESS
 /* ===========================================================================
   Writes the given number of uncompressed bytes into the compressed file.
   azwrite returns the number of bytes actually written (0 in case of error).
 */
-int azwrite (azio_stream *s, voidpc buf, unsigned len)
+unsigned int azwrite (azio_stream *s, voidpc buf, unsigned int len)
 {
-
   s->stream.next_in = (Bytef*)buf;
   s->stream.avail_in = len;
 
+
+  s->rows++;
+
   while (s->stream.avail_in != 0) 
   {
     if (s->stream.avail_out == 0) 
     {
 
       s->stream.next_out = s->outbuf;
-      if (my_write(s->file, (byte *)s->outbuf, Z_BUFSIZE, MYF(0)) != Z_BUFSIZE) 
+      if (my_write(s->file, (byte *)s->outbuf, AZ_BUFSIZE, MYF(0)) != AZ_BUFSIZE) 
       {
         s->z_err = Z_ERRNO;
         break;
       }
-      s->stream.avail_out = Z_BUFSIZE;
+      s->stream.avail_out = AZ_BUFSIZE;
     }
     s->in += s->stream.avail_in;
     s->out += s->stream.avail_out;
@@ -424,19 +516,15 @@ int azwrite (azio_stream *s, voidpc buf, unsigned len)
   }
   s->crc = crc32(s->crc, (const Bytef *)buf, len);
 
-  return (int)(len - s->stream.avail_in);
+  return (unsigned int)(len - s->stream.avail_in);
 }
 
-#endif
-
 
 /* ===========================================================================
   Flushes all pending output into the compressed file. The parameter
   flush is as in the deflate() function.
 */
-int do_flush (s, flush)
-  azio_stream *s;
-  int flush;
+int do_flush (azio_stream *s, int flush)
 {
   uInt len;
   int done = 0;
@@ -445,8 +533,9 @@ int do_flush (s, flush)
 
   s->stream.avail_in = 0; /* should be zero already anyway */
 
-  for (;;) {
-    len = Z_BUFSIZE - s->stream.avail_out;
+  for (;;) 
+  {
+    len = AZ_BUFSIZE - s->stream.avail_out;
 
     if (len != 0) {
       if ((uInt)my_write(s->file, (byte *)s->outbuf, len, MYF(0)) != len) 
@@ -455,7 +544,7 @@ int do_flush (s, flush)
         return Z_ERRNO;
       }
       s->stream.next_out = s->outbuf;
-      s->stream.avail_out = Z_BUFSIZE;
+      s->stream.avail_out = AZ_BUFSIZE;
     }
     if (done) break;
     s->out += s->stream.avail_out;
@@ -472,6 +561,11 @@ int do_flush (s, flush)
 
     if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break;
   }
+
+  if (flush == Z_FINISH)
+    s->dirty= 0; /* Mark it clean, we should be good now */
+  write_header(s);
+
   return  s->z_err == Z_STREAM_END ? Z_OK : s->z_err;
 }
 
@@ -479,11 +573,25 @@ int ZEXPORT azflush (s, flush)
   azio_stream *s;
   int flush;
 {
-  int err = do_flush (s, flush);
+  int err;
 
-  if (err) return err;
-  my_sync(s->file, MYF(0));
-  return  s->z_err == Z_STREAM_END ? Z_OK : s->z_err;
+  if (s->mode == 'r') 
+  {
+    unsigned char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE];
+    my_pread(s->file, buffer, AZHEADER_SIZE + AZMETA_BUFFER_SIZE, 0, MYF(0));
+    read_header(s, buffer); /* skip the .az header */
+
+    return Z_OK;
+  }
+  else
+  {
+    s->forced_flushes++;
+    err= do_flush(s, flush);
+
+    if (err) return err;
+    my_sync(s->file, MYF(0));
+    return  s->z_err == Z_STREAM_END ? Z_OK : s->z_err;
+  }
 }
 
 /* ===========================================================================
@@ -525,19 +633,17 @@ my_off_t azseek (s, offset, whence)
     return -1L;
   }
 
-  if (s->mode == 'w') {
-#ifdef NO_GZCOMPRESS
-    return -1L;
-#else
-    if (whence == SEEK_SET) {
+  if (s->mode == 'w') 
+  {
+    if (whence == SEEK_SET) 
       offset -= s->in;
-    }
 
     /* At this point, offset is the number of zero bytes to write. */
     /* There was a zmemzero here if inbuf was null -Brian */
-    while (offset > 0)  {
-      uInt size = Z_BUFSIZE;
-      if (offset < Z_BUFSIZE) size = (uInt)offset;
+    while (offset > 0)  
+    {
+      uInt size = AZ_BUFSIZE;
+      if (offset < AZ_BUFSIZE) size = (uInt)offset;
 
       size = azwrite(s, s->inbuf, size);
       if (size == 0) return -1L;
@@ -545,7 +651,6 @@ my_off_t azseek (s, offset, whence)
       offset -= size;
     }
     return s->in;
-#endif
   }
   /* Rest of function is for reading only */
 
@@ -580,11 +685,12 @@ my_off_t azseek (s, offset, whence)
     if (s->last) s->z_err = Z_STREAM_END;
   }
   while (offset > 0)  {
-    int size = Z_BUFSIZE;
-    if (offset < Z_BUFSIZE) size = (int)offset;
+    int error;
+    unsigned int size = AZ_BUFSIZE;
+    if (offset < AZ_BUFSIZE) size = (int)offset;
 
-    size = azread(s, s->outbuf, (uInt)size);
-    if (size <= 0) return -1L;
+    size = azread(s, s->outbuf, size, &error);
+    if (error <= 0) return -1L;
     offset -= size;
   }
   return s->out;
@@ -644,16 +750,14 @@ int azclose (azio_stream *s)
 
   if (s == NULL) return Z_STREAM_ERROR;
 
-  if (s->mode == 'w') {
-#ifdef NO_GZCOMPRESS
-    return Z_STREAM_ERROR;
-#else
+
+  if (s->mode == 'w') 
+  {
     if (do_flush (s, Z_FINISH) != Z_OK)
       return destroy(s);
 
     putLong(s->file, s->crc);
     putLong(s->file, (uLong)(s->in & 0xffffffff));
-#endif
   }
   return destroy(s);
 }

storage/archive/azlib.h

@@ -1,6 +1,8 @@
 /*
   This libary has been modified for use by the MySQL Archive Engine.
+     -Brian Aker
 */
+
 /* zlib.h -- interface of the 'zlib' general purpose compression library
   version 1.2.3, July 18th, 2005
 
@@ -34,10 +36,34 @@
 #include <zlib.h>
 
 #include "../../mysys/mysys_priv.h"
+#include <my_dir.h>
 
 #ifdef  __cplusplus
 extern "C" {
 #endif
+/* Start of MySQL Specific Information */
+
+/*
+  ulonglong + ulonglong + ulonglong + ulonglong + uchar
+*/
+#define AZMETA_BUFFER_SIZE sizeof(unsigned long long) \
+  + sizeof(unsigned long long) + sizeof(unsigned long long) + sizeof(unsigned long long) \
+  + sizeof(unsigned char)
+
+#define AZHEADER_SIZE 20
+
+#define AZ_MAGIC_POS 0
+#define AZ_VERSION_POS 1
+#define AZ_BLOCK_POS 2
+#define AZ_STRATEGY_POS 3
+#define AZ_FRM_POS 4
+#define AZ_META_POS 8
+#define AZ_START_POS 12
+#define AZ_ROW_POS 20
+#define AZ_FLUSH_POS 28
+#define AZ_CHECK_POS 36
+#define AZ_AUTOINCREMENT_POS 44
+#define AZ_DIRTY_POS 52
 
 /*
      The 'zlib' compression library provides in-memory compression and
@@ -152,7 +178,7 @@ extern "C" {
 /* The deflate compression method (the only one supported in this version) */
 
 #define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
-#define Z_BUFSIZE 16384
+#define AZ_BUFSIZE 16384
 
 
 typedef struct azio_stream {
@@ -160,8 +186,8 @@ typedef struct azio_stream {
   int      z_err;   /* error code for last stream operation */
   int      z_eof;   /* set if end of input file */
   File     file;   /* .gz file */
-  Byte     inbuf[Z_BUFSIZE];  /* input buffer */
-  Byte     outbuf[Z_BUFSIZE]; /* output buffer */
+  Byte     inbuf[AZ_BUFSIZE];  /* input buffer */
+  Byte     outbuf[AZ_BUFSIZE]; /* output buffer */
   uLong    crc;     /* crc32 of uncompressed data */
   char     *msg;    /* error message */
   int      transparent; /* 1 if input file is not a .gz file */
@@ -171,6 +197,13 @@ typedef struct azio_stream {
   my_off_t  out;     /* bytes out of deflate or inflate */
   int      back;    /* one character push-back */
   int      last;    /* true if push-back is last character */
+  unsigned char version;   /* Version */
+  unsigned int block_size;   /* Block Size */
+  unsigned long long check_point;   /* Last position we checked */
+  unsigned long long forced_flushes;   /* Forced Flushes */
+  unsigned long long rows;   /* rows */
+  unsigned long long auto_increment;   /* auto increment field */
+  unsigned char dirty;   /* State of file */
 } azio_stream;
 
                         /* basic functions */
@@ -206,7 +239,7 @@ int azdopen(azio_stream *s,File fd, int Flags);
 */
 
 
-extern int azread(azio_stream *file, voidp buf, unsigned len);
+extern unsigned int azread ( azio_stream *s, voidp buf, unsigned int len, int *error);
 /*
      Reads the given number of uncompressed bytes from the compressed file.
    If the input file was not in gzip format, gzread copies the given number
@@ -214,10 +247,10 @@ extern int azread(azio_stream *file, voidp buf, unsigned len);
      gzread returns the number of uncompressed bytes actually read (0 for
    end of file, -1 for error). */
 
-extern int azwrite (azio_stream *file, voidpc buf, unsigned len);
+extern unsigned int azwrite (azio_stream *s, voidpc buf, unsigned int len);
 /*
      Writes the given number of uncompressed bytes into the compressed file.
-   gzwrite returns the number of uncompressed bytes actually written
+   azwrite returns the number of uncompressed bytes actually written
    (0 in case of error).
 */
 

storage/archive/ha_archive.h

@@ -26,34 +26,39 @@
   ha_example.h.
 */
 
+typedef struct st_archive_record_buffer {
+  byte *buffer;
+  int length;
+} archive_record_buffer;
+
+
 typedef struct st_archive_share {
   char *table_name;
   char data_file_name[FN_REFLEN];
   uint table_name_length,use_count;
   pthread_mutex_t mutex;
   THR_LOCK lock;
-  File meta_file;           /* Meta file we use */
   azio_stream archive_write;     /* Archive file we are working with */
   bool archive_write_open;
   bool dirty;               /* Flag for if a flush should occur */
   bool crashed;             /* Meta file is crashed */
   ha_rows rows_recorded;    /* Number of rows in tables */
-  ulonglong auto_increment_value;
-  ulonglong forced_flushes;
   ulonglong mean_rec_length;
-  char real_path[FN_REFLEN];
 } ARCHIVE_SHARE;
 
 /*
   Version for file format.
-  1 - Initial Version
+  1 - Initial Version (Never Released)
+  2 - Stream Compression, seperate blobs, no packing
+  3 - One steam (row and blobs), with packing
 */
-#define ARCHIVE_VERSION 2
+#define ARCHIVE_VERSION 3
 
 class ha_archive: public handler
 {
   THR_LOCK_DATA lock;        /* MySQL lock */
   ARCHIVE_SHARE *share;      /* Shared lock info */
+  
   azio_stream archive;            /* Archive file we are working with */
   my_off_t current_position;  /* The position of the row we just read */
   byte byte_buffer[IO_SIZE]; /* Initial buffer for our string */
@@ -64,6 +69,10 @@ class ha_archive: public handler
   const byte *current_key;
   uint current_key_len;
   uint current_k_offset;
+  archive_record_buffer *record_buffer;
+
+  archive_record_buffer *create_record_buffer(ulonglong length); 
+  void destroy_record_buffer(archive_record_buffer *r);
 
 public:
   ha_archive(handlerton *hton, TABLE_SHARE *table_arg);
@@ -104,21 +113,13 @@ public:
   int rnd_next(byte *buf);
   int rnd_pos(byte * buf, byte *pos);
   int get_row(azio_stream *file_to_read, byte *buf);
-  int read_meta_file(File meta_file, ha_rows *rows, 
-                     ulonglong *auto_increment,
-                     ulonglong *forced_flushes,
-                     char *real_path);
-  int write_meta_file(File meta_file, ha_rows rows, 
-                      ulonglong auto_increment, 
-                      ulonglong forced_flushes, 
-                      char *real_path,
-                      bool dirty);
+  int get_row_version2(azio_stream *file_to_read, byte *buf);
+  int get_row_version3(azio_stream *file_to_read, byte *buf);
   ARCHIVE_SHARE *get_share(const char *table_name, TABLE *table, int *rc);
   int free_share(ARCHIVE_SHARE *share);
   int init_archive_writer();
   bool auto_repair() const { return 1; } // For the moment we just do this
   int read_data_header(azio_stream *file_to_read);
-  int write_data_header(azio_stream *file_to_write);
   void position(const byte *record);
   int info(uint);
   void update_create_info(HA_CREATE_INFO *create_info);
@@ -136,5 +137,9 @@ public:
   bool is_crashed() const;
   int check(THD* thd, HA_CHECK_OPT* check_opt);
   bool check_and_repair(THD *thd);
+  int max_row_length(const byte *buf);
+  bool fix_rec_buff(int length);
+  int unpack_row(azio_stream *file_to_read, char *record);
+  unsigned int pack_row(const byte *record);
 };