Bug #23578: Corruption prevents Optimize table from working properly with a

spatial index
 While executing OPTIMIZE TABLE on MyISAM tables the server re-creates the
 index file(s) in order to sort them physically by the key. This cannot be 
 done for R-tree indexes as it makes no sense.
 The server was not checking the type of the index and was accessing an 
 R-tree index as if it was a B-tree.
 Fixed by preventing sorting the index file if it contains an R-tree index.

myisam/mi_check.c

@@ -1758,6 +1758,12 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name)
   MI_STATE_INFO old_state;
   DBUG_ENTER("mi_sort_index");
 
+  /* cannot sort index files with R-tree indexes */
+  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
+       key++,keyinfo++)
+    if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
+      return 0;
+
   if (!(param->testflag & T_SILENT))
     printf("- Sorting index for MyISAM-table '%s'\n",name);
 
@@ -1850,6 +1856,8 @@ static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
   char llbuff[22];
   DBUG_ENTER("sort_one_index");
 
+  /* cannot walk over R-tree indices */
+  DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
   new_page_pos=param->new_file_pos;
   param->new_file_pos+=keyinfo->block_length;
 

mysql-test/r/gis-rtree.result

@@ -881,3 +881,14 @@ ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
 INSERT INTO t1(foo) VALUES ('');
 ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
 DROP TABLE t1;
+CREATE TABLE t1 (a INT AUTO_INCREMENT, b POINT NOT NULL, KEY (a), SPATIAL KEY (b));
+INSERT INTO t1 (b) VALUES (GeomFromText('POINT(1 2)'));
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+DROP TABLE t1;

mysql-test/t/gis-rtree.test

@@ -254,3 +254,20 @@ INSERT INTO t1() VALUES ();
 --error 1416
 INSERT INTO t1(foo) VALUES ('');
 DROP TABLE t1;
+
+#
+# Bug #23578: Corruption prevents Optimize table from working properly with a 
+#             spatial index
+#
+
+CREATE TABLE t1 (a INT AUTO_INCREMENT, b POINT NOT NULL, KEY (a), SPATIAL KEY (b));
+
+INSERT INTO t1 (b) VALUES (GeomFromText('POINT(1 2)'));
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+INSERT INTO t1 (b) SELECT b FROM t1;
+
+OPTIMIZE TABLE t1;
+DROP TABLE t1;