MDEV-11741 handler::ha_reset(): Assertion `bitmap_is_set_all(&table->s->all_set)' failed or server crash in mi_reset or buffer overrun or unexpected ER_CANT_REMOVE_ALL_FIELDS

MEMORY table could be renamed into a non-extistent database.

rename() is documented to return ENOENT when the source file does not
exist OR when the target directory not exist. Nonexistent source .frm
file is ok (table can still exist in the engine), nonexistent target
directory is not.

Make my_rename to use ENOTDIR for the latter case. Make RENAME TABLE
issue an appropriate error ("unknown database" instead of "unknown table")

mysql-test/r/rename.result

@@ -133,3 +133,7 @@ select * from t2;
 a
 1
 drop table tmp,t2;
+create table t1 (a int) engine=memory;
+rename table t1 to non_existent.t2;
+ERROR 42000: Unknown database 'non_existent'
+drop table t1;

mysql-test/t/rename.test

@@ -141,3 +141,10 @@ select * from tmp;
 select * from t2;
 drop table tmp,t2;
 
+#
+# MDEV-11741 handler::ha_reset(): Assertion `bitmap_is_set_all(&table->s->all_set)' failed or server crash in mi_reset or buffer overrun or unexpected ER_CANT_REMOVE_ALL_FIELDS
+#
+create table t1 (a int) engine=memory;
+--error ER_BAD_DB_ERROR
+rename table t1 to non_existent.t2;
+drop table t1;

mysys/my_rename.c

@@ -39,7 +39,10 @@ int my_rename(const char *from, const char *to, myf MyFlags)
   if (link(from, to) || unlink(from))
   {
 #endif
-    my_errno=errno;
+    if (errno == ENOENT && !access(from, F_OK))
+      my_errno= ENOTDIR;
+    else
+      my_errno= errno;
     error = -1;
     if (MyFlags & (MY_FAE+MY_WME))
       my_error(EE_LINK, MYF(ME_BELL+ME_WAITTANG),from,to,my_errno);

sql/sql_table.cc

@@ -5257,6 +5257,8 @@ mysql_rename_table(handlerton *base, const char *old_db,
   delete file;
   if (error == HA_ERR_WRONG_COMMAND)
     my_error(ER_NOT_SUPPORTED_YET, MYF(0), "ALTER TABLE");
+  else if (error ==  ENOTDIR)
+    my_error(ER_BAD_DB_ERROR, MYF(0), new_db);
   else if (error)
     my_error(ER_ERROR_ON_RENAME, MYF(0), from, to, error);
   else if (!(flags & FN_IS_TMP))