Bug #18010711 UNIQUE PREFIX INDEX ON BINARY COLUMN:

FAILING ASSERTION: FLEN == LEN

Problem:
       Broken invariant triggered when building a unique index on a
binary column and the input data contains duplicate keys. This was broken
in debug builds only.

Fix:
       Fixed length of the binary datatype can be greater than length of
the shorter prefix on which index is being created.

mysql-test/suite/innodb/r/create-index.result

@@ -0,0 +1,9 @@
+#
+# Bug #18010711	UNIQUE PREFIX INDEX ON BINARY COLUMN: FAILING
+# ASSERTION: FLEN == LEN
+#
+create table t1 (f1 binary(5)) engine=innodb;
+insert into t1 values ('w'), ('w');
+create unique index index_t1 on t1(f1(4));
+ERROR 23000: Duplicate entry 'w' for key 'index_t1'
+drop table t1;

mysql-test/suite/innodb/t/create-index.test

@@ -0,0 +1,11 @@
+--source include/have_innodb.inc
+
+--echo #
+--echo # Bug #18010711	UNIQUE PREFIX INDEX ON BINARY COLUMN: FAILING
+--echo # ASSERTION: FLEN == LEN
+--echo #
+create table t1 (f1 binary(5)) engine=innodb;
+insert into t1 values ('w'), ('w');
+--error ER_DUP_ENTRY
+create unique index index_t1 on t1(f1(4));
+drop table t1;

storage/innobase/handler/handler0alter.cc

@@ -110,16 +110,16 @@ innobase_col_to_mysql(
 		/* These column types should never be shipped to MySQL. */
 		ut_ad(0);
 
-	case DATA_FIXBINARY:
 	case DATA_FLOAT:
 	case DATA_DOUBLE:
 	case DATA_DECIMAL:
 		/* Above are the valid column types for MySQL data. */
 		ut_ad(flen == len);
 		/* fall through */
+	case DATA_FIXBINARY:
 	case DATA_CHAR:
 		/* We may have flen > len when there is a shorter
-		prefix on a CHAR column. */
+		prefix on the CHAR and BINARY column. */
 		ut_ad(flen >= len);
 #else /* UNIV_DEBUG */
 	default: