MDEV-9158 Read max size bytes from encryption key file and ignore remain bytes

Previously plugin check aes key file size to make sure its size isn't too large before reading it, this commit change the way to read only max aes key file size bytes. This way can support named pipe as a coproduct .

mysql-test/suite/encryption/r/filekeys_length_too_large.result

@@ -0,0 +1,8 @@
+# Test checks if opening a too large key file, file key plugin will only read max key file size bytes and extra bytes will be ignored.
+#Large key file will read max size bytes , which is 1MB
+call mtr.add_suppression("Syntax error");
+call mtr.add_suppression("Invalid key");
+call mtr.add_suppression("Plugin 'file_key_management' init function returned error");
+call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
+INSTALL SONAME 'file_key_management';
+FOUND 1 /read bytes: 1048576B/ in mysqld.1.err

mysql-test/suite/encryption/r/filekeys_named_pipe.result

@@ -0,0 +1,31 @@
+INSTALL SONAME 'file_key_management';
+Warnings:
+Note	1	Read from MYSQL_TMP_DIR/fifo.key , read bytes: 105B, max key file size :1048576B 
+CREATE TABLE t1(c1 BIGINT NOT NULL, b CHAR(200))  ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=1
+INSERT t1 VALUES (12345, REPEAT('1234567890', 20));
+ALTER TABLE t1 ENCRYPTION_KEY_ID=2;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=2
+ALTER TABLE t1 ENCRYPTION_KEY_ID=3;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=3
+# Reset mysqld
+DROP TABLE t1;
+UNINSTALL SONAME 'file_key_management';
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown
+# restart

mysql-test/suite/encryption/t/filekeys_length_too_large.opt

@@ -0,0 +1 @@
+--loose-file-key-management-filename=$MYSQL_TMP_DIR/filekeys-data-too-large.key

mysql-test/suite/encryption/t/filekeys_length_too_large.test

@@ -0,0 +1,33 @@
+-- source include/not_embedded.inc
+--echo # Test checks if opening a too large key file, file key plugin will only read max key file size bytes and extra bytes will be ignored.
+
+let $k=0;
+while($k<1000){
+        let $content="";
+        let $i= 1;
+        while($i <= 100){
+            let $num= `SELECT $i+$k*100`;
+            let $content=$content"$num;24e579b00c0f365a11761bdb9286a72a1148fa779d9c23dd55402627d0c87726";
+            if($i<=99){
+                let $content=$content"\n";
+            }
+            inc $i;
+        }
+        exec echo -e $content >> $MYSQL_TMP_DIR/filekeys-data-too-large.key;
+        inc $k;
+}
+--echo #Large key file will read max size bytes , which is 1MB
+let SEARCH_PATTERN=read bytes: 1048576B;
+call mtr.add_suppression("Syntax error");
+call mtr.add_suppression("Invalid key");
+call mtr.add_suppression("Plugin 'file_key_management' init function returned error");
+call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
+
+--disable_result_log
+--error 2
+INSTALL SONAME 'file_key_management';
+--enable_result_log
+
+
+--let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err
+--source include/search_pattern_in_file.inc

mysql-test/suite/encryption/t/filekeys_named_pipe.opt

@@ -0,0 +1 @@
+--loose-file-key-management-filename=$MYSQL_TMP_DIR/fifo.key

mysql-test/suite/encryption/t/filekeys_named_pipe.test

@@ -0,0 +1,30 @@
+#
+# Test read key from named pipe
+#
+write_file $MYSQL_TMP_DIR/fifo.key;
+1;11111111111111111111111111111111
+2;00000000000000000000000000000000
+3;22222222222222222222222222222222
+EOF
+--sleep 3
+
+--source include/have_innodb.inc
+--replace_result $MYSQL_TMP_DIR   MYSQL_TMP_DIR
+INSTALL SONAME 'file_key_management';
+
+CREATE TABLE t1(c1 BIGINT NOT NULL, b CHAR(200))  ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
+SHOW CREATE TABLE t1;
+INSERT t1 VALUES (12345, REPEAT('1234567890', 20));
+
+ALTER TABLE t1 ENCRYPTION_KEY_ID=2;
+SHOW CREATE TABLE t1;
+ALTER TABLE t1 ENCRYPTION_KEY_ID=3;
+SHOW CREATE TABLE t1;
+
+--echo # Reset mysqld
+DROP TABLE t1;
+remove_file $MYSQL_TMP_DIR/fifo.key;
+UNINSTALL SONAME 'file_key_management';
+
+--source include/restart_mysqld.inc
+

plugin/file_key_management/parser.cc

@@ -314,6 +314,7 @@ int Parser::parse_line(char **line_ptr, keyentry *key)
 char* Parser::read_and_decrypt_file(const char *secret)
 {
   int f;
+  ssize_t file_size= 0;
   if (!filename || !filename[0])
   {
     my_printf_error(EE_CANT_OPEN_STREAM, "file-key-management-filename is not set",
@@ -328,41 +329,28 @@ char* Parser::read_and_decrypt_file(const char *secret)
     goto err0;
   }
 
-  my_off_t file_size;
-  file_size= lseek(f, 0, SEEK_END);
-
-  if (file_size == MY_FILEPOS_ERROR || (my_off_t)lseek(f, 0, SEEK_SET) == MY_FILEPOS_ERROR)
-  {
-    my_error(EE_CANT_SEEK, MYF(0), filename, errno);
-    goto err1;
-  }
-
-  if (file_size > MAX_KEY_FILE_SIZE)
-  {
-    my_error(EE_READ, MYF(0), filename, EFBIG);
-    goto err1;
-  }
-
   //Read file into buffer
   uchar *buffer;
-  buffer= (uchar*)malloc((size_t)file_size + 1);
+  buffer= (uchar *) malloc((size_t) MAX_KEY_FILE_SIZE );
   if (!buffer)
   {
-    my_error(EE_OUTOFMEMORY, ME_ERROR_LOG| ME_FATAL, file_size);
+    my_error(EE_OUTOFMEMORY, ME_ERROR_LOG | ME_FATAL, file_size);
     goto err1;
   }
-
-  if (read(f, buffer, (int)file_size) != (int)file_size)
+  file_size= read(f, buffer, MAX_KEY_FILE_SIZE);
+  if (file_size < 0)
   {
-    my_printf_error(EE_READ,
-      "read from %s failed, errno %d",
-      MYF(ME_ERROR_LOG|ME_FATAL), filename, errno);
+    my_printf_error(EE_READ, "Read from %s failed, errno %d",
+                   ME_ERROR_LOG , filename, errno);
     goto err2;
   }
-
-// Check for file encryption
+  my_printf_error(EE_ERROR_FIRST,
+                  "Read from %s , read bytes: %zdB, max key file size :%dB ",
+                  ME_ERROR_LOG | ME_NOTE, filename, file_size,
+                  MAX_KEY_FILE_SIZE);
+  // Check for file encryption
   uchar *decrypted;
-  if (file_size > OpenSSL_prefix_len && strncmp((char*)buffer, OpenSSL_prefix, OpenSSL_prefix_len) == 0)
+  if ((size_t)file_size > OpenSSL_prefix_len && strncmp((char*)buffer, OpenSSL_prefix, OpenSSL_prefix_len) == 0)
   {
     uchar key[OpenSSL_key_len];
     uchar iv[OpenSSL_iv_len];