Bug11764310 - 57132: CONV FUNCTION CRASHES, NEGATIVE ARGUMENT TO MEMCPY

Failure to check the return state of a longlong2str() call
caused a crash. This could happen if a user executed the sql
function CONV() with certain parameters.

The patch fixes the issue by checking that the returned pointer
isn't NULL.

mysql-test/r/func_str.result

@@ -2784,6 +2784,12 @@ SELECT * FROM t1;
 format(123,2,'no_NO')
 123,00
 DROP TABLE t1;
+#
+# Bug#11764310 conv function crashes, negative argument to memcpy
+#
+SELECT CONV(1,-2147483648,-2147483648);
+CONV(1,-2147483648,-2147483648)
+
 #
 # End of 5.5 tests
 #

mysql-test/t/func_str.test

@@ -1436,6 +1436,11 @@ SHOW CREATE TABLE t1;
 SELECT * FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11764310 conv function crashes, negative argument to memcpy
+--echo #
+SELECT CONV(1,-2147483648,-2147483648);
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

sql/item_strfunc.cc

@@ -2952,8 +2952,8 @@ String *Item_func_conv::val_str(String *str)
                                    from_base, &endptr, &err);
   }
 
-  ptr= longlong2str(dec, ans, to_base);
-  if (str->copy(ans, (uint32) (ptr-ans), default_charset()))
+  if (!(ptr= longlong2str(dec, ans, to_base)) ||
+      str->copy(ans, (uint32) (ptr - ans), default_charset()))
     return make_empty_result();
   return str;
 }
@@ -3113,8 +3113,10 @@ String *Item_func_hex::val_str_ascii(String *str)
 
     if ((null_value= args[0]->null_value))
       return 0;
-    ptr= longlong2str(dec,ans,16);
-    if (str->copy(ans,(uint32) (ptr-ans), &my_charset_numeric))
+    
+    if (!(ptr= longlong2str(dec, ans, 16)) ||
+        str->copy(ans,(uint32) (ptr - ans),
+        &my_charset_numeric))
       return make_empty_result();		// End of memory
     return str;
   }