WL#4584 Internationalized number format

@ mysql-test/r/func_str.result
   Adding tests
@ mysql-test/t/func_str.test
   Adding tests
@ mysql-test/t/variables.test
   Fixing error number
@ sql/item_create.cc
   Allowing 2 and 3 arguments to format()
@ sql/item_strfunc.cc
   Adding new formatting code.
@ sql/item_strfunc.h
   Adding new contructors and "locale" member
@ sql/mysql_priv.h
   Adding number formatting members into MY_LOCALE
@ sql/sql_locale.cc
   Adding number formatting data into locale constants
@ sql/set_var.cc
   Using new error message
@ sql/share/errmgs.txt
   Adding new error message

mysql-test/r/func_str.result

@@ -2558,3 +2558,133 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	Using join buffer
 2	DERIVED	t1	ALL	NULL	NULL	NULL	NULL	2	
 drop table t1;
+Start of 5.4 tests
+SELECT format(12345678901234567890.123, 3);
+format(12345678901234567890.123, 3)
+12,345,678,901,234,567,890.123
+SELECT format(12345678901234567890.123, 3, NULL);
+format(12345678901234567890.123, 3, NULL)
+12,345,678,901,234,567,890.123
+Warnings:
+Warning	1647	Unknown locale: 'NULL'
+SELECT format(12345678901234567890.123, 3, 'ar_AE');
+format(12345678901234567890.123, 3, 'ar_AE')
+12,345,678,901,234,567,890.123
+SELECT format(12345678901234567890.123, 3, 'ar_SA');
+format(12345678901234567890.123, 3, 'ar_SA')
+12345678901234567890.123
+SELECT format(12345678901234567890.123, 3, 'be_BY');
+format(12345678901234567890.123, 3, 'be_BY')
+12.345.678.901.234.567.890,123
+SELECT format(12345678901234567890.123, 3, 'de_DE');
+format(12345678901234567890.123, 3, 'de_DE')
+12.345.678.901.234.567.890,123
+SELECT format(12345678901234567890.123, 3, 'en_IN');
+format(12345678901234567890.123, 3, 'en_IN')
+1,23,45,67,89,01,23,45,67,890.123
+SELECT format(12345678901234567890.123, 3, 'en_US');
+format(12345678901234567890.123, 3, 'en_US')
+12,345,678,901,234,567,890.123
+SELECT format(12345678901234567890.123, 3, 'it_CH');
+format(12345678901234567890.123, 3, 'it_CH')
+12'345'678'901'234'567'890,123
+SELECT format(12345678901234567890.123, 3, 'ru_RU');
+format(12345678901234567890.123, 3, 'ru_RU')
+12 345 678 901 234 567 890,123
+SELECT format(12345678901234567890.123, 3, 'ta_IN');
+format(12345678901234567890.123, 3, 'ta_IN')
+1,23,45,67,89,01,23,45,67,890.123
+CREATE TABLE t1 (fmt CHAR(5) NOT NULL);
+INSERT INTO t1 VALUES ('ar_AE');
+INSERT INTO t1 VALUES ('ar_SA');
+INSERT INTO t1 VALUES ('be_BY');
+INSERT INTO t1 VALUES ('de_DE');
+INSERT INTO t1 VALUES ('en_IN');
+INSERT INTO t1 VALUES ('en_US');
+INSERT INTO t1 VALUES ('it_CH');
+INSERT INTO t1 VALUES ('ru_RU');
+INSERT INTO t1 VALUES ('ta_IN');
+SELECT fmt, format(12345678901234567890.123, 3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(12345678901234567890.123, 3, fmt)
+ar_AE	12,345,678,901,234,567,890.123
+ar_SA	12345678901234567890.123
+be_BY	12.345.678.901.234.567.890,123
+de_DE	12.345.678.901.234.567.890,123
+en_IN	1,23,45,67,89,01,23,45,67,890.123
+en_US	12,345,678,901,234,567,890.123
+it_CH	12'345'678'901'234'567'890,123
+ru_RU	12 345 678 901 234 567 890,123
+ta_IN	1,23,45,67,89,01,23,45,67,890.123
+SELECT fmt, format(12345678901234567890.123, 0, fmt) FROM t1 ORDER BY fmt;
+fmt	format(12345678901234567890.123, 0, fmt)
+ar_AE	12,345,678,901,234,567,890
+ar_SA	12345678901234567890
+be_BY	12.345.678.901.234.567.890
+de_DE	12.345.678.901.234.567.890
+en_IN	1,23,45,67,89,01,23,45,67,890
+en_US	12,345,678,901,234,567,890
+it_CH	12'345'678'901'234'567'890
+ru_RU	12 345 678 901 234 567 890
+ta_IN	1,23,45,67,89,01,23,45,67,890
+SELECT fmt, format(12345678901234567890,     3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(12345678901234567890,     3, fmt)
+ar_AE	12,345,678,901,234,567,890.000
+ar_SA	12345678901234567890.000
+be_BY	12.345.678.901.234.567.890,000
+de_DE	12.345.678.901.234.567.890,000
+en_IN	1,23,45,67,89,01,23,45,67,890.000
+en_US	12,345,678,901,234,567,890.000
+it_CH	12'345'678'901'234'567'890,000
+ru_RU	12 345 678 901 234 567 890,000
+ta_IN	1,23,45,67,89,01,23,45,67,890.000
+SELECT fmt, format(-12345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(-12345678901234567890,    3, fmt)
+ar_AE	-12,345,678,901,234,567,890.000
+ar_SA	-12345678901234567890.000
+be_BY	-12.345.678.901.234.567.890,000
+de_DE	-12.345.678.901.234.567.890,000
+en_IN	-1,23,45,67,89,01,23,45,67,890.000
+en_US	-12,345,678,901,234,567,890.000
+it_CH	-12'345'678'901'234'567'890,000
+ru_RU	-12 345 678 901 234 567 890,000
+ta_IN	-1,23,45,67,89,01,23,45,67,890.000
+SELECT fmt, format(-02345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(-02345678901234567890,    3, fmt)
+ar_AE	-2,345,678,901,234,567,890.000
+ar_SA	-2345678901234567890.000
+be_BY	-2.345.678.901.234.567.890,000
+de_DE	-2.345.678.901.234.567.890,000
+en_IN	-23,45,67,89,01,23,45,67,890.000
+en_US	-2,345,678,901,234,567,890.000
+it_CH	-2'345'678'901'234'567'890,000
+ru_RU	-2 345 678 901 234 567 890,000
+ta_IN	-23,45,67,89,01,23,45,67,890.000
+SELECT fmt, format(-00345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(-00345678901234567890,    3, fmt)
+ar_AE	-345,678,901,234,567,890.000
+ar_SA	-345678901234567890.000
+be_BY	-345.678.901.234.567.890,000
+de_DE	-345.678.901.234.567.890,000
+en_IN	-3,45,67,89,01,23,45,67,890.000
+en_US	-345,678,901,234,567,890.000
+it_CH	-345'678'901'234'567'890,000
+ru_RU	-345 678 901 234 567 890,000
+ta_IN	-3,45,67,89,01,23,45,67,890.000
+SELECT fmt, format(-00045678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+fmt	format(-00045678901234567890,    3, fmt)
+ar_AE	-45,678,901,234,567,890.000
+ar_SA	-45678901234567890.000
+be_BY	-45.678.901.234.567.890,000
+de_DE	-45.678.901.234.567.890,000
+en_IN	-45,67,89,01,23,45,67,890.000
+en_US	-45,678,901,234,567,890.000
+it_CH	-45'678'901'234'567'890,000
+ru_RU	-45 678 901 234 567 890,000
+ta_IN	-45,67,89,01,23,45,67,890.000
+DROP TABLE t1;
+SELECT format(123, 1, 'Non-existent-locale');
+format(123, 1, 'Non-existent-locale')
+123.0
+Warnings:
+Warning	1647	Unknown locale: 'Non-existent-locale'
+End of 5.4 tests

mysql-test/t/func_str.test

@@ -1318,3 +1318,45 @@ insert into t1 values (-1),(null);
 explain select 1 as a from t1,(select decode(f1,f1) as b from t1) a;
 explain select 1 as a from t1,(select encode(f1,f1) as b from t1) a;
 drop table t1;
+
+
+
+--echo Start of 5.4 tests
+#
+# WL#4584 Internationalized number format
+#
+SELECT format(12345678901234567890.123, 3);
+SELECT format(12345678901234567890.123, 3, NULL);
+SELECT format(12345678901234567890.123, 3, 'ar_AE');
+SELECT format(12345678901234567890.123, 3, 'ar_SA');
+SELECT format(12345678901234567890.123, 3, 'be_BY');
+SELECT format(12345678901234567890.123, 3, 'de_DE');
+SELECT format(12345678901234567890.123, 3, 'en_IN');
+SELECT format(12345678901234567890.123, 3, 'en_US');
+SELECT format(12345678901234567890.123, 3, 'it_CH');
+SELECT format(12345678901234567890.123, 3, 'ru_RU');
+SELECT format(12345678901234567890.123, 3, 'ta_IN');
+
+CREATE TABLE t1 (fmt CHAR(5) NOT NULL);
+INSERT INTO t1 VALUES ('ar_AE');
+INSERT INTO t1 VALUES ('ar_SA');
+INSERT INTO t1 VALUES ('be_BY');
+INSERT INTO t1 VALUES ('de_DE');
+INSERT INTO t1 VALUES ('en_IN');
+INSERT INTO t1 VALUES ('en_US');
+INSERT INTO t1 VALUES ('it_CH');
+INSERT INTO t1 VALUES ('ru_RU');
+INSERT INTO t1 VALUES ('ta_IN');
+SELECT fmt, format(12345678901234567890.123, 3, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(12345678901234567890.123, 0, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(12345678901234567890,     3, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(-12345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(-02345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(-00345678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+SELECT fmt, format(-00045678901234567890,    3, fmt) FROM t1 ORDER BY fmt;
+DROP TABLE t1;
+
+SELECT format(123, 1, 'Non-existent-locale');
+
+--echo End of 5.4 tests
+

mysql-test/t/variables.test

@@ -559,7 +559,7 @@ select @@lc_time_names;
 --echo *** LC_TIME_NAMES: testing with string expressions
 set lc_time_names=concat('de','_','DE');
 select @@lc_time_names;
---error ER_UNKNOWN_ERROR
+--error ER_UNKNOWN_LOCALE
 set lc_time_names=concat('de','+','DE');
 select @@lc_time_names;
 --echo LC_TIME_NAMES: testing with numeric expressions
@@ -572,14 +572,14 @@ set lc_time_names=en_US;
 --echo LC_TIME_NAMES: testing NULL and a negative number:
 --error ER_WRONG_VALUE_FOR_VAR
 set lc_time_names=NULL;
---error ER_UNKNOWN_ERROR
+--error ER_UNKNOWN_LOCALE
 set lc_time_names=-1;
 select @@lc_time_names;
 --echo LC_TIME_NAMES: testing locale with the last ID:
 set lc_time_names=108;
 select @@lc_time_names;
 --echo LC_TIME_NAMES: testing a number beyond the valid ID range:
---error ER_UNKNOWN_ERROR
+--error ER_UNKNOWN_LOCALE
 set lc_time_names=109;
 select @@lc_time_names;
 --echo LC_TIME_NAMES: testing that 0 is en_US:

sql/item_create.cc

@@ -927,10 +927,10 @@ protected:
 };
 
 
-class Create_func_format : public Create_func_arg2
+class Create_func_format : public Create_native_func
 {
 public:
-  virtual Item *create(THD *thd, Item *arg1, Item *arg2);
+  virtual Item *create_native(THD *thd, LEX_STRING name, List<Item> *item_list);
 
   static Create_func_format s_singleton;
 
@@ -3352,9 +3352,34 @@ Create_func_floor::create(THD *thd, Item *arg1)
 Create_func_format Create_func_format::s_singleton;
 
 Item*
-Create_func_format::create(THD *thd, Item *arg1, Item *arg2)
+Create_func_format::create_native(THD *thd, LEX_STRING name,
+                                  List<Item> *item_list)
 {
-  return new (thd->mem_root) Item_func_format(arg1, arg2);
+  Item *func= NULL;
+  int arg_count= item_list ? item_list->elements : 0;
+
+  switch (arg_count) {
+  case 2:
+  {
+    Item *param_1= item_list->pop();
+    Item *param_2= item_list->pop();
+    func= new (thd->mem_root) Item_func_format(param_1, param_2);
+    break;
+  }
+  case 3:
+  {
+    Item *param_1= item_list->pop();
+    Item *param_2= item_list->pop();
+    Item *param_3= item_list->pop();
+    func= new (thd->mem_root) Item_func_format(param_1, param_2, param_3);
+    break;
+  }
+  default:
+    my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name.str);
+    break;
+  }
+
+  return func;
 }
 
 

sql/item_strfunc.cc

@@ -2040,9 +2040,22 @@ String *Item_func_soundex::val_str(String *str)
 
 const int FORMAT_MAX_DECIMALS= 30;
 
-Item_func_format::Item_func_format(Item *org, Item *dec)
-: Item_str_func(org, dec)
+
+MY_LOCALE *Item_func_format::get_locale(Item *item)
 {
+  DBUG_ASSERT(arg_count == 3);
+  String tmp, *locale_name= args[2]->val_str(&tmp);
+  MY_LOCALE *lc;
+  if (!locale_name ||
+      !(lc= my_locale_by_name(locale_name->c_ptr_safe())))
+  {
+    push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_WARN,
+                        ER_UNKNOWN_LOCALE,
+                        ER(ER_UNKNOWN_LOCALE),
+                        locale_name ? locale_name->c_ptr_safe() : "NULL");
+    lc= &my_locale_en_US;
+  }
+  return lc;
 }
 
 void Item_func_format::fix_length_and_dec()
@@ -2052,6 +2065,10 @@ void Item_func_format::fix_length_and_dec()
   collation.set(default_charset());
   max_length= (char_length + max_sep_count + decimals) *
     collation.collation->mbmaxlen;
+  if (arg_count == 3)
+    locale= args[2]->basic_const_item() ? get_locale(args[2]) : NULL;
+  else
+    locale= &my_locale_en_US; /* Two arguments */
 }
 
 
@@ -2063,13 +2080,12 @@ void Item_func_format::fix_length_and_dec()
 
 String *Item_func_format::val_str(String *str)
 {
-  uint32 length;
   uint32 str_length;
   /* Number of decimal digits */
   int dec;
   /* Number of characters used to represent the decimals, including '.' */
   uint32 dec_length;
-  int diff;
+  MY_LOCALE *lc;
   DBUG_ASSERT(fixed == 1);
 
   dec= (int) args[1]->val_int();
@@ -2079,6 +2095,8 @@ String *Item_func_format::val_str(String *str)
     return NULL;
   }
 
+  lc= locale ? locale : get_locale(args[2]);
+
   dec= set_zone(dec, 0, FORMAT_MAX_DECIMALS);
   dec_length= dec ? dec+1 : 0;
   null_value=0;
@@ -2093,8 +2111,6 @@ String *Item_func_format::val_str(String *str)
     my_decimal_round(E_DEC_FATAL_ERROR, res, dec, false, &rnd_dec);
     my_decimal2string(E_DEC_FATAL_ERROR, &rnd_dec, 0, 0, 0, str);
     str_length= str->length();
-    if (rnd_dec.sign())
-      str_length--;
   }
   else
   {
@@ -2107,31 +2123,51 @@ String *Item_func_format::val_str(String *str)
     if (isnan(nr))
       return str;
     str_length=str->length();
-    if (nr < 0)
-      str_length--;				// Don't count sign
-  }
-  /* We need this test to handle 'nan' values */
-  if (str_length >= dec_length+4)
-  {
-    char *tmp,*pos;
-    length= str->length()+(diff=((int)(str_length- dec_length-1))/3);
-    str= copy_if_not_alloced(&tmp_str,str,length);
-    str->length(length);
-    tmp= (char*) str->ptr()+length - dec_length-1;
-    for (pos= (char*) str->ptr()+length-1; pos != tmp; pos--)
-      pos[0]= pos[-diff];
-    while (diff)
+  }
+  /* We need this test to handle 'nan' and short values */
+  if (lc->grouping[0] > 0 &&
+      str_length >= dec_length + 1 + lc->grouping[0])
+  {
+    char buf[DECIMAL_MAX_STR_LENGTH * 2]; /* 2 - in the worst case when grouping=1 */
+    int count;
+    const char *grouping= lc->grouping;
+    char sign_length= *str->ptr() == '-' ? 1 : 0;
+    const char *src= str->ptr() + str_length - dec_length - 1;
+    const char *src_begin= str->ptr() + sign_length;
+    char *dst= buf + sizeof(buf);
+    
+    /* Put the fractional part */
+    if (dec)
     {
-      *pos= *(pos - diff);
-      pos--;
-      *pos= *(pos - diff);
-      pos--;
-      *pos= *(pos - diff);
-      pos--;
-      pos[0]=',';
-      pos--;
-      diff--;
+      dst-= (dec + 1);
+      *dst= lc->decimal_point;
+      memcpy(dst + 1, src + 2, dec);
     }
+    
+    /* Put the integer part with grouping */
+    for (count= *grouping; src >= src_begin; count--)
+    {
+      /*
+        When *grouping==0x80 (which means "end of grouping")
+        count will be initialized to -1 and
+        we'll never get into this "if" anymore.
+      */
+      if (!count)
+      {
+        *--dst= lc->thousand_sep;
+        if (grouping[1])
+          grouping++;
+        count= *grouping;
+      }
+      DBUG_ASSERT(dst > buf);
+      *--dst= *src--;
+    }
+    
+    if (sign_length) /* Put '-' */
+      *--dst= *str->ptr();
+    
+    /* Put the rest of the integer part without grouping */
+    str->copy(dst, buf + sizeof(buf) - dst, &my_charset_latin1);
   }
   return str;
 }

sql/item_strfunc.h

@@ -498,8 +498,13 @@ public:
 class Item_func_format :public Item_str_func
 {
   String tmp_str;
+  MY_LOCALE *locale;
 public:
-  Item_func_format(Item *org, Item *dec);
+  Item_func_format(Item *org, Item *dec): Item_str_func(org, dec) {}
+  Item_func_format(Item *org, Item *dec, Item *lang):
+  Item_str_func(org, dec, lang) {}
+  
+  MY_LOCALE *get_locale(Item *item);
   String *val_str(String *);
   void fix_length_and_dec();
   const char *func_name() const { return "format"; }

sql/mysql_priv.h

@@ -156,18 +156,26 @@ typedef struct my_locale_st
   TYPELIB *ab_day_names;
   uint max_month_name_length;
   uint max_day_name_length;
+  uint decimal_point;
+  uint thousand_sep;
+  const char *grouping;
 #ifdef __cplusplus 
   my_locale_st(uint number_par,
                const char *name_par, const char *descr_par, bool is_ascii_par,
                TYPELIB *month_names_par, TYPELIB *ab_month_names_par,
                TYPELIB *day_names_par, TYPELIB *ab_day_names_par,
-               uint max_month_name_length_par, uint max_day_name_length_par) : 
+               uint max_month_name_length_par, uint max_day_name_length_par,
+               uint decimal_point_par, uint thousand_sep_par,
+               const char *grouping_par) : 
     number(number_par),
     name(name_par), description(descr_par), is_ascii(is_ascii_par),
     month_names(month_names_par), ab_month_names(ab_month_names_par),
     day_names(day_names_par), ab_day_names(ab_day_names_par),
     max_month_name_length(max_month_name_length_par),
-    max_day_name_length(max_day_name_length_par)
+    max_day_name_length(max_day_name_length_par),
+    decimal_point(decimal_point_par),
+    thousand_sep(thousand_sep_par),
+    grouping(grouping_par)
   {}
 #endif
 } MY_LOCALE;

sql/set_var.cc

@@ -2916,7 +2916,7 @@ bool sys_var_thd_lc_time_names::check(THD *thd, set_var *var)
     {
       char buf[20];
       int10_to_str((int) var->value->val_int(), buf, -10);
-      my_printf_error(ER_UNKNOWN_ERROR, "Unknown locale: '%s'", MYF(0), buf);
+      my_printf_error(ER_UNKNOWN_LOCALE, ER(ER_UNKNOWN_LOCALE), MYF(0), buf);
       return 1;
     }
   }
@@ -2932,8 +2932,7 @@ bool sys_var_thd_lc_time_names::check(THD *thd, set_var *var)
     const char *locale_str= res->c_ptr();
     if (!(locale_match= my_locale_by_name(locale_str)))
     {
-      my_printf_error(ER_UNKNOWN_ERROR,
-                      "Unknown locale: '%s'", MYF(0), locale_str);
+      my_printf_error(ER_UNKNOWN_LOCALE, ER(ER_UNKNOWN_LOCALE), MYF(0), locale_str);
       return 1;
     }
   }

sql/share/errmsg.txt

@@ -6228,3 +6228,5 @@ WARN_COND_ITEM_TRUNCATED
 ER_COND_ITEM_TOO_LONG
         eng "Data too long for condition item '%s'"
 
+ER_UNKNOWN_LOCALE                                                                                                                                                                    
+        eng "Unknown locale: '%-.64s'"