Fixed bug#58026 - massive recursion and crash in regular expression

handling.

The problem was that parsing of nested regular expression involved
recursive calls. Such recursion didn't take into account the amount of
available stack space, which ended up leading to stack overflow crashes.

mysql-test/r/not_embedded_server.result

@@ -4,3 +4,7 @@ select 1;
 SHOW VARIABLES like 'slave_skip_errors';
 Variable_name	Value
 slave_skip_errors	OFF
+#
+# Bug#58026: massive recursion and crash in regular expression handling
+#
+SELECT '1' RLIKE RPAD('1', 10000, '(');

mysql-test/t/not_embedded_server.test

@@ -42,4 +42,14 @@ select 1;
 
 SHOW VARIABLES like 'slave_skip_errors';
 
+--echo #
+--echo # Bug#58026: massive recursion and crash in regular expression handling
+--echo #
+
+--disable_result_log
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT '1' RLIKE RPAD('1', 10000, '(');
+--enable_result_log
+
+
 # End of 5.1 tests

regex/my_regex.h

@@ -28,6 +28,7 @@ typedef struct {
 
 
 /* === regcomp.c === */
+typedef int (*my_regex_stack_check_t)();
 extern int my_regcomp(my_regex_t *, const char *, int, CHARSET_INFO *charset);
 #define	REG_BASIC	0000
 #define	REG_EXTENDED	0001
@@ -76,7 +77,8 @@ extern void my_regfree(my_regex_t *);
 
 /* === reginit.c === */
 
-extern void my_regex_init(CHARSET_INFO *cs);	/* Should be called for multithread progs */
+/* Should be called for multithread progs */
+extern void my_regex_init(CHARSET_INFO *cs,  my_regex_stack_check_t func);
 extern void my_regex_end(void);	/* If one wants a clean end */
 
 #ifdef __cplusplus

regex/regcomp.c

@@ -31,6 +31,9 @@ struct parse {
 	CHARSET_INFO *charset;	/* for ctype things  */
 };
 
+/* Check if there is enough stack space for recursion. */
+my_regex_stack_check_t my_regex_enough_mem_in_stack= NULL;
+
 #include "regcomp.ih"
 
 static char nuls[10];		/* place to point scanner in event of error */
@@ -117,7 +120,7 @@ CHARSET_INFO *charset;
 #	define	GOODFLAGS(f)	((f)&~REG_DUMP)
 #endif
 
-	my_regex_init(charset);	/* Init cclass if neaded */
+	my_regex_init(charset, NULL);	/* Init cclass if neaded */
 	preg->charset=charset;
 	cflags = GOODFLAGS(cflags);
 	if ((cflags&REG_EXTENDED) && (cflags&REG_NOSPEC))
@@ -222,7 +225,15 @@ int stop;			/* character this ERE should end at */
 		/* do a bunch of concatenated expressions */
 		conc = HERE();
 		while (MORE() && (c = PEEK()) != '|' && c != stop)
-			p_ere_exp(p);
+		{
+		  if (my_regex_enough_mem_in_stack &&
+		      my_regex_enough_mem_in_stack())
+		  {
+		    SETERROR(REG_ESPACE);
+		    return;
+		  }
+		  p_ere_exp(p);
+		}
 		if(REQUIRE(HERE() != conc, REG_EMPTY)) {}/* require nonempty */
 
 		if (!EAT('|'))

regex/reginit.c

@@ -4,10 +4,12 @@
 #include <m_ctype.h>
 #include <m_string.h>
 #include "cclass.h"
+#include "my_regex.h"
 
 static my_bool regex_inited=0;
+extern my_regex_stack_check_t my_regex_enough_mem_in_stack;
 
-void my_regex_init(CHARSET_INFO *cs)
+void my_regex_init(CHARSET_INFO *cs, my_regex_stack_check_t func)
 {
   char buff[CCLASS_LAST][256];
   int  count[CCLASS_LAST];
@@ -16,6 +18,7 @@ void my_regex_init(CHARSET_INFO *cs)
   if (!regex_inited)
   {
     regex_inited=1;
+    my_regex_enough_mem_in_stack= func;
     bzero((uchar*) &count,sizeof(count));
 
     for (i=1 ; i<= 255; i++)
@@ -74,6 +77,7 @@ void my_regex_end()
     int i;
     for (i=0; i < CCLASS_LAST ; i++)
       free((char*) cclasses[i].chars);
+    my_regex_enough_mem_in_stack= NULL;
     regex_inited=0;
   }
 }

sql/mysqld.cc

@@ -3034,6 +3034,19 @@ sizeof(load_default_groups)/sizeof(load_default_groups[0]);
 #endif
 
 
+#ifndef EMBEDDED_LIBRARY
+static
+int
+check_enough_stack_size()
+{
+  uchar stack_top;
+
+  return check_stack_overrun(current_thd, STACK_MIN_SIZE,
+                             &stack_top);
+}
+#endif
+
+
 /**
   Initialize one of the global date/time format variables.
 
@@ -3415,7 +3428,11 @@ static int init_common_variables(const char *conf_file_name, int argc,
 #endif
   mysys_uses_curses=0;
 #ifdef USE_REGEX
-  my_regex_init(&my_charset_latin1);
+#ifndef EMBEDDED_LIBRARY
+  my_regex_init(&my_charset_latin1, check_enough_stack_size);
+#else
+  my_regex_init(&my_charset_latin1, NULL);
+#endif
 #endif
   /*
     Process a comma-separated character set list and choose