MDEV-13173 An RLIKE that previously worked on 10.0 now returns "Got error 'pcre_exec: recursion limit of 100 exceeded' from regexp"

1. use Regexp_processor_pcre::set_recursion_limit() to set the
   recursion limit depending on the current available stack size
2. make pcre stack frame to be estimated no less than 500 bytes.
   sometimes pcre estimates it too low, even though the manual
   says 500+16 bytes (it was estimated only 188 for me, actual
   frame size was 512).
3. do it for embedded too

mysql-test/r/func_regexp_pcre.result

@@ -879,3 +879,35 @@ SELECT 1 FROM dual WHERE ('Alpha,Bravo,Charlie,Delta,Echo,Foxtrot,StrataCentral,
 1
 Warnings:
 Warning	1139	Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp
+SELECT CONCAT(REPEAT('100,',500),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$';
+CONCAT(REPEAT('100,',500),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$'
+1
+SELECT CONCAT(REPEAT('100,',600),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$';
+CONCAT(REPEAT('100,',600),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$'
+0
+Warnings:
+Warning	1139	Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp
+SELECT REGEXP_INSTR(CONCAT(REPEAT('100,',500),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$');
+REGEXP_INSTR(CONCAT(REPEAT('100,',500),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$')
+1
+SELECT REGEXP_INSTR(CONCAT(REPEAT('100,',600),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$');
+REGEXP_INSTR(CONCAT(REPEAT('100,',600),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$')
+0
+Warnings:
+Warning	1139	Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp
+SELECT LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'));
+LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'))
+671
+SELECT LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'));
+LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'))
+0
+Warnings:
+Warning	1139	Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp
+SELECT LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''));
+LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''))
+0
+SELECT LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''));
+LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''))
+803
+Warnings:
+Warning	1139	Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp

mysql-test/t/func_regexp_pcre.test

@@ -430,3 +430,22 @@ SELECT CAST(0xE001 AS BINARY) REGEXP @regCheck;
 --echo # MDEV-12420: Testing recursion overflow
 --replace_regex /[0-9]+ exceeded/NUM exceeded/
 SELECT 1 FROM dual WHERE ('Alpha,Bravo,Charlie,Delta,Echo,Foxtrot,StrataCentral,Golf,Hotel,India,Juliet,Kilo,Lima,Mike,StrataL3,November,Oscar,StrataL2,Sand,P3,P4SwitchTest,Arsys,Poppa,ExtensionMgr,Arp,Quebec,Romeo,StrataApiV2,PtReyes,Sierra,SandAcl,Arrow,Artools,BridgeTest,Tango,SandT,PAlaska,Namespace,Agent,Qos,PatchPanel,ProjectReport,Ark,Gimp,Agent,SliceAgent,Arnet,Bgp,Ale,Tommy,Central,AsicPktTestLib,Hsc,SandL3,Abuild,Pca9555,Standby,ControllerDut,CalSys,SandLib,Sb820,PointV2,BfnLib,Evpn,BfnSdk,Sflow,ManagementActive,AutoTest,GatedTest,Bgp,Sand,xinetd,BfnAgentLib,bf-utils,Hello,BfnState,Eos,Artest,Qos,Scd,ThermoMgr,Uniform,EosUtils,Eb,FanController,Central,BfnL3,BfnL2,tcp_wrappers,Victor,Environment,Route,Failover,Whiskey,Xray,Gimp,BfnFixed,Strata,SoCal,XApi,Msrp,XpProfile,tcpdump,PatchPanel,ArosTest,FhTest,Arbus,XpAcl,MacConc,XpApi,telnet,QosTest,Alpha2,BfnVlan,Stp,VxlanControllerTest,MplsAgent,Bravo2,Lanz,BfnMbb,Intf,XCtrl,Unicast,SandTunnel,L3Unicast,Ipsec,MplsTest,Rsvp,EthIntf,StageMgr,Sol,MplsUtils,Nat,Ira,P4NamespaceDut,Counters,Charlie2,Aqlc,Mlag,Power,OpenFlow,Lag,RestApi,BfdTest,strongs,Sfa,CEosUtils,Adt746,MaintenanceMode,MlagDut,EosImage,IpEth,MultiProtocol,Launcher,Max3179,Snmp,Acl,IpEthTest,PhyEee,bf-syslibs,tacc,XpL2,p4-ar-switch,p4-bf-switch,LdpTest,BfnPhy,Mirroring,Phy6,Ptp'  REGEXP '^((?!\b(Strata|StrataApi|StrataApiV2)\b).)*$');
+
+#
+# MDEV-13173 An RLIKE that previously worked on 10.0 now returns "Got error 'pcre_exec: recursion limit of 100 exceeded' from regexp"
+#
+SELECT CONCAT(REPEAT('100,',500),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$';
+--replace_regex /[0-9]+ exceeded/NUM exceeded/
+SELECT CONCAT(REPEAT('100,',600),'101') RLIKE '^(([1-9][0-9]*),)*[1-9][0-9]*$';
+
+SELECT REGEXP_INSTR(CONCAT(REPEAT('100,',500),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$');
+--replace_regex /[0-9]+ exceeded/NUM exceeded/
+SELECT REGEXP_INSTR(CONCAT(REPEAT('100,',600),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$');
+
+SELECT LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'));
+--replace_regex /[0-9]+ exceeded/NUM exceeded/
+SELECT LENGTH(REGEXP_SUBSTR(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$'));
+
+SELECT LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',500/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''));
+--replace_regex /[0-9]+ exceeded/NUM exceeded/
+SELECT LENGTH(REGEXP_REPLACE(CONCAT(REPEAT('100,',600/3),'101'), '^(([1-9][0-9]*),)*[1-9][0-9]*$', ''));

sql/item_cmpfunc.cc

@@ -5110,7 +5110,7 @@ void Regexp_processor_pcre::set_recursion_limit(THD *thd)
   DBUG_ASSERT(thd == current_thd);
   stack_used= available_stack_size(thd->thread_stack, &stack_used);
   m_pcre_extra.match_limit_recursion=
-    (my_thread_stack_size - stack_used)/my_pcre_frame_size;
+    (my_thread_stack_size - STACK_MIN_SIZE - stack_used)/my_pcre_frame_size;
 }
 
 
@@ -5372,6 +5372,12 @@ void Regexp_processor_pcre::fix_owner(Item_func *owner,
 }
 
 
+bool Item_func_regex::fix_fields(THD *thd, Item **ref)
+{
+  re.set_recursion_limit(thd);
+  return Item_bool_func::fix_fields(thd, ref);
+}
+
 void
 Item_func_regex::fix_length_and_dec()
 {
@@ -5398,6 +5404,13 @@ longlong Item_func_regex::val_int()
 }
 
 
+bool Item_func_regexp_instr::fix_fields(THD *thd, Item **ref)
+{
+  re.set_recursion_limit(thd);
+  return Item_int_func::fix_fields(thd, ref);
+}
+
+
 void
 Item_func_regexp_instr::fix_length_and_dec()
 {

sql/item_cmpfunc.h

@@ -1652,6 +1652,7 @@ public:
     DBUG_VOID_RETURN;
   }
   longlong val_int();
+  bool fix_fields(THD *thd, Item **ref);
   void fix_length_and_dec();
   const char *func_name() const { return "regexp"; }
 
@@ -1679,6 +1680,7 @@ public:
     DBUG_VOID_RETURN;
   }
   longlong val_int();
+  bool fix_fields(THD *thd, Item **ref);
   void fix_length_and_dec();
   const char *func_name() const { return "regexp_instr"; }
 };

sql/item_strfunc.cc

@@ -1343,6 +1343,13 @@ void Item_func_replace::fix_length_and_dec()
 
 
 /*********************************************************************/
+bool Item_func_regexp_replace::fix_fields(THD *thd, Item **ref)
+{
+  re.set_recursion_limit(thd);
+  return Item_str_func::fix_fields(thd, ref);
+}
+
+
 void Item_func_regexp_replace::fix_length_and_dec()
 {
   if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 3))
@@ -1478,6 +1485,13 @@ err:
 }
 
 
+bool Item_func_regexp_substr::fix_fields(THD *thd, Item **ref)
+{
+  re.set_recursion_limit(thd);
+  return Item_str_func::fix_fields(thd, ref);
+}
+
+
 void Item_func_regexp_substr::fix_length_and_dec()
 {
   if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 2))

sql/item_strfunc.h

@@ -231,6 +231,7 @@ public:
     DBUG_VOID_RETURN;
   }
   String *val_str(String *str);
+  bool fix_fields(THD *thd, Item **ref);
   void fix_length_and_dec();
   const char *func_name() const { return "regexp_replace"; }
 };
@@ -251,6 +252,7 @@ public:
     DBUG_VOID_RETURN;
   }
   String *val_str(String *str);
+  bool fix_fields(THD *thd, Item **ref);
   void fix_length_and_dec();
   const char *func_name() const { return "regexp_substr"; }
 };

sql/mysqld.cc

@@ -3451,7 +3451,6 @@ sizeof(load_default_groups)/sizeof(load_default_groups[0]);
 #endif
 
 
-#ifndef EMBEDDED_LIBRARY
 /**
   This function is used to check for stack overrun for pathological
   cases of  regular expressions and 'like' expressions.
@@ -3480,8 +3479,6 @@ check_enough_stack_size(int recurse_level)
     return 0;
   return check_enough_stack_size_slow();
 }
-#endif
-
 
 
 /*
@@ -3503,11 +3500,12 @@ static void init_pcre()
 {
   pcre_malloc= pcre_stack_malloc= my_str_malloc_mysqld;
   pcre_free= pcre_stack_free= my_str_free_mysqld;
-#ifndef EMBEDDED_LIBRARY
   pcre_stack_guard= check_enough_stack_size_slow;
   /* See http://pcre.org/original/doc/html/pcrestack.html */
-  my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16;
-#endif
+  my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0);
+  // pcre can underestimate its stack usage. Use a safe value, as in the manual
+  set_if_bigger(my_pcre_frame_size, 500);
+  my_pcre_frame_size += 16; // Again, safety margin, see the manual
 }