MDEV-33281 Implement optimizer hints

- Using Lex_ident_sys to scan identifiers, like the SQL parser does.

  This fixes handling of double-quote-delimited and backtick-delimited identifiers,
  as well as handling of non-ASCII identifiers.

  Unescaping and converting from the client character set to the system
  character set is now done using Lex_ident_cli_st and Lex_ident_sys,
  like it's done in the SQL tokenizer/parser.
  Adding helper methods to_ident_cli() and to_ident_sys()
  in Optimizer_hint_parser::Token.

- Fixing the hint parser to report a syntax error when an empty identifiers:
    SELECT /*+ BKA(``) */ * FROM t1;

- Moving a part of the code from opt_hints_parser.h to opt_hints_parser.cc

  Moving these method definitions:
  - Optimizer_hint_tokenizer::find_keyword()
  - Optimizer_hint_tokenizer::get_token()

  to avoid huge pieces of the code in the header file.

- A Lex_ident_cli_st cleanup
  Fixing a few Lex_ident_cli_st methods to return Lex_ident_cli_st &
  instead of void, to use them easier in the caller code.

- Fixing the hint parser to display the correct line number

  Adding a new data type Lex_comment_st
  (a combination of LEX_CSTRING and a line number)
  Using it in sql_yacc.yy

- Getting rid of redundant dependencies on sql_hints_parser.h

  Moving void LEX::resolve_optimizer_hints() from sql_lex.h to sql_lex.cc

  Adding a class Optimizer_hint_parser_output, deriving from
  Optimizer_hint_parser::Hint_list. Fixing the hint parser to
  return a pointer to an allocated instance of Optimizer_hint_parser_output
  rather than an instance of Optimizer_hint_parser::Hint_list.
  This allows to use a forward declaration of Optimizer_hint_parser_output
  in sql_lex.h and thus avoid dependencies on sql_hints_parser.h.

mysql-test/main/opt_hints.result

@@ -1215,7 +1215,7 @@ EXPLAIN EXTENDED SELECT /*+ QB_NAME(``) */ 1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 Warnings:
-Warning	1064	Optimizer hint syntax error near '`) */ 1' at line 1
+Warning	1064	Optimizer hint syntax error near '``) */ 1' at line 1
 Note	1003	select 1 AS `1`
 SET NAMES utf8;
 EXPLAIN EXTENDED SELECT /*+ QB_NAME(`\BF``\BF`) */ 1;

sql/lex_ident_cli.h

@@ -85,21 +85,26 @@ public:
 struct Lex_ident_cli_st: public Lex_string_with_metadata_st
 {
 public:
-  void set_keyword(const char *s, size_t len)
+  Lex_ident_cli_st & set_keyword(const char *s, size_t len)
   {
     set(s, len, false, '\0');
+    return *this;
   }
-  void set_ident(const char *s, size_t len, bool is_8bit)
+  Lex_ident_cli_st & set_ident(const char *s, size_t len, bool is_8bit)
   {
     set(s, len, is_8bit, '\0');
+    return *this;
   }
-  void set_ident_quoted(const char *s, size_t len, bool is_8bit, char quote)
+  Lex_ident_cli_st & set_ident_quoted(const char *s, size_t len,
+                                      bool is_8bit, char quote)
   {
     set(s, len, is_8bit, quote);
+    return *this;
   }
-  void set_unquoted(const LEX_CSTRING *s, bool is_8bit)
+  Lex_ident_cli_st & set_unquoted(const LEX_CSTRING *s, bool is_8bit)
   {
     set(s, is_8bit, '\0');
+    return *this;
   }
   const char *pos() const { return str - is_quoted(); }
   const char *end() const { return str + length + is_quoted(); }

sql/opt_hints.cc

@@ -18,6 +18,7 @@
 #include "sql_lex.h"
 #include "sql_select.h"
 #include "opt_hints.h"
+#include "opt_hints_parser.h"
 
 /**
   Information about hints. Sould be
@@ -70,11 +71,13 @@ static int cmp_lex_string(const LEX_CSTRING *s,
 }
 
 
+static const Lex_ident_sys null_ident_sys;
+
 static void print_warn(THD *thd, uint err_code, opt_hints_enum hint_type,
                 bool hint_state,
-                const LEX_CSTRING *qb_name_arg,
-                const LEX_CSTRING *table_name_arg,
-                const LEX_CSTRING *key_name_arg/*,                PT_hint *hint*/)
+                const Lex_ident_sys *qb_name_arg,
+                const Lex_ident_sys *table_name_arg,
+                const Lex_ident_sys *key_name_arg/*,        PT_hint *hint*/)
 {
   String str;
 
@@ -187,11 +190,11 @@ static Opt_hints_qb *get_qb_hints(Parse_context *pc)
 */
 
 static Opt_hints_qb *find_qb_hints(Parse_context *pc,
-                                   const LEX_CSTRING *qb_name,
+                                   const Lex_ident_sys &qb_name,
                                    opt_hints_enum hint_type,
                                    bool hint_state)
 {
-  if (qb_name->length == 0) // no QB NAME is used
+  if (qb_name.length == 0) // no QB NAME is used
     return pc->select->opt_hints_qb;
 
   Opt_hints_qb *qb= static_cast<Opt_hints_qb *>
@@ -200,7 +203,7 @@ static Opt_hints_qb *find_qb_hints(Parse_context *pc,
   if (qb == NULL)
   {
     print_warn(pc->thd, ER_WARN_UNKNOWN_QB_NAME, hint_type, hint_state,
-               qb_name, NULL, NULL);
+               &qb_name, NULL, NULL);
   }
   return qb;
 }
@@ -219,7 +222,7 @@ static Opt_hints_qb *find_qb_hints(Parse_context *pc,
 */
 
 static Opt_hints_table *get_table_hints(Parse_context *pc,
-                                        const LEX_CSTRING *table_name,
+                                        const Lex_ident_sys &table_name,
                                         Opt_hints_qb *qb)
 {
   Opt_hints_table *tab=
@@ -248,12 +251,12 @@ bool Opt_hints::get_switch(opt_hints_enum type_arg) const
 }
 
 
-Opt_hints* Opt_hints::find_by_name(const LEX_CSTRING *name_arg) const
+Opt_hints* Opt_hints::find_by_name(const LEX_CSTRING &name_arg) const
 {
   for (uint i= 0; i < child_array.size(); i++)
   {
     const LEX_CSTRING *name= child_array[i]->get_name();
-    if (name && !cmp_lex_string(name, name_arg))
+    if (name && !cmp_lex_string(name, &name_arg))
       return child_array[i];
   }
   return NULL;
@@ -334,7 +337,7 @@ PT_hint *Opt_hints_global::get_complex_hints(uint type)
 Opt_hints_qb::Opt_hints_qb(Opt_hints *opt_hints_arg,
                            MEM_ROOT *mem_root_arg,
                            uint select_number_arg)
-  : Opt_hints(NULL, opt_hints_arg, mem_root_arg),
+  : Opt_hints(Lex_ident_sys(), opt_hints_arg, mem_root_arg),
     select_number(select_number_arg)
 {
   sys_name.str= buff;
@@ -344,7 +347,7 @@ Opt_hints_qb::Opt_hints_qb(Opt_hints *opt_hints_arg,
 
 
 Opt_hints_table *Opt_hints_qb::adjust_table_hints(TABLE *table,
-                                                  const LEX_CSTRING *alias)
+                                                  const Lex_ident_table &alias)
 {
   Opt_hints_table *tab= static_cast<Opt_hints_table *>(find_by_name(alias));
 
@@ -531,8 +534,8 @@ bool Optimizer_hint_parser::Table_level_hint::resolve(Parse_context *pc) const
        at_query_block_name_opt_table_name_list= *this)
   {
     // this is @ query_block_name opt_table_name_list
-    const Query_block_name &qb_name= *this;
-    Opt_hints_qb *qb= find_qb_hints(pc, &qb_name, hint_type, hint_state);
+    const Lex_ident_sys qb_name_sys= Query_block_name::to_ident_sys(pc->thd);
+    Opt_hints_qb *qb= find_qb_hints(pc, qb_name_sys, hint_type, hint_state);
     if (qb == NULL)
       return false;
     if (at_query_block_name_opt_table_name_list.is_empty())
@@ -540,7 +543,7 @@ bool Optimizer_hint_parser::Table_level_hint::resolve(Parse_context *pc) const
       // e.g. BKA(@qb1)
       if (qb->set_switch(hint_state, hint_type, false))
         print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                   &qb_name, NULL, NULL/*, this*/);
+                   &qb_name_sys, NULL, NULL/*, this*/);
       return false;
     }
     else
@@ -549,12 +552,13 @@ bool Optimizer_hint_parser::Table_level_hint::resolve(Parse_context *pc) const
       const Opt_table_name_list &opt_table_name_list= *this;
       for (const Table_name &table : opt_table_name_list)
       {
-        Opt_hints_table *tab= get_table_hints(pc, &table, qb);
+        const Lex_ident_sys table_name_sys= table.to_ident_sys(pc->thd);
+        Opt_hints_table *tab= get_table_hints(pc, table_name_sys, qb);
         if (!tab)
           return true; // OLEGS: why no warning?
         if (tab->set_switch(hint_state, hint_type, true))
           print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                     &qb_name, &table, NULL/*, this*/);
+                     &qb_name_sys, &table_name_sys, NULL/*, this*/);
       }
     }
   }
@@ -563,7 +567,7 @@ bool Optimizer_hint_parser::Table_level_hint::resolve(Parse_context *pc) const
     // this is opt_hint_param_table_list
     const Opt_table_name_list &table_name_list= *this;
     const Opt_hint_param_table_list &opt_hint_param_table_list= *this;
-    Opt_hints_qb *qb= find_qb_hints(pc, &null_clex_str, hint_type, hint_state);
+    Opt_hints_qb *qb= find_qb_hints(pc, Lex_ident_sys(), hint_type, hint_state);
     if (qb == NULL)
       return false;
     if (table_name_list.is_empty() && opt_hint_param_table_list.is_empty())
@@ -571,36 +575,39 @@ bool Optimizer_hint_parser::Table_level_hint::resolve(Parse_context *pc) const
       // e.g. BKA()
       if (qb->set_switch(hint_state, hint_type, false))
         print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                   &null_clex_str, NULL, NULL/*, this*/);
+                   &null_ident_sys, NULL, NULL/*, this*/);
       return false;
     }
     for (const Table_name &table : table_name_list)
     {
       // e.g. BKA(t1, t2)
-      Opt_hints_table *tab= get_table_hints(pc, &table, qb);
+      const Lex_ident_sys table_name_sys= table.to_ident_sys(pc->thd);
+      Opt_hints_table *tab= get_table_hints(pc, table_name_sys, qb);
       if (!tab)
         return true; // OLEGS: no warning?
       if (tab->set_switch(hint_state, hint_type, true))
         print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                   &null_clex_str, &table, NULL/*, this*/);
+                   &null_ident_sys, &table_name_sys, NULL/*, this*/);
     }
   
     for (const Hint_param_table &table : opt_hint_param_table_list)
     {
       // e.g. BKA(t1@qb1, t2@qb2)
-      const Query_block_name &qb_name= table;
-      const Table_name &table_name= table;
-      Opt_hints_qb *qb= find_qb_hints(pc, &qb_name, hint_type, hint_state);
+      const Lex_ident_sys qb_name_sys= table.Query_block_name::
+                                         to_ident_sys(pc->thd);
+      Opt_hints_qb *qb= find_qb_hints(pc, qb_name_sys, hint_type, hint_state);
       if (qb == NULL)
         return false;
       // OLEGS: todo
-      Opt_hints_table *tab= get_table_hints(pc, &table_name, qb);
+      const Lex_ident_sys table_name_sys= table.Table_name::
+                                            to_ident_sys(pc->thd);
+      Opt_hints_table *tab= get_table_hints(pc, table_name_sys, qb);
       if (!tab)
         return true; // OLEGS: why no warning?
   
       if (tab->set_switch(hint_state, hint_type, true))
          print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                    &qb_name, &table_name, NULL/*, this*/);
+                    &qb_name_sys, &table_name_sys, NULL/*, this*/);
     }
   }
   return false;
@@ -637,14 +644,15 @@ bool Optimizer_hint_parser::Index_level_hint::resolve(Parse_context *pc) const
   }
   
   const Hint_param_table_ext &table_ext= *this;
-  const Query_block_name &qb_name= table_ext;
-  const Table_name &table_name= table_ext;
-  
-  Opt_hints_qb *qb= find_qb_hints(pc, &qb_name, hint_type, hint_state);
+  const Lex_ident_sys qb_name_sys= table_ext.Query_block_name::
+                                     to_ident_sys(pc->thd);
+  const Lex_ident_sys table_name_sys= table_ext.Table_name::
+                                        to_ident_sys(pc->thd);
+  Opt_hints_qb *qb= find_qb_hints(pc, qb_name_sys, hint_type, hint_state);
   if (qb == NULL)
     return false;
 
-  Opt_hints_table *tab= get_table_hints(pc, &table_name, qb);
+  Opt_hints_table *tab= get_table_hints(pc, table_name_sys, qb);
   if (!tab)
     return true; // OLEGS: why no warning?
   
@@ -652,22 +660,23 @@ bool Optimizer_hint_parser::Index_level_hint::resolve(Parse_context *pc) const
   {
     if (tab->set_switch(hint_state, hint_type, false))
       print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                 &qb_name, &table_name, NULL/*, this*/);
+                 &qb_name_sys, &table_name_sys, NULL/*, this*/);
     return false;
   }
 
   for (const Hint_param_index &index_name : *this)
   {
-    Opt_hints_key *idx= (Opt_hints_key *)tab->find_by_name(&index_name);
+    const Lex_ident_sys index_name_sys= index_name.to_ident_sys(pc->thd);
+    Opt_hints_key *idx= (Opt_hints_key *)tab->find_by_name(index_name_sys);
     if (!idx)
     {
-      idx= new Opt_hints_key(&index_name, tab, pc->thd->mem_root);
+      idx= new Opt_hints_key(index_name_sys, tab, pc->thd->mem_root);
       tab->register_child(idx);
     }
 
     if (idx->set_switch(hint_state, hint_type, true))
       print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, hint_type, hint_state,
-                 &qb_name, &table_name, &index_name/*, this*/);
+                 &qb_name_sys, &table_name_sys, &index_name_sys/*, this*/);
   }
 
   return false;
@@ -680,17 +689,17 @@ bool Optimizer_hint_parser::Qb_name_hint::resolve(Parse_context *pc) const
 
   DBUG_ASSERT(qb);
 
-  const Query_block_name &qb_name= *this;
+  const Lex_ident_sys qb_name_sys= Query_block_name::to_ident_sys(pc->thd);
 
-  if (qb->get_name() ||                         // QB name is already set
-      qb->get_parent()->find_by_name(&qb_name)) // Name is already used
+  if (qb->get_name() ||                            // QB name is already set
+      qb->get_parent()->find_by_name(qb_name_sys)) // Name is already used
   {
     print_warn(pc->thd, ER_WARN_CONFLICTING_HINT, QB_NAME_HINT_ENUM, false,
                NULL, NULL, NULL/*, this*/);
     return false;
   }
 
-  qb->set_name(&qb_name);
+  qb->set_name(qb_name_sys);
   return false;
 }
 

sql/opt_hints.h

@@ -145,7 +145,7 @@ class Opt_hints : public Sql_alloc
     table name for table level and key name
     for key level.
   */
-  const LEX_CSTRING *name;
+  Lex_ident_sys name;
   /*
     Parent object. There is no parent for global level,
     for query block level parent is Opt_hints_global object,
@@ -165,7 +165,7 @@ class Opt_hints : public Sql_alloc
 
 public:
 
-  Opt_hints(const LEX_CSTRING *name_arg,
+  Opt_hints(const Lex_ident_sys &name_arg,
             Opt_hints *parent_arg,
             MEM_ROOT *mem_root_arg)
     : name(name_arg), parent(parent_arg), child_array(mem_root_arg),
@@ -209,8 +209,11 @@ public:
   */
   bool get_switch(opt_hints_enum type_arg) const;
 
-  virtual const LEX_CSTRING *get_name() const { return name; }
-  void set_name(const LEX_CSTRING *name_arg) { name= name_arg; }
+  virtual const LEX_CSTRING *get_name() const
+  {
+    return name.str ? &name : nullptr;
+  }
+  void set_name(const Lex_ident_sys &name_arg) { name= name_arg; }
   Opt_hints *get_parent() const { return parent; }
   void set_resolved() { resolved= true; }
   bool is_resolved() const { return resolved; }
@@ -249,7 +252,7 @@ public:
     @return  hint if found,
              NULL otherwise
   */
-  Opt_hints *find_by_name(const LEX_CSTRING *name_arg) const;
+  Opt_hints *find_by_name(const LEX_CSTRING &name_arg) const;
   /**
     Print all hints except of QB_NAME hint.
 
@@ -303,7 +306,7 @@ public:
   PT_hint_max_execution_time *max_exec_time;
  
   Opt_hints_global(MEM_ROOT *mem_root_arg)
-    : Opt_hints(NULL, NULL, mem_root_arg)
+    : Opt_hints(Lex_ident_sys(), NULL, mem_root_arg)
   {
     max_exec_time= NULL;
   }
@@ -375,7 +378,8 @@ public:
     @return  pointer Opt_hints_table object if this object is found,
              NULL otherwise.
   */
-  Opt_hints_table *adjust_table_hints(TABLE *table, const LEX_CSTRING *alias);
+  Opt_hints_table *adjust_table_hints(TABLE *table,
+                                      const Lex_ident_table &alias);
 };
 
 
@@ -388,7 +392,7 @@ class Opt_hints_table : public Opt_hints
 public:
   Mem_root_array<Opt_hints_key*, true> keyinfo_array;
 
-  Opt_hints_table(const LEX_CSTRING *table_name_arg,
+  Opt_hints_table(const Lex_ident_sys &table_name_arg,
                   Opt_hints_qb *qb_hints_arg,
                   MEM_ROOT *mem_root_arg)
     : Opt_hints(table_name_arg, qb_hints_arg, mem_root_arg),
@@ -429,7 +433,7 @@ class Opt_hints_key : public Opt_hints
 {
 public:
 
-  Opt_hints_key(const LEX_CSTRING *key_name_arg,
+  Opt_hints_key(const Lex_ident_sys &key_name_arg,
                 Opt_hints_table *table_hints_arg,
                 MEM_ROOT *mem_root_arg)
     : Opt_hints(key_name_arg, table_hints_arg, mem_root_arg)

sql/opt_hints_parser.cc

@@ -33,6 +33,85 @@ Parse_context::Parse_context(THD *thd, st_select_lex *select)
 {}
 
 
+Optimizer_hint_tokenizer::TokenID
+Optimizer_hint_tokenizer::find_keyword(const LEX_CSTRING &str)
+{
+  switch (str.length)
+  {
+  case 3:
+    if ("BKA"_Lex_ident_column.streq(str)) return TokenID::keyword_BKA;
+    if ("BNL"_Lex_ident_column.streq(str)) return TokenID::keyword_BNL;
+    if ("MRR"_Lex_ident_column.streq(str)) return TokenID::keyword_MRR;
+    break;
+
+  case 6:
+    if ("NO_BKA"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_BKA;
+    if ("NO_BNL"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_BNL;
+    if ("NO_ICP"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_ICP;
+    if ("NO_MRR"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_MRR;
+    break;
+
+  case 7:
+    if ("QB_NAME"_Lex_ident_column.streq(str))
+      return TokenID::keyword_QB_NAME;
+    break;
+
+  case 21:
+    if ("NO_RANGE_OPTIMIZATION"_Lex_ident_column.streq(str))
+      return TokenID::keyword_NO_RANGE_OPTIMIZATION;
+    break;
+  }
+  return TokenID::tIDENT;
+}
+
+
+Optimizer_hint_tokenizer::Token
+Optimizer_hint_tokenizer::get_token(CHARSET_INFO *cs)
+{
+  get_spaces();
+  if (eof())
+    return Token(Lex_cstring(m_ptr, m_ptr), TokenID::tEOF);
+  const char head= m_ptr[0];
+  if (head == '`' || head=='"')
+  {
+    const Token_with_metadata delimited_ident= get_quoted_string();
+    /*
+      Consider only non-empty quoted strings as identifiers.
+      Table and index names cannot be empty in MariaDB.
+      Let's also disallow empty query block names.
+      Note, table aliases can actually be empty:
+        SELECT ``.a FROM t1 ``;
+      But let's disallow them in hints for simplicity, to handle
+      all identifiers in the same way in the hint parser.
+    */
+    if (delimited_ident.length > 2)
+      return Token(delimited_ident, TokenID::tIDENT);
+    /*
+      If the string is empty, "unget" it to have a good
+      syntax error position in the message text.
+      The point is to include the empty string in the error message:
+        EXPLAIN EXTENDED SELECT ... QB_NAME(``) ...;  -->
+        Optimizer hint syntax error near '``) ...' at line 1
+    */
+    m_ptr-= delimited_ident.length;
+    return Token(Lex_cstring(m_ptr, m_ptr), TokenID::tNULL);
+  }
+  const Token_with_metadata ident= get_ident();
+  if (ident.length)
+    return Token(ident, ident.m_extended_chars ?
+                 TokenID::tIDENT : find_keyword(ident));
+  if (!get_char(','))
+    return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tCOMMA);
+  if (!get_char('@'))
+    return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tAT);
+  if (!get_char('('))
+    return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tLPAREN);
+  if (!get_char(')'))
+    return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tRPAREN);
+  return Token(Lex_cstring(m_ptr, m_ptr), TokenID::tNULL);
+}
+
+
 // This method is for debug purposes
 bool Optimizer_hint_parser::parse_token_list(THD *thd)
 {
@@ -52,15 +131,22 @@ bool Optimizer_hint_parser::parse_token_list(THD *thd)
   return true; // Success
 }
 
-
-void Optimizer_hint_parser::push_warning_syntax_error(THD *thd)
+void Optimizer_hint_parser::push_warning_syntax_error(THD *thd,
+                                                      uint start_lineno)
 {
+  DBUG_ASSERT(m_start <= m_ptr);
+  DBUG_ASSERT(m_ptr <= m_end);
   const char *msg= ER_THD(thd, ER_WARN_OPTIMIZER_HINT_SYNTAX_ERROR);
   ErrConvString txt(m_look_ahead_token.str, strlen(m_look_ahead_token.str),
                     thd->variables.character_set_client);
+  /*
+    start_lineno is the line number on which the whole hint started.
+    Add the line number of the current tokenizer position inside the hint
+    (in case hints are written in multiple lines).
+  */
   push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
                       ER_PARSE_ERROR, ER_THD(thd, ER_PARSE_ERROR),
-                      msg, txt.ptr(), 1);
+                      msg, txt.ptr(), start_lineno + lineno());
 }
 
 

sql/opt_hints_parser.h

@@ -18,7 +18,7 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA
 */
 
-
+#include "lex_ident_sys.h"
 #include "simple_tokenizer.h"
 #include "sql_list.h"
 #include "simple_parser.h"
@@ -76,40 +76,6 @@ public:
     tIDENT
   };
 
-protected:
-
-  TokenID find_keyword(const LEX_CSTRING &str)
-  {
-    switch (str.length)
-    {
-    case 3:
-      if ("BKA"_Lex_ident_column.streq(str)) return TokenID::keyword_BKA;
-      if ("BNL"_Lex_ident_column.streq(str)) return TokenID::keyword_BNL;
-      if ("MRR"_Lex_ident_column.streq(str)) return TokenID::keyword_MRR;
-      break;
-
-    case 6:
-      if ("NO_BKA"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_BKA;
-      if ("NO_BNL"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_BNL;
-      if ("NO_ICP"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_ICP;
-      if ("NO_MRR"_Lex_ident_column.streq(str)) return TokenID::keyword_NO_MRR;
-      break;
-
-    case 7:
-      if ("QB_NAME"_Lex_ident_column.streq(str))
-        return TokenID::keyword_QB_NAME;
-      break;
-
-    case 21:
-      if ("NO_RANGE_OPTIMIZATION"_Lex_ident_column.streq(str))
-        return TokenID::keyword_NO_RANGE_OPTIMIZATION;
-      break;
-    }
-    return TokenID::tIDENT;
-  }
-
-public:
-
   class Token: public Lex_cstring
   {
   protected:
@@ -132,32 +98,9 @@ public:
     }
   };
 
-  Token get_token(CHARSET_INFO *cs)
-  {
-    get_spaces();
-    if (eof())
-      return Token(Lex_cstring(m_ptr, m_ptr), TokenID::tEOF);
-    const char head= m_ptr[0];
-    if (head == '`' || head=='"')
-    {
-      const Token_with_metadata delimited_ident= get_quoted_string();
-      if (delimited_ident.length)
-        return Token(delimited_ident, TokenID::tIDENT);
-    }
-    const Token_with_metadata ident= get_ident();
-    if (ident.length)
-      return Token(ident, ident.m_extended_chars ?
-                   TokenID::tIDENT : find_keyword(ident));
-    if (!get_char(','))
-      return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tCOMMA);
-    if (!get_char('@'))
-      return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tAT);
-    if (!get_char('('))
-      return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tLPAREN);
-    if (!get_char(')'))
-      return Token(Lex_cstring(m_ptr - 1, 1), TokenID::tRPAREN);
-    return Token(Lex_cstring(m_ptr, m_ptr), TokenID::tNULL);
-  }
+protected:
+  Token get_token(CHARSET_INFO *cs);
+  static TokenID find_keyword(const LEX_CSTRING &str);
 };
 
 
@@ -167,6 +110,7 @@ class Optimizer_hint_parser: public Optimizer_hint_tokenizer,
 private:
   Token m_look_ahead_token;
   THD *m_thd;
+  const char *m_start;
   bool m_syntax_error;
   bool m_fatal_error;
 public:
@@ -174,6 +118,7 @@ public:
    :Optimizer_hint_tokenizer(cs, hint),
     m_look_ahead_token(get_token(cs)),
     m_thd(thd),
+    m_start(hint.str),
     m_syntax_error(false),
     m_fatal_error(false)
   { }
@@ -187,6 +132,24 @@ public:
     m_fatal_error= true;
     return false;
   }
+  // Calculate the line number inside the whole hint
+  uint lineno(const char *ptr) const
+  {
+    DBUG_ASSERT(m_start <= ptr);
+    DBUG_ASSERT(ptr <= m_end);
+    uint lineno= 0;
+    for ( ; ptr >= m_start; ptr--)
+    {
+      if (*ptr == '\n')
+        lineno++;
+    }
+    return lineno;
+  }
+  uint lineno() const
+  {
+    return lineno(m_ptr);
+  }
+
   TokenID look_ahead_token_id() const
   {
     return is_error() ? TokenID::tNULL : m_look_ahead_token.id();
@@ -249,7 +212,7 @@ public:
 
   bool parse_token_list(THD *thd); // For debug purposes
 
-  void push_warning_syntax_error(THD *thd);
+  void push_warning_syntax_error(THD *thd, uint lineno);
 
 
 private:
@@ -280,6 +243,18 @@ private:
   {
   public:
     using TOKEN::TOKEN;
+    Lex_ident_cli_st to_ident_cli() const
+    {
+      Lex_ident_cli_st cli;
+      if (length >= 2 && (str[0] == '`' || str[0] == '"'))
+        return cli.set_ident_quoted(str + 1, length - 2, true, str[0]);
+      return cli.set_ident(str, length, true);
+    }
+    Lex_ident_sys to_ident_sys(THD *thd) const
+    {
+      const Lex_ident_cli_st cli= to_ident_cli();
+      return Lex_ident_sys(thd, &cli);
+    }
   };
 
   class LParen: public TOKEN<PARSER, TokenID::tLPAREN>
@@ -627,4 +602,17 @@ public:
 
 };
 
+
+/*
+  This wrapper class is needed to use a forward declaration in sql_lex.h
+  instead of including the entire opt_hints_parser.h.
+  (forward declarations of qualified nested classes are not possible in C++)
+*/
+class Optimizer_hint_parser_output: public Optimizer_hint_parser::Hint_list
+{
+public:
+  using Hint_list::Hint_list;
+};
+
+
 #endif // OPT_HINTS_PARSER

sql/sql_base.cc

@@ -8323,7 +8323,7 @@ bool setup_tables(THD *thd, Name_resolution_context *context,
           !table_list->opt_hints_table)        // Table hints are not adjusted yet
       {
         table_list->opt_hints_table=
-            qb_hints->adjust_table_hints(table_list->table, &table_list->alias);
+            qb_hints->adjust_table_hints(table_list->table, table_list->alias);
       }
     }
     if (select_insert && !is_insert_tables_num_set)

sql/sql_lex.cc

@@ -2496,16 +2496,17 @@ int Lex_input_stream::lex_one_token(YYSTYPE *yylval, THD *thd)
       else
       {
         in_comment= PRESERVE_COMMENT;
-        yylval->lex_str.str= m_ptr;
+        yylval->lex_comment.lineno= yylineno;
+        yylval->lex_comment.str= m_ptr;
         yySkip();                  // Accept /
         yySkip();                  // Accept *
         /* regular comments can have zero comments inside. */
         if ((comment_closed= ! consume_comment(0)) && hint_comment)
         {
-          if (yylval->lex_str.str[2]=='+')
+          if (yylval->lex_comment.str[2] == '+')
           {
             next_state= MY_LEX_START;
-            yylval->lex_str.length= m_ptr - yylval->lex_str.str;
+            yylval->lex_comment.length= m_ptr - yylval->lex_comment.str;
             restore_in_comment_state();
             return HINT_COMMENT;
           }
@@ -12862,8 +12863,8 @@ bool SELECT_LEX_UNIT::is_derived_eliminated() const
   rc == nullptr  false            no hints, empty hints, hint parse error
   rc == nullptr  true             fatal error, such as EOM
 */
-Optimizer_hint_parser::Hint_list *
-LEX::parse_optimizer_hints(const LEX_CSTRING &hints_str)
+Optimizer_hint_parser_output *
+LEX::parse_optimizer_hints(const Lex_comment_st &hints_str)
 {
   DBUG_ASSERT(!hints_str.str || hints_str.length >= 5);
   if (!hints_str.str)
@@ -12875,7 +12876,7 @@ LEX::parse_optimizer_hints(const LEX_CSTRING &hints_str)
   Optimizer_hint_parser p(thd, thd->charset(),
                           Lex_cstring(hints_str.str + 3, hints_str.length - 5));
   // Parse hints
-  Optimizer_hint_parser::Hints hints(&p);
+  Optimizer_hint_parser_output hints(&p);
   DBUG_ASSERT(!p.is_error() || !hints);
 
   if (p.is_fatal_error())
@@ -12890,10 +12891,25 @@ LEX::parse_optimizer_hints(const LEX_CSTRING &hints_str)
 
   if (!hints) // Hint parsing failed with a syntax error
   {
-    p.push_warning_syntax_error(thd);
+    p.push_warning_syntax_error(thd, hints_str.lineno);
     return nullptr; // Continue and ignore hints.
   }
 
   // Hints were not empty and were parsed without errors
-  return new (thd->mem_root) Optimizer_hint_parser::Hint_list(std::move(hints));
+  return new (thd->mem_root) Optimizer_hint_parser_output(std::move(hints));
+}
+
+
+void LEX::resolve_optimizer_hints()
+{
+  SELECT_LEX *select_lex;
+  if (likely(select_stack_top))
+    select_lex= select_stack[select_stack_top - 1];
+  else
+    select_lex= nullptr;
+  if (select_lex && select_lex->parsed_optimizer_hints)
+  {
+    Parse_context pc(thd, select_lex);
+    select_lex->parsed_optimizer_hints->resolve(&pc);
+  }
 }

sql/sql_lex.h

@@ -40,7 +40,6 @@
 #include "table.h"
 #include "sql_class.h"                // enum enum_column_usage
 #include "select_handler.h"
-#include "opt_hints_parser.h"
 
 /* Used for flags of nesting constructs */
 #define SELECT_NESTING_MAP_SIZE 64
@@ -49,6 +48,17 @@ typedef Bitmap<SELECT_NESTING_MAP_SIZE> nesting_map;
 /* YACC and LEX Definitions */
 
 
+struct Lex_comment_st: public LEX_CSTRING
+{
+  uint lineno;
+  void init()
+  {
+    LEX_CSTRING::operator=({nullptr, 0});
+    lineno= 0;
+  }
+};
+
+
 struct Lex_column_list_privilege_st
 {
   List<Lex_ident_sys> *m_columns;
@@ -170,6 +180,7 @@ class select_handler;
 class Pushdown_select;
 class Opt_hints_global;
 class Opt_hints_qb;
+class Optimizer_hint_parser_output;
 
 #define ALLOC_ROOT_SET 1024
 
@@ -1262,7 +1273,7 @@ public:
   /* it is for correct printing SELECT options */
   thr_lock_type lock_type;  
 
-  Optimizer_hint_parser::Hint_list *parsed_optimizer_hints;
+  Optimizer_hint_parser_output *parsed_optimizer_hints;
 
   /** System Versioning */
   int vers_setup_conds(THD *thd, TABLE_LIST *tables);
@@ -1560,7 +1571,7 @@ public:
   bool is_unit_nest() { return (nest_flags & UNIT_NEST_FL); }
   void mark_as_unit_nest() { nest_flags= UNIT_NEST_FL; }
   bool is_sj_conversion_prohibited(THD *thd);
-  void set_optimizer_hints(Optimizer_hint_parser::Hint_list *hl)
+  void set_optimizer_hints(Optimizer_hint_parser_output *hl)
   { 
     parsed_optimizer_hints= hl;
   }
@@ -3736,19 +3747,7 @@ public:
     DBUG_RETURN(select_lex);
   }
 
-  void resolve_optimizer_hints()
-  {
-    SELECT_LEX *select_lex;
-    if (likely(select_stack_top))
-      select_lex= select_stack[select_stack_top - 1];
-    else
-      select_lex= nullptr;
-    if (select_lex && select_lex->parsed_optimizer_hints)
-    {
-      Parse_context pc(thd, select_lex);
-      select_lex->parsed_optimizer_hints->resolve(&pc);
-    }
-  }
+  void resolve_optimizer_hints();
 
   SELECT_LEX *current_select_or_default()
   {
@@ -4993,8 +4992,8 @@ public:
     return nullptr;
   }
 
-  Optimizer_hint_parser::Hint_list *
-    parse_optimizer_hints(const LEX_CSTRING &hint);
+  Optimizer_hint_parser_output *
+    parse_optimizer_hints(const Lex_comment_st &hint);
 };
 
 

sql/sql_yacc.yy

@@ -206,6 +206,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)()
 
   /* structs */
   LEX_CSTRING lex_str;
+  Lex_comment_st lex_comment;
   Lex_ident_cli_st kwd;
   Lex_ident_cli_st ident_cli;
   Lex_ident_sys_st ident_sys;
@@ -278,7 +279,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)()
   TABLE_LIST *table_list;
   Table_ident *table;
   Qualified_column_ident *qualified_column_ident;
-  Optimizer_hint_parser::Hint_list *opt_hints;
+  Optimizer_hint_parser_output *opt_hints;
   char *simple_string;
   const char *const_simple_string;
   chooser_compare_func_creator boolfunc2creator;
@@ -1323,6 +1324,9 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize);
 %left EMPTY_FROM_CLAUSE
 %right INTO
 
+%type <lex_comment>
+        HINT_COMMENT opt_hint_comment
+
 %type <lex_str>
         DECIMAL_NUM FLOAT_NUM NUM LONG_NUM
         HEX_NUM HEX_STRING
@@ -1333,7 +1337,6 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize);
         opt_constraint constraint opt_ident
         sp_block_label sp_control_label opt_place opt_db
         udt_name
-        HINT_COMMENT opt_hint_comment
 
 %type <ident_sys>
         IDENT_sys
@@ -8959,7 +8962,7 @@ table_value_constructor:
 	;
 
 opt_hint_comment:
-          /*empty */   { $$= null_clex_str; }
+          /*empty */   { $$.init(); }
         | HINT_COMMENT { $$= $1; }
         ;