Browse Source

Regenerate tokenizer data

pull/1239/head
Nikita Popov 11 years ago
parent
commit
a5f2be38d4
  1. 2
      ext/tokenizer/tokenizer_data.c

2
ext/tokenizer/tokenizer_data.c

@ -40,6 +40,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_PRINT", T_PRINT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_YIELD_FROM", T_YIELD_FROM, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_POW_EQUAL", T_POW_EQUAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT);
@ -179,6 +180,7 @@ char *get_token_type_name(int token_type)
case T_PRINT: return "T_PRINT";
case T_YIELD: return "T_YIELD";
case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
case T_YIELD_FROM: return "T_YIELD_FROM";
case T_POW_EQUAL: return "T_POW_EQUAL";
case T_SR_EQUAL: return "T_SR_EQUAL";
case T_SL_EQUAL: return "T_SL_EQUAL";

Loading…
Cancel
Save