Use ZSTR_CHAR in token_get_all()

ext/tokenizer/tokenizer.c

@@ -106,12 +106,29 @@ PHP_MINFO_FUNCTION(tokenizer)
 }
 /* }}} */
 
+static void add_token(zval *return_value, int token_type,
+		unsigned char *text, size_t leng, int lineno) {
+	if (token_type >= 256) {
+		zval keyword;
+		array_init(&keyword);
+		add_next_index_long(&keyword, token_type);
+		add_next_index_stringl(&keyword, (char *) text, leng);
+		add_next_index_long(&keyword, lineno);
+		add_next_index_zval(return_value, &keyword);
+	} else {
+		if (leng == 1) {
+			add_next_index_str(return_value, ZSTR_CHAR(text[0]));
+		} else {
+			add_next_index_stringl(return_value, (char *) text, leng);
+		}
+	}
+}
+
 static zend_bool tokenize(zval *return_value, zend_string *source)
 {
 	zval source_zval;
 	zend_lex_state original_lex_state;
 	zval token;
-	zval keyword;
 	int token_type;
 	int token_line = 1;
 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
@@ -129,15 +146,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 
 	ZVAL_UNDEF(&token);
 	while ((token_type = lex_scan(&token))) {
-		if (token_type >= 256) {
-			array_init(&keyword);
-			add_next_index_long(&keyword, token_type);
-			add_next_index_stringl(&keyword, (char *)zendtext, zendleng);
-			add_next_index_long(&keyword, token_line);
-			add_next_index_zval(return_value, &keyword);
-		} else {
-			add_next_index_stringl(return_value, (char *)zendtext, zendleng);
-		}
+		add_token(return_value, token_type, zendtext, zendleng, token_line);
 
 		if (Z_TYPE(token) != IS_UNDEF) {
 			zval_dtor(&token);
@@ -152,11 +161,8 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 			) {
 				/* fetch the rest into a T_INLINE_HTML */
 				if (zendcursor != zendlimit) {
-					array_init(&keyword);
-					add_next_index_long(&keyword, T_INLINE_HTML);
-					add_next_index_stringl(&keyword, (char *)zendcursor, zendlimit - zendcursor);
-					add_next_index_long(&keyword, token_line);
-					add_next_index_zval(return_value, &keyword);
+					add_token(return_value, T_INLINE_HTML,
+						zendcursor, zendlimit - zendcursor, token_line);
 				}
 				break;
 			}
@@ -181,22 +187,13 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 void on_event(zend_php_scanner_event event, int token, int line, void *context)
 {
 	zval *token_stream = (zval *) context;
-	zval keyword;
 	HashTable *tokens_ht;
 	zval *token_zv;
 
 	switch (event) {
 		case ON_TOKEN:
 			if (token == END) break;
-			if (token >= 256) {
-				array_init(&keyword);
-				add_next_index_long(&keyword, token);
-				add_next_index_stringl(&keyword, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
-				add_next_index_long(&keyword, line);
-				add_next_index_zval(token_stream, &keyword);
-			} else {
-				add_next_index_stringl(token_stream, (char *)LANG_SCNG(yy_text), LANG_SCNG(yy_leng));
-			}
+			add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
 			break;
 		case ON_FEEDBACK:
 			tokens_ht = Z_ARRVAL_P(token_stream);
@@ -207,12 +204,8 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 			break;
 		case ON_STOP:
 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
-				array_init(&keyword);
-				add_next_index_long(&keyword, T_INLINE_HTML);
-				add_next_index_stringl(&keyword,
-					(char *)LANG_SCNG(yy_cursor), LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor));
-				add_next_index_long(&keyword, CG(zend_lineno));
-				add_next_index_zval(token_stream, &keyword);
+				add_token(token_stream, T_INLINE_HTML, LANG_SCNG(yy_cursor),
+					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno));
 			}
 			break;
 	}