fix: arrays in expansion concats, _word_in_replacement should be _exp…

…ansion_word
tree-sitter · Aug 29, 2023 · e225c58 · e225c58
1 parent d35dd56
commit e225c58
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 31 deletions.
diff --git a/grammar.js b/grammar.js
@@ -60,7 +60,7 @@ module.exports = grammar({
     $.regex,
     $._regex_no_slash,
     $._regex_no_space,
-    $._word_in_replacement,
+    $._expansion_word,
     $.extglob_pattern,
     $._bare_dollar,
     $._brace_start,
@@ -618,6 +618,7 @@ module.exports = grammar({
       $.expansion,
       $._simple_variable_name,
       $.variable_name,
+      $.string,
     )),
 
     _arithmetic_binary_expression: $ => prec.left(2, choice(
@@ -806,7 +807,6 @@ module.exports = grammar({
           $.raw_string,
           $.ansi_c_string,
           alias($._expansion_word, $.word),
-          alias($._word_in_replacement, $.word),
         ),
       )),
     )),
@@ -827,9 +827,9 @@ module.exports = grammar({
             $._literal,
             seq(
               $.command_substitution,
-              alias($._word_in_replacement, $.word),
+              alias($._expansion_word, $.word),
             ),
-            alias($._word_in_replacement, $.word),
+            alias($._expansion_word, $.word),
           ),
           field('operator', optional('/')),
         )),
@@ -879,7 +879,7 @@ module.exports = grammar({
         $.raw_string,
         $.command_substitution,
         alias($._expansion_word, $.word),
-        alias($._word_in_replacement, $.word),
+        $.array,
       ),
       repeat1(seq(
         choice($._concat, alias(/`\s*`/, '``')),
@@ -892,7 +892,7 @@ module.exports = grammar({
           $.raw_string,
           $.command_substitution,
           alias($._expansion_word, $.word),
-          alias($._word_in_replacement, $.word),
+          $.array,
         ),
       )),
     )),
@@ -921,18 +921,6 @@ module.exports = grammar({
 
     comment: _ => token(prec(-10, /#.*/)),
 
-    _expansion_word: _ => token(prec(-9, seq(
-      choice(
-        noneOf(...['$', '"', '{', '}', '(', ')', '\'', '\\s']),
-        seq('\\', noneOf('\\s')),
-      ),
-      repeat(choice(
-        noneOf(...['$', '"', '{', '}', '(', ')', '\'', '\\s']),
-        seq('\\', noneOf('\\s')),
-        '\\ ',
-      )),
-    ))),
-
     _comment_word: _ => token(prec(-8, seq(
       choice(
         noneOf(...SPECIAL_CHARACTERS),

diff --git a/src/scanner.c b/src/scanner.c
@@ -52,7 +52,7 @@ enum TokenType {
     REGEX,
     REGEX_NO_SLASH,
     REGEX_NO_SPACE,
-    WORD_IN_REPLACEMENT,
+    EXPANSION_WORD,
     EXTGLOB_PATTERN,
     BARE_DOLLAR,
     BRACE_START,
@@ -89,6 +89,7 @@ static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
 
 static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
 
+
 static inline bool in_error_recovery(const bool *valid_symbols) {
     return valid_symbols[HEREDOC_START] && valid_symbols[HEREDOC_END] &&
            valid_symbols[FILE_DESCRIPTOR] && valid_symbols[EMPTY_VALUE] &&
@@ -338,7 +339,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
             }
         }
         if (iswspace(lexer->lookahead) && valid_symbols[CLOSING_BRACE] &&
-            !valid_symbols[WORD_IN_REPLACEMENT]) {
+            !valid_symbols[EXPANSION_WORD]) {
             lexer->result_symbol = CONCAT;
             return true;
         }
@@ -476,9 +477,9 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
                 lexer->mark_end(lexer);
                 advance(lexer);
                 if (lexer->lookahead == '}' && valid_symbols[CLOSING_BRACE]) {
-                    if (valid_symbols[WORD_IN_REPLACEMENT]) {
+                    if (valid_symbols[EXPANSION_WORD]) {
                         lexer->mark_end(lexer);
-                        lexer->result_symbol = WORD_IN_REPLACEMENT;
+                        lexer->result_symbol = EXPANSION_WORD;
                         return true;
                     }
                     return false;
@@ -496,7 +497,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
             if ((lexer->lookahead == ' ' || lexer->lookahead == '\t' ||
                  lexer->lookahead == '\r' ||
                  (lexer->lookahead == '\n' && !valid_symbols[NEWLINE])) &&
-                !valid_symbols[WORD_IN_REPLACEMENT]) {
+                !valid_symbols[EXPANSION_WORD]) {
                 skip(lexer);
             } else if (lexer->lookahead == '\\') {
                 skip(lexer);
@@ -561,7 +562,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
             if (lexer->lookahead == '{') {
                 goto brace_start;
             }
-            if (valid_symbols[WORD_IN_REPLACEMENT]) {
+            if (valid_symbols[EXPANSION_WORD]) {
                 goto word_in_replacement;
             }
             if (valid_symbols[EXTGLOB_PATTERN]) {
@@ -882,7 +883,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
     }
 
 word_in_replacement:
-    if (valid_symbols[WORD_IN_REPLACEMENT]) {
+    if (valid_symbols[EXPANSION_WORD]) {
         bool advanced_once = false;
         for (;;) {
             if (lexer->lookahead == '\"') {
@@ -900,7 +901,7 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
 
             if (lexer->lookahead == '}') {
                 lexer->mark_end(lexer);
-                lexer->result_symbol = WORD_IN_REPLACEMENT;
+                lexer->result_symbol = EXPANSION_WORD;
                 return advanced_once;
             }
 

diff --git a/test/corpus/literals.txt b/test/corpus/literals.txt
@@ -1119,6 +1119,8 @@ echo $((bar=1))
 echo $((-1, 1))
 echo $((! -a || ~ +b || ++c || --d))
 echo $((foo-- || bar++))
+(("${MULTIBUILD_VARIANTS}" > 1))
+$(("$(stat --printf '%05a' "${save_file}")" & 07177))
 
 --------------------------------------------------------------------------------
 
@@ -1211,7 +1213,25 @@ echo $((foo-- || bar++))
         (binary_expression
           (postfix_expression
             (variable_name))
-          (variable_name))))))
+          (variable_name)))))
+  (command
+    (command_name
+      (arithmetic_expansion
+        (binary_expression
+          (string (expansion (variable_name)))
+          (number)))))
+  (command
+    (command_name
+      (arithmetic_expansion
+        (binary_expression
+          (string
+            (command_substitution
+              (command
+                (command_name (word))
+                (word)
+                (raw_string)
+                (string (expansion (variable_name))))))
+          (number))))))
 
 ================================================================================
 Concatenation with double backticks

diff --git a/test/corpus/statements.txt b/test/corpus/statements.txt
@@ -927,8 +927,6 @@ fi
 [[ -f "${EROOT}/usr/share/php/.packagexml/${MY_P}.xml" && \
 	-x "${EROOT}/usr/bin/peardev" ]]
 
-(("${MULTIBUILD_VARIANTS}" > 1))
-
 [[ ${test} == @($(IFS='|'; echo "${skip[*]}")) ]]
 
 [[ ${SRC_URI} == */${a}* ]]
@@ -1007,8 +1005,6 @@ fi
         (unary_expression
           (test_operator)
           (string (expansion (variable_name)))))))
-  (test_command
-    (binary_expression (string (expansion (variable_name))) (number)))
   (test_command
     (binary_expression
       (expansion (variable_name))