Fix: string/symbol array literals nesting and escaping (crystal-lang#…

…5667)

i# ase enter the commit message for your changes. Lines starting

spec/compiler/parser/parser_spec.cr

@@ -1615,8 +1615,26 @@ describe "Parser" do
     assert_syntax_error %(case x; when /x/; 2; when /x/; end), "duplicate when /x/ in case"
     assert_syntax_error %(case x; when X; 2; when X; end), "duplicate when X in case"
 
+    it_parses "%w{one  two}", (["one".string, "two".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{one\ntwo}", (["one".string, "two".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{one\ttwo}", (["one".string, "two".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{\n}", ([] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{one\\ two}", (["one two".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{one{} two}", (["one{}".string, "two".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{\\{one}", (["{one".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%w{one\\}}", (["one}".string] of ASTNode).array_of(Path.global("String"))
+    it_parses "%i(one\\ two)", (["one two".symbol] of ASTNode).array_of(Path.global("Symbol"))
+    it_parses "%i{(one two)}", (["(one".symbol, "two)".symbol] of ASTNode).array_of(Path.global("Symbol"))
+    it_parses "%i((one two))", (["(one".symbol, "two)".symbol] of ASTNode).array_of(Path.global("Symbol"))
+    it_parses "%i(foo(bar) baz)", (["foo(bar)".symbol, "baz".symbol] of ASTNode).array_of(Path.global("Symbol"))
+    it_parses "%i{foo\\nbar baz}", (["foo\\nbar".symbol, "baz".symbol] of ASTNode).array_of(Path.global("Symbol"))
+
     assert_syntax_error "%w(", "Unterminated string array literal"
+    assert_syntax_error "%w{one}}", "expecting token 'EOF', not '}'"
+    assert_syntax_error "%w{{one}", "Unterminated string array literal"
     assert_syntax_error "%i(", "Unterminated symbol array literal"
+    assert_syntax_error "%i{one}}", "expecting token 'EOF', not '}'"
+    assert_syntax_error "%i{{one}", "Unterminated symbol array literal"
     assert_syntax_error "%x(", "Unterminated command literal"
     assert_syntax_error "%r(", "Unterminated regular expression"
     assert_syntax_error "%q(", "Unterminated string literal"

src/compiler/crystal/syntax/lexer.cr

@@ -2477,7 +2477,40 @@ module Crystal
       end
 
       start = current_pos
-      while !current_char.ascii_whitespace? && current_char != '\0' && current_char != @token.delimiter_state.end
+      sub_start = start
+      value = String::Builder.new
+
+      escaped = false
+      while true
+        case current_char
+        when Char::ZERO
+          break # raise is handled by parser
+        when @token.delimiter_state.end
+          unless escaped
+            if @token.delimiter_state.open_count == 0
+              break
+            else
+              @token.delimiter_state = @token.delimiter_state.with_open_count_delta(-1)
+            end
+          end
+        when @token.delimiter_state.nest
+          unless escaped
+            @token.delimiter_state = @token.delimiter_state.with_open_count_delta(+1)
+          end
+        when .ascii_whitespace?
+          break unless escaped
+        else
+          if escaped
+            value << '\\'
+          end
+        end
+
+        escaped = current_char == '\\'
+        if escaped
+          value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
+          sub_start = current_pos + 1
+        end
+
         next_char
       end
 
@@ -2486,8 +2519,10 @@ module Crystal
         return @token
       end
 
+      value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
+
       @token.type = :STRING
-      @token.value = string_range(start)
+      @token.value = value.to_s
       set_token_raw_from_start(start)
 
       @token