Merge pull request lcompilers#502 from certik/string2

Parse strings of the "a" "b" "c" kind

src/lpython/parser/parser.yy

@@ -235,6 +235,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg)
 %type <ast> primary
 %type <vec_ast> sep
 %type <ast> sep_one
+%type <ast> string
 
 // Precedence
 
@@ -664,10 +665,14 @@ function_call
     | primary "(" keyword_items ")" { $$ = CALL_03($1, $3, @$); }
     ;
 
+string
+    : string TK_STRING { $$ = STRING2($1, $2, @$); } // TODO
+    | TK_STRING { $$ = STRING1($1, @$); }
+
 expr
     : id { $$ = $1; }
     | TK_INTEGER { $$ = INTEGER($1, @$); }
-    | TK_STRING { $$ = STRING($1, @$); }
+    | string { $$ = $1; }
     | TK_REAL { $$ = FLOAT($1, @$); }
     | TK_IMAG_NUM { $$ = COMPLEX($1, @$); }
     | TK_TRUE { $$ = BOOL(true, @$); }
@@ -679,7 +684,7 @@ expr
     | "{" expr_list "}" { $$ = SET($2, @$); }
     | "{" expr_list "," "}" { $$ = SET($2, @$); }
     | id "[" tuple_list "]" { $$ = SUBSCRIPT_01($1, $3, @$); }
-    | TK_STRING "[" tuple_list "]" { $$ = SUBSCRIPT_02($1, $3, @$); }
+    | string "[" tuple_list "]" { $$ = SUBSCRIPT_02($1, $3, @$); }
     | expr "." id { $$ = ATTRIBUTE_REF($1, $3, @$); }
     | "{" "}" { $$ = DICT_01(@$); }
     | "{" dict_list "}" { $$ = DICT_02($2, @$); }

src/lpython/parser/semantics.h

@@ -389,11 +389,17 @@ Vec<ast_t*> MERGE_EXPR(Allocator &al, ast_t *x, ast_t *y) {
 #define COMPARE(x, op, y, l) make_Compare_t(p.m_a, l, \
         EXPR(x), cmpopType::op, EXPRS(A2LIST(p.m_a, y)), 1)
 
+char* concat_string(Allocator &al, ast_t *a, char *b) {
+    char *s = down_cast2<ConstantStr_t>(a)->m_value;
+    return LFortran::s2c(al, std::string(s) + std::string(b));
+}
+
 #define SYMBOL(x, l) make_Name_t(p.m_a, l, \
         x.c_str(p.m_a), expr_contextType::Load)
 // `x.int_n` is of type BigInt but we store the int64_t directly in AST
 #define INTEGER(x, l) make_ConstantInt_t(p.m_a, l, x, nullptr)
-#define STRING(x, l) make_ConstantStr_t(p.m_a, l, x.c_str(p.m_a), nullptr)
+#define STRING1(x, l) make_ConstantStr_t(p.m_a, l, x.c_str(p.m_a), nullptr)
+#define STRING2(x, y, l) make_ConstantStr_t(p.m_a, l, concat_string(p.m_a, x, y.c_str(p.m_a)), nullptr)
 #define FLOAT(x, l) make_ConstantFloat_t(p.m_a, l, x, nullptr)
 #define COMPLEX(x, l) make_ConstantComplex_t(p.m_a, l, 0, x, nullptr)
 #define BOOL(x, l) make_ConstantBool_t(p.m_a, l, x, nullptr)
@@ -432,7 +438,7 @@ expr_t* CHECK_TUPLE(expr_t *x) {
 #define SUBSCRIPT_01(value, slice, l) make_Subscript_t(p.m_a, l, \
         EXPR(value), CHECK_TUPLE(EXPR(slice)), expr_contextType::Load)
 #define SUBSCRIPT_02(s, slice, l) make_Subscript_t(p.m_a, l, \
-        EXPR(STRING(s, l)), CHECK_TUPLE(EXPR(slice)), expr_contextType::Load)
+        EXPR(s), CHECK_TUPLE(EXPR(slice)), expr_contextType::Load)
 
 static inline ast_t* SLICE(Allocator &al, Location &l,
         ast_t *lower, ast_t *upper, ast_t *_step) {

tests/parser/statements1.py

@@ -51,6 +51,20 @@
 12+3j
 .12+.001j
 "String"
+"String " "String"
+'String ' 'String'
+'String ' "String"
+"String " "String"[1:]
+x = ("String "
+"String")
+x = ("String " +
+"String")
+x = ("String " \
+"String")
+x = "String " \
+"String"
+x = "String "
+"String"
 True
 False
 

tests/reference/ast_new-statements1-e081093.json

@@ -2,11 +2,11 @@
     "basename": "ast_new-statements1-e081093",
     "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
     "infile": "tests/parser/statements1.py",
-    "infile_hash": "4aaaf4ec5106c798f6b639b7dab587b4b9c8412b2029b384445e4b20",
+    "infile_hash": "9110211f2f42a7add5e5f1d8d27205c9b52f90a922e82c5f1c47fabc",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "ast_new-statements1-e081093.stdout",
-    "stdout_hash": "6fa494880a0bec62c30b15379701a5d5c277817940419d8379312bb6",
+    "stdout_hash": "fbd2a56fb302bc7ecc52f37193f7646e3eb3c0e78d18d19c451b7427",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/ast_new-statements1-e081093.stdout

@@ -1 +1 @@
-(Module [(Pass) (Break) (Continue) (Raise () ()) (Raise (Call (Name NameError Load) [(ConstantStr "String" ())] []) ()) (Raise (Name RuntimeError Load) (Name exc Load)) (Assert (Compare (Call (Name len Load) [(Name marks Load)] []) NotEq [(ConstantInt 0 ())]) (ConstantStr "List is empty." ())) (Assert (Compare (Name x Load) Eq [(ConstantStr "String" ())]) ()) (Assign [(Name x Store)] (ConstantInt 1 ()) ()) (Assign [(Tuple [(Name x Store) (Name y Store)] Store)] (Call (Name x Load) [] []) ()) (Assign [(Name x Store) (Name y Store)] (ConstantInt 1 ()) ()) (Assign [(Tuple [(Name x Store) (Name y Store)] Store)] (Tuple [(ConstantInt 1 ()) (ConstantInt 2 ())] Store) ()) (Assign [(Subscript (Name x Load) (Name i Load) Store)] (Tuple [(ConstantInt 1 ()) (ConstantInt 2 ())] Store) ()) (AugAssign (Name x Store) Add (ConstantInt 1 ())) (AnnAssign (Name x Store) (Name i64 Load) () 1) (AnnAssign (Name y Store) (Name i32 Load) (ConstantInt 1 ()) 1) (Delete [(Name x Del)]) (Delete [(Name x Del) (Name y Del)]) (Return ()) (Return (BinOp (Name a Load) Add (Name b Load))) (Return (Call (Name x Load) [(Name a Load)] [])) (Return (Tuple [(Name x Store) (Name y Store)] Store)) (Global [a]) (Global [a b]) (Nonlocal [a]) (Nonlocal [a b]) (Expr (ConstantInt 123 ())) (Expr (UnaryOp USub (ConstantInt 123 ()))) (Expr (UnaryOp USub (ConstantInt 291 ()))) (Expr (ConstantInt 6844 ())) (Expr (UnaryOp USub (ConstantInt 83 ()))) (Expr (ConstantInt 87 ())) (Expr (UnaryOp USub (ConstantInt 13 ()))) (Expr (ConstantInt 13 ())) (Expr (ConstantFloat 123.000000 ())) (Expr (ConstantFloat 123.450000 ())) (Expr (ConstantFloat 123400000000.000000 ())) (Expr (BinOp (ConstantInt 12 ()) Add (ConstantComplex 0.000000 3.000000 ()))) (Expr (BinOp (ConstantFloat 0.120000 ()) Add (ConstantComplex 0.000000 0.001000 ()))) (Expr (ConstantStr "String" ())) (Expr (ConstantBool .true. ())) (Expr (ConstantBool .false. ())) (Expr (BinOp (BinOp (Name x Load) Add (Name y Load)) Mult (Name z Load))) (Expr (BinOp (Name x Load) Sub (Name y Load))) (Expr (BinOp (Name x Load) Mult (Name y Load))) (Expr (BinOp (Name x Load) Div (Name y Load))) (Expr (BinOp (Name x Load) Mod (Name y Load))) (Expr (UnaryOp USub (Name y Load))) (Expr (UnaryOp UAdd (Name y Load))) (Expr (UnaryOp Invert (Name y Load))) (Expr (BinOp (Name x Load) Pow (Name y Load))) (Expr (BinOp (Name x Load) FloorDiv (Name y Load))) (Expr (BinOp (Name x Load) MatMult (Name y Load))) (Expr (BinOp (Name x Load) BitAnd (Name y Load))) (Expr (BinOp (Name x Load) BitOr (Name y Load))) (Expr (BinOp (Name x Load) BitXor (Name y Load))) (Expr (BinOp (Name x Load) LShift (Name y Load))) (Expr (BinOp (Name x Load) RShift (Name y Load))) (Expr (Compare (Name x Load) Eq [(Name y Load)])) (Expr (Compare (Name x Load) NotEq [(Name y Load)])) (Expr (Compare (Name x Load) Lt [(Name y Load)])) (Expr (Compare (Name x Load) LtE [(Name y Load)])) (Expr (Compare (Name x Load) Gt [(Name y Load)])) (Expr (Compare (Name x Load) GtE [(Name y Load)])) (AnnAssign (Name i Store) (Name i32 Load) (ConstantInt 4 ()) 1) (If (Compare (ConstantInt 2 ()) Gt [(Name i Load)]) [(Pass)] []) (If (Compare (Name i Load) Gt [(ConstantInt 5 ())]) [(Break)] []) (If (BoolOp And [(Compare (Name i Load) Eq [(ConstantInt 5 ())]) (Compare (Name i Load) Lt [(ConstantInt 10 ())])]) [(Assign [(Name i Store)] (ConstantInt 3 ()) ())] []) (For (Name i Store) (Call (Name range Load) [(Name N Load)] []) [(Assign [(Subscript (Name c Load) (Name i Load) Store)] (BinOp (Subscript (Name a Load) (Name i Load) Load) Add (BinOp (Name scalar Load) Mult (Subscript (Name b Load) (Name i Load) Load))) ())] [] ()) (Assign [(Name x Store)] (NamedExpr (Name y Store) (ConstantInt 0 ())) ()) (If (NamedExpr (Name a Store) (Call (Name ord Load) [(ConstantStr "3" ())] [])) [(Assign [(Name x Store)] (ConstantInt 1 ()) ())] []) (Assign [(Name a Store)] (Set [(ConstantInt 1 ()) (ConstantInt 2 ()) (ConstantInt 3 ())]) ())] [])
+(Module [(Pass) (Break) (Continue) (Raise () ()) (Raise (Call (Name NameError Load) [(ConstantStr "String" ())] []) ()) (Raise (Name RuntimeError Load) (Name exc Load)) (Assert (Compare (Call (Name len Load) [(Name marks Load)] []) NotEq [(ConstantInt 0 ())]) (ConstantStr "List is empty." ())) (Assert (Compare (Name x Load) Eq [(ConstantStr "String" ())]) ()) (Assign [(Name x Store)] (ConstantInt 1 ()) ()) (Assign [(Tuple [(Name x Store) (Name y Store)] Store)] (Call (Name x Load) [] []) ()) (Assign [(Name x Store) (Name y Store)] (ConstantInt 1 ()) ()) (Assign [(Tuple [(Name x Store) (Name y Store)] Store)] (Tuple [(ConstantInt 1 ()) (ConstantInt 2 ())] Store) ()) (Assign [(Subscript (Name x Load) (Name i Load) Store)] (Tuple [(ConstantInt 1 ()) (ConstantInt 2 ())] Store) ()) (AugAssign (Name x Store) Add (ConstantInt 1 ())) (AnnAssign (Name x Store) (Name i64 Load) () 1) (AnnAssign (Name y Store) (Name i32 Load) (ConstantInt 1 ()) 1) (Delete [(Name x Del)]) (Delete [(Name x Del) (Name y Del)]) (Return ()) (Return (BinOp (Name a Load) Add (Name b Load))) (Return (Call (Name x Load) [(Name a Load)] [])) (Return (Tuple [(Name x Store) (Name y Store)] Store)) (Global [a]) (Global [a b]) (Nonlocal [a]) (Nonlocal [a b]) (Expr (ConstantInt 123 ())) (Expr (UnaryOp USub (ConstantInt 123 ()))) (Expr (UnaryOp USub (ConstantInt 291 ()))) (Expr (ConstantInt 6844 ())) (Expr (UnaryOp USub (ConstantInt 83 ()))) (Expr (ConstantInt 87 ())) (Expr (UnaryOp USub (ConstantInt 13 ()))) (Expr (ConstantInt 13 ())) (Expr (ConstantFloat 123.000000 ())) (Expr (ConstantFloat 123.450000 ())) (Expr (ConstantFloat 123400000000.000000 ())) (Expr (BinOp (ConstantInt 12 ()) Add (ConstantComplex 0.000000 3.000000 ()))) (Expr (BinOp (ConstantFloat 0.120000 ()) Add (ConstantComplex 0.000000 0.001000 ()))) (Expr (ConstantStr "String" ())) (Expr (ConstantStr "String String" ())) (Expr (ConstantStr "String String" ())) (Expr (ConstantStr "String String" ())) (Expr (Subscript (ConstantStr "String String" ()) (Slice (ConstantInt 1 ()) () ()) Load)) (Assign [(Name x Store)] (ConstantStr "String String" ()) ()) (Assign [(Name x Store)] (BinOp (ConstantStr "String " ()) Add (ConstantStr "String" ())) ()) (Assign [(Name x Store)] (ConstantStr "String String" ()) ()) (Assign [(Name x Store)] (ConstantStr "String String" ()) ()) (Assign [(Name x Store)] (ConstantStr "String " ()) ()) (Expr (ConstantStr "String" ())) (Expr (ConstantBool .true. ())) (Expr (ConstantBool .false. ())) (Expr (BinOp (BinOp (Name x Load) Add (Name y Load)) Mult (Name z Load))) (Expr (BinOp (Name x Load) Sub (Name y Load))) (Expr (BinOp (Name x Load) Mult (Name y Load))) (Expr (BinOp (Name x Load) Div (Name y Load))) (Expr (BinOp (Name x Load) Mod (Name y Load))) (Expr (UnaryOp USub (Name y Load))) (Expr (UnaryOp UAdd (Name y Load))) (Expr (UnaryOp Invert (Name y Load))) (Expr (BinOp (Name x Load) Pow (Name y Load))) (Expr (BinOp (Name x Load) FloorDiv (Name y Load))) (Expr (BinOp (Name x Load) MatMult (Name y Load))) (Expr (BinOp (Name x Load) BitAnd (Name y Load))) (Expr (BinOp (Name x Load) BitOr (Name y Load))) (Expr (BinOp (Name x Load) BitXor (Name y Load))) (Expr (BinOp (Name x Load) LShift (Name y Load))) (Expr (BinOp (Name x Load) RShift (Name y Load))) (Expr (Compare (Name x Load) Eq [(Name y Load)])) (Expr (Compare (Name x Load) NotEq [(Name y Load)])) (Expr (Compare (Name x Load) Lt [(Name y Load)])) (Expr (Compare (Name x Load) LtE [(Name y Load)])) (Expr (Compare (Name x Load) Gt [(Name y Load)])) (Expr (Compare (Name x Load) GtE [(Name y Load)])) (AnnAssign (Name i Store) (Name i32 Load) (ConstantInt 4 ()) 1) (If (Compare (ConstantInt 2 ()) Gt [(Name i Load)]) [(Pass)] []) (If (Compare (Name i Load) Gt [(ConstantInt 5 ())]) [(Break)] []) (If (BoolOp And [(Compare (Name i Load) Eq [(ConstantInt 5 ())]) (Compare (Name i Load) Lt [(ConstantInt 10 ())])]) [(Assign [(Name i Store)] (ConstantInt 3 ()) ())] []) (For (Name i Store) (Call (Name range Load) [(Name N Load)] []) [(Assign [(Subscript (Name c Load) (Name i Load) Store)] (BinOp (Subscript (Name a Load) (Name i Load) Load) Add (BinOp (Name scalar Load) Mult (Subscript (Name b Load) (Name i Load) Load))) ())] [] ()) (Assign [(Name x Store)] (NamedExpr (Name y Store) (ConstantInt 0 ())) ()) (If (NamedExpr (Name a Store) (Call (Name ord Load) [(ConstantStr "3" ())] [])) [(Assign [(Name x Store)] (ConstantInt 1 ()) ())] []) (Assign [(Name a Store)] (Set [(ConstantInt 1 ()) (ConstantInt 2 ()) (ConstantInt 3 ())]) ())] [])