Merge pull request lcompilers#1084 from Thirumalai-Shaktivel/fix_pars…

…er_issues

src/lpython/parser/parser.yy

@@ -266,8 +266,6 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg)
 %type <args_> lambda_parameter_list_no_posonly
 %type <var_kw> lambda_parameter_list_starargs
 %type <vec_arg> lambda_defparameter_list
-%type <ast> await_expr
-%type <ast> expr_or_await
 %type <ast> yield_expr
 
 // Precedence
@@ -287,6 +285,7 @@ void yyerror(YYLTYPE *yyloc, LFortran::Parser &p, const std::string &msg)
 %left "%" "//" "/" "@" "*"
 %precedence UNARY
 %right "**"
+%precedence AWAIT
 %precedence "."
 
 %start units
@@ -393,24 +392,14 @@ multi_line_statement
     | while_statement
     ;
 
-await_expr
-    : KW_AWAIT tuple_list { $$ = AWAIT($2, @$); }
-    ;
-
 yield_expr
     : KW_YIELD { $$ = YIELD_01(@$); }
     | KW_YIELD tuple_list { $$ = YIELD_02($2, @$); }
     | KW_YIELD_FROM tuple_list { $$ = YIELD_03($2, @$); }
     ;
 
-expr_or_await
-    : expr { $$ = $1; }
-    | await_expr { $$ = $1; }
-    ;
-
 expression_statment
     : tuple_list { $$ = EXPR_01($1, @$); }
-    | await_expr { $$ = EXPR_01($1, @$); }
     | yield_expr { $$ = EXPR_01($1, @$); }
     | ternary_if_statement { $$ = EXPR_01($1, @$); }
     ;
@@ -445,15 +434,14 @@ target_list
 
 assignment_statement
     : target_list tuple_list { $$ = ASSIGNMENT($1, $2, @$); }
-    | target_list await_expr { $$ = ASSIGNMENT($1, $2, @$); }
     | target_list yield_expr { $$ = ASSIGNMENT($1, $2, @$); }
     | target_list ternary_if_statement { $$ = ASSIGNMENT($1, $2, @$); }
     | target_list tuple_list TK_TYPE_COMMENT {
         $$ = ASSIGNMENT2($1, $2, $3, @$); }
     ;
 
 augassign_statement
-    : expr augassign_op expr_or_await { $$ = AUGASSIGN_01($1, $2, $3, @$); }
+    : expr augassign_op expr { $$ = AUGASSIGN_01($1, $2, $3, @$); }
     ;
 
 augassign_op
@@ -485,7 +473,6 @@ delete_statement
 return_statement
     : KW_RETURN { $$ = RETURN_01(@$); }
     | KW_RETURN tuple_list { $$ = RETURN_02($2, @$); }
-    | KW_RETURN await_expr { $$ = RETURN_02($2, @$); }
     | KW_RETURN yield_expr { $$ = RETURN_02($2, @$); }
     ;
 
@@ -625,7 +612,6 @@ with_as_items
     | "(" expr_list "," expr KW_AS expr_list comma_opt ")" {
         $$ = withitem_to_list(p.m_a, WITH_ITEM_01(TUPLE_01(TUPLE_($2, $4), @$),
                               TUPLE_01($6, @$), @$)); }
-    | await_expr { $$ = withitem_to_list(p.m_a, WITH_ITEM_02($1, @$));}
     ;
 
 with_statement
@@ -851,14 +837,14 @@ primary
     ;
 
 comp_if_items
-    : comp_if_items KW_IF expr_or_await { $$ = $1; LIST_ADD($$, $3); }
-    | KW_IF expr_or_await { LIST_NEW($$); LIST_ADD($$, $2); }
+    : comp_if_items KW_IF expr { $$ = $1; LIST_ADD($$, $3); }
+    | KW_IF expr { LIST_NEW($$); LIST_ADD($$, $2); }
     ;
 
 comp_for
-    : KW_FOR id_list KW_IN expr_or_await {
+    : KW_FOR id_list KW_IN expr {
         $$ = COMP_FOR_01(ID_TUPLE_01($2, @$), $4, @$); }
-    | KW_FOR id_list "," KW_IN expr_or_await {
+    | KW_FOR id_list "," KW_IN expr {
         $$ = COMP_FOR_01(ID_TUPLE_03($2, @$), $5, @$); }
     | KW_FOR id_list KW_IN expr comp_if_items {
         $$ = COMP_FOR_02(ID_TUPLE_01($2, @$), $4, $5, @$); }
@@ -1010,9 +996,9 @@ lambda_expression
     ;
 
 comprehension
-    : "[" expr_or_await comp_for_items "]" { $$ = LIST_COMP_1($2, $3, @$); }
-    | "{" expr_or_await comp_for_items "}" { $$ = SET_COMP_1($2, $3, @$); }
-    | "{" expr ":" expr_or_await comp_for_items "}" {
+    : "[" expr comp_for_items "]" { $$ = LIST_COMP_1($2, $3, @$); }
+    | "{" expr comp_for_items "}" { $$ = SET_COMP_1($2, $3, @$); }
+    | "{" expr ":" expr comp_for_items "}" {
         $$ = DICT_COMP_1($2, $4, $5, @$); }
     | "(" expr comp_for_items ")" { $$ = COMP_EXPR_1($2, $3, @$); }
     ;
@@ -1028,6 +1014,7 @@ expr
     | KW_NONE { $$ = NONE(@$); }
     | TK_ELLIPSIS { $$ = ELLIPSIS(@$); }
     | "(" expr ")" { $$ = $2; }
+    | KW_AWAIT expr %prec AWAIT { $$ = AWAIT($2, @$); }
     | "(" yield_expr ")" { $$ = $2; }
     | "(" TK_TYPE_IGNORE expr ")" { $$ = $3; extract_type_comment(p, @$, $2); }
     | "(" ")" { $$ = TUPLE_EMPTY(@$); }

src/lpython/parser/tokenizer.re

@@ -481,6 +481,7 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             "is" whitespace? "\\" newline whitespace? "not" whitespace { RET(TK_IS_NOT) }
             "not" whitespace "in" "\\" newline { RET(TK_NOT_IN) }
             "not" whitespace "in" whitespace { RET(TK_NOT_IN) }
+            "not" whitespace "in" newline { RET(TK_NOT_IN) }
             "not" whitespace? "\\" newline whitespace? "in" "\\" newline { RET(TK_NOT_IN) }
             "not" whitespace? "\\" newline whitespace? "in" whitespace { RET(TK_NOT_IN) }
 

tests/parser/async1.py

@@ -24,3 +24,10 @@ async def test_02():
     [s for c in [f(''), f('abc'), f(''), f(['de', 'fg'])]
                             for s in await c]
     return (i * 2 for i in range(n) if await wrap(i))
+
+async def t():
+    results.append(await anext(g))
+    self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10)))
+    x = -await bar()
+    return (await bar() + await wrap()() + await db['b']()()() +
+                        await bar() * 1000 + await DB.b()())

tests/parser/statements2.py

@@ -20,14 +20,18 @@
 [x for x in G if self.ring.is_unit(x[0])][0]
 
 if x not in\
-    z: ... 
+    z: ...
 
 if (x not in\
-    z): ... 
+    z): ...
 
 if x not\
     in\
-    z: ... 
+    z: ...
+
+if (x not in
+    z): ...
+
 
 def imatmul(a, b):
     "Same as a @= b."

tests/reference/ast_new-async1-b3d07ed.json

@@ -2,11 +2,11 @@
     "basename": "ast_new-async1-b3d07ed",
     "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
     "infile": "tests/parser/async1.py",
-    "infile_hash": "d089cff932e221cd4faac8d2802e18e16833322a746eea933291caf7",
+    "infile_hash": "7c5dba5bb5fe728a9452de16f8164730885146b2d93e49be8642d96a",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "ast_new-async1-b3d07ed.stdout",
-    "stdout_hash": "4447a9366a389765da43bf8ae8cef7ec165c2cf8e8925ced842bbc66",
+    "stdout_hash": "1580c6f5bca7b5bd74f5cca0f2dc7f165fe454d2a19f12897ea133bd",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/ast_new-async1-b3d07ed.stdout

@@ -1 +1 @@
-(Module [(AsyncFunctionDef func ([] [(param1 () ()) (param2 () ())] [] [] [] [] []) [(Expr (Await (Call (Attribute (Name asyncio Load) sleep Load) [(ConstantInt 1 ())] []))) (Expr (Call (Name do_something Load) [] [])) (AsyncFor (Name x Store) (Name y Load) [(Expr (Call (Name do_something Load) [(Name x Load)] []))] [] ()) (AsyncWith [((Call (Name open Load) [(ConstantStr "examples/expr2.py" ()) (ConstantStr "r" ())] []) (Name file Store))] [(Assign [(Name x Store)] (Call (Attribute (Name file Load) read Load) [] []) ())] ())] [] () ()) (AsyncFunctionDef test_01 ([] [] [] [] [] [] []) [(Assign [(Name results Store)] (Await (Call (Attribute (Name tasks Load) gather Load) [(Starred (ListComp (Call (Attribute (Name ag Load) aclose Load) [] []) [((Name ag Store) (Name closing_agens Load) [] 0)]) Load)] [(return_exceptions (ConstantBool .true. ()))])) ()) (With [((Await (Name lock Load)) ())] [(Pass)] ()) (AugAssign (Name data Store) Add (Await (Call (Attribute (Attribute (Name self Load) loop Load) sock_recv Load) [(Name sock Load) (Name DATA_SIZE Load)] []))) (Return (Await (Call (Attribute (Name self Load) run_in_executor Load) [(ConstantNone ()) (Name getaddr_func Load) (Name host Load) (Name port Load) (Name family Load) (Name type Load) (Name proto Load) (Name flags Load)] [])))] [] () ()) (AsyncFunctionDef test_02 ([] [] [] [] [] [] []) [(Assign [(Name items Store)] (ListComp (Await (Call (Attribute (Name q Load) get Load) [] [])) [((Name _ Store) (Call (Name range Load) [(ConstantInt 3 ())] []) [] 0)]) ()) (Expr (SetComp (Await (Name c Load)) [((Name c Store) (List [(Call (Name f Load) [(ConstantInt 1 ())] []) (Call (Name f Load) [(ConstantInt 41 ())] [])] Load) [] 0)])) (Expr (DictComp (Name i Load) (Await (Name c Load)) [((Tuple [(Name i Store) (Name c Store)] Store) (Call (Name enumerate Load) [(List [(Call (Name f Load) [(ConstantInt 1 ())] []) (Call (Name f Load) [(ConstantInt 41 ())] [])] Load)] []) [] 0)])) (Expr (ListComp (Name s Load) [((Name c Store) (List [(Call (Name f Load) [(ConstantStr "" ())] []) (Call (Name f Load) [(ConstantStr "abc" ())] []) (Call (Name f Load) [(ConstantStr "" ())] []) (Call (Name f Load) [(List [(ConstantStr "de" ()) (ConstantStr "fg" ())] Load)] [])] Load) [] 0) ((Name s Store) (Await (Name c Load)) [] 0)])) (Return (GeneratorExp (BinOp (Name i Load) Mult (ConstantInt 2 ())) [((Name i Store) (Call (Name range Load) [(Name n Load)] []) [(Await (Call (Name wrap Load) [(Name i Load)] []))] 0)]))] [] () ())] [])
+(Module [(AsyncFunctionDef func ([] [(param1 () ()) (param2 () ())] [] [] [] [] []) [(Expr (Await (Call (Attribute (Name asyncio Load) sleep Load) [(ConstantInt 1 ())] []))) (Expr (Call (Name do_something Load) [] [])) (AsyncFor (Name x Store) (Name y Load) [(Expr (Call (Name do_something Load) [(Name x Load)] []))] [] ()) (AsyncWith [((Call (Name open Load) [(ConstantStr "examples/expr2.py" ()) (ConstantStr "r" ())] []) (Name file Store))] [(Assign [(Name x Store)] (Call (Attribute (Name file Load) read Load) [] []) ())] ())] [] () ()) (AsyncFunctionDef test_01 ([] [] [] [] [] [] []) [(Assign [(Name results Store)] (Await (Call (Attribute (Name tasks Load) gather Load) [(Starred (ListComp (Call (Attribute (Name ag Load) aclose Load) [] []) [((Name ag Store) (Name closing_agens Load) [] 0)]) Load)] [(return_exceptions (ConstantBool .true. ()))])) ()) (With [((Await (Name lock Load)) ())] [(Pass)] ()) (AugAssign (Name data Store) Add (Await (Call (Attribute (Attribute (Name self Load) loop Load) sock_recv Load) [(Name sock Load) (Name DATA_SIZE Load)] []))) (Return (Await (Call (Attribute (Name self Load) run_in_executor Load) [(ConstantNone ()) (Name getaddr_func Load) (Name host Load) (Name port Load) (Name family Load) (Name type Load) (Name proto Load) (Name flags Load)] [])))] [] () ()) (AsyncFunctionDef test_02 ([] [] [] [] [] [] []) [(Assign [(Name items Store)] (ListComp (Await (Call (Attribute (Name q Load) get Load) [] [])) [((Name _ Store) (Call (Name range Load) [(ConstantInt 3 ())] []) [] 0)]) ()) (Expr (SetComp (Await (Name c Load)) [((Name c Store) (List [(Call (Name f Load) [(ConstantInt 1 ())] []) (Call (Name f Load) [(ConstantInt 41 ())] [])] Load) [] 0)])) (Expr (DictComp (Name i Load) (Await (Name c Load)) [((Tuple [(Name i Store) (Name c Store)] Store) (Call (Name enumerate Load) [(List [(Call (Name f Load) [(ConstantInt 1 ())] []) (Call (Name f Load) [(ConstantInt 41 ())] [])] Load)] []) [] 0)])) (Expr (ListComp (Name s Load) [((Name c Store) (List [(Call (Name f Load) [(ConstantStr "" ())] []) (Call (Name f Load) [(ConstantStr "abc" ())] []) (Call (Name f Load) [(ConstantStr "" ())] []) (Call (Name f Load) [(List [(ConstantStr "de" ()) (ConstantStr "fg" ())] Load)] [])] Load) [] 0) ((Name s Store) (Await (Name c Load)) [] 0)])) (Return (GeneratorExp (BinOp (Name i Load) Mult (ConstantInt 2 ())) [((Name i Store) (Call (Name range Load) [(Name n Load)] []) [(Await (Call (Name wrap Load) [(Name i Load)] []))] 0)]))] [] () ()) (AsyncFunctionDef t ([] [] [] [] [] [] []) [(Expr (Call (Attribute (Name results Load) append Load) [(Await (Call (Name anext Load) [(Name g Load)] []))] [])) (Expr (Call (Attribute (Name self Load) assertIn Load) [(ConstantStr "..." ()) (Call (Name repr Load) [(Await (Call (Attribute (Name asyncio Load) wait_for Load) [(Call (Name func Load) [] [])] [(timeout (ConstantInt 10 ()))]))] [])] [])) (Assign [(Name x Store)] (UnaryOp USub (Await (Call (Name bar Load) [] []))) ()) (Return (BinOp (BinOp (BinOp (BinOp (Await (Call (Name bar Load) [] [])) Add (Await (Call (Call (Name wrap Load) [] []) [] []))) Add (Await (Call (Call (Call (Subscript (Name db Load) (ConstantStr "b" ()) Load) [] []) [] []) [] []))) Add (BinOp (Await (Call (Name bar Load) [] [])) Mult (ConstantInt 1000 ()))) Add (Await (Call (Call (Attribute (Name DB Load) b Load) [] []) [] []))))] [] () ())] [])

tests/reference/ast_new-statements2-c4cdc5f.json

@@ -2,11 +2,11 @@
     "basename": "ast_new-statements2-c4cdc5f",
     "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
     "infile": "tests/parser/statements2.py",
-    "infile_hash": "3be5c5c199bae9d20d51a30cb785a08faa69aa7cb13f18aae44ffe01",
+    "infile_hash": "8c96f6788e951a113e775f497394a372018d04b4f7f910d304471017",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "ast_new-statements2-c4cdc5f.stdout",
-    "stdout_hash": "b740926cb114f97c24252d42bc2a0a1b374c3c3d6f286c9d869f1512",
+    "stdout_hash": "24a915b924b34f39fd72c541e9132c6e887faee540d6a9f5f63b0b4e",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

tests/reference/ast_new-statements2-c4cdc5f.stdout

@@ -1 +1 @@
-(Module [(Expr (Call (Call (Call (Name getattr Load) [(Name x Load) (Name y Load)] []) [(Name a Load) (Name b Load)] []) [(ConstantInt 5 ())] [(a (Name a Load)) (b (ConstantInt 0 ()))])) (Expr (Call (Call (Name func Load) [(Name a Load) (Name _op Load)] []) [(Name s Load)] [])) (Expr (Call (Call (Name test Load) [] []) [] [])) (Assign [(Name a Store)] (Subscript (List [(ConstantInt 1 ()) (ConstantInt 2 ())] Load) (ConstantInt 0 ()) Load) ()) (Assign [(Tuple [(Name a Store) (Name b Store)] Store)] (Subscript (BinOp (List [(ConstantInt 1 ()) (ConstantInt 2 ())] Load) Add (List [(ConstantInt 0 ()) (ConstantInt 0 ())] Load)) (Slice () (ConstantInt 2 ()) ()) Load) ()) (Assign [(Tuple [(Name a Store) (Name b Store)] Store)] (Subscript (BinOp (Call (Attribute (Name c Load) d Load) [] []) Add (List [(ConstantInt 1 ()) (ConstantInt 0 ())] Load)) (Slice () () ()) Load) ()) (Expr (Subscript (Dict [(ConstantStr "a" ()) (ConstantStr "b" ())] [(Name a Load) (Name b Load)]) (Name val Load) Load)) (If (Compare (Subscript (Subscript (Name args Load) (Name i Load) Load) (Slice () (ConstantInt 1 ()) ()) Load) In [(List [(ConstantStr "" ()) (ConstantStr "." ())] Load)]) [(Pass)] []) (Assign [(Name x Store)] (List [(Subscript (ListComp (BinOp (UnaryOp USub (Name c Load)) Mod (Name self Load)) [((Name c Store) (Call (Name reversed Load) [(Attribute (Attribute (Name T Load) rep Load) rep Load)] []) [] 0)]) (Slice () (UnaryOp USub (ConstantInt 1 ())) ()) Load)] Load) ()) (Expr (Subscript (ListComp (Name x Load) [((Name x Store) (Name G Load) [(Call (Attribute (Attribute (Name self Load) ring Load) is_unit Load) [(Subscript (Name x Load) (ConstantInt 0 ()) Load)] [])] 0)]) (ConstantInt 0 ()) Load)) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (FunctionDef imatmul ([] [(a () ()) (b () ())] [] [] [] [] []) [(Expr (ConstantStr "Same as a @= b." ())) (AugAssign (Name a Store) MatMult (Name b Load)) (Return (Name a Load))] [] () ())] [])
+(Module [(Expr (Call (Call (Call (Name getattr Load) [(Name x Load) (Name y Load)] []) [(Name a Load) (Name b Load)] []) [(ConstantInt 5 ())] [(a (Name a Load)) (b (ConstantInt 0 ()))])) (Expr (Call (Call (Name func Load) [(Name a Load) (Name _op Load)] []) [(Name s Load)] [])) (Expr (Call (Call (Name test Load) [] []) [] [])) (Assign [(Name a Store)] (Subscript (List [(ConstantInt 1 ()) (ConstantInt 2 ())] Load) (ConstantInt 0 ()) Load) ()) (Assign [(Tuple [(Name a Store) (Name b Store)] Store)] (Subscript (BinOp (List [(ConstantInt 1 ()) (ConstantInt 2 ())] Load) Add (List [(ConstantInt 0 ()) (ConstantInt 0 ())] Load)) (Slice () (ConstantInt 2 ()) ()) Load) ()) (Assign [(Tuple [(Name a Store) (Name b Store)] Store)] (Subscript (BinOp (Call (Attribute (Name c Load) d Load) [] []) Add (List [(ConstantInt 1 ()) (ConstantInt 0 ())] Load)) (Slice () () ()) Load) ()) (Expr (Subscript (Dict [(ConstantStr "a" ()) (ConstantStr "b" ())] [(Name a Load) (Name b Load)]) (Name val Load) Load)) (If (Compare (Subscript (Subscript (Name args Load) (Name i Load) Load) (Slice () (ConstantInt 1 ()) ()) Load) In [(List [(ConstantStr "" ()) (ConstantStr "." ())] Load)]) [(Pass)] []) (Assign [(Name x Store)] (List [(Subscript (ListComp (BinOp (UnaryOp USub (Name c Load)) Mod (Name self Load)) [((Name c Store) (Call (Name reversed Load) [(Attribute (Attribute (Name T Load) rep Load) rep Load)] []) [] 0)]) (Slice () (UnaryOp USub (ConstantInt 1 ())) ()) Load)] Load) ()) (Expr (Subscript (ListComp (Name x Load) [((Name x Store) (Name G Load) [(Call (Attribute (Attribute (Name self Load) ring Load) is_unit Load) [(Subscript (Name x Load) (ConstantInt 0 ()) Load)] [])] 0)]) (ConstantInt 0 ()) Load)) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (If (Compare (Name x Load) NotIn [(Name z Load)]) [(Expr (ConstantEllipsis ()))] []) (FunctionDef imatmul ([] [(a () ()) (b () ())] [] [] [] [] []) [(Expr (ConstantStr "Same as a @= b." ())) (AugAssign (Name a Store) MatMult (Name b Load)) (Return (Name a Load))] [] () ())] [])