Merge pull request lcompilers#1068 from akshanshbhatt/pr_1023

[Parser] Fix a bug in tokenizer related to docstring
yashsain · Aug 31, 2022 · a227df8 · a227df8
2 parents 0536281 + 88bc051
commit a227df8
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 5 deletions.
diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re
@@ -281,12 +281,12 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             string1 = '"' ('\\'[^\x00] | [^"\x00\n\\])* '"';
             string2 = "'" ("\\"[^\x00] | [^'\x00\n\\])* "'";
             string3 = '"""' ( '\\'[^\x00]
-                            | ('"' | '"' '\\'+) [^"\x00\\]
+                            | ('"' | '"' '\\'+ '"' | '"' '\\'+) [^"\x00\\]
                             | ('""' | '""' '\\'+) [^"\x00\\]
                             | [^"\x00\\] )*
                       '"""';
             string4 = "'''" ( "\\"[^\x00]
-                            | ("'" | "'" "\\"+) [^'\x00\\]
+                            | ("'" | "'" "\\"+ "'" | "'" "\\"+) [^'\x00\\]
                             | ("''" | "''" "\\"+) [^'\x00\\]
                             | [^'\x00\\] )*
                       "'''";

diff --git a/tests/parser/string2.py b/tests/parser/string2.py
@@ -15,3 +15,10 @@ class a:
     def test():
         return r"\\LaTeX"
 """)
+
+r"""
+    "\\"
+"""
+'''
+    '\\'
+'''
diff --git a/tests/reference/ast_new-string2-44323ea.json b/tests/reference/ast_new-string2-44323ea.json
@@ -2,11 +2,11 @@
     "basename": "ast_new-string2-44323ea",
     "cmd": "lpython --show-ast --new-parser --no-color {infile} -o {outfile}",
     "infile": "tests/parser/string2.py",
-    "infile_hash": "fc6c3d133d5b93c5b5f052b9d86c2b1f270149aa48dba8ae8ced3e86",
+    "infile_hash": "02d7463c30650df9791af7669856669f0a948670cea5d26642f9a30c",
     "outfile": null,
     "outfile_hash": null,
     "stdout": "ast_new-string2-44323ea.stdout",
-    "stdout_hash": "cf514f21d4a57de66c5fbfc107a6ab58818a09359ef9ba651896d71f",
+    "stdout_hash": "c53164138eddb106cd7e13ab552e7b9f5978e3a7649708ece0ceb1db",
     "stderr": null,
     "stderr_hash": null,
     "returncode": 0

diff --git a/tests/reference/ast_new-string2-44323ea.stdout b/tests/reference/ast_new-string2-44323ea.stdout
@@ -5,4 +5,8 @@
 class a:
     def test():
         return r"\\LaTeX"
-" ())] []))] [])
+" ())] [])) (Expr (ConstantStr "
+    "\\"
+" ())) (Expr (ConstantStr "
+    '\\'
+" ()))] [])