Merge pull request lcompilers#260 from certik/lf_remove5

Remove the Fortran parser completely

.gitignore

@@ -3,7 +3,6 @@
 src/bin/lpython
 src/bin/parse
 src/bin/parse2
-src/lpython/parser/parser.output
 tests/output
 src/lpython/tests/test_asm
 src/lpython/tests/test_ast
@@ -55,17 +54,6 @@ _CPack_Packages
 version
 lpython/ast/ast.py
 lpython/asr/asr.py
-lpython/parser/fortran.tokens
-lpython/parser/fortran.interp
-lpython/parser/fortranLexer.py
-lpython/parser/fortranLexer.tokens
-lpython/parser/fortranLexer.interp
-lpython/parser/fortranParser.py
-lpython/parser/fortranVisitor.py
-src/lpython/parser/parser.tab.cc
-src/lpython/parser/parser.tab.hh
-src/lpython/parser/tokenizer.cpp
-src/lpython/parser/preprocessor.cpp
 src/lpython/ast.h
 src/lpython/asr.h
 src/libasr/asr.h

build0.sh

@@ -14,11 +14,3 @@ python grammar/asdl_py.py
 python grammar/asdl_cpp.py grammar/Python.asdl src/lpython/python_ast.h
 # Generate a Fortran ASR from ASR.asdl (C++)
 python grammar/asdl_cpp.py grammar/ASR.asdl src/libasr/asr.h
-
-# Generate the tokenizer and parser
-(cd src/lpython/parser && re2c -W -b tokenizer.re -o tokenizer.cpp)
-(cd src/lpython/parser && re2c -W -b preprocessor.re -o preprocessor.cpp)
-(cd src/lpython/parser && bison -Wall -d -r all parser.yy)
-
-grep -n "'" src/lpython/parser/parser.yy && echo "Single quote not allowed" && exit 1
-echo "OK"

build1_win.sh

@@ -3,6 +3,3 @@
 bash ci/version.sh
 python grammar/asdl_cpp.py
 python grammar/asdl_cpp.py grammar/ASR.asdl src/libasr/asr.h
-pushd src/lpython/parser && re2c -W -b tokenizer.re -o tokenizer.cpp && popd
-pushd src/lpython/parser && re2c -W -b preprocessor.re -o preprocessor.cpp && popd
-pushd src/lpython/parser && bison -Wall -d -r all parser.yy && popd

ci/build.xsh

@@ -35,11 +35,6 @@ python grammar/asdl_cpp.py grammar/Python.asdl src/lpython/python_ast.h
 # Generate a Python AST from Python.asdl (Python)
 python grammar/asdl_py.py
 
-# Generate the tokenizer and parser
-pushd src/lpython/parser && re2c -W -b tokenizer.re -o tokenizer.cpp && popd
-pushd src/lpython/parser && re2c -W -b preprocessor.re -o preprocessor.cpp && popd
-pushd src/lpython/parser && bison -Wall -d -r all parser.yy && popd
-
 $lpython_version=$(cat version).strip()
 $dest="lpython-" + $lpython_version
 bash ci/create_source_tarball0.sh

src/bin/lpython.cpp

@@ -7,8 +7,6 @@
 #include <bin/CLI11.hpp>
 
 #include <libasr/stacktrace.h>
-#include <lpython/parser/parser.h>
-#include <lpython/parser/preprocessor.h>
 #include <lpython/pickle.h>
 #include <lpython/semantics/python_ast_to_asr.h>
 #include <lpython/mod_to_asr.h>
@@ -35,7 +33,6 @@
 #include <libasr/string_utils.h>
 #include <lpython/utils.h>
 #include <lpython/python_serialization.h>
-#include <lpython/parser/parser.tab.hh>
 
 #include <cpp-terminal/terminal.h>
 #include <cpp-terminal/prompt0.h>
@@ -97,49 +94,6 @@ std::string get_kokkos_dir()
 
 #endif
 
-int emit_prescan(const std::string &infile, CompilerOptions &compiler_options)
-{
-    std::string input = read_file(infile);
-    LFortran::LocationManager lm;
-    lm.in_filename = infile;
-    lm.init_simple(input);
-    std::string prescan = LFortran::fix_continuation(input, lm,
-        compiler_options.fixed_form);
-    std::cout << prescan << std::endl;
-    return 0;
-}
-
-int emit_tokens(const std::string &infile, bool line_numbers, const CompilerOptions &compiler_options)
-{
-    std::string input = read_file(infile);
-    // Src -> Tokens
-    Allocator al(64*1024*1024);
-    std::vector<int> toks;
-    std::vector<LFortran::YYSTYPE> stypes;
-    std::vector<LFortran::Location> locations;
-    LFortran::diag::Diagnostics diagnostics;
-    auto res = LFortran::tokens(al, input, diagnostics, &stypes, &locations);
-    LFortran::LocationManager lm;
-    lm.in_filename = infile;
-    lm.init_simple(input);
-    std::cerr << diagnostics.render(input, lm, compiler_options);
-    if (res.ok) {
-        toks = res.result;
-    } else {
-        LFORTRAN_ASSERT(diagnostics.has_error())
-        return 1;
-    }
-
-    for (size_t i=0; i < toks.size(); i++) {
-        std::cout << LFortran::pickle(toks[i], stypes[i]);
-        if (line_numbers) {
-            std::cout << " " << locations[i].first << ":" << locations[i].last;
-        }
-        std::cout << std::endl;
-    }
-    return 0;
-}
-
 int emit_ast(const std::string &infile,
     const std::string &runtime_library_dir,
     CompilerOptions &compiler_options)
@@ -509,8 +463,6 @@ int main(int argc, char *argv[])
         std::string arg_o;
         std::vector<std::string> arg_files;
         bool arg_version = false;
-        bool show_prescan = false;
-        bool show_tokens = false;
         bool show_ast = false;
         bool show_asr = false;
         bool show_cpp = false;
@@ -561,8 +513,6 @@ int main(int argc, char *argv[])
 
         // LPython specific options
         app.add_flag("--cpp", compiler_options.c_preprocessor, "Enable C preprocessing");
-        app.add_flag("--show-prescan", show_prescan, "Show tokens for the given file and exit");
-        app.add_flag("--show-tokens", show_tokens, "Show tokens for the given file and exit");
         app.add_flag("--show-ast", show_ast, "Show AST for the given python file and exit");
         app.add_flag("--show-asr", show_asr, "Show ASR for the given python file and exit");
         app.add_flag("--show-llvm", show_llvm, "Show LLVM IR for the given file and exit");
@@ -703,10 +653,6 @@ int main(int argc, char *argv[])
             outfile = basename + ".s";
         } else if (arg_c) {
             outfile = basename + ".o";
-        } else if (show_prescan) {
-            outfile = basename + ".prescan";
-        } else if (show_tokens) {
-            outfile = basename + ".tokens";
         } else if (show_ast) {
             outfile = basename + ".ast";
         } else if (show_asr) {
@@ -721,12 +667,6 @@ int main(int argc, char *argv[])
         //     return emit_c_preprocessor(arg_file, compiler_options);
         // }
 
-        if (show_prescan) {
-            return emit_prescan(arg_file, compiler_options);
-        }
-        if (show_tokens) {
-            return emit_tokens(arg_file, false, compiler_options);
-        }
         std::vector<ASRPass> passes;
         if (arg_pass != "") {
             if (arg_pass == "do_loops") {

src/lpython/CMakeLists.txt

@@ -1,17 +1,11 @@
 set(SRC
-    parser/preprocessor.cpp
-    parser/tokenizer.cpp
-    parser/parser.tab.cc
-    parser/parser.cpp
-
     semantics/python_ast_to_asr.cpp
 
     fortran_evaluator.cpp
 
     pickle.cpp
     python_serialization.cpp
     ast_serialization.cpp
-    cwrapper.cpp
 
     ast_to_src.cpp
     ast_to_openmp.cpp
@@ -27,21 +21,13 @@ if (WITH_XEUS)
 #        fortran_kernel.cpp
     )
 endif()
-if (WITH_JSON)
-    set(SRC ${SRC}
-        ast_to_json.cpp
-    )
-endif()
 add_library(lfortran_lib ${SRC})
 target_link_libraries(lfortran_lib asr lfortran_runtime_static ZLIB::ZLIB)
 target_include_directories(lfortran_lib BEFORE PUBLIC ${lfortran_SOURCE_DIR}/src)
 target_include_directories(lfortran_lib BEFORE PUBLIC ${lfortran_BINARY_DIR}/src)
 if (WITH_XEUS)
     target_link_libraries(lfortran_lib xeus)
 endif()
-if (WITH_JSON)
-    target_link_libraries(lfortran_lib p::rapidjson)
-endif()
 if (WITH_BFD)
     target_link_libraries(lfortran_lib p::bfd)
 endif()

src/lpython/ast_serialization.cpp

@@ -2,8 +2,6 @@
 
 #include <libasr/config.h>
 #include <lpython/ast_serialization.h>
-#include <lpython/parser/parser.h>
-#include <lpython/parser/parser.tab.hh>
 #include <libasr/asr_utils.h>
 #include <libasr/asr_verify.h>
 #include <libasr/bwriter.h>