Skip to content

Commit

Permalink
Add tokenizer and parser
Browse files Browse the repository at this point in the history
  • Loading branch information
certik committed Aug 13, 2019
1 parent daddc0f commit eae3132
Show file tree
Hide file tree
Showing 20 changed files with 753 additions and 9 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ lfortran/parser/fortranLexer.tokens
lfortran/parser/fortranLexer.interp
lfortran/parser/fortranParser.py
lfortran/parser/fortranVisitor.py
src/lfortran/parser/parser.tab.cc
src/lfortran/parser/parser.tab.hh
src/lfortran/parser/tokenizer.cpp


# Python
Expand Down
10 changes: 5 additions & 5 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ before_script:

prepare tarball:
stage: tarball
image: registry.gitlab.com/lfortran/ci-images:prepare-1.4.0
image: registry.gitlab.com/lfortran/ci-images:prepare-1.5.1
script:
- export CLASSPATH="$HOME/antlr-4.7-complete.jar:$CLASSPATH"
- sudo ln /usr/bin/pip3 /usr/bin/pip
Expand Down Expand Up @@ -83,7 +83,7 @@ full:
# Build and upload documentation
documentation:
stage: build
image: registry.gitlab.com/lfortran/ci-images:jupyter-1.4.0
image: registry.gitlab.com/lfortran/ci-images:jupyter-1.5.1
script:
- sudo apt-get update
- sudo apt-get install -yq --no-install-recommends openssh-client
Expand Down Expand Up @@ -117,7 +117,7 @@ downloads_update:
# compilers, just the linker `ld` and the musl C library.
minimal:
stage: build
image: registry.gitlab.com/lfortran/ci-images:minimal-1.4.0
image: registry.gitlab.com/lfortran/ci-images:minimal-1.5.1
script:
- export PATH="$HOME/conda_root/bin:$PATH"
- tar xzf dist/lfortran-${lfortran_version}.tar.gz
Expand All @@ -127,7 +127,7 @@ minimal:
- cd ..
- rm -r lfortran lfortran-${lfortran_version}
- rm lfort
- sudo apt-get remove -yq gcc libc6-dev
- sudo apt-get remove -yq gcc g++ libc6-dev
- sudo apt-get autoremove -yq
- py.test --pyargs lfortran
- lfort --ld-musl examples/expr2.f90 -o a.out
Expand All @@ -136,7 +136,7 @@ minimal:
# Test Jupyter notebooks
jupyter:
stage: build
image: registry.gitlab.com/lfortran/ci-images:jupyter-1.4.0
image: registry.gitlab.com/lfortran/ci-images:jupyter-1.5.1
script:
- export PATH="$HOME/conda_root/bin:$PATH"
- tar xzf dist/lfortran-${lfortran_version}.tar.gz
Expand Down
10 changes: 9 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.5 FATAL_ERROR)

project(lfort C CXX)
project(lfortran C CXX)

if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release
Expand All @@ -12,6 +12,14 @@ if (NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR
message(FATAL_ERROR "CMAKE_BUILD_TYPE must be one of: Debug, Release (current value: '${CMAKE_BUILD_TYPE}')")
endif ()

# Enable C++11 support in all compilers.
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|Intel")
set(CXX11_OPTIONS "-std=c++11")
endif ()
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CXX11_OPTIONS}")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CXX11_OPTIONS}")


enable_testing()

message("\n")
Expand Down
5 changes: 5 additions & 0 deletions build0.bat
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,8 @@ python grammar\asdl_py.py grammar\ASR.asdl lfortran\asr\asr.py ..ast.utils
cd grammar
call antlr4 -Dlanguage=Python3 -no-listener -visitor fortran.g4 -o ..\lfortran\parser
cd ..

cd src\lfortran\parser
re2c -W -b tokenizer.re -o tokenizer.cpp
bison -Wall -d parser.yy
cd ..\..\..
6 changes: 5 additions & 1 deletion build0.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,8 @@ python grammar/asdl_py.py grammar/ASR.asdl lfortran/asr/asr.py ..ast.utils

# Generate a parse tree from fortran.g4
antlr4="java org.antlr.v4.Tool"
(cd grammar; $antlr4 -Dlanguage=Python3 -no-listener -visitor fortran.g4 -o ../lfortran/parser)
(cd grammar && $antlr4 -Dlanguage=Python3 -no-listener -visitor fortran.g4 -o ../lfortran/parser)

# Generate the tokenizer and parser
(cd src/lfortran/parser && re2c -W -b tokenizer.re -o tokenizer.cpp)
(cd src/lfortran/parser && bison -Wall -d parser.yy)
8 changes: 8 additions & 0 deletions ci/appveyor_build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ set BUILD_TYPE=Debug
call :check
call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
call :check
cd src\lfortran\parser
call :check
re2c -W -b tokenizer.re -o tokenizer.cpp
call :check
bison -Wall -d parser.yy
call :check
cd ..\..\..
call :check
set lfortran_version=0.0+git
call :check
pip install scikit-build
Expand Down
2 changes: 1 addition & 1 deletion ci/appveyor_install.bat
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ call conda info -a
call :check
call conda update -q conda
call :check
call conda install python=3.7
call conda install -c conda-forge python=3.7 re2c m2-bison
call :check

cd grammar
Expand Down
5 changes: 4 additions & 1 deletion ci/azure_install_macos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -ex
conda config --set always_yes yes --set changeps1 no
conda info -a
conda update -q conda
conda install python=3.7
conda install -c conda-forge python=3.7 re2c bison m4

python grammar/asdl_py.py
python grammar/asdl_py.py grammar/ASR.asdl lfortran/asr/asr.py ..ast.utils
Expand All @@ -15,6 +15,9 @@ curl -O https://www.antlr.org/download/antlr-4.7-complete.jar
java -cp antlr-4.7-complete.jar org.antlr.v4.Tool -Dlanguage=Python3 -no-listener -visitor fortran.g4 -o ../lfortran/parser
cd ..

(cd src/lfortran/parser && re2c -W -b tokenizer.re -o tokenizer.cpp)
(cd src/lfortran/parser && bison -Wall -d parser.yy)

ci/version.sh
lfortran_version=$(<version)

Expand Down
1 change: 1 addition & 0 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
add_subdirectory(lfortran)
add_subdirectory(runtime)
11 changes: 11 additions & 0 deletions src/lfortran/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
set(SRC
parser/tokenizer.cpp
parser/parser.tab.cc
)
add_library(lfortran_lib ${SRC})
target_include_directories(lfortran_lib BEFORE PUBLIC ${lfortran_SOURCE_DIR}/src)
install(TARGETS lfortran_lib
RUNTIME DESTINATION bin
ARCHIVE DESTINATION lib
LIBRARY DESTINATION lib
)
50 changes: 50 additions & 0 deletions src/lfortran/assert.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#ifndef LFORTRAN_ASSERT_H
#define LFORTRAN_ASSERT_H

// LFORTRAN_ASSERT uses internal functions to perform as assert
// so that there is no effect with NDEBUG
#if defined(WITH_LFORTRAN_ASSERT)

#if !defined(LFORTRAN_ASSERT)
#define stringize(s) #s
#define XSTR(s) stringize(s)
#define LFORTRAN_ASSERT(cond) \
{ \
if (!(cond)) { \
std::cerr << "LFORTRAN_ASSERT failed: " << __FILE__ \
<< "\nfunction " << __func__ << "(), line number " \
<< __LINE__ << " at \n" \
<< XSTR(cond) << "\n"; \
abort(); \
} \
}
#endif // !defined(LFORTRAN_ASSERT)

#if !defined(LFORTRAN_ASSERT_MSG)
#define LFORTRAN_ASSERT_MSG(cond, msg) \
{ \
if (!(cond)) { \
std::cerr << "LFORTRAN_ASSERT failed: " << __FILE__ \
<< "\nfunction " << __func__ << "(), line number " \
<< __LINE__ << " at \n" \
<< XSTR(cond) << "\n" \
<< "ERROR MESSAGE:\n" \
<< msg << "\n"; \
abort(); \
} \
}
#endif // !defined(LFORTRAN_ASSERT_MSG)

#else // defined(WITH_LFORTRAN_ASSERT)

#define LFORTRAN_ASSERT(cond)
#define LFORTRAN_ASSERT_MSG(cond, msg)

#endif // defined(WITH_LFORTRAN_ASSERT)

#define LFORTRAN_ERROR(description) \
std::cerr << description; \
std::cerr << "\n"; \
abort();

#endif // LFORTRAN_ASSERT_H
140 changes: 140 additions & 0 deletions src/lfortran/casts.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
#ifndef LFORTRAN_CASTS_H
#define LFORTRAN_CASTS_H

#include <iostream>
#include <limits>
//#include <lfortran/config.h>
#include <lfortran/assert.h>

namespace LFortran
{

// Reference modifications.
template <typename T>
struct remove_reference {
typedef T type;
};
template <typename T>
struct remove_reference<T &> {
typedef T type;
};

// Use implicit_cast as a safe version of static_cast or const_cast
// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
// a const pointer to Foo).
// When you use implicit_cast, the compiler checks that the cast is safe.
// Such explicit implicit_casts are necessary in surprisingly many
// situations where C++ demands an exact type match instead of an
// argument type convertable to a target type.
//
// The From type can be inferred, so the preferred syntax for using
// implicit_cast is the same as for static_cast etc.:
//
// implicit_cast<ToType>(expr)

template <typename To, typename From>
inline To implicit_cast(const From &f)
{
return f;
}

// When you upcast (that is, cast a pointer from type Foo to type
// SuperclassOfFoo), it's fine to use implicit_cast<>, since upcasts
// always succeed. When you downcast (that is, cast a pointer from
// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because
// how do you know the pointer is really of type SubclassOfFoo? It
// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus,
// when you downcast, you should use this macro. In debug mode, we
// use dynamic_cast<> to double-check the downcast is legal (we die
// if it's not). In normal mode, we do the efficient static_cast<>
// instead. Thus, it's important to test in debug mode to make sure
// the cast is legal!
// This is the only place in the code we should use dynamic_cast<>.
// In particular, you SHOULDN'T be using dynamic_cast<> in order to
// do RTTI (eg code like this:
// if (dynamic_cast<Subclass1>(foo)) HandleASubclass1Object(foo);
// if (dynamic_cast<Subclass2>(foo)) HandleASubclass2Object(foo);
// You should design the code some other way not to need this.

template <typename To, typename From> // use like this: down_cast<T*>(foo).
inline To down_cast(From *f) // Only accept pointers.
{
// Ensures that To is a sub-type of From *. This test is here only
// for compile-time type checking, and has no overhead in an
// optimized build at run-time, as it will be optimized away
// completely.
if (false) {
implicit_cast<From *, To>(0);
}

LFORTRAN_ASSERT(f == NULL || dynamic_cast<To>(f) != NULL);

return static_cast<To>(f);
}

template <typename To, typename From> // use like this: down_cast<T&>(foo);
inline To down_cast(From &f)
{
typedef typename remove_reference<To>::type *ToAsPointer;
// Ensures that To is a sub-type of From *. This test is here only
// for compile-time type checking, and has no overhead in an
// optimized build at run-time, as it will be optimized away
// completely.
if (false) {
implicit_cast<From *, ToAsPointer>(0);
}

LFORTRAN_ASSERT(dynamic_cast<ToAsPointer>(&f) != NULL);

return *static_cast<ToAsPointer>(&f);
}

template <typename To, typename From>
inline To
numeric_cast(From f,
typename std::enable_if<(std::is_signed<From>::value
&& std::is_signed<To>::value)
|| (std::is_unsigned<From>::value
&& std::is_unsigned<To>::value)>::type
* = nullptr)
{
LFORTRAN_ASSERT(f <= std::numeric_limits<To>::max());
LFORTRAN_ASSERT(f >= std::numeric_limits<To>::min());
return static_cast<To>(f);
}

template <typename To, typename From>
inline To numeric_cast(
From f,
typename std::enable_if<(std::is_signed<From>::value
&& std::is_unsigned<To>::value)>::type * = nullptr)
{
#ifdef WITH_LFORTRAN_ASSERT
// Above ifdef is needed to avoid a warning about unused typedefs
typedef typename std::make_unsigned<From>::type unsigned_from_type;
LFORTRAN_ASSERT(f >= 0);
LFORTRAN_ASSERT(static_cast<unsigned_from_type>(f)
<= std::numeric_limits<To>::max());
#endif
return static_cast<To>(f);
}

template <typename To, typename From>
inline To numeric_cast(
From f,
typename std::enable_if<(std::is_unsigned<From>::value
&& std::is_signed<To>::value)>::type * = nullptr)
{
#ifdef WITH_LFORTRAN_ASSERT
typedef typename std::make_unsigned<To>::type unsigned_to_type;
LFORTRAN_ASSERT(
f <= static_cast<unsigned_to_type>(std::numeric_limits<To>::max()));

#endif
return static_cast<To>(f);
}

} // LFortran

#endif // LFORTRAN_CASTS_H
Loading

0 comments on commit eae3132

Please sign in to comment.