tests.toml

# Possible tests:
# ast ... run the Parser and output AST, compare against reference version
# ast_new ... run the New Parser and output AST, compare against reference version
# asr ... run the Semantics and output ASR, compare against reference version
# llvm ... run the Semantics and output LLVM, compare against reference version
# llvm_dbg ... run the Semantics and output LLVM with debug information
# tokens ... output Tokens, compare against reference version
# run ... compiles and executes the specified file;
#         checks both runtime errors and successful runs
# run_with_dbg ... runtime checks with debug information enabled

# tests

[[test]]
filename = "doconcurrentloop_01.py"
ast = true
asr = true
cpp = true

[[test]]
filename = "complex1.py"
ast = true
asr = true

[[test]]
filename = "complex2.py"
# cpp = true

[[test]]
filename = "constants1.py"
ast = true
asr = true

[[test]]
filename = "subscript1.py"
ast = true
asr = true

[[test]]
filename = "list1.py"
ast = true
asr = true

[[test]]
filename = "tuple1.py"
ast = true
asr = true

[[test]]
filename = "dictionary1.py"
ast = true
asr = true

[[test]]
filename = "expr1.py"
ast = true
asr = true

[[test]]
filename = "expr2.py"
ast = true
asr = true
cpp = true
wat = true
python = true

[[test]]
filename = "expr4.py"
ast = true
asr = true
python = true

[[test]]
filename = "expr5.py"
ast = true
asr = true
cpp = true
python = true

[[test]]
filename = "expr6.py"
ast = true
asr = true
cpp = true
python = true

[[test]]
filename = "expr7.py"
ast = true
asr = true
cpp = true
c = true

[[test]]
filename = "expr8.py"
ast = true
asr = true
cpp = true
#llvm = true

[[test]]
filename = "expr9.py"
ast = true
asr = true
cpp = true
wat = true

[[test]]
filename = "expr10.py"
ast = true
asr = true

[[test]]
filename = "expr11.py"
ast = true
asr = true
python = true

[[test]]
filename = "expr12.py"
ast = true
asr = true
cpp = true

[[test]]
filename = "expr13.py"
ast = true
asr = true

[[test]]
filename = "expr14.py"
llvm = true
wat = true
python = true

[[test]]
filename = "expr15.py"
cpp = true

[[test]]
filename = "expr16.py"
asr = true

[[test]]
filename = "expr17.py"
python = true

[[test]]
filename = "expr_01.py"
ast = true
asr = true
llvm = true
llvm_dbg = true

[[test]]
filename = "../integration_tests/array_01_decl.py"
asr = true

[[test]]
filename = "../integration_tests/array_02_decl.py"
asr = true

[[test]]
filename = "../integration_tests/expr_07.py"
asr = true

[[test]]
filename = "../integration_tests/test_numpy_03.py"
asr = true

[[test]]
filename = "../integration_tests/test_numpy_04.py"
asr = true

[[test]]
filename = "../integration_tests/expr_01.py"
asr = true
c = true

[[test]]
filename = "../integration_tests/expr_09.py"
asr = true

[[test]]
filename = "../integration_tests/elemental_01.py"
asr = true

[[test]]
filename = "../integration_tests/expr_10.py"
asr = true

[[test]]
filename = "../integration_tests/expr_11.py"
c = true
cpp = true

[[test]]
filename = "../integration_tests/expr_12.py"
asr = true
c = true

[[test]]
filename = "../integration_tests/expr_14.py"
asr = true

[[test]]
filename = "../integration_tests/modules_02.py"
asr = true
asr_json = true

[[test]]
filename = "../integration_tests/test_import_02.py"
c = true

[[test]]
filename = "../integration_tests/import_order_01.py"
c = true
disable_main = true

[[test]]
filename = "../integration_tests/vec_01.py"
asr = true
pass = "loop_vectorise"

[[test]]
filename = "../integration_tests/func_inline_01.py"
asr = true
llvm = true
pass_with_llvm = true
pass = "inline_function_calls"
fast = true

# to make sure static keyword is actually present in function declaration
[[test]]
filename = "../integration_tests/func_static_01.py"
c = true

[[test]]
filename = "loop1.py"
ast = true
asr = true
cpp = true
c = true
wat = true

[[test]]
filename = "loop3.py"
ast = true
asr = true
cpp = true

[[test]]
filename = "loop4.py"
asr = true
c = true
cpp = true

[[test]]
filename = "assert1.py"
ast = true
asr = true
cpp = true
llvm = true
python = true

[[test]]
filename = "assign1.py"
ast = true
asr = true
python = true

[[test]]
filename = "assign2.py"
asr = true

[[test]]
filename = "set1.py"
ast = true
asr = true

[[test]]
filename = "bool1.py"
llvm = true
wat = true

[[test]]
filename = "global_scope1.py"
ast = true
asr = true

[[test]]
filename = "c_interop1.py"
asr = true
c = true

[[test]]
filename = "test_end_sep_keywords.py"
asr = true

[[test]]
filename = "lpython1.py"
llvm = true

[[test]]
filename = "print_str.py"
wat = true

[[test]]
filename = "intent_01.py"
asr = true

# integration_tests

[[test]]
filename = "../integration_tests/test_list_repeat2.py"
cpp = true

[[test]]
filename = "../integration_tests/test_builtin.py"
asr = true

[[test]]
filename = "../integration_tests/test_complex_01.py"
asr = true

[[test]]
filename = "../integration_tests/test_complex_02.py"
asr = true

[[test]]
filename = "../integration_tests/structs_01.py"
asr = true

[[test]]
filename = "../integration_tests/structs_02.py"
asr = true

[[test]]
filename = "../integration_tests/structs_03.py"
asr = true

[[test]]
filename = "../integration_tests/structs_04.py"
asr = true

[[test]]
filename = "../integration_tests/structs_05.py"
asr = true

[[test]]
filename = "../integration_tests/structs_16.py"
asr = true
pass = "class_constructor"
cumulative = true

[[test]]
filename = "../integration_tests/callback_01.py"
asr = true

[[test]]
filename = "../integration_tests/bindc_01.py"
asr = true
llvm = true

[[test]]
filename = "../integration_tests/bindc_02.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_abs.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_str.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_bool.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_len.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_pow.py"
asr = true
cpp = true

[[test]]
filename = "../integration_tests/test_builtin_int.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_float.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_round.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_bin.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_hex.py"
asr = true

[[test]]
filename = "../integration_tests/test_builtin_oct.py"
asr = true

[[test]]
filename = "../integration_tests/test_max_min.py"
asr = true

[[test]]
filename = "../integration_tests/expr_05.py"
asr = true

[[test]]
filename = "../integration_tests/test_unary_op_03.py"
asr = true
llvm = true
cpp = true

[[test]]
filename = "../integration_tests/test_bool_binop.py"
asr = true

[[test]]
filename = "../integration_tests/test_issue_518.py"
llvm = true
c = true

[[test]]
filename = "../integration_tests/test_c_interop_01.py"
asr = true

[[test]]
filename = "../integration_tests/print_01.py"
c = true
cpp = true

[[test]]
filename = "../integration_tests/variable_decl_03.py"
c = true

[[test]]
filename = "../integration_tests/print_02.py"
asr = true
pass = "print_list_tuple"

[[test]]
filename = "../integration_tests/print_list_tuple_03.py"
asr = true
pass = "print_list_tuple"

[[test]]
filename = "../integration_tests/print_04.py"
llvm = true

[[test]]
filename = "../integration_tests/test_list_item_mixed_print.py"
run = true

[[test]]
filename = "../integration_tests/generics_01.py"
asr = true

[[test]]
filename = "../integration_tests/generics_02.py"
asr = true

[[test]]
filename = "../integration_tests/generics_array_01.py"
asr = true

[[test]]
filename = "../integration_tests/generics_array_02.py"
asr = true

[[test]]
filename = "../integration_tests/generics_array_03.py"
asr = true

[[test]]
filename = "../integration_tests/generics_list_01.py"
asr = true

[[test]]
filename = "../integration_tests/global_syms_01.py"
asr = true

[[test]]
filename = "cast.py"
asr = true

# tests/tokens

[[test]]
filename = "tokens/docstring1.py"
tokens = true

[[test]]
filename = "tokens/comment1.py"
tokens = true

[[test]]
filename = "tokens/comment2.py"
tokens = true
ast_new = true

[[test]]
filename = "tokens/symbols1.py"
tokens = true

[[test]]
filename = "tokens/numbers1.py"
tokens = true

[[test]]
filename = "tokens/indent1.py"
tokens = true

[[test]]
filename = "tokens/indent2.py"
tokens = true

# tests/parser

[[test]]
filename = "parser/import1.py"
ast_new = true

[[test]]
filename = "parser/statements1.py"
ast_new = true

[[test]]
filename = "parser/statements2.py"
ast_new = true

[[test]]
filename = "parser/yield.py"
ast_new = true

[[test]]
filename = "parser/if1.py"
ast_new = true

[[test]]
filename = "parser/for1.py"
ast_new = true

[[test]]
filename = "parser/comprehension1.py"
ast_new = true

[[test]]
filename = "parser/conditional_expr1.py"
ast_new = true

[[test]]
filename = "parser/try1.py"
ast_new = true

[[test]]
filename = "parser/function_def1.py"
ast_new = true

[[test]]
filename = "parser/function_def2.py"
ast_new = true

[[test]]
filename = "parser/function_def3.py"
ast_new = true

[[test]]
filename = "parser/class_def1.py"
ast_new = true

[[test]]
filename = "parser/class_def2.py"
ast_new = true

[[test]]
filename = "parser/dictionary1.py"
ast_new = true

[[test]]
filename = "parser/with1.py"
ast_new = true

[[test]]
filename = "parser/async1.py"
ast_new = true

[[test]]
filename = "parser/slice1.py"
ast_new = true

[[test]]
filename = "parser/ellipsis1.py"
ast = true

[[test]]
filename = "parser/ellipsis2.py"
ast_new = true

[[test]]
filename = "parser/while1.py"
ast_new = true

[[test]]
filename = "parser/string1.py"
ast_new = true

[[test]]
filename = "parser/string2.py"
ast_new = true

[[test]]
filename = "parser/string3.py"
ast_new = true

[[test]]
filename = "parser/global1.py"
ast = true
ast_new = true

[[test]]
filename = "parser/if2.py"
ast_new = true

[[test]]
filename = "parser/for2.py"
ast_new = true

[[test]]
filename = "parser/boolOp1.py"
ast_new = true

[[test]]
filename = "parser/lambda1.py"
ast_new = true

[[test]]
filename = "parser/lambda2.py"
ast_new = true

[[test]]
filename = "parser/tuple1.py"
ast_new = true

[[test]]
filename = "errors/lambda_01.py"
asr = true

[[test]]
filename = "errors/test_bit_length.py"
asr = true

[[test]]
filename = "errors/string_01.py"
asr = true

[[test]]
filename = "errors/string_02.py"
asr = true

[[test]]
filename = "errors/structs_01.py"
asr = true

[[test]]
filename = "errors/arrays_01.py"
asr = true

[[test]]
filename = "errors/arrays_02.py"
asr = true

[[test]]
filename = "errors/arrays_03.py"
asr = true

[[test]]
filename = "errors/arrays_04.py"
asr = true

[[test]]
filename = "errors/arrays_05.py"
asr = true

[[test]]
filename = "errors/arrays_06.py"
asr = true

[[test]]
filename = "errors/arrays_07.py"
asr = true

[[test]]
filename = "errors/arrays_08.py"
asr = true

[[test]]
filename = "errors/arrays_09.py"
asr = true

[[test]]
filename = "errors/arrays_10.py"
asr = true

[[test]]
filename = "errors/arrays_11.py"
asr = true

[[test]]
filename = "errors/arrays_12.py"
asr = true

[[test]]
filename = "errors/arrays_13.py"
asr = true

[[test]]
filename = "errors/arrays_14.py"
asr = true

[[test]]
filename = "errors/structs_02.py"
asr = true

[[test]]
filename = "errors/structs_03.py"
asr = true

[[test]]
filename = "errors/structs_04.py"
asr = true

[[test]]
filename = "structs_01.py"
asr = true

[[test]]
filename = "errors/structs_06.py"
asr = true

[[test]]
filename = "errors/structs_07.py"
asr = true

[[test]]
filename = "errors/structs_08.py"
asr = true

[[test]]
filename = "errors/structs_09.py"
asr = true

[[test]]
filename = "errors/structs_10.py"
asr = true

[[test]]
filename = "structs_11.py"
llvm = true

[[test]]
filename = "errors/const_01.py"
asr = true

[[test]]
filename = "errors/const_02.py"
asr = true

[[test]]
filename = "errors/test_compare_01.py"
asr = true

[[test]]
filename = "errors/test_compare_02.py"
asr = true

[[test]]
filename = "parser/type_comment1.py"
ast_new = true

[[test]]
filename = "parser/unicode.py"
ast_new = true

[[test]]
filename = "parser/match_stmt1.py"
ast_new = true

# tests/errors

[[test]]
filename = "errors/test_async.py"
asr = true

[[test]]
filename = "errors/test_str_indexing.py"
asr = true

[[test]]
filename = "errors/test_set_indexing.py"
asr = true

[[test]]
filename = "errors/test_str_slicing.py"
asr = true

[[test]]
filename = "errors/test_str_slicing2.py"
asr = true

[[test]]
filename = "errors/test_str_slicing3.py"
asr = true

[[test]]
filename = "errors/test_str_slicing4.py"
asr = true

[[test]]
filename = "errors/test_list_slicing.py"
asr = true

[[test]]
filename = "errors/test_const_list_append.py"
asr = true

[[test]]
filename = "errors/test_const_list_clear.py"
asr = true

[[test]]
filename = "errors/test_const_list_insert.py"
asr = true

[[test]]
filename = "errors/test_const_list_pop.py"
asr = true

[[test]]
filename = "errors/test_const_list_reverse.py"
asr = true

[[test]]
filename = "errors/test_const_list_remove.py"
asr = true

[[test]]
filename = "errors/test_annassign_01.py"
asr = true

[[test]]
filename = "errors/test_annassign_02.py"
asr = true

[[test]]
filename = "errors/test_annassign_03.py"
asr = true

[[test]]
filename = "errors/test_annassign_type_mismatch.py"
asr = true

[[test]]
filename = "errors/test_annassign_type_mismatch2.py"
asr = true

[[test]]
filename = "errors/test_append_type_mismatch.py"
asr = true

[[test]]
filename = "errors/test_list_concat.py"
asr = true

[[test]]
filename = "errors/test_list_count.py"
asr = true

[[test]]
filename = "errors/test_list_index.py"
run = true

[[test]]
filename = "errors/test_list_index2.py"
run = true

[[test]]
filename = "errors/test_list_index3.py"
run = true

[[test]]
filename = "errors/test_list_index4.py"
run = true

[[test]]
filename = "errors/test_list1.py"
asr = true

[[test]]
filename = "errors/test_list2.py"
asr = true

[[test]]
filename = "errors/test_list3.py"
asr = true

[[test]]
filename = "errors/test_list4.py"
asr = true

[[test]]
filename = "errors/test_set1.py"
asr = true

[[test]]
filename = "errors/test_set2.py"
asr = true

[[test]]
filename = "errors/test_set3.py"
asr = true

[[test]]
filename = "errors/test_set4.py"
asr = true

[[test]]
filename = "errors/test_pow.py"
asr = true

[[test]]
filename = "errors/test_pow1.py"
asr = true

[[test]]
filename = "errors/test_pow2.py"
asr = true

[[test]]
filename = "errors/test_pow3.py"
asr = true

[[test]]
filename = "errors/test_pow4.py"
asr = true

[[test]]
filename = "errors/test_operator_01.py"
asr = true

[[test]]
filename = "errors/test_ord.py"
asr = true

[[test]]
filename = "errors/test_chr.py"
asr = true

[[test]]
filename = "errors/test_assign1.py"
asr = true

[[test]]
filename = "errors/test_assign2.py"
asr = true

[[test]]
filename = "errors/test_assign3.py"
asr = true

[[test]]
filename = "errors/test_assign4.py"
asr = true

[[test]]
filename = "errors/test_assign5.py"
asr = true

[[test]]
filename = "errors/test_assign6.py"
asr = true

[[test]]
filename = "errors/test_assign7.py"
asr = true

[[test]]
filename = "errors/test_assign8.py"
asr = true

[[test]]
filename = "errors/test_assign9.py"
asr = true

[[test]]
filename = "errors/test_binop1.py"
asr = true

[[test]]
filename = "errors/test_binop2.py"
asr = true

[[test]]
filename = "errors/test_binop3.py"
asr = true

[[test]]
filename = "errors/test_dict1.py"
asr = true

[[test]]
filename = "errors/test_dict8.py"
asr = true

[[test]]
filename = "errors/test_dict9.py"
asr = true

[[test]]
filename = "errors/test_dict10.py"
asr = true

[[test]]
filename = "errors/test_dict11.py"
asr = true

[[test]]
filename = "errors/test_dict12.py"
asr = true

[[test]]
filename = "errors/test_dict13.py"
asr = true

[[test]]
filename = "errors/test_dict14.py"
run = true

[[test]]
filename = "errors/test_dict15.py"
run = true

[[test]]
filename = "errors/test_dict16.py"
run = true

[[test]]
filename = "errors/test_zero_division.py"
asr = true

[[test]]
filename = "errors/test_zero_division2.py"
asr = true

[[test]]
filename = "errors/test_zero_division3.py"
asr = true

[[test]]
filename = "errors/test_zero_division4.py"
asr = true

[[test]]
filename = "errors/test_len1.py"
asr = true

[[test]]
filename = "errors/test_return1.py"
asr = true

[[test]]
filename = "errors/test_return2.py"
asr = true

[[test]]
filename = "errors/test_dict2.py"
asr = true

[[test]]
filename = "errors/test_dict3.py"
asr = true

[[test]]
filename = "errors/test_dict4.py"
asr = true

[[test]]
filename = "errors/test_dict5.py"
asr = true

[[test]]
filename = "errors/test_dict6.py"
asr = true

[[test]]
filename = "errors/test_dict7.py"
asr = true

[[test]]
filename = "errors/test_func_args.py"
asr = true

[[test]]
filename = "errors/test_str_to_int.py"
asr = true

[[test]]
filename = "errors/test_bitwise_on_float.py"
asr = true

[[test]]
filename = "errors/test_bitwise_on_complex.py"
asr = true

[[test]]
filename = "errors/test_print1.py"
asr = true

[[test]]
filename = "errors/test_print2.py"
asr = true

[[test]]
filename = "errors/test_pointer_types.py"
asr = true

[[test]]
filename = "errors/test_unsupported_type.py"
asr = true

[[test]]
filename = "errors/test_int_semantic_error.py"
asr = true

[[test]]
filename = "errors/test_float_semantic_error.py"
asr = true

[[test]]
filename = "errors/generics_error_01.py"
asr = true

[[test]]
filename = "errors/generics_error_02.py"
asr = true

[[test]]
filename = "errors/generics_error_03.py"
asr = true

[[test]]
filename = "errors/test_str_capitalize.py"
asr = true

[[test]]
filename = "errors/test_tuple1.py"
asr = true

[[test]]
filename = "errors/test_tuple2.py"
asr = true

[[test]]
filename = "errors/test_tuple3.py"
asr = true

[[test]]
filename = "errors/test_for2.py"
asr = true

[[test]]
filename = "errors/test_import_01.py"
asr = true

[[test]]
filename = "errors/test_import_02.py"
asr = true

[[test]]
filename = "errors/type_mismatch.py"
asr = true

[[test]]
filename = "errors/test_type_mismatch_01.py"
asr = true

# tests/tokens/errors

[[test]]
filename = "tokens/errors/paren1.py"
tokens = true

[[test]]
filename = "tokens/errors/paren2.py"
tokens = true

[[test]]
filename = "tokens/errors/paren3.py"
tokens = true

[[test]]
filename = "tokens/errors/paren4.py"
tokens = true

[[test]]
filename = "tokens/errors/indent1.py"
tokens = true

[[test]]
filename = "tokens/errors/indent2.py"
tokens = true

[[test]]
filename = "tokens/errors/indent3.py"
tokens = true

[[test]]
filename = "errors/test_literal.py"
tokens = true

[[test]]
filename = "errors/kwargs_01_error.py"
asr = true

[[test]]
filename = "errors/kwargs_02_error.py"
asr = true

[[test]]
filename = "errors/kwargs_03_error.py"
asr = true

[[test]]
filename = "errors/enum_01.py"
asr = true

[[test]]
filename = "errors/enum_02.py"
asr = true

[[test]]
filename = "errors/loop_01.py"
asr = true

[[test]]
filename = "errors/loop_02.py"
asr = true

[[test]]
filename = "errors/loop_03.py"
asr = true

[[test]]
filename = "errors/test_logical_compare_01.py"
asr = true

[[test]]
filename = "errors/bindc_01.py"
asr = true

[[test]]
filename = "errors/bindc_02.py"
asr = true

[[test]]
filename = "errors/bindc_03.py"
asr = true

[[test]]
filename = "errors/bindc_04.py"
asr = true

[[test]]
filename = "errors/bindc_10e.py"
asr = true

[[test]]
filename = "errors/cptr_01.py"
asr = true

[[test]]
filename = "errors/func_01.py"
asr = true

[[test]]
filename = "errors/func_02.py"
asr = true

[[test]]
filename = "errors/func_03.py"
asr = true

[[test]]
filename = "errors/func_04.py"
asr = true

[[test]]
filename = "errors/func_05.py"
asr = true

[[test]]
filename = "errors/func_06.py"
asr = true

[[test]]
filename = "errors/func_07.py"
asr = true

[[test]]
filename = "errors/func_08.py"
asr = true

[[test]]
filename = "errors/unsigned_01.py"
asr = true

[[test]]
filename = "errors/unsigned_02.py"
asr = true

[[test]]
filename = "errors/unsigned_03.py"
asr = true

[[test]]
filename = "errors/unsigned_04.py"
asr = true

[[test]]
filename = "errors/prefix_string_01.py"
ast = true

[[test]]
filename = "errors/prefix_string_02.py"
ast = true

# tests/runtime_errors
[[test]]
filename = "runtime_errors/test_list_01.py"
run = true

[[test]]
filename = "runtime_errors/test_list_02.py"
run = true

[[test]]
filename = "runtime_errors/test_str_01.py"
run = true

[[test]]
filename = "runtime_errors/test_str_02.py"
run = true

[[test]]
filename = "runtime_errors/test_assert_01.py"
run_with_dbg = true

[[test]]
filename = "runtime_errors/test_assert_02.py"
run_with_dbg = true

[[test]]
filename = "runtime_errors/test_assert_03.py"
run_with_dbg = true

[[test]]
filename = "runtime_errors/test_quit_01.py"
run_with_dbg = true

[[test]]
filename = "runtime_errors/test_raise_01.py"
run_with_dbg = true

[[test]]
filename = "errors/test_optional.py"
asr = true