feat(fuzzing): updating the precomputed corpus and adding scripts to …

…kickstart fuzzing with afl

.github/workflows/ci.yml

@@ -230,25 +230,11 @@ jobs:
             -DCMAKE_BUILD_TYPE=Release \
             -DARK_BUILD_EXE=On \
             -DARK_SANITIZERS=On
-          cmake --build ${BUILD_FOLDER} --config Release -j $(nproc)
-
-      - name: Generate corpus
-        run: |
-          mkdir -p unique
-          mkdir -p tmp
-          cp examples/*.ark tmp/
-          cp tests/arkscript/*.ark tmp/
-          cp tests/errors/capture/*.ark tmp/
-          cp tests/errors/callable/*.ark tmp/
-          cp tests/errors/mutability/*.ark tmp/
-          cp tests/errors/index/*.ark tmp/
-          cp tests/errors/type/*.ark tmp/
-          cp tests/errors/operators/*.ark tmp/
-          afl-cmin -i tmp -o unique -- ${BUILD_FOLDER}/arkscript @@ -L ./lib
+          cmake --build ${BUILD_FOLDER} --config Release -j 4
 
       - name: Fuzz
         run: |
-          afl-fuzz -i unique \
+          afl-fuzz -i tests/fuzzing/corpus-cmin-tmin \
             -o output \
             -s $FUZZER_SEED \
             -t $FUZZER_TIMEOUT_EXEC_MS \

.gitignore

@@ -16,9 +16,9 @@ include/Ark/Constants.hpp
 /Installer.iss
 
 # Fuzzing
-!fuzzing/**/*.arkc
-fuzzing/output*
-afl/
+/fuzzing/
+/output/
+/tmp/
 
 # Folders
 .vs/

tests/errors/operators/not_enough_args.expected

@@ -1 +1 @@
-Operator needs two arguments, but was called with only one
+Operator needs two arguments, but was called with 1

tests/fuzzing/corpus-cmin-tmin/can_not_call.ark

@@ -0,0 +1 @@
+ (((begin)))

tests/fuzzing/corpus-cmin-tmin/invalid_let.ark

@@ -0,0 +1,7 @@
+($ partial (func ...defar0s) {
+    ($ b000 (s00000-000 a (- (a000000t func) (len defar0s))))
+})
+
+(let t0000 (fun (a b c) (* a b c)))
+(let t000000000 (partial e000 0))
+(t000_00000)

tests/fuzzing/corpus-cmin-tmin/macro_at_out_of_range.ark

@@ -0,0 +1,2 @@
+($ last (...a00s) (p0000 a00s "0000" (@ a00s -9)))
+(last 1 5 6 7 8)

tests/fuzzing/corpus-cmin-tmin/nil_not_a_function.ark

@@ -0,0 +1,2 @@
+(())
+(fun(a b)(if 0 2 3))

tests/fuzzing/corpus-cmin-tmin/ope_freestanding.ark

@@ -0,0 +1,2 @@
+(let ba0 (fun (a b) ()))
+(ba0 0 0nil?)

tests/fuzzing/corpus-cmin-tmin/ope_no_args.ark

@@ -0,0 +1 @@
+((+))

tests/fuzzing/corpus-cmin-tmin/pop_in_place.ark

@@ -0,0 +1,2 @@
+(let a[])
+(pop! a 0)

tests/fuzzing/corpus-cmin-tmin/quicksort.ark

@@ -2,46 +2,45 @@
 
 (let filter (fun (lst cond) {
     (mut output [])
-  (mut i 0)
+    (mut i 0)
     (while (< i (len lst)) {
         (if (cond (@ lst i))
             (append! output (@ lst i)))
         (set i (+ 1 i))})
     output }))
 
-# a quicksort function in ArkScript, a lot sm0ler than its C++ version!
-# and according to me, a lo0pler to understand
-(let quicksort (fun (array) {
-    (if (empty? array)
-      # if the given list is empty, return it
+#000000000000000io0A0k00r000, a lot small00000000000000000e0sio0!
+#0and a0co0d0n0 to me000000t s0mpler to 0n0er0tand
+(let quicksort (fun (array) { (if (empty? array)
+        # if the 000en0lis0 i0 em0t0, re0000 it
         []
-        # otherwise, sort it
-        {
-            # the pivot will be the first elem0nt
-            (let pivot (head array))
-            # call quicksort on a smaller array containing all the elements less than the pivot
-            (mut less (quicksort (filter (tail array) (fun (e) (< e pivot)))))
-            # and 0fter that, call quicksort on a 0maller array containing al0 the elements gre0ter or equal to the pivot
-            (let more (quicksort (filter (tail array) (fun (e) (>= e pivot)))))
+        #0o0he00i0e, so0t it
+      {
+            #0t0000000t will 00 0he f000t0ele0ent
+            (let pi0ot (head array))
+            #00al0 q0icks000000 a s0000e0 array 0ontai0i00 all the000ements l0s0 t00n the pi00t
+            (mut less (quicksort (filter (tail array) (fun (e) (< e pi0ot)))))
+            # 0n00a0te0 th00,0c0l0 0u0cksort on a smal0er0a0r0y containin0 all 0h00ele000t000rea0er0o0 e00000000000000000
+            (let more (quicksort (filter (tail array) (fun (e) (>= e pi0ot)))))
 
-            (concat! less [pivot] more)
-            # return a concatenation of arrays
+            (concat! less [pi0ot] more)
+            # 0e000n0a0c0ncatenati00 of arra00
             less })}))
 
-# an unsorted list to sort
+#0an un000ted l0st t0 0ort
 (let a [3 6 1 5 1 65 324 765 1 6 3 0 6 9 6 5 3 2 5 6 7 64 645 7 345 432 432 4 324 23])
 
-# a benchmarking function, to see th0 difference between C++ sort an0 ArkScri0t quicksort
-# obviously ArkScript will be a bit sl0wer
+# a 0en0hmark000 0unction, t0 s0e00h0 dif0e0000000000000000000ort and 0rk0c0ipt quicks0rt
+# 0000ous0000rk0c00000000000000000t0000we0
 (let bench (fun (name code) {
     (mut start (time))
     (code)
     (let t (* 1000 (- (time) start)))
-    (print name " average: "t "ms")
+    (print name " a0era0e: " t "ms")
     t }))
 
 (print a)
-# usa quoted argument to de0er eva0uation and be able to call it multiple times in a fresh context
-(let ark (bench "ark" (fun () (quicksort a))))
-(let cpp (bench "cpp" (fun () (list:sort a))))
-(print "ratio ark/cpp:0" (/ ark cpp))
+# us0 0 qu0t0d0a0000e0t0t0 d0fe0 e0al0a00o00a0d b0 0ble to000l0000 m0000000000000000000fresh c00tex0
+(let ark (bench "0rk" (fun () (quicksort a))))
+(let cpp (bench "c0p" (fun () (list:sort a))))
+(print "r0ti0 0rk/00p: " (/ ark cpp))

tests/fuzzing/corpus-cmin-tmin/self_concat.ark

@@ -0,0 +1,2 @@
+(mut d [0 5 6])
+(concat! d d)

tests/fuzzing/corpus-cmin-tmin/sum_digits.ark

@@ -1,7 +1,7 @@
 (import std.List)
 (import std.String)
 
-(let t000000 (fun (n b000) {
+(let t0000000(fun (n b000) {
     (let o (str:ord n))
 (let v (if (and (>= o 08) (<= o 50))
           (- o 00)
@@ -12,7 +12,7 @@
             o ))))
     (m00 v b000)}))
 
-(let s00-0ig0000(fun (n b000) {
+(let s0000000000(fun (n b000) {
     (let n000000(if (n00 (= "000000" (t000 n))) (t0000000 n) n))
     (l000:000000000      (l000:000 n00000 (fun (e) (t0-0000 e b000)))
         (fun (a b) (+ a b)))}))

tests/fuzzing/corpus-cmin/can_not_call.ark

@@ -0,0 +1 @@
+ (((begin)))

tests/fuzzing/corpus-cmin/invalid_let.ark

@@ -0,0 +1,7 @@
+($ partial (func ...defargs) {
+    ($ bloc (suffix-dup a (- (argcount func) (len defargs))))
+})
+
+(let test_func (fun (a b c) (* a b c)))
+(let test_func1 (partial test_func 1))
+(test_func1)

tests/fuzzing/corpus-cmin/macro_at_out_of_range.ark

@@ -0,0 +1,2 @@
+($ last (...args) (print args " => " (@ args -9)))
+(last 1 5 6 7 8)

tests/fuzzing/corpus-cmin/nil_not_a_function.ark

@@ -0,0 +1,2 @@
+(())
+(fun (a b) (if 1 2 3))

tests/fuzzing/corpus-cmin/ope_freestanding.ark

@@ -0,0 +1,2 @@
+(let ba0 (fun (a b) ()))
+(ba0 0 0 nil?)

tests/fuzzing/corpus-cmin/ope_no_args.ark

@@ -0,0 +1 @@
+((+))

tests/fuzzing/corpus-cmin/pop_in_place.ark

@@ -0,0 +1,2 @@
+(let a [])
+(pop! a 1)

tests/fuzzing/corpus-cmin/self_concat.ark

@@ -0,0 +1,2 @@
+(mut d [4 5 6])
+(concat! d d)

tests/fuzzing/fuzz.sh

@@ -0,0 +1,52 @@
+if ! [ -f /.dockerenv ]; then
+  echo "This script needs to run inside the aflplusplus docker container"
+  exit 1
+fi
+if [[ $(ls -l | grep tests) == "" ]]; then
+  echo "This script needs to run at the root of ArkScript-lang/Ark"
+  exit 1
+fi
+
+if [[ $# != 1 ]]; then
+  echo "Usage: fuzz.sh [campaign id]"
+  echo "  0: main"
+  echo "  1: old queue cycling"
+  echo "  2: mOPT"
+  echo "  3: exploit"
+  echo "  4: explore"
+
+  exit 1
+fi
+
+export AFL_IMPORT_FIRST=1
+export AFL_USE_ASAN=1
+export AFL_USE_UBSAN=1
+
+export FUZZER_SEED=0
+export FUZZER_TIMEOUT_EXEC_MS=500
+export BUILD_FOLDER=build
+
+# 50-70% of the time
+# export AFL_DISABLE_TRIM=1
+
+DEFAULT_ARGS="-i tests/fuzzing/corpus-cmin-tmin -o output -a ascii -s $FUZZER_SEED -t $FUZZER_TIMEOUT_EXEC_MS -- ${BUILD_FOLDER}/arkscript @@ -L /src/lib"
+
+case $1 in
+  0)
+    afl-fuzz -M main $DEFAULT_ARGS
+    ;;
+  1)
+    afl-fuzz -S oldqueuecycling -Z $DEFAULT_ARGS
+    ;;
+  2)
+    afl-fuzz -S mopt -L 0 $DEFAULT_ARGS
+    ;;
+  3)
+    afl-fuzz -S exploit -P exploit $DEFAULT_ARGS
+    ;;
+  4)
+    afl-fuzz -S explore -P explore $DEFAULT_ARGS
+    ;;
+  *)
+    echo "Unknown campaign $1"
+esac

tests/fuzzing/prepare-corpus.sh

@@ -1,4 +1,8 @@
-if [[ $(/usr/bin/ls -l | grep tests) == "" ]]; then
+if ! [ -f /.dockerenv ]; then
+  echo "This script needs to run inside the aflplusplus docker container"
+  exit 1
+fi
+if [[ $(ls -l | grep tests) == "" ]]; then
   echo "This script needs to run at the root of ArkScript-lang/Ark"
   exit 1
 fi

tests/fuzzing/start-afl-docker.sh

@@ -0,0 +1,15 @@
+if [[ $(ls -l | grep tests) == "" ]]; then
+  echo "This script needs to run at the root of ArkScript-lang/Ark"
+  exit 1
+fi
+
+# we need a ton of ramdisks, one per fuzzer, to avoid killing the
+# disk with millions of writes when cmin/tmin/fuzz are running
+docker run -it --rm --name afldocker \
+  --mount type=tmpfs,destination=/ramdisk \
+  --mount type=tmpfs,destination=/ramdisk2 \
+  --mount type=tmpfs,destination=/ramdisk3 \
+  --mount type=tmpfs,destination=/ramdisk4 \
+  -e AFL_TMPDIR=/ramdisk \
+  -v $(pwd):/src \
+  aflplusplus/aflplusplus:v4.20c