Update fuzzer

tests/.gitignore

@@ -2,3 +2,4 @@
 /csmith-*
 /test*.c
 /reduce-criteria.sh
+/creduce_bug_*

tests/README.md

@@ -14,7 +14,11 @@ Fuzzer randomly generates input C program and feeds it to your implementation fo
 
 After the fuzzing stage, you may found a buggy input program in `test_polished.c`. However, it is highly likely that the input program is too big (about 3~5K lines of code) to manually inspect. In this stage, we use `creduce` to reduce the buggy input program as much as possible. Reduced buggy input program is saved to `test_reduced.c`.
 
-**[NOTICE]** The buggy input program after reducing stage may contain undefined behaviors. To workaround this, please refer to this issue: [#218](https://github.com/kaist-cp/cs420/issues/218)
+**[NOTICE]** The buggy input program after reducing stage may contain undefined behaviors. To workaround this, we recommend you to (1) use `--clang-analyze` option to use clang static analyzer for reducing, or (2) do manual binary search to reduce the program by following this:
+
+- If the code is 4000 lines, delete the latter 2000, and try again
+- If there is no error, undo and delete only the latter 1000. If there is, delete lines 1000 to 2000, etc.
+- It is important that you do not delete upper lines you know are safe (e.g, have lines 1000 to 2000 alive when checking 2000to 3000) as they might have some include for macros that can lead to compile errors if left out.
 
 For now, fuzzer is supported only for Homework 1 (C AST Printer) and Homework 2 (IRgen).
 

tests/fuzz.py

@@ -155,7 +155,7 @@ def polish(src, inc_path):
 
     return src_replaced
 
-def make_reduce_criteria(tests_dir, fuzz_arg):
+def make_reduce_criteria(tests_dir, fuzz_arg, analyze):
     """Make executable reduce_criteria.sh
     """
     # Make shell script i.e. dependent to KECC path
@@ -164,6 +164,7 @@ def make_reduce_criteria(tests_dir, fuzz_arg):
         "$FUZZ_ARG": fuzz_arg,
         "$KECC_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/kecc"))),
         "$FUZZ_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/fuzz"))),
+        "$CLANG_ANALYZE": str(analyze).lower(),
     }
     with open(os.path.join(tests_dir, "reduce-criteria-template.sh"), "r") as t:
         temp = t.read()
@@ -183,14 +184,14 @@ def make_reduce_criteria(tests_dir, fuzz_arg):
         proc.kill()
         raise e
 
-def creduce(tests_dir, fuzz_arg):
+def creduce(tests_dir, fuzz_arg, analyze):
     """Reduce `tests/test_polished.c` to `tests/test_reduced.c`
 
     First, we copy test_polished.c to test_reduced.c.
     Then, when Creduce reduces test_reduced.c, it overwrites partially reduced program to itself.
     Original file is moved to test_reduced.c.orig which is then identical to test_polished.c.
     """
-    make_reduce_criteria(tests_dir, fuzz_arg)
+    make_reduce_criteria(tests_dir, fuzz_arg, analyze)
 
     try:
         args = ["cp", "test_polished.c", "test_reduced.c"]
@@ -204,7 +205,7 @@ def creduce(tests_dir, fuzz_arg):
 
     try:
         # --tidy: Do not make a backup copy of each file to reduce as file.orig
-        args = ["creduce", "--tidy", "./reduce-criteria.sh", "test_reduced.c"]
+        args = ["creduce", "--tidy", "--timeout", "20", "./reduce-criteria.sh", "test_reduced.c"]
         proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=tests_dir)
         (out, err) = proc.communicate()
         if proc.returncode != 0:
@@ -269,6 +270,7 @@ if __name__ == "__main__":
     parser.add_argument('--skip-build', action='store_true', help="Skipping cargo build")
     parser.add_argument('--easy', action='store_true', help="Generate more easy code by csmith option")
     parser.add_argument('--seed', type=int, help="Provide seed of fuzz generation", default=-1)
+    parser.add_argument('--clang-analyze', action='store_true', help="Use clang static analyzer for reducing. It prevents undefined behaviors coming from reduced program, but perhaps take a long time to do so")
     args = parser.parse_args()
 
     if args.print and args.irgen:
@@ -301,6 +303,6 @@ if __name__ == "__main__":
         print("Skip building. Please run `cargo build --features=build-bin --release --bin fuzz --bin kecc` to manually build.")
 
     if args.reduce:
-        creduce(tests_dir, fuzz_arg)
+        creduce(tests_dir, fuzz_arg, args.clang_analyze)
     else:
         fuzz(tests_dir, fuzz_arg, args.num, args.easy)

tests/reduce-criteria-template.sh

@@ -6,7 +6,7 @@ rm -f out*.txt
 #ulimit -v 2000000                                                                                                                                                                                          
 
 if
-  (! gcc test_reduced.c > cc_out.txt 2>&1 ||\
+  (! gcc -Wall -Wextra test_reduced.c > out_gcc.txt 2>&1 ||\
   ! $KECC_BIN --parse test_reduced.c >/dev/null 2>&1)
 then
   exit 1
@@ -14,58 +14,66 @@ fi
 
 if
   [ $FUZZ_ARG = '-i' ] &&\
-  (! clang -pedantic -Wall -Werror=strict-prototypes -c test_reduced.c > out.txt 2>&1 ||\
-  grep 'main-return-type' out.txt ||\
-  grep 'conversions than data arguments' out.txt ||\
-  grep 'int-conversion' out.txt ||\
-  grep 'ordered comparison between pointer and zero' out.txt ||\
-  grep 'ordered comparison between pointer and integer' out.txt ||\
-  grep 'eliding middle term' out.txt ||\
-  grep 'end of non-void function' out.txt ||\
-  grep 'invalid in C99' out.txt ||\
-  grep 'specifies type' out.txt ||\
-  grep 'should return a value' out.txt ||\
-  grep 'uninitialized' out.txt ||\
-  grep 'incompatible pointer to' out.txt ||\
-  grep 'incompatible integer to' out.txt ||\
-  grep 'type specifier missing' out.txt ||\
-  grep 'implicit-function-declaration' out.txt ||\
-  grep 'infinite-recursion' out.txt ||\
-  grep 'pointer-bool-conversion' out.txt ||\
-  grep 'non-void function does not return a value' out.txt ||\
-  grep 'too many arguments in call' out.txt ||\
-  grep 'declaration does not declare anything' out.txt ||\
-  grep 'not equal to a null pointer is always true' out.txt ||\
-  grep 'empty struct is a GNU extension' out.txt ||\
-  ! gcc -Wall -Wextra test_reduced.c > outa.txt 2>&1 ||\
-  grep 'uninitialized' outa.txt ||\
-  grep 'without a cast' outa.txt ||\
-  grep 'control reaches end' outa.txt ||\
-  grep 'return type defaults' outa.txt ||\
-  grep 'cast from pointer to integer' outa.txt ||\
-  grep 'useless type name in empty declaration' outa.txt ||\
-  grep 'no semicolon at end' outa.txt ||\
-  grep 'type defaults to' outa.txt ||\
-  grep 'too few arguments for format' outa.txt ||\
+  (! clang -pedantic -Wall -Werror=strict-prototypes -c test_reduced.c > out_clang.txt 2>&1 ||\
+  grep 'main-return-type' out_clang.txt ||\
+  grep 'conversions than data arguments' out_clang.txt ||\
+  grep 'int-conversion' out_clang.txt ||\
+  grep 'ordered comparison between pointer and zero' out_clang.txt ||\
+  grep 'ordered comparison between pointer and integer' out_clang.txt ||\
+  grep 'eliding middle term' out_clang.txt ||\
+  grep 'end of non-void function' out_clang.txt ||\
+  grep 'invalid in C99' out_clang.txt ||\
+  grep 'specifies type' out_clang.txt ||\
+  grep 'should return a value' out_clang.txt ||\
+  grep 'uninitialized' out_clang.txt ||\
+  grep 'incompatible pointer to' out_clang.txt ||\
+  grep 'incompatible integer to' out_clang.txt ||\
+  grep 'type specifier missing' out_clang.txt ||\
+  grep 'implicit-function-declaration' out_clang.txt ||\
+  grep 'infinite-recursion' out_clang.txt ||\
+  grep 'pointer-bool-conversion' out_clang.txt ||\
+  grep 'non-void function does not return a value' out_clang.txt ||\
+  grep 'too many arguments in call' out_clang.txt ||\
+  grep 'declaration does not declare anything' out_clang.txt ||\
+  grep 'not equal to a null pointer is always true' out_clang.txt ||\
+  grep 'empty struct is a GNU extension' out_clang.txt ||\
+  grep 'uninitialized' out_gcc.txt ||\
+  grep 'without a cast' out_gcc.txt ||\
+  grep 'control reaches end' out_gcc.txt ||\
+  grep 'return type defaults' out_gcc.txt ||\
+  grep 'cast from pointer to integer' out_gcc.txt ||\
+  grep 'useless type name in empty declaration' out_gcc.txt ||\
+  grep 'no semicolon at end' out_gcc.txt ||\
+  grep 'type defaults to' out_gcc.txt ||\
+  grep 'too few arguments for format' out_gcc.txt ||\
   grep 'incompatible pointer' out_gcc.txt ||\
-  grep 'ordered comparison of pointer with integer' outa.txt ||\
-  grep 'declaration does not declare anything' outa.txt ||\
-  grep 'expects type' outa.txt ||\
-  grep 'pointer from integer' outa.txt ||\
-  grep 'incompatible implicit' outa.txt ||\
-  grep 'excess elements in struct initializer' outa.txt ||\
-  grep 'comparison between pointer and integer' outa.txt ||\
-  grep 'division by zero' outa.txt ||\
-  ! clang -Wall -Wextra --analyze -c test_reduced.c > outb.txt 2>&1 ||\
-  grep 'garbage value' outb.txt)
+  grep 'ordered comparison of pointer with integer' out_gcc.txt ||\
+  grep 'declaration does not declare anything' out_gcc.txt ||\
+  grep 'expects type' out_gcc.txt ||\
+  grep 'pointer from integer' out_gcc.txt ||\
+  grep 'incompatible implicit' out_gcc.txt ||\
+  grep 'excess elements in struct initializer' out_gcc.txt ||\
+  grep 'comparison between pointer and integer' out_gcc.txt ||\
+  grep 'division by zero' out_gcc.txt)
 then
   exit 1
 fi
 
-$FUZZ_BIN $FUZZ_ARG test_reduced.c
-if [ "$?" = 101 ]
+if
+  [ $FUZZ_ARG = '-i' ] &&\
+  $CLANG_ANALYZE &&\
+  (! clang --analyze -c test_reduced.c > out_analyzer.txt 2>&1 ||\
+  grep 'garbage value' out_analyzer.txt)
 then
-  exit 0
-else
   exit 1
 fi
+
+$FUZZ_BIN $FUZZ_ARG test_reduced.c > out_fuzz.txt 2>&1
+
+if
+  [ $FUZZ_ARG = '-p' ]
+then
+  grep 'panicked' out_fuzz.txt
+else
+  grep 'assertion failed' out_fuzz.txt
+fi