kmc7468/cs420
1
0
Fork 0
mirror of https://github.com/kmc7468/cs420.git synced 2025-12-16 07:28:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update fuzzer

Browse Source
This commit is contained in:
Minseong Jang
2022-04-04 20:34:26 +09:00
parent 3a36e47985
commit e567681ed9
4 changed files with 69 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/.gitignore vendored
View File

@@ -2,3 +2,4 @@
/csmith-* /csmith-*
/test*.c /test*.c
/reduce-criteria.sh /reduce-criteria.sh
/creduce_bug_*

6
tests/README.md
View File

@@ -14,7 +14,11 @@ Fuzzer randomly generates input C program and feeds it to your implementation fo
After the fuzzing stage, you may found a buggy input program in `test_polished.c`. However, it is highly likely that the input program is too big (about 3~5K lines of code) to manually inspect. In this stage, we use `creduce` to reduce the buggy input program as much as possible. Reduced buggy input program is saved to `test_reduced.c`. After the fuzzing stage, you may found a buggy input program in `test_polished.c`. However, it is highly likely that the input program is too big (about 3~5K lines of code) to manually inspect. In this stage, we use `creduce` to reduce the buggy input program as much as possible. Reduced buggy input program is saved to `test_reduced.c`.
**[NOTICE]** The buggy input program after reducing stage may contain undefined behaviors. To workaround this, please refer to this issue: [#218](https://github.com/kaist-cp/cs420/issues/218) **[NOTICE]** The buggy input program after reducing stage may contain undefined behaviors. To workaround this, we recommend you to (1) use `--clang-analyze` option to use clang static analyzer for reducing, or (2) do manual binary search to reduce the program by following this:
- If the code is 4000 lines, delete the latter 2000, and try again
- If there is no error, undo and delete only the latter 1000. If there is, delete lines 1000 to 2000, etc.
- It is important that you do not delete upper lines you know are safe (e.g, have lines 1000 to 2000 alive when checking 2000to 3000) as they might have some include for macros that can lead to compile errors if left out.
For now, fuzzer is supported only for Homework 1 (C AST Printer) and Homework 2 (IRgen). For now, fuzzer is supported only for Homework 1 (C AST Printer) and Homework 2 (IRgen).

12
tests/fuzz.py
View File

@@ -155,7 +155,7 @@ def polish(src, inc_path):
return src_replaced return src_replaced
def make_reduce_criteria(tests_dir, fuzz_arg): def make_reduce_criteria(tests_dir, fuzz_arg, analyze):
"""Make executable reduce_criteria.sh """Make executable reduce_criteria.sh
""" """
# Make shell script i.e. dependent to KECC path # Make shell script i.e. dependent to KECC path
@@ -164,6 +164,7 @@ def make_reduce_criteria(tests_dir, fuzz_arg):
"$FUZZ_ARG": fuzz_arg, "$FUZZ_ARG": fuzz_arg,
"$KECC_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/kecc"))), "$KECC_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/kecc"))),
"$FUZZ_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/fuzz"))), "$FUZZ_BIN": str(os.path.abspath(os.path.join(tests_dir, "../target/release/fuzz"))),
"$CLANG_ANALYZE": str(analyze).lower(),
} }
with open(os.path.join(tests_dir, "reduce-criteria-template.sh"), "r") as t: with open(os.path.join(tests_dir, "reduce-criteria-template.sh"), "r") as t:
temp = t.read() temp = t.read()
@@ -183,14 +184,14 @@ def make_reduce_criteria(tests_dir, fuzz_arg):
proc.kill() proc.kill()
raise e raise e
def creduce(tests_dir, fuzz_arg): def creduce(tests_dir, fuzz_arg, analyze):
"""Reduce `tests/test_polished.c` to `tests/test_reduced.c` """Reduce `tests/test_polished.c` to `tests/test_reduced.c`
First, we copy test_polished.c to test_reduced.c. First, we copy test_polished.c to test_reduced.c.
Then, when Creduce reduces test_reduced.c, it overwrites partially reduced program to itself. Then, when Creduce reduces test_reduced.c, it overwrites partially reduced program to itself.
Original file is moved to test_reduced.c.orig which is then identical to test_polished.c. Original file is moved to test_reduced.c.orig which is then identical to test_polished.c.
""" """
make_reduce_criteria(tests_dir, fuzz_arg) make_reduce_criteria(tests_dir, fuzz_arg, analyze)
try: try:
args = ["cp", "test_polished.c", "test_reduced.c"] args = ["cp", "test_polished.c", "test_reduced.c"]
@@ -204,7 +205,7 @@ def creduce(tests_dir, fuzz_arg):
try: try:
# --tidy: Do not make a backup copy of each file to reduce as file.orig # --tidy: Do not make a backup copy of each file to reduce as file.orig
args = ["creduce", "--tidy", "./reduce-criteria.sh", "test_reduced.c"] args = ["creduce", "--tidy", "--timeout", "20", "./reduce-criteria.sh", "test_reduced.c"]
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=tests_dir) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=tests_dir)
(out, err) = proc.communicate() (out, err) = proc.communicate()
if proc.returncode != 0: if proc.returncode != 0:
@@ -269,6 +270,7 @@ if __name__ == "__main__":
parser.add_argument('--skip-build', action='store_true', help="Skipping cargo build") parser.add_argument('--skip-build', action='store_true', help="Skipping cargo build")
parser.add_argument('--easy', action='store_true', help="Generate more easy code by csmith option") parser.add_argument('--easy', action='store_true', help="Generate more easy code by csmith option")
parser.add_argument('--seed', type=int, help="Provide seed of fuzz generation", default=-1) parser.add_argument('--seed', type=int, help="Provide seed of fuzz generation", default=-1)
parser.add_argument('--clang-analyze', action='store_true', help="Use clang static analyzer for reducing. It prevents undefined behaviors coming from reduced program, but perhaps take a long time to do so")
args = parser.parse_args() args = parser.parse_args()
if args.print and args.irgen: if args.print and args.irgen:
@@ -301,6 +303,6 @@ if __name__ == "__main__":
print("Skip building. Please run `cargo build --features=build-bin --release --bin fuzz --bin kecc` to manually build.") print("Skip building. Please run `cargo build --features=build-bin --release --bin fuzz --bin kecc` to manually build.")
if args.reduce: if args.reduce:
creduce(tests_dir, fuzz_arg) creduce(tests_dir, fuzz_arg, args.clang_analyze)
else: else:
fuzz(tests_dir, fuzz_arg, args.num, args.easy) fuzz(tests_dir, fuzz_arg, args.num, args.easy)

104
tests/reduce-criteria-template.sh
View File

@@ -6,7 +6,7 @@ rm -f out*.txt
#ulimit -v 2000000 #ulimit -v 2000000
if if
(! gcc test_reduced.c > cc_out.txt 2>&1 ||\ (! gcc -Wall -Wextra test_reduced.c > out_gcc.txt 2>&1 ||\
! $KECC_BIN --parse test_reduced.c >/dev/null 2>&1) ! $KECC_BIN --parse test_reduced.c >/dev/null 2>&1)
then then
exit 1 exit 1
@@ -14,58 +14,66 @@ fi
if if
[ $FUZZ_ARG = '-i' ] &&\ [ $FUZZ_ARG = '-i' ] &&\
(! clang -pedantic -Wall -Werror=strict-prototypes -c test_reduced.c > out.txt 2>&1 ||\ (! clang -pedantic -Wall -Werror=strict-prototypes -c test_reduced.c > out_clang.txt 2>&1 ||\
grep 'main-return-type' out.txt ||\ grep 'main-return-type' out_clang.txt ||\
grep 'conversions than data arguments' out.txt ||\ grep 'conversions than data arguments' out_clang.txt ||\
grep 'int-conversion' out.txt ||\ grep 'int-conversion' out_clang.txt ||\
grep 'ordered comparison between pointer and zero' out.txt ||\ grep 'ordered comparison between pointer and zero' out_clang.txt ||\
grep 'ordered comparison between pointer and integer' out.txt ||\ grep 'ordered comparison between pointer and integer' out_clang.txt ||\
grep 'eliding middle term' out.txt ||\ grep 'eliding middle term' out_clang.txt ||\
grep 'end of non-void function' out.txt ||\ grep 'end of non-void function' out_clang.txt ||\
grep 'invalid in C99' out.txt ||\ grep 'invalid in C99' out_clang.txt ||\
grep 'specifies type' out.txt ||\ grep 'specifies type' out_clang.txt ||\
grep 'should return a value' out.txt ||\ grep 'should return a value' out_clang.txt ||\
grep 'uninitialized' out.txt ||\ grep 'uninitialized' out_clang.txt ||\
grep 'incompatible pointer to' out.txt ||\ grep 'incompatible pointer to' out_clang.txt ||\
grep 'incompatible integer to' out.txt ||\ grep 'incompatible integer to' out_clang.txt ||\
grep 'type specifier missing' out.txt ||\ grep 'type specifier missing' out_clang.txt ||\
grep 'implicit-function-declaration' out.txt ||\ grep 'implicit-function-declaration' out_clang.txt ||\
grep 'infinite-recursion' out.txt ||\ grep 'infinite-recursion' out_clang.txt ||\
grep 'pointer-bool-conversion' out.txt ||\ grep 'pointer-bool-conversion' out_clang.txt ||\
grep 'non-void function does not return a value' out.txt ||\ grep 'non-void function does not return a value' out_clang.txt ||\
grep 'too many arguments in call' out.txt ||\ grep 'too many arguments in call' out_clang.txt ||\
grep 'declaration does not declare anything' out.txt ||\ grep 'declaration does not declare anything' out_clang.txt ||\
grep 'not equal to a null pointer is always true' out.txt ||\ grep 'not equal to a null pointer is always true' out_clang.txt ||\
grep 'empty struct is a GNU extension' out.txt ||\ grep 'empty struct is a GNU extension' out_clang.txt ||\
! gcc -Wall -Wextra test_reduced.c > outa.txt 2>&1 ||\ grep 'uninitialized' out_gcc.txt ||\
grep 'uninitialized' outa.txt ||\ grep 'without a cast' out_gcc.txt ||\
grep 'without a cast' outa.txt ||\ grep 'control reaches end' out_gcc.txt ||\
grep 'control reaches end' outa.txt ||\ grep 'return type defaults' out_gcc.txt ||\
grep 'return type defaults' outa.txt ||\ grep 'cast from pointer to integer' out_gcc.txt ||\
grep 'cast from pointer to integer' outa.txt ||\ grep 'useless type name in empty declaration' out_gcc.txt ||\
grep 'useless type name in empty declaration' outa.txt ||\ grep 'no semicolon at end' out_gcc.txt ||\
grep 'no semicolon at end' outa.txt ||\ grep 'type defaults to' out_gcc.txt ||\
grep 'type defaults to' outa.txt ||\ grep 'too few arguments for format' out_gcc.txt ||\
grep 'too few arguments for format' outa.txt ||\
grep 'incompatible pointer' out_gcc.txt ||\ grep 'incompatible pointer' out_gcc.txt ||\
grep 'ordered comparison of pointer with integer' outa.txt ||\ grep 'ordered comparison of pointer with integer' out_gcc.txt ||\
grep 'declaration does not declare anything' outa.txt ||\ grep 'declaration does not declare anything' out_gcc.txt ||\
grep 'expects type' outa.txt ||\ grep 'expects type' out_gcc.txt ||\
grep 'pointer from integer' outa.txt ||\ grep 'pointer from integer' out_gcc.txt ||\
grep 'incompatible implicit' outa.txt ||\ grep 'incompatible implicit' out_gcc.txt ||\
grep 'excess elements in struct initializer' outa.txt ||\ grep 'excess elements in struct initializer' out_gcc.txt ||\
grep 'comparison between pointer and integer' outa.txt ||\ grep 'comparison between pointer and integer' out_gcc.txt ||\
grep 'division by zero' outa.txt ||\ grep 'division by zero' out_gcc.txt)
! clang -Wall -Wextra --analyze -c test_reduced.c > outb.txt 2>&1 ||\
grep 'garbage value' outb.txt)
then then
exit 1 exit 1
fi fi
$FUZZ_BIN $FUZZ_ARG test_reduced.c if
if [ "$?" = 101 ] [ $FUZZ_ARG = '-i' ] &&\
$CLANG_ANALYZE &&\
(! clang --analyze -c test_reduced.c > out_analyzer.txt 2>&1 ||\
grep 'garbage value' out_analyzer.txt)
then then
exit 0
else
exit 1 exit 1
fi fi
$FUZZ_BIN $FUZZ_ARG test_reduced.c > out_fuzz.txt 2>&1
if
[ $FUZZ_ARG = '-p' ]
then
grep 'panicked' out_fuzz.txt
else
grep 'assertion failed' out_fuzz.txt
fi