aboutsummaryrefslogtreecommitdiff
path: root/afl-cmin
diff options
context:
space:
mode:
Diffstat (limited to 'afl-cmin')
-rwxr-xr-xafl-cmin217
1 files changed, 182 insertions, 35 deletions
diff --git a/afl-cmin b/afl-cmin
index 853c9398..4aaf3953 100755
--- a/afl-cmin
+++ b/afl-cmin
@@ -1,11 +1,15 @@
#!/usr/bin/env sh
+SYS=$(uname -s)
+test "$SYS" = "Darwin" && {
+ echo Error: afl-cmin does not work on Apple currently. please use afl-cmin.bash instead.
+ exit 1
+}
export AFL_QUIET=1
export ASAN_OPTIONS=detect_leaks=0
THISPATH=`dirname ${0}`
export PATH="${THISPATH}:$PATH"
awk -f - -- ${@+"$@"} <<'EOF'
#!/usr/bin/awk -f
-
# awk script to minimize a test corpus of input files
#
# based on afl-cmin bash script written by Michal Zalewski
@@ -103,30 +107,38 @@ function usage() {
" -o dir - output directory for minimized files\n" \
"\n" \
"Execution control settings:\n" \
+" -T tasks - how many parallel tasks to run (default: 1, all=nproc)\n" \
" -f file - location read by the fuzzed program (stdin)\n" \
" -m megs - memory limit for child process ("mem_limit" MB)\n" \
-" -t msec - run time limit for child process (none)\n" \
+" -t msec - run time limit for child process (default: 5000)\n" \
" -O - use binary-only instrumentation (FRIDA mode)\n" \
" -Q - use binary-only instrumentation (QEMU mode)\n" \
" -U - use unicorn-based instrumentation (unicorn mode)\n" \
+" -X - use Nyx mode\n" \
"\n" \
"Minimization settings:\n" \
+" -A - allow crashes and timeouts (not recommended)\n" \
" -C - keep crashing inputs, reject everything else\n" \
" -e - solve for edge coverage only, ignore hit counts\n" \
"\n" \
"For additional tips, please consult README.md\n" \
"\n" \
"Environment variables used:\n" \
-"AFL_ALLOW_TMP: allow unsafe use of input/output directories under {/var}/tmp\n" \
"AFL_CRASH_EXITCODE: optional child exit code to be interpreted as crash\n" \
"AFL_FORKSRV_INIT_TMOUT: time the fuzzer waits for the forkserver to come up\n" \
"AFL_KEEP_TRACES: leave the temporary <out_dir>/.traces directory\n" \
"AFL_KILL_SIGNAL: Signal delivered to child processes on timeout (default: SIGKILL)\n" \
+"AFL_FORK_SERVER_KILL_SIGNAL: Signal delivered to fork server processes on\n" \
+" termination (default: SIGTERM). If this is not set and AFL_KILL_SIGNAL is\n" \
+" set, this will be set to the same value as AFL_KILL_SIGNAL.\n" \
"AFL_NO_FORKSRV: run target via execve instead of using the forkserver\n" \
+"AFL_CMIN_ALLOW_ANY: write tuples for crashing inputs also\n" \
"AFL_PATH: path for the afl-showmap binary if not found anywhere in PATH\n" \
"AFL_PRINT_FILENAMES: If set, the filename currently processed will be " \
"printed to stdout\n" \
"AFL_SKIP_BIN_CHECK: skip afl instrumentation checks for target binary\n"
+"AFL_CUSTOM_MUTATOR_LIBRARY: custom mutator library (post_process and send)\n"
+"AFL_PYTHON_MODULE: custom mutator library (post_process and send)\n"
exit 1
}
@@ -135,22 +147,35 @@ function exists_and_is_executable(binarypath) {
}
BEGIN {
- print "corpus minimization tool for afl++ (awk version)\n"
+ if (0 != system( "test -t 1")) {
+ redirected = 1
+ } else {
+ redirected = 0
+ }
+
+ print "corpus minimization tool for AFL++ (awk version)\n"
# defaults
extra_par = ""
AFL_CMIN_CRASHES_ONLY = ""
+ AFL_CMIN_ALLOW_ANY = ""
# process options
Opterr = 1 # default is to diagnose
Optind = 1 # skip ARGV[0]
- while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCOQU?")) != -1) {
+ while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXYT:?")) != -1) {
if (_go_c == "i") {
if (!Optarg) usage()
if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
in_dir = Optarg
continue
} else
+ if (_go_c == "T") {
+ if (!Optarg) usage()
+ if (threads) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
+ threads = Optarg
+ continue
+ } else
if (_go_c == "o") {
if (!Optarg) usage()
if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
@@ -180,6 +205,10 @@ BEGIN {
AFL_CMIN_CRASHES_ONLY = "AFL_CMIN_CRASHES_ONLY=1 "
continue
} else
+ if (_go_c == "A") {
+ AFL_CMIN_ALLOW_ANY = "AFL_CMIN_ALLOW_ANY=1 "
+ continue
+ } else
if (_go_c == "e") {
extra_par = extra_par " -e"
continue
@@ -201,6 +230,12 @@ BEGIN {
extra_par = extra_par " -U"
unicorn_mode = 1
continue
+ } else
+ if (_go_c == "X" || _go_c == "Y") {
+ if (nyx_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
+ extra_par = extra_par " -X"
+ nyx_mode = 1
+ continue
} else
if (_go_c == "?") {
exit 1
@@ -209,7 +244,7 @@ BEGIN {
} # while options
if (!mem_limit) mem_limit = "none"
- if (!timeout) timeout = "none"
+ if (!timeout) timeout = "5000"
# get program args
i = 0
@@ -236,15 +271,22 @@ BEGIN {
"pwd" | getline dirlist[4] # current directory
for (dirind in dirlist) {
dir = dirlist[dirind]
-
if (dir ~ /^(\/var)?\/tmp/) {
- print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr"
- exit 1
+ print "[-] Warning: do not use this script in /tmp or /var/tmp for security reasons." > "/dev/stderr"
}
}
delete dirlist
}
+ if (threads && stdin_file) {
+ print "[-] Error: -T and -f cannot be used together." > "/dev/stderr"
+ exit 1
+ }
+
+ if (!threads && !stdin_file && !nyx_mode) {
+ print "[*] Are you aware of the '-T all' parallelize option that improves the speed for large/slow corpuses?"
+ }
+
# If @@ is specified, but there's no -f, let's come up with a temporary input
# file name.
@@ -275,9 +317,12 @@ BEGIN {
exit 1
}
- if (target_bin && !exists_and_is_executable(target_bin)) {
- "command -v "target_bin" 2>/dev/null" | getline tnew
+ if (!nyx_mode && target_bin && !exists_and_is_executable(target_bin)) {
+
+ cmd = "command -v "target_bin" 2>/dev/null"
+ cmd | getline tnew
+ close(cmd)
if (!tnew || !exists_and_is_executable(tnew)) {
print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr"
exit 1
@@ -285,7 +330,18 @@ BEGIN {
target_bin = tnew
}
- if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode) {
+ if (0 == system ( "grep -aq AFL_DUMP_MAP_SIZE " target_bin )) {
+ echo "[!] Trying to obtain the map size of the target ..."
+ get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin
+ get_map_size | getline mapsize
+ close(get_map_size)
+ if (mapsize && mapsize > 65535 && mapsize < 100000000) {
+ AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" "
+ print "[+] Setting "AFL_MAP_SIZE
+ }
+ }
+
+ if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode && !nyx_mode) {
if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) {
print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr"
exit 1
@@ -308,12 +364,28 @@ BEGIN {
system("rm -rf "trace_dir" 2>/dev/null");
system("rm "out_dir"/id[:_]* 2>/dev/null")
- "ls "out_dir"/* 2>/dev/null | wc -l" | getline noofentries
+ cmd = "ls "out_dir"/* 2>/dev/null | wc -l"
+ cmd | getline noofentries
+ close(cmd)
if (0 == system( "test -d "out_dir" -a "noofentries" -gt 0" )) {
print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr"
exit 1
}
+ if (threads) {
+ cmd = "nproc"
+ cmd | getline nproc
+ close(cmd)
+ if (threads == "all") {
+ threads = nproc
+ } else {
+ if (!(threads > 1 && threads <= nproc)) {
+ print "[-] Error: -T option must be between 1 and "nproc" or \"all\"." > "/dev/stderr"
+ exit 1
+ }
+ }
+ }
+
# Check for the more efficient way to copy files...
if (0 != system("mkdir -p -m 0700 "trace_dir)) {
print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr"
@@ -323,12 +395,14 @@ BEGIN {
if (stdin_file) {
# truncate input file
printf "" > stdin_file
- close( stdin_file )
+ close(stdin_file)
}
# First we look in PATH
if (0 == system("command -v afl-showmap >/dev/null 2>&1")) {
- "command -v afl-showmap 2>/dev/null" | getline showmap
+ cmd = "command -v afl-showmap 2>/dev/null"
+ cmd | getline showmap
+ close(cmd)
} else {
# then we look in the current directory
if (0 == system("test -x ./afl-showmap")) {
@@ -350,13 +424,15 @@ BEGIN {
# yuck, gnu stat is option incompatible to bsd stat
# we use a heuristic to differentiate between
# GNU stat and other stats
- "stat --version 2>/dev/null" | getline statversion
- if (statversion ~ /GNU coreutils/) {
+ cmd = "stat --version 2>/dev/null"
+ cmd | getline statversion
+ close(cmd)
+ if (statversion ~ /GNU coreutils/ || statversion ~ /BusyBox/) {
stat_format = "-c '%s %n'" # GNU
} else {
stat_format = "-f '%z %N'" # *BSD, MacOS
}
- cmdline = "(cd "in_dir" && find . \\( ! -name \".*\" -a -type d \\) -o -type f -exec stat "stat_format" \\{\\} + | sort -k1n -k2r)"
+ cmdline = "(cd "in_dir" && find . \\( ! -name \".*\" -a -type d \\) -o -type f -exec stat "stat_format" \\{\\} + | sort -k1n -k2r) | grep -Ev '^0'"
#cmdline = "ls "in_dir" | (cd "in_dir" && xargs stat "stat_format" 2>/dev/null) | sort -k1n -k2r"
#cmdline = "(cd "in_dir" && stat "stat_format" *) | sort -k1n -k2r"
#cmdline = "(cd "in_dir" && ls | xargs stat "stat_format" ) | sort -k1n -k2r"
@@ -369,6 +445,7 @@ BEGIN {
infilesSmallToBigFullMap[infilesSmallToBigFull[i]] = infilesSmallToBig[i]
i++
}
+ close(cmdline)
in_count = i
first_file = infilesSmallToBigFull[0]
@@ -393,10 +470,10 @@ BEGIN {
print "[*] Testing the target binary..."
if (!stdin_file) {
- system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"")
+ system(AFL_MAP_SIZE "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"")
} else {
system("cp \""in_dir"/"first_file"\" "stdin_file)
- system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+ system(AFL_MAP_SIZE "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
}
first_count = 0
@@ -405,6 +482,7 @@ BEGIN {
while ((getline < runtest) > 0) {
++first_count
}
+ close(runtest)
if (first_count) {
print "[+] OK, "first_count" tuples recorded."
@@ -417,33 +495,90 @@ BEGIN {
}
}
+ if (in_count < threads) {
+ threads = in_count
+ print "[!] WARNING: less inputs than threads, reducing threads to "threads" and likely the overhead of threading makes things slower..."
+ }
+
# Let's roll!
#############################
# STEP 1: Collecting traces #
#############################
+ if (threads) {
+
+ inputsperfile = int(in_count / threads)
+ if (in_count % threads) {
+ inputsperfile++;
+ }
+
+ cnt = 0;
+ tmpfile=out_dir "/.filelist"
+ for (instance = 1; instance < threads; instance++) {
+ for (i = 0; i < inputsperfile; i++) {
+ print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."instance
+ cnt++
+ }
+ }
+ for (; cnt < in_count; cnt++) {
+ print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."threads
+ }
+
+ }
+
print "[*] Obtaining traces for "in_count" input files in '"in_dir"'."
cur = 0;
- if (!stdin_file) {
- print " Processing "in_count" files (forkserver mode)..."
-# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string
- retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
+
+ if (threads > 1) {
+
+ print "[*] Creating " threads " parallel tasks with about " inputsperfile " items each."
+ for (i = 1; i <= threads; i++) {
+
+ if (!stdin_file) {
+# print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &"
+ retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &")
+ } else {
+ stdin_file=tmpfile"."i".stdin"
+# print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &"
+ retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &")
+ }
+ }
+ print "[*] Waiting for parallel tasks to complete ..."
+ # wait for all processes to finish
+ ok=0
+ while (ok < threads) {
+ ok=0
+ for (i = 1; i <= threads; i++) {
+ if (system("test -f "tmpfile"."i".done") == 0) {
+ ok++
+ }
+ }
+ }
+ print "[*] Done!"
+ system("rm -f "tmpfile"*")
} else {
- print " Processing "in_count" files (forkserver mode)..."
+ if (!stdin_file) {
+ print " Processing "in_count" files (forkserver mode)..."
+# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string
+ retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
+ } else {
+ print " Processing "in_count" files (forkserver mode)..."
# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
- retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
- }
+ retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+ }
- if (retval && !AFL_CMIN_CRASHES_ONLY) {
- print "[!] Exit code "retval" != 0 received from afl-showmap, terminating..."
+ if (retval && (!AFL_CMIN_CRASHES_ONLY && !AFL_CMIN_ALLOW_ANY)) {
+ print "[!] Exit code "retval" != 0 received from afl-showmap (this means a crashing or timeout input is likely present), terminating..."
- if (!ENVIRON["AFL_KEEP_TRACES"]) {
- system("rm -rf "trace_dir" 2>/dev/null")
- system("rmdir "out_dir)
+ if (!ENVIRON["AFL_KEEP_TRACES"]) {
+ system("rm -rf "trace_dir" 2>/dev/null")
+ system("rmdir "out_dir)
+ }
+ exit retval
}
- exit retval
+
}
#######################################################
@@ -463,9 +598,19 @@ BEGIN {
while (cur < in_count) {
fn = infilesSmallToBig[cur]
++cur
- printf "\r Processing file "cur"/"in_count
+ if (redirected == 0) { printf "\r Processing file "cur"/"in_count }
+ else { print " Processing file "cur"/"in_count }
# create path for the trace file from afl-showmap
tracefile_path = trace_dir"/"fn
+ # ensure the file size is not zero
+ cmd = "du -b "tracefile_path
+ "ls -l "tracefile_path
+ cmd | getline output
+ close(cmd)
+ split(output, result, "\t")
+ if (result[1] == 0) {
+ print "[!] WARNING: file "fn" is crashing the target, ignoring..."
+ }
# gather all keys, and count them
while ((getline line < tracefile_path) > 0) {
key = line
@@ -502,7 +647,9 @@ BEGIN {
key = field[nrFields]
++tcnt;
- printf "\r Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..."
+ if (redirected == 0) { printf "\r Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..." }
+ else { print " Processing tuple "tcnt"/"tuple_count" with count "key_count[key]"..." }
+
if (key in keyAlreadyKnown) {
continue
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-30 17:41:33 +0000