From 26be7408f79be7207ba198f1c9f640968ab7859d Mon Sep 17 00:00:00 2001 From: John McMaster Date: Tue, 11 Dec 2018 12:24:46 -0800 Subject: [PATCH 1/5] int_loop: move into .sh for easier editing Signed-off-by: John McMaster --- fuzzers/int_loop.mk | 22 ++-------------------- fuzzers/int_loop.sh | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 20 deletions(-) create mode 100755 fuzzers/int_loop.sh diff --git a/fuzzers/int_loop.mk b/fuzzers/int_loop.mk index f4819201..e7891582 100644 --- a/fuzzers/int_loop.mk +++ b/fuzzers/int_loop.mk @@ -33,26 +33,8 @@ build/todo.txt: build/pips_int_l.txt maketodo.py # XXX: conider moving to script run: - \ - set -ex; \ - make clean; \ - mkdir -p todo; \ - while \ - make cleanprj; \ - make build/todo.txt || exit 1; \ - test -s build/todo.txt; \ - do \ - i=$$((i+1)); \ - cp build/todo.txt todo/$${i}.txt; \ - cp build/todo_all.txt todo/$${i}_all.txt; \ - if make database; then \ - make pushdb; \ - fi; \ - if [ "$(QUICK)" = "Y" ] ; then \ - break; \ - fi \ - done; \ - true + $(MAKE) clean + MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(FUZDIR)/../int_loop.sh touch run.ok clean: diff --git a/fuzzers/int_loop.sh b/fuzzers/int_loop.sh new file mode 100755 index 00000000..f6f620a5 --- /dev/null +++ b/fuzzers/int_loop.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -ex +MAKE=${MAKE:-make} +MAKEFLAGS=${MAKEFLAGS:-} +echo "make: ${MAKE} ${MAKEFLAGS}" +echo $MAKE +mkdir -p todo; +while + ${MAKE} ${MAKEFLAGS} cleanprj; + ${MAKE} ${MAKEFLAGS} build/todo.txt || exit 1; + test -s build/todo.txt; +do + i=$((i+1)); + cp build/todo.txt todo/${i}.txt; + cp build/todo_all.txt todo/${i}_all.txt; + if ${MAKE} ${MAKEFLAGS} database; then + ${MAKE} ${MAKEFLAGS} pushdb; + fi; + if [ "$QUICK" = "Y" ] ; then + break; + fi +done; +exit 0 + From c1bd5d0194d579e44e2bf33d6e5fed4cd83dc4ad Mon Sep 17 00:00:00 2001 From: John McMaster Date: Tue, 11 Dec 2018 14:11:55 -0800 Subject: [PATCH 2/5] int_loop.sh: while => if conditional Signed-off-by: John McMaster --- fuzzers/int_loop.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fuzzers/int_loop.sh b/fuzzers/int_loop.sh index f6f620a5..363eb6a3 100755 --- a/fuzzers/int_loop.sh +++ b/fuzzers/int_loop.sh @@ -5,11 +5,14 @@ MAKEFLAGS=${MAKEFLAGS:-} echo "make: ${MAKE} ${MAKEFLAGS}" echo $MAKE mkdir -p todo; -while +while true; do ${MAKE} ${MAKEFLAGS} cleanprj; ${MAKE} ${MAKEFLAGS} build/todo.txt || exit 1; - test -s build/todo.txt; -do + echo "Remaining: " $(wc -l build/todo_all.txt) + if [ \! -s build/todo.txt ] ; then + break + fi + i=$((i+1)); cp build/todo.txt todo/${i}.txt; cp build/todo_all.txt todo/${i}_all.txt; From 7669859c1a453ab6ec46f7852ff87890e0d2f30f Mon Sep 17 00:00:00 2001 From: John McMaster Date: Tue, 11 Dec 2018 14:44:01 -0800 Subject: [PATCH 3/5] int_loop_check: finer grained termination Signed-off-by: John McMaster --- fuzzers/int_loop.mk | 4 +- fuzzers/int_loop.sh | 30 +++++++- fuzzers/int_loop_check.py | 146 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 fuzzers/int_loop_check.py diff --git a/fuzzers/int_loop.mk b/fuzzers/int_loop.mk index e7891582..a91b292d 100644 --- a/fuzzers/int_loop.mk +++ b/fuzzers/int_loop.mk @@ -1,5 +1,7 @@ # WARNING: N cannot be reduced or -m will always fail N := 10 +# See int_loop_check.py +CHECK_ARGS := --zero-entries --timeout-iters 100 SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) # Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit @@ -34,7 +36,7 @@ build/todo.txt: build/pips_int_l.txt maketodo.py # XXX: conider moving to script run: $(MAKE) clean - MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(FUZDIR)/../int_loop.sh + XRAY_DIR=${XRAY_DIR} MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(XRAY_DIR)/fuzzers/int_loop.sh --check-args "$(CHECK_ARGS)" touch run.ok clean: diff --git a/fuzzers/int_loop.sh b/fuzzers/int_loop.sh index 363eb6a3..9c968b3a 100755 --- a/fuzzers/int_loop.sh +++ b/fuzzers/int_loop.sh @@ -1,4 +1,31 @@ #!/usr/bin/env bash + +usage() { + echo "Run makefile until termination condition" + echo "usage: int_loop.sh [args]" + echo "--check-args int_loop_check.py args" +} + +check_args= +while [[ $# -gt 0 ]]; do + case "$1" in + --check-args) + check_args=$2 + shift + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unrecognized argument" + usage + exit 1 + ;; + esac +done + set -ex MAKE=${MAKE:-make} MAKEFLAGS=${MAKEFLAGS:-} @@ -8,8 +35,7 @@ mkdir -p todo; while true; do ${MAKE} ${MAKEFLAGS} cleanprj; ${MAKE} ${MAKEFLAGS} build/todo.txt || exit 1; - echo "Remaining: " $(wc -l build/todo_all.txt) - if [ \! -s build/todo.txt ] ; then + if python3 ${XRAY_DIR}/fuzzers/int_loop_check.py $check_args ; then break fi diff --git a/fuzzers/int_loop_check.py b/fuzzers/int_loop_check.py new file mode 100644 index 00000000..25d66f95 --- /dev/null +++ b/fuzzers/int_loop_check.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import sys, re +import os +import glob +import hashlib + + +# len(txt.split("\n"))) is off by 1 +def wc(fn): + i = 0 + with open(fn) as f: + for i, _l in enumerate(f, 1): + pass + return i + + +def bytehex(x): + return ''.join('{:02x}'.format(x) for x in x) + + +def calc_stable_iters(todo_dir, max_iter): + m5s = [] + wcs = [] + m5_last = None + stablen = 0 + for fni in range(1, max_iter + 1, 1): + fn = "%s/%u_all.txt" % (todo_dir, fni) + txt = open(fn, "r").read() + m5 = hashlib.md5(txt.encode("ascii")).hexdigest() + + m5s.append(m5) + wc_this = wc(fn) + wcs.append(wc_this) + + if m5_last == m5: + stablen += 1 + else: + stablen = 1 + + print( + "% 4u %s % 6u lines % 6u stable" % + (fni, m5[0:8], wc_this, stablen)) + + m5_last = m5 + + return stablen + + +def run( + todo_dir, + min_iters=None, + stable_iters=None, + timeout_iters=None, + zero_entries=None, + verbose=False): + timeout_fn = "%s/timeout" % todo_dir + # make clean removes todo dir, but helps debugging + if os.path.exists(timeout_fn): + print("WARNING: removing %s" % timeout_fn) + os.remove(timeout_fn) + + alls = glob.glob("%s/*_all.txt" % todo_dir) + max_iter = 0 + for fn in alls: + n = int(re.match(r".*/([0-9]*)_all.txt", fn).group(1)) + max_iter = max(max_iter, n) + + if max_iter == 0: + print("Incomplete: no iters") + sys.exit(1) + + verbose and print("Max iter: %u, need: %s" % (max_iter, min_iters)) + + fn = "%s/%u_all.txt" % (todo_dir, max_iter) + txt = open(fn, "r").read() + nbytes = len(txt) + + stablen = calc_stable_iters(todo_dir, max_iter) + + if min_iters is not None and max_iter < min_iters: + print("Incomplete: not enough iters") + sys.exit(1) + + if timeout_iters is not None and max_iter > timeout_iters: + print("ERROR: timeout (max %u, got %u)" % (timeout_iters, max_iter)) + with open(timeout_fn, "w") as _f: + pass + sys.exit(1) + + if zero_entries and nbytes: + print("%s: %u bytes, %s lines" % (fn, nbytes, wc(fn))) + print("Incomplete: need zero entries") + sys.exit(1) + + if stable_iters: + if stablen < stable_iters: + print( + "Incomplete: insufficient stable iters (got %s, need %s)" % + (stablen, stable_iters)) + sys.exit(1) + + print("Complete!") + sys.exit(0) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description= + "Check int_loop completion. Exits 0 on done, 1 if more loops are needed" + ) + + parser.add_argument('--verbose', action='store_true', help='') + parser.add_argument('--todo-dir', default="todo", help='') + parser.add_argument( + '--min-iters', default=None, help='Minimum total number of iterations') + parser.add_argument( + '--stable-iters', + default=None, + help='Number of iterations without any change') + parser.add_argument( + '--timeout-iters', + default=None, + help='Max number of entries before creating todo/timeout') + parser.add_argument( + '--zero-entries', + action="store_true", + help='Must be no unsolved entries in latest') + args = parser.parse_args() + + def zint(x): + return None if x is None else int(x) + + run( + args.todo_dir, + zint(args.min_iters), + zint(args.stable_iters), + zint(args.timeout_iters), + args.zero_entries, + verbose=args.verbose) + + +if __name__ == '__main__': + main() From 4d81a436d4589f54980fd11ec35e978d2ab4eba9 Mon Sep 17 00:00:00 2001 From: John McMaster Date: Tue, 11 Dec 2018 14:47:49 -0800 Subject: [PATCH 4/5] int_loop: more aggressive default termination Signed-off-by: John McMaster --- fuzzers/int_loop.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzzers/int_loop.mk b/fuzzers/int_loop.mk index a91b292d..5ee8393a 100644 --- a/fuzzers/int_loop.mk +++ b/fuzzers/int_loop.mk @@ -1,7 +1,8 @@ # WARNING: N cannot be reduced or -m will always fail N := 10 # See int_loop_check.py -CHECK_ARGS := --zero-entries --timeout-iters 100 +# rempips took 35 iters once, so set 50 as a good start point +CHECK_ARGS := --zero-entries --timeout-iters 50 SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) # Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit From ed5d1b07e54c5834bbcad9c516f71b4bac3bea1f Mon Sep 17 00:00:00 2001 From: John McMaster Date: Tue, 11 Dec 2018 15:12:21 -0800 Subject: [PATCH 5/5] int_loop: check for timeout Signed-off-by: John McMaster --- fuzzers/int_loop.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fuzzers/int_loop.sh b/fuzzers/int_loop.sh index 9c968b3a..d62c0394 100755 --- a/fuzzers/int_loop.sh +++ b/fuzzers/int_loop.sh @@ -38,6 +38,10 @@ while true; do if python3 ${XRAY_DIR}/fuzzers/int_loop_check.py $check_args ; then break fi + if [ -f build/timeout ] ; then + echo "ERROR: timeout" + exit 1 + fi i=$((i+1)); cp build/todo.txt todo/${i}.txt;