diff --git a/fuzzers/int_loop.mk b/fuzzers/int_loop.mk index e7891582..a91b292d 100644 --- a/fuzzers/int_loop.mk +++ b/fuzzers/int_loop.mk @@ -1,5 +1,7 @@ # WARNING: N cannot be reduced or -m will always fail N := 10 +# See int_loop_check.py +CHECK_ARGS := --zero-entries --timeout-iters 100 SPECIMENS := $(addprefix build/specimen_,$(shell seq -f '%03.0f' $(N))) SPECIMENS_OK := $(addsuffix /OK,$(SPECIMENS)) # Individual fuzzer directory, such as ~/prjxray/fuzzers/010-lutinit @@ -34,7 +36,7 @@ build/todo.txt: build/pips_int_l.txt maketodo.py # XXX: conider moving to script run: $(MAKE) clean - MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(FUZDIR)/../int_loop.sh + XRAY_DIR=${XRAY_DIR} MAKE="$(MAKE)" MAKEFLAGS="$(MAKEFLAGS)" QUICK=$(QUICK) $(XRAY_DIR)/fuzzers/int_loop.sh --check-args "$(CHECK_ARGS)" touch run.ok clean: diff --git a/fuzzers/int_loop.sh b/fuzzers/int_loop.sh index 363eb6a3..9c968b3a 100755 --- a/fuzzers/int_loop.sh +++ b/fuzzers/int_loop.sh @@ -1,4 +1,31 @@ #!/usr/bin/env bash + +usage() { + echo "Run makefile until termination condition" + echo "usage: int_loop.sh [args]" + echo "--check-args int_loop_check.py args" +} + +check_args= +while [[ $# -gt 0 ]]; do + case "$1" in + --check-args) + check_args=$2 + shift + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unrecognized argument" + usage + exit 1 + ;; + esac +done + set -ex MAKE=${MAKE:-make} MAKEFLAGS=${MAKEFLAGS:-} @@ -8,8 +35,7 @@ mkdir -p todo; while true; do ${MAKE} ${MAKEFLAGS} cleanprj; ${MAKE} ${MAKEFLAGS} build/todo.txt || exit 1; - echo "Remaining: " $(wc -l build/todo_all.txt) - if [ \! -s build/todo.txt ] ; then + if python3 ${XRAY_DIR}/fuzzers/int_loop_check.py $check_args ; then break fi diff --git a/fuzzers/int_loop_check.py b/fuzzers/int_loop_check.py new file mode 100644 index 00000000..25d66f95 --- /dev/null +++ b/fuzzers/int_loop_check.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import sys, re +import os +import glob +import hashlib + + +# len(txt.split("\n"))) is off by 1 +def wc(fn): + i = 0 + with open(fn) as f: + for i, _l in enumerate(f, 1): + pass + return i + + +def bytehex(x): + return ''.join('{:02x}'.format(x) for x in x) + + +def calc_stable_iters(todo_dir, max_iter): + m5s = [] + wcs = [] + m5_last = None + stablen = 0 + for fni in range(1, max_iter + 1, 1): + fn = "%s/%u_all.txt" % (todo_dir, fni) + txt = open(fn, "r").read() + m5 = hashlib.md5(txt.encode("ascii")).hexdigest() + + m5s.append(m5) + wc_this = wc(fn) + wcs.append(wc_this) + + if m5_last == m5: + stablen += 1 + else: + stablen = 1 + + print( + "% 4u %s % 6u lines % 6u stable" % + (fni, m5[0:8], wc_this, stablen)) + + m5_last = m5 + + return stablen + + +def run( + todo_dir, + min_iters=None, + stable_iters=None, + timeout_iters=None, + zero_entries=None, + verbose=False): + timeout_fn = "%s/timeout" % todo_dir + # make clean removes todo dir, but helps debugging + if os.path.exists(timeout_fn): + print("WARNING: removing %s" % timeout_fn) + os.remove(timeout_fn) + + alls = glob.glob("%s/*_all.txt" % todo_dir) + max_iter = 0 + for fn in alls: + n = int(re.match(r".*/([0-9]*)_all.txt", fn).group(1)) + max_iter = max(max_iter, n) + + if max_iter == 0: + print("Incomplete: no iters") + sys.exit(1) + + verbose and print("Max iter: %u, need: %s" % (max_iter, min_iters)) + + fn = "%s/%u_all.txt" % (todo_dir, max_iter) + txt = open(fn, "r").read() + nbytes = len(txt) + + stablen = calc_stable_iters(todo_dir, max_iter) + + if min_iters is not None and max_iter < min_iters: + print("Incomplete: not enough iters") + sys.exit(1) + + if timeout_iters is not None and max_iter > timeout_iters: + print("ERROR: timeout (max %u, got %u)" % (timeout_iters, max_iter)) + with open(timeout_fn, "w") as _f: + pass + sys.exit(1) + + if zero_entries and nbytes: + print("%s: %u bytes, %s lines" % (fn, nbytes, wc(fn))) + print("Incomplete: need zero entries") + sys.exit(1) + + if stable_iters: + if stablen < stable_iters: + print( + "Incomplete: insufficient stable iters (got %s, need %s)" % + (stablen, stable_iters)) + sys.exit(1) + + print("Complete!") + sys.exit(0) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description= + "Check int_loop completion. Exits 0 on done, 1 if more loops are needed" + ) + + parser.add_argument('--verbose', action='store_true', help='') + parser.add_argument('--todo-dir', default="todo", help='') + parser.add_argument( + '--min-iters', default=None, help='Minimum total number of iterations') + parser.add_argument( + '--stable-iters', + default=None, + help='Number of iterations without any change') + parser.add_argument( + '--timeout-iters', + default=None, + help='Max number of entries before creating todo/timeout') + parser.add_argument( + '--zero-entries', + action="store_true", + help='Must be no unsolved entries in latest') + args = parser.parse_args() + + def zint(x): + return None if x is None else int(x) + + run( + args.todo_dir, + zint(args.min_iters), + zint(args.stable_iters), + zint(args.timeout_iters), + args.zero_entries, + verbose=args.verbose) + + +if __name__ == '__main__': + main()