From 1adee642245581829a3464ed233a1fe6ceab8971 Mon Sep 17 00:00:00 2001
From: Brian Taylor <lbwnet@comcast.net>
Date: Fri, 1 Dec 2023 15:54:17 -0800
Subject: [PATCH] Add scripts for running the paranoia tests in parallel on
 Linux with valgrind.

---
 examples/paranoia/README.txt  | 26 +++++++++++++++++++++
 examples/paranoia/runtests.sh | 39 +++++++++++++++++++++++++++++++
 examples/paranoia/textract.py | 44 +++++++++++++++++++++++++++++++++++
 3 files changed, 109 insertions(+)
 create mode 100644 examples/paranoia/README.txt
 create mode 100755 examples/paranoia/runtests.sh
 create mode 100644 examples/paranoia/textract.py

diff --git a/examples/paranoia/README.txt b/examples/paranoia/README.txt
new file mode 100644
index 000000000..515b742ef
--- /dev/null
+++ b/examples/paranoia/README.txt
@@ -0,0 +1,26 @@
+
+To run the paranoia test suite in parallel on Linux with valgrind:
+
+1. Download the paranoia tests (paranoia.7z) from the ngspice Quality web page.
+
+2. p7zip -d paranoia.7z
+Rename the the unzipped directory to a name without spaces which would
+otherwise confuse valgrind.
+
+3. cd into the renamed unzipped directory.
+
+4. copy runtests.sh and textract.py from the examples/paranoia directory in
+your git repository to the current directory.
+
+5. If your computer has several cores, you can modify the -j4 in the line
+  time parallel -j4 bash ::: $2/*
+in runtests.sh and increase the number of parallel jobs.
+
+6. ./runtests.sh <paranoia_shell_script> <test_area_directory>
+For example:
+  ./runtests.sh paranoia_test.sh testdir
+Note that the test area directory must not exist before you invoke runtests.sh.
+
+Now relax and drink a cup of coffee. If you don't want to run the tests in
+parallel, it will take several cups.
+
diff --git a/examples/paranoia/runtests.sh b/examples/paranoia/runtests.sh
new file mode 100755
index 000000000..69b3973ff
--- /dev/null
+++ b/examples/paranoia/runtests.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+if test -n "$1" && test -n "$2" ; then
+  if test -d "$2" || test -e "$2" ; then
+    echo "$2 already exists, remove it first"
+    exit 1
+  fi
+python3 textract.py $1 $2
+else
+echo "arg 1 is the paranoia test script"
+echo "arg 2 is the test script working directory"
+exit 1
+fi
+
+SECONDS=0
+
+time parallel -j4 bash ::: $2/*
+
+wait
+NGSPICE_OK="`ngspice -v | awk '/level/ {print $2;}'` done"
+
+echo "*******************************************"
+echo "vlog files with errors found by valgrind:"
+grep -L "ERROR SUMMARY: 0 errors from 0 context" ./*.vlog
+echo "*******************************************"
+echo "log files with ngspice errors:"
+grep -L "$NGSPICE_OK" ./*.log
+echo "*******************************************"
+echo "log files with convergence issues:"
+grep -l "Too many iterations without convergence" ./*.log
+echo "*******************************************"
+echo "log files with messages containing 'error':"
+grep -i -l "error" ./*.log
+echo "*******************************************"
+
+ELAPSED="Elapsed: $(($SECONDS / 3600))hrs $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec"
+echo
+echo $ELAPSED
+
diff --git a/examples/paranoia/textract.py b/examples/paranoia/textract.py
new file mode 100644
index 000000000..a81cf6a75
--- /dev/null
+++ b/examples/paranoia/textract.py
@@ -0,0 +1,44 @@
+import os
+import sys
+
+
+testnum = 1
+
+
+def writeit(cd, cmd, outd):
+    global testnum
+    pwd = os.getcwd()
+    outfname = outd + '/testfile' + str(testnum) + '.sh'
+    outf = open(outfname, 'w')
+    testnum = testnum + 1
+    outf.write('#!/bin/bash\n')
+    outf.write('NGSPICE="ngspice -i "\n')
+    p1 = 'VALGRIND="valgrind --leak-check=full --suppressions='
+    p2 = p1 + pwd + '/ignore_shared_libs.supp"\n'
+    outf.write(p2)
+    outf.write(cd)
+    if cmd.endswith('&\n'):
+        outf.write(cmd[:-2] + '\n')
+    else:
+        outf.write(cmd)
+    os.chmod(outfname, 0o777)
+    outf.close()
+    return 0
+
+
+def main():
+    infile = sys.argv[1]
+    outdir = sys.argv[2]
+    os.mkdir(outdir)
+    inp = open(infile, 'r')
+    for line in inp:
+        if line.startswith('cd '):
+            cdname = line
+        elif line.startswith('$VALGRIND'):
+            writeit(cdname, line, outdir)
+    inp.close()
+    return 0
+
+
+if __name__ == '__main__':
+    main()