From 5088487cf7a42a7f48293f80ed31a0bbc7cfdea3 Mon Sep 17 00:00:00 2001
From: Matt Guthaus <mrg@ucsc.edu>
Date: Thu, 26 Jul 2018 15:45:17 -0700
Subject: [PATCH] Update delay tests to output useful information for debug.

---
 compiler/tests/21_hspice_delay_test.py  | 14 ++++++++++----
 compiler/tests/21_ngspice_delay_test.py | 12 +++++++++---
 compiler/tests/testutils.py             | 20 +++++++++++---------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/compiler/tests/21_hspice_delay_test.py b/compiler/tests/21_hspice_delay_test.py
index 2b89d3ac..99f20cef 100755
--- a/compiler/tests/21_hspice_delay_test.py
+++ b/compiler/tests/21_hspice_delay_test.py
@@ -49,7 +49,7 @@ class timing_sram_test(openram_test):
         loads = [tech.spice["msflop_in_cap"]*4]
         slews = [tech.spice["rise_time"]*2]
         data = d.analyze(probe_address, probe_data, slews, loads)
-        #print data
+
         if OPTS.tech_name == "freepdk45":
             golden_data = {'leakage_power': 0.0006964536000000001,
                            'delay_lh': [0.0573055],
@@ -76,14 +76,20 @@ class timing_sram_test(openram_test):
             self.assertTrue(False) # other techs fail
         # Check if no too many or too few results
         self.assertTrue(len(data.keys())==len(golden_data.keys()))
+
         # Check each result
+        data_matches = True
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.isclose(data[k][i],golden_data[k][i],0.15)
+                    if not self.isclose(k,data[k][i],golden_data[k][i],0.15):
+                        data_matches = False
             else:
-                self.isclose(data[k],golden_data[k],0.15)
-
+                self.isclose(k,data[k],golden_data[k],0.15)
+        if not data_matches:
+            debug.info(0,str(data))
+        self.assertTrue(data_matches)
+        
         globals.end_openram()
         
 # instantiate a copdsay of the class to actually run the test
diff --git a/compiler/tests/21_ngspice_delay_test.py b/compiler/tests/21_ngspice_delay_test.py
index d10b9e61..cdc0eba5 100755
--- a/compiler/tests/21_ngspice_delay_test.py
+++ b/compiler/tests/21_ngspice_delay_test.py
@@ -47,7 +47,7 @@ class timing_sram_test(openram_test):
         loads = [tech.spice["msflop_in_cap"]*4]
         slews = [tech.spice["rise_time"]*2]
         data = d.analyze(probe_address, probe_data, slews, loads)
-        #print data
+
         if OPTS.tech_name == "freepdk45":
             golden_data = {'leakage_power': 0.0007348262,
                            'delay_lh': [0.05799613],
@@ -75,13 +75,19 @@ class timing_sram_test(openram_test):
 
         # Check if no too many or too few results
         self.assertTrue(len(data.keys())==len(golden_data.keys()))
+        
         # Check each result
+        data_matches = True
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.isclose(data[k][i],golden_data[k][i],0.15)
+                    if not self.isclose(k,data[k][i],golden_data[k][i],0.15):
+                        data_matches = False
             else:
-                self.isclose(data[k],golden_data[k],0.15)
+                self.isclose(k,data[k],golden_data[k],0.15)
+        if not data_matches:
+            debug.info(0,str(data))
+        self.assertTrue(data_matches)
 
         globals.end_openram()
 
diff --git a/compiler/tests/testutils.py b/compiler/tests/testutils.py
index 6c7feddd..52861d7b 100644
--- a/compiler/tests/testutils.py
+++ b/compiler/tests/testutils.py
@@ -62,21 +62,23 @@ class openram_test(unittest.TestCase):
         
 
 
-    def isclose(self, value1,value2,error_tolerance=1e-2):
+    def isclose(self,key,value,actual_value,error_tolerance=1e-2):
         """ This is used to compare relative values. """
         import debug
-        relative_diff = abs(value1 - value2) / max(value1,value2)
+        relative_diff = abs(value - actual_value) / max(value,actual_value)
         check = relative_diff <= error_tolerance
         if not check:
-            self.fail("NOT CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
+            debug.warning("NOT CLOSE\t{0: <10}\t{1:.3f}\t{2:.3f}\tdiff={3:.1f}%".format(key,value,actual_value,relative_diff*100))
+            return False
         else:
-            debug.info(2,"CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
-
-    def relative_compare(self, value1,value2,error_tolerance):
-        """ This is used to compare relative values. """
-        if (value1==value2): # if we don't need a relative comparison!
+            debug.info(2,"CLOSE\t{0: <10}\t{1:.3f}\t{2:.3f}\tdiff={3:.1f}%".format(key,value,actual_value,relative_diff*100))
             return True
-        return (abs(value1 - value2) / max(value1,value2) <= error_tolerance)
+
+    def relative_compare(self, value,actual_value,error_tolerance):
+        """ This is used to compare relative values. """
+        if (value==actual_value): # if we don't need a relative comparison!
+            return True
+        return (abs(value - actual_value) / max(value,actual_value) <= error_tolerance)
 
     def isapproxdiff(self, f1, f2, error_tolerance=0.001):
         """Compare two files.