Modify unit tests to distinguish between FAIL and ERROR. Move comparison utilities into our derived unit test class.

compiler/debug.py

@@ -14,14 +14,13 @@ def check(check,str):
      index) = inspect.getouterframes(inspect.currentframe())[1]
     if not check:
         print("ERROR: file {0}: line {1}: {2}".format(os.path.basename(filename),line_number,str))
-        sys.exit(-1)
+        assert 0
 
-def error(str,return_value=None):
+def error(str,return_value):
     (frame, filename, line_number, function_name, lines,
      index) = inspect.getouterframes(inspect.currentframe())[1]
     print("ERROR: file {0}: line {1}: {2}".format(os.path.basename(filename),line_number,str))
-    if return_value:
+    assert return_value==0
-        sys.exit(return_value)
 
 def warning(str):
     (frame, filename, line_number, function_name, lines,

compiler/tests/21_hspice_delay_test.py

@@ -4,7 +4,7 @@ Run a regresion test on various srams
 """
 
 import unittest
-from testutils import header,openram_test,isclose
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -77,9 +77,9 @@ class timing_sram_test(openram_test):
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.assertTrue(isclose(data[k][i],golden_data[k][i],0.15))
+                    self.isclose(data[k][i],golden_data[k][i],0.15)
             else:
-                self.assertTrue(isclose(data[k],golden_data[k],0.15))
+                self.isclose(data[k],golden_data[k],0.15)
 
                 
         # reset these options

compiler/tests/21_hspice_setuphold_test.py

@@ -4,7 +4,7 @@ Run a regresion test on various srams
 """
 
 import unittest
-from testutils import header,openram_test,isclose
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -54,9 +54,9 @@ class timing_setup_test(openram_test):
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.assertTrue(isclose(data[k][i],golden_data[k][i],0.15))
+                    self.isclose(data[k][i],golden_data[k][i],0.15)
             else:
-                self.assertTrue(isclose(data[k],golden_data[k],0.15))
+                self.isclose(data[k],golden_data[k],0.15)
 
         OPTS.check_lvsdrc = True
         OPTS.analytical_delay = True

compiler/tests/21_ngspice_delay_test.py

@@ -4,7 +4,7 @@ Run a regresion test on various srams
 """
 
 import unittest
-from testutils import header,openram_test,isclose
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -76,9 +76,9 @@ class timing_sram_test(openram_test):
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.assertTrue(isclose(data[k][i],golden_data[k][i],0.15))
+                    self.isclose(data[k][i],golden_data[k][i],0.15)
             else:
-                self.assertTrue(isclose(data[k],golden_data[k],0.15))
+                self.isclose(data[k],golden_data[k],0.15)
 
         # reset these options
         OPTS.check_lvsdrc = True

compiler/tests/21_ngspice_setuphold_test.py

@@ -4,7 +4,7 @@ Run a regresion test on various srams
 """
 
 import unittest
-from testutils import header,openram_test,isclose
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -53,9 +53,9 @@ class timing_setup_test(openram_test):
         for k in data.keys():
             if type(data[k])==list:
                 for i in range(len(data[k])):
-                    self.assertTrue(isclose(data[k][i],golden_data[k][i],0.15))
+                    self.isclose(data[k][i],golden_data[k][i],0.15)
             else:
-                self.assertTrue(isclose(data[k],golden_data[k],0.15))
+                self.isclose(data[k],golden_data[k],0.15)
 
         # reset these options
         OPTS.check_lvsdrc = True

compiler/tests/23_lib_sram_model_test.py

@@ -4,7 +4,7 @@ Check the .lib file for an SRAM
 """
 
 import unittest
-from testutils import header,openram_test,isapproxdiff
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -36,7 +36,7 @@ class lib_test(openram_test):
 
         # let's diff the result with a golden model
         golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),filename)
-        self.assertEqual(isapproxdiff(libname,golden,0.15),True)
+        self.isapproxdiff(libname,golden,0.15)
 
         globals.end_openram()
 

compiler/tests/23_lib_sram_prune_test.py

@@ -4,7 +4,7 @@ Check the .lib file for an SRAM with pruning
 """
 
 import unittest
-from testutils import header,openram_test,isapproxdiff
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -40,7 +40,7 @@ class lib_test(openram_test):
         
         # let's diff the result with a golden model
         golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),filename)
-        self.assertEqual(isapproxdiff(libname,golden,0.30),True)
+        self.isapproxdiff(libname,golden,0.30)
 
         OPTS.analytical_delay = True
         reload(characterizer)

compiler/tests/23_lib_sram_test.py

@@ -4,7 +4,7 @@ Check the .lib file for an SRAM
 """
 
 import unittest
-from testutils import header,openram_test,isapproxdiff
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -40,7 +40,7 @@ class lib_test(openram_test):
         
         # let's diff the result with a golden model
         golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),filename)
-        self.assertEqual(isapproxdiff(libname,golden,0.15),True)
+        self.isapproxdiff(libname,golden,0.15)
 
         OPTS.analytical_delay = True
         OPTS.trim_netlist = True

compiler/tests/24_lef_sram_test.py

@@ -4,7 +4,7 @@ Check the LEF file for an SRMA
 """
 
 import unittest
-from testutils import header,openram_test,isdiff
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -37,7 +37,7 @@ class lef_test(openram_test):
 
         # let's diff the result with a golden model
         golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),leffile)
-        self.assertEqual(isdiff(lefname,golden),True)
+        self.isdiff(lefname,golden)
 
         os.system("rm {0}".format(gdsname))
         os.system("rm {0}".format(lefname))

compiler/tests/25_verilog_sram_test.py

@@ -4,7 +4,7 @@ Check the  .v file for an SRAM
 """
 
 import unittest
-from testutils import header,openram_test,isdiff
+from testutils import header,openram_test
 import sys,os
 sys.path.append(os.path.join(sys.path[0],".."))
 import globals
@@ -35,7 +35,7 @@ class verilog_test(openram_test):
 
         # let's diff the result with a golden model
         golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),vfile)
-        self.assertEqual(isdiff(vname,golden),True)
+        self.isdiff(vname,golden)
 
         os.system("rm {0}".format(vname))
 

compiler/tests/testutils.py

@@ -27,16 +27,17 @@ class openram_test(unittest.TestCase):
 
         import verify
         try:
-            self.assertFalse(verify.run_drc(a.name, tempgds)==0)
+            self.assertTrue(verify.run_drc(a.name, tempgds)==0)
         except:
             self.reset()
-            raise Exception('DRC failed: {}'.format(a.name))
+            self.fail("DRC failed: {}".format(a.name))
+
             
         try:
-            self.assertFalse(verify.run_lvs(a.name, tempgds, tempspice)==0)
+            self.assertTrue(verify.run_lvs(a.name, tempgds, tempspice)==0)
         except:
             self.reset()
-            raise Exception('LVS failed: {}'.format(a.name))
+            self.fail("LVS mismatch: {}".format(a.name))
 
         self.cleanup()            
 
@@ -55,108 +56,106 @@ class openram_test(unittest.TestCase):
         import design
         design.design.name_map=[]
 
-def isclose(value1,value2,error_tolerance=1e-2):
+    def isclose(self, value1,value2,error_tolerance=1e-2):
-    """ This is used to compare relative values. """
+        """ This is used to compare relative values. """
-    import debug
+        import debug
-    relative_diff = abs(value1 - value2) / max(value1,value2)
+        relative_diff = abs(value1 - value2) / max(value1,value2)
-    check = relative_diff <= error_tolerance
+        check = relative_diff <= error_tolerance
-    if not check:
+        if not check:
-        debug.info(1,"NOT CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
+            debug.info(1,"NOT CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
-    else:
+        else:
-        debug.info(2,"CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
+            debug.info(2,"CLOSE {0} {1} relative diff={2}".format(value1,value2,relative_diff))
-    return (check)
+        return (check)
 
-def relative_compare(value1,value2,error_tolerance):
+    def relative_compare(self, value1,value2,error_tolerance):
-    """ This is used to compare relative values. """
+        """ This is used to compare relative values. """
-    if (value1==value2): # if we don't need a relative comparison!
+        if (value1==value2): # if we don't need a relative comparison!
-        return True
+            return True
-    return (abs(value1 - value2) / max(value1,value2) <= error_tolerance)
+        return (abs(value1 - value2) / max(value1,value2) <= error_tolerance)
 
-def isapproxdiff(f1, f2, error_tolerance=0.001):
+    def isapproxdiff(self, f1, f2, error_tolerance=0.001):
-    """Compare two files.
+        """Compare two files.
 
-    Arguments:
+        Arguments:
-
-    f1 -- First file name
-
-    f2 -- Second file name
-
-    Return value:
-
-    True if the files are the same, False otherwise.
-
-    """
-    import re
-    import debug
-
-    with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
-        while True:
-            b1 = fp1.readline()
-            b2 = fp2.readline()
-            #print "b1:",b1,
-            #print "b2:",b2,
-
-            # 1. Find all of the floats using a regex
-            numeric_const_pattern = r"""
-            [-+]? # optional sign
-            (?:
-            (?: \d* \. \d+ ) # .1 .12 .123 etc 9.1 etc 98.1 etc
-            |
-            (?: \d+ \.? ) # 1. 12. 123. etc 1 12 123 etc
-            )
-            # followed by optional exponent part if desired
-            (?: [Ee] [+-]? \d+ ) ?
-            """
-            rx = re.compile(numeric_const_pattern, re.VERBOSE)
-            b1_floats=rx.findall(b1)
-            b2_floats=rx.findall(b2)
-            debug.info(3,"b1_floats: "+str(b1_floats))
-            debug.info(3,"b2_floats: "+str(b2_floats))
         
-            # 2. Remove the floats from the string
+        f1 -- First file name
-            for f in b1_floats:
+        
-                b1=b1.replace(str(f),"",1)
+        f2 -- Second file name
-            for f in b2_floats:
+
-                b2=b2.replace(str(f),"",1)
+        Return value:
-            #print "b1:",b1,
+        
-            #print "b2:",b2,
+        True if the files are the same, False otherwise.
+        
+        """
+        import re
+        import debug
+
+        with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
+            while True:
+                b1 = fp1.readline()
+                b2 = fp2.readline()
+                #print "b1:",b1,
+                #print "b2:",b2,
+
+                # 1. Find all of the floats using a regex
+                numeric_const_pattern = r"""
+                [-+]? # optional sign
+                (?:
+                (?: \d* \. \d+ ) # .1 .12 .123 etc 9.1 etc 98.1 etc
+                |
+                (?: \d+ \.? ) # 1. 12. 123. etc 1 12 123 etc
+                )
+                # followed by optional exponent part if desired
+                (?: [Ee] [+-]? \d+ ) ?
+                """
+                rx = re.compile(numeric_const_pattern, re.VERBOSE)
+                b1_floats=rx.findall(b1)
+                b2_floats=rx.findall(b2)
+                debug.info(3,"b1_floats: "+str(b1_floats))
+                debug.info(3,"b2_floats: "+str(b2_floats))
+        
+                # 2. Remove the floats from the string
+                for f in b1_floats:
+                    b1=b1.replace(str(f),"",1)
+                for f in b2_floats:
+                    b2=b2.replace(str(f),"",1)
+                #print "b1:",b1,
+                #print "b2:",b2,
             
-            # 3. Check if remaining string matches
+                # 3. Check if remaining string matches
-            if b1 != b2:
+                if b1 != b2:
-                debug.info(1,"Line: {0}\n!=\nLine: {1}".format(b1,b2))
+                    self.fail("Line: {0}\n!=\nLine: {1}".format(b1,b2))
-                return False
 
-        # 4. Now compare that the floats match
+                # 4. Now compare that the floats match
-            if len(b1_floats)!=len(b2_floats):
+                if len(b1_floats)!=len(b2_floats):
-                debug.info(1,"Len {0} != {1}".format(len(b1_floats),len(b2_floats)))
+                    self.fail("Len {0} != {1}".format(len(b1_floats),len(b2_floats)))
-                return False
+                for (f1,f2) in zip(b1_floats,b2_floats):
-            for (f1,f2) in zip(b1_floats,b2_floats):
+                    if not relative_compare(float(f1),float(f2),error_tolerance):
-                if not relative_compare(float(f1),float(f2),error_tolerance):
+                        self.fail("Float {0} != {1}".format(f1,f2))
-                    debug.info(1, "Float {0} != {1}".format(f1,f2))
-                    return False
 
-            if not b1:
+                if not b1:
-                return True
+                    return
 
 
-def isdiff(file1,file2):
+
-    """ This is used to compare two files and display the diff if they are different.. """
+    def isdiff(self,file1,file2):
-    import debug
+        """ This is used to compare two files and display the diff if they are different.. """
-    import filecmp
+        import debug
-    import difflib
+        import filecmp
-    check = filecmp.cmp(file1,file2)
+        import difflib
-    if not check:
+        check = filecmp.cmp(file1,file2)
-        debug.info(2,"MISMATCH {0} {1}".format(file1,file2))
+        if not check:
-        f1 = open(file1,"r")
+            debug.info(2,"MISMATCH {0} {1}".format(file1,file2))
-        s1 = f1.readlines()
+            f1 = open(file1,"r")
-        f2 = open(file2,"r")
+            s1 = f1.readlines()
-        s2 = f2.readlines()
+            f2 = open(file2,"r")
-        for line in difflib.unified_diff(s1, s2):
+            s2 = f2.readlines()
-            debug.info(3,line)
+            for line in difflib.unified_diff(s1, s2):
-        debug.error("MISMATCH {0} {1}".format(file1,file2))            
+                debug.info(3,line)
-    else:
+            self.fail("MISMATCH {0} {1}".format(file1,file2))
-        debug.info(2,"MATCH {0} {1}".format(file1,file2))
+        else:
-    return (check)
+            debug.info(2,"MATCH {0} {1}".format(file1,file2))
+
 
 def header(filename, technology):
     tst = "Running Test for:"