From 6b967c08dd6d00f39750bebc8878a74365d7f031 Mon Sep 17 00:00:00 2001 From: Matt Guthaus Date: Fri, 27 Jul 2018 09:34:44 -0700 Subject: [PATCH] Updated output messages in timing test comparisons. Added output to show which lines differ and what their line numbers are.. Added output to show relative difference of approximate compares. Added output to include file names that mismatch. --- compiler/tests/23_lib_sram_model_test.py | 2 +- compiler/tests/23_lib_sram_prune_test.py | 2 +- ..._2_16_1_scn3me_subm_TT_5p0V_25C_pruned.lib | 58 +++---- compiler/tests/testutils.py | 153 ++++++++++++------ 4 files changed, 136 insertions(+), 79 deletions(-) diff --git a/compiler/tests/23_lib_sram_model_test.py b/compiler/tests/23_lib_sram_model_test.py index a41460f7..5d1d641c 100755 --- a/compiler/tests/23_lib_sram_model_test.py +++ b/compiler/tests/23_lib_sram_model_test.py @@ -40,7 +40,7 @@ class lib_test(openram_test): newname = filename.replace(".lib","_analytical.lib") libname = "{0}/{1}".format(OPTS.openram_temp,filename) golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),newname) - self.isapproxdiff(libname,golden,0.15) + self.assertTrue(self.isapproxdiff(libname,golden,0.15)) globals.end_openram() diff --git a/compiler/tests/23_lib_sram_prune_test.py b/compiler/tests/23_lib_sram_prune_test.py index 4a49bc74..35de23aa 100755 --- a/compiler/tests/23_lib_sram_prune_test.py +++ b/compiler/tests/23_lib_sram_prune_test.py @@ -49,7 +49,7 @@ class lib_test(openram_test): newname = filename.replace(".lib","_pruned.lib") libname = "{0}/{1}".format(OPTS.openram_temp,filename) golden = "{0}/golden/{1}".format(os.path.dirname(os.path.realpath(__file__)),newname) - self.isapproxdiff(libname,golden,0.40) + self.assertTrue(self.isapproxdiff(libname,golden,0.40)) reload(characterizer) globals.end_openram() diff --git a/compiler/tests/golden/sram_2_16_1_scn3me_subm_TT_5p0V_25C_pruned.lib b/compiler/tests/golden/sram_2_16_1_scn3me_subm_TT_5p0V_25C_pruned.lib index b37a777f..39924746 100644 --- a/compiler/tests/golden/sram_2_16_1_scn3me_subm_TT_5p0V_25C_pruned.lib +++ b/compiler/tests/golden/sram_2_16_1_scn3me_subm_TT_5p0V_25C_pruned.lib @@ -78,27 +78,31 @@ cell (sram_2_16_1_scn3me_subm){ dont_use : true; map_only : true; dont_touch : true; - area : 134589.78; + area : 142800.38999999998; leakage_power () { when : "CSb"; - value : 0.0004764706; + value : 0.0252988; } cell_leakage_power : 0; - bus(DATA){ + bus(DIN){ bus_type : DATA; - direction : inout; + direction : in; max_capacitance : 78.5936; min_capacitance : 2.45605; - three_state : "!OEb & !clk"; memory_write(){ address : ADDR; clocked_on : clk; } + bus(DOUT){ + bus_type : DATA; + direction : out; + max_capacitance : 78.5936; + min_capacitance : 2.45605; memory_read(){ address : ADDR; } - pin(DATA[1:0]){ + pin(DOUT[1:0]){ timing(){ timing_type : setup_rising; related_pin : "clk"; @@ -130,26 +134,26 @@ cell (sram_2_16_1_scn3me_subm){ timing(){ timing_sense : non_unate; related_pin : "clk"; - timing_type : falling_edge; + timing_type : rising_edge; cell_rise(CELL_TABLE) { - values("0.458, 0.504, 0.871",\ - "0.461, 0.506, 0.874",\ - "0.5, 0.544, 0.912"); + values("0.928, 0.959, 1.113",\ + "0.931, 0.962, 1.116",\ + "0.985, 1.018, 1.176"); } cell_fall(CELL_TABLE) { - values("0.573, 0.649, 1.251",\ - "0.577, 0.652, 1.254",\ - "0.618, 0.69, 1.29"); + values("11.214, 11.27, 11.756",\ + "11.22, 11.27, 11.761",\ + "11.268, 11.323, 11.809"); } rise_transition(CELL_TABLE) { - values("0.153, 0.233, 1.085",\ - "0.154, 0.234, 1.084",\ - "0.158, 0.237, 1.084"); + values("0.599, 0.624, 0.968",\ + "0.599, 0.623, 0.97",\ + "0.598, 0.623, 0.967"); } fall_transition(CELL_TABLE) { - values("0.276, 0.356, 1.5",\ - "0.277, 0.357, 1.5",\ - "0.278, 0.363, 1.5"); + values("11.157, 11.165, 11.446",\ + "11.159, 11.162, 1.271",\ + "11.158, 11.165, 11.47"); } } } @@ -298,19 +302,19 @@ cell (sram_2_16_1_scn3me_subm){ internal_power(){ when : "!CSb & clk & !WEb"; rise_power(scalar){ - values("4.42361814306"); + values("2.033783461111111"); } fall_power(scalar){ - values("4.42361814306"); + values("2.033783461111111"); } } internal_power(){ when : "!CSb & !clk & WEb"; rise_power(scalar){ - values("4.97118480973"); + values("2.032705683333334"); } fall_power(scalar){ - values("4.97118480973"); + values("2.032705683333334"); } } internal_power(){ @@ -326,20 +330,20 @@ cell (sram_2_16_1_scn3me_subm){ timing_type :"min_pulse_width"; related_pin : clk; rise_constraint(scalar) { - values("1.875"); + values("9.6875"); } fall_constraint(scalar) { - values("1.875"); + values("9.6875"); } } timing(){ timing_type :"minimum_period"; related_pin : clk; rise_constraint(scalar) { - values("3.75"); + values("19.375"); } fall_constraint(scalar) { - values("3.75"); + values("19.375"); } } } diff --git a/compiler/tests/testutils.py b/compiler/tests/testutils.py index 250483cf..be216761 100644 --- a/compiler/tests/testutils.py +++ b/compiler/tests/testutils.py @@ -79,7 +79,7 @@ class openram_test(unittest.TestCase): if not data_matches: import pprint data_string=pprint.pformat(data) - debug.info(0,"Data exceeded {:.1f}% tolerance:\n".format(error_tolerance*100)+data_string) + debug.error("Data exceeded {:.1f}% tolerance:\n".format(error_tolerance*100)+data_string) return data_matches @@ -87,14 +87,35 @@ class openram_test(unittest.TestCase): def isclose(self,key,value,actual_value,error_tolerance=1e-2): """ This is used to compare relative values. """ import debug - relative_diff = abs(value - actual_value) / max(value,actual_value) + relative_diff = self.relative_diff(value,actual_value) check = relative_diff <= error_tolerance - if not check: - debug.warning("NOT CLOSE\t{0: <10}\t{1:.3f}\t{2:.3f}\tdiff={3:.1f}%".format(key,value,actual_value,relative_diff*100)) - return False - else: + if check: debug.info(2,"CLOSE\t{0: <10}\t{1:.3f}\t{2:.3f}\tdiff={3:.1f}%".format(key,value,actual_value,relative_diff*100)) return True + else: + debug.error("NOT CLOSE\t{0: <10}\t{1:.3f}\t{2:.3f}\tdiff={3:.1f}%".format(key,value,actual_value,relative_diff*100)) + return False + + def relative_diff(self, value1, value2): + """ Compute the relative difference of two values and normalize to the largest. + If largest value is 0, just return the difference.""" + + # Edge case to avoid divide by zero + if value1==0 and value2==0: + return 0.0 + + # Don't need relative, exact compare + if value1==value2: + return 0.0 + + # Get normalization value + norm_value = abs(max(value1, value2)) + # Edge case where greater is a zero + if norm_value == 0: + min_value = abs(min(value1, value2)) + + return abs(value1 - value2) / norm_value + def relative_compare(self, value,actual_value,error_tolerance): """ This is used to compare relative values. """ @@ -102,14 +123,14 @@ class openram_test(unittest.TestCase): return True return (abs(value - actual_value) / max(value,actual_value) <= error_tolerance) - def isapproxdiff(self, f1, f2, error_tolerance=0.001): + def isapproxdiff(self, filename1, filename2, error_tolerance=0.001): """Compare two files. Arguments: - f1 -- First file name + filename1 -- First file name - f2 -- Second file name + filename2 -- Second file name Return value: @@ -130,61 +151,93 @@ class openram_test(unittest.TestCase): (?: [Ee] [+-]? \d+ ) ? """ rx = re.compile(numeric_const_pattern, re.VERBOSE) - with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2: - while True: - b1 = fp1.readline().decode('utf-8') - b2 = fp2.readline().decode('utf-8') - #print "b1:",b1, - #print "b2:",b2, + fp1 = open(filename1, 'rb') + fp2 = open(filename2, 'rb') + mismatches=0 + line_num=0 + while True: + line_num+=1 + line1 = fp1.readline().decode('utf-8') + line2 = fp2.readline().decode('utf-8') + #print("line1:",line1) + #print("line2:",line2) + + # 1. Find all of the floats using a regex + line1_floats=rx.findall(line1) + line2_floats=rx.findall(line2) + debug.info(3,"line1_floats: "+str(line1_floats)) + debug.info(3,"line2_floats: "+str(line2_floats)) - # 1. Find all of the floats using a regex - b1_floats=rx.findall(b1) - b2_floats=rx.findall(b2) - debug.info(3,"b1_floats: "+str(b1_floats)) - debug.info(3,"b2_floats: "+str(b2_floats)) - - # 2. Remove the floats from the string - for f in b1_floats: - b1=b1.replace(f,"",1) - for f in b2_floats: - b2=b2.replace(f,"",1) - #print "b1:",b1, - #print "b2:",b2, - # 3. Check if remaining string matches - if b1 != b2: - self.fail("MISMATCH Line: {0}\n!=\nLine: {1}".format(b1,b2)) + # 2. Remove the floats from the string + for f in line1_floats: + line1=line1.replace(f,"",1) + for f in line2_floats: + line2=line2.replace(f,"",1) + #print("line1:",line1) + #print("line2:",line2) - # 4. Now compare that the floats match - if len(b1_floats)!=len(b2_floats): - self.fail("MISMATCH Length {0} != {1}".format(len(b1_floats),len(b2_floats))) - for (f1,f2) in zip(b1_floats,b2_floats): - if not self.relative_compare(float(f1),float(f2),error_tolerance): - self.fail("MISMATCH Float {0} != {1}".format(f1,f2)) + # 3. Convert to floats rather than strings + line1_floats = [float(x) for x in line1_floats] + line2_floats = [float(x) for x in line1_floats] + + # 4. Check if remaining string matches + if line1 != line2: + if mismatches==0: + debug.error("Mismatching files:\nfile1={0}\nfile2={1}".format(filename1,filename2)) + mismatches += 1 + debug.error("MISMATCH Line ({0}):\n{1}\n!=\n{2}".format(line_num,line1.rstrip('\n'),line2.rstrip('\n'))) - if not b1 and not b2: - return + # 5. Now compare that the floats match + elif len(line1_floats)!=len(line2_floats): + if mismatches==0: + debug.error("Mismatching files:\nfile1={0}\nfile2={1}".format(filename1,filename2)) + mismatches += 1 + debug.error("MISMATCH Line ({0}) Length {1} != {2}".format(line_num,len(line1_floats),len(line2_floats))) + else: + for (float1,float2) in zip(line1_floats,line2_floats): + relative_diff = self.relative_diff(float1,float2) + check = relative_diff <= error_tolerance + if not check: + if mismatches==0: + debug.error("Mismatching files:\nfile1={0}\nfile2={1}".format(filename1,filename2)) + mismatches += 1 + debug.error("MISMATCH Line ({0}) Float {1} != {2} diff: {3:.1f}%".format(line_num,float1,float2,relative_diff*100)) + + # Only show the first 10 mismatch lines + if not line1 and not line2 or mismatches>10: + fp1.close() + fp2.close() + return mismatches==0 + + # Never reached + return False - - def isdiff(self,file1,file2): + def isdiff(self,filename1,filename2): """ This is used to compare two files and display the diff if they are different.. """ import debug import filecmp import difflib - check = filecmp.cmp(file1,file2) + check = filecmp.cmp(filename1,filename2) if not check: - debug.info(2,"MISMATCH {0} {1}".format(file1,file2)) - f1 = open(file1,"r") - s1 = f1.readlines() - f2 = open(file2,"r") + debug.error("MISMATCH file1={0} file2={1}".format(filename1,filename2)) + f1 = open(filename1,"r") + s1 = f1.readlines().decode('utf-8') + f1.close() + f2 = open(filename2,"r").decode('utf-8') s2 = f2.readlines() + f2.close() + mismatches=0 for line in difflib.unified_diff(s1, s2): - debug.info(3,line) - self.fail("MISMATCH {0} {1}".format(file1,file2)) + mismatches += 1 + self.error("DIFF LINES:",line) + if mismatches>10: + return False + return False else: - debug.info(2,"MATCH {0} {1}".format(file1,file2)) - + debug.info(2,"MATCH {0} {1}".format(filename1,filename2)) + return True def header(filename, technology): # Skip the header for gitlab regression