From 19e99d1c7b29281b1a1f12bf73d9de4ef25c9752 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 3 Feb 2021 14:19:11 -0800 Subject: [PATCH 01/18] Enable parallel regression testing. --- compiler/characterizer/stimuli.py | 6 +-- compiler/globals.py | 15 +++++++- compiler/options.py | 7 ++-- compiler/tests/regress.py | 61 ++++++++++++++++++++++++++++++- compiler/tests/testutils.py | 3 +- 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/compiler/characterizer/stimuli.py b/compiler/characterizer/stimuli.py index eed7a0d6..035e9e58 100644 --- a/compiler/characterizer/stimuli.py +++ b/compiler/characterizer/stimuli.py @@ -312,12 +312,12 @@ class stimuli(): cmd = "{0} {1} -c {2}xa.cfg -o {2}xa -mt {3}".format(OPTS.spice_exe, temp_stim, OPTS.openram_temp, - OPTS.num_threads) + OPTS.num_sim_threads) valid_retcode=0 elif OPTS.spice_name == "hspice": # TODO: Should make multithreading parameter a configuration option cmd = "{0} -mt {1} -i {2} -o {3}timing".format(OPTS.spice_exe, - OPTS.num_threads, + OPTS.num_sim_threads, temp_stim, OPTS.openram_temp) valid_retcode=0 @@ -326,7 +326,7 @@ class stimuli(): # Measurements can't be made with a raw file set in ngspice # -r {2}timing.raw ng_cfg = open("{}.spiceinit".format(OPTS.openram_temp), "w") - ng_cfg.write("set num_threads={}\n".format(OPTS.num_threads)) + ng_cfg.write("set num_threads={}\n".format(OPTS.num_sim_threads)) ng_cfg.close() cmd = "{0} -b -o {2}timing.lis {1}".format(OPTS.spice_exe, diff --git a/compiler/globals.py b/compiler/globals.py index 57f1577b..8ea989df 100644 --- a/compiler/globals.py +++ b/compiler/globals.py @@ -18,6 +18,8 @@ import sys import re import copy import importlib +import getpass + VERSION = "1.1.9" NAME = "OpenRAM v{}".format(VERSION) @@ -113,6 +115,10 @@ def parse_args(): if OPTS.tech_name == "s8": OPTS.tech_name = "sky130" + if OPTS.openram_temp: + # If they define the temp directory, we can only use one thread at a time! + OPTS.num_threads = 1 + return (options, args) @@ -133,8 +139,9 @@ def print_banner(): debug.print_raw("|=========" + user_info.center(60) + "=========|") dev_info = "Development help: openram-dev-group@ucsc.edu" debug.print_raw("|=========" + dev_info.center(60) + "=========|") - temp_info = "Temp dir: {}".format(OPTS.openram_temp) - debug.print_raw("|=========" + temp_info.center(60) + "=========|") + if OPTS.openram_temp: + temp_info = "Temp dir: {}".format(OPTS.openram_temp) + debug.print_raw("|=========" + temp_info.center(60) + "=========|") debug.print_raw("|=========" + "See LICENSE for license info".center(60) + "=========|") debug.print_raw("|==============================================================================|") @@ -414,6 +421,10 @@ def setup_paths(): if "__pycache__" not in full_path: sys.path.append("{0}".format(full_path)) + # Use a unique temp directory + if not OPTS.openram_temp: + OPTS.openram_temp = "/tmp/openram_{0}_{1}_temp/".format(getpass.getuser(), + os.getpid()) if not OPTS.openram_temp.endswith('/'): OPTS.openram_temp += "/" debug.info(1, "Temporary files saved in " + OPTS.openram_temp) diff --git a/compiler/options.py b/compiler/options.py index 87083a7e..91bae758 100644 --- a/compiler/options.py +++ b/compiler/options.py @@ -74,9 +74,8 @@ class options(optparse.Values): # If user defined the temporary location in their environment, use it openram_temp = os.path.abspath(os.environ.get("OPENRAM_TMP")) except: - # Else use a unique temporary directory - openram_temp = "/tmp/openram_{0}_{1}_temp/".format(getpass.getuser(), - os.getpid()) + openram_temp = None + # This is the verbosity level to control debug information. 0 is none, 1 # is minimal, etc. verbose_level = 0 @@ -135,6 +134,8 @@ class options(optparse.Values): # Number of threads to use num_threads = 2 + # Number of threads to use in ngspice/hspice + num_sim_threads = 2 # Should we print out the banner at startup print_banner = True diff --git a/compiler/tests/regress.py b/compiler/tests/regress.py index bac5d8e1..bd8ba019 100755 --- a/compiler/tests/regress.py +++ b/compiler/tests/regress.py @@ -12,6 +12,9 @@ import unittest import sys, os sys.path.append(os.getenv("OPENRAM_HOME")) import globals +from subunit import ProtocolTestCase, TestProtocolClient +from subunit.test_results import AutoTimingTestResultDecorator +from testtools import ConcurrentTestSuite (OPTS, args) = globals.parse_args() del sys.argv[1:] @@ -39,17 +42,71 @@ all_tests = list(filter(nametest.search, files)) filtered_tests = list(filter(lambda i: i not in skip_tests, all_tests)) filtered_tests.sort() +num_threads = OPTS.num_threads + + +def partition_unit_tests(suite, num_threads): + partitions = [list() for x in range(num_threads)] + for index, test in enumerate(suite): + partitions[index % num_threads].append(test) + return partitions + + +def fork_tests(num_threads): + results = [] + test_partitions = partition_unit_tests(suite, num_threads) + suite._tests[:] = [] + + def do_fork(suite): + + for test_partition in test_partitions: + test_suite = unittest.TestSuite(test_partition) + test_partition[:] = [] + c2pread, c2pwrite = os.pipe() + pid = os.fork() + if pid == 0: + # PID of 0 is a child + try: + # Open a stream to write to the parent + stream = os.fdopen(c2pwrite, 'wb', 0) + os.close(c2pread) + sys.stdin.close() + test_suite_result = AutoTimingTestResultDecorator(TestProtocolClient(stream)) + test_suite.run(test_suite_result) + except: + try: + stream.write(traceback.format_exc()) + finally: + os._exit(1) + os._exit(0) + else: + # PID >0 is the parent + # Collect all of the child streams and append to the results + os.close(c2pwrite) + stream = os.fdopen(c2pread, 'rb', 0) + test = ProtocolTestCase(stream) + results.append(test) + return results + return do_fork + + # import all of the modules filenameToModuleName = lambda f: os.path.splitext(f)[0] moduleNames = map(filenameToModuleName, filtered_tests) modules = map(__import__, moduleNames) + suite = unittest.TestSuite() load = unittest.defaultTestLoader.loadTestsFromModule suite.addTests(map(load, modules)) test_runner = unittest.TextTestRunner(verbosity=2, stream=sys.stderr) -test_result = test_runner.run(suite) - +if num_threads == 1: + final_suite = suite +else: + final_suite = ConcurrentTestSuite(suite, fork_tests(num_threads)) + +test_result = test_runner.run(final_suite) + import verify verify.print_drc_stats() verify.print_lvs_stats() diff --git a/compiler/tests/testutils.py b/compiler/tests/testutils.py index eddee87d..b6468749 100644 --- a/compiler/tests/testutils.py +++ b/compiler/tests/testutils.py @@ -315,7 +315,8 @@ def header(filename, technology): print("|=========" + technology.center(60) + "=========|") print("|=========" + filename.center(60) + "=========|") from globals import OPTS - print("|=========" + OPTS.openram_temp.center(60) + "=========|") + if OPTS.openram_temp: + print("|=========" + OPTS.openram_temp.center(60) + "=========|") print("|==============================================================================|") From 181808c326dd12ede44d8adda1d1310127fbefcf Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 3 Feb 2021 14:26:09 -0800 Subject: [PATCH 02/18] Change number of threads in CI --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27431cb2..bcfcc64d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: freepdk45: stage: test script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -t freepdk45 + - coverage run -p $OPENRAM_HOME/tests/regress.py -j 16 -t freepdk45 artifacts: paths: - .coverage.* @@ -19,7 +19,7 @@ freepdk45: scn4m_subm: stage: test script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -t scn4m_subm + - coverage run -p $OPENRAM_HOME/tests/regress.py -j 16 -t scn4m_subm artifacts: paths: - .coverage.* From e043aaffb3a91c64aa1b5cb6143b9cb36f6b3c56 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 3 Feb 2021 15:17:28 -0800 Subject: [PATCH 03/18] Don't print DRC/LVS/PEX run stats in regress.py --- compiler/tests/regress.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/tests/regress.py b/compiler/tests/regress.py index bd8ba019..285f8c7d 100755 --- a/compiler/tests/regress.py +++ b/compiler/tests/regress.py @@ -107,9 +107,9 @@ else: test_result = test_runner.run(final_suite) -import verify -verify.print_drc_stats() -verify.print_lvs_stats() -verify.print_pex_stats() +# import verify +# verify.print_drc_stats() +# verify.print_lvs_stats() +# verify.print_pex_stats() sys.exit(not test_result.wasSuccessful()) From 18cddb4b233d36408404e756804492452d538e82 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 3 Feb 2021 16:22:44 -0800 Subject: [PATCH 04/18] Reduce to 6 threads. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bcfcc64d..d0677eaf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: freepdk45: stage: test script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -j 16 -t freepdk45 + - coverage run -p $OPENRAM_HOME/tests/regress.py -j 6 -t freepdk45 artifacts: paths: - .coverage.* @@ -19,7 +19,7 @@ freepdk45: scn4m_subm: stage: test script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -j 16 -t scn4m_subm + - coverage run -p $OPENRAM_HOME/tests/regress.py -j 6 -t scn4m_subm artifacts: paths: - .coverage.* From ed4fdbc8bebd15416aeaeda98199b17fe9e1c0f7 Mon Sep 17 00:00:00 2001 From: mrg Date: Fri, 5 Feb 2021 09:42:48 -0800 Subject: [PATCH 05/18] Add initial requirements.txt --- requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a165c2b1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +scikit-learn>=0.22.2 +coverage>=4.5.2 +scipy>=1.3.3 +numpy>=1.17.4 +subunit>=1.4.0 From 5a0c75d573dfa7cab187b86f910c26c4e2f59618 Mon Sep 17 00:00:00 2001 From: mrg Date: Fri, 5 Feb 2021 09:48:14 -0800 Subject: [PATCH 06/18] Fix subunit name. Add unittest requirement. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a165c2b1..2a8879b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ scikit-learn>=0.22.2 coverage>=4.5.2 scipy>=1.3.3 numpy>=1.17.4 -subunit>=1.4.0 +python-subunit>=1.4.0 +unittest2>=1.1.0 From b83d93cc9a8e5379f98836423b2e0e5808d9ca71 Mon Sep 17 00:00:00 2001 From: mrg Date: Fri, 5 Feb 2021 17:20:08 -0800 Subject: [PATCH 07/18] GitHub Actions CI flow. --- .github/workflows/ci.yml | 47 ++++++++++++++++++++ .gitlab-ci.yml | 48 --------------------- compiler/globals.py | 12 ++++++ compiler/tests/30_openram_back_end_test.py | 7 +-- compiler/tests/30_openram_front_end_test.py | 7 +-- 5 files changed, 61 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..512df925 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: ci +on: [push] +jobs: + setup: + runs-on: self-hosted + steps: + - name: Check out repository + uses: actions/checkout@v1 + - name: Hello + run: echo "Hello, world!" + scn4me_subm: + # Run this first since it is faster + needs: setup + runs-on: self-hosted + steps: + - name: SCMOS test + run: | + . /home/github-runner/setup-paths.sh + export OPENRAM_HOME="`pwd`/compiler" + export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm + freepdk45: + # Run this second and only if the first passes + needs: scn4me_subm + runs-on: self-hosted + steps: + - name: FreePDK45 test + run: | + . /home/github-runner/setup-paths.sh + export OPENRAM_HOME="`pwd`/compiler" + export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 + coverage: + needs: [scn4me_subm, freepdk45] + runs-on: self-hosted + steps: + - name: Coverage stats + run: | + python3-coverage combine + python3-coverage report + python3-coverage html -d coverage_html + - name: Archive coverage + uses: actions/upload-artifact@v2 + with: + name: code-coverage-report + path: coverage_html/ + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index d0677eaf..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,48 +0,0 @@ -before_script: - - . /home/gitlab-runner/setup-paths.sh - - export OPENRAM_HOME="`pwd`/compiler" - - export OPENRAM_TECH="`pwd`/technology:/home/PDKs/skywater-tech" - -stages: - - test - - coverage - -freepdk45: - stage: test - script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -j 6 -t freepdk45 - artifacts: - paths: - - .coverage.* - expire_in: 1 week - -scn4m_subm: - stage: test - script: - - coverage run -p $OPENRAM_HOME/tests/regress.py -j 6 -t scn4m_subm - artifacts: - paths: - - .coverage.* - expire_in: 1 week - -# s8: -# stage: test -# script: -# - coverage run -p $OPENRAM_HOME/tests/regress.py -t s8 -# artifacts: -# paths: -# - .coverage.* -# expire_in: 1 week - -coverage: - stage: coverage - script: - - coverage combine - - coverage report - - coverage html -d coverage_html - artifacts: - paths: - - coverage_html - expire_in: 1 week - coverage: '/TOTAL.+ ([0-9]{1,3}%)/' - diff --git a/compiler/globals.py b/compiler/globals.py index 8ea989df..ae290be9 100644 --- a/compiler/globals.py +++ b/compiler/globals.py @@ -19,6 +19,7 @@ import re import copy import importlib import getpass +import subprocess VERSION = "1.1.9" @@ -161,6 +162,17 @@ def check_versions(): # or, this could be done in each module (e.g. verify, characterizer, etc.) global OPTS + def cmd_exists(cmd): + return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + + if cmd_exists("coverage"): + OPTS.coverage_exe = "coverage run -p " + elif cmd_exists("python3-coverage"): + OPTS.coverage_exe = "python3-coverage run -p " + else: + OPTS.coverage_exe = "" + debug.warning("Failed to find coverage installation. This can be installed with pip3 install coverage") + try: import coverage OPTS.coverage = 1 diff --git a/compiler/tests/30_openram_back_end_test.py b/compiler/tests/30_openram_back_end_test.py index 7a2cc012..af8773a2 100755 --- a/compiler/tests/30_openram_back_end_test.py +++ b/compiler/tests/30_openram_back_end_test.py @@ -46,12 +46,7 @@ class openram_back_end_test(openram_test): if OPTS.spice_name: options += " -s {}".format(OPTS.spice_name) - # Always perform code coverage - if OPTS.coverage == 0: - debug.warning("Failed to find coverage installation. This can be installed with pip3 install coverage") - exe_name = "{0}/openram.py ".format(OPENRAM_HOME) - else: - exe_name = "coverage run -p {0}/openram.py ".format(OPENRAM_HOME) + exe_name = "{0}{1}/openram.py ".format(OPTS.coverage_exe, OPENRAM_HOME) config_name = "{0}/tests/configs/config_back_end.py".format(OPENRAM_HOME) cmd = "{0} -o {1} -p {2} {3} {4} 2>&1 > {5}/output.log".format(exe_name, out_file, diff --git a/compiler/tests/30_openram_front_end_test.py b/compiler/tests/30_openram_front_end_test.py index d5089bbc..b80c6d7d 100755 --- a/compiler/tests/30_openram_front_end_test.py +++ b/compiler/tests/30_openram_front_end_test.py @@ -46,12 +46,7 @@ class openram_front_end_test(openram_test): if OPTS.spice_name: options += " -s {}".format(OPTS.spice_name) - # Always perform code coverage - if OPTS.coverage == 0: - debug.warning("Failed to find coverage installation. This can be installed with pip3 install coverage") - exe_name = "{0}/openram.py ".format(OPENRAM_HOME) - else: - exe_name = "coverage run -p {0}/openram.py ".format(OPENRAM_HOME) + exe_name = "{0}{1}/openram.py ".format(OPTS.coverage_exe, OPENRAM_HOME) config_name = "{0}/tests/configs/config_front_end.py".format(OPENRAM_HOME) cmd = "{0} -n -o {1} -p {2} {3} {4} 2>&1 > {5}/output.log".format(exe_name, out_file, From 29c3d46be68fac2410ecc46cfce02bb09e0b8542 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 10:23:06 -0800 Subject: [PATCH 08/18] Warn about threads forced to 1 --- compiler/globals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/globals.py b/compiler/globals.py index 8ea989df..7afc2c8b 100644 --- a/compiler/globals.py +++ b/compiler/globals.py @@ -117,6 +117,7 @@ def parse_args(): if OPTS.openram_temp: # If they define the temp directory, we can only use one thread at a time! + debug.warning("num_threads forced to 1 due to shared temp directory {}".format(OPTS.openram_temp)) OPTS.num_threads = 1 return (options, args) From b82b7aaf280bf624017dd484360ba0522c0e75eb Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 12:10:04 -0800 Subject: [PATCH 09/18] PEP8 format --- compiler/characterizer/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/compiler/characterizer/__init__.py b/compiler/characterizer/__init__.py index 040658f0..0d9fbe83 100644 --- a/compiler/characterizer/__init__.py +++ b/compiler/characterizer/__init__.py @@ -7,8 +7,7 @@ # import os import debug -import globals -from globals import OPTS,find_exe,get_tool +from globals import OPTS, find_exe, get_tool from .lib import * from .delay import * from .elmore import * @@ -21,7 +20,7 @@ from .model_check import * from .analytical_util import * from .regression_model import * -debug.info(1,"Initializing characterizer...") +debug.info(1, "Initializing characterizer...") OPTS.spice_exe = "" if not OPTS.analytical_delay: @@ -30,17 +29,17 @@ if not OPTS.analytical_delay: if OPTS.spice_name != "": OPTS.spice_exe=find_exe(OPTS.spice_name) if OPTS.spice_exe=="" or OPTS.spice_exe==None: - debug.error("{0} not found. Unable to perform characterization.".format(OPTS.spice_name),1) + debug.error("{0} not found. Unable to perform characterization.".format(OPTS.spice_name), 1) else: - (OPTS.spice_name,OPTS.spice_exe) = get_tool("spice",["hspice", "ngspice", "ngspice.exe", "xa"]) + (OPTS.spice_name, OPTS.spice_exe) = get_tool("spice", ["ngspice", "ngspice.exe", "hspice", "xa"]) # set the input dir for spice files if using ngspice if OPTS.spice_name == "ngspice": os.environ["NGSPICE_INPUT_DIR"] = "{0}".format(OPTS.openram_temp) if OPTS.spice_exe == "": - debug.error("No recognizable spice version found. Unable to perform characterization.",1) + debug.error("No recognizable spice version found. Unable to perform characterization.", 1) else: - debug.info(1,"Analytical model enabled.") + debug.info(1, "Analytical model enabled.") From 7610f23fc7b303bfa085e1011d99ce7f2fe4c3c4 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 15:39:12 -0800 Subject: [PATCH 10/18] Sub temp directory. Add github archive. --- .github/workflows/ci.yml | 14 +++++++++++++- compiler/globals.py | 15 +++++---------- compiler/options.py | 2 +- compiler/tests/regress.py | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 512df925..6215641e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,16 @@ jobs: . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" + export OPENRAM_TMP="`pwd`/scn4me_subm" python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm + - name: Archive + uses: actions/upload-artifact@v2 + with: + name: scn4me_subm Archives + path: $OPENRAM_TMP/ freepdk45: # Run this second and only if the first passes - needs: scn4me_subm + needs: setup runs-on: self-hosted steps: - name: FreePDK45 test @@ -29,7 +35,13 @@ jobs: . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" + export OPENRAM_TMP="`pwd`/freepdk45" python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 + - name: Archive + uses: actions/upload-artifact@v2 + with: + name: FreePDK45 Archives + path: $OPENRAM_TMP/ coverage: needs: [scn4me_subm, freepdk45] runs-on: self-hosted diff --git a/compiler/globals.py b/compiler/globals.py index d6cff8a4..8a4cff3d 100644 --- a/compiler/globals.py +++ b/compiler/globals.py @@ -116,11 +116,6 @@ def parse_args(): if OPTS.tech_name == "s8": OPTS.tech_name = "sky130" - if OPTS.openram_temp: - # If they define the temp directory, we can only use one thread at a time! - debug.warning("num_threads forced to 1 due to shared temp directory {}".format(OPTS.openram_temp)) - OPTS.num_threads = 1 - return (options, args) @@ -426,7 +421,7 @@ def setup_paths(): # Add all of the subdirs to the python path # These subdirs are modules and don't need # to be added: characterizer, verify - subdirlist = [ item for item in os.listdir(OPENRAM_HOME) if os.path.isdir(os.path.join(OPENRAM_HOME, item)) ] + subdirlist = [item for item in os.listdir(OPENRAM_HOME) if os.path.isdir(os.path.join(OPENRAM_HOME, item))] for subdir in subdirlist: full_path = "{0}/{1}".format(OPENRAM_HOME, subdir) debug.check(os.path.isdir(full_path), @@ -434,10 +429,10 @@ def setup_paths(): if "__pycache__" not in full_path: sys.path.append("{0}".format(full_path)) - # Use a unique temp directory - if not OPTS.openram_temp: - OPTS.openram_temp = "/tmp/openram_{0}_{1}_temp/".format(getpass.getuser(), - os.getpid()) + # Use a unique temp subdirectory + OPTS.openram_temp += "/openram_{0}_{1}_temp/".format(getpass.getuser(), + os.getpid()) + if not OPTS.openram_temp.endswith('/'): OPTS.openram_temp += "/" debug.info(1, "Temporary files saved in " + OPTS.openram_temp) diff --git a/compiler/options.py b/compiler/options.py index 91bae758..4c04cdb0 100644 --- a/compiler/options.py +++ b/compiler/options.py @@ -74,7 +74,7 @@ class options(optparse.Values): # If user defined the temporary location in their environment, use it openram_temp = os.path.abspath(os.environ.get("OPENRAM_TMP")) except: - openram_temp = None + openram_temp = "/tmp" # This is the verbosity level to control debug information. 0 is none, 1 # is minimal, etc. diff --git a/compiler/tests/regress.py b/compiler/tests/regress.py index 285f8c7d..878182a5 100755 --- a/compiler/tests/regress.py +++ b/compiler/tests/regress.py @@ -73,7 +73,7 @@ def fork_tests(num_threads): sys.stdin.close() test_suite_result = AutoTimingTestResultDecorator(TestProtocolClient(stream)) test_suite.run(test_suite_result) - except: + except EBADF: try: stream.write(traceback.format_exc()) finally: From 8435908afa7b7d7db337cd9508bf8552e5f287c7 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 15:40:17 -0800 Subject: [PATCH 11/18] Remove tab. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6215641e..b3b5b48a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" - export OPENRAM_TMP="`pwd`/scn4me_subm" + export OPENRAM_TMP="`pwd`/scn4me_subm" python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm - name: Archive uses: actions/upload-artifact@v2 @@ -35,7 +35,7 @@ jobs: . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" - export OPENRAM_TMP="`pwd`/freepdk45" + export OPENRAM_TMP="`pwd`/freepdk45" python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 - name: Archive uses: actions/upload-artifact@v2 From 7d7f849b30557ce79bc21697e2e145b1ec6e4082 Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 16:55:24 -0800 Subject: [PATCH 12/18] Separate checkouts for runners --- .github/workflows/ci.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b5b48a..4f9573b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,42 +1,35 @@ name: ci on: [push] jobs: - setup: + scn4me_subm: runs-on: self-hosted steps: - name: Check out repository uses: actions/checkout@v1 - - name: Hello - run: echo "Hello, world!" - scn4me_subm: - # Run this first since it is faster - needs: setup - runs-on: self-hosted - steps: - name: SCMOS test run: | . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/scn4me_subm" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -k -t scn4m_subm - name: Archive uses: actions/upload-artifact@v2 with: name: scn4me_subm Archives path: $OPENRAM_TMP/ freepdk45: - # Run this second and only if the first passes - needs: setup runs-on: self-hosted steps: + - name: Check out repository + uses: actions/checkout@v1 - name: FreePDK45 test run: | . /home/github-runner/setup-paths.sh export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/freepdk45" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -k -t freepdk45 - name: Archive uses: actions/upload-artifact@v2 with: From d236f78718eb02a0e558b44e13039df66fc67b9d Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 16:57:09 -0800 Subject: [PATCH 13/18] Change from 32 to 20 threads each. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f9573b0..d6bd8f55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/scn4me_subm" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -k -t scn4m_subm + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -k -t scn4m_subm - name: Archive uses: actions/upload-artifact@v2 with: @@ -29,7 +29,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/freepdk45" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -k -t freepdk45 + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -k -t freepdk45 - name: Archive uses: actions/upload-artifact@v2 with: From b57487f5e504f654c37895f4d34de60d44ef472d Mon Sep 17 00:00:00 2001 From: mrg Date: Wed, 10 Feb 2021 21:37:33 -0800 Subject: [PATCH 14/18] Only upload archive on failure, always do coverage check. --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6bd8f55..2df69733 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,9 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/scn4me_subm" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -k -t scn4m_subm + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -t scn4m_subm - name: Archive + if: ${{ failure() }} uses: actions/upload-artifact@v2 with: name: scn4me_subm Archives @@ -29,13 +30,15 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/freepdk45" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -k -t freepdk45 + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -t freepdk45 - name: Archive + if: ${{ failure() }} uses: actions/upload-artifact@v2 with: name: FreePDK45 Archives path: $OPENRAM_TMP/ coverage: + if: ${{ always() }} needs: [scn4me_subm, freepdk45] runs-on: self-hosted steps: From d354a847e611135b3b1914f5b7e5fbdd29714199 Mon Sep 17 00:00:00 2001 From: mrg Date: Sat, 13 Feb 2021 23:54:16 -0800 Subject: [PATCH 15/18] Remove gitlab badges. --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index dd7055e0..da68361b 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,9 @@ [![License: BSD 3-clause](./images/license_badge.svg)](./LICENSE) Master: -[![Pipeline Status](https://scone.soe.ucsc.edu:8888/mrg/OpenRAM/badges/master/pipeline.svg)](https://github.com/VLSIDA/OpenRAM/commits/master) -![Coverage](https://scone.soe.ucsc.edu:8888/mrg/OpenRAM/badges/master/coverage.svg) [![Download](./images/download-stable-blue.svg)](https://github.com/VLSIDA/OpenRAM/archive/master.zip) Dev: -[![Pipeline Status](https://scone.soe.ucsc.edu:8888/mrg/OpenRAM/badges/dev/pipeline.svg)](https://github.com/VLSIDA/OpenRAM/commits/dev) -![Coverage](https://scone.soe.ucsc.edu:8888/mrg/OpenRAM/badges/dev/coverage.svg) [![Download](./images/download-unstable-blue.svg)](https://github.com/VLSIDA/OpenRAM/archive/dev.zip) An open-source static random access memory (SRAM) compiler. From 33bc9a597c12e9ae26d8e57dda620eae089e9e9c Mon Sep 17 00:00:00 2001 From: mrg Date: Mon, 15 Feb 2021 08:19:08 -0800 Subject: [PATCH 16/18] Remove dashes for Python module name warning. --- ...{riscv-freepdk45-8kbyte.py => riscv_freepdk45_8kbyte.py} | 0 ...m-16kbyte-1rw1r.py => riscv_scn4m_subm_16kbyte_1rw1r.py} | 0 ...ubm-1kbyte-1rw1r.py => riscv_scn4m_subm_1kbyte_1rw1r.py} | 0 ...bm-2kbyte-1rw1r.py => riscv_scn4m_subm_2skbyte_1rw1r.py} | 0 ...4m_subm-32kbyte.py => riscv_scn4m_subm_32kbyte_1rw1r.py} | 0 ...ubm-4kbyte-1rw1r.py => riscv_scn4m_subm_4kbyte_1rw1r.py} | 0 ...ubm-8kbyte-1rw1r.py => riscv_scn4m_subm_8kbyte_1rw1r.py} | 0 ...iscv-sky130-1kbyte-1rw.py => riscv_sky130_1kbyte_1rw.py} | 0 ...-sky130-1kbyte-1rw1r.py => riscv_sky130_1kbyte_1rw1r.py} | 6 +++--- ...iscv-sky130-2kbyte-1rw.py => riscv_sky130_2kbyte_1rw.py} | 0 ...-sky130-2kbyte-1rw1r.py => riscv_sky130_2kbyte_1rw1r.py} | 0 ...iscv-sky130-4kbyte-1rw.py => riscv_sky130_4kbyte_1rw.py} | 0 ...-sky130-4kbyte-1rw1r.py => riscv_sky130_4kbyte_1rw1r.py} | 0 13 files changed, 3 insertions(+), 3 deletions(-) rename compiler/example_configs/{riscv-freepdk45-8kbyte.py => riscv_freepdk45_8kbyte.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-16kbyte-1rw1r.py => riscv_scn4m_subm_16kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-1kbyte-1rw1r.py => riscv_scn4m_subm_1kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-2kbyte-1rw1r.py => riscv_scn4m_subm_2skbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-32kbyte.py => riscv_scn4m_subm_32kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-4kbyte-1rw1r.py => riscv_scn4m_subm_4kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-scn4m_subm-8kbyte-1rw1r.py => riscv_scn4m_subm_8kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-sky130-1kbyte-1rw.py => riscv_sky130_1kbyte_1rw.py} (100%) rename compiler/example_configs/{riscv-sky130-1kbyte-1rw1r.py => riscv_sky130_1kbyte_1rw1r.py} (90%) rename compiler/example_configs/{riscv-sky130-2kbyte-1rw.py => riscv_sky130_2kbyte_1rw.py} (100%) rename compiler/example_configs/{riscv-sky130-2kbyte-1rw1r.py => riscv_sky130_2kbyte_1rw1r.py} (100%) rename compiler/example_configs/{riscv-sky130-4kbyte-1rw.py => riscv_sky130_4kbyte_1rw.py} (100%) rename compiler/example_configs/{riscv-sky130-4kbyte-1rw1r.py => riscv_sky130_4kbyte_1rw1r.py} (100%) diff --git a/compiler/example_configs/riscv-freepdk45-8kbyte.py b/compiler/example_configs/riscv_freepdk45_8kbyte.py similarity index 100% rename from compiler/example_configs/riscv-freepdk45-8kbyte.py rename to compiler/example_configs/riscv_freepdk45_8kbyte.py diff --git a/compiler/example_configs/riscv-scn4m_subm-16kbyte-1rw1r.py b/compiler/example_configs/riscv_scn4m_subm_16kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-16kbyte-1rw1r.py rename to compiler/example_configs/riscv_scn4m_subm_16kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-scn4m_subm-1kbyte-1rw1r.py b/compiler/example_configs/riscv_scn4m_subm_1kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-1kbyte-1rw1r.py rename to compiler/example_configs/riscv_scn4m_subm_1kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-scn4m_subm-2kbyte-1rw1r.py b/compiler/example_configs/riscv_scn4m_subm_2skbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-2kbyte-1rw1r.py rename to compiler/example_configs/riscv_scn4m_subm_2skbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-scn4m_subm-32kbyte.py b/compiler/example_configs/riscv_scn4m_subm_32kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-32kbyte.py rename to compiler/example_configs/riscv_scn4m_subm_32kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-scn4m_subm-4kbyte-1rw1r.py b/compiler/example_configs/riscv_scn4m_subm_4kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-4kbyte-1rw1r.py rename to compiler/example_configs/riscv_scn4m_subm_4kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-scn4m_subm-8kbyte-1rw1r.py b/compiler/example_configs/riscv_scn4m_subm_8kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-scn4m_subm-8kbyte-1rw1r.py rename to compiler/example_configs/riscv_scn4m_subm_8kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-sky130-1kbyte-1rw.py b/compiler/example_configs/riscv_sky130_1kbyte_1rw.py similarity index 100% rename from compiler/example_configs/riscv-sky130-1kbyte-1rw.py rename to compiler/example_configs/riscv_sky130_1kbyte_1rw.py diff --git a/compiler/example_configs/riscv-sky130-1kbyte-1rw1r.py b/compiler/example_configs/riscv_sky130_1kbyte_1rw1r.py similarity index 90% rename from compiler/example_configs/riscv-sky130-1kbyte-1rw1r.py rename to compiler/example_configs/riscv_sky130_1kbyte_1rw1r.py index 20463a99..d0b47857 100644 --- a/compiler/example_configs/riscv-sky130-1kbyte-1rw1r.py +++ b/compiler/example_configs/riscv_sky130_1kbyte_1rw1r.py @@ -2,7 +2,7 @@ word_size = 32 num_words = 256 write_size = 8 -local_array_size = 16 +#local_array_size = 16 num_rw_ports = 1 num_r_ports = 1 @@ -11,9 +11,9 @@ num_w_ports = 0 tech_name = "sky130" nominal_corner_only = True -route_supplies = False +#route_supplies = False check_lvsdrc = True -perimeter_pins = False +#perimeter_pins = False #netlist_only = True #analytical_delay = False output_name = "sram_{0}rw{1}r{2}w_{3}_{4}_{5}".format(num_rw_ports, diff --git a/compiler/example_configs/riscv-sky130-2kbyte-1rw.py b/compiler/example_configs/riscv_sky130_2kbyte_1rw.py similarity index 100% rename from compiler/example_configs/riscv-sky130-2kbyte-1rw.py rename to compiler/example_configs/riscv_sky130_2kbyte_1rw.py diff --git a/compiler/example_configs/riscv-sky130-2kbyte-1rw1r.py b/compiler/example_configs/riscv_sky130_2kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-sky130-2kbyte-1rw1r.py rename to compiler/example_configs/riscv_sky130_2kbyte_1rw1r.py diff --git a/compiler/example_configs/riscv-sky130-4kbyte-1rw.py b/compiler/example_configs/riscv_sky130_4kbyte_1rw.py similarity index 100% rename from compiler/example_configs/riscv-sky130-4kbyte-1rw.py rename to compiler/example_configs/riscv_sky130_4kbyte_1rw.py diff --git a/compiler/example_configs/riscv-sky130-4kbyte-1rw1r.py b/compiler/example_configs/riscv_sky130_4kbyte_1rw1r.py similarity index 100% rename from compiler/example_configs/riscv-sky130-4kbyte-1rw1r.py rename to compiler/example_configs/riscv_sky130_4kbyte_1rw1r.py From f5c86f70a3843dd73ed2b65b27f038cefe61c5b3 Mon Sep 17 00:00:00 2001 From: mrg Date: Mon, 15 Feb 2021 08:19:37 -0800 Subject: [PATCH 17/18] Change to 32 threads --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2df69733..00b23115 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/scn4me_subm" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -t scn4m_subm + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm - name: Archive if: ${{ failure() }} uses: actions/upload-artifact@v2 @@ -30,7 +30,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/freepdk45" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 20 -t freepdk45 + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 - name: Archive if: ${{ failure() }} uses: actions/upload-artifact@v2 From c3156be7b1bdecbb37310634034847ffdfcb6025 Mon Sep 17 00:00:00 2001 From: mrg Date: Mon, 15 Feb 2021 12:02:22 -0800 Subject: [PATCH 18/18] Change from 32 to 48 threads --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00b23115..6a7cd73b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/scn4me_subm" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t scn4m_subm + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 48 -t scn4m_subm - name: Archive if: ${{ failure() }} uses: actions/upload-artifact@v2 @@ -30,7 +30,7 @@ jobs: export OPENRAM_HOME="`pwd`/compiler" export OPENRAM_TECH="`pwd`/technology:/software/PDKs/skywater-tech" export OPENRAM_TMP="`pwd`/freepdk45" - python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 32 -t freepdk45 + python3-coverage run -p $OPENRAM_HOME/tests/regress.py -j 48 -t freepdk45 - name: Archive if: ${{ failure() }} uses: actions/upload-artifact@v2