From 7852634ffa8024401f8f0830625d208635a69def Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 03:07:52 -0400 Subject: [PATCH 01/12] bugfix for setup.py sdist --- MANIFEST.in | 11 +++++++++++ setup.py | 24 ++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..8088e14c0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +recursive-include src/tl/tl *.cc *.h +recursive-include src/db/db *.cc *.h +recursive-include src/gsi/gsi *.cc *.h +recursive-include src/rdb/rdb *.cc *.h +recursive-include src/pya/pya *.cc *.h +recursive-include src/pymod *.cc *.h +include src/plugins/*/db_plugin/*.cc +include src/plugins/*/*/db_plugin/*.cc +include src/plugins/*/db_plugin/*.h +include src/plugins/*/*/db_plugin/*.h +recursive-include src/plugins/common *.h diff --git a/setup.py b/setup.py index f7b4d2c44..d29d9d1a4 100644 --- a/setup.py +++ b/setup.py @@ -191,7 +191,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev1" + return "0.26.0.dev2" config = Config() @@ -201,13 +201,15 @@ config = Config() _tl_path = os.path.join("src", "tl", "tl") -_tl_sources = glob.glob(os.path.join(_tl_path, "*.cc")) +_tl_sources = set(glob.glob(os.path.join(_tl_path, "*.cc"))) # Exclude sources which are compatible with Qt only -_tl_sources.remove(os.path.join(_tl_path, "tlHttpStreamQt.cc")) -_tl_sources.remove(os.path.join(_tl_path, "tlHttpStreamNoQt.cc")) -_tl_sources.remove(os.path.join(_tl_path, "tlFileSystemWatcher.cc")) -_tl_sources.remove(os.path.join(_tl_path, "tlDeferredExecutionQt.cc")) +# Caveat, in source distribution tarballs from pypi, these files will +# not exist. So we need an error-free discard method instead of list's remove. +_tl_sources.discard(os.path.join(_tl_path, "tlHttpStreamQt.cc")) +_tl_sources.discard(os.path.join(_tl_path, "tlHttpStreamNoQt.cc")) +_tl_sources.discard(os.path.join(_tl_path, "tlFileSystemWatcher.cc")) +_tl_sources.discard(os.path.join(_tl_path, "tlDeferredExecutionQt.cc")) _tl = Extension(config.root + '._tl', define_macros=config.macros() + [('MAKE_TL_LIBRARY', 1)], @@ -215,7 +217,7 @@ _tl = Extension(config.root + '._tl', libraries=['curl', 'expat'], extra_link_args=config.link_args('_tl'), extra_compile_args=config.compile_args('_tl'), - sources=_tl_sources) + sources=list(_tl_sources)) # ------------------------------------------------------------------ # _gsi dependency library @@ -249,10 +251,12 @@ _pya = Extension(config.root + '._pya', # _db dependency library _db_path = os.path.join("src", "db", "db") -_db_sources = glob.glob(os.path.join(_db_path, "*.cc")) +_db_sources = set(glob.glob(os.path.join(_db_path, "*.cc"))) # Not a real source: -_db_sources.remove(os.path.join(_db_path, "fonts.cc")) +# Caveat, in source distribution tarballs from pypi, these files will +# not exist. So we need an error-free discard method instead of list's remove. +_db_sources.discard(os.path.join(_db_path, "fonts.cc")) _db = Extension(config.root + '._db', define_macros=config.macros() + [('MAKE_DB_LIBRARY', 1)], @@ -261,7 +265,7 @@ _db = Extension(config.root + '._db', language='c++', extra_link_args=config.link_args('_db'), extra_compile_args=config.compile_args('_db'), - sources=_db_sources) + sources=list(_db_sources)) # ------------------------------------------------------------------ # _rdb dependency library From e7ebd88b2cdd0b3bf42888bcce46569e519aeee6 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 9 Oct 2018 00:35:28 -0400 Subject: [PATCH 02/12] critical bugfix for pymod (db plugins failed to import) Cause: dbInit.cc:133 hardcodes the macos extension for shared libraries ('.dylib'), but setuptools builds it with '.so'. Current solution: Since the main build's qmake respects the dylib standard, I've adapted setuptools to use '.dylib'. --- .travis.yml | 1 + setup.py | 28 +++++++++++++++++++++++++--- test-klayout-db-plugins.py | 13 +++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 test-klayout-db-plugins.py diff --git a/.travis.yml b/.travis.yml index 691fa5f30..a70551b6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -232,6 +232,7 @@ script: python setup.py build; python setup.py bdist_wheel; python setup.py install; + python test-klayout-db-plugins.py; mkdir -p deploy/dist-pymod; cp -a dist/* deploy/dist-pymod/; python -c 'import klayout.db as db; print(dir(db))'; diff --git a/setup.py b/setup.py index d29d9d1a4..a1f2c56e8 100644 --- a/setup.py +++ b/setup.py @@ -60,10 +60,10 @@ import os import platform import distutils.sysconfig as sysconfig from distutils.errors import CompileError +import distutils.command.build_ext import multiprocessing N_cores = multiprocessing.cpu_count() - # monkey-patch for parallel compilation # from https://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): @@ -103,6 +103,24 @@ if sys.version_info[0] * 10 + sys.version_info[1] > 26: import distutils.ccompiler distutils.ccompiler.CCompiler.compile = parallelCCompile + +from distutils.command.build_ext import build_ext +_old_get_ext_filename = build_ext.get_ext_filename + + +def patched_get_ext_filename(self, ext_name): + r"""Convert the name of an extension (eg. "foo.bar") into the name + of the file from which it will be loaded (eg. "foo/bar.so", or + "foo\bar.pyd"). + """ + filename = _old_get_ext_filename(self, ext_name) + # Making sure this matches qmake's default extension .dylib, instead of .so + if platform.system() == "Darwin" and '_dbpi' in ext_name: + filename = filename.replace('.so', '.dylib') + return filename + + +distutils.command.build_ext.build_ext.get_ext_filename = patched_get_ext_filename # ---------------------------------------------------------------------------------------- @@ -121,6 +139,7 @@ class Config(object): self.build_platlib = build_cmd.build_platlib self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + if self.ext_suffix is None: self.ext_suffix = ".so" @@ -131,7 +150,10 @@ class Config(object): Returns the library name for a given module The library name is usually decorated (i.e. "tl" -> "tl.cpython-35m-x86_64-linux-gnu.so"). """ - return mod + self.ext_suffix + ext_suffix = self.ext_suffix + if platform.system() == "Darwin" and '_dbpi' in mod: + ext_suffix = ext_suffix.replace('.so', '.dylib') + return mod + ext_suffix def path_of(self, mod): """ @@ -191,7 +213,7 @@ class Config(object): """ Gets the version string """ - return "0.26.0.dev2" + return "0.26.0.dev4" config = Config() diff --git a/test-klayout-db-plugins.py b/test-klayout-db-plugins.py new file mode 100644 index 000000000..ab05fdea0 --- /dev/null +++ b/test-klayout-db-plugins.py @@ -0,0 +1,13 @@ +import klayout.db as kdb + +layout = kdb.Layout() + +layer1 = layout.layer(kdb.LayerInfo('1/0')) + +TOP = layout.create_cell("TOP") + +box = kdb.DBox(kdb.DPoint(-4500, -4500), kdb.DPoint(4500, 4500)) + +TOP.shapes(layer1).insert(box) + +layout.write('test.gds') From bda1304500a1ff6dbd6cef8fe811a8b236b700c7 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Sun, 7 Oct 2018 22:56:44 -0400 Subject: [PATCH 03/12] fixed macos10.11 version with py3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 691fa5f30..4b21af936 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ matrix: os: osx osx_image: xcode8 # macOS 10.11 env: - - MATRIX_EVAL="brew update; brew config; brew upgrade python;" + - MATRIX_EVAL="brew update; brew config; brew upgrade python; brew postinstall python; ls -l /usr/local/opt/python/libexec/bin/; shopt -s expand_aliases; alias python='/usr/local/opt/python/libexec/bin/python'; alias pip='/usr/local/opt/python/libexec/bin/pip';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true From 5dcc01f0f3b19b12d700c828165073621ad60bb8 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 01:12:51 -0400 Subject: [PATCH 04/12] adding linux pymod py3.7 with clang instead of gcc --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4b21af936..5c2d9ab95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,6 +74,19 @@ matrix: - PYTHON_BUILD=true - BREW_BUNDLE=false + - name: "klayout python3.7 package" + os: linux + dist: trusty # Ubuntu 14.04 + sudo: false + language: python + python: '3.7-dev' + env: + - MATRIX_EVAL="" + - PIP_UPDATE="1" + - PYTHON_BUILD=true + - BREW_BUNDLE=false + - CC=clang + - name: "klayout python3.6 package" os: linux dist: trusty # Ubuntu 14.04 From a2e357cf9cfca1ed51c87b3e8896c1c9e5241696 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 01:20:17 -0400 Subject: [PATCH 05/12] testing new compile flags --- .travis.yml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5c2d9ab95..d0cb08029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,6 +86,7 @@ matrix: - PYTHON_BUILD=true - BREW_BUNDLE=false - CC=clang + - CXX=clang++ - name: "klayout python3.6 package" os: linux diff --git a/setup.py b/setup.py index f7b4d2c44..211d6a257 100644 --- a/setup.py +++ b/setup.py @@ -149,7 +149,7 @@ class Config(object): return [] else: return ["-Wno-strict-aliasing", # Avoids many "type-punned pointer" warnings - "-std=c++0x", # because we use unordered_map/unordered_set + "-std=c++11", # because we use unordered_map/unordered_set ] def link_args(self, mod): From 6d69f22ece641b07481d4888b9618e63ec2479c4 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 02:03:46 -0400 Subject: [PATCH 06/12] building and linking in linux with clang and clang++, respectively (smaller binaries) --- .travis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.travis.yml b/.travis.yml index d0cb08029..26aefaebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -99,6 +99,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ - name: "klayout python2.7 package" os: linux @@ -111,6 +113,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ - name: "klayout python2.6 package" os: linux @@ -123,6 +127,8 @@ matrix: - PIP_UPDATE="0" # setuptools installed from last pip has syntax error on py 2.6 - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ - name: "klayout python3.3 package" os: linux @@ -135,6 +141,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ - name: "klayout python3.4 package" os: linux @@ -147,6 +155,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ - name: "klayout python3.5 package" os: linux @@ -159,6 +169,8 @@ matrix: - PIP_UPDATE="1" - PYTHON_BUILD=true - BREW_BUNDLE=false + - CC=clang + - CXX=clang++ # KLayout builds for mac # Python 3 From 69bc566e0f91b0b9b948dc78d2b064f0f651cc5e Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 01:54:41 -0400 Subject: [PATCH 07/12] adding python 3.5 and 3.4 for osx 10.13 --- .travis.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26aefaebe..956702401 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,11 +44,31 @@ matrix: - PYTHON_BUILD=true - BREW_BUNDLE=true - - name: "klayout python3.6.5_1 osx10.13" + - name: "klayout python3.6.6 osx10.13" os: osx osx_image: xcode9.4 # macOS 10.13 env: - - MATRIX_EVAL="brew update; brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f2a764ef944b1080be64bd88dca9a1d80130c558/Formula/python.rb; brew switch python 3.6.5_1; shopt -s expand_aliases; alias python='python3'; alias pip='pip3';" + - MATRIX_EVAL="brew update; brew install sashkab/python/python36; brew link --force --overwrite python36; shopt -s expand_aliases; alias python='/usr/local/opt/python36/bin/python3.6'; alias pip='/usr/local/opt/python36/bin/pip3.6';" + - ARCHFLAGS="-std=c++11" + - PIP_UPDATE="1" + - PYTHON_BUILD=true + - BREW_BUNDLE=false + + - name: "klayout python3.5.6 osx10.13" + os: osx + osx_image: xcode9.4 # macOS 10.13 + env: + - MATRIX_EVAL="brew update; brew install sashkab/python/python35; brew link --force --overwrite python35; shopt -s expand_aliases; alias python='/usr/local/opt/python35/bin/python3.5'; alias pip='/usr/local/opt/python35/bin/pip3.5';" + - ARCHFLAGS="-std=c++11" + - PIP_UPDATE="1" + - PYTHON_BUILD=true + - BREW_BUNDLE=false + + - name: "klayout python3.4.9 osx10.13" + os: osx + osx_image: xcode9.4 # macOS 10.13 + env: + - MATRIX_EVAL="brew update; brew install sashkab/python/python34; brew link --force --overwrite python34; shopt -s expand_aliases; alias python='/usr/local/opt/python34/bin/python3.4'; alias pip='/usr/local/opt/python34/bin/pip3.4';" - ARCHFLAGS="-std=c++11" - PIP_UPDATE="1" - PYTHON_BUILD=true From f007d2d75882c9491e43339e7963508c6a811bc6 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 8 Oct 2018 11:27:06 -0400 Subject: [PATCH 08/12] latest version of ruby not necessary for dropbox-deployment --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 956702401..0898c58d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -257,7 +257,6 @@ matrix: before_install: - env - - rvm install ruby --latest - gem install dropbox-deployment - eval "${MATRIX_EVAL}" - if [ "$BREW_BUNDLE" = true ]; then From c78c2a02286ac60c2114e1cba00f736d00708ef7 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 9 Oct 2018 18:00:38 -0400 Subject: [PATCH 09/12] unit-testing pymod with tl, rdb, and db (+plugins) tests --- .travis.yml | 2 +- test-klayout-db-plugins.py | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 test-klayout-db-plugins.py diff --git a/.travis.yml b/.travis.yml index a70551b6f..036ea13b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -232,7 +232,7 @@ script: python setup.py build; python setup.py bdist_wheel; python setup.py install; - python test-klayout-db-plugins.py; + python -m unittest testdata/pymod/import_db.py testdata/pymod/import_rdb.py testdata/pymod/import_tl.py; mkdir -p deploy/dist-pymod; cp -a dist/* deploy/dist-pymod/; python -c 'import klayout.db as db; print(dir(db))'; diff --git a/test-klayout-db-plugins.py b/test-klayout-db-plugins.py deleted file mode 100644 index ab05fdea0..000000000 --- a/test-klayout-db-plugins.py +++ /dev/null @@ -1,13 +0,0 @@ -import klayout.db as kdb - -layout = kdb.Layout() - -layer1 = layout.layer(kdb.LayerInfo('1/0')) - -TOP = layout.create_cell("TOP") - -box = kdb.DBox(kdb.DPoint(-4500, -4500), kdb.DPoint(4500, 4500)) - -TOP.shapes(layer1).insert(box) - -layout.write('test.gds') From dd9d46da3844feade0a7bea89494a04a3c849925 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 14 Oct 2018 23:34:25 +0200 Subject: [PATCH 10/12] Activated stream unit tests. --- src/tl/unit_tests/unit_tests.pro | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index 59776da0c..89d63ec6c 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -30,6 +30,7 @@ SOURCES = \ tlVariant.cc \ tlXMLParser.cc \ tlUri.cc \ + tlStreamTests.cc \ tlWebDAV.cc \ tlHttpStream.cc \ tlInt128Support.cc \ From 90c03140b3e3198fbe3166f6191977c2ffd6e9cd Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 14 Oct 2018 23:42:10 +0200 Subject: [PATCH 11/12] Fixed #185 for pymod branch --- src/pya/pya/pyaModule.cc | 6 ++++++ testdata/python/dbTransTest.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/src/pya/pya/pyaModule.cc b/src/pya/pya/pyaModule.cc index 9670a0736..ecfb35c3c 100644 --- a/src/pya/pya/pyaModule.cc +++ b/src/pya/pya/pyaModule.cc @@ -2687,6 +2687,12 @@ PythonModule::make_classes (const char *mod_name) add_python_doc (*c, mt, int (mid), tl::to_string (tr ("This method is also available as 'str(object)'"))); } + } else if (name == "hash" && m_first->compatible_with_num_args (0)) { + + // The hash method is also routed via the tp_hash implementation + alt_names.push_back ("__hash__"); + add_python_doc (*c, mt, int (mid), tl::to_string (tr ("This method is also available as 'hash(object)'"))); + } else if (name == "inspect" && m_first->compatible_with_num_args (0)) { // The str method is also routed via the tp_str implementation diff --git a/testdata/python/dbTransTest.py b/testdata/python/dbTransTest.py index 8f7643f45..b3423b01e 100644 --- a/testdata/python/dbTransTest.py +++ b/testdata/python/dbTransTest.py @@ -530,6 +530,14 @@ class DBTransTests(unittest.TestCase): self.assertEqual(t1.hash() == t3.hash(), False) self.assertEqual(t1.hash() == t4a.hash(), False) self.assertEqual(t1.hash() == t4b.hash(), False) + self.assertEqual(hash(t1) == hash(t2), True) + self.assertEqual(hash(t1) == hash(t3), False) + self.assertEqual(hash(t1) == hash(t4a), False) + self.assertEqual(hash(t1) == hash(t4b), False) + self.assertEqual(t1.__hash__() == t2.__hash__(), True) + self.assertEqual(t1.__hash__() == t3.__hash__(), False) + self.assertEqual(t1.__hash__() == t4a.__hash__(), False) + self.assertEqual(t1.__hash__() == t4b.__hash__(), False) # Transformations can't be used as hash keys currently if False: From 69c508827863406b6995dd3aecd0243f306455ed Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 14 Oct 2018 23:52:08 +0200 Subject: [PATCH 12/12] Updated Changelog (merge from master) --- Changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog b/Changelog index d7f3d501b..1f0dbe239 100644 --- a/Changelog +++ b/Changelog @@ -17,6 +17,9 @@ Performance issue with many layers with width >1 * Bugfix: https://github.com/klayoutmatthias/klayout/issues/175 Painting issue with texts +* Bugfix: https://github.com/klayoutmatthias/klayout/issues/185 + Hash values available as __hash__ standard method now + for Python * Bugfix: some potential memory corruption issues fixed During the efforts for making the code base compatible with MSVC, some potential candidates for memory corruption