From ca3d840cb2e73f8561e68e8c19e8958ff20dc9ca Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 28 May 2022 00:33:08 +0200 Subject: [PATCH] Alternative implementation for resources for the non-Qt case. --- scripts/pyqrc.py | 154 ++++++++++++++++++++++++++ src/drc/drc/drc.pro | 2 +- src/klayout.pri | 12 ++ src/tl/tl/tl.pro | 2 + src/tl/tl/tlResources.cc | 127 +++++++++++++++++++++ src/tl/tl/tlResources.h | 74 +++++++++++++ src/tl/tl/tlStream.cc | 75 ++++++++++--- src/tl/unit_tests/tlDeflateTests.cc | 9 +- src/tl/unit_tests/tlResourcesTests.cc | 91 +++++++++++++++ src/tl/unit_tests/unit_tests.pro | 1 + 10 files changed, 524 insertions(+), 23 deletions(-) create mode 100755 scripts/pyqrc.py create mode 100644 src/tl/tl/tlResources.cc create mode 100644 src/tl/tl/tlResources.h create mode 100644 src/tl/unit_tests/tlResourcesTests.cc diff --git a/scripts/pyqrc.py b/scripts/pyqrc.py new file mode 100755 index 000000000..f0cbe3f9f --- /dev/null +++ b/scripts/pyqrc.py @@ -0,0 +1,154 @@ + +# A lean substitute for QRC which is employed in the non-Qt case + +import xml.etree.ElementTree as et +import argparse +import zlib +import os + + +# A class providing the generator + +class RCFile(object): + + def __init__(self, alias, path): + self.path = path + self.alias = alias + + def write(self, file, index): + + f = open(self.path, "rb") + raw_data = f.read() + f.close() + + data = zlib.compress(raw_data) + compressed = "true" + + cb = "{" + bc = "}" + cls = "Resource" + str(index) + file.write(f"\n// Resource file {self.path} as {self.alias}\n") + file.write( "namespace {\n") + file.write( "\n") + file.write(f" class {cls}\n") + file.write( " {\n") + file.write(f" {cls}() {cb}\n") + file.write(f" static bool compressed = {compressed};\n") + file.write(f" static const char *name = \"{self.alias}\";\n") + file.write( " static const unsigned char data[] = {") + n = 0 + for b in data: + if n == 0: + file.write("\n ") + file.write("0x%02x," % b) + n += 1 + if n == 16: + n = 0 + file.write( "\n") + file.write( " };\n") + file.write( " m_id = tl::register_resource(name, compressed, data, sizeof(data) / sizeof(data[0]));\n") + file.write( " }\n") + file.write(f" ~{cls}() {cb}\n") + file.write( " tl::unregister_resource(m_id)\n") + file.write( " }\n") + file.write( " tl::resource_id_type m_id;\n") + file.write(f" {bc} resource_instance{index}\n") + file.write( "\n") + file.write( "}\n") + + +class RCGenerator(object): + + def __init__(self): + self.files = [] + + def append(self, path, alias): + self.files.append(RCFile(path, alias)) + + def dump_files(self): + for f in self.files: + print(f.path) + + def write(self, file): + file.write(f""" +/* + + KLayout Layout Viewer + Copyright (C) 2006-2022 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +/** +* DO NOT EDIT THIS FILE. +* This file has been created automatically +*/ + +#include "tlResources.h" +""") + i = 1 + for f in self.files: + f.write(file, i) + i += 1 + + +# The main code + +generator = RCGenerator() + +# argument parsing + +parser = argparse.ArgumentParser(description='Lean QRC parser') +parser.add_argument('input', type=str, nargs='+', + help='The QRC input file') +parser.add_argument('--output', '-o', type=str, nargs='?', + help='The C++ output file') +parser.add_argument('--path', '-p', type=str, nargs='?', + help='Path to the input files (default is current directory)') + +args = parser.parse_args() + +# read the input file + +for input in args.input: + + root_node = et.parse(input).getroot() + + for qresource in root_node.findall('qresource'): + prefix = qresource.get('prefix') + for file in qresource.findall('file'): + alias = file.get('alias') + path = file.text + if alias is None: + alias = path + if prefix is not None: + alias = prefix + "/" + alias + if args.path is not None: + path = os.path.join(args.path, path) + else: + path = os.path.join(os.path.dirname(input), path) + generator.append(alias, path) + +# produce the output file + +if args.output is not None: + f = open(args.output, "w") + generator.write(f) + f.close() +else: + generator.dump_files() + + diff --git a/src/drc/drc/drc.pro b/src/drc/drc/drc.pro index 2304dc132..c43e875d0 100644 --- a/src/drc/drc/drc.pro +++ b/src/drc/drc/drc.pro @@ -13,7 +13,7 @@ HEADERS = \ drcCommon.h \ drcForceLink.h \ -!equals(HAVE_QT, "0") { +!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") { RESOURCES = \ drcResources.qrc } diff --git a/src/klayout.pri b/src/klayout.pri index 9bf6debac..ad0504dd2 100644 --- a/src/klayout.pri +++ b/src/klayout.pri @@ -182,6 +182,17 @@ equals(HAVE_QT, "0") { QT = + # fake qrc made with python + !equals(HAVE_PYTHON, "0") { + new_qrc.output = qrc_${QMAKE_FILE_BASE}.cc + new_qrc.commands = $$PYTHON $$PWD/../scripts/pyqrc.py ${QMAKE_FILE_NAME} -o ${QMAKE_FILE_OUT} + new_qrc.depend_command = $$PYTHON $$PWD/../scripts/pyqrc.py ${QMAKE_FILE_NAME} + new_qrc.input = RESOURCES + new_qrc.variable_out = SOURCES + new_qrc.CONFIG += dep_lines + QMAKE_EXTRA_COMPILERS += new_qrc + } + } else { DEFINES += HAVE_QT @@ -290,3 +301,4 @@ DEFINES += \ KLAYOUT_TINY_VERSION=$$KLAYOUT_TINY_VERSION \ VERSION = $$KLAYOUT_VERSION + diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 0b1d91ead..2fc72a1fe 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -26,6 +26,7 @@ SOURCES = \ tlLog.cc \ tlObject.cc \ tlProgress.cc \ + tlResources.cc \ tlScriptError.cc \ tlSleep.cc \ tlStaticObjects.cc \ @@ -76,6 +77,7 @@ HEADERS = \ tlObject.h \ tlObjectCollection.h \ tlProgress.h \ + tlResources.h \ tlReuseVector.h \ tlScriptError.h \ tlSleep.h \ diff --git a/src/tl/tl/tlResources.cc b/src/tl/tl/tlResources.cc new file mode 100644 index 000000000..b09febe0d --- /dev/null +++ b/src/tl/tl/tlResources.cc @@ -0,0 +1,127 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2022 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlResources.h" + +#include +#include + +namespace tl +{ + +namespace { + +class ResourceDict +{ +public: + struct DictEntry + { + const unsigned char *data; + size_t data_size; + bool compressed; + }; + + ResourceDict () { } + + resource_id_type add (const char *name, bool compressed, const unsigned char *data, size_t data_size) + { + m_dict[std::string (name)] = m_entries.size (); + m_entries.push_back (DictEntry ()); + m_entries.back ().data = data; + m_entries.back ().data_size = data_size; + m_entries.back ().compressed = compressed; + return m_entries.size () - 1; + } + + void remove (resource_id_type id) + { + if (id < m_entries.size ()) { + m_entries [id].data = 0; + m_entries [id].data_size = 0; + } + } + + DictEntry *entry (const char *name) + { + auto i = m_dict.find (std::string (name)); + if (i != m_dict.end () && i->second < m_entries.size ()) { + return &m_entries [i->second]; + } else { + return 0; + } + } + +private: + std::map m_dict; + std::vector m_entries; +}; + +} + +static ResourceDict *ms_dict = 0; + +resource_id_type register_resource (const char *name, bool compressed, const unsigned char *data, size_t data_size) +{ + if (! ms_dict) { + ms_dict = new ResourceDict (); + } + return ms_dict->add (name, compressed, data, data_size); +} + +void unregister_resource (size_t id) +{ + if (ms_dict) { + ms_dict->remove (id); + } +} + +tl::InputStream *get_resource (const char *name) +{ + if (! ms_dict) { + return 0; + } + + ResourceDict::DictEntry *entry = ms_dict->entry (name); + if (! entry || ! entry->data) { + return 0; + } + + if (entry->compressed) { + + tl_assert (entry->data_size > 6); + + // NOTE: zlib compression (used in pyqrc) adds two bytes header before the data block and + // 4 bytes after (CRC32) + auto stream = new tl::InputStream (new tl::InputMemoryStream ((const char *) entry->data + 2, entry->data_size - 6)); + stream->inflate (); + return stream; + + } else { + + // raw data + return new tl::InputStream (new tl::InputMemoryStream ((const char *) entry->data, entry->data_size)); + + } +} + +} + diff --git a/src/tl/tl/tlResources.h b/src/tl/tl/tlResources.h new file mode 100644 index 000000000..3fbdaef0d --- /dev/null +++ b/src/tl/tl/tlResources.h @@ -0,0 +1,74 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2022 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlResources +#define HDR_tlResources + +#include "tlAssert.h" +#include "tlStream.h" + +#include + +namespace tl +{ + +typedef size_t resource_id_type; + +/** + * @brief A facility for retrieving resource data similar to Qt resources + * + * This feature is intended to substitute Qt resources when Qt is not available. + */ + +/** + * @brief Registers the resource data under the given name + * + * @param name The name of the resource + * @param compressed True, if the data is gzip-compressed + * @param data The data pointer - needs to stay valid during the lifetime of the application + * @param data_size The size of the data block + * + * The return value is an Id by which the resource can be unregistered. + */ +TL_PUBLIC resource_id_type register_resource (const char *name, bool compressed, const unsigned char *data, size_t data_size); + +/** + * @brief Registers the resource data under the given name + * + * @param id The id of the resource to unregister (see "register_resource") + */ +TL_PUBLIC void unregister_resource (size_t id); + +/** + * @brief Gets the resource data as a stream + * + * @param name The resource name + * @return A tl::InputStream object delivering the data or 0 if there is no such resource + * It is the responsibility of the called to delete the input stream. + */ +TL_PUBLIC tl::InputStream *get_resource (const char *name); + +} + +#endif + diff --git a/src/tl/tl/tlStream.cc b/src/tl/tl/tlStream.cc index 06b45849f..f8984e094 100644 --- a/src/tl/tl/tlStream.cc +++ b/src/tl/tl/tlStream.cc @@ -285,7 +285,7 @@ InputStream::get (size_t n, bool bypass_inflate) delete mp_inflate; mp_inflate = 0; } - } + } if (m_blen < n) { @@ -341,16 +341,36 @@ std::string InputStream::read_all (size_t max_count) { std::string str; - while (max_count > 0) { - size_t n = std::min (max_count, std::max (size_t (1), m_blen)); - const char *b = get (n); - if (b) { - str += std::string (b, n); - max_count -= n; - } else { - break; + + if (mp_inflate) { + + // Inflate is special - it does not have a guaranteed byte delivery, so we have to go the + // hard way and pick the file byte by byte + while (max_count > 0) { + const char *b = get (1); + if (b) { + str += *b; + --max_count; + } else { + break; + } } - } + + } else { + + while (max_count > 0) { + size_t n = std::min (max_count, std::max (size_t (1), m_blen)); + const char *b = get (n); + if (b) { + str += std::string (b, n); + max_count -= n; + } else { + break; + } + } + + } + return str; } @@ -358,15 +378,34 @@ std::string InputStream::read_all () { std::string str; - while (true) { - size_t n = std::max (size_t (1), m_blen); - const char *b = get (n); - if (b) { - str += std::string (b, n); - } else { - break; + + if (mp_inflate) { + + // Inflate is special - it does not have a guaranteed byte delivery, so we have to go the + // hard way and pick the file byte by byte + while (true) { + const char *b = get (1); + if (b) { + str += *b; + } else { + break; + } } - } + + } else { + + while (true) { + size_t n = std::max (size_t (1), m_blen); + const char *b = get (n); + if (b) { + str += std::string (b, n); + } else { + break; + } + } + + } + return str; } diff --git a/src/tl/unit_tests/tlDeflateTests.cc b/src/tl/unit_tests/tlDeflateTests.cc index f7be73257..948f90ce7 100644 --- a/src/tl/unit_tests/tlDeflateTests.cc +++ b/src/tl/unit_tests/tlDeflateTests.cc @@ -30,7 +30,7 @@ TEST(1) { unsigned char data[] = { - // gzip header + // gzip header: // 0x1f, 0x8b, 0x08, 0x08, // 0xed, 0x11, 0x07, 0x50, // 0x00, 0x03, @@ -39,9 +39,10 @@ TEST(1) 0x56, 0x00, 0xa2, 0x44, 0x85, 0x92, 0xd4, 0xe2, 0x12, 0x85, 0x18, 0x45, - 0x2e, 0x00, 0x20, 0xc7, - 0x43, 0x6a, 0x12, 0x00, - 0x00, 0x00 + 0x2e, 0x00, + // gzip tail (8 bytes): + // 0x20, 0xc7, 0x43, 0x6a, CRC32 + // 0x12, 0x00, 0x00, 0x00 uncompressed file size }; tl::InputMemoryStream ims ((const char *) data, sizeof (data)); diff --git a/src/tl/unit_tests/tlResourcesTests.cc b/src/tl/unit_tests/tlResourcesTests.cc new file mode 100644 index 000000000..044daa017 --- /dev/null +++ b/src/tl/unit_tests/tlResourcesTests.cc @@ -0,0 +1,91 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2022 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlUnitTest.h" +#include "tlResources.h" + +#include + +// uncompressed resources + +TEST(1) +{ + unsigned char hw[] = "hello, world!\n"; + + const char *name; + std::unique_ptr s; + + name = "__test_resource1"; + tl::resource_id_type id = tl::register_resource (name, false, hw, sizeof (hw)); + + s.reset (tl::get_resource ("__doesnotexist")); + EXPECT_EQ (s.get () == 0, true); + + s.reset (tl::get_resource (name)); + EXPECT_EQ (s.get () == 0, false); + if (s) { + std::string data = s->read_all (); + EXPECT_EQ (data.size (), strlen ((const char *) hw) + 1); + EXPECT_EQ (data, std::string ((const char *) hw, sizeof (hw))); + } + + tl::unregister_resource (id); + s.reset (tl::get_resource (name)); + EXPECT_EQ (s.get () == 0, true); +} + +// compressed resources + +TEST(2) +{ + const unsigned char hw[] = { + 0x78,0x9c, // zlib header + // data: + 0xcb,0x48,0xcd,0xc9,0xc9,0xd7,0x51,0x28,0xcf,0x2f,0xca,0x49,0x51,0xe4, + 0x02,0x00, + 0x26,0xb2,0x04,0xb4, // zlib CRC + }; + unsigned char hw_decoded[] = "hello, world!\n"; + + const char *name; + std::unique_ptr s; + + name = "__test_resource2"; + tl::resource_id_type id = tl::register_resource (name, true, hw, sizeof (hw)); + + s.reset (tl::get_resource ("__doesnotexist")); + EXPECT_EQ (s.get () == 0, true); + + s.reset (tl::get_resource (name)); + EXPECT_EQ (s.get () == 0, false); + if (s) { + std::string data = s->read_all (); + EXPECT_EQ (data.size (), strlen ((const char *) hw_decoded)); + EXPECT_EQ (data, std::string ((const char *) hw_decoded, sizeof (hw_decoded) - 1)); + } + + tl::unregister_resource (id); + s.reset (tl::get_resource (name)); + EXPECT_EQ (s.get () == 0, true); +} + diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index 768faad17..05fe5f202 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -27,6 +27,7 @@ SOURCES = \ tlLongIntTests.cc \ tlMathTests.cc \ tlObjectTests.cc \ + tlResourcesTests.cc \ tlReuseVectorTests.cc \ tlStableVectorTests.cc \ tlStreamTests.cc \