From c8bc64a90e2d7c89b708a3154e9cf9c9333c8669 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 11:41:09 +0200 Subject: [PATCH] WIP --- src/tl/tl/tl.pro | 2 + src/tl/tl/tlXMLParser.cc | 1206 +++++-------------- src/tl/tl/tlXMLParser.h | 1263 ++++++++++++-------- src/tl/tl/tlXMLReader.cc | 913 ++++++++++++++ src/tl/tl/tlXMLReader.h | 215 ++++ src/tl/unit_tests/tlProtocolBufferTests.cc | 37 +- 6 files changed, 2266 insertions(+), 1370 deletions(-) create mode 100644 src/tl/tl/tlXMLReader.cc create mode 100644 src/tl/tl/tlXMLReader.h diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 0773588d1..98c1a41b9 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -45,6 +45,7 @@ SOURCES = \ tlUnitTest.cc \ tlInt128Support.cc \ tlXMLParser.cc \ + tlXMLReader.cc \ tlXMLWriter.cc \ tlThreadedWorkers.cc \ tlThreads.cc \ @@ -113,6 +114,7 @@ HEADERS = \ tlInt128Support.h \ tlDefs.h \ tlXMLParser.h \ + tlXMLReader.h \ tlXMLWriter.h \ tlThreadedWorkers.h \ tlThreads.h \ diff --git a/src/tl/tl/tlXMLParser.cc b/src/tl/tl/tlXMLParser.cc index 55174fc75..01959e579 100644 --- a/src/tl/tl/tlXMLParser.cc +++ b/src/tl/tl/tlXMLParser.cc @@ -26,891 +26,11 @@ #include "tlLog.h" #include "tlAssert.h" #include "tlProgress.h" +#include "tlProtocolBuffer.h" #include #include -#if defined(HAVE_EXPAT) - -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData -{ -public: - XMLSourcePrivateData (tl::InputStream *stream) - : mp_stream_holder (stream), - m_has_error (false) - { - mp_stream = stream; - } - - XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) - : mp_stream_holder (stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - XMLSourcePrivateData (tl::InputStream &stream) - : m_has_error (false) - { - mp_stream = &stream; - } - - XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) - : mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = &stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - int read (char *data, size_t n) - { - try { - - if (mp_progress.get ()) { - mp_progress->set (mp_stream->pos ()); - } - - size_t n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { - *data++ = *rd; - } - - return int (n0 - n); - - } catch (tl::Exception &ex) { - m_error = ex.msg (); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - - const std::string &error_msg () const - { - return m_error; - } - - void reset () - { - mp_stream->reset (); - } - -private: - std::unique_ptr mp_stream_holder; - tl::InputStream *mp_stream; - std::unique_ptr mp_progress; - bool m_has_error; - std::string m_error; -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) - : m_copy (string) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (s, progress_message)); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLSourcePrivateData (s)); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); -void XMLCALL end_element_handler (void *user_data, const XML_Char *name); -void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); - -static std::string get_lname (const std::string &name) -{ - size_t colon = name.find (':'); - if (colon != std::string::npos) { - return std::string (name, colon + 1, name.size () - colon - 1); - } else { - return name; - } -} - -class XMLParserPrivateData -{ -public: - XMLParserPrivateData () - : mp_struct_handler (0) - { - mp_parser = XML_ParserCreate ("UTF-8"); - tl_assert (mp_parser != NULL); - } - - ~XMLParserPrivateData () - { - if (mp_parser != NULL) { - XML_ParserFree (mp_parser); - } - } - - void start_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->start_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void end_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->end_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void cdata (const std::string &cdata) - { - try { - mp_struct_handler->characters (cdata); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) - { - m_has_error = false; - mp_struct_handler = &struct_handler; - - // Just in case we want to reuse it ... - XML_ParserReset (mp_parser, NULL); - XML_SetUserData (mp_parser, (void *) this); - XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); - XML_SetCharacterDataHandler (mp_parser, cdata_handler); - - const int chunk = 65536; - char buffer [chunk]; - - int n; - - do { - - n = source.source ()->read (buffer, chunk); - if (n < 0) { - break; - } - - XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); - if (status == XML_STATUS_ERROR) { - m_has_error = true; - m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); - m_error_line = XML_GetErrorLineNumber (mp_parser); - m_error_column = XML_GetErrorColumnNumber (mp_parser); - } - - } while (n == chunk && !m_has_error); - } - - void check_error () - { - if (m_has_error) { - throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); - } - } - -private: - void error (tl::Exception &ex) - { - m_has_error = true; - m_error_line = XML_GetCurrentLineNumber (mp_parser); - m_error_column = XML_GetCurrentColumnNumber (mp_parser); - m_error = ex.msg (); - } - - XML_Parser mp_parser; - XMLStructureHandler *mp_struct_handler; - bool m_has_error; - std::string m_error; - int m_error_line, m_error_column; -}; - -void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->start_element (std::string (name)); -} - -void end_element_handler (void *user_data, const XML_Char *name) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->end_element (std::string (name)); -} - -void cdata_handler (void *user_data, const XML_Char *s, int len) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->cdata (std::string (s, size_t (len))); -} - - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - mp_data->parse (source, struct_handler); - - // throws an exception if there is an error - mp_data->check_error (); -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#elif defined(HAVE_QT) - -#include -#include -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// A SAX handler for the Qt implementation - -class SAXHandler - : public QXmlDefaultHandler -{ -public: - SAXHandler (XMLStructureHandler *sh); - - virtual bool characters (const QString &ch); - virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); - virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); - virtual bool error (const QXmlParseException &exception); - virtual bool fatalError (const QXmlParseException &exception); - virtual bool warning (const QXmlParseException &exception); - virtual QString errorString () const; - - void setDocumentLocator (QXmlLocator *locator); - - const tl::XMLLocatedException *exception () const - { - return m_error.get (); - } - -private: - QXmlLocator *mp_locator; - XMLStructureHandler *mp_struct_handler; - std::unique_ptr m_error; - std::string m_error_string; -}; - -// -------------------------------------------------------------------------------------------------------- -// trureHandler implementation - -SAXHandler::SAXHandler (XMLStructureHandler *sh) - : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) -{ - // .. nothing yet .. -} - -void -SAXHandler::setDocumentLocator (QXmlLocator *locator) -{ - mp_locator = locator; -} - -bool -SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->start_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->end_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::characters (const QString &t) -{ - try { - mp_struct_handler->characters (tl::to_string (t)); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -QString -SAXHandler::errorString () const -{ - return tl::to_qstring (m_error_string); -} - - -bool -SAXHandler::error (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::fatalError (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::warning (const QXmlParseException &ex) -{ - tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); - tl::warn << lex.msg (); - // continue - return true; -} - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData - : public QXmlInputSource -{ -public: - XMLSourcePrivateData () - : QXmlInputSource () - { - // .. nothing yet .. - } - - XMLSourcePrivateData (QIODevice *dev) - : QXmlInputSource (dev) - { - // .. nothing yet .. - } -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (string.c_str ())); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp)); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp, int (len))); - set_source (source); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// StreamIODevice definition and implementation - -class StreamIODevice - : public QIODevice -{ -public: - StreamIODevice (tl::InputStream &stream) - : mp_stream (&stream), - mp_progress (0), - m_has_error (false) - { - open (QIODevice::ReadOnly); - } - - StreamIODevice (tl::InputStream &stream, const std::string &progress_message) - : mp_stream (&stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (0), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path, const std::string &progress_message) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - ~StreamIODevice () - { - if (mp_progress) { - delete mp_progress; - mp_progress = 0; - } - } - - virtual bool isSequential () const - { - return true; - } - - qint64 writeData (const char *, qint64) - { - tl_assert (false); - } - - qint64 readData (char *data, qint64 n) - { - try { - - if (mp_progress) { - mp_progress->set (mp_stream->pos ()); - } - - qint64 n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { - // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). - if (*rd != '\r') { - *data++ = *rd; - --n; - } - } - - if (n0 == n) { - return -1; - } else { - return n0 - n; - } - - } catch (tl::Exception &ex) { - setErrorString (tl::to_qstring (ex.msg ())); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - -private: - tl::InputStream *mp_stream; - std::unique_ptr mp_stream_holder; - tl::AbsoluteProgress *mp_progress; - bool m_has_error; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -class XMLStreamSourcePrivateData - : public XMLSourcePrivateData -{ -public: - XMLStreamSourcePrivateData (StreamIODevice *io) - : XMLSourcePrivateData (io), mp_io (io) - { - // .. nothing yet .. - } - - virtual void fetchData () - { - QXmlInputSource::fetchData (); - - // This feature is actually missing in the original implementation: throw an exception on error - if (mp_io->has_error ()) { - throw tl::Exception (tl::to_string (mp_io->errorString ())); - } - } - -private: - std::unique_ptr mp_io; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -class XMLParserPrivateData - : public QXmlSimpleReader -{ -public: - XMLParserPrivateData () : QXmlSimpleReader () { } -}; - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - SAXHandler handler (&struct_handler); - - mp_data->setContentHandler (&handler); - mp_data->setErrorHandler (&handler); - - bool result = mp_data->parse (source.source (), false /*=not incremental*/); - if (! result && handler.exception ()) { - throw tl::XMLLocatedException (*handler.exception ()); - } -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#else - -namespace tl -{ - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - // .. nothing yet .. -} - -void -XMLSource::reset () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *, size_t) -{ - tl_assert (false); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &, const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::XMLFileSource (const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) -{ - tl_assert (false); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &) -{ - tl_assert (false); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -XMLParser::XMLParser () - : mp_data (0) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - // .. nothing yet .. -} - -void -XMLParser::parse (XMLSource &, XMLStructureHandler &) -{ - tl_assert (false); -} - -bool -XMLParser::is_available () -{ - return false; -} - -} - -#endif - namespace tl { // ----------------------------------------------------------------- @@ -942,8 +62,140 @@ XMLElementProxy::~XMLElementProxy () mp_ptr = 0; } +// ----------------------------------------------------------------- +// XMLElementList implementation + +static size_t s_oid = 0; + +XMLElementList::XMLElementList () + : m_oid (++s_oid) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementBase &e) + : m_oid (++s_oid) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (XMLElementBase *e) + : m_oid (++s_oid) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::XMLElementList (const std::string &name, const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (d.m_name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d, const XMLElementBase &e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (const XMLElementList &d, XMLElementBase *e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +void XMLElementList::append (const XMLElementBase &e) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +void XMLElementList::append (XMLElementBase *e) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::iterator XMLElementList::begin () const +{ + return m_elements.begin (); +} + +XMLElementList::iterator XMLElementList::end () const +{ + return m_elements.end (); +} + +XMLElementList XMLElementList::empty () +{ + return XMLElementList (); +} + +// ----------------------------------------------------------------- // XMLElementBase implementation +static std::string parse_name (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + return std::string (n, 0, hash); + } else { + return n; + } +} + +static int parse_tag (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + tl::Extractor ex (n.c_str () + hash + 1); + int tag; + if (ex.try_read (tag)) { + return tag; + } + } + return -1; +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList &children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (new XMLElementList (children)), m_owns_child_list (true) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList *children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (children), m_owns_child_list (false) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const XMLElementBase &d) + : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) +{ + if (m_owns_child_list) { + mp_children = new XMLElementList (*d.mp_children); + } else { + mp_children = d.mp_children; + } +} + +XMLElementBase::~XMLElementBase () +{ + if (m_owns_child_list) { + delete const_cast (mp_children); + mp_children = 0; + } +} + void XMLElementBase::write_indent (tl::OutputStream &os, int indent) { @@ -975,6 +227,127 @@ XMLElementBase::write_string (tl::OutputStream &os, const std::string &s) } } +bool +XMLElementBase::check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const +{ + if (m_name == "*") { + return true; + } else { + return m_name == lname; // no namespace currently + } +} + +XMLElementBase::Cardinality +XMLElementBase::cardinality () const +{ + return Zero; +} + +XMLElementBase::iterator +XMLElementBase::begin () const +{ + return mp_children->begin (); +} + +XMLElementBase::iterator +XMLElementBase::end () const +{ + return mp_children->end (); +} + +std::string +XMLElementBase::name4code () const +{ + std::string res; + const char *n = name ().c_str (); + + if (! isalpha (*n) && *n != '_') { + res += '_'; + } + + while (*n) { + if (*n == '-') { + res += '_'; + } else if (isalnum (*n) || *n == '_') { + res += *n; + } + ++n; + } + + return res; +} + +std::string +XMLElementBase::create_def (std::map > &messages) const +{ + std::string res; + + auto m = messages.find (oid ()); + if (m != messages.end ()) { + + res += "message " + m->second.second + " {\n"; + + for (auto i = begin (); i != end (); ++i) { + const XMLElementBase *e = i->get (); + Cardinality c = e->cardinality (); + std::string entry = e->create_def_entry (messages); + if (! entry.empty ()) { + res += " "; + if (c != Zero) { + if (c == Many) { + res += "repeated "; + } else { + res += "optional "; + } + res += entry + "\n"; + } + } + } + + res += "}"; + + } + + return res; +} + +void +XMLElementBase::collect_messages (std::map > &messages) const +{ + for (auto i = begin (); i != end (); ++i) { + i->get ()->collect_messages (messages); + } +} + +std::string +XMLElementBase::make_message_name () const +{ + std::string res = mp_children->name (); + if (! res.empty ()) { + return res; + } + + // Capitalize names + std::string n4c = name4code (); + + bool upcase = true; + + const char *n = n4c.c_str (); + while (*n) { + if (*n == '_') { + upcase = true; + } else if (upcase) { + res += toupper (*n); + upcase = false; + } else { + res += *n; + } + ++n; + } + + return res; +} + // -------------------------------------------------------------------------------------------------------- // trureHandler implementation @@ -1066,5 +439,76 @@ XMLWriterState::XMLWriterState () // .. nothing yet .. } +// -------------------------------------------------------------------- +// PBParser implementation + +PBParser::PBParser () +{ + // .. nothing yet .. +} + +PBParser::~PBParser () +{ + // .. nothing yet .. +} + +void +PBParser::parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state) +{ + mp_state = reader_state; + parse_element (root, reader); +} + +void +PBParser::parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader) +{ + while (! reader.at_end ()) { + + int tag = reader.read_tag (); + + const XMLElementBase *new_element = 0; + if (parent) { + for (XMLElementBase::iterator c = parent->begin (); c != parent->end (); ++c) { + if ((*c)->tag () == tag) { + new_element = (*c).get (); + break; + } + } + } + + if (! new_element) { + reader.skip (); + } else { + new_element->pb_create (parent, *mp_state); + new_element->pb_parse (this, reader); + new_element->pb_finish (parent, *mp_state); + } + + } +} + +void +PBParser::expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name) +{ + int tag = reader.read_tag (); + if (tag != name_tag) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with ID %d (got %d)")), name_tag, tag)); + } + + std::string n; + reader.read (n); + if (n != name) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with string '%s' (got '%s')")), name, n)); + } +} + +// -------------------------------------------------------------------- +// PBWriterState implementation + +PBWriterState::PBWriterState () +{ + // .. nothing yet .. +} + } diff --git a/src/tl/tl/tlXMLParser.h b/src/tl/tl/tlXMLParser.h index 226217228..fab046eff 100644 --- a/src/tl/tl/tlXMLParser.h +++ b/src/tl/tl/tlXMLParser.h @@ -29,6 +29,8 @@ #include #include +#include "tlXMLReader.h" +#include "tlProtocolBuffer.h" #include "tlAssert.h" #include "tlInternational.h" #include "tlString.h" @@ -40,339 +42,12 @@ namespace tl { /** - * @brief A basic XML parser error exception class + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. */ -class TL_PUBLIC XMLException : public tl::Exception -{ -public: - XMLException (const char *msg) - : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - XMLException (const std::string &msg) - : Exception (fmt (-1, -1).c_str (), msg.c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - /** - * @brief Raw (unprefixed) message of the XML parser - */ - const std::string & - raw_msg () const - { - return m_msg; - } - -protected: - XMLException (const std::string &msg, int line, int column) - : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), - m_msg (msg) - { - // .. nothing yet .. - } - -private: - std::string m_msg; - - static std::string fmt (int line, int /*column*/) - { - if (line < 0) { - return tl::to_string (tr ("XML parser error: %s")).c_str (); - } else { - return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); - } - } -}; - -/** - * @brief A XML parser error exception class that additionally provides line and column information - */ - -class TL_PUBLIC XMLLocatedException : public XMLException -{ -public: - XMLLocatedException (const std::string &msg, int line, int column) - : XMLException (msg, line, column), - m_line (line), m_column (column) - { - // .. nothing yet .. - } - - /** - * @brief Line number information of the exception - */ - int line () const - { - return m_line; - } - - /** - * @brief Column number information of the exception - */ - int column () const - { - return m_column; - } - -private: - int m_line; - int m_column; -}; - -/** - * @brief An object wrapper base class for target object management - * - * Implementations of this class through the XMLReaderProxy templates - * manage pointers to certain objects. - */ - -class TL_PUBLIC XMLReaderProxyBase -{ -public: - XMLReaderProxyBase () { } - virtual ~XMLReaderProxyBase () { } - virtual void release () = 0; - virtual void detach () = 0; -}; - -/** - * @brief An object wrapper base class for target object management specialized to a certain class - */ - -template -class TL_PUBLIC_TEMPLATE XMLReaderProxy - : public XMLReaderProxyBase -{ -public: - XMLReaderProxy (Obj *obj, bool owns_obj) - : mp_obj (obj), m_owns_obj (owns_obj) - { } - - virtual ~XMLReaderProxy () { } - - virtual void release () - { - if (m_owns_obj && mp_obj) { - delete mp_obj; - } - mp_obj = 0; - } - - virtual void detach () - { - m_owns_obj = false; - } - - Obj *ptr () const - { - return mp_obj; - } - -private: - Obj *mp_obj; - bool m_owns_obj; -}; - -/** - * @brief Helper class: A class tag - */ - -template -struct XMLObjTag -{ - XMLObjTag() { } - typedef Obj obj; -}; - -/** - * @brief Helper class: The reader state - * - * The reader state mainly comprises of a stack of objects being parsed and - * a string in which to collect cdata. - */ - -class TL_PUBLIC XMLReaderState -{ -public: - /** - * @brief Default constructor - */ - XMLReaderState (); - - /** - * @brief Destructor - */ - ~XMLReaderState (); - - /** - * @brief Push a new object on the stack - */ - template - void push (XMLObjTag /*tag*/) - { - m_objects.push_back (new XMLReaderProxy (new Obj (), true)); - } - - /** - * @brief Push an existing object on the stack - */ - template - void push (Obj *obj) - { - m_objects.push_back (new XMLReaderProxy (obj, false)); - } - - /** - * @brief Push an existing object on the stack with the ownership flag - */ - template - void push (Obj *obj, bool owner) - { - m_objects.push_back (new XMLReaderProxy (obj, owner)); - } - - /** - * @brief Get the top object - */ - template - Obj *back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Get the top object and release - */ - template - Obj *detach_back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->detach (); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Pop an object from the stack - */ - template - void pop (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->release (); - delete m_objects.back (); - m_objects.pop_back (); - } - - /** - * @brief Empty predicate: true, if no more object is on the stack - */ - bool empty () const - { - return m_objects.empty (); - } - - /** - * @brief Obtain the parent object from the stack - */ - template - Obj *parent (XMLObjTag /*tag*/) - { - tl_assert (m_objects.size () > 1); - return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); - } - - /** - * @brief The cdata string collected - */ - std::string cdata; - -private: - std::vector m_objects; -}; - -// The opaque source type -class XMLSourcePrivateData; - -/** - * @brief A generic XML text source class - * - * This class is the base class providing input for - * the Qt XML parser and basically maps to a QXmlInputSource object - * for compatibility with the "libparsifal" branch. - */ - -class TL_PUBLIC XMLSource -{ -public: - XMLSource (); - ~XMLSource (); - - XMLSourcePrivateData *source () - { - return mp_source; - } - - void reset (); - -protected: - void set_source (XMLSourcePrivateData *source) - { - mp_source = source; - } - -private: - XMLSourcePrivateData *mp_source; -}; - -/** - * @brief A specialization of XMLSource to receive a string - */ - -class TL_PUBLIC XMLStringSource : public XMLSource -{ -public: - XMLStringSource (const std::string &string); - XMLStringSource (const char *cp); - XMLStringSource (const char *cp, size_t len); - ~XMLStringSource (); - -private: - std::string m_copy; -}; - -/** - * @brief A specialization of XMLSource to receive from a file - */ - -class TL_PUBLIC XMLFileSource : public XMLSource -{ -public: - XMLFileSource (const std::string &path); - XMLFileSource (const std::string &path, const std::string &progress_message); - ~XMLFileSource (); -}; - -/** - * @brief A generic stream source class - * - * This class implements a XML parser source from a tl::InputStream - */ - -class TL_PUBLIC XMLStreamSource : public XMLSource -{ -public: - XMLStreamSource (tl::InputStream &stream); - XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); - ~XMLStreamSource (); -}; - +class XMLReaderState; +class XMLReaderState; // ----------------------------------------------------------------- // The C++ structure definition interface (for use cases see tlXMLParser.ut) @@ -387,6 +62,18 @@ struct pass_by_ref_tag { pass_by_ref_tag () { } }; +struct pb_zero_cardinality_tag { + pb_zero_cardinality_tag () { } +}; + +struct pb_single_cardinality_tag { + pb_single_cardinality_tag () { } +}; + +struct pb_many_cardinality_tag { + pb_many_cardinality_tag () { } +}; + /** * @brief The XML handler implementation * @@ -483,66 +170,208 @@ public: typedef std::list children_list; typedef children_list::const_iterator iterator; - XMLElementList () + XMLElementList (); + XMLElementList (const XMLElementBase &e); + XMLElementList (XMLElementBase *e); + XMLElementList (const std::string &name, const XMLElementList &d); + XMLElementList (const XMLElementList &d); + XMLElementList (const XMLElementList &d, const XMLElementBase &e); + XMLElementList (const XMLElementList &d, XMLElementBase *e); + + void append (const XMLElementBase &e); + void append (XMLElementBase *e); + + iterator begin () const; + iterator end () const; + + static XMLElementList empty (); + + size_t oid () const { - // .. nothing yet .. + return m_oid; } - XMLElementList (const XMLElementBase &e) + const std::string &name () const { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - XMLElementList (const XMLElementList &d, const XMLElementBase &e) - : m_elements (d.m_elements) - { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (const XMLElementList &d, XMLElementBase *e) - : m_elements (d.m_elements) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - void append (const XMLElementBase &e) - { - m_elements.push_back (XMLElementProxy (e)); - } - - void append (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - iterator begin () const - { - return m_elements.begin (); - } - - iterator end () const - { - return m_elements.end (); - } - - static XMLElementList empty () - { - return XMLElementList (); + return m_name; } private: std::list m_elements; + size_t m_oid; + std::string m_name; +}; + +/** + * @brief An object wrapper base class for target object management + * + * Implementations of this class through the XMLReaderProxy templates + * manage pointers to certain objects. + */ + +class TL_PUBLIC XMLReaderProxyBase +{ +public: + XMLReaderProxyBase () { } + virtual ~XMLReaderProxyBase () { } + virtual void release () = 0; + virtual void detach () = 0; +}; + +/** + * @brief An object wrapper base class for target object management specialized to a certain class + */ + +template +class TL_PUBLIC_TEMPLATE XMLReaderProxy + : public XMLReaderProxyBase +{ +public: + XMLReaderProxy (Obj *obj, bool owns_obj) + : mp_obj (obj), m_owns_obj (owns_obj) + { } + + virtual ~XMLReaderProxy () { } + + virtual void release () + { + if (m_owns_obj && mp_obj) { + delete mp_obj; + } + mp_obj = 0; + } + + virtual void detach () + { + m_owns_obj = false; + } + + Obj *ptr () const + { + return mp_obj; + } + +private: + Obj *mp_obj; + bool m_owns_obj; +}; + +/** + * @brief Helper class: A class tag + */ + +template +struct XMLObjTag +{ + XMLObjTag() { } + typedef Obj obj; +}; + +/** + * @brief Helper class: The reader state + * + * The reader state mainly comprises of a stack of objects being parsed and + * a string in which to collect cdata. + */ + +class TL_PUBLIC XMLReaderState +{ +public: + /** + * @brief Default constructor + */ + XMLReaderState (); + + /** + * @brief Destructor + */ + ~XMLReaderState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (XMLObjTag /*tag*/) + { + m_objects.push_back (new XMLReaderProxy (new Obj (), true)); + } + + /** + * @brief Push an existing object on the stack + */ + template + void push (Obj *obj) + { + m_objects.push_back (new XMLReaderProxy (obj, false)); + } + + /** + * @brief Push an existing object on the stack with the ownership flag + */ + template + void push (Obj *obj, bool owner) + { + m_objects.push_back (new XMLReaderProxy (obj, owner)); + } + + /** + * @brief Get the top object + */ + template + Obj *back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Get the top object and release + */ + template + Obj *detach_back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->detach (); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Pop an object from the stack + */ + template + void pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->release (); + delete m_objects.back (); + m_objects.pop_back (); + } + + /** + * @brief Empty predicate: true, if no more object is on the stack + */ + bool empty () const + { + return m_objects.empty (); + } + + /** + * @brief Obtain the parent object from the stack + */ + template + Obj *parent (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 1); + return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); + } + + /** + * @brief The cdata string collected + */ + std::string cdata; + +private: + std::vector m_objects; }; /** @@ -592,6 +421,78 @@ private: std::vector m_objects; }; +/** + * @brief The PB parser class + * This is the main entry point. It will take a PB reader, the structure (in "root") and + * a reader state initialized with the top level object. + */ +class TL_PUBLIC PBParser +{ +public: + PBParser (); + ~PBParser (); + + void parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state); + void parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader); + + void expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name); + + XMLReaderState &reader_state () + { + return *mp_state; + } + +private: + XMLReaderState *mp_state; +}; + +/** + * @brief Helper class: A stack of const objects being written (PB binding) + */ + +class TL_PUBLIC PBWriterState +{ +public: + /** + * @brief Default constructor + */ + PBWriterState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (const Obj *obj) + { + m_objects.push_back (obj); + } + + /** + * @brief Pop an object from the stack + */ + template + const Obj *pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + const Obj *obj = reinterpret_cast (m_objects.back ()); + m_objects.pop_back (); + return obj; + } + + /** + * @brief Obtain the parent object from the stack + */ + template + const Obj *back (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 0); + return reinterpret_cast (m_objects.end () [-1]); + } + +private: + std::vector m_objects; +}; + /** * @brief The XML element base object * @@ -606,35 +507,14 @@ class TL_PUBLIC XMLElementBase public: typedef XMLElementList::iterator iterator; - XMLElementBase (const std::string &name, const XMLElementList &children) - : m_name (name), mp_children (new XMLElementList (children)), m_owns_child_list (true) - { - // .. nothing yet .. - } + enum Cardinality { + Zero, Single, Many + }; - XMLElementBase (const std::string &name, const XMLElementList *children) - : m_name (name), mp_children (children), m_owns_child_list (false) - { - // .. nothing yet .. - } - - XMLElementBase (const XMLElementBase &d) - : m_name (d.m_name), m_owns_child_list (d.m_owns_child_list) - { - if (m_owns_child_list) { - mp_children = new XMLElementList (*d.mp_children); - } else { - mp_children = d.mp_children; - } - } - - virtual ~XMLElementBase () - { - if (m_owns_child_list) { - delete const_cast (mp_children); - mp_children = 0; - } - } + XMLElementBase (const std::string &name, const XMLElementList &children); + XMLElementBase (const std::string &name, const XMLElementList *children); + XMLElementBase (const XMLElementBase &d); + virtual ~XMLElementBase (); virtual XMLElementBase *clone () const = 0; @@ -648,32 +528,66 @@ public: static void write_indent (tl::OutputStream &os, int indent); static void write_string (tl::OutputStream &os, const std::string &s); + virtual void pb_create (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; + virtual void pb_finish (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + + virtual void pb_write (const XMLElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } + + virtual Cardinality cardinality () const; + + size_t oid () const + { + return mp_children->oid (); + } + + int tag () const + { + return m_tag; + } + const std::string &name () const { return m_name; } - bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const + bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const; + + /** + * @brief Returns a name suitable for code + * Specifically, hyphens are replaced by underscores. + */ + std::string name4code () const; + + iterator begin () const; + iterator end () const; + + std::string create_def (std::map > &messages) const; + +protected: + virtual void collect_messages (std::map > &messages) const; + virtual std::string create_def_entry (std::map > &messages) const = 0; + + std::string make_message_name () const; + + static Cardinality get_cardinality (tl::pb_zero_cardinality_tag) { - if (m_name == "*") { - return true; - } else { - return m_name == lname; // no namespace currently - } + return Zero; } - iterator begin () const + static Cardinality get_cardinality (tl::pb_single_cardinality_tag) { - return mp_children->begin (); + return Single; } - iterator end () const + static Cardinality get_cardinality (tl::pb_many_cardinality_tag) { - return mp_children->end (); + return Many; } private: std::string m_name; + int m_tag; const XMLElementList *mp_children; bool m_owns_child_list; }; @@ -767,6 +681,64 @@ public: return (! r.at_end ()); } + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const + { + XMLObjTag tag; + objs.push (tag); + } + + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + reader.open (); + parser->parse_element (this, reader); + reader.close (); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + typedef typename Read::tag read_tag_type; + pb_write_obj (r (), tag (), writer, read_tag_type (), objs); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > &messages) const + { + if (messages.find (oid ()) == messages.end ()) { + messages [oid ()] = std::make_pair (this, make_message_name ()); + XMLElementBase::collect_messages (messages); + } + } + + virtual std::string create_def_entry (std::map > &messages) const + { + auto m = messages.find (oid ()); + if (m != messages.end ()) { + return m->second.second + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } else { + return std::string (); + } + } + private: Read m_r; Write m_w; @@ -791,6 +763,43 @@ private: } objs.pop (tag); } + + // this write helper is used if the reader delivers an object by value + void pb_write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_value_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (! writer.is_counting ()) { + break; + } + } + } + + void pb_write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_ref_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (writer.is_counting ()) { + break; + } + } + } }; /** @@ -802,23 +811,23 @@ private: template class TL_PUBLIC_TEMPLATE XMLElementWithParentRef - : public XMLElementBase + : public XMLElement { public: XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElement (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElement (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const XMLElementWithParentRef &d) - : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) + : XMLElement (d) { // .. nothing yet .. } @@ -835,11 +844,6 @@ public: objs.push (new Obj (objs.back (parent_tag)), true); } - virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const - { - // .. nothing yet .. - } - virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const { XMLObjTag tag; @@ -848,53 +852,18 @@ public: objs.pop (tag); } - virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - XMLElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - XMLElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } - } - - virtual bool has_any (XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } - -private: - Read m_r; - Write m_w; - - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); + XMLObjTag parent_tag; + objs.push (new Obj (objs.back (parent_tag)), true); } - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, XMLWriterState &objs) const + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } + XMLObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } }; @@ -992,10 +961,278 @@ public: return (! r.at_end ()); } + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + + XMLReaderState value_obj; + value_obj.push (tag); + + read (reader, *value_obj.back (tag)); + m_w (*parser->reader_state ().back (parent_tag), value_obj); + + value_obj.pop (tag); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + Read r (m_r); + r.start (* objs.back (parent_tag)); + while (! r.at_end ()) { + write (writer, tag (), r ()); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > & /*messages*/) const + { + // no messages here. + } + + virtual std::string create_def_entry (std::map > & /*messages*/) const + { + const Value *v = 0; + return typestring (v) + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } + private: Read m_r; Write m_w; Converter m_c; + + // write incarnations + void write (tl::ProtocolBufferWriterBase &writer, int tag, float v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, double v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint8_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int8_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint16_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int16_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, bool v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, const std::string &v) const + { + writer.write (tag, v); + } + + template + void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const + { + writer.write (tag, m_c.pb_encode (v)); + } + + // read incarnations + void read (tl::ProtocolBufferReaderBase &reader, float &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, double &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint8_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int8_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint16_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int16_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, bool &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, std::string &v) const + { + reader.read (v); + } + + template + void read (tl::ProtocolBufferReaderBase &reader, T &v) const + { + typename Converter::pb_type vv; + reader.read (vv); + m_c.pb_decode (vv, v); + } + + // type strings + std::string typestring (const float *) const + { + return "float"; + } + + std::string typestring (const double *) const + { + return "double"; + } + + std::string typestring (const uint8_t *) const + { + return "uint32"; + } + + std::string typestring (const uint16_t *) const + { + return "uint32"; + } + + std::string typestring (const uint32_t *) const + { + return "uint32"; + } + + std::string typestring (const uint64_t *) const + { + return "uint64"; + } + + std::string typestring (const int8_t *) const + { + return "sint32"; + } + + std::string typestring (const int16_t *) const + { + return "sint32"; + } + + std::string typestring (const int32_t *) const + { + return "sint32"; + } + + std::string typestring (const int64_t *) const + { + return "sint64"; + } + + std::string typestring (const bool *) const + { + return "uint32"; + } + + std::string typestring (const std::string *) const + { + return "string"; + } + + template + std::string typestring (const T *) const + { + const typename Converter::pb_type *v = 0; + return typestring (v); + } }; /** @@ -1143,11 +1380,90 @@ public: tl_assert (rs.empty ()); } + /** + * @brief Serializes the given object (root) to the writer + */ + void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const + { + PBWriterState writer_state; + writer_state.push (& root); + + writer.write (tag (), name ()); + + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, writer_state); + } + } + + /** + * @brief Deserializes the given object (root) from the reader + */ + void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const + { + XMLObjTag self_tag; + XMLReaderState rs; + rs.push (&root); + PBParser h; + h.expect_header (reader, tag (), name ()); + h.parse (reader, this, &rs); + rs.pop (self_tag); + tl_assert (rs.empty ()); + } + + /** + * @brief Produces a definition for the protoc compiler + */ + std::string create_def () const + { + std::map > msgs; + collect_messages (msgs); + msgs[oid ()] = std::make_pair (this, make_message_name ()); + + std::string res = "// created from KLayout proto definition '" + name () + "'\n\n"; + res += "syntax = \"proto2\";"; + + for (auto i = msgs.begin (); i != msgs.end (); ++i) { + std::string entry = i->second.first->create_def (msgs); + if (! entry.empty ()) { + res += "\n"; + res += "\n"; + res += entry; + } + } + + return res; + } + private: virtual void write (const XMLElementBase*, tl::OutputStream &, int, XMLWriterState &) const { // .. see write (os) } + + virtual void pb_write (const XMLElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const + { + // disable base class implementation + } + + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const + { + // disable base class implementation + } + + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual std::string create_def_entry (std::map > &) const + { + return std::string (); + } }; /** @@ -1267,6 +1583,7 @@ template struct XMLMemberDummyReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_zero_cardinality_tag cardinality; XMLMemberDummyReadAdaptor () { @@ -1298,6 +1615,7 @@ template struct XMLMemberReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberReadAdaptor (Value Parent::*member) : mp_member (member), mp_owner (0), m_done (false) @@ -1336,6 +1654,7 @@ template struct XMLMemberAccRefReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1374,6 +1693,7 @@ template struct XMLMemberAccReadAdaptor { typedef pass_by_value_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberAccReadAdaptor (Value (Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1412,6 +1732,7 @@ template struct XMLMemberIterReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_many_cardinality_tag cardinality; XMLMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) : mp_begin (begin), mp_end (end) diff --git a/src/tl/tl/tlXMLReader.cc b/src/tl/tl/tlXMLReader.cc new file mode 100644 index 000000000..49f0509dc --- /dev/null +++ b/src/tl/tl/tlXMLReader.cc @@ -0,0 +1,913 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlXMLReader.h" +#include "tlXMLParser.h" +#include "tlString.h" +#include "tlLog.h" +#include "tlAssert.h" +#include "tlProgress.h" + +#include +#include + +#if defined(HAVE_EXPAT) + +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData +{ +public: + XMLSourcePrivateData (tl::InputStream *stream) + : mp_stream_holder (stream), + m_has_error (false) + { + mp_stream = stream; + } + + XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) + : mp_stream_holder (stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + XMLSourcePrivateData (tl::InputStream &stream) + : m_has_error (false) + { + mp_stream = &stream; + } + + XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) + : mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = &stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + int read (char *data, size_t n) + { + try { + + if (mp_progress.get ()) { + mp_progress->set (mp_stream->pos ()); + } + + size_t n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { + *data++ = *rd; + } + + return int (n0 - n); + + } catch (tl::Exception &ex) { + m_error = ex.msg (); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + + const std::string &error_msg () const + { + return m_error; + } + + void reset () + { + mp_stream->reset (); + } + +private: + std::unique_ptr mp_stream_holder; + tl::InputStream *mp_stream; + std::unique_ptr mp_progress; + bool m_has_error; + std::string m_error; +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) + : m_copy (string) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (s, progress_message)); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLSourcePrivateData (s)); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); +void XMLCALL end_element_handler (void *user_data, const XML_Char *name); +void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); + +static std::string get_lname (const std::string &name) +{ + size_t colon = name.find (':'); + if (colon != std::string::npos) { + return std::string (name, colon + 1, name.size () - colon - 1); + } else { + return name; + } +} + +class XMLParserPrivateData +{ +public: + XMLParserPrivateData () + : mp_struct_handler (0) + { + mp_parser = XML_ParserCreate ("UTF-8"); + tl_assert (mp_parser != NULL); + } + + ~XMLParserPrivateData () + { + if (mp_parser != NULL) { + XML_ParserFree (mp_parser); + } + } + + void start_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->start_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void end_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->end_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void cdata (const std::string &cdata) + { + try { + mp_struct_handler->characters (cdata); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) + { + m_has_error = false; + mp_struct_handler = &struct_handler; + + // Just in case we want to reuse it ... + XML_ParserReset (mp_parser, NULL); + XML_SetUserData (mp_parser, (void *) this); + XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler (mp_parser, cdata_handler); + + const int chunk = 65536; + char buffer [chunk]; + + int n; + + do { + + n = source.source ()->read (buffer, chunk); + if (n < 0) { + break; + } + + XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); + if (status == XML_STATUS_ERROR) { + m_has_error = true; + m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); + m_error_line = XML_GetErrorLineNumber (mp_parser); + m_error_column = XML_GetErrorColumnNumber (mp_parser); + } + + } while (n == chunk && !m_has_error); + } + + void check_error () + { + if (m_has_error) { + throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); + } + } + +private: + void error (tl::Exception &ex) + { + m_has_error = true; + m_error_line = XML_GetCurrentLineNumber (mp_parser); + m_error_column = XML_GetCurrentColumnNumber (mp_parser); + m_error = ex.msg (); + } + + XML_Parser mp_parser; + XMLStructureHandler *mp_struct_handler; + bool m_has_error; + std::string m_error; + int m_error_line, m_error_column; +}; + +void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->start_element (std::string (name)); +} + +void end_element_handler (void *user_data, const XML_Char *name) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->end_element (std::string (name)); +} + +void cdata_handler (void *user_data, const XML_Char *s, int len) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->cdata (std::string (s, size_t (len))); +} + + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + mp_data->parse (source, struct_handler); + + // throws an exception if there is an error + mp_data->check_error (); +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#elif defined(HAVE_QT) + +#include +#include +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// A SAX handler for the Qt implementation + +class SAXHandler + : public QXmlDefaultHandler +{ +public: + SAXHandler (XMLStructureHandler *sh); + + virtual bool characters (const QString &ch); + virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); + virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); + virtual bool error (const QXmlParseException &exception); + virtual bool fatalError (const QXmlParseException &exception); + virtual bool warning (const QXmlParseException &exception); + virtual QString errorString () const; + + void setDocumentLocator (QXmlLocator *locator); + + const tl::XMLLocatedException *exception () const + { + return m_error.get (); + } + +private: + QXmlLocator *mp_locator; + XMLStructureHandler *mp_struct_handler; + std::unique_ptr m_error; + std::string m_error_string; +}; + +// -------------------------------------------------------------------------------------------------------- +// trureHandler implementation + +SAXHandler::SAXHandler (XMLStructureHandler *sh) + : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) +{ + // .. nothing yet .. +} + +void +SAXHandler::setDocumentLocator (QXmlLocator *locator) +{ + mp_locator = locator; +} + +bool +SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->start_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->end_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::characters (const QString &t) +{ + try { + mp_struct_handler->characters (tl::to_string (t)); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +QString +SAXHandler::errorString () const +{ + return tl::to_qstring (m_error_string); +} + + +bool +SAXHandler::error (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::fatalError (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::warning (const QXmlParseException &ex) +{ + tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); + tl::warn << lex.msg (); + // continue + return true; +} + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData + : public QXmlInputSource +{ +public: + XMLSourcePrivateData () + : QXmlInputSource () + { + // .. nothing yet .. + } + + XMLSourcePrivateData (QIODevice *dev) + : QXmlInputSource (dev) + { + // .. nothing yet .. + } +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (string.c_str ())); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp)); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp, int (len))); + set_source (source); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// StreamIODevice definition and implementation + +class StreamIODevice + : public QIODevice +{ +public: + StreamIODevice (tl::InputStream &stream) + : mp_stream (&stream), + mp_progress (0), + m_has_error (false) + { + open (QIODevice::ReadOnly); + } + + StreamIODevice (tl::InputStream &stream, const std::string &progress_message) + : mp_stream (&stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (0), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path, const std::string &progress_message) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + ~StreamIODevice () + { + if (mp_progress) { + delete mp_progress; + mp_progress = 0; + } + } + + virtual bool isSequential () const + { + return true; + } + + qint64 writeData (const char *, qint64) + { + tl_assert (false); + } + + qint64 readData (char *data, qint64 n) + { + try { + + if (mp_progress) { + mp_progress->set (mp_stream->pos ()); + } + + qint64 n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { + // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). + if (*rd != '\r') { + *data++ = *rd; + --n; + } + } + + if (n0 == n) { + return -1; + } else { + return n0 - n; + } + + } catch (tl::Exception &ex) { + setErrorString (tl::to_qstring (ex.msg ())); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + +private: + tl::InputStream *mp_stream; + std::unique_ptr mp_stream_holder; + tl::AbsoluteProgress *mp_progress; + bool m_has_error; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +class XMLStreamSourcePrivateData + : public XMLSourcePrivateData +{ +public: + XMLStreamSourcePrivateData (StreamIODevice *io) + : XMLSourcePrivateData (io), mp_io (io) + { + // .. nothing yet .. + } + + virtual void fetchData () + { + QXmlInputSource::fetchData (); + + // This feature is actually missing in the original implementation: throw an exception on error + if (mp_io->has_error ()) { + throw tl::Exception (tl::to_string (mp_io->errorString ())); + } + } + +private: + std::unique_ptr mp_io; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +class XMLParserPrivateData + : public QXmlSimpleReader +{ +public: + XMLParserPrivateData () : QXmlSimpleReader () { } +}; + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + SAXHandler handler (&struct_handler); + + mp_data->setContentHandler (&handler); + mp_data->setErrorHandler (&handler); + + bool result = mp_data->parse (source.source (), false /*=not incremental*/); + if (! result && handler.exception ()) { + throw tl::XMLLocatedException (*handler.exception ()); + } +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#else + +namespace tl +{ + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + // .. nothing yet .. +} + +void +XMLSource::reset () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *, size_t) +{ + tl_assert (false); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &, const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::XMLFileSource (const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) +{ + tl_assert (false); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &) +{ + tl_assert (false); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +XMLParser::XMLParser () + : mp_data (0) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + // .. nothing yet .. +} + +void +XMLParser::parse (XMLSource &, XMLStructureHandler &) +{ + tl_assert (false); +} + +bool +XMLParser::is_available () +{ + return false; +} + +} + +#endif diff --git a/src/tl/tl/tlXMLReader.h b/src/tl/tl/tlXMLReader.h new file mode 100644 index 000000000..3b8423e3d --- /dev/null +++ b/src/tl/tl/tlXMLReader.h @@ -0,0 +1,215 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlXMLReader +#define HDR_tlXMLReader + +#include "tlCommon.h" + +#include +#include + +#include "tlAssert.h" +#include "tlInternational.h" +#include "tlString.h" +#include "tlStream.h" + +class QIODevice; + +namespace tl +{ + +/** + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. + */ + +class ProtocolBufferReaderBase; +class ProtocolBufferWriterBase; + +/** + * @brief A basic XML parser error exception class + */ + +class TL_PUBLIC XMLException : public tl::Exception +{ +public: + XMLException (const char *msg) + : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + XMLException (const std::string &msg) + : Exception (fmt (-1, -1).c_str (), msg.c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + /** + * @brief Raw (unprefixed) message of the XML parser + */ + const std::string & + raw_msg () const + { + return m_msg; + } + +protected: + XMLException (const std::string &msg, int line, int column) + : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), + m_msg (msg) + { + // .. nothing yet .. + } + +private: + std::string m_msg; + + static std::string fmt (int line, int /*column*/) + { + if (line < 0) { + return tl::to_string (tr ("XML parser error: %s")).c_str (); + } else { + return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); + } + } +}; + +/** + * @brief A XML parser error exception class that additionally provides line and column information + */ + +class TL_PUBLIC XMLLocatedException : public XMLException +{ +public: + XMLLocatedException (const std::string &msg, int line, int column) + : XMLException (msg, line, column), + m_line (line), m_column (column) + { + // .. nothing yet .. + } + + /** + * @brief Line number information of the exception + */ + int line () const + { + return m_line; + } + + /** + * @brief Column number information of the exception + */ + int column () const + { + return m_column; + } + +private: + int m_line; + int m_column; +}; + +// The opaque source type +class XMLSourcePrivateData; + +/** + * @brief A generic XML text source class + * + * This class is the base class providing input for + * the Qt XML parser and basically maps to a QXmlInputSource object + * for compatibility with the "libparsifal" branch. + */ + +class TL_PUBLIC XMLSource +{ +public: + XMLSource (); + ~XMLSource (); + + XMLSourcePrivateData *source () + { + return mp_source; + } + + void reset (); + +protected: + void set_source (XMLSourcePrivateData *source) + { + mp_source = source; + } + +private: + XMLSourcePrivateData *mp_source; +}; + +/** + * @brief A specialization of XMLSource to receive a string + */ + +class TL_PUBLIC XMLStringSource : public XMLSource +{ +public: + XMLStringSource (const std::string &string); + XMLStringSource (const char *cp); + XMLStringSource (const char *cp, size_t len); + ~XMLStringSource (); + +private: + std::string m_copy; +}; + +/** + * @brief A specialization of XMLSource to receive from a file + */ + +class TL_PUBLIC XMLFileSource : public XMLSource +{ +public: + XMLFileSource (const std::string &path); + XMLFileSource (const std::string &path, const std::string &progress_message); + ~XMLFileSource (); +}; + +/** + * @brief A generic stream source class + * + * This class implements a XML parser source from a tl::InputStream + */ + +class TL_PUBLIC XMLStreamSource : public XMLSource +{ +public: + XMLStreamSource (tl::InputStream &stream); + XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); + ~XMLStreamSource (); +}; + +} // namespace tl + +#endif + diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index 75289aed1..000ffda67 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -20,16 +20,14 @@ */ -#include "tlProtocolBufferStruct.h" +#include "tlXMLParser.h" +#include "tlProtocolBuffer.h" #include "tlUnitTest.h" #include "tlFileUtils.h" #include #include -// @@@ -// Missing: all kind of variants (uint8_t, ...), float - // Basic tests of reader and writer TEST (1_BasicTypes) { @@ -205,20 +203,20 @@ struct Root { const Child &get_child () const { return m_child; } }; -static tl::PBElementList child_struct ("Child", - tl::pb_make_member (&Child::txt, "txt", 1) + - tl::pb_make_member (&Child::d, "d", 2) + - tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children", 3, &child_struct) +static tl::XMLElementList child_struct ("Child", + tl::make_member (&Child::txt, "txt#1") + + tl::make_member (&Child::d, "d#2") + + tl::make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children#3", &child_struct) ); -static tl::PBStruct structure ("pbtest-struct", 88888888, - tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub", 1) + - tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub", 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children", 3, &child_struct) + - tl::pb_make_element (&Root::get_child, &Root::set_child, "child", 4, &child_struct) + - tl::pb_make_member (&Root::m, "m", 5) + - tl::pb_make_member (&Root::get_mi, &Root::set_mi, "mi", 6) + - tl::pb_make_member (&Root::b, "b", 7) +static tl::XMLStruct structure ("pbtest-struct#88888888", + tl::make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub#1") + + tl::make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub#2") + + tl::make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children#3", &child_struct) + + tl::make_element (&Root::get_child, &Root::set_child, "child#4", &child_struct) + + tl::make_member (&Root::m, "m#5") + + tl::make_member (&Root::get_mi, &Root::set_mi, "mi#6") + + tl::make_member (&Root::b, "b#7") ); static void build_struct (Root &root) @@ -405,10 +403,13 @@ struct TestClassEnumConverter break; } } + + void from_string (const std::string &, TestClass::enum_type) const { } + std::string to_string (TestClass::enum_type) const { return std::string (); } }; -tl::PBStruct tc_structure ("pbtest-tc", 1, - tl::pb_make_member (&TestClass::e, "e", 2, TestClassEnumConverter ()) +tl::XMLStruct tc_structure ("pbtest-tc#1", + tl::make_member (&TestClass::e, "e#2", TestClassEnumConverter ()) ); TEST (101_Converter)