From abe70971414b0667c5afac8e32107464c5a93673 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Mon, 8 Jul 2024 18:58:56 +0200 Subject: [PATCH 01/22] WIP --- src/tl/tl/tl.pro | 2 + src/tl/tl/tlProtocolBuffer.cc | 0 src/tl/tl/tlProtocolBuffer.h | 272 ++++++++++++++++++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 src/tl/tl/tlProtocolBuffer.cc create mode 100644 src/tl/tl/tlProtocolBuffer.h diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 7d1cf0c15..27b27f14a 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -29,6 +29,7 @@ SOURCES = \ tlObject.cc \ tlProgress.cc \ tlPixelBuffer.cc \ + tlProtocolBuffer.cc \ tlResources.cc \ tlScriptError.cc \ tlSleep.cc \ @@ -85,6 +86,7 @@ HEADERS = \ tlObjectCollection.h \ tlProgress.h \ tlPixelBuffer.h \ + tlProtocolBuffer.h \ tlResources.h \ tlReuseVector.h \ tlScriptError.h \ diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc new file mode 100644 index 000000000..e69de29bb diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h new file mode 100644 index 000000000..1b461a3f2 --- /dev/null +++ b/src/tl/tl/tlProtocolBuffer.h @@ -0,0 +1,272 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_tlProtocolBuffer +#define HDR_tlProtocolBuffer + +#include "tlCommon.h" +#include "tlStream.h" + +namespace tl +{ + +// Representation of VARINT values +typedef size_t pb_varint; + +enum PBWireType +{ + PB_VARINT = 0, + PB_I64 = 1, + PB_LEN = 2, + PB_SGROUP = 3, + PB_EGROUP = 4, + PB_I32 = 5 +}; + +/** + * @brief An exception thrown by the ProtocolBuffer reader on a read error + */ +class TL_PUBLIC ProtocolBufferReaderError + : public tl::Exception +{ +public: + ProtocolBufferReaderError (const std::string &msg, size_t position) + : tl::Exception (msg), m_position (position) + { } + + virtual std::string msg () const + { + return basic_msg () + tl::to_string (tr (", at position ") + tl::to_string (m_position)); + } + +private: + size_t m_position; +}; + +/** + * @brief A reader for ProtocolBuffer files and streams + * + * This is a low-level decoder for ProtocolBuffer files. + * + * The following LEN-type related concepts need to be implemented by the client code: + * - submessages + * - maps + * - packed repetitions + * - strings + * + * Unknown tags need to be skipped with "skip". + * + * Submessages: if a corresponding tag is encountered with "is_seq()" true, the + * reader code needs to call "open" to enter the sequence and read tags until + * "at_end" is true. Then, call "close" to leave the sequence. + * + * Packed repetitions: same a submessages, but single values are read + * with one of the "read" types. + * + * Maps are read like submessages with key/values as tags 1 and 2. + * + * Strings: if a corresponding tag is encountered, use "read(s)" to read + * the string. "is_seq()" is required to be true, i.e. wire type is LEN. + */ +class TL_PUBLIC ProtocolBufferReader +{ +public: + /** + * @brief Creates a ProtocolBuffer parser from the given stream + */ + ProtocolBufferReader (tl::InputStream &input); + + /** + * @brief Reads a new tag + * + * @returns The message ID + */ + int read_tag (); + + /** + * @brief Gets the current wire type + */ + PBWireType type () const + { + return m_type; + } + + /** + * @brief Returns true, if the current message is a LEN type sequence + * Such messages can be read into strings or "open" can be used on them to + * open a submessage, map or packed repetition. + */ + bool is_seq () const + { + return m_type == PB_LEN; + } + + /** + * @brief Skips the current tag + */ + void skip (); + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a double value. + */ + void read (double &d); + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a float value. + */ + void read (float &f); + + /** + * @brief Reads a string from the current message + * Throws a reader error if the current tag's value is not compatible with a string. + */ + void read (std::string &s); + + /** + * @brief Reads a uint32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint32_t. + */ + void read (uint32_t &ui32, bool fixed = false); + + /** + * @brief Reads a int32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int32_t. + */ + void read (int32_t &i32, bool fixed = false); + + /** + * @brief Reads a uint64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint64_t. + */ + void read (uint64_t &ui64, bool fixed = false); + + /** + * @brief Reads a int64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int64_t. + */ + void read (int64_t &i64, bool fixed = false); + + /** + * @brief Reads a boolean value from the current message + * Throws a reader error if the current tag's value is not compatible with a bool. + */ + void read (bool &b); + + /** + * @brief Opens a LEN sequence + * After calling "open", the parser will continue reading messages, but + * "at_end" will report true on the end of the sequence, not at the end of the + * file. + * Thie method will throw an exception if not in a message of LEN type. + */ + void open (); + + /** + * @brief Closes the LEN sequence + * This method will jump to the end of the sequence and continue reading + * messages from the previous block or file. + */ + void close (); + + /** + * @brief Returns true if at the end of the file or end of a block + */ + bool at_end () const; + +private: + tl::InputStream m_stream; + PBWireType m_type; + size_t m_pos; + std::vector m_block_end; + + pb_varint read_varint (); + bool has_bytes (size_t n); + void skip_bytes (size_t n); + void error (const std::string &msg); +}; + +/** + * @brief The Protocol buffer writer + * + * Scalar types are easy to write: just use the "write" methods. + * This includes strings. + * + * Submessages and packed sequences are special as byte counting is + * required. The writer uses a two-pass approach. This means: + * + * 1. On writing a submessage or packed repetition, call "begin_seq" + * with the message tag and "counting" set to true. + * 2. Write the elements using "write" or the submessage writing + * scheme recursively. + * 3. At the end of the sequence, call "end_seq". + * 4. if "is_counting()" is false, repeat steps 1 to 3 with + * "counting" set to false on "begin_seq". + */ +class TL_PUBLIC ProtocolBufferWriter +{ +public: + /** + * @brief Creates a writer for the given stream + */ + ProtocolBufferWriter (tl::OutputStream &stream); + + /** + * @brief Writes a scalar tag with the given value + */ + void write (int tag, float v); + void write (int tag, double v); + void write (int tag, uint32_t v, bool fixed = false); + void write (int tag, int32_t v, bool fixed = false); + void write (int tag, uint64_t v, bool fixed = false); + void write (int tag, int64_t v, bool fixed = false); + void write (int tag, bool b); + void write (int tag, const std::string &s); + + /** + * @brief Returns true if the writer is in counting mode + */ + bool is_counting () const; + + /** + * @brief Initiates a new sequence. See class documentation for details. + */ + void begin_seq (int tag, bool counting); + + /** + * @brief Ends a sequence. See class documentation for details. + */ + void end_seq (); + +private: + size_t count_bytes_for (pb_varint v); + void write_varint (pb_varint v); + + tl::OutputStream m_stream; + size_t m_bytes_counted; + std::vector m_byte_counter_stack; +}; + +} + +#endif From b3ec079c136b3a0c8d145335099c1c481560adc3 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 18:05:45 +0200 Subject: [PATCH 02/22] WIP: implementation of protobuf generic reader and writer --- src/tl/tl/tlProtocolBuffer.cc | 479 ++++++++++++++++++++++++++++++++++ src/tl/tl/tlProtocolBuffer.h | 18 +- 2 files changed, 488 insertions(+), 9 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index e69de29bb..1f676e574 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -0,0 +1,479 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlProtocolBuffer.h" + +namespace tl +{ + +// ---------------------------------------------------------------------------------- + +ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input) + : mp_stream (&input), m_type (PBWireType (0)), m_pos (0), m_pos_before (0) +{ + // .. nothing yet .. +} + +int +ProtocolBufferReader::read_tag () +{ + uint32_t value = 0; + read (value); + m_type = PBWireType (value & 7); + return int (value >> 3); +} + +void +ProtocolBufferReader::skip () +{ + if (m_type == PB_VARINT) { + + while ((*get (1) & 0x80) != 0) { + // continue reading until the byte's MSB is 0 + } + + } else if (m_type == PB_I64) { + + get (8); + + } else if (m_type == PB_I32) { + + get (4); + + } else if (m_type == PB_LEN) { + + size_t value = 0; + read (value); + skip_bytes (value); + + } +} + +void +ProtocolBufferReader::read (double &d) +{ + uint64_t value = 0; + read (value); + d = *reinterpret_cast (&value); +} + +void +ProtocolBufferReader::read (float &f) +{ + uint32_t value = 0; + read (value); + f = *reinterpret_cast (&value); +} + +void +ProtocolBufferReader::read (std::string &s) +{ + size_t value = 0; + read (value); + + s = std::string (); + s.reserve (value); + + const size_t chunk_size = 1024; + while (value > 0) { + size_t l = std::min (chunk_size, value); + s += std::string (get (l), l); + value -= l; + } +} + +void +ProtocolBufferReader::read (uint32_t &ui32, bool fixed) +{ + if (! fixed) { + + pb_varint ui64 = read_varint (); + if (ui64 > std::numeric_limits::max ()) { + error (tl::to_string (tr ("32 bit value overflow"))); + } + + ui32 = uint32_t (ui64); + + } else { + + ui32 = 0; + for (unsigned int i = 0; i < sizeof (ui32) && ! at_end (); ++i) { + ui32 <<= 8; + ui32 |= (unsigned char) *get (1); + } + + } +} + +void +ProtocolBufferReader::read (int32_t &i32, bool fixed) +{ + uint32_t ui32; + read (ui32, fixed); + + if (! fixed) { + if (ui32 & 1) { + i32 = -(int32_t (ui32 >> 1) + 1); + } else { + i32 = int32_t (ui32 >> 1); + } + } else { + i32 = int32_t (ui32); + } +} + +void +ProtocolBufferReader::read (uint64_t &ui64, bool fixed) +{ + if (! fixed) { + + ui64 = read_varint (); + + } else { + + ui64 = 0; + for (unsigned int i = 0; i < sizeof (ui64) && ! at_end (); ++i) { + ui64 <<= 8; + ui64 |= (unsigned char) *get (1); + } + + } +} + +void +ProtocolBufferReader::read (int64_t &i64, bool fixed) +{ + uint64_t ui64; + read (ui64, fixed); + + if (! fixed) { + if (ui64 & 1) { + i64 = -(int64_t (ui64 >> 1) + 1); + } else { + i64 = int64_t (ui64 >> 1); + } + } else { + i64 = int64_t (ui64); + } +} + +void +ProtocolBufferReader::read (bool &b) +{ + uint32_t ui32; + read (ui32); + b = (ui32 != 0); +} + +void +ProtocolBufferReader::open () +{ + size_t value = 0; + read (value); + m_seq_counts.push_back (value); +} + +void +ProtocolBufferReader::close () +{ + if (! m_seq_counts.empty ()) { + skip_bytes (m_seq_counts.back ()); + m_seq_counts.pop_back (); + } +} + +bool +ProtocolBufferReader::at_end () const +{ + if (m_seq_counts.empty ()) { + const char *cp = mp_stream->get (1); + if (cp) { + mp_stream->unget (1); + return false; + } else { + return true; + } + } else { + return m_seq_counts.back () == 0; + } +} + +const char * +ProtocolBufferReader::get (size_t n) +{ + m_pos_before = m_pos; + m_pos += n; + if (! m_seq_counts.empty ()) { + if (m_seq_counts.back () < n) { + error (tl::to_string (tr ("sequence underflow"))); + } + m_seq_counts.back () -= n; + } + + const char *cp = mp_stream->get (n); + if (! cp) { + error (tl::to_string (tr ("unexpected end of file"))); + } + + return cp; +} + +pb_varint +ProtocolBufferReader::read_varint () +{ + pb_varint v = 0; + + while (true) { + const char *cp = get (1); + if (! cp) { + error (tl::to_string (tr ("unexpected end of file"))); + } + if ((v & 0xfe00000000000000l) != 0) { + error (tl::to_string (tr ("64 bit integer overflow"))); + } + v <<= 7; + v |= (unsigned char) (*cp & 0x7f); + if ((*cp & 0x80) == 0) { + break; + } + } + + return v; +} + +void +ProtocolBufferReader::skip_bytes (size_t n) +{ + const size_t chunk_size = 1024; + while (n > 0) { + size_t l = std::min (chunk_size, n); + if (! mp_stream->get (l)) { + error (tl::to_string (tr ("unexpected end of file"))); + } + n -= l; + } +} + +void +ProtocolBufferReader::error (const std::string &msg) +{ + throw ProtocolBufferReaderError (msg + tl::to_string (tr (", in: ")) + mp_stream->source (), m_pos_before); +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) + : mp_stream (&stream), m_bytes_counted (0) +{ + // .. nothing yet .. +} + +void ProtocolBufferWriter::write (int tag, float v) +{ + write (tag, *reinterpret_cast (&v)); +} + +void ProtocolBufferWriter::write (int tag, double v) +{ + write (tag, *reinterpret_cast (&v)); +} + +void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) +{ + if (fixed) { + + write_varint (pb_varint ((tag << 3) + PB_I32)); + + if (is_counting ()) { + + m_byte_counter_stack.back () += sizeof (v); + + } else { + + char b[sizeof (v)]; + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[sizeof (v) - 1 - i] = (char) v; + v >>= 8; + } + + mp_stream->put (b, sizeof (v)); + + } + + } else { + + write_varint (pb_varint ((tag << 3) + PB_VARINT)); + write_varint (pb_varint (v)); + + } +} + +void ProtocolBufferWriter::write (int tag, int32_t v, bool fixed) +{ + if (fixed) { + write (tag, uint32_t (v), true); + } else { + if (v < 0) { + write (tag, ((uint32_t (-v) - 1) << 1) + 1, false); + } else { + write (tag, uint32_t (v), false); + } + } +} + +void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) +{ + if (fixed) { + + write_varint (pb_varint ((tag << 3) + PB_I64)); + + if (is_counting ()) { + + m_byte_counter_stack.back () += sizeof (v); + + } else { + + char b[sizeof (v)]; + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[sizeof (v) - 1 - i] = (char) v; + v >>= 8; + } + + mp_stream->put (b, sizeof (v)); + + } + + } else { + + write_varint (pb_varint ((tag << 3) + PB_VARINT)); + write_varint (pb_varint (v)); + + } +} + +void ProtocolBufferWriter::write (int tag, int64_t v, bool fixed) +{ + if (fixed) { + write (tag, uint64_t (v), true); + } else { + if (v < 0) { + write (tag, ((uint64_t (-v) - 1) << 1) + 1, false); + } else { + write (tag, uint64_t (v), false); + } + } +} + +void ProtocolBufferWriter::write (int tag, bool b) +{ + write (tag, uint32_t (b ? 1 : 0)); +} + +void ProtocolBufferWriter::write (int tag, const std::string &s) +{ + write_varint (pb_varint ((tag << 3) + PB_LEN)); + write_varint (s.size ()); + + if (is_counting ()) { + m_byte_counter_stack.back () += s.size (); + } else { + mp_stream->put (s.c_str (), s.size ()); + } +} + +bool ProtocolBufferWriter::is_counting () const +{ + return ! m_byte_counter_stack.empty (); +} + +void ProtocolBufferWriter::begin_seq (int tag, bool counting) +{ + if (counting) { + + // header only + m_byte_counter_stack.push_back (0); + write_varint (pb_varint ((tag << 3) + PB_LEN)); + + // body only + m_byte_counter_stack.push_back (0); + + } else { + + write_varint (pb_varint ((tag << 3) + PB_LEN)); + write_varint (m_bytes_counted); + + } +} + +void ProtocolBufferWriter::end_seq () +{ + if (is_counting ()) { + + m_bytes_counted = m_byte_counter_stack.back (); + m_byte_counter_stack.pop_back (); + + // just for adding the required bytes + write_varint (m_bytes_counted); + + // now add header bytes + m_bytes_counted += m_byte_counter_stack.back (); + m_byte_counter_stack.pop_back (); + + } +} + +void +ProtocolBufferWriter::write_varint (pb_varint v) +{ + if (is_counting ()) { + + size_t n = 0; + while (true) { + ++n; + if (v < 0x80) { + break; + } + v >>= 7; + } + + m_byte_counter_stack.back () += n; + + } else { + + char b[16]; + char *cp = b; + while (true) { + if (v < 0x80) { + *cp++ = char (v); + break; + } else { + *cp++ = (char (v) | 0x80); + } + v >>= 7; + } + + m_byte_counter_stack.back () += (cp - b); + + } +} + +} diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index 1b461a3f2..011f3b7a7 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -30,7 +30,7 @@ namespace tl { // Representation of VARINT values -typedef size_t pb_varint; +typedef uint64_t pb_varint; enum PBWireType { @@ -55,7 +55,7 @@ public: virtual std::string msg () const { - return basic_msg () + tl::to_string (tr (", at position ") + tl::to_string (m_position)); + return basic_msg () + tl::to_string (tr (", at position ")) + tl::to_string (m_position); } private: @@ -97,7 +97,7 @@ public: /** * @brief Reads a new tag - * + * This method will also set the current write type. * @returns The message ID */ int read_tag (); @@ -178,7 +178,7 @@ public: * After calling "open", the parser will continue reading messages, but * "at_end" will report true on the end of the sequence, not at the end of the * file. - * Thie method will throw an exception if not in a message of LEN type. + * This method will throw an exception if not in a message of LEN type. */ void open (); @@ -195,15 +195,16 @@ public: bool at_end () const; private: - tl::InputStream m_stream; + tl::InputStream *mp_stream; PBWireType m_type; size_t m_pos; - std::vector m_block_end; + size_t m_pos_before; + std::vector m_seq_counts; pb_varint read_varint (); - bool has_bytes (size_t n); void skip_bytes (size_t n); void error (const std::string &msg); + const char *get (size_t n); }; /** @@ -259,10 +260,9 @@ public: void end_seq (); private: - size_t count_bytes_for (pb_varint v); void write_varint (pb_varint v); - tl::OutputStream m_stream; + tl::OutputStream *mp_stream; size_t m_bytes_counted; std::vector m_byte_counter_stack; }; From 4e5da4ee9eb579d4e28003e76222572ea88947a8 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 18:39:39 +0200 Subject: [PATCH 03/22] WIP --- src/tl/tl/tl.pro | 2 + src/tl/tl/tlProtocolBufferStruct.cc | 31 + src/tl/tl/tlProtocolBufferStruct.h | 1786 +++++++++++++++++++++++++++ 3 files changed, 1819 insertions(+) create mode 100644 src/tl/tl/tlProtocolBufferStruct.cc create mode 100644 src/tl/tl/tlProtocolBufferStruct.h diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 27b27f14a..0773588d1 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -30,6 +30,7 @@ SOURCES = \ tlProgress.cc \ tlPixelBuffer.cc \ tlProtocolBuffer.cc \ + tlProtocolBufferStruct.cc \ tlResources.cc \ tlScriptError.cc \ tlSleep.cc \ @@ -87,6 +88,7 @@ HEADERS = \ tlProgress.h \ tlPixelBuffer.h \ tlProtocolBuffer.h \ + tlProtocolBufferStruct.h \ tlResources.h \ tlReuseVector.h \ tlScriptError.h \ diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc new file mode 100644 index 000000000..2749ea44b --- /dev/null +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -0,0 +1,31 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlProtocolBufferStruct.h" + + +namespace tl +{ + + + +} diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h new file mode 100644 index 000000000..65ca763ce --- /dev/null +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -0,0 +1,1786 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#ifndef HDR_tlProtocolBufferStruct +#define HDR_tlProtocolBufferStruct + +#include "tlCommon.h" +#include "tlProtocolBuffer.h" + +namespace tl +{ + +/** + * @brief A basic PB parser error exception class + */ + +class TL_PUBLIC PBException : public tl::Exception +{ +public: + PBException (const char *msg) + : Exception (tl::to_string (tr ("Protocol buffer parser error: %s")).c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + PBException (const std::string &msg) + : Exception (fmt (0).c_str (), msg.c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + /** + * @brief Raw (unprefixed) message of the PB parser + */ + const std::string & + raw_msg () const + { + return m_msg; + } + +protected: + PBException (const std::string &msg, size_t pos) + : Exception (fmt (pos).c_str (), msg.c_str (), pos), + m_msg (msg) + { + // .. nothing yet .. + } + +private: + std::string m_msg; + + static std::string fmt (size_t pos) + { + if (pos == 0) { + return tl::to_string (tr ("Protocol buffer parser error: %s")).c_str (); + } else { + return tl::to_string (tr ("Protocol buffer parser error: %s at position %ul")).c_str (); + } + } +}; + +/** + * @brief A PB parser error exception class that additionally provides line and column information + */ + +class TL_PUBLIC PBLocatedException : public PBException +{ +public: + PBLocatedException (const std::string &msg, size_t pos) + : PBException (msg, pos), + m_pos (pos) + { + // .. nothing yet .. + } + + /** + * @brief Line number information of the exception + */ + size_t pos () const + { + return m_pos; + } + +private: + size_t m_pos; +}; + +/** + * @brief An object wrapper base class for target object management + * + * Implementations of this class through the PBReaderProxy templates + * manage pointers to certain objects. + */ + +class TL_PUBLIC PBReaderProxyBase +{ +public: + PBReaderProxyBase () { } + virtual ~PBReaderProxyBase () { } + virtual void release () = 0; + virtual void detach () = 0; +}; + +/** + * @brief An object wrapper base class for target object management specialized to a certain class + */ + +template +class TL_PUBLIC_TEMPLATE PBReaderProxy + : public PBReaderProxyBase +{ +public: + PBReaderProxy (Obj *obj, bool owns_obj) + : mp_obj (obj), m_owns_obj (owns_obj) + { } + + virtual ~PBReaderProxy () { } + + virtual void release () + { + if (m_owns_obj && mp_obj) { + delete mp_obj; + } + mp_obj = 0; + } + + virtual void detach () + { + m_owns_obj = false; + } + + Obj *ptr () const + { + return mp_obj; + } + +private: + Obj *mp_obj; + bool m_owns_obj; +}; + +/** + * @brief Helper class: A class tag + */ + +template +struct PBObjTag +{ + PBObjTag() { } + typedef Obj obj; +}; + +/** + * @brief Helper class: The reader state + * + * The reader state mainly consists of a stack of objects being parsed. + */ + +class TL_PUBLIC PBReaderState +{ +public: + /** + * @brief Default constructor + */ + PBReaderState (); + + /** + * @brief Destructor + */ + ~PBReaderState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (PBObjTag /*tag*/) + { + m_objects.push_back (new PBReaderProxy (new Obj (), true)); + } + + /** + * @brief Push an existing object on the stack + */ + template + void push (Obj *obj) + { + m_objects.push_back (new PBReaderProxy (obj, false)); + } + + /** + * @brief Push an existing object on the stack with the ownership flag + */ + template + void push (Obj *obj, bool owner) + { + m_objects.push_back (new PBReaderProxy (obj, owner)); + } + + /** + * @brief Get the top object + */ + template + Obj *back (PBObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Get the top object and release + */ + template + Obj *detach_back (PBObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->detach (); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Pop an object from the stack + */ + template + void pop (PBObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->release (); + delete m_objects.back (); + m_objects.pop_back (); + } + + /** + * @brief Empty predicate: true, if no more object is on the stack + */ + bool empty () const + { + return m_objects.empty (); + } + + /** + * @brief Obtain the parent object from the stack + */ + template + Obj *parent (PBObjTag /*tag*/) + { + tl_assert (m_objects.size () > 1); + return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); + } + +private: + std::vector m_objects; +}; + +// ----------------------------------------------------------------- +// The C++ structure definition interface (for use cases see tlProtocolBufferStruct.ut) + +class TL_PUBLIC PBElementBase; + +struct pass_by_value_tag { + pass_by_value_tag () { } +}; + +struct pass_by_ref_tag { + pass_by_ref_tag () { } +}; + +/** + * @brief The PB parser class + * This is the main entry point. It will take a PB reader, the structure (in "root") and + * a reader state initialized with the top level object. + */ +class TL_PUBLIC PBParser +{ +public: + PBParser (); + ~PBParser (); + + void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state); + + template + void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, Obj &obj) + { + PBReaderState rs; + rs.push (&obj); + parse (reader, root, &rs); + } + +private: + std::vector m_stack; + const PBElementBase *mp_root; + PBReaderState *mp_state; +}; + +/** + * @brief PBElementProxy element of the PB structure definition + * + * The purpose of this class is to provide a wrapper around the + * different derivations of the PBElementBase class, so we can + * pack them into a vector. The proxy objects can be copied + * duplicating the wrapped objects with their "clone" methods. + * The proxy object can be given the pointer to the element + * in two ways: the reference version creates a duplicate, the + * pointer version transfers the ownership of the PBElementBase + * object. + */ + +class TL_PUBLIC PBElementProxy +{ +public: + PBElementProxy (const PBElementProxy &d); + PBElementProxy (const PBElementBase &d); + PBElementProxy (PBElementBase *d); + + ~PBElementProxy (); + + PBElementBase *operator-> () const + { + return mp_ptr; + } + + PBElementBase *get () const + { + return mp_ptr; + } + +private: + PBElementBase *mp_ptr; +}; + +/** + * @brief A list of PB elements + * + * This class provides a list of PB elements. + * A list can be created by using the + operator and + * supplying various objects based on PBElementBase. + */ + +class TL_PUBLIC PBElementList +{ +public: + typedef std::list children_list; + typedef children_list::const_iterator iterator; + + PBElementList () + { + // .. nothing yet .. + } + + PBElementList (const PBElementBase &e) + { + m_elements.push_back (PBElementProxy (e)); + } + + PBElementList (PBElementBase *e) + { + if (e) { + m_elements.push_back (PBElementProxy (e)); + } + } + + PBElementList (const PBElementList &d, const PBElementBase &e) + : m_elements (d.m_elements) + { + m_elements.push_back (PBElementProxy (e)); + } + + PBElementList (const PBElementList &d, PBElementBase *e) + : m_elements (d.m_elements) + { + if (e) { + m_elements.push_back (PBElementProxy (e)); + } + } + + void append (const PBElementBase &e) + { + m_elements.push_back (PBElementProxy (e)); + } + + void append (PBElementBase *e) + { + if (e) { + m_elements.push_back (PBElementProxy (e)); + } + } + + iterator begin () const + { + return m_elements.begin (); + } + + iterator end () const + { + return m_elements.end (); + } + + static PBElementList empty () + { + return PBElementList (); + } + +private: + std::list m_elements; +}; + +/** + * @brief Helper class: A stack of const objects being written + */ + +class TL_PUBLIC PBWriterState +{ +public: + /** + * @brief Default constructor + */ + PBWriterState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (const Obj *obj) + { + m_objects.push_back (obj); + } + + /** + * @brief Pop an object from the stack + */ + template + const Obj *pop (PBObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + const Obj *obj = reinterpret_cast (m_objects.back ()); + m_objects.pop_back (); + return obj; + } + + /** + * @brief Obtain the parent object from the stack + */ + template + const Obj *back (PBObjTag /*tag*/) + { + tl_assert (m_objects.size () > 0); + return reinterpret_cast (m_objects.end () [-1]); + } + +private: + std::vector m_objects; +}; + +/** + * @brief The PB element base object + * + * This class is the base class for objects implementing + * the parser handler semantics. The basic methods are + * create (create an actual object), read (read value), + * finish (finalize the actual object). + * + * Writing of objects is also handled through this class. + * The basic method is "write". + */ + +class TL_PUBLIC PBElementBase +{ +public: + typedef PBElementList::iterator iterator; + + PBElementBase (int tag, const PBElementList &children) + : m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) + { + // .. nothing yet .. + } + + PBElementBase (int tag, const PBElementList *children) + : m_tag (tag), mp_children (children), m_owns_child_list (false) + { + // .. nothing yet .. + } + + PBElementBase (const PBElementBase &d) + : m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) + { + if (m_owns_child_list) { + mp_children = new PBElementList (*d.mp_children); + } else { + mp_children = d.mp_children; + } + } + + virtual ~PBElementBase () + { + if (m_owns_child_list) { + delete const_cast (mp_children); + mp_children = 0; + } + } + + virtual PBElementBase *clone () const = 0; + +#if 0 // @@@ + virtual void create (const PBElementBase *parent, PBReaderState &objs, const std::string &uri, const std::string &lname, const std::string &qname) const = 0; + virtual void cdata (const std::string &cdata, PBReaderState &objs) const = 0; + virtual void finish (const PBElementBase *parent, PBReaderState &objs, const std::string &uri, const std::string &lname, const std::string &qname) const = 0; + + virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter & /*writer*/, PBWriterState & /*objs*/) const { } + virtual bool has_any (PBWriterState & /*objs*/) const { return false; } +#endif + + int tag () const + { + return m_tag; + } + + iterator begin () const + { + return mp_children->begin (); + } + + iterator end () const + { + return mp_children->end (); + } + +private: + int m_tag; + const PBElementList *mp_children; + bool m_owns_child_list; +}; + +/** + * @brief A PB child element + * + * This class is a PB structure component describing a child + * element in the PB tree. There is no limit about the number of + * child elements. + * This object must be provided a pointer to a factory method (in + * the parent's class), a name and a list of children (which can be + * empty). + * Write is a class providing a Obj &operator(Parent &) operator. + * It is supposed to create a new instance of Obj within Parent. + * Read is a class providing a start(const Parent &) method to start + * iterating over the instances, a const Obj &operator() const for + * access and a bool at_end() method to determine if the iterator + * is at the end and next() to increment the iterator. + */ + +template +class TL_PUBLIC_TEMPLATE PBElement + : public PBElementBase +{ +public: + PBElement (const Read &r, const Write &w, int tag, const PBElementList &children) + : PBElementBase (tag, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + PBElement (const Read &r, const Write &w, int tag, const PBElementList *children) + : PBElementBase (tag, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + PBElement (const PBElement &d) + : PBElementBase (d), m_r (d.m_r), m_w (d.m_w) + { + // .. nothing yet .. + } + + virtual PBElementBase *clone () const + { + return new PBElement (*this); + } + +#if 0 // @@@ + virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + PBObjTag tag; + objs.push (tag); + } + + virtual void cdata (const std::string & /*cdata*/, PBReaderState & /*objs*/) const + { + // .. nothing yet .. + } + + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + PBObjTag tag; + PBObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } + + virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + PBElementBase::write_indent (os, indent); + os << "<" << this->name () << ">\n"; + typedef typename Read::tag read_tag_type; + read_tag_type read_tag; + write_obj (r (), os, indent, read_tag, objs); + PBElementBase::write_indent (os, indent); + os << "name () << ">\n"; + r.next (); + } + } + + virtual bool has_any (PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + return (! r.at_end ()); + } +#endif + +private: + Read m_r; + Write m_w; + +#if 0 // @@@ + // this write helper is used if the reader delivers an object by value + void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, PBWriterState &objs) const + { + PBObjTag tag; + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } + + void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, PBWriterState &objs) const + { + PBObjTag tag; + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } +#endif +}; + +/** + * @brief A PB child element with is instantiated with a parent reference + * + * This declares a PB element with a constructor that takes a (object *) arguments + * to the parent. + */ + +template +class TL_PUBLIC_TEMPLATE PBElementWithParentRef + : public PBElementBase +{ +public: + PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList &children) + : PBElementBase (tag, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList *children) + : PBElementBase (tag, children), m_r (r), m_w (w) + { + // .. nothing yet .. + } + + PBElementWithParentRef (const PBElementWithParentRef &d) + : PBElementBase (d), m_r (d.m_r), m_w (d.m_w) + { + // .. nothing yet .. + } + + virtual PBElementBase *clone () const + { + return new PBElementWithParentRef (*this); + } + + #if 0 // @@@ + virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + PBObjTag tag; + PBObjTag parent_tag; + objs.push (new Obj (objs.back (parent_tag)), true); + } + + virtual void cdata (const std::string & /*cdata*/, PBReaderState & /*objs*/) const + { + // .. nothing yet .. + } + + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + PBObjTag tag; + PBObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } + + virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + PBElementBase::write_indent (os, indent); + os << "<" << this->name () << ">\n"; + typedef typename Read::tag read_tag_type; + read_tag_type read_tag; + write_obj (r (), os, indent, read_tag, objs); + PBElementBase::write_indent (os, indent); + os << "name () << ">\n"; + r.next (); + } + } + + virtual bool has_any (PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + return (! r.at_end ()); + } +#endif + +private: + Read m_r; + Write m_w; + +#if 0 // @@@ + // this write helper is used if the reader delivers an object by value + void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, PBWriterState &objs) const + { + PBObjTag tag; + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } + + void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, PBWriterState &objs) const + { + PBObjTag tag; + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, indent + 1, objs); + } + objs.pop (tag); + } +#endif +}; + +/** + * @brief A PB child element mapped to a member + * + * This class is a PB structure component describing a child + * element in the PB tree. The value of the child element + * is directly mapped to a member of the parent's class. + * This object must be provided two adaptors: + * Write is a writer providing an operator() (Parent &, const Value &) + * that adds a new value or sets the value. + * Read is a class providing a start(const Parent &) method to start + * iterating over the instances, a const Value &operator() const for + * access and a bool at_end() method to determine if the iterator + * is at the end and next() to increment the iterator. + */ + +template +class TL_PUBLIC_TEMPLATE PBMember + : public PBElementBase +{ +public: + PBMember (const Read &r, const Write &w, int tag, Converter c = Converter ()) + : PBElementBase (tag, PBElementList::empty ()), m_r (r), m_w (w), m_c (c) + { + // .. nothing yet .. + } + + PBMember (const PBMember &d) + : PBElementBase (d), m_r (d.m_r), m_w (d.m_w), m_c (d.m_c) + { + // .. nothing yet .. + } + + virtual PBElementBase *clone () const + { + return new PBMember (*this); + } + + #if 0 // @@@ + virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + objs.cdata = ""; + } + + virtual void cdata (const std::string &cd, PBReaderState &objs) const + { + objs.cdata += cd; + } + + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + PBObjTag tag; + PBObjTag parent_tag; + + PBReaderState value_obj; + value_obj.push (tag); + + m_c.from_string (objs.cdata, *value_obj.back (tag)); + m_w (*objs.back (parent_tag), value_obj); + + value_obj.pop (tag); + } + + virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (* objs.back (parent_tag)); + while (! r.at_end ()) { + + std::string value = m_c.to_string (r ()); + + write_indent (os, indent); + + if (value.empty ()) { + os << "<" << name () << "/>\n"; + } else { + os << "<" << name () << ">"; + write_string (os, value); + os << "\n"; + } + + r.next (); + + } + } + + virtual bool has_any (PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (*objs.back (parent_tag)); + return (! r.at_end ()); + } +#endif + +private: + Read m_r; + Write m_w; + Converter m_c; +}; + +/** + * @brief The root element of the PB structure + * + * The root element is also the handler implementation. + * It must be supplied a pointer to an actual root object, + * a name and a list of children. + */ + +template +class TL_PUBLIC_TEMPLATE PBStruct + : public PBElementBase +{ +public: + PBStruct (int tag, const PBElementList *children) + : PBElementBase (tag, children) + { + // .. nothing yet .. + } + + PBStruct (int tag, const PBElementList &children) + : PBElementBase (tag, children) + { + // .. nothing yet .. + } + + PBStruct (const PBStruct &d) + : PBElementBase (d) + { + // .. nothing yet .. + } + + virtual PBElementBase *clone () const + { + return new PBStruct (*this); + } + +#if 0 // @@@ + virtual void create (const PBElementBase *, PBReaderState &, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + // .. nothing yet .. + } + + virtual void cdata (const std::string &, PBReaderState &) const + { + // .. nothing yet .. + } + + virtual void finish (const PBElementBase *, PBReaderState &, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + // .. nothing yet .. + } + + void write (tl::OutputStream &os, const Obj &root) const + { + PBWriterState writer_state; + writer_state.push (& root); + + os << "\n"; + os << "<" << this->name () << ">\n"; + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, os, 1, writer_state); + } + os << "name () << ">\n"; + + os.flush (); + } + + void parse (PBSource &source, Obj &root) const + { + PBObjTag tag; + PBParser p; + PBReaderState rs; + rs.push (&root); + PBStructureHandler h (this, &rs); + p.parse (source, h); + rs.pop (tag); + tl_assert (rs.empty ()); + } +#endif + +private: +#if 0 // @@@ + virtual void write (const PBElementBase*, tl::OutputStream &, int, PBWriterState &) const + { + // .. see write (os) + } +#endif +}; + +/** + * @brief Utility: add a PB element to a list of some + */ +inline PBElementList +operator+ (const PBElementList &l, const PBElementBase &e) +{ + return PBElementList (l, e); +} + +/** + * @brief Utility: add a PB element to a list of some + * + * This version transfers the ownership of the PBElementBase object. + */ +inline PBElementList +operator+ (const PBElementList &l, PBElementBase *e) +{ + return PBElementList (l, e); +} + + +// Several adaptor classes needed to implement the various make_* utilities + +template +struct PBMemberDummyWriteAdaptor +{ + PBMemberDummyWriteAdaptor () + { + // .. nothing yet .. + } + + void operator () (Parent &, PBReaderState &) const + { + // .. nothing yet .. + } +}; + +template +struct PBMemberWriteAdaptor +{ + PBMemberWriteAdaptor (Value Parent::*member) + : mp_member (member) + { + // .. nothing yet .. + } + + void operator () (Parent &owner, PBReaderState &reader) const + { + PBObjTag tag; + owner.*mp_member = *reader.back (tag); + } + +private: + Value Parent::*mp_member; +}; + +template +struct PBMemberAccRefWriteAdaptor +{ + PBMemberAccRefWriteAdaptor (void (Parent::*member) (const Value &)) + : mp_member (member) + { + // .. nothing yet .. + } + + void operator () (Parent &owner, PBReaderState &reader) const + { + PBObjTag tag; + (owner.*mp_member) (*reader.back (tag)); + } + +private: + void (Parent::*mp_member) (const Value &); +}; + +template +struct PBMemberTransferWriteAdaptor +{ + PBMemberTransferWriteAdaptor (void (Parent::*member) (Value *)) + : mp_member (member) + { + // .. nothing yet .. + } + + void operator () (Parent &owner, PBReaderState &reader) const + { + PBObjTag tag; + (owner.*mp_member) (reader.detach_back (tag)); + } + +private: + void (Parent::*mp_member) (Value *); +}; + +template +struct PBMemberAccWriteAdaptor +{ + PBMemberAccWriteAdaptor (void (Parent::*member) (Value)) + : mp_member (member) + { + // .. nothing yet .. + } + + void operator () (Parent &owner, PBReaderState &reader) const + { + PBObjTag tag; + (owner.*mp_member) (*reader.back (tag)); + } + +private: + void (Parent::*mp_member) (Value); +}; + +template +struct PBMemberDummyReadAdaptor +{ + typedef pass_by_ref_tag tag; + + PBMemberDummyReadAdaptor () + { + // .. nothing yet .. + } + + Value operator () () const + { + return Value (); + } + + bool at_end () const + { + return true; + } + + void start (const Parent &) + { + // .. nothing yet .. + } + + void next () + { + // .. nothing yet .. + } +}; + +template +struct PBMemberReadAdaptor +{ + typedef pass_by_ref_tag tag; + + PBMemberReadAdaptor (Value Parent::*member) + : mp_member (member), mp_owner (0), m_done (false) + { + // .. nothing yet .. + } + + const Value &operator () () const + { + return mp_owner->*mp_member; + } + + bool at_end () const + { + return m_done; + } + + void start (const Parent &owner) + { + mp_owner = &owner; + m_done = false; + } + + void next () + { + m_done = true; + } + +private: + Value Parent::*mp_member; + const Parent *mp_owner; + bool m_done; +}; + +template +struct PBMemberAccRefReadAdaptor +{ + typedef pass_by_ref_tag tag; + + PBMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) + : mp_member (member), mp_owner (0), m_done (false) + { + // .. nothing yet .. + } + + const Value &operator () () const + { + return (mp_owner->*mp_member) (); + } + + bool at_end () const + { + return m_done; + } + + void start (const Parent &owner) + { + mp_owner = &owner; + m_done = false; + } + + void next () + { + m_done = true; + } + +private: + const Value &(Parent::*mp_member) () const; + const Parent *mp_owner; + bool m_done; +}; + +template +struct PBMemberAccReadAdaptor +{ + typedef pass_by_value_tag tag; + + PBMemberAccReadAdaptor (Value (Parent::*member) () const) + : mp_member (member), mp_owner (0), m_done (false) + { + // .. nothing yet .. + } + + Value operator () () const + { + return (mp_owner->*mp_member) (); + } + + bool at_end () const + { + return m_done; + } + + void start (const Parent &owner) + { + mp_owner = &owner; + m_done = false; + } + + void next () + { + m_done = true; + } + +private: + Value (Parent::*mp_member) () const; + const Parent *mp_owner; + bool m_done; +}; + +template +struct PBMemberIterReadAdaptor +{ + typedef pass_by_ref_tag tag; + + PBMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) + : mp_begin (begin), mp_end (end) + { + // .. nothing yet .. + } + + Value operator () () const + { + return *m_iter; + } + + bool at_end () const + { + return m_iter == m_end; + } + + void start (const Parent &parent) + { + m_iter = (parent.*mp_begin) (); + m_end = (parent.*mp_end) (); + } + + void next () + { + ++m_iter; + } + +private: + Iter (Parent::*mp_begin) () const; + Iter (Parent::*mp_end) () const; + Iter m_iter, m_end; +}; + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberDummyReadAdaptor (), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList *children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberDummyReadAdaptor (), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberWriteAdaptor > +pb_make_element (Value Parent::*member, int tag, const PBElementList *children) +{ + return PBElement, PBMemberWriteAdaptor > ( + PBMemberReadAdaptor (member), + PBMemberWriteAdaptor (member), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberDummyReadAdaptor (), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberDummyReadAdaptor (), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccWriteAdaptor > +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccWriteAdaptor > +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccWriteAdaptor > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberWriteAdaptor > +pb_make_element (Value Parent::*member, int tag, const PBElementList &children) +{ + return PBElement, PBMemberWriteAdaptor > ( + PBMemberReadAdaptor (member), + PBMemberWriteAdaptor (member), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberAccRefWriteAdaptor > +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElement, PBMemberAccRefWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +/** + * @brief Utility: create a PBElement object + */ +template +PBElement, PBMemberTransferWriteAdaptor > +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElement, PBMemberTransferWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberAccRefWriteAdaptor > +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberAccRefWriteAdaptor > +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +{ + return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberTransferWriteAdaptor > +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberTransferWriteAdaptor > +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +{ + return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberAccRefWriteAdaptor > +pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +{ + return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberAccRefWriteAdaptor (setter), tag, children); +} + +template +PBElementWithParentRef, PBMemberTransferWriteAdaptor > +pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +{ + return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberTransferWriteAdaptor (setter), tag, children); +} + +/** + * @brief A helper class providing string to value (and back) conversion + */ + +template +struct PBStdConverter +{ + std::string to_string (const Value &v) const + { + return tl::to_string (v); + } + + void from_string (const std::string &s, Value &v) const + { + tl::from_string (s, v); + } +}; + +/** + * @brief Utility: create a PBMember object without read & write capability + */ +template +PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > +pb_make_member (int tag) +{ + return PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > ( + PBMemberDummyReadAdaptor (), + PBMemberDummyWriteAdaptor (), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberWriteAdaptor , PBStdConverter > +pb_make_member (Value Parent::*member, int tag) +{ + return PBMember, PBMemberWriteAdaptor , PBStdConverter > ( + PBMemberReadAdaptor (member), + PBMemberWriteAdaptor (member), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > +pb_make_member (void (Parent::*setter) (const Value &), int tag) +{ + return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( + PBMemberDummyReadAdaptor (), + PBMemberAccRefWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccWriteAdaptor , PBStdConverter > +pb_make_member (void (Parent::*setter) (Value), int tag) +{ + return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( + PBMemberDummyReadAdaptor (), + PBMemberAccWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > +pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag) +{ + return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccWriteAdaptor , PBStdConverter > +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag) +{ + return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccWriteAdaptor , PBStdConverter > +pb_make_member (const Value & (Parent::*getter) () const, void (Parent::*setter) (Value), int tag) +{ + return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag) +{ + return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( + PBMemberAccReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > +pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag) +{ + return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( + PBMemberIterReadAdaptor (begin, end), + PBMemberAccRefWriteAdaptor (setter), tag); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberWriteAdaptor , Converter> +pb_make_member (Value Parent::*member, int tag, Converter conv) +{ + return PBMember, PBMemberWriteAdaptor , Converter> ( + PBMemberReadAdaptor (member), + PBMemberWriteAdaptor (member), tag, conv); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , Converter> +pb_make_member (void (Parent::*setter) (const Value &), int tag, Converter conv) +{ + return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( + PBMemberDummyReadAdaptor (), + PBMemberAccRefWriteAdaptor (setter), tag, conv); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberDummyWriteAdaptor , Converter> +pb_make_member (void (Parent::*setter) (Value), int tag, Converter conv) +{ + return PBMember, PBMemberDummyWriteAdaptor , Converter> ( + PBMemberAccReadAdaptor (setter), + PBMemberDummyWriteAdaptor (), tag, conv); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , Converter> +pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, Converter conv) +{ + return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( + PBMemberAccRefReadAdaptor (getter), + PBMemberAccRefWriteAdaptor (setter), tag, conv); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccWriteAdaptor , Converter> +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, Converter conv) +{ + return PBMember, PBMemberAccWriteAdaptor , Converter> ( + PBMemberAccReadAdaptor (getter), + PBMemberAccWriteAdaptor (setter), tag, conv); +} + +/** + * @brief Utility: create a PBMember object + */ +template +PBMember, PBMemberAccRefWriteAdaptor , Converter> +pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, Converter conv) +{ + return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( + PBMemberIterReadAdaptor (begin, end), + PBMemberAccRefWriteAdaptor (setter), tag, conv); +} + +} + +#endif From b6e38b8fdf4b0856a1bd0f3fea34f5db6ffdcf61 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 20:06:20 +0200 Subject: [PATCH 04/22] WIP --- src/tl/tl/tlProtocolBufferStruct.h | 217 ++++++++++++++------- src/tl/unit_tests/tlProtocolBufferTests.cc | 206 +++++++++++++++++++ src/tl/unit_tests/unit_tests.pro | 1 + 3 files changed, 350 insertions(+), 74 deletions(-) create mode 100644 src/tl/unit_tests/tlProtocolBufferTests.cc diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index 65ca763ce..1c7a6d316 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -298,14 +298,6 @@ public: void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state); - template - void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, Obj &obj) - { - PBReaderState rs; - rs.push (&obj); - parse (reader, root, &rs); - } - private: std::vector m_stack; const PBElementBase *mp_root; @@ -525,10 +517,11 @@ public: virtual void cdata (const std::string &cdata, PBReaderState &objs) const = 0; virtual void finish (const PBElementBase *parent, PBReaderState &objs, const std::string &uri, const std::string &lname, const std::string &qname) const = 0; - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter & /*writer*/, PBWriterState & /*objs*/) const { } virtual bool has_any (PBWriterState & /*objs*/) const { return false; } #endif + virtual void write (const PBElementBase*, tl::ProtocolBufferWriter &, PBWriterState &) const { } + int tag () const { return m_tag; @@ -641,32 +634,64 @@ public: } #endif + virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const + { + PBObjTag parent_tag; + + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + typedef typename Read::tag read_tag_type; + read_tag_type read_tag; + write_obj (r (), tag (), writer, read_tag, objs); + r.next (); + } + } + +protected: + Read &rear () { return m_r; } + Write &write () { return m_w; } + private: Read m_r; Write m_w; -#if 0 // @@@ // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, PBWriterState &objs) const + void write_obj (Obj obj, int tag, tl::ProtocolBufferWriter &writer, tl::pass_by_value_tag, PBWriterState &objs) const { - PBObjTag tag; - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); + PBObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (! writer.is_counting ()) { + break; + } } - objs.pop (tag); } - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, PBWriterState &objs) const + void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriter &writer, tl::pass_by_ref_tag, PBWriterState &objs) const { - PBObjTag tag; - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); + PBObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (! writer.is_counting ()) { + break; + } } - objs.pop (tag); } -#endif }; /** @@ -678,23 +703,23 @@ private: template class TL_PUBLIC_TEMPLATE PBElementWithParentRef - : public PBElementBase + : public PBElement { public: PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList &children) - : PBElementBase (tag, children), m_r (r), m_w (w) + : PBElement (r, w, tag, children) { // .. nothing yet .. } PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList *children) - : PBElementBase (tag, children), m_r (r), m_w (w) + : PBElement (r, w, tag, children) { // .. nothing yet .. } PBElementWithParentRef (const PBElementWithParentRef &d) - : PBElementBase (d), m_r (d.m_r), m_w (d.m_w) + : PBElement (d) { // .. nothing yet .. } @@ -750,33 +775,6 @@ public: return (! r.at_end ()); } #endif - -private: - Read m_r; - Write m_w; - -#if 0 // @@@ - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, PBWriterState &objs) const - { - PBObjTag tag; - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); - } - - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, PBWriterState &objs) const - { - PBObjTag tag; - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); - } -#endif }; /** @@ -874,10 +872,87 @@ public: } #endif + virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const + { + PBObjTag parent_tag; + Read r (m_r); + r.start (* objs.back (parent_tag)); + while (! r.at_end ()) { + write (writer, tag (), r ()); + r.next (); + } + } + private: Read m_r; Write m_w; Converter m_c; + + void write (tl::ProtocolBufferWriter &writer, int tag, float v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, double v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, uint8_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, int8_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, uint16_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, int16_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, uint32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, int32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, uint64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, int64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, bool v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriter &writer, int tag, const std::string &v) const + { + writer.write (tag, v); + } + + template + void write (tl::ProtocolBufferWriter &writer, int tag, const T &v) const + { + writer.write (tag, m_c.to_string (v)); + } }; /** @@ -893,14 +968,14 @@ class TL_PUBLIC_TEMPLATE PBStruct : public PBElementBase { public: - PBStruct (int tag, const PBElementList *children) - : PBElementBase (tag, children) + PBStruct (const PBElementList *children) + : PBElementBase (0, children) { // .. nothing yet .. } - PBStruct (int tag, const PBElementList &children) - : PBElementBase (tag, children) + PBStruct (const PBElementList &children) + : PBElementBase (0, children) { // .. nothing yet .. } @@ -931,42 +1006,36 @@ public: { // .. nothing yet .. } +#endif - void write (tl::OutputStream &os, const Obj &root) const + void write (tl::ProtocolBufferWriter &writer, const Obj &root) const { PBWriterState writer_state; writer_state.push (& root); - os << "\n"; - os << "<" << this->name () << ">\n"; - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, 1, writer_state); - } - os << "name () << ">\n"; + // @@@ writer.write (0, name ()); - os.flush (); + for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, writer, writer_state); + } } - void parse (PBSource &source, Obj &root) const + void parse (tl::ProtocolBufferReader &reader, Obj &root) const { PBObjTag tag; - PBParser p; PBReaderState rs; rs.push (&root); - PBStructureHandler h (this, &rs); - p.parse (source, h); + PBParser h; + h.parse (reader, this, &rs); rs.pop (tag); tl_assert (rs.empty ()); } -#endif private: -#if 0 // @@@ - virtual void write (const PBElementBase*, tl::OutputStream &, int, PBWriterState &) const + virtual void write (const PBElementBase*, tl::ProtocolBufferWriter &, PBWriterState &) const { // .. see write (os) } -#endif }; /** diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc new file mode 100644 index 000000000..de172ae48 --- /dev/null +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -0,0 +1,206 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include "tlProtocolBufferStruct.h" +#include "tlUnitTest.h" +#include "tlFileUtils.h" + +#include +#include + + +struct Child { + Child () : txt (""), d(-1), live(true) { } + ~Child () { tl_assert (live); live = false; } + std::string txt; + double d; + bool live; + bool operator== (const Child &x) const { return txt == x.txt && fabs (d - x.d) < 1e-9 && children == x.children; } + std::vector children; + std::vector::const_iterator begin_children() const { return children.begin (); } + std::vector::const_iterator end_children() const { return children.end (); } + void add_child (const Child &c) { children.push_back (c); } + void add_child_ptr (Child *c) { children.push_back (*c); delete c; } +}; + +struct Root { + long m; + unsigned int mi; + std::vector m_subs; + std::vector m_isubs; + std::vector m_children; + Child m_child; + + Root () : m(0), mi(0) { } + + bool operator== (const Root &x) const { return m == x.m && + mi == x.mi && m_subs == x.m_subs && + m_isubs == x.m_isubs && m_children == x.m_children + && m_child == x.m_child; } + + int get_mi () const { return mi; } + void set_mi (int i) { mi = i; } + void add_sub (const double &s) { + m_subs.push_back (s); + } + void add_isub (const int &s) { + m_isubs.push_back (s); + } + std::vector::const_iterator begin_subs () const { + return m_subs.begin (); + } + std::vector::const_iterator end_subs () const { + return m_subs.end (); + } + std::vector::const_iterator begin_isubs () const { + return m_isubs.begin (); + } + std::vector::const_iterator end_isubs () const { + return m_isubs.end (); + } + void add_child_ptr (Child *c) { + m_children.push_back (*c); + delete c; + } + void add_child (const Child &c) { + m_children.push_back (c); + } + std::vector::const_iterator begin_children () const { + return m_children.begin (); + } + std::vector::const_iterator end_children () const { + return m_children.end (); + } + void set_child (const Child &child) { m_child = child; } + const Child &get_child () const { return m_child; } +}; + +TEST (1) +{ + Root root; + + tl::PBStruct structure ( + tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, + tl::pb_make_member (&Child::txt, 1) + + tl::pb_make_member (&Child::d, 2) + ) + + tl::pb_make_element (&Root::get_child, &Root::set_child, 4, + tl::pb_make_member (&Child::txt, 1) + + tl::pb_make_member (&Child::d, 2) + ) + + tl::pb_make_member (&Root::m, 5) + + tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + ); + + root.add_sub (0.5); + root.add_sub (7.5); + root.add_isub (42); + root.add_isub (1700000000); + + Child c1; + c1.txt = "c1"; + c1.d = 1.0; + root.add_child (c1); + + Child c2; + c2.txt = "c2"; + c2.d = 2.0; + + Child c21; + c21.txt = "c21"; + c21.d = 2.1; + c2.add_child (c21); + + Child c22; + c22.txt = "c22"; + c22.d = 2.2; + c2.add_child (c22); + + Child c23; + c23.txt = "c23"; + c23.d = 2.3; + c2.add_child (c23); + + root.add_child (c2); + + std::string fn = tl::combine_path (tl::testtmp (), "pb_test.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + } + + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + // TODO: adjust + EXPECT_EQ (error, ""); + EXPECT_EQ (root.m_subs.size (), size_t (2)); + EXPECT_EQ (root.m_subs [0], 1.0); + EXPECT_EQ (root.m_subs [1], -2.5); + EXPECT_EQ (root.m_isubs.size (), size_t (1)); + EXPECT_EQ (root.m_isubs [0], -100); + EXPECT_EQ (root.m, 10); + EXPECT_EQ (root.mi, (unsigned int) 21); + EXPECT_EQ (root.m_children.size (), size_t (2)); + EXPECT_EQ (root.m_children [0].txt, " Text "); + EXPECT_EQ (fabs (root.m_children [0].d - 0.001) < 1e-12, true); + EXPECT_EQ (root.m_children [1].txt, "T2"); + EXPECT_EQ (root.m_children [1].d, -1.0); + EXPECT_EQ (root.m_child.txt, "Single child"); + EXPECT_EQ (root.m_child.d, -1.0); + + // write .. + tl::OutputMemoryStream out; + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + } + + // and read again. + Root rsave (root); + + try { + error.clear (); + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + root = Root (); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == rsave, true); +} diff --git a/src/tl/unit_tests/unit_tests.pro b/src/tl/unit_tests/unit_tests.pro index 3d8b06676..3d3562342 100644 --- a/src/tl/unit_tests/unit_tests.pro +++ b/src/tl/unit_tests/unit_tests.pro @@ -33,6 +33,7 @@ SOURCES = \ tlObjectTests.cc \ tlOptionalTests.cc \ tlPixelBufferTests.cc \ + tlProtocolBufferTests.cc \ tlResourcesTests.cc \ tlReuseVectorTests.cc \ tlStableVectorTests.cc \ From 2f7d572db013d38d7a7242a36902c8c92dae8305 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 21:00:17 +0200 Subject: [PATCH 05/22] WIP --- src/tl/tl/tlProtocolBufferStruct.cc | 42 +++++ src/tl/tl/tlProtocolBufferStruct.h | 255 +++++++++++++--------------- 2 files changed, 163 insertions(+), 134 deletions(-) diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index 2749ea44b..2d074e305 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -26,6 +26,48 @@ namespace tl { +PBParser::PBParser () +{ + // .. nothing yet .. +} +PBParser::~PBParser () +{ + // .. nothing yet .. +} + +void +PBParser::parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state) +{ + mp_state = reader_state; + parse_element (root, reader); +} + +void +PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReader &reader) +{ + while (! reader.at_end ()) { + + int tag = reader.read_tag (); + + const PBElementBase *new_element = 0; + if (parent) { + for (PBElementBase::iterator c = parent->begin (); c != parent->end (); ++c) { + if ((*c)->tag () == tag) { + new_element = (*c).get (); + break; + } + } + } + + if (! new_element) { + reader.skip (); + } else { + new_element->create (parent, *mp_state); + new_element->parse (this, reader); + } + + } +} } diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index 1c7a6d316..abf6a4f3c 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -297,10 +297,14 @@ public: ~PBParser (); void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state); + void parse_element (const PBElementBase *parent, tl::ProtocolBufferReader &reader); + + PBReaderState &reader_state () + { + return *mp_state; + } private: - std::vector m_stack; - const PBElementBase *mp_root; PBReaderState *mp_state; }; @@ -512,15 +516,11 @@ public: virtual PBElementBase *clone () const = 0; -#if 0 // @@@ - virtual void create (const PBElementBase *parent, PBReaderState &objs, const std::string &uri, const std::string &lname, const std::string &qname) const = 0; - virtual void cdata (const std::string &cdata, PBReaderState &objs) const = 0; - virtual void finish (const PBElementBase *parent, PBReaderState &objs, const std::string &uri, const std::string &lname, const std::string &qname) const = 0; + virtual void create (const PBElementBase *parent, PBReaderState &objs) const = 0; + virtual void parse (PBParser *, tl::ProtocolBufferReader &) const = 0; + virtual void finish (const PBElementBase *parent, PBReaderState &objs) const = 0; - virtual bool has_any (PBWriterState & /*objs*/) const { return false; } -#endif - - virtual void write (const PBElementBase*, tl::ProtocolBufferWriter &, PBWriterState &) const { } + virtual void write (const PBElementBase *, tl::ProtocolBufferWriter &, PBWriterState &) const { } int tag () const { @@ -588,19 +588,13 @@ public: return new PBElement (*this); } -#if 0 // @@@ - virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void create (const PBElementBase *, PBReaderState &objs) const { PBObjTag tag; objs.push (tag); } - virtual void cdata (const std::string & /*cdata*/, PBReaderState & /*objs*/) const - { - // .. nothing yet .. - } - - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs) const { PBObjTag tag; PBObjTag parent_tag; @@ -608,32 +602,11 @@ public: objs.pop (tag); } - virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const + virtual void parse (PBParser *parser, tl::ProtocolBufferReader &reader) const { - PBObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - PBElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - PBElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } + parser->parse_element (this, reader); } - virtual bool has_any (PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } -#endif - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const { PBObjTag parent_tag; @@ -729,52 +702,20 @@ public: return new PBElementWithParentRef (*this); } - #if 0 // @@@ - virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void create (const PBElementBase *, PBReaderState &objs) const { PBObjTag tag; PBObjTag parent_tag; objs.push (new Obj (objs.back (parent_tag)), true); } - virtual void cdata (const std::string & /*cdata*/, PBReaderState & /*objs*/) const - { - // .. nothing yet .. - } - - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs) const { PBObjTag tag; PBObjTag parent_tag; m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } - - virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - PBElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - PBElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } - } - - virtual bool has_any (PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } -#endif }; /** @@ -814,18 +755,17 @@ public: return new PBMember (*this); } - #if 0 // @@@ - virtual void create (const PBElementBase *, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void create (const PBElementBase *, PBReaderState &) const { - objs.cdata = ""; + // .. nothing yet .. } - virtual void cdata (const std::string &cd, PBReaderState &objs) const + virtual void finish (const PBElementBase *, PBReaderState &) const { - objs.cdata += cd; + // .. nothing yet .. } - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + virtual void parse (PBParser *parser, tl::ProtocolBufferReader &reader) const { PBObjTag tag; PBObjTag parent_tag; @@ -833,45 +773,12 @@ public: PBReaderState value_obj; value_obj.push (tag); - m_c.from_string (objs.cdata, *value_obj.back (tag)); - m_w (*objs.back (parent_tag), value_obj); + read (reader, *value_obj.back (tag)); + m_w (*parser->reader_state ().back (parent_tag), value_obj); value_obj.pop (tag); } - virtual void write (const PBElementBase * /*parent*/, tl::OutputStream &os, int indent, PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (* objs.back (parent_tag)); - while (! r.at_end ()) { - - std::string value = m_c.to_string (r ()); - - write_indent (os, indent); - - if (value.empty ()) { - os << "<" << name () << "/>\n"; - } else { - os << "<" << name () << ">"; - write_string (os, value); - os << "\n"; - } - - r.next (); - - } - } - - virtual bool has_any (PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } -#endif - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const { PBObjTag parent_tag; @@ -888,6 +795,7 @@ private: Write m_w; Converter m_c; + // write incarnations void write (tl::ProtocolBufferWriter &writer, int tag, float v) const { writer.write (tag, v); @@ -953,6 +861,87 @@ private: { writer.write (tag, m_c.to_string (v)); } + + // read incarnations + void read (tl::ProtocolBufferReader &reader, float &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, double &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, uint8_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReader &reader, int8_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReader &reader, uint16_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReader &reader, int16_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReader &reader, uint32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, int32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, uint64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, int64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, bool &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReader &reader, std::string &v) const + { + reader.read (v); + } + + template + void read (tl::ProtocolBufferReader &reader, const T &v) const + { + std::string vv; + reader.read (vv); + m_c.from_string (vv, v); + } }; /** @@ -991,23 +980,6 @@ public: return new PBStruct (*this); } -#if 0 // @@@ - virtual void create (const PBElementBase *, PBReaderState &, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const - { - // .. nothing yet .. - } - - virtual void cdata (const std::string &, PBReaderState &) const - { - // .. nothing yet .. - } - - virtual void finish (const PBElementBase *, PBReaderState &, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const - { - // .. nothing yet .. - } -#endif - void write (tl::ProtocolBufferWriter &writer, const Obj &root) const { PBWriterState writer_state; @@ -1034,7 +1006,22 @@ public: private: virtual void write (const PBElementBase*, tl::ProtocolBufferWriter &, PBWriterState &) const { - // .. see write (os) + // disable base class implementation + } + + virtual void parse (PBParser *, tl::ProtocolBufferReader &) const + { + // disable base class implementation + } + + virtual void create (const PBElementBase *, PBReaderState &) const + { + // disable base class implementation + } + + virtual void finish (const PBElementBase * /*parent*/, PBReaderState &) const + { + // disable base class implementation } }; From 64b6512bc1f8b87b284762c76724561c3a20cf71 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 22:04:49 +0200 Subject: [PATCH 06/22] WIP: debugging --- src/tl/tl/tlProtocolBuffer.cc | 36 ++--- src/tl/tl/tlProtocolBuffer.h | 8 +- src/tl/tl/tlProtocolBufferStruct.cc | 55 ++++++++ src/tl/unit_tests/tlProtocolBufferTests.cc | 145 +++++++++++++++++++-- 4 files changed, 210 insertions(+), 34 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index 1f676e574..279512474 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -36,6 +36,7 @@ ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input) int ProtocolBufferReader::read_tag () { + m_type = PB_VARINT; uint32_t value = 0; read (value); m_type = PBWireType (value & 7); @@ -102,9 +103,9 @@ ProtocolBufferReader::read (std::string &s) } void -ProtocolBufferReader::read (uint32_t &ui32, bool fixed) +ProtocolBufferReader::read (uint32_t &ui32) { - if (! fixed) { + if (m_type != PB_I32) { pb_varint ui64 = read_varint (); if (ui64 > std::numeric_limits::max ()) { @@ -125,12 +126,12 @@ ProtocolBufferReader::read (uint32_t &ui32, bool fixed) } void -ProtocolBufferReader::read (int32_t &i32, bool fixed) +ProtocolBufferReader::read (int32_t &i32) { uint32_t ui32; - read (ui32, fixed); + read (ui32); - if (! fixed) { + if (m_type != PB_I32) { if (ui32 & 1) { i32 = -(int32_t (ui32 >> 1) + 1); } else { @@ -142,9 +143,9 @@ ProtocolBufferReader::read (int32_t &i32, bool fixed) } void -ProtocolBufferReader::read (uint64_t &ui64, bool fixed) +ProtocolBufferReader::read (uint64_t &ui64) { - if (! fixed) { + if (m_type != PB_I64) { ui64 = read_varint (); @@ -160,12 +161,12 @@ ProtocolBufferReader::read (uint64_t &ui64, bool fixed) } void -ProtocolBufferReader::read (int64_t &i64, bool fixed) +ProtocolBufferReader::read (int64_t &i64) { uint64_t ui64; - read (ui64, fixed); + read (ui64); - if (! fixed) { + if (m_type != PB_I64) { if (ui64 & 1) { i64 = -(int64_t (ui64 >> 1) + 1); } else { @@ -241,6 +242,7 @@ pb_varint ProtocolBufferReader::read_varint () { pb_varint v = 0; + unsigned int s = 0; while (true) { const char *cp = get (1); @@ -250,8 +252,8 @@ ProtocolBufferReader::read_varint () if ((v & 0xfe00000000000000l) != 0) { error (tl::to_string (tr ("64 bit integer overflow"))); } - v <<= 7; - v |= (unsigned char) (*cp & 0x7f); + v |= (pb_varint ((unsigned char) (*cp & 0x7f)) << s); + s += 7; if ((*cp & 0x80) == 0) { break; } @@ -289,12 +291,12 @@ ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) void ProtocolBufferWriter::write (int tag, float v) { - write (tag, *reinterpret_cast (&v)); + write (tag, *reinterpret_cast (&v), true); } void ProtocolBufferWriter::write (int tag, double v) { - write (tag, *reinterpret_cast (&v)); + write (tag, *reinterpret_cast (&v), true); } void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) @@ -335,7 +337,7 @@ void ProtocolBufferWriter::write (int tag, int32_t v, bool fixed) if (v < 0) { write (tag, ((uint32_t (-v) - 1) << 1) + 1, false); } else { - write (tag, uint32_t (v), false); + write (tag, uint32_t (v) << 1, false); } } } @@ -378,7 +380,7 @@ void ProtocolBufferWriter::write (int tag, int64_t v, bool fixed) if (v < 0) { write (tag, ((uint64_t (-v) - 1) << 1) + 1, false); } else { - write (tag, uint64_t (v), false); + write (tag, uint64_t (v) << 1, false); } } } @@ -471,7 +473,7 @@ ProtocolBufferWriter::write_varint (pb_varint v) v >>= 7; } - m_byte_counter_stack.back () += (cp - b); + mp_stream->put (b, cp - b); } } diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index 011f3b7a7..fdb7f7d1b 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -147,25 +147,25 @@ public: * @brief Reads a uint32_t value from the current message * Throws a reader error if the current tag's value is not compatible with a uint32_t. */ - void read (uint32_t &ui32, bool fixed = false); + void read (uint32_t &ui32); /** * @brief Reads a int32_t value from the current message * Throws a reader error if the current tag's value is not compatible with a int32_t. */ - void read (int32_t &i32, bool fixed = false); + void read (int32_t &i32); /** * @brief Reads a uint64_t value from the current message * Throws a reader error if the current tag's value is not compatible with a uint64_t. */ - void read (uint64_t &ui64, bool fixed = false); + void read (uint64_t &ui64); /** * @brief Reads a int64_t value from the current message * Throws a reader error if the current tag's value is not compatible with a int64_t. */ - void read (int64_t &i64, bool fixed = false); + void read (int64_t &i64); /** * @brief Reads a boolean value from the current message diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index 2d074e305..b41f1b459 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -26,6 +26,9 @@ namespace tl { + // -------------------------------------------------------------------- +// PBParser implementation + PBParser::PBParser () { // .. nothing yet .. @@ -70,4 +73,56 @@ PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReader & } } +// -------------------------------------------------------------------- +// PBElementProxy implementation + +PBElementProxy::PBElementProxy (const PBElementProxy &d) + : mp_ptr (d.mp_ptr->clone ()) +{ + // .. nothing yet .. +} + +PBElementProxy::PBElementProxy (const PBElementBase &d) + : mp_ptr (d.clone ()) +{ + // .. nothing yet .. +} + +PBElementProxy::PBElementProxy (PBElementBase *d) + : mp_ptr (d) +{ + // .. nothing yet .. +} + +PBElementProxy::~PBElementProxy () +{ + delete mp_ptr; + mp_ptr = 0; +} + +// -------------------------------------------------------------------- +// PBReaderState implementation + +PBReaderState::PBReaderState () +{ + // .. nothing yet .. +} + +PBReaderState::~PBReaderState () +{ + for (std::vector ::const_iterator o = m_objects.begin (); o != m_objects.end (); ++o) { + (*o)->release (); + delete *o; + } + m_objects.clear (); +} + +// -------------------------------------------------------------------- +// PBWriterState implementation + +PBWriterState::PBWriterState () +{ + // .. nothing yet .. +} + } diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index de172ae48..d5ae4db05 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -27,6 +27,116 @@ #include #include +// @@@ +// Missing: all kind of variants (uint8_t, ...), float + +// Basic tests of reader and writer +TEST (1_BasicTypes) +{ + tl::OutputMemoryStream out; + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + writer.write (1, std::string ("xyz_abc")); + writer.write (2, float (1.5)); + writer.write (3, double (2.5)); + writer.write (4, true); + writer.write (5, int32_t (-100000)); + writer.write (6, int32_t (100000)); + writer.write (7, uint32_t (200000)); + writer.write (8, int64_t (-10000000000)); + writer.write (9, int64_t (10000000000)); + writer.write (10, uint64_t (20000000000)); + writer.write (11, int32_t (-100000), true); + writer.write (12, int32_t (100000), true); + writer.write (13, uint32_t (200000), true); + writer.write (14, int64_t (-10000000000), true); + writer.write (15, int64_t (10000000000), true); + writer.write (16, uint64_t (20000000000), true); + } + + { + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + tl::ProtocolBufferReader reader (is); + + std::string s; + bool b; + float f; + double d; + uint32_t ui32; + int32_t i32; + uint64_t ui64; + int64_t i64; + + EXPECT_EQ (reader.read_tag (), 1); + s = ""; + reader.read (s); + EXPECT_EQ (s, "xyz_abc"); + EXPECT_EQ (reader.read_tag (), 2); + f = 0; + reader.read (f); + EXPECT_EQ (f, 1.5); + EXPECT_EQ (reader.read_tag (), 3); + d = 0; + reader.read (d); + EXPECT_EQ (d, 2.5); + EXPECT_EQ (reader.read_tag (), 4); + b = false; + reader.read (b); + EXPECT_EQ (b, true); + EXPECT_EQ (reader.read_tag (), 5); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, -100000); + EXPECT_EQ (reader.read_tag (), 6); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, 100000); + EXPECT_EQ (reader.read_tag (), 7); + ui32 = 0; + reader.read (ui32); + EXPECT_EQ (ui32, 200000u); + EXPECT_EQ (reader.read_tag (), 8); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, -10000000000); + EXPECT_EQ (reader.read_tag (), 9); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, 10000000000); + EXPECT_EQ (reader.read_tag (), 10); + ui64 = 0; + reader.read (ui64); + EXPECT_EQ (ui64, 20000000000u); + EXPECT_EQ (reader.read_tag (), 11); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, -100000); + EXPECT_EQ (reader.read_tag (), 12); + i32 = 0; + reader.read (i32); + EXPECT_EQ (i32, 100000); + EXPECT_EQ (reader.read_tag (), 13); + ui32 = 0; + reader.read (ui32); + EXPECT_EQ (ui32, 200000u); + EXPECT_EQ (reader.read_tag (), 14); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, -10000000000); + EXPECT_EQ (reader.read_tag (), 15); + i64 = 0; + reader.read (i64); + EXPECT_EQ (i64, 10000000000); + EXPECT_EQ (reader.read_tag (), 16); + ui64 = 0; + reader.read (ui64); + EXPECT_EQ (ui64, 20000000000u); + EXPECT_EQ (reader.at_end (), true); + } +} struct Child { Child () : txt (""), d(-1), live(true) { } @@ -94,7 +204,7 @@ struct Root { const Child &get_child () const { return m_child; } }; -TEST (1) +TEST (100) { Root root; @@ -115,8 +225,9 @@ TEST (1) root.add_sub (0.5); root.add_sub (7.5); - root.add_isub (42); - root.add_isub (1700000000); + root.add_isub (420000000); + root.m = -1700000; + root.set_mi (21); Child c1; c1.txt = "c1"; @@ -144,6 +255,11 @@ TEST (1) root.add_child (c2); + Child sc; + sc.txt = "single"; + sc.d = 4.2e6; + root.set_child (sc); + std::string fn = tl::combine_path (tl::testtmp (), "pb_test.pb"); { @@ -152,6 +268,8 @@ TEST (1) structure.write (writer, root); } + root = Root (); + std::string error; try { tl::InputStream is (fn); @@ -164,19 +282,20 @@ TEST (1) // TODO: adjust EXPECT_EQ (error, ""); EXPECT_EQ (root.m_subs.size (), size_t (2)); - EXPECT_EQ (root.m_subs [0], 1.0); - EXPECT_EQ (root.m_subs [1], -2.5); + EXPECT_EQ (root.m_subs [0], 0.5); + EXPECT_EQ (root.m_subs [1], 7.5); EXPECT_EQ (root.m_isubs.size (), size_t (1)); - EXPECT_EQ (root.m_isubs [0], -100); - EXPECT_EQ (root.m, 10); + EXPECT_EQ (root.m_isubs [0], 420000000); + EXPECT_EQ (root.m, -1700000); EXPECT_EQ (root.mi, (unsigned int) 21); EXPECT_EQ (root.m_children.size (), size_t (2)); - EXPECT_EQ (root.m_children [0].txt, " Text "); - EXPECT_EQ (fabs (root.m_children [0].d - 0.001) < 1e-12, true); - EXPECT_EQ (root.m_children [1].txt, "T2"); - EXPECT_EQ (root.m_children [1].d, -1.0); - EXPECT_EQ (root.m_child.txt, "Single child"); - EXPECT_EQ (root.m_child.d, -1.0); + EXPECT_EQ (root.m_children [0].txt, "c1"); + EXPECT_EQ (root.m_children [1].d, 1.0); + EXPECT_EQ (root.m_children [1].txt, "c2"); + EXPECT_EQ (root.m_children [1].d, 2.0); + EXPECT_EQ (root.m_children [1].end_children () - root.m_children [1].begin_children (), 3); + EXPECT_EQ (root.m_child.txt, "single"); + EXPECT_EQ (root.m_child.d, 4.2e6); // write .. tl::OutputMemoryStream out; From f6da5e11c6d785380d056185c624043bac7a0ebb Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 10 Aug 2024 23:22:54 +0200 Subject: [PATCH 07/22] WIP: debugging, first tests, basic test case working --- src/tl/tl/tlProtocolBuffer.cc | 129 ++++++++++++++++++--- src/tl/tl/tlProtocolBuffer.h | 11 +- src/tl/tl/tlProtocolBufferStruct.cc | 1 + src/tl/tl/tlProtocolBufferStruct.h | 4 +- src/tl/unit_tests/tlProtocolBufferTests.cc | 50 ++++++-- 5 files changed, 167 insertions(+), 28 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index 279512474..92b0292ea 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -21,10 +21,14 @@ */ #include "tlProtocolBuffer.h" +#include "tlLog.h" namespace tl { +// @@@ +// Missing: readers should check for proper wire type (i.e. read(int64)->either VARINT or I64, not I32 + // ---------------------------------------------------------------------------------- ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input) @@ -284,11 +288,17 @@ ProtocolBufferReader::error (const std::string &msg) // ---------------------------------------------------------------------------------- ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) - : mp_stream (&stream), m_bytes_counted (0) + : mp_stream (&stream), m_bytes_counted (0), m_debug (false), m_debug_pos (0) { // .. nothing yet .. } +void ProtocolBufferWriter::set_debug (bool f) +{ + m_debug = f; + m_debug_pos = 0; +} + void ProtocolBufferWriter::write (int tag, float v) { write (tag, *reinterpret_cast (&v), true); @@ -303,7 +313,7 @@ void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) { if (fixed) { - write_varint (pb_varint ((tag << 3) + PB_I32)); + write_varint (pb_varint ((tag << 3) + PB_I32), true); if (is_counting ()) { @@ -311,19 +321,25 @@ void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) } else { + auto vv = v; + char b[sizeof (v)]; for (unsigned int i = 0; i < sizeof (v); ++i) { b[sizeof (v) - 1 - i] = (char) v; v >>= 8; } + if (m_debug) { + dump (b, sizeof (v), "I32", tl::to_string (vv)); + } + mp_stream->put (b, sizeof (v)); } } else { - write_varint (pb_varint ((tag << 3) + PB_VARINT)); + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); write_varint (pb_varint (v)); } @@ -346,7 +362,7 @@ void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) { if (fixed) { - write_varint (pb_varint ((tag << 3) + PB_I64)); + write_varint (pb_varint ((tag << 3) + PB_I64), true); if (is_counting ()) { @@ -354,19 +370,25 @@ void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) } else { + auto vv = v; + char b[sizeof (v)]; for (unsigned int i = 0; i < sizeof (v); ++i) { b[sizeof (v) - 1 - i] = (char) v; v >>= 8; } + if (m_debug) { + dump (b, sizeof (v), "I64", tl::to_string (vv)); + } + mp_stream->put (b, sizeof (v)); } } else { - write_varint (pb_varint ((tag << 3) + PB_VARINT)); + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); write_varint (pb_varint (v)); } @@ -392,13 +414,21 @@ void ProtocolBufferWriter::write (int tag, bool b) void ProtocolBufferWriter::write (int tag, const std::string &s) { - write_varint (pb_varint ((tag << 3) + PB_LEN)); + write_varint (pb_varint ((tag << 3) + PB_LEN), true); write_varint (s.size ()); if (is_counting ()) { + m_byte_counter_stack.back () += s.size (); + } else { + + if (m_debug) { + dump (s.c_str (), s.size (), "(string)", s); + } + mp_stream->put (s.c_str (), s.size ()); + } } @@ -411,16 +441,15 @@ void ProtocolBufferWriter::begin_seq (int tag, bool counting) { if (counting) { - // header only - m_byte_counter_stack.push_back (0); - write_varint (pb_varint ((tag << 3) + PB_LEN)); + if (is_counting ()) { + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + } - // body only m_byte_counter_stack.push_back (0); } else { - write_varint (pb_varint ((tag << 3) + PB_LEN)); + write_varint (pb_varint ((tag << 3) + PB_LEN), true); write_varint (m_bytes_counted); } @@ -434,17 +463,15 @@ void ProtocolBufferWriter::end_seq () m_byte_counter_stack.pop_back (); // just for adding the required bytes - write_varint (m_bytes_counted); - - // now add header bytes - m_bytes_counted += m_byte_counter_stack.back (); - m_byte_counter_stack.pop_back (); + if (is_counting ()) { + write_varint (m_bytes_counted); + } } } void -ProtocolBufferWriter::write_varint (pb_varint v) +ProtocolBufferWriter::write_varint (pb_varint v, bool id) { if (is_counting ()) { @@ -461,6 +488,8 @@ ProtocolBufferWriter::write_varint (pb_varint v) } else { + auto vv = v; + char b[16]; char *cp = b; while (true) { @@ -473,9 +502,75 @@ ProtocolBufferWriter::write_varint (pb_varint v) v >>= 7; } + if (m_debug) { + if (id) { + unsigned int wt = v & 7; + std::string wire_type; + if (wt == PB_EGROUP) { + wire_type = "EGROUP"; + } else if (wt == PB_SGROUP) { + wire_type = "SGROUP"; + } else if (wt == PB_VARINT) { + wire_type = "VARINT"; + } else if (wt == PB_I32) { + wire_type = "I32"; + } else if (wt == PB_I64) { + wire_type = "I64"; + } else if (wt == PB_LEN) { + wire_type = "LEN"; + } + dump (b, cp - b, "(id)", "#" + tl::to_string (vv >> 3) + " " + wire_type); + } else { + dump (b, cp - b, "VARINT", tl::to_string (vv)); + } + } + mp_stream->put (b, cp - b); } } +void +ProtocolBufferWriter::dump (const char *cp, size_t n, const std::string &type, const std::string &value) +{ + bool first = true; + size_t nn = n; + + while (n > 0) { + + std::string line; + if (first) { + line += tl::sprintf ("%08ld", m_debug_pos); + } else { + line += " "; + } + line += " "; + + for (unsigned int i = 0; i < 8; ++i) { + if (n > 0) { + line += tl::sprintf ("%02x", (unsigned int) ((unsigned char) *cp)); + ++cp; + --n; + } else { + line += " "; + } + line += " "; + } + + if (first) { + line += " "; + line += type; + line += " "; + line += value; + } + + tl::info << line; + + first = false; + + } + + m_debug_pos += nn; +} + } diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index fdb7f7d1b..106401719 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -259,12 +259,21 @@ public: */ void end_seq (); + /** + * @brief Enables or disables debug mode + * In debug mode, the stream will be dumped in a human readable form + */ + void set_debug (bool f); + private: - void write_varint (pb_varint v); + void write_varint (pb_varint v, bool id = false); + void dump (const char *cp, size_t n, const std::string &type, const std::string &value); tl::OutputStream *mp_stream; size_t m_bytes_counted; std::vector m_byte_counter_stack; + bool m_debug; + size_t m_debug_pos; }; } diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index b41f1b459..a52bc879d 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -68,6 +68,7 @@ PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReader & } else { new_element->create (parent, *mp_state); new_element->parse (this, reader); + new_element->finish (parent, *mp_state); } } diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index abf6a4f3c..613bc3024 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -604,7 +604,9 @@ public: virtual void parse (PBParser *parser, tl::ProtocolBufferReader &reader) const { + reader.open (); parser->parse_element (this, reader); + reader.close (); } virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const @@ -660,7 +662,7 @@ private: } objs.pop (self_tag); writer.end_seq (); - if (! writer.is_counting ()) { + if (writer.is_counting ()) { break; } } diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index d5ae4db05..fb73a859b 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -155,17 +155,18 @@ struct Child { struct Root { long m; unsigned int mi; + bool b; std::vector m_subs; std::vector m_isubs; std::vector m_children; Child m_child; - Root () : m(0), mi(0) { } + Root () : m(0), mi(0), b(false) { } bool operator== (const Root &x) const { return m == x.m && mi == x.mi && m_subs == x.m_subs && m_isubs == x.m_isubs && m_children == x.m_children - && m_child == x.m_child; } + && m_child == x.m_child && b == x.b; } int get_mi () const { return mi; } void set_mi (int i) { mi = i; } @@ -204,23 +205,26 @@ struct Root { const Child &get_child () const { return m_child; } }; -TEST (100) +TEST (100_BasicStruct) { Root root; + tl::PBElementList child_struct = + tl::pb_make_member (&Child::txt, 1) + + tl::pb_make_member (&Child::d, 2) + + tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, 3, &child_struct); + tl::PBStruct structure ( tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, - tl::pb_make_member (&Child::txt, 1) + - tl::pb_make_member (&Child::d, 2) - ) + + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, &child_struct) + tl::pb_make_element (&Root::get_child, &Root::set_child, 4, tl::pb_make_member (&Child::txt, 1) + tl::pb_make_member (&Child::d, 2) ) + tl::pb_make_member (&Root::m, 5) + - tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + + tl::pb_make_member (&Root::b, 7) ); root.add_sub (0.5); @@ -228,6 +232,7 @@ TEST (100) root.add_isub (420000000); root.m = -1700000; root.set_mi (21); + root.b = true; Child c1; c1.txt = "c1"; @@ -265,6 +270,7 @@ TEST (100) { tl::OutputStream os (fn); tl::ProtocolBufferWriter writer (os); + // For development: writer.set_debug (true); structure.write (writer, root); } @@ -287,10 +293,11 @@ TEST (100) EXPECT_EQ (root.m_isubs.size (), size_t (1)); EXPECT_EQ (root.m_isubs [0], 420000000); EXPECT_EQ (root.m, -1700000); + EXPECT_EQ (root.b, true); EXPECT_EQ (root.mi, (unsigned int) 21); EXPECT_EQ (root.m_children.size (), size_t (2)); EXPECT_EQ (root.m_children [0].txt, "c1"); - EXPECT_EQ (root.m_children [1].d, 1.0); + EXPECT_EQ (root.m_children [0].d, 1.0); EXPECT_EQ (root.m_children [1].txt, "c2"); EXPECT_EQ (root.m_children [1].d, 2.0); EXPECT_EQ (root.m_children [1].end_children () - root.m_children [1].begin_children (), 3); @@ -322,4 +329,29 @@ TEST (100) EXPECT_EQ (error, ""); EXPECT_EQ (root == rsave, true); + + // write empty object + out.clear (); + root = Root (); + + { + tl::OutputStream os (out); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, root); + } + + // and read again. + try { + error.clear (); + tl::InputMemoryStream s2 (out.data (), out.size ()); + tl::InputStream is (s2); + root = Root (); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == Root (), true); } From ee06ac2f7a4c8d2b8b8a56c1fefe61df2c9b10bb Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 14:42:34 +0200 Subject: [PATCH 08/22] WIP: bug fixing, tested binary format with protoscope --- src/tl/tl/tlProtocolBuffer.cc | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index 92b0292ea..58dae9f76 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -121,9 +121,10 @@ ProtocolBufferReader::read (uint32_t &ui32) } else { ui32 = 0; - for (unsigned int i = 0; i < sizeof (ui32) && ! at_end (); ++i) { + const char *cp = get (sizeof (ui32)); + for (unsigned int i = 0; i < sizeof (ui32); ++i) { ui32 <<= 8; - ui32 |= (unsigned char) *get (1); + ui32 |= (unsigned char) cp [sizeof (ui32) - 1 - i]; } } @@ -156,9 +157,10 @@ ProtocolBufferReader::read (uint64_t &ui64) } else { ui64 = 0; - for (unsigned int i = 0; i < sizeof (ui64) && ! at_end (); ++i) { + const char *cp = get (sizeof (ui64)); + for (unsigned int i = 0; i < sizeof (ui64); ++i) { ui64 <<= 8; - ui64 |= (unsigned char) *get (1); + ui64 |= (unsigned char) cp [sizeof (ui64) - 1 - i]; } } @@ -194,6 +196,10 @@ ProtocolBufferReader::open () { size_t value = 0; read (value); + if (! m_seq_counts.empty ()) { + // take out the following bytes from the current sequence + m_seq_counts.back () -= value; + } m_seq_counts.push_back (value); } @@ -325,7 +331,7 @@ void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) char b[sizeof (v)]; for (unsigned int i = 0; i < sizeof (v); ++i) { - b[sizeof (v) - 1 - i] = (char) v; + b[i] = (char) v; v >>= 8; } @@ -374,7 +380,7 @@ void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) char b[sizeof (v)]; for (unsigned int i = 0; i < sizeof (v); ++i) { - b[sizeof (v) - 1 - i] = (char) v; + b[i] = (char) v; v >>= 8; } @@ -464,6 +470,7 @@ void ProtocolBufferWriter::end_seq () // just for adding the required bytes if (is_counting ()) { + m_byte_counter_stack.back () += m_bytes_counted; write_varint (m_bytes_counted); } From 0f067e1dd84732478da0bdccec80a144739e867d Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 15:27:32 +0200 Subject: [PATCH 09/22] WIP --- src/tl/tl/tlProtocolBuffer.cc | 348 ++++++++++++++++++++-------- src/tl/tl/tlProtocolBuffer.h | 248 +++++++++++++++++--- src/tl/tl/tlProtocolBufferStruct.cc | 4 +- src/tl/tl/tlProtocolBufferStruct.h | 40 ++-- 4 files changed, 487 insertions(+), 153 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index 58dae9f76..0099fdf39 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -292,17 +292,78 @@ ProtocolBufferReader::error (const std::string &msg) } // ---------------------------------------------------------------------------------- +// Writer utilities -ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) - : mp_stream (&stream), m_bytes_counted (0), m_debug (false), m_debug_pos (0) +inline uint32_t zigzag_encode (int32_t v) { - // .. nothing yet .. + if (v < 0) { + return ((uint32_t (-v) - 1) << 1) + 1; + } else { + return uint32_t (v) << 1; + } } -void ProtocolBufferWriter::set_debug (bool f) +inline uint64_t zigzag_encode (int64_t v) { - m_debug = f; - m_debug_pos = 0; + if (v < 0) { + return ((uint64_t (-v) - 1) << 1) + 1; + } else { + return uint64_t (v) << 1; + } +} + +inline void little_endian_encode (char *b, uint32_t v) +{ + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[i] = (char) v; + v >>= 8; + } +} + +inline void little_endian_encode (char *b, uint64_t v) +{ + for (unsigned int i = 0; i < sizeof (v); ++i) { + b[i] = (char) v; + v >>= 8; + } +} + +inline size_t count_varint_bytes (pb_varint v) +{ + size_t n = 0; + while (true) { + ++n; + if (v < 0x80) { + break; + } + v >>= 7; + } + return n; +} + +const size_t max_varint_bytes = (sizeof (pb_varint) * 8) / 7 + 1; + +inline size_t varint_encode (char *b, pb_varint v) +{ + char *cp = b; + while (true) { + if (v < 0x80) { + *cp++ = char (v); + break; + } else { + *cp++ = (char (v) | 0x80); + } + v >>= 7; + } + return cp - b; +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) + : mp_stream (&stream), m_bytes_counted (0) +{ + // .. nothing yet .. } void ProtocolBufferWriter::write (int tag, float v) @@ -322,25 +383,11 @@ void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) write_varint (pb_varint ((tag << 3) + PB_I32), true); if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - - auto vv = v; - char b[sizeof (v)]; - for (unsigned int i = 0; i < sizeof (v); ++i) { - b[i] = (char) v; - v >>= 8; - } - - if (m_debug) { - dump (b, sizeof (v), "I32", tl::to_string (vv)); - } - + little_endian_encode (b, v); mp_stream->put (b, sizeof (v)); - } } else { @@ -356,11 +403,7 @@ void ProtocolBufferWriter::write (int tag, int32_t v, bool fixed) if (fixed) { write (tag, uint32_t (v), true); } else { - if (v < 0) { - write (tag, ((uint32_t (-v) - 1) << 1) + 1, false); - } else { - write (tag, uint32_t (v) << 1, false); - } + write (tag, zigzag_encode (v), false); } } @@ -371,25 +414,11 @@ void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) write_varint (pb_varint ((tag << 3) + PB_I64), true); if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - - auto vv = v; - char b[sizeof (v)]; - for (unsigned int i = 0; i < sizeof (v); ++i) { - b[i] = (char) v; - v >>= 8; - } - - if (m_debug) { - dump (b, sizeof (v), "I64", tl::to_string (vv)); - } - + little_endian_encode (b, v); mp_stream->put (b, sizeof (v)); - } } else { @@ -405,11 +434,7 @@ void ProtocolBufferWriter::write (int tag, int64_t v, bool fixed) if (fixed) { write (tag, uint64_t (v), true); } else { - if (v < 0) { - write (tag, ((uint64_t (-v) - 1) << 1) + 1, false); - } else { - write (tag, uint64_t (v) << 1, false); - } + write (tag, zigzag_encode (v), false); } } @@ -424,17 +449,9 @@ void ProtocolBufferWriter::write (int tag, const std::string &s) write_varint (s.size ()); if (is_counting ()) { - m_byte_counter_stack.back () += s.size (); - } else { - - if (m_debug) { - dump (s.c_str (), s.size (), "(string)", s); - } - mp_stream->put (s.c_str (), s.size ()); - } } @@ -477,68 +494,199 @@ void ProtocolBufferWriter::end_seq () } } -void -ProtocolBufferWriter::write_varint (pb_varint v, bool id) +void ProtocolBufferWriter::write_varint (pb_varint v, bool /*id*/) { if (is_counting ()) { - size_t n = 0; - while (true) { - ++n; - if (v < 0x80) { - break; - } - v >>= 7; - } - - m_byte_counter_stack.back () += n; + m_byte_counter_stack.back () += count_varint_bytes (v); } else { - auto vv = v; + char b [max_varint_bytes]; + size_t n = varint_encode (b, v); + mp_stream->put (b, n); - char b[16]; - char *cp = b; - while (true) { - if (v < 0x80) { - *cp++ = char (v); - break; - } else { - *cp++ = (char (v) | 0x80); - } - v >>= 7; + } +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferDumper::ProtocolBufferDumper () + : m_bytes_counted (0), m_debug_pos (0) +{ + // .. nothing yet .. +} + +void ProtocolBufferDumper::write (int tag, float v) +{ + write (tag, *reinterpret_cast (&v), true); +} + +void ProtocolBufferDumper::write (int tag, double v) +{ + write (tag, *reinterpret_cast (&v), true); +} + +void ProtocolBufferDumper::write (int tag, uint32_t v, bool fixed) +{ + if (fixed) { + + write_varint (pb_varint ((tag << 3) + PB_I32), true); + + if (is_counting ()) { + m_byte_counter_stack.back () += sizeof (v); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I32", tl::to_string (v)); } - if (m_debug) { - if (id) { - unsigned int wt = v & 7; - std::string wire_type; - if (wt == PB_EGROUP) { - wire_type = "EGROUP"; - } else if (wt == PB_SGROUP) { - wire_type = "SGROUP"; - } else if (wt == PB_VARINT) { - wire_type = "VARINT"; - } else if (wt == PB_I32) { - wire_type = "I32"; - } else if (wt == PB_I64) { - wire_type = "I64"; - } else if (wt == PB_LEN) { - wire_type = "LEN"; - } - dump (b, cp - b, "(id)", "#" + tl::to_string (vv >> 3) + " " + wire_type); - } else { - dump (b, cp - b, "VARINT", tl::to_string (vv)); - } + } else { + + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v)); + + } +} + +void ProtocolBufferDumper::write (int tag, int32_t v, bool fixed) +{ + if (fixed) { + write (tag, uint32_t (v), true); + } else { + write (tag, zigzag_encode (v), false); + } +} + +void ProtocolBufferDumper::write (int tag, uint64_t v, bool fixed) +{ + if (fixed) { + + write_varint (pb_varint ((tag << 3) + PB_I64), true); + + if (is_counting ()) { + m_byte_counter_stack.back () += sizeof (v); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I64", tl::to_string (v)); } - mp_stream->put (b, cp - b); + } else { + + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v)); + + } +} + +void ProtocolBufferDumper::write (int tag, int64_t v, bool fixed) +{ + if (fixed) { + write (tag, uint64_t (v), true); + } else { + write (tag, zigzag_encode (v), false); + } +} + +void ProtocolBufferDumper::write (int tag, bool b) +{ + write (tag, uint32_t (b ? 1 : 0)); +} + +void ProtocolBufferDumper::write (int tag, const std::string &s) +{ + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + write_varint (s.size ()); + + if (is_counting ()) { + + m_byte_counter_stack.back () += s.size (); + + } else { + + dump (s.c_str (), s.size (), "(string)", s); + + } +} + +bool ProtocolBufferDumper::is_counting () const +{ + return ! m_byte_counter_stack.empty (); +} + +void ProtocolBufferDumper::begin_seq (int tag, bool counting) +{ + if (counting) { + + if (is_counting ()) { + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + } + + m_byte_counter_stack.push_back (0); + + } else { + + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + write_varint (m_bytes_counted); + + } +} + +void ProtocolBufferDumper::end_seq () +{ + if (is_counting ()) { + + m_bytes_counted = m_byte_counter_stack.back (); + m_byte_counter_stack.pop_back (); + + // just for adding the required bytes + if (is_counting ()) { + m_byte_counter_stack.back () += m_bytes_counted; + write_varint (m_bytes_counted); + } } } void -ProtocolBufferWriter::dump (const char *cp, size_t n, const std::string &type, const std::string &value) +ProtocolBufferDumper::write_varint (pb_varint v, bool id) +{ + if (is_counting ()) { + + m_byte_counter_stack.back () += count_varint_bytes (v); + + } else { + + char b[max_varint_bytes]; + size_t n = varint_encode (b, v); + + if (id) { + tl::PBWireType wt = tl::PBWireType (v & 7); + std::string wire_type; + if (wt == PB_EGROUP) { + wire_type = "EGROUP"; + } else if (wt == PB_SGROUP) { + wire_type = "SGROUP"; + } else if (wt == PB_VARINT) { + wire_type = "VARINT"; + } else if (wt == PB_I32) { + wire_type = "I32"; + } else if (wt == PB_I64) { + wire_type = "I64"; + } else if (wt == PB_LEN) { + wire_type = "LEN"; + } + dump (b, n, "(id)", "#" + tl::to_string (v >> 3) + " " + wire_type); + } else { + dump (b, n, "VARINT", tl::to_string (v)); + } + + } +} + +void +ProtocolBufferDumper::dump (const char *cp, size_t n, const std::string &type, const std::string &value) { bool first = true; size_t nn = n; diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index 106401719..61acf4b04 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -67,27 +67,126 @@ private: * * This is a low-level decoder for ProtocolBuffer files. * - * The following LEN-type related concepts need to be implemented by the client code: - * - submessages - * - maps - * - packed repetitions - * - strings + * Use "read_tag" to read a new tag. Unknown tags must be skipped. + * Use "skip" to skip an entry. * - * Unknown tags need to be skipped with "skip". * - * Submessages: if a corresponding tag is encountered with "is_seq()" true, the - * reader code needs to call "open" to enter the sequence and read tags until - * "at_end" is true. Then, call "close" to leave the sequence. + */ +class TL_PUBLIC ProtocolBufferReaderBase +{ +public: + /** + * @brief Constructor + */ + ProtocolBufferReaderBase () + { + // .. nothing yet .. + } + + /** + * @brief Destructor + */ + virtual ~ProtocolBufferReaderBase () + { + // .. nothing yet .. + } + + /** + * @brief Reads a new tag + * This method will also set the current write type. + * @returns The message ID + */ + virtual int read_tag () = 0; + + /** + * @brief Gets the current wire type + */ + virtual PBWireType type () const = 0; + + /** + * @brief Skips the current tag + */ + virtual void skip () = 0; + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a double value. + */ + virtual void read (double &d) = 0; + + /** + * @brief Reads a floating-point value from the current message + * Throws a reader error if the current tag's value is not compatible with a float value. + */ + virtual void read (float &f) = 0; + + /** + * @brief Reads a string from the current message + * Throws a reader error if the current tag's value is not compatible with a string. + */ + virtual void read (std::string &s) = 0; + + /** + * @brief Reads a uint32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint32_t. + */ + virtual void read (uint32_t &ui32) = 0; + + /** + * @brief Reads a int32_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int32_t. + */ + virtual void read (int32_t &i32) = 0; + + /** + * @brief Reads a uint64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a uint64_t. + */ + virtual void read (uint64_t &ui64) = 0; + + /** + * @brief Reads a int64_t value from the current message + * Throws a reader error if the current tag's value is not compatible with a int64_t. + */ + virtual void read (int64_t &i64) = 0; + + /** + * @brief Reads a boolean value from the current message + * Throws a reader error if the current tag's value is not compatible with a bool. + */ + virtual void read (bool &b) = 0; + + /** + * @brief Opens a LEN sequence + * After calling "open", the parser will continue reading messages, but + * "at_end" will report true on the end of the sequence, not at the end of the + * file. + * This method will throw an exception if not in a message of LEN type. + */ + virtual void open () = 0; + + /** + * @brief Closes the LEN sequence + * This method will jump to the end of the sequence and continue reading + * messages from the previous block or file. + */ + virtual void close () = 0; + + /** + * @brief Returns true if at the end of the file or end of a block + */ + virtual bool at_end () const = 0; +}; + +/** + * @brief A reader for ProtocolBuffer files and streams * - * Packed repetitions: same a submessages, but single values are read - * with one of the "read" types. - * - * Maps are read like submessages with key/values as tags 1 and 2. - * - * Strings: if a corresponding tag is encountered, use "read(s)" to read - * the string. "is_seq()" is required to be true, i.e. wire type is LEN. + * This is the reader implementation for binary files + * as described here: + * https://protobuf.dev/programming-guides/encoding/ */ class TL_PUBLIC ProtocolBufferReader + : public ProtocolBufferReaderBase { public: /** @@ -110,16 +209,6 @@ public: return m_type; } - /** - * @brief Returns true, if the current message is a LEN type sequence - * Such messages can be read into strings or "open" can be used on them to - * open a submessage, map or packed repetition. - */ - bool is_seq () const - { - return m_type == PB_LEN; - } - /** * @brief Skips the current tag */ @@ -224,7 +313,62 @@ private: * 4. if "is_counting()" is false, repeat steps 1 to 3 with * "counting" set to false on "begin_seq". */ +class TL_PUBLIC ProtocolBufferWriterBase +{ +public: + /** + * @brief Constructor + */ + ProtocolBufferWriterBase () + { + // .. nothing yet .. + } + + /** + * @brief Destructor + */ + virtual ~ProtocolBufferWriterBase () + { + // .. nothing yet .. + } + + /** + * @brief Writes a scalar tag with the given value + */ + virtual void write (int tag, float v) = 0; + virtual void write (int tag, double v) = 0; + virtual void write (int tag, uint32_t v, bool fixed = false) = 0; + virtual void write (int tag, int32_t v, bool fixed = false) = 0; + virtual void write (int tag, uint64_t v, bool fixed = false) = 0; + virtual void write (int tag, int64_t v, bool fixed = false) = 0; + virtual void write (int tag, bool b) = 0; + virtual void write (int tag, const std::string &s) = 0; + + /** + * @brief Returns true if the writer is in counting mode + */ + virtual bool is_counting () const = 0; + + /** + * @brief Initiates a new sequence. See class documentation for details. + */ + virtual void begin_seq (int tag, bool counting) = 0; + + /** + * @brief Ends a sequence. See class documentation for details. + */ + virtual void end_seq () = 0; +}; + +/** + * @brief A writer for ProtocolBuffer files and streams + * + * This is the writer implementation for binary files + * as described here: + * https://protobuf.dev/programming-guides/encoding/ + */ class TL_PUBLIC ProtocolBufferWriter + : public ProtocolBufferWriterBase { public: /** @@ -259,20 +403,62 @@ public: */ void end_seq (); +private: + void write_varint (pb_varint v, bool id = false); + + tl::OutputStream *mp_stream; + size_t m_bytes_counted; + std::vector m_byte_counter_stack; +}; + +/** + * @brief A writer implementation that dumps the file content to tl::info + * + * This implementation does a halfway job of producing binary files, + * but only insofar it is needed for dumping the binary data. + */ +class TL_PUBLIC ProtocolBufferDumper + : public ProtocolBufferWriterBase +{ +public: /** - * @brief Enables or disables debug mode - * In debug mode, the stream will be dumped in a human readable form + * @brief Creates the writer */ - void set_debug (bool f); + ProtocolBufferDumper (); + + /** + * @brief Writes a scalar tag with the given value + */ + void write (int tag, float v); + void write (int tag, double v); + void write (int tag, uint32_t v, bool fixed = false); + void write (int tag, int32_t v, bool fixed = false); + void write (int tag, uint64_t v, bool fixed = false); + void write (int tag, int64_t v, bool fixed = false); + void write (int tag, bool b); + void write (int tag, const std::string &s); + + /** + * @brief Returns true if the writer is in counting mode + */ + bool is_counting () const; + + /** + * @brief Initiates a new sequence. See class documentation for details. + */ + void begin_seq (int tag, bool counting); + + /** + * @brief Ends a sequence. See class documentation for details. + */ + void end_seq (); private: void write_varint (pb_varint v, bool id = false); void dump (const char *cp, size_t n, const std::string &type, const std::string &value); - tl::OutputStream *mp_stream; size_t m_bytes_counted; std::vector m_byte_counter_stack; - bool m_debug; size_t m_debug_pos; }; diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index a52bc879d..2df45d4ff 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -40,14 +40,14 @@ PBParser::~PBParser () } void -PBParser::parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state) +PBParser::parse (tl::ProtocolBufferReaderBase &reader, const PBElementBase *root, PBReaderState *reader_state) { mp_state = reader_state; parse_element (root, reader); } void -PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReader &reader) +PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBase &reader) { while (! reader.at_end ()) { diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index 613bc3024..560cd89f9 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -296,8 +296,8 @@ public: PBParser (); ~PBParser (); - void parse (tl::ProtocolBufferReader &reader, const PBElementBase *root, PBReaderState *reader_state); - void parse_element (const PBElementBase *parent, tl::ProtocolBufferReader &reader); + void parse (tl::ProtocolBufferReaderBase &reader, const PBElementBase *root, PBReaderState *reader_state); + void parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBase &reader); PBReaderState &reader_state () { @@ -517,7 +517,7 @@ public: virtual PBElementBase *clone () const = 0; virtual void create (const PBElementBase *parent, PBReaderState &objs) const = 0; - virtual void parse (PBParser *, tl::ProtocolBufferReader &) const = 0; + virtual void parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; virtual void finish (const PBElementBase *parent, PBReaderState &objs) const = 0; virtual void write (const PBElementBase *, tl::ProtocolBufferWriter &, PBWriterState &) const { } @@ -602,7 +602,7 @@ public: objs.pop (tag); } - virtual void parse (PBParser *parser, tl::ProtocolBufferReader &reader) const + virtual void parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const { reader.open (); parser->parse_element (this, reader); @@ -767,7 +767,7 @@ public: // .. nothing yet .. } - virtual void parse (PBParser *parser, tl::ProtocolBufferReader &reader) const + virtual void parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const { PBObjTag tag; PBObjTag parent_tag; @@ -865,17 +865,17 @@ private: } // read incarnations - void read (tl::ProtocolBufferReader &reader, float &v) const + void read (tl::ProtocolBufferReaderBase &reader, float &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, double &v) const + void read (tl::ProtocolBufferReaderBase &reader, double &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, uint8_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, uint8_t &v) const { uint32_t vv = 0; reader.read (vv); @@ -883,7 +883,7 @@ private: v = vv; } - void read (tl::ProtocolBufferReader &reader, int8_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, int8_t &v) const { int32_t vv = 0; reader.read (vv); @@ -891,7 +891,7 @@ private: v = vv; } - void read (tl::ProtocolBufferReader &reader, uint16_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, uint16_t &v) const { uint32_t vv = 0; reader.read (vv); @@ -899,7 +899,7 @@ private: v = vv; } - void read (tl::ProtocolBufferReader &reader, int16_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, int16_t &v) const { int32_t vv = 0; reader.read (vv); @@ -907,38 +907,38 @@ private: v = vv; } - void read (tl::ProtocolBufferReader &reader, uint32_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, uint32_t &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, int32_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, int32_t &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, uint64_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, uint64_t &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, int64_t &v) const + void read (tl::ProtocolBufferReaderBase &reader, int64_t &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, bool &v) const + void read (tl::ProtocolBufferReaderBase &reader, bool &v) const { reader.read (v); } - void read (tl::ProtocolBufferReader &reader, std::string &v) const + void read (tl::ProtocolBufferReaderBase &reader, std::string &v) const { reader.read (v); } template - void read (tl::ProtocolBufferReader &reader, const T &v) const + void read (tl::ProtocolBufferReaderBase &reader, const T &v) const { std::string vv; reader.read (vv); @@ -994,7 +994,7 @@ public: } } - void parse (tl::ProtocolBufferReader &reader, Obj &root) const + void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const { PBObjTag tag; PBReaderState rs; @@ -1011,7 +1011,7 @@ private: // disable base class implementation } - virtual void parse (PBParser *, tl::ProtocolBufferReader &) const + virtual void parse (PBParser *, tl::ProtocolBufferReaderBase &) const { // disable base class implementation } From 7357fe50e2aa2d109a5b25d91a80af8ad04c2694 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 15:51:36 +0200 Subject: [PATCH 10/22] WIP --- src/tl/tl/tlProtocolBuffer.cc | 306 +++++++++++------------------ src/tl/tl/tlProtocolBuffer.h | 119 ++++------- src/tl/tl/tlProtocolBufferStruct.h | 40 ++-- 3 files changed, 170 insertions(+), 295 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index 0099fdf39..a6a62dab1 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -360,45 +360,28 @@ inline size_t varint_encode (char *b, pb_varint v) // ---------------------------------------------------------------------------------- -ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) - : mp_stream (&stream), m_bytes_counted (0) +ProtocolBufferWriterBase::ProtocolBufferWriterBase () + : m_bytes_counted (0) { // .. nothing yet .. } -void ProtocolBufferWriter::write (int tag, float v) +ProtocolBufferWriterBase::~ProtocolBufferWriterBase () +{ + // .. nothing yet .. +} + +void ProtocolBufferWriterBase::write (int tag, float v) { write (tag, *reinterpret_cast (&v), true); } -void ProtocolBufferWriter::write (int tag, double v) +void ProtocolBufferWriterBase::write (int tag, double v) { write (tag, *reinterpret_cast (&v), true); } -void ProtocolBufferWriter::write (int tag, uint32_t v, bool fixed) -{ - if (fixed) { - - write_varint (pb_varint ((tag << 3) + PB_I32), true); - - if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - char b[sizeof (v)]; - little_endian_encode (b, v); - mp_stream->put (b, sizeof (v)); - } - - } else { - - write_varint (pb_varint ((tag << 3) + PB_VARINT), true); - write_varint (pb_varint (v)); - - } -} - -void ProtocolBufferWriter::write (int tag, int32_t v, bool fixed) +void ProtocolBufferWriterBase::write (int tag, int32_t v, bool fixed) { if (fixed) { write (tag, uint32_t (v), true); @@ -407,29 +390,7 @@ void ProtocolBufferWriter::write (int tag, int32_t v, bool fixed) } } -void ProtocolBufferWriter::write (int tag, uint64_t v, bool fixed) -{ - if (fixed) { - - write_varint (pb_varint ((tag << 3) + PB_I64), true); - - if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - char b[sizeof (v)]; - little_endian_encode (b, v); - mp_stream->put (b, sizeof (v)); - } - - } else { - - write_varint (pb_varint ((tag << 3) + PB_VARINT), true); - write_varint (pb_varint (v)); - - } -} - -void ProtocolBufferWriter::write (int tag, int64_t v, bool fixed) +void ProtocolBufferWriterBase::write (int tag, int64_t v, bool fixed) { if (fixed) { write (tag, uint64_t (v), true); @@ -438,29 +399,46 @@ void ProtocolBufferWriter::write (int tag, int64_t v, bool fixed) } } -void ProtocolBufferWriter::write (int tag, bool b) +void ProtocolBufferWriterBase::write (int tag, bool b) { write (tag, uint32_t (b ? 1 : 0)); } -void ProtocolBufferWriter::write (int tag, const std::string &s) +void ProtocolBufferWriterBase::write (int tag, uint32_t v, bool fixed) { - write_varint (pb_varint ((tag << 3) + PB_LEN), true); - write_varint (s.size ()); - - if (is_counting ()) { - m_byte_counter_stack.back () += s.size (); + if (fixed) { + write_varint (pb_varint ((tag << 3) + PB_I32), true); + write_fixed (v); } else { - mp_stream->put (s.c_str (), s.size ()); + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v), false); } } -bool ProtocolBufferWriter::is_counting () const +void ProtocolBufferWriterBase::write (int tag, uint64_t v, bool fixed) +{ + if (fixed) { + write_varint (pb_varint ((tag << 3) + PB_I64), true); + write_fixed (v); + } else { + write_varint (pb_varint ((tag << 3) + PB_VARINT), true); + write_varint (pb_varint (v), false); + } +} + +void ProtocolBufferWriterBase::write (int tag, const std::string &s) +{ + write_varint (pb_varint ((tag << 3) + PB_LEN), true); + write_varint (s.size (), false); + write_bytes (s); +} + +bool ProtocolBufferWriterBase::is_counting () const { return ! m_byte_counter_stack.empty (); } -void ProtocolBufferWriter::begin_seq (int tag, bool counting) +void ProtocolBufferWriterBase::begin_seq (int tag, bool counting) { if (counting) { @@ -473,12 +451,12 @@ void ProtocolBufferWriter::begin_seq (int tag, bool counting) } else { write_varint (pb_varint ((tag << 3) + PB_LEN), true); - write_varint (m_bytes_counted); + write_varint (m_bytes_counted, false); } } -void ProtocolBufferWriter::end_seq () +void ProtocolBufferWriterBase::end_seq () { if (is_counting ()) { @@ -488,164 +466,104 @@ void ProtocolBufferWriter::end_seq () // just for adding the required bytes if (is_counting ()) { m_byte_counter_stack.back () += m_bytes_counted; - write_varint (m_bytes_counted); + write_varint (m_bytes_counted, false); } } } +void ProtocolBufferWriterBase::add_bytes (size_t n) +{ + tl_assert (! m_byte_counter_stack.empty ()); + m_byte_counter_stack.back () += n; +} + +// ---------------------------------------------------------------------------------- + +ProtocolBufferWriter::ProtocolBufferWriter (tl::OutputStream &stream) + : mp_stream (&stream) +{ + // .. nothing yet .. +} + +void ProtocolBufferWriter::write_fixed (uint32_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + mp_stream->put (b, sizeof (v)); + } +} + +void ProtocolBufferWriter::write_fixed (uint64_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + mp_stream->put (b, sizeof (v)); + } +} + +void ProtocolBufferWriter::write_bytes (const std::string &s) +{ + if (is_counting ()) { + add_bytes (s.size ()); + } else { + mp_stream->put (s.c_str (), s.size ()); + } +} + void ProtocolBufferWriter::write_varint (pb_varint v, bool /*id*/) { if (is_counting ()) { - - m_byte_counter_stack.back () += count_varint_bytes (v); - + add_bytes (count_varint_bytes (v)); } else { - char b [max_varint_bytes]; size_t n = varint_encode (b, v); mp_stream->put (b, n); - } } // ---------------------------------------------------------------------------------- ProtocolBufferDumper::ProtocolBufferDumper () - : m_bytes_counted (0), m_debug_pos (0) + : m_debug_pos (0) { // .. nothing yet .. } -void ProtocolBufferDumper::write (int tag, float v) +void ProtocolBufferDumper::write_fixed (uint32_t v) { - write (tag, *reinterpret_cast (&v), true); -} - -void ProtocolBufferDumper::write (int tag, double v) -{ - write (tag, *reinterpret_cast (&v), true); -} - -void ProtocolBufferDumper::write (int tag, uint32_t v, bool fixed) -{ - if (fixed) { - - write_varint (pb_varint ((tag << 3) + PB_I32), true); - - if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - char b[sizeof (v)]; - little_endian_encode (b, v); - dump (b, sizeof (v), "I32", tl::to_string (v)); - } - - } else { - - write_varint (pb_varint ((tag << 3) + PB_VARINT), true); - write_varint (pb_varint (v)); - - } -} - -void ProtocolBufferDumper::write (int tag, int32_t v, bool fixed) -{ - if (fixed) { - write (tag, uint32_t (v), true); - } else { - write (tag, zigzag_encode (v), false); - } -} - -void ProtocolBufferDumper::write (int tag, uint64_t v, bool fixed) -{ - if (fixed) { - - write_varint (pb_varint ((tag << 3) + PB_I64), true); - - if (is_counting ()) { - m_byte_counter_stack.back () += sizeof (v); - } else { - char b[sizeof (v)]; - little_endian_encode (b, v); - dump (b, sizeof (v), "I64", tl::to_string (v)); - } - - } else { - - write_varint (pb_varint ((tag << 3) + PB_VARINT), true); - write_varint (pb_varint (v)); - - } -} - -void ProtocolBufferDumper::write (int tag, int64_t v, bool fixed) -{ - if (fixed) { - write (tag, uint64_t (v), true); - } else { - write (tag, zigzag_encode (v), false); - } -} - -void ProtocolBufferDumper::write (int tag, bool b) -{ - write (tag, uint32_t (b ? 1 : 0)); -} - -void ProtocolBufferDumper::write (int tag, const std::string &s) -{ - write_varint (pb_varint ((tag << 3) + PB_LEN), true); - write_varint (s.size ()); - if (is_counting ()) { - - m_byte_counter_stack.back () += s.size (); - + add_bytes (sizeof (v)); } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I32", tl::to_string (v)); + } +} +void ProtocolBufferDumper::write_fixed (uint64_t v) +{ + if (is_counting ()) { + add_bytes (sizeof (v)); + } else { + char b[sizeof (v)]; + little_endian_encode (b, v); + dump (b, sizeof (v), "I64", tl::to_string (v)); + } +} + +void ProtocolBufferDumper::write_bytes (const std::string &s) +{ + if (is_counting ()) { + add_bytes (s.size ()); + } else { dump (s.c_str (), s.size (), "(string)", s); - - } -} - -bool ProtocolBufferDumper::is_counting () const -{ - return ! m_byte_counter_stack.empty (); -} - -void ProtocolBufferDumper::begin_seq (int tag, bool counting) -{ - if (counting) { - - if (is_counting ()) { - write_varint (pb_varint ((tag << 3) + PB_LEN), true); - } - - m_byte_counter_stack.push_back (0); - - } else { - - write_varint (pb_varint ((tag << 3) + PB_LEN), true); - write_varint (m_bytes_counted); - - } -} - -void ProtocolBufferDumper::end_seq () -{ - if (is_counting ()) { - - m_bytes_counted = m_byte_counter_stack.back (); - m_byte_counter_stack.pop_back (); - - // just for adding the required bytes - if (is_counting ()) { - m_byte_counter_stack.back () += m_bytes_counted; - write_varint (m_bytes_counted); - } - } } @@ -654,7 +572,7 @@ ProtocolBufferDumper::write_varint (pb_varint v, bool id) { if (is_counting ()) { - m_byte_counter_stack.back () += count_varint_bytes (v); + add_bytes (count_varint_bytes (v)); } else { diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index 61acf4b04..0d7d8b4d6 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -319,45 +319,52 @@ public: /** * @brief Constructor */ - ProtocolBufferWriterBase () - { - // .. nothing yet .. - } + ProtocolBufferWriterBase (); /** * @brief Destructor */ - virtual ~ProtocolBufferWriterBase () - { - // .. nothing yet .. - } + virtual ~ProtocolBufferWriterBase (); /** - * @brief Writes a scalar tag with the given value + * @brief Writes a scalar element with the given value and tag */ - virtual void write (int tag, float v) = 0; - virtual void write (int tag, double v) = 0; - virtual void write (int tag, uint32_t v, bool fixed = false) = 0; - virtual void write (int tag, int32_t v, bool fixed = false) = 0; - virtual void write (int tag, uint64_t v, bool fixed = false) = 0; - virtual void write (int tag, int64_t v, bool fixed = false) = 0; - virtual void write (int tag, bool b) = 0; - virtual void write (int tag, const std::string &s) = 0; + + // implicit types + void write (int tag, float v); + void write (int tag, double v); + void write (int tag, int32_t v, bool fixed = false); + void write (int tag, int64_t v, bool fixed = false); + void write (int tag, uint32_t v, bool fixed = false); + void write (int tag, uint64_t v, bool fixed = false); + void write (int tag, bool b); + void write (int tag, const std::string &s); /** * @brief Returns true if the writer is in counting mode */ - virtual bool is_counting () const = 0; + bool is_counting () const; /** * @brief Initiates a new sequence. See class documentation for details. */ - virtual void begin_seq (int tag, bool counting) = 0; + void begin_seq (int tag, bool counting); /** * @brief Ends a sequence. See class documentation for details. */ - virtual void end_seq () = 0; + void end_seq (); + +protected: + virtual void write_bytes (const std::string &s) = 0; + virtual void write_fixed (uint32_t v) = 0; + virtual void write_fixed (uint64_t v) = 0; + virtual void write_varint (pb_varint v, bool id) = 0; + void add_bytes (size_t n); + +private: + size_t m_bytes_counted; + std::vector m_byte_counter_stack; }; /** @@ -376,39 +383,14 @@ public: */ ProtocolBufferWriter (tl::OutputStream &stream); - /** - * @brief Writes a scalar tag with the given value - */ - void write (int tag, float v); - void write (int tag, double v); - void write (int tag, uint32_t v, bool fixed = false); - void write (int tag, int32_t v, bool fixed = false); - void write (int tag, uint64_t v, bool fixed = false); - void write (int tag, int64_t v, bool fixed = false); - void write (int tag, bool b); - void write (int tag, const std::string &s); - - /** - * @brief Returns true if the writer is in counting mode - */ - bool is_counting () const; - - /** - * @brief Initiates a new sequence. See class documentation for details. - */ - void begin_seq (int tag, bool counting); - - /** - * @brief Ends a sequence. See class documentation for details. - */ - void end_seq (); +protected: + virtual void write_bytes (const std::string &s); + virtual void write_fixed (uint32_t v); + virtual void write_fixed (uint64_t v); + virtual void write_varint (pb_varint v, bool id = false); private: - void write_varint (pb_varint v, bool id = false); - tl::OutputStream *mp_stream; - size_t m_bytes_counted; - std::vector m_byte_counter_stack; }; /** @@ -426,39 +408,14 @@ public: */ ProtocolBufferDumper (); - /** - * @brief Writes a scalar tag with the given value - */ - void write (int tag, float v); - void write (int tag, double v); - void write (int tag, uint32_t v, bool fixed = false); - void write (int tag, int32_t v, bool fixed = false); - void write (int tag, uint64_t v, bool fixed = false); - void write (int tag, int64_t v, bool fixed = false); - void write (int tag, bool b); - void write (int tag, const std::string &s); - - /** - * @brief Returns true if the writer is in counting mode - */ - bool is_counting () const; - - /** - * @brief Initiates a new sequence. See class documentation for details. - */ - void begin_seq (int tag, bool counting); - - /** - * @brief Ends a sequence. See class documentation for details. - */ - void end_seq (); - -private: - void write_varint (pb_varint v, bool id = false); +protected: + virtual void write_bytes (const std::string &s); + virtual void write_fixed (uint32_t v); + virtual void write_fixed (uint64_t v); + virtual void write_varint (pb_varint v, bool id = false); void dump (const char *cp, size_t n, const std::string &type, const std::string &value); - size_t m_bytes_counted; - std::vector m_byte_counter_stack; +private: size_t m_debug_pos; }; diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index 560cd89f9..68fd22e7e 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -520,7 +520,7 @@ public: virtual void parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; virtual void finish (const PBElementBase *parent, PBReaderState &objs) const = 0; - virtual void write (const PBElementBase *, tl::ProtocolBufferWriter &, PBWriterState &) const { } + virtual void write (const PBElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } int tag () const { @@ -609,7 +609,7 @@ public: reader.close (); } - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const + virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const { PBObjTag parent_tag; @@ -632,7 +632,7 @@ private: Write m_w; // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, int tag, tl::ProtocolBufferWriter &writer, tl::pass_by_value_tag, PBWriterState &objs) const + void write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_value_tag, PBWriterState &objs) const { PBObjTag self_tag; @@ -650,7 +650,7 @@ private: } } - void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriter &writer, tl::pass_by_ref_tag, PBWriterState &objs) const + void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_ref_tag, PBWriterState &objs) const { PBObjTag self_tag; @@ -781,7 +781,7 @@ public: value_obj.pop (tag); } - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriter &writer, PBWriterState &objs) const + virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const { PBObjTag parent_tag; Read r (m_r); @@ -798,68 +798,68 @@ private: Converter m_c; // write incarnations - void write (tl::ProtocolBufferWriter &writer, int tag, float v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, float v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, double v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, double v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, uint8_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint8_t v) const { writer.write (tag, (uint32_t) v); } - void write (tl::ProtocolBufferWriter &writer, int tag, int8_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, int8_t v) const { writer.write (tag, (int32_t) v); } - void write (tl::ProtocolBufferWriter &writer, int tag, uint16_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint16_t v) const { writer.write (tag, (uint32_t) v); } - void write (tl::ProtocolBufferWriter &writer, int tag, int16_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, int16_t v) const { writer.write (tag, (int32_t) v); } - void write (tl::ProtocolBufferWriter &writer, int tag, uint32_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint32_t v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, int32_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, int32_t v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, uint64_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint64_t v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, int64_t v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, int64_t v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, bool v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, bool v) const { writer.write (tag, v); } - void write (tl::ProtocolBufferWriter &writer, int tag, const std::string &v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, const std::string &v) const { writer.write (tag, v); } template - void write (tl::ProtocolBufferWriter &writer, int tag, const T &v) const + void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const { writer.write (tag, m_c.to_string (v)); } @@ -982,7 +982,7 @@ public: return new PBStruct (*this); } - void write (tl::ProtocolBufferWriter &writer, const Obj &root) const + void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const { PBWriterState writer_state; writer_state.push (& root); @@ -1006,7 +1006,7 @@ public: } private: - virtual void write (const PBElementBase*, tl::ProtocolBufferWriter &, PBWriterState &) const + virtual void write (const PBElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const { // disable base class implementation } From 7d3b3dcb143f20f936f948ef9803bf08665b87d5 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 16:55:38 +0200 Subject: [PATCH 11/22] WIP: generalization, more diagnostics and checks --- src/tl/tl/tlProtocolBuffer.cc | 25 ++++++++----- src/tl/tl/tlProtocolBuffer.h | 11 +++++- src/tl/tl/tlProtocolBufferStruct.cc | 15 ++++++++ src/tl/tl/tlProtocolBufferStruct.h | 41 +++++++++++++++++----- src/tl/unit_tests/tlProtocolBufferTests.cc | 8 +++-- 5 files changed, 80 insertions(+), 20 deletions(-) diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index a6a62dab1..b12b8f761 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -26,9 +26,6 @@ namespace tl { -// @@@ -// Missing: readers should check for proper wire type (i.e. read(int64)->either VARINT or I64, not I32 - // ---------------------------------------------------------------------------------- ProtocolBufferReader::ProtocolBufferReader (tl::InputStream &input) @@ -92,6 +89,10 @@ ProtocolBufferReader::read (float &f) void ProtocolBufferReader::read (std::string &s) { + if (m_type != PB_LEN) { + error (tl::to_string (tr ("Expected a LEN wire type for a string"))); + } + size_t value = 0; read (value); @@ -109,7 +110,7 @@ ProtocolBufferReader::read (std::string &s) void ProtocolBufferReader::read (uint32_t &ui32) { - if (m_type != PB_I32) { + if (m_type == PB_VARINT || m_type == PB_LEN) { pb_varint ui64 = read_varint (); if (ui64 > std::numeric_limits::max ()) { @@ -118,7 +119,7 @@ ProtocolBufferReader::read (uint32_t &ui32) ui32 = uint32_t (ui64); - } else { + } else if (m_type == PB_I32) { ui32 = 0; const char *cp = get (sizeof (ui32)); @@ -127,6 +128,8 @@ ProtocolBufferReader::read (uint32_t &ui32) ui32 |= (unsigned char) cp [sizeof (ui32) - 1 - i]; } + } else { + error (tl::to_string (tr ("Expected a VARINT or I32 wire type"))); } } @@ -150,11 +153,11 @@ ProtocolBufferReader::read (int32_t &i32) void ProtocolBufferReader::read (uint64_t &ui64) { - if (m_type != PB_I64) { + if (m_type == PB_VARINT || m_type == PB_LEN) { ui64 = read_varint (); - } else { + } else if (m_type == PB_I64) { ui64 = 0; const char *cp = get (sizeof (ui64)); @@ -163,6 +166,8 @@ ProtocolBufferReader::read (uint64_t &ui64) ui64 |= (unsigned char) cp [sizeof (ui64) - 1 - i]; } + } else { + error (tl::to_string (tr ("Expected a VARINT or I64 wire type"))); } } @@ -194,6 +199,10 @@ ProtocolBufferReader::read (bool &b) void ProtocolBufferReader::open () { + if (m_type != PB_LEN) { + error (tl::to_string (tr ("Expected a LEN wire type for a submessage"))); + } + size_t value = 0; read (value); if (! m_seq_counts.empty ()) { @@ -259,7 +268,7 @@ ProtocolBufferReader::read_varint () if (! cp) { error (tl::to_string (tr ("unexpected end of file"))); } - if ((v & 0xfe00000000000000l) != 0) { + if (s + 7 > 64 && pb_varint ((unsigned char) (*cp & 0x7f)) >> (s + 7 - 64) != 0) { error (tl::to_string (tr ("64 bit integer overflow"))); } v |= (pb_varint ((unsigned char) (*cp & 0x7f)) << s); diff --git a/src/tl/tl/tlProtocolBuffer.h b/src/tl/tl/tlProtocolBuffer.h index 0d7d8b4d6..1b8ab25ce 100644 --- a/src/tl/tl/tlProtocolBuffer.h +++ b/src/tl/tl/tlProtocolBuffer.h @@ -176,6 +176,11 @@ public: * @brief Returns true if at the end of the file or end of a block */ virtual bool at_end () const = 0; + + /** + * @brief Emits an error at the current position + */ + virtual void error (const std::string &msg) = 0; }; /** @@ -283,6 +288,11 @@ public: */ bool at_end () const; + /** + * @brief Emits an error at the current position + */ + void error (const std::string &msg); + private: tl::InputStream *mp_stream; PBWireType m_type; @@ -292,7 +302,6 @@ private: pb_varint read_varint (); void skip_bytes (size_t n); - void error (const std::string &msg); const char *get (size_t n); }; diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index 2df45d4ff..6a675031f 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -74,6 +74,21 @@ PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBa } } +void +PBParser::expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name) +{ + int tag = reader.read_tag (); + if (tag != name_tag) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with ID %d (got %d)")), name_tag, tag)); + } + + std::string n; + reader.read (n); + if (n != name) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with string '%s' (got '%s')")), name, n)); + } +} + // -------------------------------------------------------------------- // PBElementProxy implementation diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index 68fd22e7e..d569622dc 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -299,6 +299,8 @@ public: void parse (tl::ProtocolBufferReaderBase &reader, const PBElementBase *root, PBReaderState *reader_state); void parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBase &reader); + void expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name); + PBReaderState &reader_state () { return *mp_state; @@ -949,9 +951,16 @@ private: /** * @brief The root element of the PB structure * - * The root element is also the handler implementation. - * It must be supplied a pointer to an actual root object, - * a name and a list of children. + * The root object is also a element node. It belongs to an object + * representing the root of the object hierarchy. Correspondingly, + * child elements (members, elements - aka submessages) need to be + * specified. + * + * In addition, the root element has a name and a name tag. + * The name / name tag are emitted as the first element in the + * message, providing a file header with fixed content. This + * allows identifying the file as a encoded protocol buffer + * file for a specific object type. */ template @@ -959,20 +968,20 @@ class TL_PUBLIC_TEMPLATE PBStruct : public PBElementBase { public: - PBStruct (const PBElementList *children) - : PBElementBase (0, children) + PBStruct (const std::string &name, int name_tag, const PBElementList *children) + : PBElementBase (0, children), m_name (name), m_name_tag (name_tag) { // .. nothing yet .. } - PBStruct (const PBElementList &children) - : PBElementBase (0, children) + PBStruct (const std::string &name, int name_tag, const PBElementList &children) + : PBElementBase (0, children), m_name (name), m_name_tag (name_tag) { // .. nothing yet .. } PBStruct (const PBStruct &d) - : PBElementBase (d) + : PBElementBase (d), m_name (d.name ()), m_name_tag (d.name_tag ()) { // .. nothing yet .. } @@ -982,12 +991,22 @@ public: return new PBStruct (*this); } + const std::string &name () const + { + return m_name; + } + + int name_tag () const + { + return m_name_tag; + } + void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const { PBWriterState writer_state; writer_state.push (& root); - // @@@ writer.write (0, name ()); + writer.write (name_tag (), name ()); for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { c->get ()->write (this, writer, writer_state); @@ -1000,6 +1019,7 @@ public: PBReaderState rs; rs.push (&root); PBParser h; + h.expect_header (reader, m_name_tag, m_name); h.parse (reader, this, &rs); rs.pop (tag); tl_assert (rs.empty ()); @@ -1025,6 +1045,9 @@ private: { // disable base class implementation } + + std::string m_name; + int m_name_tag; }; /** diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index fb73a859b..919dcb319 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -214,7 +214,7 @@ TEST (100_BasicStruct) tl::pb_make_member (&Child::d, 2) + tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, 3, &child_struct); - tl::PBStruct structure ( + tl::PBStruct structure ("pbtest-struct", 88888888, tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, &child_struct) + @@ -270,8 +270,12 @@ TEST (100_BasicStruct) { tl::OutputStream os (fn); tl::ProtocolBufferWriter writer (os); - // For development: writer.set_debug (true); structure.write (writer, root); + /* + for debugging: + tl::ProtocolBufferDumper dumper; + structure.write (dumper, root); + */ } root = Root (); From edfac96db2ea1ca7093154e6e8f56cd7bc2b7bc5 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 18:55:51 +0200 Subject: [PATCH 12/22] WIP: converters for protocol buffers --- src/tl/tl/tlProtocolBufferStruct.h | 14 +-- src/tl/unit_tests/tlProtocolBufferTests.cc | 100 +++++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index d569622dc..e3624e3e8 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -863,7 +863,7 @@ private: template void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const { - writer.write (tag, m_c.to_string (v)); + writer.write (tag, m_c.pb_encode (v)); } // read incarnations @@ -940,11 +940,11 @@ private: } template - void read (tl::ProtocolBufferReaderBase &reader, const T &v) const + void read (tl::ProtocolBufferReaderBase &reader, T &v) const { - std::string vv; + typename Converter::pb_type vv; reader.read (vv); - m_c.from_string (vv, v); + m_c.pb_decode (vv, v); } }; @@ -1671,12 +1671,14 @@ pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::* template struct PBStdConverter { - std::string to_string (const Value &v) const + typedef std::string pb_type; + + pb_type pb_encode (const Value &v) const { return tl::to_string (v); } - void from_string (const std::string &s, Value &v) const + void pb_decode (const pb_type &s, Value &v) const { tl::from_string (s, v); } diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index 919dcb319..cf413878e 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -359,3 +359,103 @@ TEST (100_BasicStruct) EXPECT_EQ (error, ""); EXPECT_EQ (root == Root (), true); } + +struct TestClass +{ + enum enum_type { A, B, C }; + + TestClass () : e (A) { } + enum_type e; +}; + +struct TestClassEnumConverter +{ + typedef uint32_t pb_type; + + pb_type pb_encode (TestClass::enum_type e) const + { + switch (e) { + case TestClass::A: + return 17; + case TestClass::B: + return 18; + case TestClass::C: + return 19; + default: + return 0; + } + } + + void pb_decode (uint32_t value, TestClass::enum_type &e) const + { + switch (value) { + case 17: + e = TestClass::A; + break; + case 18: + e = TestClass::B; + break; + case 19: + e = TestClass::C; + break; + default: + e = TestClass::enum_type (0); + break; + } + } +}; + +TEST (101_Converter) +{ + TestClass tc; + + tl::PBStruct structure ("pbtest-tc", 1, + tl::pb_make_member (&TestClass::e, 2, TestClassEnumConverter ()) + ); + + tc.e = TestClass::A; + std::string fn = tl::combine_path (tl::testtmp (), "pb_101a.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, tc); + } + + tc = TestClass (); + + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (tc.e, TestClass::A); + + tc.e = TestClass::B; + + fn = tl::combine_path (tl::testtmp (), "pb_101b.pb"); + + { + tl::OutputStream os (fn); + tl::ProtocolBufferWriter writer (os); + structure.write (writer, tc); + } + + tc = TestClass (); + + error.clear (); + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (tc.e, TestClass::B); +} + From e616b656276c18e4c859245a1cb3f367da69e123 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 19:44:53 +0200 Subject: [PATCH 13/22] More tests for PB, some bug fixes --- src/tl/tl/tlProtocolBuffer.cc | 22 ++- src/tl/unit_tests/tlProtocolBufferTests.cc | 210 ++++++++++++++++++--- testdata/pb/struct1.pb | Bin 0 -> 149 bytes testdata/pb/struct2_invalid_header.pb | Bin 0 -> 147 bytes testdata/pb/struct3_invalid_header.pb | Bin 0 -> 149 bytes testdata/pb/struct4_deviations.pb | Bin 0 -> 184 bytes testdata/pb/struct5_overflow.pb | 2 + testdata/pb/struct6_invalid_type.pb | Bin 0 -> 20 bytes testdata/pb/struct7_invalid_type.pb | Bin 0 -> 16 bytes testdata/pb/struct8_invalid_type.pb | 2 + 10 files changed, 199 insertions(+), 37 deletions(-) create mode 100644 testdata/pb/struct1.pb create mode 100644 testdata/pb/struct2_invalid_header.pb create mode 100644 testdata/pb/struct3_invalid_header.pb create mode 100644 testdata/pb/struct4_deviations.pb create mode 100644 testdata/pb/struct5_overflow.pb create mode 100644 testdata/pb/struct6_invalid_type.pb create mode 100644 testdata/pb/struct7_invalid_type.pb create mode 100644 testdata/pb/struct8_invalid_type.pb diff --git a/src/tl/tl/tlProtocolBuffer.cc b/src/tl/tl/tlProtocolBuffer.cc index b12b8f761..126b0ffdf 100644 --- a/src/tl/tl/tlProtocolBuffer.cc +++ b/src/tl/tl/tlProtocolBuffer.cc @@ -63,8 +63,7 @@ ProtocolBufferReader::skip () } else if (m_type == PB_LEN) { - size_t value = 0; - read (value); + size_t value = read_varint (); skip_bytes (value); } @@ -93,8 +92,7 @@ ProtocolBufferReader::read (std::string &s) error (tl::to_string (tr ("Expected a LEN wire type for a string"))); } - size_t value = 0; - read (value); + size_t value = read_varint (); s = std::string (); s.reserve (value); @@ -110,7 +108,7 @@ ProtocolBufferReader::read (std::string &s) void ProtocolBufferReader::read (uint32_t &ui32) { - if (m_type == PB_VARINT || m_type == PB_LEN) { + if (m_type == PB_VARINT) { pb_varint ui64 = read_varint (); if (ui64 > std::numeric_limits::max ()) { @@ -153,7 +151,7 @@ ProtocolBufferReader::read (int32_t &i32) void ProtocolBufferReader::read (uint64_t &ui64) { - if (m_type == PB_VARINT || m_type == PB_LEN) { + if (m_type == PB_VARINT) { ui64 = read_varint (); @@ -203,8 +201,7 @@ ProtocolBufferReader::open () error (tl::to_string (tr ("Expected a LEN wire type for a submessage"))); } - size_t value = 0; - read (value); + size_t value = read_varint (); if (! m_seq_counts.empty ()) { // take out the following bytes from the current sequence m_seq_counts.back () -= value; @@ -284,6 +281,15 @@ ProtocolBufferReader::read_varint () void ProtocolBufferReader::skip_bytes (size_t n) { + m_pos_before = m_pos; + m_pos += n; + if (! m_seq_counts.empty ()) { + if (m_seq_counts.back () < n) { + error (tl::to_string (tr ("sequence underflow"))); + } + m_seq_counts.back () -= n; + } + const size_t chunk_size = 1024; while (n > 0) { size_t l = std::min (chunk_size, n); diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index cf413878e..68ea64f06 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -205,28 +205,26 @@ struct Root { const Child &get_child () const { return m_child; } }; -TEST (100_BasicStruct) +static tl::PBElementList child_struct = + tl::pb_make_member (&Child::txt, 1) + + tl::pb_make_member (&Child::d, 2) + + tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, 3, &child_struct); + +static tl::PBStruct structure ("pbtest-struct", 88888888, + tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, &child_struct) + + tl::pb_make_element (&Root::get_child, &Root::set_child, 4, + tl::pb_make_member (&Child::txt, 1) + + tl::pb_make_member (&Child::d, 2) + ) + + tl::pb_make_member (&Root::m, 5) + + tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + + tl::pb_make_member (&Root::b, 7) +); + +static void build_struct (Root &root) { - Root root; - - tl::PBElementList child_struct = - tl::pb_make_member (&Child::txt, 1) + - tl::pb_make_member (&Child::d, 2) + - tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, 3, &child_struct); - - tl::PBStruct structure ("pbtest-struct", 88888888, - tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + - tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, &child_struct) + - tl::pb_make_element (&Root::get_child, &Root::set_child, 4, - tl::pb_make_member (&Child::txt, 1) + - tl::pb_make_member (&Child::d, 2) - ) + - tl::pb_make_member (&Root::m, 5) + - tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + - tl::pb_make_member (&Root::b, 7) - ); - root.add_sub (0.5); root.add_sub (7.5); root.add_isub (420000000); @@ -264,6 +262,12 @@ TEST (100_BasicStruct) sc.txt = "single"; sc.d = 4.2e6; root.set_child (sc); +} + +TEST (100_BasicStruct) +{ + Root root; + build_struct (root); std::string fn = tl::combine_path (tl::testtmp (), "pb_test.pb"); @@ -405,21 +409,21 @@ struct TestClassEnumConverter } }; +tl::PBStruct tc_structure ("pbtest-tc", 1, + tl::pb_make_member (&TestClass::e, 2, TestClassEnumConverter ()) +); + TEST (101_Converter) { TestClass tc; - tl::PBStruct structure ("pbtest-tc", 1, - tl::pb_make_member (&TestClass::e, 2, TestClassEnumConverter ()) - ); - tc.e = TestClass::A; std::string fn = tl::combine_path (tl::testtmp (), "pb_101a.pb"); { tl::OutputStream os (fn); tl::ProtocolBufferWriter writer (os); - structure.write (writer, tc); + tc_structure.write (writer, tc); } tc = TestClass (); @@ -428,7 +432,7 @@ TEST (101_Converter) try { tl::InputStream is (fn); tl::ProtocolBufferReader reader (is); - structure.parse (reader, tc); + tc_structure.parse (reader, tc); } catch (tl::Exception &ex) { error = ex.msg (); } @@ -442,7 +446,7 @@ TEST (101_Converter) { tl::OutputStream os (fn); tl::ProtocolBufferWriter writer (os); - structure.write (writer, tc); + tc_structure.write (writer, tc); } tc = TestClass (); @@ -451,7 +455,7 @@ TEST (101_Converter) try { tl::InputStream is (fn); tl::ProtocolBufferReader reader (is); - structure.parse (reader, tc); + tc_structure.parse (reader, tc); } catch (tl::Exception &ex) { error = ex.msg (); } @@ -459,3 +463,151 @@ TEST (101_Converter) EXPECT_EQ (tc.e, TestClass::B); } +TEST (101_ExternalFiles) +{ + Root root_au; + build_struct (root_au); + + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct1.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == root_au, true); +} + +TEST (102_InvalidHeader) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct2_invalid_header.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 49), "Expected header field with ID 88888888 (got 4444)"); +} + +TEST (103_InvalidHeader) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct3_invalid_header.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 71), "Expected header field with string 'pbtest-struct' (got 'xxxxxxxxxxxxx')"); +} + +// "last one wins", unknown IDs are skipped +TEST (104_Deviations) +{ + Root root_au; + build_struct (root_au); + + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct4_deviations.pb"); + + Root root; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + structure.parse (reader, root); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (root == root_au, true); +} + +TEST (105_Int32Overflow) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct5_overflow.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 21), "32 bit value overflow"); +} + +TEST (106_InvalidType) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct6_invalid_type.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type"); +} + +TEST (107_InvalidType) +{ + // uses I32 to represent enum - works too, not just VARINT + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct7_invalid_type.pb"); + + TestClass tc; + tc.e = TestClass::B; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (error, ""); + EXPECT_EQ (tc.e, TestClass::A); +} + +TEST (108_InvalidType) +{ + std::string fn = tl::combine_path (tl::testsrc (), "testdata/pb/struct8_invalid_type.pb"); + + TestClass tc; + std::string error; + try { + tl::InputStream is (fn); + tl::ProtocolBufferReader reader (is); + tc_structure.parse (reader, tc); + } catch (tl::Exception &ex) { + error = ex.msg (); + } + + EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type"); +} + diff --git a/testdata/pb/struct1.pb b/testdata/pb/struct1.pb new file mode 100644 index 0000000000000000000000000000000000000000..b786e0d3c39520f2d2cd9743fd9be9eeb1539cfc GIT binary patch literal 149 zcmX@~xa%?#Z$VN?YH^8faY<2WatS8`7(B3tGUOZt8jc*Dz%0ef#guF)2$B6@FJ%kl z8bP@X4pMwv%*jTEf@jZw0E}-WIBO;dFhb>x1=GNQ$w5hwi>){_FFhv}Y{UfC07s4e Mht4w^Xjw1<04G5uw*UYD literal 0 HcmV?d00001 diff --git a/testdata/pb/struct2_invalid_header.pb b/testdata/pb/struct2_invalid_header.pb new file mode 100644 index 0000000000000000000000000000000000000000..8fb9360701e0a1903020878cb0e53c658ad34f56 GIT binary patch literal 147 zcmaDfm5H|?sU)?yM7Ox4s5H5RlK~7K*h3j|4gw8Fj!s~f;^ksWHWY-&ez2Fa1#*p` zTm}azJ}%~DBSXQnXFvePHxisR69gEc^2UN`V8G;{B*?{9oSB!NlL|Is0&9Sy#{NU+ J84a{77y&{yBrE^` literal 0 HcmV?d00001 diff --git a/testdata/pb/struct3_invalid_header.pb b/testdata/pb/struct3_invalid_header.pb new file mode 100644 index 0000000000000000000000000000000000000000..0c1e687ef262e19505bd862e439445fcaa9aa12d GIT binary patch literal 149 zcmX@~xa%?#Zv`^oWB`K)_E3hLgFwTPqZ62=c)6I84Fw^xAMB-Ufm|afm%%}bkBd3k z$WZX?84!T+jRa@S1OY~|p1`9?2Nj)z6 literal 0 HcmV?d00001 diff --git a/testdata/pb/struct5_overflow.pb b/testdata/pb/struct5_overflow.pb new file mode 100644 index 000000000..14e328697 --- /dev/null +++ b/testdata/pb/struct5_overflow.pb @@ -0,0 +1,2 @@ + + pbtest-tc€Ôª? \ No newline at end of file diff --git a/testdata/pb/struct6_invalid_type.pb b/testdata/pb/struct6_invalid_type.pb new file mode 100644 index 0000000000000000000000000000000000000000..4331c5da28dc17b38e01771552863cf6cfdb40b9 GIT binary patch literal 20 Wcmd<$EJ!LzEiTb5Nfs1jfB*nBSp&=f literal 0 HcmV?d00001 diff --git a/testdata/pb/struct7_invalid_type.pb b/testdata/pb/struct7_invalid_type.pb new file mode 100644 index 0000000000000000000000000000000000000000..bec039f513e8e736d303a4267e6c7563cd115c13 GIT binary patch literal 16 Xcmd<$EJ!LzEiTb5Nfs4kU|;|MCOret literal 0 HcmV?d00001 diff --git a/testdata/pb/struct8_invalid_type.pb b/testdata/pb/struct8_invalid_type.pb new file mode 100644 index 000000000..02d42f890 --- /dev/null +++ b/testdata/pb/struct8_invalid_type.pb @@ -0,0 +1,2 @@ + + pbtest-tcabc \ No newline at end of file From b7cc3b87df2b2bbfd464314fd1038bccf430ea83 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 20:58:20 +0200 Subject: [PATCH 14/22] WIP --- src/tl/tl/tlProtocolBufferStruct.h | 252 +++++++++++---------- src/tl/unit_tests/tlProtocolBufferTests.cc | 25 +- 2 files changed, 142 insertions(+), 135 deletions(-) diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index e3624e3e8..c22635594 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -486,20 +486,20 @@ class TL_PUBLIC PBElementBase public: typedef PBElementList::iterator iterator; - PBElementBase (int tag, const PBElementList &children) - : m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) + PBElementBase (const std::string &name, int tag, const PBElementList &children) + : m_name (name), m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) { // .. nothing yet .. } - PBElementBase (int tag, const PBElementList *children) - : m_tag (tag), mp_children (children), m_owns_child_list (false) + PBElementBase (const std::string &name, int tag, const PBElementList *children) + : m_name (name), m_tag (tag), mp_children (children), m_owns_child_list (false) { // .. nothing yet .. } PBElementBase (const PBElementBase &d) - : m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) + : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) { if (m_owns_child_list) { mp_children = new PBElementList (*d.mp_children); @@ -529,6 +529,11 @@ public: return m_tag; } + const std::string &name () const + { + return m_name; + } + iterator begin () const { return mp_children->begin (); @@ -540,6 +545,7 @@ public: } private: + std::string m_name; int m_tag; const PBElementList *mp_children; bool m_owns_child_list; @@ -567,14 +573,14 @@ class TL_PUBLIC_TEMPLATE PBElement : public PBElementBase { public: - PBElement (const Read &r, const Write &w, int tag, const PBElementList &children) - : PBElementBase (tag, children), m_r (r), m_w (w) + PBElement (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList &children) + : PBElementBase (name, tag, children), m_r (r), m_w (w) { // .. nothing yet .. } - PBElement (const Read &r, const Write &w, int tag, const PBElementList *children) - : PBElementBase (tag, children), m_r (r), m_w (w) + PBElement (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList *children) + : PBElementBase (name, tag, children), m_r (r), m_w (w) { // .. nothing yet .. } @@ -683,14 +689,14 @@ class TL_PUBLIC_TEMPLATE PBElementWithParentRef : public PBElement { public: - PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList &children) - : PBElement (r, w, tag, children) + PBElementWithParentRef (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList &children) + : PBElement (r, w, name, tag, children) { // .. nothing yet .. } - PBElementWithParentRef (const Read &r, const Write &w, int tag, const PBElementList *children) - : PBElement (r, w, tag, children) + PBElementWithParentRef (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList *children) + : PBElement (r, w, name, tag, children) { // .. nothing yet .. } @@ -742,8 +748,8 @@ class TL_PUBLIC_TEMPLATE PBMember : public PBElementBase { public: - PBMember (const Read &r, const Write &w, int tag, Converter c = Converter ()) - : PBElementBase (tag, PBElementList::empty ()), m_r (r), m_w (w), m_c (c) + PBMember (const Read &r, const Write &w, const std::string &name, int tag, Converter c = Converter ()) + : PBElementBase (name, tag, PBElementList::empty ()), m_r (r), m_w (w), m_c (c) { // .. nothing yet .. } @@ -969,62 +975,69 @@ class TL_PUBLIC_TEMPLATE PBStruct { public: PBStruct (const std::string &name, int name_tag, const PBElementList *children) - : PBElementBase (0, children), m_name (name), m_name_tag (name_tag) + : PBElementBase (name, name_tag, children) { // .. nothing yet .. } PBStruct (const std::string &name, int name_tag, const PBElementList &children) - : PBElementBase (0, children), m_name (name), m_name_tag (name_tag) + : PBElementBase (name, name_tag, children) { // .. nothing yet .. } PBStruct (const PBStruct &d) - : PBElementBase (d), m_name (d.name ()), m_name_tag (d.name_tag ()) + : PBElementBase (d) { // .. nothing yet .. } + /** + * @brief Creates a deep copy of the structure + */ virtual PBElementBase *clone () const { return new PBStruct (*this); } - const std::string &name () const - { - return m_name; - } - - int name_tag () const - { - return m_name_tag; - } - + /** + * @brief Serializes the given object (root) to the writer + */ void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const { PBWriterState writer_state; writer_state.push (& root); - writer.write (name_tag (), name ()); + writer.write (tag (), name ()); for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { c->get ()->write (this, writer, writer_state); } } + /** + * @brief Deserializes the given object (root) from the reader + */ void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const { - PBObjTag tag; + PBObjTag self_tag; PBReaderState rs; rs.push (&root); PBParser h; - h.expect_header (reader, m_name_tag, m_name); + h.expect_header (reader, tag (), name ()); h.parse (reader, this, &rs); - rs.pop (tag); + rs.pop (self_tag); tl_assert (rs.empty ()); } + /** + * @brief Produces a definition for the protoc compiler + */ + std::string create_def () const + { + return std::string (); // @@@ + } + private: virtual void write (const PBElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const { @@ -1045,9 +1058,6 @@ private: { // disable base class implementation } - - std::string m_name; - int m_name_tag; }; /** @@ -1351,11 +1361,11 @@ private: */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +pb_make_element (void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1363,11 +1373,11 @@ pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElemen */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList *children) +pb_make_element (void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberDummyReadAdaptor (), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1375,11 +1385,11 @@ pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1387,11 +1397,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1399,11 +1409,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberAccWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList *children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag, children); + PBMemberAccWriteAdaptor (setter), name, tag, children); } /** @@ -1411,11 +1421,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1423,11 +1433,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1435,11 +1445,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value */ template PBElement, PBMemberAccWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList *children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag, children); + PBMemberAccWriteAdaptor (setter), name, tag, children); } /** @@ -1447,11 +1457,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value */ template PBElement, PBMemberWriteAdaptor > -pb_make_element (Value Parent::*member, int tag, const PBElementList *children) +pb_make_element (Value Parent::*member, const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberWriteAdaptor > ( PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), tag, children); + PBMemberWriteAdaptor (member), name, tag, children); } /** @@ -1459,11 +1469,11 @@ pb_make_element (Value Parent::*member, int tag, const PBElementList *children) */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1471,11 +1481,11 @@ pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, v */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1483,11 +1493,11 @@ pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, v */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element (void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1495,11 +1505,11 @@ pb_make_element (void (Parent::*setter) (const Value &), int tag, const PBElemen */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element (void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberDummyReadAdaptor (), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1507,11 +1517,11 @@ pb_make_element (void (Parent::*setter) (Value *), int tag, const PBElementList */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1519,11 +1529,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1531,11 +1541,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberAccWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList &children) +pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag, children); + PBMemberAccWriteAdaptor (setter), name, tag, children); } /** @@ -1543,11 +1553,11 @@ pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1555,11 +1565,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1567,11 +1577,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value */ template PBElement, PBMemberAccWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, const PBElementList &children) +pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccWriteAdaptor > ( PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag, children); + PBMemberAccWriteAdaptor (setter), name, tag, children); } /** @@ -1579,11 +1589,11 @@ pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value */ template PBElement, PBMemberWriteAdaptor > -pb_make_element (Value Parent::*member, int tag, const PBElementList &children) +pb_make_element (Value Parent::*member, const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberWriteAdaptor > ( PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), tag, children); + PBMemberWriteAdaptor (member), name, tag, children); } /** @@ -1591,11 +1601,11 @@ pb_make_element (Value Parent::*member, int tag, const PBElementList &children) */ template PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberAccRefWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } /** @@ -1603,65 +1613,65 @@ pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, v */ template PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElement, PBMemberTransferWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList *children) +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) { return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), int tag, const PBElementList *children) +pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) { return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, const PBElementList &children) +pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) { return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), tag, children); + PBMemberAccRefWriteAdaptor (setter), name, tag, children); } template PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), int tag, const PBElementList &children) +pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) { return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), tag, children); + PBMemberTransferWriteAdaptor (setter), name, tag, children); } /** @@ -1689,11 +1699,11 @@ struct PBStdConverter */ template PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > -pb_make_member (int tag) +pb_make_member (const std::string &name, int tag) { return PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > ( PBMemberDummyReadAdaptor (), - PBMemberDummyWriteAdaptor (), tag); + PBMemberDummyWriteAdaptor (), name, tag); } /** @@ -1701,11 +1711,11 @@ pb_make_member (int tag) */ template PBMember, PBMemberWriteAdaptor , PBStdConverter > -pb_make_member (Value Parent::*member, int tag) +pb_make_member (Value Parent::*member, const std::string &name, int tag) { return PBMember, PBMemberWriteAdaptor , PBStdConverter > ( PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), tag); + PBMemberWriteAdaptor (member), name, tag); } /** @@ -1713,11 +1723,11 @@ pb_make_member (Value Parent::*member, int tag) */ template PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (void (Parent::*setter) (const Value &), int tag) +pb_make_member (void (Parent::*setter) (const Value &), const std::string &name, int tag) { return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), tag); + PBMemberAccRefWriteAdaptor (setter), name, tag); } /** @@ -1725,11 +1735,11 @@ pb_make_member (void (Parent::*setter) (const Value &), int tag) */ template PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (void (Parent::*setter) (Value), int tag) +pb_make_member (void (Parent::*setter) (Value), const std::string &name, int tag) { return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( PBMemberDummyReadAdaptor (), - PBMemberAccWriteAdaptor (setter), tag); + PBMemberAccWriteAdaptor (setter), name, tag); } /** @@ -1737,11 +1747,11 @@ pb_make_member (void (Parent::*setter) (Value), int tag) */ template PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag) +pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) { return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag); + PBMemberAccRefWriteAdaptor (setter), name, tag); } /** @@ -1749,11 +1759,11 @@ pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag) +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag) { return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag); + PBMemberAccWriteAdaptor (setter), name, tag); } /** @@ -1761,11 +1771,11 @@ pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value) */ template PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (const Value & (Parent::*getter) () const, void (Parent::*setter) (Value), int tag) +pb_make_member (const Value & (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag) { return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag); + PBMemberAccWriteAdaptor (setter), name, tag); } /** @@ -1773,11 +1783,11 @@ pb_make_member (const Value & (Parent::*getter) () const, void (Parent::*setter) */ template PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag) +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) { return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag); + PBMemberAccRefWriteAdaptor (setter), name, tag); } /** @@ -1785,11 +1795,11 @@ pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (const */ template PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag) +pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) { return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), tag); + PBMemberAccRefWriteAdaptor (setter), name, tag); } /** @@ -1797,11 +1807,11 @@ pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, vo */ template PBMember, PBMemberWriteAdaptor , Converter> -pb_make_member (Value Parent::*member, int tag, Converter conv) +pb_make_member (Value Parent::*member, const std::string &name, int tag, Converter conv) { return PBMember, PBMemberWriteAdaptor , Converter> ( PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), tag, conv); + PBMemberWriteAdaptor (member), name, tag, conv); } /** @@ -1809,11 +1819,11 @@ pb_make_member (Value Parent::*member, int tag, Converter conv) */ template PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (void (Parent::*setter) (const Value &), int tag, Converter conv) +pb_make_member (void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) { return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), tag, conv); + PBMemberAccRefWriteAdaptor (setter), name, tag, conv); } /** @@ -1821,11 +1831,11 @@ pb_make_member (void (Parent::*setter) (const Value &), int tag, Converter conv) */ template PBMember, PBMemberDummyWriteAdaptor , Converter> -pb_make_member (void (Parent::*setter) (Value), int tag, Converter conv) +pb_make_member (void (Parent::*setter) (Value), const std::string &name, int tag, Converter conv) { return PBMember, PBMemberDummyWriteAdaptor , Converter> ( PBMemberAccReadAdaptor (setter), - PBMemberDummyWriteAdaptor (), tag, conv); + PBMemberDummyWriteAdaptor (), name, tag, conv); } /** @@ -1833,11 +1843,11 @@ pb_make_member (void (Parent::*setter) (Value), int tag, Converter conv) */ template PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), int tag, Converter conv) +pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) { return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), tag, conv); + PBMemberAccRefWriteAdaptor (setter), name, tag, conv); } /** @@ -1845,11 +1855,11 @@ pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) */ template PBMember, PBMemberAccWriteAdaptor , Converter> -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), int tag, Converter conv) +pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, Converter conv) { return PBMember, PBMemberAccWriteAdaptor , Converter> ( PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), tag, conv); + PBMemberAccWriteAdaptor (setter), name, tag, conv); } /** @@ -1857,11 +1867,11 @@ pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value) */ template PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), int tag, Converter conv) +pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) { return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), tag, conv); + PBMemberAccRefWriteAdaptor (setter), name, tag, conv); } } diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index 68ea64f06..e50b33a45 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -206,21 +206,18 @@ struct Root { }; static tl::PBElementList child_struct = - tl::pb_make_member (&Child::txt, 1) + - tl::pb_make_member (&Child::d, 2) + - tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, 3, &child_struct); + tl::pb_make_member (&Child::txt, "txt", 1) + + tl::pb_make_member (&Child::d, "d", 2) + + tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children", 3, &child_struct); static tl::PBStruct structure ("pbtest-struct", 88888888, - tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, 1) + - tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, 3, &child_struct) + - tl::pb_make_element (&Root::get_child, &Root::set_child, 4, - tl::pb_make_member (&Child::txt, 1) + - tl::pb_make_member (&Child::d, 2) - ) + - tl::pb_make_member (&Root::m, 5) + - tl::pb_make_member (&Root::get_mi, &Root::set_mi, 6) + - tl::pb_make_member (&Root::b, 7) + tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub", 1) + + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub", 2) + + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "childredn", 3, &child_struct) + + tl::pb_make_element (&Root::get_child, &Root::set_child, "child", 4, &child_struct) + + tl::pb_make_member (&Root::m, "m", 5) + + tl::pb_make_member (&Root::get_mi, &Root::set_mi, "mi", 6) + + tl::pb_make_member (&Root::b, "b", 7) ); static void build_struct (Root &root) @@ -410,7 +407,7 @@ struct TestClassEnumConverter }; tl::PBStruct tc_structure ("pbtest-tc", 1, - tl::pb_make_member (&TestClass::e, 2, TestClassEnumConverter ()) + tl::pb_make_member (&TestClass::e, "e", 2, TestClassEnumConverter ()) ); TEST (101_Converter) From 971ef3ff727d978c6d539f65bb18d17dea55d3ef Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 11 Aug 2024 23:13:26 +0200 Subject: [PATCH 15/22] Ability to create proto specs from structures --- src/tl/tl/tlProtocolBufferStruct.cc | 226 +++++++++++++- src/tl/tl/tlProtocolBufferStruct.h | 324 ++++++++++++++------- src/tl/unit_tests/tlProtocolBufferTests.cc | 35 ++- 3 files changed, 479 insertions(+), 106 deletions(-) diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc index 6a675031f..25ecbd43a 100644 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ b/src/tl/tl/tlProtocolBufferStruct.cc @@ -21,12 +21,234 @@ */ #include "tlProtocolBufferStruct.h" - +#include namespace tl { - // -------------------------------------------------------------------- +static size_t s_oid = 0; + +// -------------------------------------------------------------------- +// PBElementList implementation + +PBElementList::PBElementList () + : m_oid (++s_oid) +{ + // .. nothing yet .. +} + +PBElementList::PBElementList (const PBElementBase &e) + : m_oid (++s_oid) +{ + m_elements.push_back (PBElementProxy (e)); +} + +PBElementList::PBElementList (PBElementBase *e) + : m_oid (++s_oid) +{ + if (e) { + m_elements.push_back (PBElementProxy (e)); + } +} + +PBElementList::PBElementList (const std::string &name, const PBElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (name) +{ + // .. nothing yet .. +} + +PBElementList::PBElementList (const PBElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (d.m_name) +{ + // .. nothing yet .. +} + +PBElementList::PBElementList (const PBElementList &d, const PBElementBase &e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + m_elements.push_back (PBElementProxy (e)); +} + +PBElementList::PBElementList (const PBElementList &d, PBElementBase *e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + if (e) { + m_elements.push_back (PBElementProxy (e)); + } +} + +void PBElementList::append (const PBElementBase &e) +{ + m_elements.push_back (PBElementProxy (e)); +} + +void PBElementList::append (PBElementBase *e) +{ + if (e) { + m_elements.push_back (PBElementProxy (e)); + } +} + +PBElementList::iterator PBElementList::begin () const +{ + return m_elements.begin (); +} + +PBElementList::iterator PBElementList::end () const +{ + return m_elements.end (); +} + +PBElementList PBElementList::empty () +{ + return PBElementList (); +} + +// -------------------------------------------------------------------- +// PBElementBase implementation + +PBElementBase::PBElementBase (const std::string &name, int tag, const PBElementList &children) + : m_name (name), m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) +{ + // .. nothing yet .. +} + +PBElementBase::PBElementBase (const std::string &name, int tag, const PBElementList *children) + : m_name (name), m_tag (tag), mp_children (children), m_owns_child_list (false) +{ + // .. nothing yet .. +} + +PBElementBase::PBElementBase (const PBElementBase &d) + : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) +{ + if (m_owns_child_list) { + mp_children = new PBElementList (*d.mp_children); + } else { + mp_children = d.mp_children; + } +} + +PBElementBase::~PBElementBase () +{ + if (m_owns_child_list) { + delete const_cast (mp_children); + mp_children = 0; + } +} + +PBElementBase::Cardinality +PBElementBase::cardinality () const +{ + return Zero; +} + +PBElementBase::iterator +PBElementBase::begin () const +{ + return mp_children->begin (); +} + +PBElementBase::iterator +PBElementBase::end () const +{ + return mp_children->end (); +} + +std::string +PBElementBase::name4code () const +{ + std::string res; + const char *n = name ().c_str (); + + if (! isalpha (*n) && *n != '_') { + res += '_'; + } + + while (*n) { + if (*n == '-') { + res += '_'; + } else if (isalnum (*n) || *n == '_') { + res += *n; + } + ++n; + } + + return res; +} + +std::string +PBElementBase::create_def (std::map > &messages) const +{ + std::string res; + + auto m = messages.find (oid ()); + if (m != messages.end ()) { + + res += "message " + m->second.second + " {\n"; + + for (auto i = begin (); i != end (); ++i) { + const PBElementBase *e = i->get (); + Cardinality c = e->cardinality (); + std::string entry = e->create_def_entry (messages); + if (! entry.empty ()) { + res += " "; + if (c != Zero) { + if (c == Many) { + res += "repeated "; + } else { + res += "optional "; + } + res += entry + "\n"; + } + } + } + + res += "}"; + + } + + return res; +} + +void +PBElementBase::collect_messages (std::map > &messages) const +{ + for (auto i = begin (); i != end (); ++i) { + i->get ()->collect_messages (messages); + } +} + +std::string +PBElementBase::make_message_name () const +{ + std::string res = mp_children->name (); + if (! res.empty ()) { + return res; + } + + // Capitalize names + std::string n4c = name4code (); + + bool upcase = true; + + const char *n = n4c.c_str (); + while (*n) { + if (*n == '_') { + upcase = true; + } else if (upcase) { + res += toupper (*n); + upcase = false; + } else { + res += *n; + } + ++n; + } + + return res; +} + +// -------------------------------------------------------------------- // PBParser implementation PBParser::PBParser () diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h index c22635594..c4bacf3de 100644 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ b/src/tl/tl/tlProtocolBufferStruct.h @@ -26,6 +26,8 @@ #include "tlCommon.h" #include "tlProtocolBuffer.h" +#include + namespace tl { @@ -277,12 +279,24 @@ private: class TL_PUBLIC PBElementBase; -struct pass_by_value_tag { - pass_by_value_tag () { } +struct pb_pass_by_value_tag { + pb_pass_by_value_tag () { } }; -struct pass_by_ref_tag { - pass_by_ref_tag () { } +struct pb_pass_by_ref_tag { + pb_pass_by_ref_tag () { } +}; + +struct pb_zero_cardinality_tag { + pb_zero_cardinality_tag () { } +}; + +struct pb_single_cardinality_tag { + pb_single_cardinality_tag () { } +}; + +struct pb_many_cardinality_tag { + pb_many_cardinality_tag () { } }; /** @@ -360,66 +374,36 @@ public: typedef std::list children_list; typedef children_list::const_iterator iterator; - PBElementList () + PBElementList (); + PBElementList (const PBElementBase &e); + PBElementList (PBElementBase *e); + PBElementList (const std::string &name, const PBElementList &d); + PBElementList (const PBElementList &d); + PBElementList (const PBElementList &d, const PBElementBase &e); + PBElementList (const PBElementList &d, PBElementBase *e); + + void append (const PBElementBase &e); + void append (PBElementBase *e); + + iterator begin () const; + iterator end () const; + + static PBElementList empty (); + + size_t oid () const { - // .. nothing yet .. + return m_oid; } - PBElementList (const PBElementBase &e) + const std::string &name () const { - m_elements.push_back (PBElementProxy (e)); - } - - PBElementList (PBElementBase *e) - { - if (e) { - m_elements.push_back (PBElementProxy (e)); - } - } - - PBElementList (const PBElementList &d, const PBElementBase &e) - : m_elements (d.m_elements) - { - m_elements.push_back (PBElementProxy (e)); - } - - PBElementList (const PBElementList &d, PBElementBase *e) - : m_elements (d.m_elements) - { - if (e) { - m_elements.push_back (PBElementProxy (e)); - } - } - - void append (const PBElementBase &e) - { - m_elements.push_back (PBElementProxy (e)); - } - - void append (PBElementBase *e) - { - if (e) { - m_elements.push_back (PBElementProxy (e)); - } - } - - iterator begin () const - { - return m_elements.begin (); - } - - iterator end () const - { - return m_elements.end (); - } - - static PBElementList empty () - { - return PBElementList (); + return m_name; } private: std::list m_elements; + size_t m_oid; + std::string m_name; }; /** @@ -486,35 +470,14 @@ class TL_PUBLIC PBElementBase public: typedef PBElementList::iterator iterator; - PBElementBase (const std::string &name, int tag, const PBElementList &children) - : m_name (name), m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) - { - // .. nothing yet .. - } + enum Cardinality { + Zero, Single, Many + }; - PBElementBase (const std::string &name, int tag, const PBElementList *children) - : m_name (name), m_tag (tag), mp_children (children), m_owns_child_list (false) - { - // .. nothing yet .. - } - - PBElementBase (const PBElementBase &d) - : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) - { - if (m_owns_child_list) { - mp_children = new PBElementList (*d.mp_children); - } else { - mp_children = d.mp_children; - } - } - - virtual ~PBElementBase () - { - if (m_owns_child_list) { - delete const_cast (mp_children); - mp_children = 0; - } - } + PBElementBase (const std::string &name, int tag, const PBElementList &children); + PBElementBase (const std::string &name, int tag, const PBElementList *children); + PBElementBase (const PBElementBase &d); + virtual ~PBElementBase (); virtual PBElementBase *clone () const = 0; @@ -524,6 +487,13 @@ public: virtual void write (const PBElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } + virtual Cardinality cardinality () const; + + size_t oid () const + { + return mp_children->oid (); + } + int tag () const { return m_tag; @@ -534,14 +504,36 @@ public: return m_name; } - iterator begin () const + /** + * @brief Returns a name suitable for code + * Specifically, hyphens are replaced by underscores. + */ + std::string name4code () const; + + iterator begin () const; + iterator end () const; + + std::string create_def (std::map > &messages) const; + +protected: + virtual void collect_messages (std::map > &messages) const; + virtual std::string create_def_entry (std::map > &messages) const = 0; + + std::string make_message_name () const; + + static Cardinality get_cardinality (tl::pb_zero_cardinality_tag) { - return mp_children->begin (); + return Zero; } - iterator end () const + static Cardinality get_cardinality (tl::pb_single_cardinality_tag) { - return mp_children->end (); + return Single; + } + + static Cardinality get_cardinality (tl::pb_many_cardinality_tag) + { + return Many; } private: @@ -625,12 +617,35 @@ public: r.start (*objs.back (parent_tag)); while (! r.at_end ()) { typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), tag (), writer, read_tag, objs); + write_obj (r (), tag (), writer, read_tag_type (), objs); r.next (); } } + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > &messages) const + { + if (messages.find (oid ()) == messages.end ()) { + messages [oid ()] = std::make_pair (this, make_message_name ()); + PBElementBase::collect_messages (messages); + } + } + + virtual std::string create_def_entry (std::map > &messages) const + { + auto m = messages.find (oid ()); + if (m != messages.end ()) { + return m->second.second + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } else { + return std::string (); + } + } + protected: Read &rear () { return m_r; } Write &write () { return m_w; } @@ -640,7 +655,7 @@ private: Write m_w; // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_value_tag, PBWriterState &objs) const + void write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pb_pass_by_value_tag, PBWriterState &objs) const { PBObjTag self_tag; @@ -658,7 +673,7 @@ private: } } - void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_ref_tag, PBWriterState &objs) const + void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pb_pass_by_ref_tag, PBWriterState &objs) const { PBObjTag self_tag; @@ -800,6 +815,23 @@ public: } } + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > & /*messages*/) const + { + // no messages here. + } + + virtual std::string create_def_entry (std::map > & /*messages*/) const + { + const Value *v = 0; + return typestring (v) + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } + private: Read m_r; Write m_w; @@ -952,6 +984,74 @@ private: reader.read (vv); m_c.pb_decode (vv, v); } + + // type strings + std::string typestring (const float *) const + { + return "float"; + } + + std::string typestring (const double *) const + { + return "double"; + } + + std::string typestring (const uint8_t *) const + { + return "uint32"; + } + + std::string typestring (const uint16_t *) const + { + return "uint32"; + } + + std::string typestring (const uint32_t *) const + { + return "uint32"; + } + + std::string typestring (const uint64_t *) const + { + return "uint64"; + } + + std::string typestring (const int8_t *) const + { + return "sint32"; + } + + std::string typestring (const int16_t *) const + { + return "sint32"; + } + + std::string typestring (const int32_t *) const + { + return "sint32"; + } + + std::string typestring (const int64_t *) const + { + return "sint64"; + } + + std::string typestring (const bool *) const + { + return "uint32"; + } + + std::string typestring (const std::string *) const + { + return "string"; + } + + template + std::string typestring (const T *) const + { + const typename Converter::pb_type *v = 0; + return typestring (v); + } }; /** @@ -1035,7 +1135,23 @@ public: */ std::string create_def () const { - return std::string (); // @@@ + std::map > msgs; + collect_messages (msgs); + msgs[oid ()] = std::make_pair (this, make_message_name ()); + + std::string res = "// created from KLayout proto definition '" + name () + "'\n\n"; + res += "syntax = \"proto2\";"; + + for (auto i = msgs.begin (); i != msgs.end (); ++i) { + std::string entry = i->second.first->create_def (msgs); + if (! entry.empty ()) { + res += "\n"; + res += "\n"; + res += entry; + } + } + + return res; } private: @@ -1054,10 +1170,15 @@ private: // disable base class implementation } - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &) const + virtual void finish (const PBElementBase *, PBReaderState &) const { // disable base class implementation } + + virtual std::string create_def_entry (std::map > &) const + { + return std::string (); + } }; /** @@ -1176,7 +1297,8 @@ private: template struct PBMemberDummyReadAdaptor { - typedef pass_by_ref_tag tag; + typedef pb_pass_by_ref_tag tag; + typedef pb_zero_cardinality_tag cardinality; PBMemberDummyReadAdaptor () { @@ -1207,7 +1329,8 @@ struct PBMemberDummyReadAdaptor template struct PBMemberReadAdaptor { - typedef pass_by_ref_tag tag; + typedef pb_pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; PBMemberReadAdaptor (Value Parent::*member) : mp_member (member), mp_owner (0), m_done (false) @@ -1245,7 +1368,8 @@ private: template struct PBMemberAccRefReadAdaptor { - typedef pass_by_ref_tag tag; + typedef pb_pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; PBMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1283,7 +1407,8 @@ private: template struct PBMemberAccReadAdaptor { - typedef pass_by_value_tag tag; + typedef pb_pass_by_value_tag tag; + typedef pb_single_cardinality_tag cardinality; PBMemberAccReadAdaptor (Value (Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1321,7 +1446,8 @@ private: template struct PBMemberIterReadAdaptor { - typedef pass_by_ref_tag tag; + typedef pb_pass_by_ref_tag tag; + typedef pb_many_cardinality_tag cardinality; PBMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) : mp_begin (begin), mp_end (end) diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index e50b33a45..75289aed1 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -205,15 +205,16 @@ struct Root { const Child &get_child () const { return m_child; } }; -static tl::PBElementList child_struct = - tl::pb_make_member (&Child::txt, "txt", 1) + - tl::pb_make_member (&Child::d, "d", 2) + - tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children", 3, &child_struct); +static tl::PBElementList child_struct ("Child", + tl::pb_make_member (&Child::txt, "txt", 1) + + tl::pb_make_member (&Child::d, "d", 2) + + tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children", 3, &child_struct) +); static tl::PBStruct structure ("pbtest-struct", 88888888, tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub", 1) + tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub", 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "childredn", 3, &child_struct) + + tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children", 3, &child_struct) + tl::pb_make_element (&Root::get_child, &Root::set_child, "child", 4, &child_struct) + tl::pb_make_member (&Root::m, "m", 5) + tl::pb_make_member (&Root::get_mi, &Root::set_mi, "mi", 6) + @@ -608,3 +609,27 @@ TEST (108_InvalidType) EXPECT_EQ (std::string (error, 0, 34), "Expected a VARINT or I32 wire type"); } +TEST (109_DumpSpec) +{ + EXPECT_EQ (structure.create_def (), + "// created from KLayout proto definition 'pbtest-struct'\n" + "\n" + "syntax = \"proto2\";\n" + "\n" + "message Child {\n" + " optional string txt = 1;\n" + " optional double d = 2;\n" + " repeated Child children = 3;\n" + "}\n" + "\n" + "message PbtestStruct {\n" + " repeated double sub = 1;\n" + " repeated sint32 isub = 2;\n" + " repeated Child children = 3;\n" + " optional Child child = 4;\n" + " optional sint64 m = 5;\n" + " optional sint32 mi = 6;\n" + " optional uint32 b = 7;\n" + "}" + ); +} From c8bc64a90e2d7c89b708a3154e9cf9c9333c8669 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 11:41:09 +0200 Subject: [PATCH 16/22] WIP --- src/tl/tl/tl.pro | 2 + src/tl/tl/tlXMLParser.cc | 1206 +++++-------------- src/tl/tl/tlXMLParser.h | 1263 ++++++++++++-------- src/tl/tl/tlXMLReader.cc | 913 ++++++++++++++ src/tl/tl/tlXMLReader.h | 215 ++++ src/tl/unit_tests/tlProtocolBufferTests.cc | 37 +- 6 files changed, 2266 insertions(+), 1370 deletions(-) create mode 100644 src/tl/tl/tlXMLReader.cc create mode 100644 src/tl/tl/tlXMLReader.h diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 0773588d1..98c1a41b9 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -45,6 +45,7 @@ SOURCES = \ tlUnitTest.cc \ tlInt128Support.cc \ tlXMLParser.cc \ + tlXMLReader.cc \ tlXMLWriter.cc \ tlThreadedWorkers.cc \ tlThreads.cc \ @@ -113,6 +114,7 @@ HEADERS = \ tlInt128Support.h \ tlDefs.h \ tlXMLParser.h \ + tlXMLReader.h \ tlXMLWriter.h \ tlThreadedWorkers.h \ tlThreads.h \ diff --git a/src/tl/tl/tlXMLParser.cc b/src/tl/tl/tlXMLParser.cc index 55174fc75..01959e579 100644 --- a/src/tl/tl/tlXMLParser.cc +++ b/src/tl/tl/tlXMLParser.cc @@ -26,891 +26,11 @@ #include "tlLog.h" #include "tlAssert.h" #include "tlProgress.h" +#include "tlProtocolBuffer.h" #include #include -#if defined(HAVE_EXPAT) - -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData -{ -public: - XMLSourcePrivateData (tl::InputStream *stream) - : mp_stream_holder (stream), - m_has_error (false) - { - mp_stream = stream; - } - - XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) - : mp_stream_holder (stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - XMLSourcePrivateData (tl::InputStream &stream) - : m_has_error (false) - { - mp_stream = &stream; - } - - XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) - : mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = &stream; - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - } - - int read (char *data, size_t n) - { - try { - - if (mp_progress.get ()) { - mp_progress->set (mp_stream->pos ()); - } - - size_t n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { - *data++ = *rd; - } - - return int (n0 - n); - - } catch (tl::Exception &ex) { - m_error = ex.msg (); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - - const std::string &error_msg () const - { - return m_error; - } - - void reset () - { - mp_stream->reset (); - } - -private: - std::unique_ptr mp_stream_holder; - tl::InputStream *mp_stream; - std::unique_ptr mp_progress; - bool m_has_error; - std::string m_error; -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) - : m_copy (string) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLSourcePrivateData (new tl::InputStream (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLSourcePrivateData (s, progress_message)); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLSourcePrivateData (s)); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); -void XMLCALL end_element_handler (void *user_data, const XML_Char *name); -void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); - -static std::string get_lname (const std::string &name) -{ - size_t colon = name.find (':'); - if (colon != std::string::npos) { - return std::string (name, colon + 1, name.size () - colon - 1); - } else { - return name; - } -} - -class XMLParserPrivateData -{ -public: - XMLParserPrivateData () - : mp_struct_handler (0) - { - mp_parser = XML_ParserCreate ("UTF-8"); - tl_assert (mp_parser != NULL); - } - - ~XMLParserPrivateData () - { - if (mp_parser != NULL) { - XML_ParserFree (mp_parser); - } - } - - void start_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->start_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void end_element (const std::string &name) - { - try { - // TODO: Provide namespace URI? - mp_struct_handler->end_element (std::string (), get_lname (name), name); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void cdata (const std::string &cdata) - { - try { - mp_struct_handler->characters (cdata); - } catch (tl::Exception &ex) { - error (ex); - } - } - - void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) - { - m_has_error = false; - mp_struct_handler = &struct_handler; - - // Just in case we want to reuse it ... - XML_ParserReset (mp_parser, NULL); - XML_SetUserData (mp_parser, (void *) this); - XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); - XML_SetCharacterDataHandler (mp_parser, cdata_handler); - - const int chunk = 65536; - char buffer [chunk]; - - int n; - - do { - - n = source.source ()->read (buffer, chunk); - if (n < 0) { - break; - } - - XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); - if (status == XML_STATUS_ERROR) { - m_has_error = true; - m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); - m_error_line = XML_GetErrorLineNumber (mp_parser); - m_error_column = XML_GetErrorColumnNumber (mp_parser); - } - - } while (n == chunk && !m_has_error); - } - - void check_error () - { - if (m_has_error) { - throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); - } - } - -private: - void error (tl::Exception &ex) - { - m_has_error = true; - m_error_line = XML_GetCurrentLineNumber (mp_parser); - m_error_column = XML_GetCurrentColumnNumber (mp_parser); - m_error = ex.msg (); - } - - XML_Parser mp_parser; - XMLStructureHandler *mp_struct_handler; - bool m_has_error; - std::string m_error; - int m_error_line, m_error_column; -}; - -void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->start_element (std::string (name)); -} - -void end_element_handler (void *user_data, const XML_Char *name) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->end_element (std::string (name)); -} - -void cdata_handler (void *user_data, const XML_Char *s, int len) -{ - XMLParserPrivateData *d = reinterpret_cast (user_data); - d->cdata (std::string (s, size_t (len))); -} - - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - mp_data->parse (source, struct_handler); - - // throws an exception if there is an error - mp_data->check_error (); -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#elif defined(HAVE_QT) - -#include -#include -#include - -namespace tl -{ - -// -------------------------------------------------------------------- -// A SAX handler for the Qt implementation - -class SAXHandler - : public QXmlDefaultHandler -{ -public: - SAXHandler (XMLStructureHandler *sh); - - virtual bool characters (const QString &ch); - virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); - virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); - virtual bool error (const QXmlParseException &exception); - virtual bool fatalError (const QXmlParseException &exception); - virtual bool warning (const QXmlParseException &exception); - virtual QString errorString () const; - - void setDocumentLocator (QXmlLocator *locator); - - const tl::XMLLocatedException *exception () const - { - return m_error.get (); - } - -private: - QXmlLocator *mp_locator; - XMLStructureHandler *mp_struct_handler; - std::unique_ptr m_error; - std::string m_error_string; -}; - -// -------------------------------------------------------------------------------------------------------- -// trureHandler implementation - -SAXHandler::SAXHandler (XMLStructureHandler *sh) - : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) -{ - // .. nothing yet .. -} - -void -SAXHandler::setDocumentLocator (QXmlLocator *locator) -{ - mp_locator = locator; -} - -bool -SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->start_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) -{ - std::string uri (tl::to_string (qs_uri)); - std::string lname (tl::to_string (qs_lname)); - std::string qname (tl::to_string (qs_qname)); - - try { - mp_struct_handler->end_element (uri, lname, qname); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -bool -SAXHandler::characters (const QString &t) -{ - try { - mp_struct_handler->characters (tl::to_string (t)); - } catch (tl::XMLException &ex) { - m_error_string = ex.raw_msg (); - return false; - } catch (tl::Exception &ex) { - m_error_string = ex.msg (); - return false; - } - - // successful - return true; -} - -QString -SAXHandler::errorString () const -{ - return tl::to_qstring (m_error_string); -} - - -bool -SAXHandler::error (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::fatalError (const QXmlParseException &ex) -{ - m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); - // stop reading - return false; -} - -bool -SAXHandler::warning (const QXmlParseException &ex) -{ - tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); - tl::warn << lex.msg (); - // continue - return true; -} - -// -------------------------------------------------------------------- -// SourcePrivateData implementation - -class XMLSourcePrivateData - : public QXmlInputSource -{ -public: - XMLSourcePrivateData () - : QXmlInputSource () - { - // .. nothing yet .. - } - - XMLSourcePrivateData (QIODevice *dev) - : QXmlInputSource (dev) - { - // .. nothing yet .. - } -}; - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - delete mp_source; - mp_source = 0; -} - -void -XMLSource::reset () -{ - mp_source->reset (); -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &string) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (string.c_str ())); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp)); - set_source (source); -} - -XMLStringSource::XMLStringSource (const char *cp, size_t len) -{ - XMLSourcePrivateData *source = new XMLSourcePrivateData (); - source->setData (QByteArray (cp, int (len))); - set_source (source); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// StreamIODevice definition and implementation - -class StreamIODevice - : public QIODevice -{ -public: - StreamIODevice (tl::InputStream &stream) - : mp_stream (&stream), - mp_progress (0), - m_has_error (false) - { - open (QIODevice::ReadOnly); - } - - StreamIODevice (tl::InputStream &stream, const std::string &progress_message) - : mp_stream (&stream), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (0), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - open (QIODevice::ReadOnly); - } - - StreamIODevice (const std::string &path, const std::string &progress_message) - : mp_stream_holder (new tl::InputStream (path)), - mp_progress (new AbsoluteProgress (progress_message, 100)), - m_has_error (false) - { - mp_stream = mp_stream_holder.get (); - mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); - mp_progress->set_unit (1024 * 1024); - open (QIODevice::ReadOnly); - } - - ~StreamIODevice () - { - if (mp_progress) { - delete mp_progress; - mp_progress = 0; - } - } - - virtual bool isSequential () const - { - return true; - } - - qint64 writeData (const char *, qint64) - { - tl_assert (false); - } - - qint64 readData (char *data, qint64 n) - { - try { - - if (mp_progress) { - mp_progress->set (mp_stream->pos ()); - } - - qint64 n0 = n; - for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { - // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). - if (*rd != '\r') { - *data++ = *rd; - --n; - } - } - - if (n0 == n) { - return -1; - } else { - return n0 - n; - } - - } catch (tl::Exception &ex) { - setErrorString (tl::to_qstring (ex.msg ())); - m_has_error = true; - return -1; - } - } - - bool has_error () const - { - return m_has_error; - } - -private: - tl::InputStream *mp_stream; - std::unique_ptr mp_stream_holder; - tl::AbsoluteProgress *mp_progress; - bool m_has_error; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -class XMLStreamSourcePrivateData - : public XMLSourcePrivateData -{ -public: - XMLStreamSourcePrivateData (StreamIODevice *io) - : XMLSourcePrivateData (io), mp_io (io) - { - // .. nothing yet .. - } - - virtual void fetchData () - { - QXmlInputSource::fetchData (); - - // This feature is actually missing in the original implementation: throw an exception on error - if (mp_io->has_error ()) { - throw tl::Exception (tl::to_string (mp_io->errorString ())); - } - } - -private: - std::unique_ptr mp_io; -}; - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); -} - -XMLFileSource::XMLFileSource (const std::string &path) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &s) -{ - set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -class XMLParserPrivateData - : public QXmlSimpleReader -{ -public: - XMLParserPrivateData () : QXmlSimpleReader () { } -}; - -XMLParser::XMLParser () - : mp_data (new XMLParserPrivateData ()) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - delete mp_data; - mp_data = 0; -} - -void -XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) -{ - SAXHandler handler (&struct_handler); - - mp_data->setContentHandler (&handler); - mp_data->setErrorHandler (&handler); - - bool result = mp_data->parse (source.source (), false /*=not incremental*/); - if (! result && handler.exception ()) { - throw tl::XMLLocatedException (*handler.exception ()); - } -} - -bool -XMLParser::is_available () -{ - return true; -} - -} - -#else - -namespace tl -{ - -// -------------------------------------------------------------------- -// XMLSource implementation - -XMLSource::XMLSource () - : mp_source (0) -{ - // .. nothing yet .. -} - -XMLSource::~XMLSource () -{ - // .. nothing yet .. -} - -void -XMLSource::reset () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStringSource implementation - -XMLStringSource::XMLStringSource (const std::string &) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *) -{ - tl_assert (false); -} - -XMLStringSource::XMLStringSource (const char *, size_t) -{ - tl_assert (false); -} - -XMLStringSource::~XMLStringSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLFileSource implementation - -XMLFileSource::XMLFileSource (const std::string &, const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::XMLFileSource (const std::string &) -{ - tl_assert (false); -} - -XMLFileSource::~XMLFileSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLStreamSource implementation - -XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) -{ - tl_assert (false); -} - -XMLStreamSource::XMLStreamSource (tl::InputStream &) -{ - tl_assert (false); -} - -XMLStreamSource::~XMLStreamSource () -{ - // .. nothing yet .. -} - -// -------------------------------------------------------------------- -// XMLParser implementation - -XMLParser::XMLParser () - : mp_data (0) -{ - // .. nothing yet .. -} - -XMLParser::~XMLParser () -{ - // .. nothing yet .. -} - -void -XMLParser::parse (XMLSource &, XMLStructureHandler &) -{ - tl_assert (false); -} - -bool -XMLParser::is_available () -{ - return false; -} - -} - -#endif - namespace tl { // ----------------------------------------------------------------- @@ -942,8 +62,140 @@ XMLElementProxy::~XMLElementProxy () mp_ptr = 0; } +// ----------------------------------------------------------------- +// XMLElementList implementation + +static size_t s_oid = 0; + +XMLElementList::XMLElementList () + : m_oid (++s_oid) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementBase &e) + : m_oid (++s_oid) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (XMLElementBase *e) + : m_oid (++s_oid) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::XMLElementList (const std::string &name, const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d) + : m_elements (d.m_elements), m_oid (d.m_oid), m_name (d.m_name) +{ + // .. nothing yet .. +} + +XMLElementList::XMLElementList (const XMLElementList &d, const XMLElementBase &e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +XMLElementList::XMLElementList (const XMLElementList &d, XMLElementBase *e) + : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +void XMLElementList::append (const XMLElementBase &e) +{ + m_elements.push_back (XMLElementProxy (e)); +} + +void XMLElementList::append (XMLElementBase *e) +{ + if (e) { + m_elements.push_back (XMLElementProxy (e)); + } +} + +XMLElementList::iterator XMLElementList::begin () const +{ + return m_elements.begin (); +} + +XMLElementList::iterator XMLElementList::end () const +{ + return m_elements.end (); +} + +XMLElementList XMLElementList::empty () +{ + return XMLElementList (); +} + +// ----------------------------------------------------------------- // XMLElementBase implementation +static std::string parse_name (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + return std::string (n, 0, hash); + } else { + return n; + } +} + +static int parse_tag (const std::string &n) +{ + auto hash = n.find ("#"); + if (hash != std::string::npos) { + tl::Extractor ex (n.c_str () + hash + 1); + int tag; + if (ex.try_read (tag)) { + return tag; + } + } + return -1; +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList &children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (new XMLElementList (children)), m_owns_child_list (true) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const std::string &name, const XMLElementList *children) + : m_name (parse_name (name)), m_tag (parse_tag (name)), mp_children (children), m_owns_child_list (false) +{ + // .. nothing yet .. +} + +XMLElementBase::XMLElementBase (const XMLElementBase &d) + : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) +{ + if (m_owns_child_list) { + mp_children = new XMLElementList (*d.mp_children); + } else { + mp_children = d.mp_children; + } +} + +XMLElementBase::~XMLElementBase () +{ + if (m_owns_child_list) { + delete const_cast (mp_children); + mp_children = 0; + } +} + void XMLElementBase::write_indent (tl::OutputStream &os, int indent) { @@ -975,6 +227,127 @@ XMLElementBase::write_string (tl::OutputStream &os, const std::string &s) } } +bool +XMLElementBase::check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const +{ + if (m_name == "*") { + return true; + } else { + return m_name == lname; // no namespace currently + } +} + +XMLElementBase::Cardinality +XMLElementBase::cardinality () const +{ + return Zero; +} + +XMLElementBase::iterator +XMLElementBase::begin () const +{ + return mp_children->begin (); +} + +XMLElementBase::iterator +XMLElementBase::end () const +{ + return mp_children->end (); +} + +std::string +XMLElementBase::name4code () const +{ + std::string res; + const char *n = name ().c_str (); + + if (! isalpha (*n) && *n != '_') { + res += '_'; + } + + while (*n) { + if (*n == '-') { + res += '_'; + } else if (isalnum (*n) || *n == '_') { + res += *n; + } + ++n; + } + + return res; +} + +std::string +XMLElementBase::create_def (std::map > &messages) const +{ + std::string res; + + auto m = messages.find (oid ()); + if (m != messages.end ()) { + + res += "message " + m->second.second + " {\n"; + + for (auto i = begin (); i != end (); ++i) { + const XMLElementBase *e = i->get (); + Cardinality c = e->cardinality (); + std::string entry = e->create_def_entry (messages); + if (! entry.empty ()) { + res += " "; + if (c != Zero) { + if (c == Many) { + res += "repeated "; + } else { + res += "optional "; + } + res += entry + "\n"; + } + } + } + + res += "}"; + + } + + return res; +} + +void +XMLElementBase::collect_messages (std::map > &messages) const +{ + for (auto i = begin (); i != end (); ++i) { + i->get ()->collect_messages (messages); + } +} + +std::string +XMLElementBase::make_message_name () const +{ + std::string res = mp_children->name (); + if (! res.empty ()) { + return res; + } + + // Capitalize names + std::string n4c = name4code (); + + bool upcase = true; + + const char *n = n4c.c_str (); + while (*n) { + if (*n == '_') { + upcase = true; + } else if (upcase) { + res += toupper (*n); + upcase = false; + } else { + res += *n; + } + ++n; + } + + return res; +} + // -------------------------------------------------------------------------------------------------------- // trureHandler implementation @@ -1066,5 +439,76 @@ XMLWriterState::XMLWriterState () // .. nothing yet .. } +// -------------------------------------------------------------------- +// PBParser implementation + +PBParser::PBParser () +{ + // .. nothing yet .. +} + +PBParser::~PBParser () +{ + // .. nothing yet .. +} + +void +PBParser::parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state) +{ + mp_state = reader_state; + parse_element (root, reader); +} + +void +PBParser::parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader) +{ + while (! reader.at_end ()) { + + int tag = reader.read_tag (); + + const XMLElementBase *new_element = 0; + if (parent) { + for (XMLElementBase::iterator c = parent->begin (); c != parent->end (); ++c) { + if ((*c)->tag () == tag) { + new_element = (*c).get (); + break; + } + } + } + + if (! new_element) { + reader.skip (); + } else { + new_element->pb_create (parent, *mp_state); + new_element->pb_parse (this, reader); + new_element->pb_finish (parent, *mp_state); + } + + } +} + +void +PBParser::expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name) +{ + int tag = reader.read_tag (); + if (tag != name_tag) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with ID %d (got %d)")), name_tag, tag)); + } + + std::string n; + reader.read (n); + if (n != name) { + reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with string '%s' (got '%s')")), name, n)); + } +} + +// -------------------------------------------------------------------- +// PBWriterState implementation + +PBWriterState::PBWriterState () +{ + // .. nothing yet .. +} + } diff --git a/src/tl/tl/tlXMLParser.h b/src/tl/tl/tlXMLParser.h index 226217228..fab046eff 100644 --- a/src/tl/tl/tlXMLParser.h +++ b/src/tl/tl/tlXMLParser.h @@ -29,6 +29,8 @@ #include #include +#include "tlXMLReader.h" +#include "tlProtocolBuffer.h" #include "tlAssert.h" #include "tlInternational.h" #include "tlString.h" @@ -40,339 +42,12 @@ namespace tl { /** - * @brief A basic XML parser error exception class + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. */ -class TL_PUBLIC XMLException : public tl::Exception -{ -public: - XMLException (const char *msg) - : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - XMLException (const std::string &msg) - : Exception (fmt (-1, -1).c_str (), msg.c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - /** - * @brief Raw (unprefixed) message of the XML parser - */ - const std::string & - raw_msg () const - { - return m_msg; - } - -protected: - XMLException (const std::string &msg, int line, int column) - : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), - m_msg (msg) - { - // .. nothing yet .. - } - -private: - std::string m_msg; - - static std::string fmt (int line, int /*column*/) - { - if (line < 0) { - return tl::to_string (tr ("XML parser error: %s")).c_str (); - } else { - return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); - } - } -}; - -/** - * @brief A XML parser error exception class that additionally provides line and column information - */ - -class TL_PUBLIC XMLLocatedException : public XMLException -{ -public: - XMLLocatedException (const std::string &msg, int line, int column) - : XMLException (msg, line, column), - m_line (line), m_column (column) - { - // .. nothing yet .. - } - - /** - * @brief Line number information of the exception - */ - int line () const - { - return m_line; - } - - /** - * @brief Column number information of the exception - */ - int column () const - { - return m_column; - } - -private: - int m_line; - int m_column; -}; - -/** - * @brief An object wrapper base class for target object management - * - * Implementations of this class through the XMLReaderProxy templates - * manage pointers to certain objects. - */ - -class TL_PUBLIC XMLReaderProxyBase -{ -public: - XMLReaderProxyBase () { } - virtual ~XMLReaderProxyBase () { } - virtual void release () = 0; - virtual void detach () = 0; -}; - -/** - * @brief An object wrapper base class for target object management specialized to a certain class - */ - -template -class TL_PUBLIC_TEMPLATE XMLReaderProxy - : public XMLReaderProxyBase -{ -public: - XMLReaderProxy (Obj *obj, bool owns_obj) - : mp_obj (obj), m_owns_obj (owns_obj) - { } - - virtual ~XMLReaderProxy () { } - - virtual void release () - { - if (m_owns_obj && mp_obj) { - delete mp_obj; - } - mp_obj = 0; - } - - virtual void detach () - { - m_owns_obj = false; - } - - Obj *ptr () const - { - return mp_obj; - } - -private: - Obj *mp_obj; - bool m_owns_obj; -}; - -/** - * @brief Helper class: A class tag - */ - -template -struct XMLObjTag -{ - XMLObjTag() { } - typedef Obj obj; -}; - -/** - * @brief Helper class: The reader state - * - * The reader state mainly comprises of a stack of objects being parsed and - * a string in which to collect cdata. - */ - -class TL_PUBLIC XMLReaderState -{ -public: - /** - * @brief Default constructor - */ - XMLReaderState (); - - /** - * @brief Destructor - */ - ~XMLReaderState (); - - /** - * @brief Push a new object on the stack - */ - template - void push (XMLObjTag /*tag*/) - { - m_objects.push_back (new XMLReaderProxy (new Obj (), true)); - } - - /** - * @brief Push an existing object on the stack - */ - template - void push (Obj *obj) - { - m_objects.push_back (new XMLReaderProxy (obj, false)); - } - - /** - * @brief Push an existing object on the stack with the ownership flag - */ - template - void push (Obj *obj, bool owner) - { - m_objects.push_back (new XMLReaderProxy (obj, owner)); - } - - /** - * @brief Get the top object - */ - template - Obj *back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Get the top object and release - */ - template - Obj *detach_back (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->detach (); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Pop an object from the stack - */ - template - void pop (XMLObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->release (); - delete m_objects.back (); - m_objects.pop_back (); - } - - /** - * @brief Empty predicate: true, if no more object is on the stack - */ - bool empty () const - { - return m_objects.empty (); - } - - /** - * @brief Obtain the parent object from the stack - */ - template - Obj *parent (XMLObjTag /*tag*/) - { - tl_assert (m_objects.size () > 1); - return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); - } - - /** - * @brief The cdata string collected - */ - std::string cdata; - -private: - std::vector m_objects; -}; - -// The opaque source type -class XMLSourcePrivateData; - -/** - * @brief A generic XML text source class - * - * This class is the base class providing input for - * the Qt XML parser and basically maps to a QXmlInputSource object - * for compatibility with the "libparsifal" branch. - */ - -class TL_PUBLIC XMLSource -{ -public: - XMLSource (); - ~XMLSource (); - - XMLSourcePrivateData *source () - { - return mp_source; - } - - void reset (); - -protected: - void set_source (XMLSourcePrivateData *source) - { - mp_source = source; - } - -private: - XMLSourcePrivateData *mp_source; -}; - -/** - * @brief A specialization of XMLSource to receive a string - */ - -class TL_PUBLIC XMLStringSource : public XMLSource -{ -public: - XMLStringSource (const std::string &string); - XMLStringSource (const char *cp); - XMLStringSource (const char *cp, size_t len); - ~XMLStringSource (); - -private: - std::string m_copy; -}; - -/** - * @brief A specialization of XMLSource to receive from a file - */ - -class TL_PUBLIC XMLFileSource : public XMLSource -{ -public: - XMLFileSource (const std::string &path); - XMLFileSource (const std::string &path, const std::string &progress_message); - ~XMLFileSource (); -}; - -/** - * @brief A generic stream source class - * - * This class implements a XML parser source from a tl::InputStream - */ - -class TL_PUBLIC XMLStreamSource : public XMLSource -{ -public: - XMLStreamSource (tl::InputStream &stream); - XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); - ~XMLStreamSource (); -}; - +class XMLReaderState; +class XMLReaderState; // ----------------------------------------------------------------- // The C++ structure definition interface (for use cases see tlXMLParser.ut) @@ -387,6 +62,18 @@ struct pass_by_ref_tag { pass_by_ref_tag () { } }; +struct pb_zero_cardinality_tag { + pb_zero_cardinality_tag () { } +}; + +struct pb_single_cardinality_tag { + pb_single_cardinality_tag () { } +}; + +struct pb_many_cardinality_tag { + pb_many_cardinality_tag () { } +}; + /** * @brief The XML handler implementation * @@ -483,66 +170,208 @@ public: typedef std::list children_list; typedef children_list::const_iterator iterator; - XMLElementList () + XMLElementList (); + XMLElementList (const XMLElementBase &e); + XMLElementList (XMLElementBase *e); + XMLElementList (const std::string &name, const XMLElementList &d); + XMLElementList (const XMLElementList &d); + XMLElementList (const XMLElementList &d, const XMLElementBase &e); + XMLElementList (const XMLElementList &d, XMLElementBase *e); + + void append (const XMLElementBase &e); + void append (XMLElementBase *e); + + iterator begin () const; + iterator end () const; + + static XMLElementList empty (); + + size_t oid () const { - // .. nothing yet .. + return m_oid; } - XMLElementList (const XMLElementBase &e) + const std::string &name () const { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - XMLElementList (const XMLElementList &d, const XMLElementBase &e) - : m_elements (d.m_elements) - { - m_elements.push_back (XMLElementProxy (e)); - } - - XMLElementList (const XMLElementList &d, XMLElementBase *e) - : m_elements (d.m_elements) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - void append (const XMLElementBase &e) - { - m_elements.push_back (XMLElementProxy (e)); - } - - void append (XMLElementBase *e) - { - if (e) { - m_elements.push_back (XMLElementProxy (e)); - } - } - - iterator begin () const - { - return m_elements.begin (); - } - - iterator end () const - { - return m_elements.end (); - } - - static XMLElementList empty () - { - return XMLElementList (); + return m_name; } private: std::list m_elements; + size_t m_oid; + std::string m_name; +}; + +/** + * @brief An object wrapper base class for target object management + * + * Implementations of this class through the XMLReaderProxy templates + * manage pointers to certain objects. + */ + +class TL_PUBLIC XMLReaderProxyBase +{ +public: + XMLReaderProxyBase () { } + virtual ~XMLReaderProxyBase () { } + virtual void release () = 0; + virtual void detach () = 0; +}; + +/** + * @brief An object wrapper base class for target object management specialized to a certain class + */ + +template +class TL_PUBLIC_TEMPLATE XMLReaderProxy + : public XMLReaderProxyBase +{ +public: + XMLReaderProxy (Obj *obj, bool owns_obj) + : mp_obj (obj), m_owns_obj (owns_obj) + { } + + virtual ~XMLReaderProxy () { } + + virtual void release () + { + if (m_owns_obj && mp_obj) { + delete mp_obj; + } + mp_obj = 0; + } + + virtual void detach () + { + m_owns_obj = false; + } + + Obj *ptr () const + { + return mp_obj; + } + +private: + Obj *mp_obj; + bool m_owns_obj; +}; + +/** + * @brief Helper class: A class tag + */ + +template +struct XMLObjTag +{ + XMLObjTag() { } + typedef Obj obj; +}; + +/** + * @brief Helper class: The reader state + * + * The reader state mainly comprises of a stack of objects being parsed and + * a string in which to collect cdata. + */ + +class TL_PUBLIC XMLReaderState +{ +public: + /** + * @brief Default constructor + */ + XMLReaderState (); + + /** + * @brief Destructor + */ + ~XMLReaderState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (XMLObjTag /*tag*/) + { + m_objects.push_back (new XMLReaderProxy (new Obj (), true)); + } + + /** + * @brief Push an existing object on the stack + */ + template + void push (Obj *obj) + { + m_objects.push_back (new XMLReaderProxy (obj, false)); + } + + /** + * @brief Push an existing object on the stack with the ownership flag + */ + template + void push (Obj *obj, bool owner) + { + m_objects.push_back (new XMLReaderProxy (obj, owner)); + } + + /** + * @brief Get the top object + */ + template + Obj *back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Get the top object and release + */ + template + Obj *detach_back (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->detach (); + return (dynamic_cast &> (*m_objects.back ())).ptr (); + } + + /** + * @brief Pop an object from the stack + */ + template + void pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + m_objects.back ()->release (); + delete m_objects.back (); + m_objects.pop_back (); + } + + /** + * @brief Empty predicate: true, if no more object is on the stack + */ + bool empty () const + { + return m_objects.empty (); + } + + /** + * @brief Obtain the parent object from the stack + */ + template + Obj *parent (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 1); + return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); + } + + /** + * @brief The cdata string collected + */ + std::string cdata; + +private: + std::vector m_objects; }; /** @@ -592,6 +421,78 @@ private: std::vector m_objects; }; +/** + * @brief The PB parser class + * This is the main entry point. It will take a PB reader, the structure (in "root") and + * a reader state initialized with the top level object. + */ +class TL_PUBLIC PBParser +{ +public: + PBParser (); + ~PBParser (); + + void parse (tl::ProtocolBufferReaderBase &reader, const XMLElementBase *root, XMLReaderState *reader_state); + void parse_element (const XMLElementBase *parent, tl::ProtocolBufferReaderBase &reader); + + void expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name); + + XMLReaderState &reader_state () + { + return *mp_state; + } + +private: + XMLReaderState *mp_state; +}; + +/** + * @brief Helper class: A stack of const objects being written (PB binding) + */ + +class TL_PUBLIC PBWriterState +{ +public: + /** + * @brief Default constructor + */ + PBWriterState (); + + /** + * @brief Push a new object on the stack + */ + template + void push (const Obj *obj) + { + m_objects.push_back (obj); + } + + /** + * @brief Pop an object from the stack + */ + template + const Obj *pop (XMLObjTag /*tag*/) + { + tl_assert (! m_objects.empty ()); + const Obj *obj = reinterpret_cast (m_objects.back ()); + m_objects.pop_back (); + return obj; + } + + /** + * @brief Obtain the parent object from the stack + */ + template + const Obj *back (XMLObjTag /*tag*/) + { + tl_assert (m_objects.size () > 0); + return reinterpret_cast (m_objects.end () [-1]); + } + +private: + std::vector m_objects; +}; + /** * @brief The XML element base object * @@ -606,35 +507,14 @@ class TL_PUBLIC XMLElementBase public: typedef XMLElementList::iterator iterator; - XMLElementBase (const std::string &name, const XMLElementList &children) - : m_name (name), mp_children (new XMLElementList (children)), m_owns_child_list (true) - { - // .. nothing yet .. - } + enum Cardinality { + Zero, Single, Many + }; - XMLElementBase (const std::string &name, const XMLElementList *children) - : m_name (name), mp_children (children), m_owns_child_list (false) - { - // .. nothing yet .. - } - - XMLElementBase (const XMLElementBase &d) - : m_name (d.m_name), m_owns_child_list (d.m_owns_child_list) - { - if (m_owns_child_list) { - mp_children = new XMLElementList (*d.mp_children); - } else { - mp_children = d.mp_children; - } - } - - virtual ~XMLElementBase () - { - if (m_owns_child_list) { - delete const_cast (mp_children); - mp_children = 0; - } - } + XMLElementBase (const std::string &name, const XMLElementList &children); + XMLElementBase (const std::string &name, const XMLElementList *children); + XMLElementBase (const XMLElementBase &d); + virtual ~XMLElementBase (); virtual XMLElementBase *clone () const = 0; @@ -648,32 +528,66 @@ public: static void write_indent (tl::OutputStream &os, int indent); static void write_string (tl::OutputStream &os, const std::string &s); + virtual void pb_create (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; + virtual void pb_finish (const XMLElementBase *parent, XMLReaderState &objs) const = 0; + + virtual void pb_write (const XMLElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } + + virtual Cardinality cardinality () const; + + size_t oid () const + { + return mp_children->oid (); + } + + int tag () const + { + return m_tag; + } + const std::string &name () const { return m_name; } - bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const + bool check_name (const std::string & /*uri*/, const std::string &lname, const std::string & /*qname*/) const; + + /** + * @brief Returns a name suitable for code + * Specifically, hyphens are replaced by underscores. + */ + std::string name4code () const; + + iterator begin () const; + iterator end () const; + + std::string create_def (std::map > &messages) const; + +protected: + virtual void collect_messages (std::map > &messages) const; + virtual std::string create_def_entry (std::map > &messages) const = 0; + + std::string make_message_name () const; + + static Cardinality get_cardinality (tl::pb_zero_cardinality_tag) { - if (m_name == "*") { - return true; - } else { - return m_name == lname; // no namespace currently - } + return Zero; } - iterator begin () const + static Cardinality get_cardinality (tl::pb_single_cardinality_tag) { - return mp_children->begin (); + return Single; } - iterator end () const + static Cardinality get_cardinality (tl::pb_many_cardinality_tag) { - return mp_children->end (); + return Many; } private: std::string m_name; + int m_tag; const XMLElementList *mp_children; bool m_owns_child_list; }; @@ -767,6 +681,64 @@ public: return (! r.at_end ()); } + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const + { + XMLObjTag tag; + objs.push (tag); + } + + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + reader.open (); + parser->parse_element (this, reader); + reader.close (); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + + Read r (m_r); + r.start (*objs.back (parent_tag)); + while (! r.at_end ()) { + typedef typename Read::tag read_tag_type; + pb_write_obj (r (), tag (), writer, read_tag_type (), objs); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > &messages) const + { + if (messages.find (oid ()) == messages.end ()) { + messages [oid ()] = std::make_pair (this, make_message_name ()); + XMLElementBase::collect_messages (messages); + } + } + + virtual std::string create_def_entry (std::map > &messages) const + { + auto m = messages.find (oid ()); + if (m != messages.end ()) { + return m->second.second + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } else { + return std::string (); + } + } + private: Read m_r; Write m_w; @@ -791,6 +763,43 @@ private: } objs.pop (tag); } + + // this write helper is used if the reader delivers an object by value + void pb_write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_value_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (! writer.is_counting ()) { + break; + } + } + } + + void pb_write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pass_by_ref_tag, PBWriterState &objs) const + { + XMLObjTag self_tag; + + for (unsigned int pass = 0; pass < 2; ++pass) { + writer.begin_seq (tag, pass == 0); + objs.push (&obj); + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, objs); + } + objs.pop (self_tag); + writer.end_seq (); + if (writer.is_counting ()) { + break; + } + } + } }; /** @@ -802,23 +811,23 @@ private: template class TL_PUBLIC_TEMPLATE XMLElementWithParentRef - : public XMLElementBase + : public XMLElement { public: XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElement (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) - : XMLElementBase (name, children), m_r (r), m_w (w) + : XMLElement (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const XMLElementWithParentRef &d) - : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) + : XMLElement (d) { // .. nothing yet .. } @@ -835,11 +844,6 @@ public: objs.push (new Obj (objs.back (parent_tag)), true); } - virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const - { - // .. nothing yet .. - } - virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const { XMLObjTag tag; @@ -848,53 +852,18 @@ public: objs.pop (tag); } - virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - XMLElementBase::write_indent (os, indent); - os << "<" << this->name () << ">\n"; - typedef typename Read::tag read_tag_type; - read_tag_type read_tag; - write_obj (r (), os, indent, read_tag, objs); - XMLElementBase::write_indent (os, indent); - os << "name () << ">\n"; - r.next (); - } - } - - virtual bool has_any (XMLWriterState &objs) const - { - XMLObjTag parent_tag; - Read r (m_r); - r.start (*objs.back (parent_tag)); - return (! r.at_end ()); - } - -private: - Read m_r; - Write m_w; - - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } - objs.pop (tag); + XMLObjTag parent_tag; + objs.push (new Obj (objs.back (parent_tag)), true); } - void write_obj (const Obj &obj, tl::OutputStream &os, int indent, tl::pass_by_ref_tag, XMLWriterState &objs) const + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const { XMLObjTag tag; - objs.push (&obj); - for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, os, indent + 1, objs); - } + XMLObjTag parent_tag; + m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } }; @@ -992,10 +961,278 @@ public: return (! r.at_end ()); } + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // .. nothing yet .. + } + + virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + + XMLReaderState value_obj; + value_obj.push (tag); + + read (reader, *value_obj.back (tag)); + m_w (*parser->reader_state ().back (parent_tag), value_obj); + + value_obj.pop (tag); + } + + virtual void pb_write (const XMLElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const + { + XMLObjTag parent_tag; + Read r (m_r); + r.start (* objs.back (parent_tag)); + while (! r.at_end ()) { + write (writer, tag (), r ()); + r.next (); + } + } + + virtual Cardinality cardinality () const + { + typedef typename Read::cardinality cardinality_type; + return get_cardinality (cardinality_type ()); + } + + virtual void collect_messages (std::map > & /*messages*/) const + { + // no messages here. + } + + virtual std::string create_def_entry (std::map > & /*messages*/) const + { + const Value *v = 0; + return typestring (v) + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; + } + private: Read m_r; Write m_w; Converter m_c; + + // write incarnations + void write (tl::ProtocolBufferWriterBase &writer, int tag, float v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, double v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint8_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int8_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint16_t v) const + { + writer.write (tag, (uint32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int16_t v) const + { + writer.write (tag, (int32_t) v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int32_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, uint64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, int64_t v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, bool v) const + { + writer.write (tag, v); + } + + void write (tl::ProtocolBufferWriterBase &writer, int tag, const std::string &v) const + { + writer.write (tag, v); + } + + template + void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const + { + writer.write (tag, m_c.pb_encode (v)); + } + + // read incarnations + void read (tl::ProtocolBufferReaderBase &reader, float &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, double &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint8_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int8_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint16_t &v) const + { + uint32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, int16_t &v) const + { + int32_t vv = 0; + reader.read (vv); + // TODO: check for overflow? + v = vv; + } + + void read (tl::ProtocolBufferReaderBase &reader, uint32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int32_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, uint64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, int64_t &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, bool &v) const + { + reader.read (v); + } + + void read (tl::ProtocolBufferReaderBase &reader, std::string &v) const + { + reader.read (v); + } + + template + void read (tl::ProtocolBufferReaderBase &reader, T &v) const + { + typename Converter::pb_type vv; + reader.read (vv); + m_c.pb_decode (vv, v); + } + + // type strings + std::string typestring (const float *) const + { + return "float"; + } + + std::string typestring (const double *) const + { + return "double"; + } + + std::string typestring (const uint8_t *) const + { + return "uint32"; + } + + std::string typestring (const uint16_t *) const + { + return "uint32"; + } + + std::string typestring (const uint32_t *) const + { + return "uint32"; + } + + std::string typestring (const uint64_t *) const + { + return "uint64"; + } + + std::string typestring (const int8_t *) const + { + return "sint32"; + } + + std::string typestring (const int16_t *) const + { + return "sint32"; + } + + std::string typestring (const int32_t *) const + { + return "sint32"; + } + + std::string typestring (const int64_t *) const + { + return "sint64"; + } + + std::string typestring (const bool *) const + { + return "uint32"; + } + + std::string typestring (const std::string *) const + { + return "string"; + } + + template + std::string typestring (const T *) const + { + const typename Converter::pb_type *v = 0; + return typestring (v); + } }; /** @@ -1143,11 +1380,90 @@ public: tl_assert (rs.empty ()); } + /** + * @brief Serializes the given object (root) to the writer + */ + void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const + { + PBWriterState writer_state; + writer_state.push (& root); + + writer.write (tag (), name ()); + + for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { + c->get ()->pb_write (this, writer, writer_state); + } + } + + /** + * @brief Deserializes the given object (root) from the reader + */ + void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const + { + XMLObjTag self_tag; + XMLReaderState rs; + rs.push (&root); + PBParser h; + h.expect_header (reader, tag (), name ()); + h.parse (reader, this, &rs); + rs.pop (self_tag); + tl_assert (rs.empty ()); + } + + /** + * @brief Produces a definition for the protoc compiler + */ + std::string create_def () const + { + std::map > msgs; + collect_messages (msgs); + msgs[oid ()] = std::make_pair (this, make_message_name ()); + + std::string res = "// created from KLayout proto definition '" + name () + "'\n\n"; + res += "syntax = \"proto2\";"; + + for (auto i = msgs.begin (); i != msgs.end (); ++i) { + std::string entry = i->second.first->create_def (msgs); + if (! entry.empty ()) { + res += "\n"; + res += "\n"; + res += entry; + } + } + + return res; + } + private: virtual void write (const XMLElementBase*, tl::OutputStream &, int, XMLWriterState &) const { // .. see write (os) } + + virtual void pb_write (const XMLElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const + { + // disable base class implementation + } + + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const + { + // disable base class implementation + } + + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const + { + // disable base class implementation + } + + virtual std::string create_def_entry (std::map > &) const + { + return std::string (); + } }; /** @@ -1267,6 +1583,7 @@ template struct XMLMemberDummyReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_zero_cardinality_tag cardinality; XMLMemberDummyReadAdaptor () { @@ -1298,6 +1615,7 @@ template struct XMLMemberReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberReadAdaptor (Value Parent::*member) : mp_member (member), mp_owner (0), m_done (false) @@ -1336,6 +1654,7 @@ template struct XMLMemberAccRefReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1374,6 +1693,7 @@ template struct XMLMemberAccReadAdaptor { typedef pass_by_value_tag tag; + typedef pb_single_cardinality_tag cardinality; XMLMemberAccReadAdaptor (Value (Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1412,6 +1732,7 @@ template struct XMLMemberIterReadAdaptor { typedef pass_by_ref_tag tag; + typedef pb_many_cardinality_tag cardinality; XMLMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) : mp_begin (begin), mp_end (end) diff --git a/src/tl/tl/tlXMLReader.cc b/src/tl/tl/tlXMLReader.cc new file mode 100644 index 000000000..49f0509dc --- /dev/null +++ b/src/tl/tl/tlXMLReader.cc @@ -0,0 +1,913 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include "tlXMLReader.h" +#include "tlXMLParser.h" +#include "tlString.h" +#include "tlLog.h" +#include "tlAssert.h" +#include "tlProgress.h" + +#include +#include + +#if defined(HAVE_EXPAT) + +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData +{ +public: + XMLSourcePrivateData (tl::InputStream *stream) + : mp_stream_holder (stream), + m_has_error (false) + { + mp_stream = stream; + } + + XMLSourcePrivateData (tl::InputStream *stream, const std::string &progress_message) + : mp_stream_holder (stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + XMLSourcePrivateData (tl::InputStream &stream) + : m_has_error (false) + { + mp_stream = &stream; + } + + XMLSourcePrivateData (tl::InputStream &stream, const std::string &progress_message) + : mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = &stream; + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + } + + int read (char *data, size_t n) + { + try { + + if (mp_progress.get ()) { + mp_progress->set (mp_stream->pos ()); + } + + size_t n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; --n) { + *data++ = *rd; + } + + return int (n0 - n); + + } catch (tl::Exception &ex) { + m_error = ex.msg (); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + + const std::string &error_msg () const + { + return m_error; + } + + void reset () + { + mp_stream->reset (); + } + +private: + std::unique_ptr mp_stream_holder; + tl::InputStream *mp_stream; + std::unique_ptr mp_progress; + bool m_has_error; + std::string m_error; +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) + : m_copy (string) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (m_copy.c_str (), string.size ())))); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, strlen (cp))))); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (new tl::InputMemoryStream (cp, len)))); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path), progress_message)); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLSourcePrivateData (new tl::InputStream (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLSourcePrivateData (s, progress_message)); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLSourcePrivateData (s)); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +void XMLCALL start_element_handler (void *user_data, const XML_Char *name, const XML_Char **atts); +void XMLCALL end_element_handler (void *user_data, const XML_Char *name); +void XMLCALL cdata_handler (void *user_data, const XML_Char *s, int len); + +static std::string get_lname (const std::string &name) +{ + size_t colon = name.find (':'); + if (colon != std::string::npos) { + return std::string (name, colon + 1, name.size () - colon - 1); + } else { + return name; + } +} + +class XMLParserPrivateData +{ +public: + XMLParserPrivateData () + : mp_struct_handler (0) + { + mp_parser = XML_ParserCreate ("UTF-8"); + tl_assert (mp_parser != NULL); + } + + ~XMLParserPrivateData () + { + if (mp_parser != NULL) { + XML_ParserFree (mp_parser); + } + } + + void start_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->start_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void end_element (const std::string &name) + { + try { + // TODO: Provide namespace URI? + mp_struct_handler->end_element (std::string (), get_lname (name), name); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void cdata (const std::string &cdata) + { + try { + mp_struct_handler->characters (cdata); + } catch (tl::Exception &ex) { + error (ex); + } + } + + void parse (tl::XMLSource &source, XMLStructureHandler &struct_handler) + { + m_has_error = false; + mp_struct_handler = &struct_handler; + + // Just in case we want to reuse it ... + XML_ParserReset (mp_parser, NULL); + XML_SetUserData (mp_parser, (void *) this); + XML_SetElementHandler (mp_parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler (mp_parser, cdata_handler); + + const int chunk = 65536; + char buffer [chunk]; + + int n; + + do { + + n = source.source ()->read (buffer, chunk); + if (n < 0) { + break; + } + + XML_Status status = XML_Parse (mp_parser, buffer, n, n < chunk /*is final*/); + if (status == XML_STATUS_ERROR) { + m_has_error = true; + m_error = XML_ErrorString (XML_GetErrorCode (mp_parser)); + m_error_line = XML_GetErrorLineNumber (mp_parser); + m_error_column = XML_GetErrorColumnNumber (mp_parser); + } + + } while (n == chunk && !m_has_error); + } + + void check_error () + { + if (m_has_error) { + throw tl::XMLLocatedException (m_error, m_error_line, m_error_column); + } + } + +private: + void error (tl::Exception &ex) + { + m_has_error = true; + m_error_line = XML_GetCurrentLineNumber (mp_parser); + m_error_column = XML_GetCurrentColumnNumber (mp_parser); + m_error = ex.msg (); + } + + XML_Parser mp_parser; + XMLStructureHandler *mp_struct_handler; + bool m_has_error; + std::string m_error; + int m_error_line, m_error_column; +}; + +void start_element_handler (void *user_data, const XML_Char *name, const XML_Char ** /*atts*/) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->start_element (std::string (name)); +} + +void end_element_handler (void *user_data, const XML_Char *name) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->end_element (std::string (name)); +} + +void cdata_handler (void *user_data, const XML_Char *s, int len) +{ + XMLParserPrivateData *d = reinterpret_cast (user_data); + d->cdata (std::string (s, size_t (len))); +} + + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + mp_data->parse (source, struct_handler); + + // throws an exception if there is an error + mp_data->check_error (); +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#elif defined(HAVE_QT) + +#include +#include +#include + +namespace tl +{ + +// -------------------------------------------------------------------- +// A SAX handler for the Qt implementation + +class SAXHandler + : public QXmlDefaultHandler +{ +public: + SAXHandler (XMLStructureHandler *sh); + + virtual bool characters (const QString &ch); + virtual bool endElement (const QString &namespaceURI, const QString &localName, const QString &qName); + virtual bool startElement (const QString &namespaceURI, const QString &localName, const QString &qName, const QXmlAttributes &atts); + virtual bool error (const QXmlParseException &exception); + virtual bool fatalError (const QXmlParseException &exception); + virtual bool warning (const QXmlParseException &exception); + virtual QString errorString () const; + + void setDocumentLocator (QXmlLocator *locator); + + const tl::XMLLocatedException *exception () const + { + return m_error.get (); + } + +private: + QXmlLocator *mp_locator; + XMLStructureHandler *mp_struct_handler; + std::unique_ptr m_error; + std::string m_error_string; +}; + +// -------------------------------------------------------------------------------------------------------- +// trureHandler implementation + +SAXHandler::SAXHandler (XMLStructureHandler *sh) + : QXmlDefaultHandler (), mp_locator (0), mp_struct_handler (sh) +{ + // .. nothing yet .. +} + +void +SAXHandler::setDocumentLocator (QXmlLocator *locator) +{ + mp_locator = locator; +} + +bool +SAXHandler::startElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname, const QXmlAttributes & /*atts*/) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->start_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::endElement (const QString &qs_uri, const QString &qs_lname, const QString &qs_qname) +{ + std::string uri (tl::to_string (qs_uri)); + std::string lname (tl::to_string (qs_lname)); + std::string qname (tl::to_string (qs_qname)); + + try { + mp_struct_handler->end_element (uri, lname, qname); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +bool +SAXHandler::characters (const QString &t) +{ + try { + mp_struct_handler->characters (tl::to_string (t)); + } catch (tl::XMLException &ex) { + m_error_string = ex.raw_msg (); + return false; + } catch (tl::Exception &ex) { + m_error_string = ex.msg (); + return false; + } + + // successful + return true; +} + +QString +SAXHandler::errorString () const +{ + return tl::to_qstring (m_error_string); +} + + +bool +SAXHandler::error (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::fatalError (const QXmlParseException &ex) +{ + m_error.reset (new tl::XMLLocatedException (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ())); + // stop reading + return false; +} + +bool +SAXHandler::warning (const QXmlParseException &ex) +{ + tl::XMLLocatedException lex (tl::to_string (ex.message ()), ex.lineNumber (), ex.columnNumber ()); + tl::warn << lex.msg (); + // continue + return true; +} + +// -------------------------------------------------------------------- +// SourcePrivateData implementation + +class XMLSourcePrivateData + : public QXmlInputSource +{ +public: + XMLSourcePrivateData () + : QXmlInputSource () + { + // .. nothing yet .. + } + + XMLSourcePrivateData (QIODevice *dev) + : QXmlInputSource (dev) + { + // .. nothing yet .. + } +}; + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + delete mp_source; + mp_source = 0; +} + +void +XMLSource::reset () +{ + mp_source->reset (); +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &string) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (string.c_str ())); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp)); + set_source (source); +} + +XMLStringSource::XMLStringSource (const char *cp, size_t len) +{ + XMLSourcePrivateData *source = new XMLSourcePrivateData (); + source->setData (QByteArray (cp, int (len))); + set_source (source); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// StreamIODevice definition and implementation + +class StreamIODevice + : public QIODevice +{ +public: + StreamIODevice (tl::InputStream &stream) + : mp_stream (&stream), + mp_progress (0), + m_has_error (false) + { + open (QIODevice::ReadOnly); + } + + StreamIODevice (tl::InputStream &stream, const std::string &progress_message) + : mp_stream (&stream), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (0), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + open (QIODevice::ReadOnly); + } + + StreamIODevice (const std::string &path, const std::string &progress_message) + : mp_stream_holder (new tl::InputStream (path)), + mp_progress (new AbsoluteProgress (progress_message, 100)), + m_has_error (false) + { + mp_stream = mp_stream_holder.get (); + mp_progress->set_format (tl::to_string (tr ("%.0f MB"))); + mp_progress->set_unit (1024 * 1024); + open (QIODevice::ReadOnly); + } + + ~StreamIODevice () + { + if (mp_progress) { + delete mp_progress; + mp_progress = 0; + } + } + + virtual bool isSequential () const + { + return true; + } + + qint64 writeData (const char *, qint64) + { + tl_assert (false); + } + + qint64 readData (char *data, qint64 n) + { + try { + + if (mp_progress) { + mp_progress->set (mp_stream->pos ()); + } + + qint64 n0 = n; + for (const char *rd = 0; n > 0 && (rd = mp_stream->get (1)) != 0; ) { + // NOTE: we skip CR to compensate for Windows CRLF line terminators (issue #419). + if (*rd != '\r') { + *data++ = *rd; + --n; + } + } + + if (n0 == n) { + return -1; + } else { + return n0 - n; + } + + } catch (tl::Exception &ex) { + setErrorString (tl::to_qstring (ex.msg ())); + m_has_error = true; + return -1; + } + } + + bool has_error () const + { + return m_has_error; + } + +private: + tl::InputStream *mp_stream; + std::unique_ptr mp_stream_holder; + tl::AbsoluteProgress *mp_progress; + bool m_has_error; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +class XMLStreamSourcePrivateData + : public XMLSourcePrivateData +{ +public: + XMLStreamSourcePrivateData (StreamIODevice *io) + : XMLSourcePrivateData (io), mp_io (io) + { + // .. nothing yet .. + } + + virtual void fetchData () + { + QXmlInputSource::fetchData (); + + // This feature is actually missing in the original implementation: throw an exception on error + if (mp_io->has_error ()) { + throw tl::Exception (tl::to_string (mp_io->errorString ())); + } + } + +private: + std::unique_ptr mp_io; +}; + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &path, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path, progress_message))); +} + +XMLFileSource::XMLFileSource (const std::string &path) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (path))); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &s, const std::string &progress_message) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s, progress_message))); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &s) +{ + set_source (new XMLStreamSourcePrivateData (new StreamIODevice (s))); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +class XMLParserPrivateData + : public QXmlSimpleReader +{ +public: + XMLParserPrivateData () : QXmlSimpleReader () { } +}; + +XMLParser::XMLParser () + : mp_data (new XMLParserPrivateData ()) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + delete mp_data; + mp_data = 0; +} + +void +XMLParser::parse (XMLSource &source, XMLStructureHandler &struct_handler) +{ + SAXHandler handler (&struct_handler); + + mp_data->setContentHandler (&handler); + mp_data->setErrorHandler (&handler); + + bool result = mp_data->parse (source.source (), false /*=not incremental*/); + if (! result && handler.exception ()) { + throw tl::XMLLocatedException (*handler.exception ()); + } +} + +bool +XMLParser::is_available () +{ + return true; +} + +} + +#else + +namespace tl +{ + +// -------------------------------------------------------------------- +// XMLSource implementation + +XMLSource::XMLSource () + : mp_source (0) +{ + // .. nothing yet .. +} + +XMLSource::~XMLSource () +{ + // .. nothing yet .. +} + +void +XMLSource::reset () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStringSource implementation + +XMLStringSource::XMLStringSource (const std::string &) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *) +{ + tl_assert (false); +} + +XMLStringSource::XMLStringSource (const char *, size_t) +{ + tl_assert (false); +} + +XMLStringSource::~XMLStringSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLFileSource implementation + +XMLFileSource::XMLFileSource (const std::string &, const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::XMLFileSource (const std::string &) +{ + tl_assert (false); +} + +XMLFileSource::~XMLFileSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLStreamSource implementation + +XMLStreamSource::XMLStreamSource (tl::InputStream &, const std::string &) +{ + tl_assert (false); +} + +XMLStreamSource::XMLStreamSource (tl::InputStream &) +{ + tl_assert (false); +} + +XMLStreamSource::~XMLStreamSource () +{ + // .. nothing yet .. +} + +// -------------------------------------------------------------------- +// XMLParser implementation + +XMLParser::XMLParser () + : mp_data (0) +{ + // .. nothing yet .. +} + +XMLParser::~XMLParser () +{ + // .. nothing yet .. +} + +void +XMLParser::parse (XMLSource &, XMLStructureHandler &) +{ + tl_assert (false); +} + +bool +XMLParser::is_available () +{ + return false; +} + +} + +#endif diff --git a/src/tl/tl/tlXMLReader.h b/src/tl/tl/tlXMLReader.h new file mode 100644 index 000000000..3b8423e3d --- /dev/null +++ b/src/tl/tl/tlXMLReader.h @@ -0,0 +1,215 @@ + +/* + + KLayout Layout Viewer + Copyright (C) 2006-2024 Matthias Koefferlein + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#ifndef HDR_tlXMLReader +#define HDR_tlXMLReader + +#include "tlCommon.h" + +#include +#include + +#include "tlAssert.h" +#include "tlInternational.h" +#include "tlString.h" +#include "tlStream.h" + +class QIODevice; + +namespace tl +{ + +/** + * NOTE: This XML parser package also supports a ProtocolBuffer flavor. + * This allows binding the same scheme to efficient binary PB format. + */ + +class ProtocolBufferReaderBase; +class ProtocolBufferWriterBase; + +/** + * @brief A basic XML parser error exception class + */ + +class TL_PUBLIC XMLException : public tl::Exception +{ +public: + XMLException (const char *msg) + : Exception (tl::to_string (tr ("XML parser error: %s")).c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + XMLException (const std::string &msg) + : Exception (fmt (-1, -1).c_str (), msg.c_str ()), + m_msg (msg) + { + // .. nothing yet .. + } + + /** + * @brief Raw (unprefixed) message of the XML parser + */ + const std::string & + raw_msg () const + { + return m_msg; + } + +protected: + XMLException (const std::string &msg, int line, int column) + : Exception (fmt (line, column).c_str (), msg.c_str (), line, column), + m_msg (msg) + { + // .. nothing yet .. + } + +private: + std::string m_msg; + + static std::string fmt (int line, int /*column*/) + { + if (line < 0) { + return tl::to_string (tr ("XML parser error: %s")).c_str (); + } else { + return tl::to_string (tr ("XML parser error: %s in line %d, column %d")).c_str (); + } + } +}; + +/** + * @brief A XML parser error exception class that additionally provides line and column information + */ + +class TL_PUBLIC XMLLocatedException : public XMLException +{ +public: + XMLLocatedException (const std::string &msg, int line, int column) + : XMLException (msg, line, column), + m_line (line), m_column (column) + { + // .. nothing yet .. + } + + /** + * @brief Line number information of the exception + */ + int line () const + { + return m_line; + } + + /** + * @brief Column number information of the exception + */ + int column () const + { + return m_column; + } + +private: + int m_line; + int m_column; +}; + +// The opaque source type +class XMLSourcePrivateData; + +/** + * @brief A generic XML text source class + * + * This class is the base class providing input for + * the Qt XML parser and basically maps to a QXmlInputSource object + * for compatibility with the "libparsifal" branch. + */ + +class TL_PUBLIC XMLSource +{ +public: + XMLSource (); + ~XMLSource (); + + XMLSourcePrivateData *source () + { + return mp_source; + } + + void reset (); + +protected: + void set_source (XMLSourcePrivateData *source) + { + mp_source = source; + } + +private: + XMLSourcePrivateData *mp_source; +}; + +/** + * @brief A specialization of XMLSource to receive a string + */ + +class TL_PUBLIC XMLStringSource : public XMLSource +{ +public: + XMLStringSource (const std::string &string); + XMLStringSource (const char *cp); + XMLStringSource (const char *cp, size_t len); + ~XMLStringSource (); + +private: + std::string m_copy; +}; + +/** + * @brief A specialization of XMLSource to receive from a file + */ + +class TL_PUBLIC XMLFileSource : public XMLSource +{ +public: + XMLFileSource (const std::string &path); + XMLFileSource (const std::string &path, const std::string &progress_message); + ~XMLFileSource (); +}; + +/** + * @brief A generic stream source class + * + * This class implements a XML parser source from a tl::InputStream + */ + +class TL_PUBLIC XMLStreamSource : public XMLSource +{ +public: + XMLStreamSource (tl::InputStream &stream); + XMLStreamSource (tl::InputStream &stream, const std::string &progress_message); + ~XMLStreamSource (); +}; + +} // namespace tl + +#endif + diff --git a/src/tl/unit_tests/tlProtocolBufferTests.cc b/src/tl/unit_tests/tlProtocolBufferTests.cc index 75289aed1..000ffda67 100644 --- a/src/tl/unit_tests/tlProtocolBufferTests.cc +++ b/src/tl/unit_tests/tlProtocolBufferTests.cc @@ -20,16 +20,14 @@ */ -#include "tlProtocolBufferStruct.h" +#include "tlXMLParser.h" +#include "tlProtocolBuffer.h" #include "tlUnitTest.h" #include "tlFileUtils.h" #include #include -// @@@ -// Missing: all kind of variants (uint8_t, ...), float - // Basic tests of reader and writer TEST (1_BasicTypes) { @@ -205,20 +203,20 @@ struct Root { const Child &get_child () const { return m_child; } }; -static tl::PBElementList child_struct ("Child", - tl::pb_make_member (&Child::txt, "txt", 1) + - tl::pb_make_member (&Child::d, "d", 2) + - tl::pb_make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children", 3, &child_struct) +static tl::XMLElementList child_struct ("Child", + tl::make_member (&Child::txt, "txt#1") + + tl::make_member (&Child::d, "d#2") + + tl::make_element (&Child::begin_children, &Child::end_children, &Child::add_child, "children#3", &child_struct) ); -static tl::PBStruct structure ("pbtest-struct", 88888888, - tl::pb_make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub", 1) + - tl::pb_make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub", 2) + - tl::pb_make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children", 3, &child_struct) + - tl::pb_make_element (&Root::get_child, &Root::set_child, "child", 4, &child_struct) + - tl::pb_make_member (&Root::m, "m", 5) + - tl::pb_make_member (&Root::get_mi, &Root::set_mi, "mi", 6) + - tl::pb_make_member (&Root::b, "b", 7) +static tl::XMLStruct structure ("pbtest-struct#88888888", + tl::make_member (&Root::begin_subs, &Root::end_subs, &Root::add_sub, "sub#1") + + tl::make_member (&Root::begin_isubs, &Root::end_isubs, &Root::add_isub, "isub#2") + + tl::make_element (&Root::begin_children, &Root::end_children, &Root::add_child, "children#3", &child_struct) + + tl::make_element (&Root::get_child, &Root::set_child, "child#4", &child_struct) + + tl::make_member (&Root::m, "m#5") + + tl::make_member (&Root::get_mi, &Root::set_mi, "mi#6") + + tl::make_member (&Root::b, "b#7") ); static void build_struct (Root &root) @@ -405,10 +403,13 @@ struct TestClassEnumConverter break; } } + + void from_string (const std::string &, TestClass::enum_type) const { } + std::string to_string (TestClass::enum_type) const { return std::string (); } }; -tl::PBStruct tc_structure ("pbtest-tc", 1, - tl::pb_make_member (&TestClass::e, "e", 2, TestClassEnumConverter ()) +tl::XMLStruct tc_structure ("pbtest-tc#1", + tl::make_member (&TestClass::e, "e#2", TestClassEnumConverter ()) ); TEST (101_Converter) From 6857b55498d3484d7164bf0267408ffce9676b81 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 11:47:18 +0200 Subject: [PATCH 17/22] WIP --- src/db/db/dbStream.h | 1 + src/tl/tl/tl.pro | 2 - src/tl/tl/tlProtocolBufferStruct.cc | 366 ----- src/tl/tl/tlProtocolBufferStruct.h | 2005 --------------------------- src/tl/tl/tlXMLParser.h | 12 + 5 files changed, 13 insertions(+), 2373 deletions(-) delete mode 100644 src/tl/tl/tlProtocolBufferStruct.cc delete mode 100644 src/tl/tl/tlProtocolBufferStruct.h diff --git a/src/db/db/dbStream.h b/src/db/db/dbStream.h index 97dfee6ef..ace7531aa 100644 --- a/src/db/db/dbStream.h +++ b/src/db/db/dbStream.h @@ -154,6 +154,7 @@ class StreamOptionsReadAdaptor { public: typedef tl::pass_by_ref_tag tag; + typedef tl::pb_zero_cardinality_tag cardinality; StreamOptionsReadAdaptor () : mp_options (0), m_done (false) diff --git a/src/tl/tl/tl.pro b/src/tl/tl/tl.pro index 98c1a41b9..508bc5357 100644 --- a/src/tl/tl/tl.pro +++ b/src/tl/tl/tl.pro @@ -30,7 +30,6 @@ SOURCES = \ tlProgress.cc \ tlPixelBuffer.cc \ tlProtocolBuffer.cc \ - tlProtocolBufferStruct.cc \ tlResources.cc \ tlScriptError.cc \ tlSleep.cc \ @@ -89,7 +88,6 @@ HEADERS = \ tlProgress.h \ tlPixelBuffer.h \ tlProtocolBuffer.h \ - tlProtocolBufferStruct.h \ tlResources.h \ tlReuseVector.h \ tlScriptError.h \ diff --git a/src/tl/tl/tlProtocolBufferStruct.cc b/src/tl/tl/tlProtocolBufferStruct.cc deleted file mode 100644 index 25ecbd43a..000000000 --- a/src/tl/tl/tlProtocolBufferStruct.cc +++ /dev/null @@ -1,366 +0,0 @@ - -/* - - KLayout Layout Viewer - Copyright (C) 2006-2024 Matthias Koefferlein - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include "tlProtocolBufferStruct.h" -#include - -namespace tl -{ - -static size_t s_oid = 0; - -// -------------------------------------------------------------------- -// PBElementList implementation - -PBElementList::PBElementList () - : m_oid (++s_oid) -{ - // .. nothing yet .. -} - -PBElementList::PBElementList (const PBElementBase &e) - : m_oid (++s_oid) -{ - m_elements.push_back (PBElementProxy (e)); -} - -PBElementList::PBElementList (PBElementBase *e) - : m_oid (++s_oid) -{ - if (e) { - m_elements.push_back (PBElementProxy (e)); - } -} - -PBElementList::PBElementList (const std::string &name, const PBElementList &d) - : m_elements (d.m_elements), m_oid (d.m_oid), m_name (name) -{ - // .. nothing yet .. -} - -PBElementList::PBElementList (const PBElementList &d) - : m_elements (d.m_elements), m_oid (d.m_oid), m_name (d.m_name) -{ - // .. nothing yet .. -} - -PBElementList::PBElementList (const PBElementList &d, const PBElementBase &e) - : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) -{ - m_elements.push_back (PBElementProxy (e)); -} - -PBElementList::PBElementList (const PBElementList &d, PBElementBase *e) - : m_elements (d.m_elements), m_oid (++s_oid), m_name (d.m_name) -{ - if (e) { - m_elements.push_back (PBElementProxy (e)); - } -} - -void PBElementList::append (const PBElementBase &e) -{ - m_elements.push_back (PBElementProxy (e)); -} - -void PBElementList::append (PBElementBase *e) -{ - if (e) { - m_elements.push_back (PBElementProxy (e)); - } -} - -PBElementList::iterator PBElementList::begin () const -{ - return m_elements.begin (); -} - -PBElementList::iterator PBElementList::end () const -{ - return m_elements.end (); -} - -PBElementList PBElementList::empty () -{ - return PBElementList (); -} - -// -------------------------------------------------------------------- -// PBElementBase implementation - -PBElementBase::PBElementBase (const std::string &name, int tag, const PBElementList &children) - : m_name (name), m_tag (tag), mp_children (new PBElementList (children)), m_owns_child_list (true) -{ - // .. nothing yet .. -} - -PBElementBase::PBElementBase (const std::string &name, int tag, const PBElementList *children) - : m_name (name), m_tag (tag), mp_children (children), m_owns_child_list (false) -{ - // .. nothing yet .. -} - -PBElementBase::PBElementBase (const PBElementBase &d) - : m_name (d.m_name), m_tag (d.m_tag), m_owns_child_list (d.m_owns_child_list) -{ - if (m_owns_child_list) { - mp_children = new PBElementList (*d.mp_children); - } else { - mp_children = d.mp_children; - } -} - -PBElementBase::~PBElementBase () -{ - if (m_owns_child_list) { - delete const_cast (mp_children); - mp_children = 0; - } -} - -PBElementBase::Cardinality -PBElementBase::cardinality () const -{ - return Zero; -} - -PBElementBase::iterator -PBElementBase::begin () const -{ - return mp_children->begin (); -} - -PBElementBase::iterator -PBElementBase::end () const -{ - return mp_children->end (); -} - -std::string -PBElementBase::name4code () const -{ - std::string res; - const char *n = name ().c_str (); - - if (! isalpha (*n) && *n != '_') { - res += '_'; - } - - while (*n) { - if (*n == '-') { - res += '_'; - } else if (isalnum (*n) || *n == '_') { - res += *n; - } - ++n; - } - - return res; -} - -std::string -PBElementBase::create_def (std::map > &messages) const -{ - std::string res; - - auto m = messages.find (oid ()); - if (m != messages.end ()) { - - res += "message " + m->second.second + " {\n"; - - for (auto i = begin (); i != end (); ++i) { - const PBElementBase *e = i->get (); - Cardinality c = e->cardinality (); - std::string entry = e->create_def_entry (messages); - if (! entry.empty ()) { - res += " "; - if (c != Zero) { - if (c == Many) { - res += "repeated "; - } else { - res += "optional "; - } - res += entry + "\n"; - } - } - } - - res += "}"; - - } - - return res; -} - -void -PBElementBase::collect_messages (std::map > &messages) const -{ - for (auto i = begin (); i != end (); ++i) { - i->get ()->collect_messages (messages); - } -} - -std::string -PBElementBase::make_message_name () const -{ - std::string res = mp_children->name (); - if (! res.empty ()) { - return res; - } - - // Capitalize names - std::string n4c = name4code (); - - bool upcase = true; - - const char *n = n4c.c_str (); - while (*n) { - if (*n == '_') { - upcase = true; - } else if (upcase) { - res += toupper (*n); - upcase = false; - } else { - res += *n; - } - ++n; - } - - return res; -} - -// -------------------------------------------------------------------- -// PBParser implementation - -PBParser::PBParser () -{ - // .. nothing yet .. -} - -PBParser::~PBParser () -{ - // .. nothing yet .. -} - -void -PBParser::parse (tl::ProtocolBufferReaderBase &reader, const PBElementBase *root, PBReaderState *reader_state) -{ - mp_state = reader_state; - parse_element (root, reader); -} - -void -PBParser::parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBase &reader) -{ - while (! reader.at_end ()) { - - int tag = reader.read_tag (); - - const PBElementBase *new_element = 0; - if (parent) { - for (PBElementBase::iterator c = parent->begin (); c != parent->end (); ++c) { - if ((*c)->tag () == tag) { - new_element = (*c).get (); - break; - } - } - } - - if (! new_element) { - reader.skip (); - } else { - new_element->create (parent, *mp_state); - new_element->parse (this, reader); - new_element->finish (parent, *mp_state); - } - - } -} - -void -PBParser::expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name) -{ - int tag = reader.read_tag (); - if (tag != name_tag) { - reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with ID %d (got %d)")), name_tag, tag)); - } - - std::string n; - reader.read (n); - if (n != name) { - reader.error (tl::sprintf (tl::to_string (tr ("Expected header field with string '%s' (got '%s')")), name, n)); - } -} - -// -------------------------------------------------------------------- -// PBElementProxy implementation - -PBElementProxy::PBElementProxy (const PBElementProxy &d) - : mp_ptr (d.mp_ptr->clone ()) -{ - // .. nothing yet .. -} - -PBElementProxy::PBElementProxy (const PBElementBase &d) - : mp_ptr (d.clone ()) -{ - // .. nothing yet .. -} - -PBElementProxy::PBElementProxy (PBElementBase *d) - : mp_ptr (d) -{ - // .. nothing yet .. -} - -PBElementProxy::~PBElementProxy () -{ - delete mp_ptr; - mp_ptr = 0; -} - -// -------------------------------------------------------------------- -// PBReaderState implementation - -PBReaderState::PBReaderState () -{ - // .. nothing yet .. -} - -PBReaderState::~PBReaderState () -{ - for (std::vector ::const_iterator o = m_objects.begin (); o != m_objects.end (); ++o) { - (*o)->release (); - delete *o; - } - m_objects.clear (); -} - -// -------------------------------------------------------------------- -// PBWriterState implementation - -PBWriterState::PBWriterState () -{ - // .. nothing yet .. -} - -} diff --git a/src/tl/tl/tlProtocolBufferStruct.h b/src/tl/tl/tlProtocolBufferStruct.h deleted file mode 100644 index c4bacf3de..000000000 --- a/src/tl/tl/tlProtocolBufferStruct.h +++ /dev/null @@ -1,2005 +0,0 @@ - -/* - - KLayout Layout Viewer - Copyright (C) 2006-2024 Matthias Koefferlein - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#ifndef HDR_tlProtocolBufferStruct -#define HDR_tlProtocolBufferStruct - -#include "tlCommon.h" -#include "tlProtocolBuffer.h" - -#include - -namespace tl -{ - -/** - * @brief A basic PB parser error exception class - */ - -class TL_PUBLIC PBException : public tl::Exception -{ -public: - PBException (const char *msg) - : Exception (tl::to_string (tr ("Protocol buffer parser error: %s")).c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - PBException (const std::string &msg) - : Exception (fmt (0).c_str (), msg.c_str ()), - m_msg (msg) - { - // .. nothing yet .. - } - - /** - * @brief Raw (unprefixed) message of the PB parser - */ - const std::string & - raw_msg () const - { - return m_msg; - } - -protected: - PBException (const std::string &msg, size_t pos) - : Exception (fmt (pos).c_str (), msg.c_str (), pos), - m_msg (msg) - { - // .. nothing yet .. - } - -private: - std::string m_msg; - - static std::string fmt (size_t pos) - { - if (pos == 0) { - return tl::to_string (tr ("Protocol buffer parser error: %s")).c_str (); - } else { - return tl::to_string (tr ("Protocol buffer parser error: %s at position %ul")).c_str (); - } - } -}; - -/** - * @brief A PB parser error exception class that additionally provides line and column information - */ - -class TL_PUBLIC PBLocatedException : public PBException -{ -public: - PBLocatedException (const std::string &msg, size_t pos) - : PBException (msg, pos), - m_pos (pos) - { - // .. nothing yet .. - } - - /** - * @brief Line number information of the exception - */ - size_t pos () const - { - return m_pos; - } - -private: - size_t m_pos; -}; - -/** - * @brief An object wrapper base class for target object management - * - * Implementations of this class through the PBReaderProxy templates - * manage pointers to certain objects. - */ - -class TL_PUBLIC PBReaderProxyBase -{ -public: - PBReaderProxyBase () { } - virtual ~PBReaderProxyBase () { } - virtual void release () = 0; - virtual void detach () = 0; -}; - -/** - * @brief An object wrapper base class for target object management specialized to a certain class - */ - -template -class TL_PUBLIC_TEMPLATE PBReaderProxy - : public PBReaderProxyBase -{ -public: - PBReaderProxy (Obj *obj, bool owns_obj) - : mp_obj (obj), m_owns_obj (owns_obj) - { } - - virtual ~PBReaderProxy () { } - - virtual void release () - { - if (m_owns_obj && mp_obj) { - delete mp_obj; - } - mp_obj = 0; - } - - virtual void detach () - { - m_owns_obj = false; - } - - Obj *ptr () const - { - return mp_obj; - } - -private: - Obj *mp_obj; - bool m_owns_obj; -}; - -/** - * @brief Helper class: A class tag - */ - -template -struct PBObjTag -{ - PBObjTag() { } - typedef Obj obj; -}; - -/** - * @brief Helper class: The reader state - * - * The reader state mainly consists of a stack of objects being parsed. - */ - -class TL_PUBLIC PBReaderState -{ -public: - /** - * @brief Default constructor - */ - PBReaderState (); - - /** - * @brief Destructor - */ - ~PBReaderState (); - - /** - * @brief Push a new object on the stack - */ - template - void push (PBObjTag /*tag*/) - { - m_objects.push_back (new PBReaderProxy (new Obj (), true)); - } - - /** - * @brief Push an existing object on the stack - */ - template - void push (Obj *obj) - { - m_objects.push_back (new PBReaderProxy (obj, false)); - } - - /** - * @brief Push an existing object on the stack with the ownership flag - */ - template - void push (Obj *obj, bool owner) - { - m_objects.push_back (new PBReaderProxy (obj, owner)); - } - - /** - * @brief Get the top object - */ - template - Obj *back (PBObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Get the top object and release - */ - template - Obj *detach_back (PBObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->detach (); - return (dynamic_cast &> (*m_objects.back ())).ptr (); - } - - /** - * @brief Pop an object from the stack - */ - template - void pop (PBObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - m_objects.back ()->release (); - delete m_objects.back (); - m_objects.pop_back (); - } - - /** - * @brief Empty predicate: true, if no more object is on the stack - */ - bool empty () const - { - return m_objects.empty (); - } - - /** - * @brief Obtain the parent object from the stack - */ - template - Obj *parent (PBObjTag /*tag*/) - { - tl_assert (m_objects.size () > 1); - return (dynamic_cast &> (*m_objects.end () [-2])).ptr (); - } - -private: - std::vector m_objects; -}; - -// ----------------------------------------------------------------- -// The C++ structure definition interface (for use cases see tlProtocolBufferStruct.ut) - -class TL_PUBLIC PBElementBase; - -struct pb_pass_by_value_tag { - pb_pass_by_value_tag () { } -}; - -struct pb_pass_by_ref_tag { - pb_pass_by_ref_tag () { } -}; - -struct pb_zero_cardinality_tag { - pb_zero_cardinality_tag () { } -}; - -struct pb_single_cardinality_tag { - pb_single_cardinality_tag () { } -}; - -struct pb_many_cardinality_tag { - pb_many_cardinality_tag () { } -}; - -/** - * @brief The PB parser class - * This is the main entry point. It will take a PB reader, the structure (in "root") and - * a reader state initialized with the top level object. - */ -class TL_PUBLIC PBParser -{ -public: - PBParser (); - ~PBParser (); - - void parse (tl::ProtocolBufferReaderBase &reader, const PBElementBase *root, PBReaderState *reader_state); - void parse_element (const PBElementBase *parent, tl::ProtocolBufferReaderBase &reader); - - void expect_header (tl::ProtocolBufferReaderBase &reader, int name_tag, const std::string &name); - - PBReaderState &reader_state () - { - return *mp_state; - } - -private: - PBReaderState *mp_state; -}; - -/** - * @brief PBElementProxy element of the PB structure definition - * - * The purpose of this class is to provide a wrapper around the - * different derivations of the PBElementBase class, so we can - * pack them into a vector. The proxy objects can be copied - * duplicating the wrapped objects with their "clone" methods. - * The proxy object can be given the pointer to the element - * in two ways: the reference version creates a duplicate, the - * pointer version transfers the ownership of the PBElementBase - * object. - */ - -class TL_PUBLIC PBElementProxy -{ -public: - PBElementProxy (const PBElementProxy &d); - PBElementProxy (const PBElementBase &d); - PBElementProxy (PBElementBase *d); - - ~PBElementProxy (); - - PBElementBase *operator-> () const - { - return mp_ptr; - } - - PBElementBase *get () const - { - return mp_ptr; - } - -private: - PBElementBase *mp_ptr; -}; - -/** - * @brief A list of PB elements - * - * This class provides a list of PB elements. - * A list can be created by using the + operator and - * supplying various objects based on PBElementBase. - */ - -class TL_PUBLIC PBElementList -{ -public: - typedef std::list children_list; - typedef children_list::const_iterator iterator; - - PBElementList (); - PBElementList (const PBElementBase &e); - PBElementList (PBElementBase *e); - PBElementList (const std::string &name, const PBElementList &d); - PBElementList (const PBElementList &d); - PBElementList (const PBElementList &d, const PBElementBase &e); - PBElementList (const PBElementList &d, PBElementBase *e); - - void append (const PBElementBase &e); - void append (PBElementBase *e); - - iterator begin () const; - iterator end () const; - - static PBElementList empty (); - - size_t oid () const - { - return m_oid; - } - - const std::string &name () const - { - return m_name; - } - -private: - std::list m_elements; - size_t m_oid; - std::string m_name; -}; - -/** - * @brief Helper class: A stack of const objects being written - */ - -class TL_PUBLIC PBWriterState -{ -public: - /** - * @brief Default constructor - */ - PBWriterState (); - - /** - * @brief Push a new object on the stack - */ - template - void push (const Obj *obj) - { - m_objects.push_back (obj); - } - - /** - * @brief Pop an object from the stack - */ - template - const Obj *pop (PBObjTag /*tag*/) - { - tl_assert (! m_objects.empty ()); - const Obj *obj = reinterpret_cast (m_objects.back ()); - m_objects.pop_back (); - return obj; - } - - /** - * @brief Obtain the parent object from the stack - */ - template - const Obj *back (PBObjTag /*tag*/) - { - tl_assert (m_objects.size () > 0); - return reinterpret_cast (m_objects.end () [-1]); - } - -private: - std::vector m_objects; -}; - -/** - * @brief The PB element base object - * - * This class is the base class for objects implementing - * the parser handler semantics. The basic methods are - * create (create an actual object), read (read value), - * finish (finalize the actual object). - * - * Writing of objects is also handled through this class. - * The basic method is "write". - */ - -class TL_PUBLIC PBElementBase -{ -public: - typedef PBElementList::iterator iterator; - - enum Cardinality { - Zero, Single, Many - }; - - PBElementBase (const std::string &name, int tag, const PBElementList &children); - PBElementBase (const std::string &name, int tag, const PBElementList *children); - PBElementBase (const PBElementBase &d); - virtual ~PBElementBase (); - - virtual PBElementBase *clone () const = 0; - - virtual void create (const PBElementBase *parent, PBReaderState &objs) const = 0; - virtual void parse (PBParser *, tl::ProtocolBufferReaderBase &) const = 0; - virtual void finish (const PBElementBase *parent, PBReaderState &objs) const = 0; - - virtual void write (const PBElementBase *, tl::ProtocolBufferWriterBase &, PBWriterState &) const { } - - virtual Cardinality cardinality () const; - - size_t oid () const - { - return mp_children->oid (); - } - - int tag () const - { - return m_tag; - } - - const std::string &name () const - { - return m_name; - } - - /** - * @brief Returns a name suitable for code - * Specifically, hyphens are replaced by underscores. - */ - std::string name4code () const; - - iterator begin () const; - iterator end () const; - - std::string create_def (std::map > &messages) const; - -protected: - virtual void collect_messages (std::map > &messages) const; - virtual std::string create_def_entry (std::map > &messages) const = 0; - - std::string make_message_name () const; - - static Cardinality get_cardinality (tl::pb_zero_cardinality_tag) - { - return Zero; - } - - static Cardinality get_cardinality (tl::pb_single_cardinality_tag) - { - return Single; - } - - static Cardinality get_cardinality (tl::pb_many_cardinality_tag) - { - return Many; - } - -private: - std::string m_name; - int m_tag; - const PBElementList *mp_children; - bool m_owns_child_list; -}; - -/** - * @brief A PB child element - * - * This class is a PB structure component describing a child - * element in the PB tree. There is no limit about the number of - * child elements. - * This object must be provided a pointer to a factory method (in - * the parent's class), a name and a list of children (which can be - * empty). - * Write is a class providing a Obj &operator(Parent &) operator. - * It is supposed to create a new instance of Obj within Parent. - * Read is a class providing a start(const Parent &) method to start - * iterating over the instances, a const Obj &operator() const for - * access and a bool at_end() method to determine if the iterator - * is at the end and next() to increment the iterator. - */ - -template -class TL_PUBLIC_TEMPLATE PBElement - : public PBElementBase -{ -public: - PBElement (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList &children) - : PBElementBase (name, tag, children), m_r (r), m_w (w) - { - // .. nothing yet .. - } - - PBElement (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList *children) - : PBElementBase (name, tag, children), m_r (r), m_w (w) - { - // .. nothing yet .. - } - - PBElement (const PBElement &d) - : PBElementBase (d), m_r (d.m_r), m_w (d.m_w) - { - // .. nothing yet .. - } - - virtual PBElementBase *clone () const - { - return new PBElement (*this); - } - - virtual void create (const PBElementBase *, PBReaderState &objs) const - { - PBObjTag tag; - objs.push (tag); - } - - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs) const - { - PBObjTag tag; - PBObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); - objs.pop (tag); - } - - virtual void parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const - { - reader.open (); - parser->parse_element (this, reader); - reader.close (); - } - - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const - { - PBObjTag parent_tag; - - Read r (m_r); - r.start (*objs.back (parent_tag)); - while (! r.at_end ()) { - typedef typename Read::tag read_tag_type; - write_obj (r (), tag (), writer, read_tag_type (), objs); - r.next (); - } - } - - virtual Cardinality cardinality () const - { - typedef typename Read::cardinality cardinality_type; - return get_cardinality (cardinality_type ()); - } - - virtual void collect_messages (std::map > &messages) const - { - if (messages.find (oid ()) == messages.end ()) { - messages [oid ()] = std::make_pair (this, make_message_name ()); - PBElementBase::collect_messages (messages); - } - } - - virtual std::string create_def_entry (std::map > &messages) const - { - auto m = messages.find (oid ()); - if (m != messages.end ()) { - return m->second.second + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; - } else { - return std::string (); - } - } - -protected: - Read &rear () { return m_r; } - Write &write () { return m_w; } - -private: - Read m_r; - Write m_w; - - // this write helper is used if the reader delivers an object by value - void write_obj (Obj obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pb_pass_by_value_tag, PBWriterState &objs) const - { - PBObjTag self_tag; - - for (unsigned int pass = 0; pass < 2; ++pass) { - writer.begin_seq (tag, pass == 0); - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, writer, objs); - } - objs.pop (self_tag); - writer.end_seq (); - if (! writer.is_counting ()) { - break; - } - } - } - - void write_obj (const Obj &obj, int tag, tl::ProtocolBufferWriterBase &writer, tl::pb_pass_by_ref_tag, PBWriterState &objs) const - { - PBObjTag self_tag; - - for (unsigned int pass = 0; pass < 2; ++pass) { - writer.begin_seq (tag, pass == 0); - objs.push (&obj); - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, writer, objs); - } - objs.pop (self_tag); - writer.end_seq (); - if (writer.is_counting ()) { - break; - } - } - } -}; - -/** - * @brief A PB child element with is instantiated with a parent reference - * - * This declares a PB element with a constructor that takes a (object *) arguments - * to the parent. - */ - -template -class TL_PUBLIC_TEMPLATE PBElementWithParentRef - : public PBElement -{ -public: - PBElementWithParentRef (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList &children) - : PBElement (r, w, name, tag, children) - { - // .. nothing yet .. - } - - PBElementWithParentRef (const Read &r, const Write &w, const std::string &name, int tag, const PBElementList *children) - : PBElement (r, w, name, tag, children) - { - // .. nothing yet .. - } - - PBElementWithParentRef (const PBElementWithParentRef &d) - : PBElement (d) - { - // .. nothing yet .. - } - - virtual PBElementBase *clone () const - { - return new PBElementWithParentRef (*this); - } - - virtual void create (const PBElementBase *, PBReaderState &objs) const - { - PBObjTag tag; - PBObjTag parent_tag; - objs.push (new Obj (objs.back (parent_tag)), true); - } - - virtual void finish (const PBElementBase * /*parent*/, PBReaderState &objs) const - { - PBObjTag tag; - PBObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); - objs.pop (tag); - } -}; - -/** - * @brief A PB child element mapped to a member - * - * This class is a PB structure component describing a child - * element in the PB tree. The value of the child element - * is directly mapped to a member of the parent's class. - * This object must be provided two adaptors: - * Write is a writer providing an operator() (Parent &, const Value &) - * that adds a new value or sets the value. - * Read is a class providing a start(const Parent &) method to start - * iterating over the instances, a const Value &operator() const for - * access and a bool at_end() method to determine if the iterator - * is at the end and next() to increment the iterator. - */ - -template -class TL_PUBLIC_TEMPLATE PBMember - : public PBElementBase -{ -public: - PBMember (const Read &r, const Write &w, const std::string &name, int tag, Converter c = Converter ()) - : PBElementBase (name, tag, PBElementList::empty ()), m_r (r), m_w (w), m_c (c) - { - // .. nothing yet .. - } - - PBMember (const PBMember &d) - : PBElementBase (d), m_r (d.m_r), m_w (d.m_w), m_c (d.m_c) - { - // .. nothing yet .. - } - - virtual PBElementBase *clone () const - { - return new PBMember (*this); - } - - virtual void create (const PBElementBase *, PBReaderState &) const - { - // .. nothing yet .. - } - - virtual void finish (const PBElementBase *, PBReaderState &) const - { - // .. nothing yet .. - } - - virtual void parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const - { - PBObjTag tag; - PBObjTag parent_tag; - - PBReaderState value_obj; - value_obj.push (tag); - - read (reader, *value_obj.back (tag)); - m_w (*parser->reader_state ().back (parent_tag), value_obj); - - value_obj.pop (tag); - } - - virtual void write (const PBElementBase * /*parent*/, tl::ProtocolBufferWriterBase &writer, PBWriterState &objs) const - { - PBObjTag parent_tag; - Read r (m_r); - r.start (* objs.back (parent_tag)); - while (! r.at_end ()) { - write (writer, tag (), r ()); - r.next (); - } - } - - virtual Cardinality cardinality () const - { - typedef typename Read::cardinality cardinality_type; - return get_cardinality (cardinality_type ()); - } - - virtual void collect_messages (std::map > & /*messages*/) const - { - // no messages here. - } - - virtual std::string create_def_entry (std::map > & /*messages*/) const - { - const Value *v = 0; - return typestring (v) + " " + name4code () + " = " + tl::to_string (tag ()) + ";"; - } - -private: - Read m_r; - Write m_w; - Converter m_c; - - // write incarnations - void write (tl::ProtocolBufferWriterBase &writer, int tag, float v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, double v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, uint8_t v) const - { - writer.write (tag, (uint32_t) v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, int8_t v) const - { - writer.write (tag, (int32_t) v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, uint16_t v) const - { - writer.write (tag, (uint32_t) v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, int16_t v) const - { - writer.write (tag, (int32_t) v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, uint32_t v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, int32_t v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, uint64_t v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, int64_t v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, bool v) const - { - writer.write (tag, v); - } - - void write (tl::ProtocolBufferWriterBase &writer, int tag, const std::string &v) const - { - writer.write (tag, v); - } - - template - void write (tl::ProtocolBufferWriterBase &writer, int tag, const T &v) const - { - writer.write (tag, m_c.pb_encode (v)); - } - - // read incarnations - void read (tl::ProtocolBufferReaderBase &reader, float &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, double &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, uint8_t &v) const - { - uint32_t vv = 0; - reader.read (vv); - // TODO: check for overflow? - v = vv; - } - - void read (tl::ProtocolBufferReaderBase &reader, int8_t &v) const - { - int32_t vv = 0; - reader.read (vv); - // TODO: check for overflow? - v = vv; - } - - void read (tl::ProtocolBufferReaderBase &reader, uint16_t &v) const - { - uint32_t vv = 0; - reader.read (vv); - // TODO: check for overflow? - v = vv; - } - - void read (tl::ProtocolBufferReaderBase &reader, int16_t &v) const - { - int32_t vv = 0; - reader.read (vv); - // TODO: check for overflow? - v = vv; - } - - void read (tl::ProtocolBufferReaderBase &reader, uint32_t &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, int32_t &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, uint64_t &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, int64_t &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, bool &v) const - { - reader.read (v); - } - - void read (tl::ProtocolBufferReaderBase &reader, std::string &v) const - { - reader.read (v); - } - - template - void read (tl::ProtocolBufferReaderBase &reader, T &v) const - { - typename Converter::pb_type vv; - reader.read (vv); - m_c.pb_decode (vv, v); - } - - // type strings - std::string typestring (const float *) const - { - return "float"; - } - - std::string typestring (const double *) const - { - return "double"; - } - - std::string typestring (const uint8_t *) const - { - return "uint32"; - } - - std::string typestring (const uint16_t *) const - { - return "uint32"; - } - - std::string typestring (const uint32_t *) const - { - return "uint32"; - } - - std::string typestring (const uint64_t *) const - { - return "uint64"; - } - - std::string typestring (const int8_t *) const - { - return "sint32"; - } - - std::string typestring (const int16_t *) const - { - return "sint32"; - } - - std::string typestring (const int32_t *) const - { - return "sint32"; - } - - std::string typestring (const int64_t *) const - { - return "sint64"; - } - - std::string typestring (const bool *) const - { - return "uint32"; - } - - std::string typestring (const std::string *) const - { - return "string"; - } - - template - std::string typestring (const T *) const - { - const typename Converter::pb_type *v = 0; - return typestring (v); - } -}; - -/** - * @brief The root element of the PB structure - * - * The root object is also a element node. It belongs to an object - * representing the root of the object hierarchy. Correspondingly, - * child elements (members, elements - aka submessages) need to be - * specified. - * - * In addition, the root element has a name and a name tag. - * The name / name tag are emitted as the first element in the - * message, providing a file header with fixed content. This - * allows identifying the file as a encoded protocol buffer - * file for a specific object type. - */ - -template -class TL_PUBLIC_TEMPLATE PBStruct - : public PBElementBase -{ -public: - PBStruct (const std::string &name, int name_tag, const PBElementList *children) - : PBElementBase (name, name_tag, children) - { - // .. nothing yet .. - } - - PBStruct (const std::string &name, int name_tag, const PBElementList &children) - : PBElementBase (name, name_tag, children) - { - // .. nothing yet .. - } - - PBStruct (const PBStruct &d) - : PBElementBase (d) - { - // .. nothing yet .. - } - - /** - * @brief Creates a deep copy of the structure - */ - virtual PBElementBase *clone () const - { - return new PBStruct (*this); - } - - /** - * @brief Serializes the given object (root) to the writer - */ - void write (tl::ProtocolBufferWriterBase &writer, const Obj &root) const - { - PBWriterState writer_state; - writer_state.push (& root); - - writer.write (tag (), name ()); - - for (PBElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, writer, writer_state); - } - } - - /** - * @brief Deserializes the given object (root) from the reader - */ - void parse (tl::ProtocolBufferReaderBase &reader, Obj &root) const - { - PBObjTag self_tag; - PBReaderState rs; - rs.push (&root); - PBParser h; - h.expect_header (reader, tag (), name ()); - h.parse (reader, this, &rs); - rs.pop (self_tag); - tl_assert (rs.empty ()); - } - - /** - * @brief Produces a definition for the protoc compiler - */ - std::string create_def () const - { - std::map > msgs; - collect_messages (msgs); - msgs[oid ()] = std::make_pair (this, make_message_name ()); - - std::string res = "// created from KLayout proto definition '" + name () + "'\n\n"; - res += "syntax = \"proto2\";"; - - for (auto i = msgs.begin (); i != msgs.end (); ++i) { - std::string entry = i->second.first->create_def (msgs); - if (! entry.empty ()) { - res += "\n"; - res += "\n"; - res += entry; - } - } - - return res; - } - -private: - virtual void write (const PBElementBase*, tl::ProtocolBufferWriterBase &, PBWriterState &) const - { - // disable base class implementation - } - - virtual void parse (PBParser *, tl::ProtocolBufferReaderBase &) const - { - // disable base class implementation - } - - virtual void create (const PBElementBase *, PBReaderState &) const - { - // disable base class implementation - } - - virtual void finish (const PBElementBase *, PBReaderState &) const - { - // disable base class implementation - } - - virtual std::string create_def_entry (std::map > &) const - { - return std::string (); - } -}; - -/** - * @brief Utility: add a PB element to a list of some - */ -inline PBElementList -operator+ (const PBElementList &l, const PBElementBase &e) -{ - return PBElementList (l, e); -} - -/** - * @brief Utility: add a PB element to a list of some - * - * This version transfers the ownership of the PBElementBase object. - */ -inline PBElementList -operator+ (const PBElementList &l, PBElementBase *e) -{ - return PBElementList (l, e); -} - - -// Several adaptor classes needed to implement the various make_* utilities - -template -struct PBMemberDummyWriteAdaptor -{ - PBMemberDummyWriteAdaptor () - { - // .. nothing yet .. - } - - void operator () (Parent &, PBReaderState &) const - { - // .. nothing yet .. - } -}; - -template -struct PBMemberWriteAdaptor -{ - PBMemberWriteAdaptor (Value Parent::*member) - : mp_member (member) - { - // .. nothing yet .. - } - - void operator () (Parent &owner, PBReaderState &reader) const - { - PBObjTag tag; - owner.*mp_member = *reader.back (tag); - } - -private: - Value Parent::*mp_member; -}; - -template -struct PBMemberAccRefWriteAdaptor -{ - PBMemberAccRefWriteAdaptor (void (Parent::*member) (const Value &)) - : mp_member (member) - { - // .. nothing yet .. - } - - void operator () (Parent &owner, PBReaderState &reader) const - { - PBObjTag tag; - (owner.*mp_member) (*reader.back (tag)); - } - -private: - void (Parent::*mp_member) (const Value &); -}; - -template -struct PBMemberTransferWriteAdaptor -{ - PBMemberTransferWriteAdaptor (void (Parent::*member) (Value *)) - : mp_member (member) - { - // .. nothing yet .. - } - - void operator () (Parent &owner, PBReaderState &reader) const - { - PBObjTag tag; - (owner.*mp_member) (reader.detach_back (tag)); - } - -private: - void (Parent::*mp_member) (Value *); -}; - -template -struct PBMemberAccWriteAdaptor -{ - PBMemberAccWriteAdaptor (void (Parent::*member) (Value)) - : mp_member (member) - { - // .. nothing yet .. - } - - void operator () (Parent &owner, PBReaderState &reader) const - { - PBObjTag tag; - (owner.*mp_member) (*reader.back (tag)); - } - -private: - void (Parent::*mp_member) (Value); -}; - -template -struct PBMemberDummyReadAdaptor -{ - typedef pb_pass_by_ref_tag tag; - typedef pb_zero_cardinality_tag cardinality; - - PBMemberDummyReadAdaptor () - { - // .. nothing yet .. - } - - Value operator () () const - { - return Value (); - } - - bool at_end () const - { - return true; - } - - void start (const Parent &) - { - // .. nothing yet .. - } - - void next () - { - // .. nothing yet .. - } -}; - -template -struct PBMemberReadAdaptor -{ - typedef pb_pass_by_ref_tag tag; - typedef pb_single_cardinality_tag cardinality; - - PBMemberReadAdaptor (Value Parent::*member) - : mp_member (member), mp_owner (0), m_done (false) - { - // .. nothing yet .. - } - - const Value &operator () () const - { - return mp_owner->*mp_member; - } - - bool at_end () const - { - return m_done; - } - - void start (const Parent &owner) - { - mp_owner = &owner; - m_done = false; - } - - void next () - { - m_done = true; - } - -private: - Value Parent::*mp_member; - const Parent *mp_owner; - bool m_done; -}; - -template -struct PBMemberAccRefReadAdaptor -{ - typedef pb_pass_by_ref_tag tag; - typedef pb_single_cardinality_tag cardinality; - - PBMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) - : mp_member (member), mp_owner (0), m_done (false) - { - // .. nothing yet .. - } - - const Value &operator () () const - { - return (mp_owner->*mp_member) (); - } - - bool at_end () const - { - return m_done; - } - - void start (const Parent &owner) - { - mp_owner = &owner; - m_done = false; - } - - void next () - { - m_done = true; - } - -private: - const Value &(Parent::*mp_member) () const; - const Parent *mp_owner; - bool m_done; -}; - -template -struct PBMemberAccReadAdaptor -{ - typedef pb_pass_by_value_tag tag; - typedef pb_single_cardinality_tag cardinality; - - PBMemberAccReadAdaptor (Value (Parent::*member) () const) - : mp_member (member), mp_owner (0), m_done (false) - { - // .. nothing yet .. - } - - Value operator () () const - { - return (mp_owner->*mp_member) (); - } - - bool at_end () const - { - return m_done; - } - - void start (const Parent &owner) - { - mp_owner = &owner; - m_done = false; - } - - void next () - { - m_done = true; - } - -private: - Value (Parent::*mp_member) () const; - const Parent *mp_owner; - bool m_done; -}; - -template -struct PBMemberIterReadAdaptor -{ - typedef pb_pass_by_ref_tag tag; - typedef pb_many_cardinality_tag cardinality; - - PBMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) - : mp_begin (begin), mp_end (end) - { - // .. nothing yet .. - } - - Value operator () () const - { - return *m_iter; - } - - bool at_end () const - { - return m_iter == m_end; - } - - void start (const Parent &parent) - { - m_iter = (parent.*mp_begin) (); - m_end = (parent.*mp_end) (); - } - - void next () - { - ++m_iter; - } - -private: - Iter (Parent::*mp_begin) () const; - Iter (Parent::*mp_end) () const; - Iter m_iter, m_end; -}; - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberDummyReadAdaptor (), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberWriteAdaptor > -pb_make_element (Value Parent::*member, const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberWriteAdaptor > ( - PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberDummyReadAdaptor (), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccWriteAdaptor > -pb_make_element (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccWriteAdaptor > -pb_make_element (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccWriteAdaptor > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberWriteAdaptor > -pb_make_element (Value Parent::*member, const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberWriteAdaptor > ( - PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberAccRefWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberAccRefWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief Utility: create a PBElement object - */ -template -PBElement, PBMemberTransferWriteAdaptor > -pb_make_element (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElement, PBMemberTransferWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList *children) -{ - return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (const Value &(Parent::*getter) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList *children) -{ - return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberAccRefWriteAdaptor > -pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, const PBElementList &children) -{ - return PBElementWithParentRef, PBMemberAccRefWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), name, tag, children); -} - -template -PBElementWithParentRef, PBMemberTransferWriteAdaptor > -pb_make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (Value *), const std::string &name, int tag, const PBElementList &children) -{ - return PBElementWithParentRef, PBMemberTransferWriteAdaptor > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberTransferWriteAdaptor (setter), name, tag, children); -} - -/** - * @brief A helper class providing string to value (and back) conversion - */ - -template -struct PBStdConverter -{ - typedef std::string pb_type; - - pb_type pb_encode (const Value &v) const - { - return tl::to_string (v); - } - - void pb_decode (const pb_type &s, Value &v) const - { - tl::from_string (s, v); - } -}; - -/** - * @brief Utility: create a PBMember object without read & write capability - */ -template -PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > -pb_make_member (const std::string &name, int tag) -{ - return PBMember, PBMemberDummyWriteAdaptor , PBStdConverter > ( - PBMemberDummyReadAdaptor (), - PBMemberDummyWriteAdaptor (), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberWriteAdaptor , PBStdConverter > -pb_make_member (Value Parent::*member, const std::string &name, int tag) -{ - return PBMember, PBMemberWriteAdaptor , PBStdConverter > ( - PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (void (Parent::*setter) (const Value &), const std::string &name, int tag) -{ - return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( - PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (void (Parent::*setter) (Value), const std::string &name, int tag) -{ - return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( - PBMemberDummyReadAdaptor (), - PBMemberAccWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) -{ - return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag) -{ - return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccWriteAdaptor , PBStdConverter > -pb_make_member (const Value & (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag) -{ - return PBMember, PBMemberAccWriteAdaptor , PBStdConverter > ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) -{ - return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( - PBMemberAccReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > -pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag) -{ - return PBMember, PBMemberAccRefWriteAdaptor , PBStdConverter > ( - PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), name, tag); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberWriteAdaptor , Converter> -pb_make_member (Value Parent::*member, const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberWriteAdaptor , Converter> ( - PBMemberReadAdaptor (member), - PBMemberWriteAdaptor (member), name, tag, conv); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( - PBMemberDummyReadAdaptor (), - PBMemberAccRefWriteAdaptor (setter), name, tag, conv); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberDummyWriteAdaptor , Converter> -pb_make_member (void (Parent::*setter) (Value), const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberDummyWriteAdaptor , Converter> ( - PBMemberAccReadAdaptor (setter), - PBMemberDummyWriteAdaptor (), name, tag, conv); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (const Value &(Parent::*getter) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( - PBMemberAccRefReadAdaptor (getter), - PBMemberAccRefWriteAdaptor (setter), name, tag, conv); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccWriteAdaptor , Converter> -pb_make_member (Value (Parent::*getter) () const, void (Parent::*setter) (Value), const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberAccWriteAdaptor , Converter> ( - PBMemberAccReadAdaptor (getter), - PBMemberAccWriteAdaptor (setter), name, tag, conv); -} - -/** - * @brief Utility: create a PBMember object - */ -template -PBMember, PBMemberAccRefWriteAdaptor , Converter> -pb_make_member (Iter (Parent::*begin) () const, Iter (Parent::*end) () const, void (Parent::*setter) (const Value &), const std::string &name, int tag, Converter conv) -{ - return PBMember, PBMemberAccRefWriteAdaptor , Converter> ( - PBMemberIterReadAdaptor (begin, end), - PBMemberAccRefWriteAdaptor (setter), name, tag, conv); -} - -} - -#endif diff --git a/src/tl/tl/tlXMLParser.h b/src/tl/tl/tlXMLParser.h index fab046eff..7efb7c585 100644 --- a/src/tl/tl/tlXMLParser.h +++ b/src/tl/tl/tlXMLParser.h @@ -2092,6 +2092,18 @@ make_element_with_parent_ref (Iter (Parent::*begin) () const, Iter (Parent::*end template struct XMLStdConverter { + typedef std::string pb_type; + + pb_type pb_encode (const Value &v) const + { + return tl::to_string (v); + } + + void pb_decode (const pb_type &s, Value &v) const + { + tl::from_string (s, v); + } + std::string to_string (const Value &v) const { return tl::to_string (v); From a5e26b7b99b3b78851a8e22030d8eee35f7bfba8 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 16:26:43 +0200 Subject: [PATCH 18/22] WIP: integrating PB with XML --- src/db/db/dbConverters.h | 6 + src/db/db/dbStream.h | 2 +- src/db/db/dbTechnology.h | 1 + src/img/img/imgStream.cc | 8 +- src/lay/lay/laySaltGrain.cc | 12 +- src/laybasic/laybasic/layDispatcher.cc | 4 + src/lym/lym/lymMacro.cc | 4 +- .../pcb/db_plugin/dbGerberImportData.cc | 12 +- .../lay_plugin/layStreamImportDialog.cc | 8 +- .../net_tracer/db_plugin/dbNetTracerPlugin.cc | 86 ++++---- src/rdb/rdb/rdbFile.cc | 14 +- src/tl/tl/tlXMLParser.h | 202 +++++++++++------- 12 files changed, 226 insertions(+), 133 deletions(-) diff --git a/src/db/db/dbConverters.h b/src/db/db/dbConverters.h index 2403f6c21..3da99c447 100644 --- a/src/db/db/dbConverters.h +++ b/src/db/db/dbConverters.h @@ -40,6 +40,8 @@ namespace db template struct DB_PUBLIC_TEMPLATE TransformationConverter { + typedef T value_type; + std::string to_string (const T &t) const { return t.to_string (); @@ -58,6 +60,8 @@ struct DB_PUBLIC_TEMPLATE TransformationConverter */ struct DB_PUBLIC LayoutLayerConverter { + typedef db::LayerProperties value_type; + std::string to_string (const db::LayerProperties &p) const { return p.to_string (); @@ -79,6 +83,8 @@ struct DB_PUBLIC LayoutLayerConverter template struct DB_PUBLIC_TEMPLATE PointConverter { + typedef P value_type; + std::string to_string (const P &p) const { return tl::to_string (p.x ()) + "," + tl::to_string (p.y ()); diff --git a/src/db/db/dbStream.h b/src/db/db/dbStream.h index ace7531aa..88a30fbf5 100644 --- a/src/db/db/dbStream.h +++ b/src/db/db/dbStream.h @@ -154,7 +154,7 @@ class StreamOptionsReadAdaptor { public: typedef tl::pass_by_ref_tag tag; - typedef tl::pb_zero_cardinality_tag cardinality; + typedef tl::zero_cardinality_tag cardinality; StreamOptionsReadAdaptor () : mp_options (0), m_done (false) diff --git a/src/db/db/dbTechnology.h b/src/db/db/dbTechnology.h index 2fa1c6f5e..9477a18cd 100644 --- a/src/db/db/dbTechnology.h +++ b/src/db/db/dbTechnology.h @@ -807,6 +807,7 @@ class TechnologyComponentReadAdaptor { public: typedef tl::pass_by_ref_tag tag; + typedef tl::single_cardinality_tag cardinality; TechnologyComponentReadAdaptor (const std::string &name) : m_name (name), mp_t (0), m_done (false) diff --git a/src/img/img/imgStream.cc b/src/img/img/imgStream.cc index 4f6833aca..5de8bdddc 100644 --- a/src/img/img/imgStream.cc +++ b/src/img/img/imgStream.cc @@ -359,6 +359,8 @@ namespace { struct PointConverter { + typedef db::DPoint value_type; + std::string to_string (const db::DPoint &p) const { return p.to_string (); @@ -373,6 +375,8 @@ namespace { struct ColorMapConverter { + typedef std::pair > value_type; + std::string to_string (const std::pair > &cm) const { std::string s; @@ -427,7 +431,7 @@ tl::XMLStruct s_img_structure ("image-data", tl::make_member (&ImageProxy::max_value, &ImageProxy::set_max_value, "max-value") + tl::make_element (&ImageProxy::data_mapping, &ImageProxy::set_data_mapping, "data-mapping", tl::make_element (&img::DataMapping::false_color_nodes, "color-map", - tl::make_member >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, ColorMapConverter> (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", ColorMapConverter ()) + tl::make_member >, img::DataMapping::false_color_nodes_type::const_iterator, img::DataMapping::false_color_nodes_type, tl::XMLStringBasedConverter > (&img::DataMapping::false_color_nodes_type::begin, &img::DataMapping::false_color_nodes_type::end, &img::DataMapping::false_color_nodes_type::push_back, "color-map-entry", tl::XMLStringBasedConverter ()) ) + tl::make_member (&img::DataMapping::brightness, "brightness") + tl::make_member (&img::DataMapping::contrast, "contrast") + @@ -437,7 +441,7 @@ tl::XMLStruct s_img_structure ("image-data", tl::make_member (&img::DataMapping::blue_gain, "blue-gain") ) + tl::make_element (&ImageProxy::landmarks, &ImageProxy::set_landmarks, "landmarks", - tl::make_member (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", PointConverter ()) + tl::make_member > (&img::Object::landmarks_type::begin, &img::Object::landmarks_type::end, &img::Object::landmarks_type::push_back, "landmark", tl::XMLStringBasedConverter ()) ) + tl::make_member (&ImageProxy::begin_byte_data, &ImageProxy::end_byte_data, &ImageProxy::push_byte_data, "byte-data") + tl::make_member (&ImageProxy::begin_data, &ImageProxy::end_data, &ImageProxy::push_data, "data") diff --git a/src/lay/lay/laySaltGrain.cc b/src/lay/lay/laySaltGrain.cc index f9c3cb181..ff06fdb2b 100644 --- a/src/lay/lay/laySaltGrain.cc +++ b/src/lay/lay/laySaltGrain.cc @@ -353,6 +353,8 @@ SaltGrain::valid_version (const std::string &v) struct TimeConverter { + typedef QDateTime value_type; + std::string to_string (const QDateTime &time) const { if (time.isNull ()) { @@ -374,6 +376,8 @@ struct TimeConverter struct ImageConverter { + typedef QImage value_type; + std::string to_string (const QImage &image) const { if (image.isNull ()) { @@ -416,10 +420,10 @@ SaltGrain::xml_elements () tl::make_member (&SaltGrain::license, &SaltGrain::set_license, "license") + tl::make_member (&SaltGrain::author, &SaltGrain::set_author, "author") + tl::make_member (&SaltGrain::author_contact, &SaltGrain::set_author_contact, "author-contact") + - tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", TimeConverter ()) + - tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", TimeConverter ()) + - tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon", ImageConverter ()) + - tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", ImageConverter ()) + + tl::make_member (&SaltGrain::authored_time, &SaltGrain::set_authored_time, "authored-time", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::installed_time, &SaltGrain::set_installed_time, "installed-time", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::icon, &SaltGrain::set_icon, "icon", tl::XMLStringBasedConverter ()) + + tl::make_member (&SaltGrain::screenshot, &SaltGrain::set_screenshot, "screenshot", tl::XMLStringBasedConverter ()) + tl::make_element (&SaltGrain::begin_dependencies, &SaltGrain::end_dependencies, &SaltGrain::add_dependency, "depends", tl::make_member (&SaltGrainDependency::name, "name") + tl::make_member (&SaltGrainDependency::url, "url") + diff --git a/src/laybasic/laybasic/layDispatcher.cc b/src/laybasic/laybasic/layDispatcher.cc index 5422c4ab0..91455a406 100644 --- a/src/laybasic/laybasic/layDispatcher.cc +++ b/src/laybasic/laybasic/layDispatcher.cc @@ -108,6 +108,8 @@ Dispatcher::config_finalize () struct ConfigGetAdaptor { + typedef tl::zero_cardinality_tag cardinality; + ConfigGetAdaptor (const std::string &name) : mp_owner (0), m_done (false), m_name (name) { @@ -145,6 +147,8 @@ private: struct ConfigGetNullAdaptor { + typedef tl::zero_cardinality_tag cardinality; + ConfigGetNullAdaptor () { // .. nothing yet .. diff --git a/src/lym/lym/lymMacro.cc b/src/lym/lym/lymMacro.cc index 7c239a757..de0004a3a 100644 --- a/src/lym/lym/lymMacro.cc +++ b/src/lym/lym/lymMacro.cc @@ -150,6 +150,8 @@ bool Macro::del () struct Interpreter2s { + typedef Macro::Interpreter value_type; + std::string to_string (Macro::Interpreter i) const { switch (i) { @@ -199,7 +201,7 @@ static tl::XMLStruct xml_struct ("klayout-macro", tl::make_member (&Macro::show_in_menu, &Macro::set_show_in_menu, "show-in-menu") + tl::make_member (&Macro::group_name, &Macro::set_group_name, "group-name") + tl::make_member (&Macro::menu_path, &Macro::set_menu_path, "menu-path") + - tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", Interpreter2s ()) + + tl::make_member (&Macro::interpreter, &Macro::set_interpreter, "interpreter", tl::XMLStringBasedConverter ()) + tl::make_member (&Macro::dsl_interpreter, &Macro::set_dsl_interpreter, "dsl-interpreter-name") + tl::make_member (&Macro::text, &Macro::set_text, "text") + tl::make_member ("format") // for backward compatibility diff --git a/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc b/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc index 00b04b164..608090f3c 100644 --- a/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc +++ b/src/plugins/streamers/pcb/db_plugin/dbGerberImportData.cc @@ -166,6 +166,8 @@ GerberImportData::setup_importer (db::GerberImporter *importer) struct MountingConverter { + typedef GerberImportData::mounting_type value_type; + std::string to_string (GerberImportData::mounting_type m) const { return m == GerberImportData::MountingTop ? "top" : "bottom"; @@ -190,9 +192,9 @@ pcb_project_structure ("pcb-project", tl::make_member (&GerberImportData::border, "border") + tl::make_member (&GerberImportData::free_layer_mapping, "free-layer-mapping") + tl::make_element (&GerberImportData::layout_layers, "layout-layers", - tl::make_member::const_iterator, std::vector > (&std::vector::begin, &std::vector::end, &std::vector::push_back, "layout-layer", db::LayoutLayerConverter ()) + tl::make_member::const_iterator, std::vector > (&std::vector::begin, &std::vector::end, &std::vector::push_back, "layout-layer", tl::XMLStringBasedConverter ()) ) + - tl::make_member (&GerberImportData::mounting, "mounting", MountingConverter ()) + + tl::make_member (&GerberImportData::mounting, "mounting", tl::XMLStringBasedConverter ()) + tl::make_member (&GerberImportData::num_metal_layers, "num-metal-layers") + tl::make_member (&GerberImportData::num_via_types, "num-via-types") + tl::make_element (&GerberImportData::artwork_files, "artwork-files", @@ -217,11 +219,11 @@ pcb_project_structure ("pcb-project", ) + tl::make_element (&GerberImportData::reference_points, "reference-points", tl::make_element, std::vector >::const_iterator, std::vector > > (&std::vector >::begin, &std::vector >::end, &std::vector >::push_back, "reference-point", - tl::make_member (&std::pair ::first, "pcb", db::PointConverter ()) + - tl::make_member (&std::pair ::second, "layout", db::PointConverter ()) + tl::make_member (&std::pair ::first, "pcb", tl::XMLStringBasedConverter > ()) + + tl::make_member (&std::pair ::second, "layout", tl::XMLStringBasedConverter > ()) ) ) + - tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", db::TransformationConverter ()) + + tl::make_member (&GerberImportData::explicit_trans, "explicit-trans", tl::XMLStringBasedConverter > ()) + tl::make_member (&GerberImportData::layer_properties_file, "layer-properties-file") + tl::make_member (&GerberImportData::num_circle_points, "num-circle-points") + tl::make_member (&GerberImportData::merge_flag, "merge-flag") + diff --git a/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc b/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc index 3f5e7841d..a3c56853c 100644 --- a/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc +++ b/src/plugins/tools/import/lay_plugin/layStreamImportDialog.cc @@ -49,6 +49,8 @@ static struct { struct ModeConverter { + typedef lay::StreamImportData::mode_type value_type; + std::string to_string (lay::StreamImportData::mode_type t) const { for (unsigned int i = 0; i < sizeof (mode_strings) / sizeof (mode_strings [0]); ++i) { @@ -82,6 +84,8 @@ static struct { struct LayerModeConverter { + typedef lay::StreamImportData::layer_mode_type value_type; + std::string to_string (lay::StreamImportData::layer_mode_type t) const { for (unsigned int i = 0; i < sizeof (layer_mode_strings) / sizeof (layer_mode_strings [0]); ++i) { @@ -144,8 +148,8 @@ static tl::XMLElementList xml_elements () ) + tl::make_member (&StreamImportData::topcell, "cell-name") + tl::make_member (&StreamImportData::layer_offset, "layer-offset") + - tl::make_member (&StreamImportData::layer_mode, "layer-mode", LayerModeConverter ()) + - tl::make_member (&StreamImportData::mode, "import-mode", ModeConverter ()) + + tl::make_member (&StreamImportData::layer_mode, "layer-mode", tl::XMLStringBasedConverter ()) + + tl::make_member (&StreamImportData::mode, "import-mode", tl::XMLStringBasedConverter ()) + tl::make_element (&StreamImportData::reference_points, "reference-points", tl::make_element (&ref_point_v::begin, &ref_point_v::end, &ref_point_v::push_back, "reference-point", tl::make_member (&ref_point::first, "p1") + diff --git a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc index 1967f7bb0..ce45a45ac 100644 --- a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc +++ b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc @@ -24,50 +24,49 @@ #include "dbTechnology.h" #include "tlClassRegistry.h" -namespace tl -{ - /** - * @brief A specialization of the XMLConverter that is used to serialize the connection info - */ - - template <> - struct XMLStdConverter - { - std::string to_string (const db::NetTracerConnectionInfo &v) const - { - return v.to_string (); - } - - void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const - { - tl::Extractor ex (s.c_str ()); - v.parse (ex); - } - }; - - /** - * @brief A specialization of the XMLConverter that is used to serialize the symbol info - */ - - template <> - struct XMLStdConverter - { - std::string to_string (const db::NetTracerSymbolInfo &v) const - { - return v.to_string (); - } - - void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const - { - tl::Extractor ex (s.c_str ()); - v.parse (ex); - } - }; -} - namespace { +/** + * @brief A string converter for NetTracerConnectionInfo + */ + +struct NetTracerConnectionInfoConverter +{ + typedef db::NetTracerConnectionInfo value_type; + + std::string to_string (const db::NetTracerConnectionInfo &v) const + { + return v.to_string (); + } + + void from_string (const std::string &s, db::NetTracerConnectionInfo &v) const + { + tl::Extractor ex (s.c_str ()); + v.parse (ex); + } +}; + +/** + * @brief A specialization of the XMLConverter that is used to serialize the symbol info + */ + +struct NetTracerSymbolInfoConverter +{ + typedef db::NetTracerSymbolInfo value_type; + + std::string to_string (const db::NetTracerSymbolInfo &v) const + { + return v.to_string (); + } + + void from_string (const std::string &s, db::NetTracerSymbolInfo &v) const + { + tl::Extractor ex (s.c_str ()); + v.parse (ex); + } +}; + static const db::NetTracerConnectivity * get_fallback_default (const db::NetTracerTechnologyComponent &tc) { @@ -129,6 +128,7 @@ template struct FallbackXMLReadAdaptor { typedef tl::pass_by_ref_tag tag; + typedef tl::many_cardinality_tag cardinality; FallbackXMLReadAdaptor (Iter (db::NetTracerConnectivity::*begin) () const, Iter (db::NetTracerConnectivity::*end) () const) : mp_begin (begin), mp_end (end) @@ -195,10 +195,10 @@ public: tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols") ) + // Fallback readers for migrating pre-0.28 setups to 0.28 and backward compatibility - tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStdConverter > ( + tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStringBasedConverter > ( FallbackXMLReadAdaptor (&NetTracerConnectivity::begin, &NetTracerConnectivity::end), FallbackXMLWriteAdaptor (&NetTracerConnectivity::add), "connection") + - tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStdConverter > ( + tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStringBasedConverter > ( FallbackXMLReadAdaptor (&NetTracerConnectivity::begin_symbols, &NetTracerConnectivity::end_symbols), FallbackXMLWriteAdaptor (&NetTracerConnectivity::add_symbol), "symbols") ); diff --git a/src/rdb/rdb/rdbFile.cc b/src/rdb/rdb/rdbFile.cc index e6159b6a4..dc533bf7f 100644 --- a/src/rdb/rdb/rdbFile.cc +++ b/src/rdb/rdb/rdbFile.cc @@ -38,6 +38,8 @@ namespace rdb struct ValueConverter { + typedef std::string pb_type; + ValueConverter (rdb::Database *rdb) : mp_rdb (rdb) { @@ -53,6 +55,16 @@ struct ValueConverter value.from_string (mp_rdb, s); } + std::string pb_encode (const ValueWrapper &value) const + { + return value.to_string (mp_rdb); + } + + void pb_decode (const std::string &s, ValueWrapper &value) const + { + value.from_string (mp_rdb, s); + } + private: rdb::Database *mp_rdb; }; @@ -110,7 +122,7 @@ make_rdb_structure (rdb::Database *rdb) tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment") + tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image") + tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values", - tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter (rdb)) + tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter (rdb)) ) ) ) diff --git a/src/tl/tl/tlXMLParser.h b/src/tl/tl/tlXMLParser.h index 7efb7c585..f8d73d5b3 100644 --- a/src/tl/tl/tlXMLParser.h +++ b/src/tl/tl/tlXMLParser.h @@ -62,16 +62,16 @@ struct pass_by_ref_tag { pass_by_ref_tag () { } }; -struct pb_zero_cardinality_tag { - pb_zero_cardinality_tag () { } +struct zero_cardinality_tag { + zero_cardinality_tag () { } }; -struct pb_single_cardinality_tag { - pb_single_cardinality_tag () { } +struct single_cardinality_tag { + single_cardinality_tag () { } }; -struct pb_many_cardinality_tag { - pb_many_cardinality_tag () { } +struct many_cardinality_tag { + many_cardinality_tag () { } }; /** @@ -570,17 +570,17 @@ protected: std::string make_message_name () const; - static Cardinality get_cardinality (tl::pb_zero_cardinality_tag) + static Cardinality get_cardinality (tl::zero_cardinality_tag) { return Zero; } - static Cardinality get_cardinality (tl::pb_single_cardinality_tag) + static Cardinality get_cardinality (tl::single_cardinality_tag) { return Single; } - static Cardinality get_cardinality (tl::pb_many_cardinality_tag) + static Cardinality get_cardinality (tl::many_cardinality_tag) { return Many; } @@ -593,69 +593,37 @@ private: }; /** - * @brief A XML child element - * - * This class is a XML structure component describing a child - * element in the XML tree. There is no limit about the number of - * child elements. - * This object must be provided a pointer to a factory method (in - * the parent's class), a name and a list of children (which can be - * empty). - * Write is a class providing a Obj &operator(Parent &) operator. - * It is supposed to create a new instance of Obj within Parent. - * Read is a class providing a start(const Parent &) method to start - * iterating over the instances, a const Obj &operator() const for - * access and a bool at_end() method to determine if the iterator - * is at the end and next() to increment the iterator. + * @brief Basic implementation for XMLElement and XMLElementWithParentRef */ template -class TL_PUBLIC_TEMPLATE XMLElement +class TL_PUBLIC_TEMPLATE XMLElementImplBase : public XMLElementBase { public: - XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) + XMLElementImplBase (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) : XMLElementBase (name, children), m_r (r), m_w (w) { // .. nothing yet .. } - XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) + XMLElementImplBase (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) : XMLElementBase (name, children), m_r (r), m_w (w) { // .. nothing yet .. } - XMLElement (const XMLElement &d) + XMLElementImplBase (const XMLElementImplBase &d) : XMLElementBase (d), m_r (d.m_r), m_w (d.m_w) { // .. nothing yet .. } - virtual XMLElementBase *clone () const - { - return new XMLElement (*this); - } - - virtual void create (const XMLElementBase *, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const - { - XMLObjTag tag; - objs.push (tag); - } - virtual void cdata (const std::string & /*cdata*/, XMLReaderState & /*objs*/) const { // .. nothing yet .. } - virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const - { - XMLObjTag tag; - XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); - objs.pop (tag); - } - virtual void write (const XMLElementBase * /*parent*/, tl::OutputStream &os, int indent, XMLWriterState &objs) const { XMLObjTag parent_tag; @@ -673,7 +641,7 @@ public: } } - virtual bool has_any (XMLWriterState &objs) const + virtual bool has_any (XMLWriterState &objs) const { XMLObjTag parent_tag; Read r (m_r); @@ -681,20 +649,6 @@ public: return (! r.at_end ()); } - virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const - { - XMLObjTag tag; - objs.push (tag); - } - - virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const - { - XMLObjTag tag; - XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); - objs.pop (tag); - } - virtual void pb_parse (PBParser *parser, tl::ProtocolBufferReaderBase &reader) const { reader.open (); @@ -739,10 +693,11 @@ public: } } -private: +protected: Read m_r; Write m_w; +private: // this write helper is used if the reader delivers an object by value void write_obj (Obj obj, tl::OutputStream &os, int indent, tl::pass_by_value_tag, XMLWriterState &objs) const { @@ -773,7 +728,7 @@ private: writer.begin_seq (tag, pass == 0); objs.push (&obj); for (XMLElementBase::iterator c = this->begin (); c != this->end (); ++c) { - c->get ()->write (this, writer, objs); + c->get ()->pb_write (this, writer, objs); } objs.pop (self_tag); writer.end_seq (); @@ -802,6 +757,80 @@ private: } }; +/** + * @brief A XML child element + * + * This class is a XML structure component describing a child + * element in the XML tree. There is no limit about the number of + * child elements. + * This object must be provided a pointer to a factory method (in + * the parent's class), a name and a list of children (which can be + * empty). + * Write is a class providing a Obj &operator(Parent &) operator. + * It is supposed to create a new instance of Obj within Parent. + * Read is a class providing a start(const Parent &) method to start + * iterating over the instances, a const Obj &operator() const for + * access and a bool at_end() method to determine if the iterator + * is at the end and next() to increment the iterator. + */ + +template +class TL_PUBLIC_TEMPLATE XMLElement + : public XMLElementImplBase +{ +public: + XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) + : XMLElementImplBase (r, w, name, children) + { + // .. nothing yet .. + } + + XMLElement (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) + : XMLElementImplBase (r, w, name, children) + { + // .. nothing yet .. + } + + XMLElement (const XMLElement &d) + : XMLElementImplBase (d) + { + // .. nothing yet .. + } + + virtual XMLElementBase *clone () const + { + return new XMLElement (*this); + } + + virtual void create (const XMLElementBase *, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + XMLObjTag tag; + objs.push (tag); + } + + virtual void finish (const XMLElementBase * /*parent*/, XMLReaderState &objs, const std::string & /*uri*/, const std::string & /*lname*/, const std::string & /*qname*/) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + this->m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } + + virtual void pb_create (const XMLElementBase *, XMLReaderState &objs) const + { + XMLObjTag tag; + objs.push (tag); + } + + virtual void pb_finish (const XMLElementBase * /*parent*/, XMLReaderState &objs) const + { + XMLObjTag tag; + XMLObjTag parent_tag; + this->m_w (*objs.parent (parent_tag), objs); + objs.pop (tag); + } +}; + /** * @brief A XML child element with is instantiated with a parent reference * @@ -811,23 +840,23 @@ private: template class TL_PUBLIC_TEMPLATE XMLElementWithParentRef - : public XMLElement + : public XMLElementImplBase { public: XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList &children) - : XMLElement (r, w, name, children) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const Read &r, const Write &w, const std::string &name, const XMLElementList *children) - : XMLElement (r, w, name, children) + : XMLElementImplBase (r, w, name, children) { // .. nothing yet .. } XMLElementWithParentRef (const XMLElementWithParentRef &d) - : XMLElement (d) + : XMLElementImplBase (d) { // .. nothing yet .. } @@ -848,7 +877,7 @@ public: { XMLObjTag tag; XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } @@ -863,7 +892,7 @@ public: { XMLObjTag tag; XMLObjTag parent_tag; - m_w (*objs.parent (parent_tag), objs); + this->m_w (*objs.parent (parent_tag), objs); objs.pop (tag); } }; @@ -1297,6 +1326,11 @@ public: return false; } + virtual void pb_create (const XMLElementBase *, XMLReaderState &) const { } + virtual void pb_parse (PBParser *, tl::ProtocolBufferReaderBase &) const { } + virtual void pb_finish (const XMLElementBase *, XMLReaderState &) const { } + virtual std::string create_def_entry (std::map > &) const { return std::string (); } + private: Write m_w; Converter m_c; @@ -1583,7 +1617,7 @@ template struct XMLMemberDummyReadAdaptor { typedef pass_by_ref_tag tag; - typedef pb_zero_cardinality_tag cardinality; + typedef zero_cardinality_tag cardinality; XMLMemberDummyReadAdaptor () { @@ -1615,7 +1649,7 @@ template struct XMLMemberReadAdaptor { typedef pass_by_ref_tag tag; - typedef pb_single_cardinality_tag cardinality; + typedef single_cardinality_tag cardinality; XMLMemberReadAdaptor (Value Parent::*member) : mp_member (member), mp_owner (0), m_done (false) @@ -1654,7 +1688,7 @@ template struct XMLMemberAccRefReadAdaptor { typedef pass_by_ref_tag tag; - typedef pb_single_cardinality_tag cardinality; + typedef single_cardinality_tag cardinality; XMLMemberAccRefReadAdaptor (const Value &(Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1693,7 +1727,7 @@ template struct XMLMemberAccReadAdaptor { typedef pass_by_value_tag tag; - typedef pb_single_cardinality_tag cardinality; + typedef single_cardinality_tag cardinality; XMLMemberAccReadAdaptor (Value (Parent::*member) () const) : mp_member (member), mp_owner (0), m_done (false) @@ -1732,7 +1766,7 @@ template struct XMLMemberIterReadAdaptor { typedef pass_by_ref_tag tag; - typedef pb_many_cardinality_tag cardinality; + typedef many_cardinality_tag cardinality; XMLMemberIterReadAdaptor (Iter (Parent::*begin) () const, Iter (Parent::*end) () const) : mp_begin (begin), mp_end (end) @@ -2115,6 +2149,26 @@ struct XMLStdConverter } }; +/** + * @brief A helper class to convert a string converter to a XML converter + */ +template +struct XMLStringBasedConverter + : public StringConverter +{ + typedef std::string pb_type; + + pb_type pb_encode (const typename StringConverter::value_type &v) const + { + return StringConverter::to_string (v); + } + + void pb_decode (const pb_type &s, typename StringConverter::value_type &v) const + { + StringConverter::from_string (s, v); + } +}; + /** * @brief Utility: create a XMLMember object without read & write capability */ From 06815fc1dca9a6eefa07da4e2062b161fd86b2c3 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 17:56:40 +0200 Subject: [PATCH 19/22] Bugfix --- src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc index ce45a45ac..4617a9ad4 100644 --- a/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc +++ b/src/plugins/tools/net_tracer/db_plugin/dbNetTracerPlugin.cc @@ -191,8 +191,8 @@ public: tl::make_element ((NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::begin, (NetTracerTechnologyComponent::const_iterator (NetTracerTechnologyComponent::*) () const) &NetTracerTechnologyComponent::end, (void (NetTracerTechnologyComponent::*) (const NetTracerConnectivity &)) &NetTracerTechnologyComponent::push_back, "stack", tl::make_member (&NetTracerConnectivity::name, &NetTracerConnectivity::set_name, "name") + tl::make_member (&NetTracerConnectivity::description, &NetTracerConnectivity::set_description, "description") + - tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection") + - tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols") + tl::make_member ((NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin, (NetTracerConnectivity::const_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end, &NetTracerConnectivity::add, "connection", tl::XMLStringBasedConverter ()) + + tl::make_member ((NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::begin_symbols, (NetTracerConnectivity::const_symbol_iterator (NetTracerConnectivity::*) () const) &NetTracerConnectivity::end_symbols, &NetTracerConnectivity::add_symbol, "symbols", tl::XMLStringBasedConverter ()) ) + // Fallback readers for migrating pre-0.28 setups to 0.28 and backward compatibility tl::XMLMember, FallbackXMLWriteAdaptor , tl::XMLStringBasedConverter > ( From 2eef9c38d6634cfebfadcae9c72447ef6e5eeafc Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 19:58:24 +0200 Subject: [PATCH 20/22] WIP --- src/rdb/rdb/rdb.cc | 157 +++++++++++++++++++++++---------------- src/rdb/rdb/rdb.h | 39 +++++++--- src/rdb/rdb/rdbFile.cc | 48 ++++++------ testdata/ruby/rdbTest.rb | 2 +- 4 files changed, 147 insertions(+), 99 deletions(-) diff --git a/src/rdb/rdb/rdb.cc b/src/rdb/rdb/rdb.cc index 64f8192f4..8e624f78c 100644 --- a/src/rdb/rdb/rdb.cc +++ b/src/rdb/rdb/rdb.cc @@ -417,18 +417,18 @@ ValueBase::create_from_shape (const db::Shape &shape, const db::CplxTrans &trans // ValueWrapper implementation std::string -ValueWrapper::to_string (const Database *rdb) const +ValueWrapper::to_string () const { std::string r; r.reserve (200); - if (tag_id () > 0 && rdb) { + if (tag_id () > 0) { r += "["; - const Tag &tag = rdb->tags ().tag (tag_id ()); - if (tag.is_user_tag ()) { + auto np = rdb::Tags::name_for_id (tag_id ()); + if (np.second) { r += "#"; } - r += tl::to_word_or_quoted_string (tag.name ()); + r += tl::to_word_or_quoted_string (np.first); r += "] "; } @@ -438,14 +438,14 @@ ValueWrapper::to_string (const Database *rdb) const } void -ValueWrapper::from_string (Database *rdb, const std::string &s) +ValueWrapper::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); - from_string (rdb, ex); + from_string (ex); } void -ValueWrapper::from_string (Database *rdb, tl::Extractor &ex) +ValueWrapper::from_string (tl::Extractor &ex) { id_type tag_id = 0; @@ -456,7 +456,7 @@ ValueWrapper::from_string (Database *rdb, tl::Extractor &ex) std::string tn; ex.read_word_or_quoted (tn); - tag_id = rdb->tags ().tag (tn, user_tag).id (); + tag_id = rdb::Tags::id_for_name (tn, user_tag); ex.test ("]"); @@ -477,16 +477,16 @@ Values::operator= (const Values &d) } bool -Values::compare (const Values &other, const std::map &tag_map, const std::map &rev_tag_map) const +Values::compare (const Values &other, const std::set &common_tags) const { Values::const_iterator a = begin (), b = other.begin (); while (a != end () && b != other.end ()) { id_type t12 = 0; while (a != end () && a->tag_id () != 0) { - auto j = tag_map.find (a->tag_id ()); - if (j != tag_map.end ()) { - t12 = j->second; + auto j = common_tags.find (a->tag_id ()); + if (j != common_tags.end ()) { + t12 = a->tag_id (); break; } ++a; @@ -494,9 +494,9 @@ Values::compare (const Values &other, const std::map &tag_map, id_type t2 = 0; while (b != other.end () && b->tag_id () != 0) { - auto j = rev_tag_map.find (b->tag_id ()); - if (j != rev_tag_map.end ()) { - t2 = j->first; + auto j = common_tags.find (b->tag_id ()); + if (j != common_tags.end ()) { + t2 = b->tag_id (); break; } ++b; @@ -529,7 +529,7 @@ Values::compare (const Values &other, const std::map &tag_map, } std::string -Values::to_string (const Database *rdb) const +Values::to_string () const { std::string r; r.reserve (200); @@ -540,7 +540,7 @@ Values::to_string (const Database *rdb) const r += ";"; } - r += v->to_string (rdb); + r += v->to_string (); } @@ -548,14 +548,14 @@ Values::to_string (const Database *rdb) const } void -Values::from_string (Database *rdb, const std::string &s) +Values::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); while (! ex.at_end ()) { ValueWrapper v; - v.from_string (rdb, ex); + v.from_string (ex); add (v); @@ -905,6 +905,41 @@ Categories::import_category (Category *category) // ------------------------------------------------------------------------------------------ // Tags implementation +static tl::Mutex s_tag_namespace_mutex; +static std::map, id_type> s_tag_namespace; +static std::vector > s_tags; + +const std::pair & +Tags::name_for_id (id_type tag_id) +{ + tl::MutexLocker locker (&s_tag_namespace_mutex); + tl_assert (tag_id > 0 && tag_id <= s_tags.size ()); + return s_tags [tag_id - 1]; +} + +id_type +Tags::id_for_name (const std::string &name, bool user_flag) +{ + tl::MutexLocker locker (&s_tag_namespace_mutex); + + auto k = std::make_pair (name, user_flag); + auto t = s_tag_namespace.find (k); + if (t != s_tag_namespace.end ()) { + return t->second; + } + + id_type id = s_tags.size () + 1; + s_tag_namespace [k] = id; + s_tags.push_back (k); + return id; +} + +Tag +Tags::tag_for_name (const std::string &name, bool user_flag) +{ + return Tag (id_for_name (name, user_flag), name, user_flag); +} + Tags::Tags () { // .. nothing yet .. @@ -913,7 +948,7 @@ Tags::Tags () void Tags::clear () { - m_ids_for_names.clear (); + m_tags_per_id.clear (); m_tags.clear (); } @@ -926,26 +961,31 @@ Tags::tag (const std::string &name, bool user_tag) const Tag & Tags::tag (const std::string &name, bool user_tag) { - std::map , id_type>::const_iterator i = m_ids_for_names.find (std::make_pair (name, user_tag)); - if (i == m_ids_for_names.end ()) { - i = m_ids_for_names.insert (std::make_pair (std::make_pair (name, user_tag), m_tags.size () + 1)).first; - m_tags.push_back (Tag (i->second, name, user_tag)); + id_type id = id_for_name (name, user_tag); + auto i = m_tags_per_id.find (id); + if (i == m_tags_per_id.end ()) { + m_tags.push_back (Tag (id, name, user_tag)); + m_tags_per_id [id] = m_tags.size () - 1; + return m_tags.back (); + } else { + return m_tags [i->second]; } - return m_tags [i->second - 1]; } const Tag & Tags::tag (id_type id) const { - tl_assert (id < m_tags.size () + 1 && id > 0); - return m_tags [id - 1]; + auto i = m_tags_per_id.find (id); + tl_assert (i != m_tags_per_id.end ()); + return m_tags [i->second]; } Tag & Tags::tag (id_type id) { - tl_assert (id < m_tags.size () + 1 && id > 0); - return m_tags [id - 1]; + auto i = m_tags_per_id.find (id); + tl_assert (i != m_tags_per_id.end ()); + return m_tags [i->second]; } void @@ -958,7 +998,7 @@ Tags::import_tag (const Tag &t) bool Tags::has_tag (const std::string &name, bool user_tag) const { - return m_ids_for_names.find (std::make_pair (name, user_tag)) != m_ids_for_names.end (); + return m_tags_per_id.find (id_for_name (name, user_tag)) != m_tags_per_id.end (); } // ------------------------------------------------------------------------------------------ @@ -1006,30 +1046,25 @@ Item &Item::operator= (const Item &d) void Item::add_tag (id_type tag_id) { - if (m_tag_ids.size () <= tag_id) { - m_tag_ids.resize (tag_id + 1, false); - } - m_tag_ids [tag_id] = true; + m_tag_ids.insert (tag_id); } void Item::remove_tag (id_type tag_id) { - if (m_tag_ids.size () > tag_id) { - m_tag_ids [tag_id] = false; - } + m_tag_ids.erase (tag_id); } void Item::remove_tags () { - m_tag_ids = std::vector (); + m_tag_ids.clear (); } bool Item::has_tag (id_type tag_id) const { - return m_tag_ids.size () > tag_id && m_tag_ids [tag_id]; + return m_tag_ids.find (tag_id) != m_tag_ids.end (); } std::string @@ -1082,13 +1117,12 @@ Item::tag_str () const if (! m_tag_ids.empty ()) { - id_type tag_id = 0; - for (std::vector::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t, ++tag_id) { + for (std::set::const_iterator t = m_tag_ids.begin (); t != m_tag_ids.end (); ++t) { if (*t) { if (! r.empty ()) { r += ","; } - const Tag &tag = mp_database->tags ().tag (tag_id); + const Tag &tag = mp_database->tags ().tag (*t); if (tag.is_user_tag ()) { r += "#"; } @@ -1801,33 +1835,30 @@ namespace class ValueMapEntryCompare { public: - ValueMapEntryCompare (const std::map &tag2tag, const std::map &rev_tag2tag) + ValueMapEntryCompare (const std::set &common_tags) { - mp_tag2tag = &tag2tag; - mp_rev_tag2tag = &rev_tag2tag; + mp_common_tags = &common_tags; } bool operator() (const Item *a, const Item *b) const { - return a->values ().compare (b->values (), *mp_tag2tag, *mp_rev_tag2tag); + return a->values ().compare (b->values (), *mp_common_tags); } private: - const std::map *mp_tag2tag; - const std::map *mp_rev_tag2tag; + const std::set *mp_common_tags; }; class ValueMapEntry { public: ValueMapEntry () - : mp_tag2tag (0), mp_rev_tag2tag (0) + : mp_common_tags (0) { } - void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::map &tag2tag, const std::map &rev_tag2tag) + void build (const rdb::Database &rdb, id_type cell_id, id_type cat_id, const std::set &common_tags) { - mp_tag2tag = &tag2tag; - mp_rev_tag2tag = &rev_tag2tag; + mp_common_tags = &common_tags; auto i2i = rdb.items_by_cell_and_category (cell_id, cat_id); @@ -1841,13 +1872,13 @@ namespace m_items.push_back ((*i).operator-> ()); } - ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag); + ValueMapEntryCompare cmp (*mp_common_tags); std::sort (m_items.begin (), m_items.end (), cmp); } const Item *find (const rdb::Item &item) const { - ValueMapEntryCompare cmp (*mp_tag2tag, *mp_rev_tag2tag); + ValueMapEntryCompare cmp (*mp_common_tags); auto i = std::lower_bound (m_items.begin (), m_items.end (), &item, cmp); if (i == m_items.end ()) { @@ -1863,8 +1894,7 @@ namespace public: std::vector m_items; - const std::map *mp_tag2tag; - const std::map *mp_rev_tag2tag; + const std::set *mp_common_tags; }; } @@ -1901,16 +1931,15 @@ Database::apply (const rdb::Database &other) map_category (*c, *this, cat2cat); } - std::map tags_by_name; + std::set t1; for (auto c = tags ().begin_tags (); c != tags ().end_tags (); ++c) { - tags_by_name.insert (std::make_pair (c->name (), c->id ())); + t1.insert (c->id ()); } + std::set common_tags; for (auto c = other.tags ().begin_tags (); c != other.tags ().end_tags (); ++c) { - auto t = tags_by_name.find (c->name ()); - if (t != tags_by_name.end ()) { - tag2tag.insert (std::make_pair (t->second, c->id ())); - rev_tag2tag.insert (std::make_pair (c->id (), t->second)); + if (t1.find (c->id ()) != t1.end ()) { + common_tags.insert (c->id ()); } } @@ -1932,7 +1961,7 @@ Database::apply (const rdb::Database &other) auto vmap = value_map.find (std::make_pair (icell->second, icat->second)); if (vmap == value_map.end ()) { vmap = value_map.insert (std::make_pair (std::make_pair (icell->second, icat->second), ValueMapEntry ())).first; - vmap->second.build (other, icell->second, icat->second, tag2tag, rev_tag2tag); + vmap->second.build (other, icell->second, icat->second, common_tags); } // find a value in the reference DB diff --git a/src/rdb/rdb/rdb.h b/src/rdb/rdb/rdb.h index 808af53ba..7f0e2e339 100644 --- a/src/rdb/rdb/rdb.h +++ b/src/rdb/rdb/rdb.h @@ -629,17 +629,17 @@ public: /** * @brief Convert the values collection to a string */ - std::string to_string (const Database *rdb = 0) const; + std::string to_string () const; /** * @brief Fill the values collection from the string */ - void from_string (Database *rdb, const std::string &s); + void from_string (const std::string &s); /** * @brief Fill the values collection from an extractor */ - void from_string (Database *rdb, tl::Extractor &ex); + void from_string (tl::Extractor &ex); private: ValueBase *mp_ptr; @@ -687,7 +687,7 @@ public: * * The order of the values matters. */ - bool compare (const Values &other, const std::map &tag_map, const std::map &rev_tag_map) const; + bool compare (const Values &other, const std::set &common_tags) const; /** * @brief The const iterator (begin) @@ -760,12 +760,12 @@ public: /** * @brief Convert the values collection to a string */ - std::string to_string (const Database *rdb) const; + std::string to_string () const; /** * @brief Fill the values collection from the string */ - void from_string (Database *rdb, const std::string &s); + void from_string (const std::string &s); private: std::list m_values; @@ -1057,7 +1057,7 @@ private: size_t m_multiplicity; std::string m_comment; bool m_visited; - std::vector m_tag_ids; + std::set m_tag_ids; Database *mp_database; std::string m_image_str; @@ -1982,11 +1982,32 @@ public: */ void clear (); + /** + * @brief Gets the name and user flag for a tag ID + * + * This method will assert if the tag ID is not valid in the global namespace. + */ + static const std::pair &name_for_id (id_type tag_id); + + /** + * @brief Gets the id for a given name and user flag + * + * This will pull the ID from the global namespace. + */ + static id_type id_for_name (const std::string &name, bool user_flag); + + /** + * @brief Gets a tag object for a given name and user flag + * + * This will create the tag from the global namespace. + */ + static Tag tag_for_name (const std::string &name, bool user_flag); + private: friend class Database; - mutable std::map , id_type> m_ids_for_names; - mutable std::vector m_tags; + mutable std::map m_tags_per_id; + mutable std::vector m_tags; }; /** diff --git a/src/rdb/rdb/rdbFile.cc b/src/rdb/rdb/rdbFile.cc index dc533bf7f..1544ffcaf 100644 --- a/src/rdb/rdb/rdbFile.cc +++ b/src/rdb/rdb/rdbFile.cc @@ -40,51 +40,47 @@ struct ValueConverter { typedef std::string pb_type; - ValueConverter (rdb::Database *rdb) - : mp_rdb (rdb) + ValueConverter () { } std::string to_string (const ValueWrapper &value) const { - return value.to_string (mp_rdb); + return value.to_string (); } void from_string (const std::string &s, ValueWrapper &value) const { - value.from_string (mp_rdb, s); + value.from_string (s); } std::string pb_encode (const ValueWrapper &value) const { - return value.to_string (mp_rdb); + return value.to_string (); } void pb_decode (const std::string &s, ValueWrapper &value) const { - value.from_string (mp_rdb, s); + value.from_string (s); } - -private: - rdb::Database *mp_rdb; }; -static -tl::XMLElementList categories_format = - tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category", - tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name") + - tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description") + - tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories", - &categories_format - ) - ) -; - // generation of the RDB file XML structure static tl::XMLStruct -make_rdb_structure (rdb::Database *rdb) +make_rdb_structure () { - return tl::XMLStruct ("report-database", + static + tl::XMLElementList categories_format = + tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category", + tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name") + + tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description") + + tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories", + &categories_format + ) + ) + ; + + return tl::XMLStruct ("report-database", tl::make_member (&rdb::Database::description, &rdb::Database::set_description, "description") + tl::make_member (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file") + tl::make_member (&rdb::Database::generator, &rdb::Database::set_generator, "generator") + @@ -122,13 +118,15 @@ make_rdb_structure (rdb::Database *rdb) tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment") + tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image") + tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values", - tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter (rdb)) + tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter ()) ) ) ) ); } +static tl::XMLStruct s_rdb_struct = make_rdb_structure (); + // ------------------------------------------------------------- // Implementation of rdb::Database::save and write // TODO: move this somewhere else - with generalized functionality @@ -144,7 +142,7 @@ void rdb::Database::write (const std::string &fn) { tl::OutputStream os (fn, tl::OutputStream::OM_Auto); - make_rdb_structure (this).write (os, *this); + s_rdb_struct.write (os, *this); if (tl::verbosity () >= 10) { tl::log << "Saved RDB to " << fn; @@ -168,7 +166,7 @@ public: { tl::SelfTimer timer (tl::verbosity () >= 11, "Reading marker database file"); tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB"))); - make_rdb_structure (&db).parse (in, db); + s_rdb_struct.parse (in, db); } virtual const char *format () const diff --git a/testdata/ruby/rdbTest.rb b/testdata/ruby/rdbTest.rb index eb99bd859..26d37f38a 100644 --- a/testdata/ruby/rdbTest.rb +++ b/testdata/ruby/rdbTest.rb @@ -595,7 +595,7 @@ class RDB_TestClass < TestBase assert_equal(db.is_modified?, false) tag_id = db.tag_id("x") - assert_equal(tag_id, 1) + assert_equal(db.tag_name(tag_id), "x") db.set_tag_description(tag_id, "xdesc") assert_equal(db.tag_description(tag_id), "xdesc") From 0f833da652c82c3f52889144626b56c52bb98f4c Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 22:03:00 +0200 Subject: [PATCH 21/22] Consolidated solution for RDB merge ('apply'). Potentially one bug fixed (tag mapping) --- src/rdb/rdb/rdb.cc | 136 +++++++++++---------- src/rdb/rdb/rdb.h | 208 +++++++++++++++++---------------- src/rdb/unit_tests/rdbTests.cc | 13 ++- 3 files changed, 196 insertions(+), 161 deletions(-) diff --git a/src/rdb/rdb/rdb.cc b/src/rdb/rdb/rdb.cc index 8e624f78c..289c21d68 100644 --- a/src/rdb/rdb/rdb.cc +++ b/src/rdb/rdb/rdb.cc @@ -278,7 +278,7 @@ template <> RDB_PUBLIC bool Value::is_shape () const } -bool ValueBase::compare (const ValueBase *a, const ValueBase *b) +bool ValueBase::compare (const ValueBase *a, const ValueBase *b) { // compare is the intrinsic compare of equal type and type index for different types. if (a->type_index () == b->type_index ()) { @@ -416,7 +416,7 @@ ValueBase::create_from_shape (const db::Shape &shape, const db::CplxTrans &trans // ------------------------------------------------------------------------------------------ // ValueWrapper implementation -std::string +std::string ValueWrapper::to_string () const { std::string r; @@ -437,14 +437,14 @@ ValueWrapper::to_string () const return r; } -void +void ValueWrapper::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); from_string (ex); } -void +void ValueWrapper::from_string (tl::Extractor &ex) { id_type tag_id = 0; @@ -453,7 +453,7 @@ ValueWrapper::from_string (tl::Extractor &ex) bool user_tag = ex.test ("#"); - std::string tn; + std::string tn; ex.read_word_or_quoted (tn); tag_id = rdb::Tags::id_for_name (tn, user_tag); @@ -482,11 +482,11 @@ Values::compare (const Values &other, const std::set &common_tags) cons Values::const_iterator a = begin (), b = other.begin (); while (a != end () && b != other.end ()) { - id_type t12 = 0; + id_type t1 = 0; while (a != end () && a->tag_id () != 0) { auto j = common_tags.find (a->tag_id ()); if (j != common_tags.end ()) { - t12 = a->tag_id (); + t1 = a->tag_id (); break; } ++a; @@ -506,8 +506,8 @@ Values::compare (const Values &other, const std::set &common_tags) cons return b != other.end (); } - if (t12 != t2) { - return t12 < t2; + if (t1 != t2) { + return t1 < t2; } if (a->get () && b->get ()) { @@ -528,7 +528,7 @@ Values::compare (const Values &other, const std::set &common_tags) cons return b != other.end (); } -std::string +std::string Values::to_string () const { std::string r; @@ -547,7 +547,7 @@ Values::to_string () const return r; } -void +void Values::from_string (const std::string &s) { tl::Extractor ex (s.c_str ()); @@ -567,7 +567,7 @@ Values::from_string (const std::string &s) // ------------------------------------------------------------------------------------------ // Cell implementation -void +void Cells::import_cell (const Cell &c) { Cell *cell; @@ -582,7 +582,7 @@ Cells::import_cell (const Cell &c) cell->references ().insert (*r); } } - + // ------------------------------------------------------------------------------------------ // Cell implementation @@ -615,7 +615,7 @@ Cell::~Cell () // .. nothing yet .. } -std::pair +std::pair Cell::path_to (id_type parent_cell_id, const Database *db) const { if (parent_cell_id == id ()) { @@ -626,7 +626,7 @@ Cell::path_to (id_type parent_cell_id, const Database *db) const } } -std::pair +std::pair Cell::path_to (id_type parent_cell_id, const Database *db, std::set &visited, const db::DCplxTrans &trans) const { for (reference_iterator r = references ().begin (); r != references ().end (); ++r) { @@ -654,15 +654,15 @@ Cell::path_to (id_type parent_cell_id, const Database *db, std::set &v return std::pair (false, db::DCplxTrans ()); } -void +void Cell::import_references (const References &references) { for (References::const_iterator r = references.begin (); r != references.end (); ++r) { m_references.insert (*r); } } - -std::string + +std::string Cell::qname () const { if (m_variant.empty ()) { @@ -687,7 +687,7 @@ References::References (Cell *cell) // .. nothing yet .. } -void +void References::set_database (Database *database) { mp_database = database; @@ -705,20 +705,20 @@ Reference::Reference (References *references) // .. nothing yet .. } -void +void Reference::set_trans_str (const std::string &s) { tl::Extractor ex (s.c_str ()); ex.read (m_trans); } -std::string +std::string Reference::trans_str () const { return m_trans.to_string (); } -void +void Reference::set_parent_cell_qname (const std::string &qname) { tl_assert (mp_database != 0); @@ -729,7 +729,7 @@ Reference::set_parent_cell_qname (const std::string &qname) m_parent_cell_id = cell->id (); } -std::string +std::string Reference::parent_cell_qname () const { tl_assert (mp_database != 0); @@ -787,7 +787,7 @@ Category::sub_categories () return *mp_sub_categories; } -std::string +std::string Category::path () const { std::vector path_elements; @@ -810,7 +810,7 @@ Category::path () const return r; } -void +void Category::import_sub_categories (Categories *categories) { if (mp_sub_categories) { @@ -826,7 +826,7 @@ Category::import_sub_categories (Categories *categories) } } -void +void Category::set_database (Database *database) { mp_database = database; @@ -838,14 +838,14 @@ Category::set_database (Database *database) // ------------------------------------------------------------------------------------------ // Categories implementation -void +void Categories::clear () { m_categories.clear (); m_categories_by_name.clear (); } -void +void Categories::add_category (Category *cat) { cat->set_database (mp_database.get ()); @@ -853,7 +853,7 @@ Categories::add_category (Category *cat) m_categories_by_name.insert (std::make_pair (cat->name (), cat)); } -void +void Categories::set_database (Database *database) { mp_database = database; @@ -863,7 +863,7 @@ Categories::set_database (Database *database) } Category * -Categories::category_by_name (const char *path) +Categories::category_by_name (const char *path) { std::string component; tl::Extractor ex (path); @@ -881,7 +881,7 @@ Categories::category_by_name (const char *path) return 0; } -void +void Categories::import_category (Category *category) { Category *cat; @@ -945,13 +945,31 @@ Tags::Tags () // .. nothing yet .. } -void +void Tags::clear () { m_tags_per_id.clear (); m_tags.clear (); } +void +Tags::remove_tag (id_type id) +{ + auto j = m_tags_per_id.find (id); + if (j != m_tags_per_id.end ()) { + + m_tags.erase (m_tags.begin () + j->second); + + for (auto i = m_tags_per_id.begin (); i != m_tags_per_id.end (); ++i) { + if (i->second > j->second) { + i->second -= 1; + } + } + m_tags_per_id.erase (id); + + } +} + const Tag & Tags::tag (const std::string &name, bool user_tag) const { @@ -995,7 +1013,7 @@ Tags::import_tag (const Tag &t) tt.set_description (t.description ()); } -bool +bool Tags::has_tag (const std::string &name, bool user_tag) const { return m_tags_per_id.find (id_for_name (name, user_tag)) != m_tags_per_id.end (); @@ -1043,31 +1061,31 @@ Item &Item::operator= (const Item &d) return *this; } -void +void Item::add_tag (id_type tag_id) { m_tag_ids.insert (tag_id); } -void +void Item::remove_tag (id_type tag_id) { m_tag_ids.erase (tag_id); } -void +void Item::remove_tags () { m_tag_ids.clear (); } -bool +bool Item::has_tag (id_type tag_id) const { return m_tag_ids.find (tag_id) != m_tag_ids.end (); } -std::string +std::string Item::cell_qname () const { tl_assert (mp_database != 0); @@ -1076,7 +1094,7 @@ Item::cell_qname () const return cell->qname (); } -void +void Item::set_cell_qname (const std::string &qname) { tl_assert (mp_database != 0); @@ -1096,7 +1114,7 @@ Item::category_name () const return category->path (); } -void +void Item::set_category_name (const std::string &path) { tl_assert (mp_database != 0); @@ -1107,7 +1125,7 @@ Item::set_category_name (const std::string &path) m_category_id = category->id (); } -std::string +std::string Item::tag_str () const { tl_assert (mp_database != 0); @@ -1135,7 +1153,7 @@ Item::tag_str () const return r; } -void +void Item::set_tag_str (const std::string &tags) { tl_assert (mp_database != 0); @@ -1153,7 +1171,7 @@ Item::set_tag_str (const std::string &tags) } #if defined(HAVE_QT) -void +void Item::set_image (const QImage &image) { if (image.isNull ()) { @@ -1341,7 +1359,7 @@ Database::import_tags (const Tags &tags) } } -void +void Database::import_categories (Categories *categories) { set_modified (); @@ -1352,7 +1370,7 @@ Database::import_categories (Categories *categories) mp_categories->set_database (this); } -void +void Database::import_cells (const Cells &cells) { set_modified (); @@ -1399,13 +1417,13 @@ Database::create_category (Categories *container, const std::string &name) } Category * -Database::category_by_name_non_const (const std::string &name) +Database::category_by_name_non_const (const std::string &name) { return mp_categories->category_by_name (name.c_str ()); } Category * -Database::category_by_id_non_const (id_type id) +Database::category_by_id_non_const (id_type id) { std::map ::const_iterator c = m_categories_by_id.find (id); if (c != m_categories_by_id.end ()) { @@ -1460,7 +1478,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons } else { new_cell = new Cell (++m_next_id, name, std::string (), layout_name); - + } m_cells.add_cell (new_cell); @@ -1468,7 +1486,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons m_cells_by_qname.insert (std::make_pair (new_cell->qname (), new_cell)); } else { - + new_cell = new Cell (++m_next_id, name, variant, layout_name); m_cells.add_cell (new_cell); m_cells_by_id.insert (std::make_pair (new_cell->id (), new_cell)); @@ -1496,7 +1514,7 @@ Database::create_cell (const std::string &name, const std::string &variant, cons } other_cell->set_variant (tl::to_string (variant_index + 1)); - + m_cells_by_qname.insert (std::make_pair (other_cell->qname (), other_cell)); } @@ -1542,21 +1560,21 @@ Database::cell_by_id_non_const (id_type id) } } -void +void Database::set_tag_description (id_type tag_id, const std::string &description) { set_modified (); tags_non_const ().tag (tag_id).set_description (description); } -void +void Database::add_item_tag (const Item *item, id_type tag) { set_modified (); const_cast (item)->add_tag (tag); } -void +void Database::remove_item_tag (const Item *item, id_type tag) { set_modified (); @@ -1571,7 +1589,7 @@ Database::set_item_comment (const Item *item, const std::string &comment) } #if defined(HAVE_QT) -void +void Database::set_item_image (const Item *item, const QImage &image) { set_modified (); @@ -1586,14 +1604,14 @@ Database::set_item_image_str (const Item *item, const std::string &image_str) const_cast (item)->set_image_str (image_str); } -void +void Database::set_item_multiplicity (const Item *item, size_t n) { set_modified (); const_cast (item)->set_multiplicity (n); } -void +void Database::set_item_visited (const Item *item_c, bool visited) { if (visited != item_c->visited ()) { @@ -1654,7 +1672,7 @@ Database::create_item (id_type cell_id, id_type category_id) static std::list empty_list; -std::pair +std::pair Database::items_by_cell_and_category (id_type cell_id, id_type category_id) const { std::map , std::list >::const_iterator i = m_items_by_cell_and_category_id.find (std::make_pair (cell_id, category_id)); @@ -1731,7 +1749,7 @@ Database::num_items (id_type cell_id, id_type category_id) const } } -size_t +size_t Database::num_items_visited (id_type cell_id, id_type category_id) const { std::map , size_t>::const_iterator n = m_num_items_visited_by_cell_and_category.find (std::make_pair (cell_id, category_id)); @@ -1915,8 +1933,6 @@ Database::apply (const rdb::Database &other) { std::map cell2cell; std::map cat2cat; - std::map tag2tag; - std::map rev_tag2tag; for (auto c = other.cells ().begin (); c != other.cells ().end (); ++c) { // TODO: do we have a consistent scheme of naming variants? What requirements diff --git a/src/rdb/rdb/rdb.h b/src/rdb/rdb/rdb.h index 7f0e2e339..3747ada65 100644 --- a/src/rdb/rdb/rdb.h +++ b/src/rdb/rdb/rdb.h @@ -68,10 +68,10 @@ class Items; /** * @brief A report item's category * - * An item is member of exactly one category. This can be a check for example. + * An item is member of exactly one category. This can be a check for example. * A category is described by a name and a description string. An Id is provided * to reference this category from actual report items. - * Categories can be organized hierarchically for which a category collection + * Categories can be organized hierarchically for which a category collection * is provided and member of the individual category. * * A category can only be created by the database object, since the @@ -112,7 +112,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -130,7 +130,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_name (const std::string &d) + void set_name (const std::string &d) { m_name = d; } @@ -155,7 +155,7 @@ public: /** * @brief The description string (setter) */ - void set_description (const std::string &d) + void set_description (const std::string &d) { m_description = d; } @@ -181,7 +181,7 @@ public: /** * @brief The parent (owner) of this category (getter) */ - Category *parent () + Category *parent () { return mp_parent; } @@ -192,7 +192,7 @@ public: * This method is provided for persistency application only. It should not be used otherwise. * The Category object will take over ownership over the sub categories. */ - void import_sub_categories (Categories *categories); + void import_sub_categories (Categories *categories); /** * @brief Report the number of items @@ -242,7 +242,7 @@ private: * @brief Default constructor * * Creates a category object with empty name and description. - * This constructor is private to allow only the database to produce a + * This constructor is private to allow only the database to produce a * category object. */ Category (const std::string &name); @@ -262,9 +262,9 @@ private: { m_num_items_visited += d; } - + /** - * @brief Add an offset to the number of items + * @brief Add an offset to the number of items */ void add_to_num_items (int d) { @@ -279,7 +279,7 @@ private: m_num_items = 0; m_num_items_visited = 0; } - + /** * @brief Set the database reference */ @@ -293,10 +293,10 @@ private: /** * @brief The collection of categories * - * A generic collection of categories used for the root node and + * A generic collection of categories used for the root node and * sub-category nodes. */ -class RDB_PUBLIC Categories +class RDB_PUBLIC Categories { public: typedef tl::shared_collection::const_iterator const_iterator; @@ -327,39 +327,39 @@ public: /** * @brief Iterate the categories inside this collection (begin iterator) */ - const_iterator begin () const - { - return m_categories.begin (); + const_iterator begin () const + { + return m_categories.begin (); } /** * @brief Iterate the categories inside this collection (end iterator) */ - const_iterator end () const - { - return m_categories.end (); + const_iterator end () const + { + return m_categories.end (); } /** * @brief Iterate the categories inside this collection (begin iterator) */ - iterator begin () - { - return m_categories.begin (); + iterator begin () + { + return m_categories.begin (); } /** * @brief Iterate the categories inside this collection (end iterator) */ - iterator end () - { - return m_categories.end (); + iterator end () + { + return m_categories.end (); } /** * @brief Find a category by name * - * The name is actually a path expression which specifies the category starting from the given + * The name is actually a path expression which specifies the category starting from the given * node with a '.' separated path notation. I.e. 'a.b' is the sub-category 'b' of category 'a'. * If no such category can be found, 0 is returned. */ @@ -371,7 +371,7 @@ public: /** * @brief Find a category by name (non-const version) * - * The name is actually a path expression which specifies the category starting from the given + * The name is actually a path expression which specifies the category starting from the given * node with a '.' separated path notation. I.e. 'a.b' is the sub-category 'b' of category 'a'. * If no such category can be found, 0 is returned. */ @@ -388,7 +388,7 @@ public: * This method is provided for persistency application only. It should not be used otherwise. * This will take over ownership over the category. */ - void import_category (Category *category); + void import_category (Category *category); /** * @brief Gets the database reference @@ -403,7 +403,7 @@ public: friend class Category; tl::shared_collection m_categories; - std::map m_categories_by_name; + std::map m_categories_by_name; tl::weak_ptr mp_database; Categories () @@ -422,7 +422,7 @@ public: * A value has a value (as the name says) and an optional tag id. Tag id's identify * the value and make the value a named one. */ -class RDB_PUBLIC ValueBase +class RDB_PUBLIC ValueBase { public: ValueBase () @@ -496,14 +496,14 @@ public: return m_value; } - C &value () + C &value () { return m_value; } - int type_index () const - { - return type_index_of (); + int type_index () const + { + return type_index_of (); } bool compare (const ValueBase *other) const; @@ -535,7 +535,7 @@ RDB_PUBLIC_TEMPLATE ValueBase *make_value (const T &value) /** * @brief A class encapsulating ValueBase pointer */ -class RDB_PUBLIC ValueWrapper +class RDB_PUBLIC ValueWrapper { public: /** @@ -592,7 +592,7 @@ public: /** * @brief Get the pointer */ - ValueBase *get () + ValueBase *get () { return mp_ptr; } @@ -627,7 +627,7 @@ public: } /** - * @brief Convert the values collection to a string + * @brief Convert the values collection to a string */ std::string to_string () const; @@ -672,19 +672,16 @@ public: } /** - * @brief Assignment + * @brief Assignment */ Values &operator= (const Values &d); /** * @brief Compare two value sets (less operator) * - * This compare function will use the tag mapping provided by tag map ("this" tag id to "other" tag id). - * Values with tags not listed in the tag map will not be compared. + * This compare function will use the tags provide in "common_tags". Tags outside this set are ignored. * Untagged values (tag_id 0) will be compared always. * - * "rev_tag_map" needs to be the reverse of "tag_map". - * * The order of the values matters. */ bool compare (const Values &other, const std::set &common_tags) const; @@ -708,7 +705,7 @@ public: /** * @brief The non-const iterator (begin) */ - iterator begin () + iterator begin () { return m_values.begin (); } @@ -758,7 +755,7 @@ public: } /** - * @brief Convert the values collection to a string + * @brief Convert the values collection to a string */ std::string to_string () const; @@ -774,8 +771,8 @@ private: /** * @brief A report item * - * A report item is one information item in the report. - * The value of a report item is manyfold. Values can be keyed, + * A report item is one information item in the report. + * The value of a report item is manyfold. Values can be keyed, * i.e. multiple values can be present with different keys. * Each value can be of different types where the type is specified by a type Id. */ @@ -789,12 +786,12 @@ public: Item (Items *items); /** - * @brief Copy constructor + * @brief Copy constructor */ Item (const Item &d); /** - * @brief Assignment + * @brief Assignment */ Item &operator= (const Item &d); @@ -816,7 +813,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_cell_id (id_type id) + void set_cell_id (id_type id) { m_cell_id = id; } @@ -846,7 +843,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_category_id (id_type id) + void set_category_id (id_type id) { m_category_id = id; } @@ -1073,9 +1070,9 @@ private: }; /** - * @brief An item reference + * @brief An item reference * - * This is basically a wrapper for a pointer that correctly + * This is basically a wrapper for a pointer that correctly * maps const * and non-const * values through the operator-> * overloads. */ @@ -1141,7 +1138,7 @@ public: /** * @brief Construct an item list with a database reference */ - Items (Database *database) + Items (Database *database) : mp_database (database) { // .. nothing yet .. @@ -1150,33 +1147,33 @@ public: /** * @brief Iterate the items inside this collection (begin iterator) */ - const_iterator begin () const - { - return m_items.begin (); + const_iterator begin () const + { + return m_items.begin (); } /** * @brief Iterate the items inside this collection (end iterator) */ - const_iterator end () const - { - return m_items.end (); + const_iterator end () const + { + return m_items.end (); } /** * @brief Iterate the items inside this collection (non-const begin iterator) */ iterator begin () - { - return m_items.begin (); + { + return m_items.begin (); } /** * @brief Iterate the items inside this collection (non-const end iterator) */ iterator end () - { - return m_items.end (); + { + return m_items.end (); } /** @@ -1195,7 +1192,7 @@ public: * * This method is provided for persistency application only. It should not be used otherwise. */ - Item &back () + Item &back () { return m_items.back (); } @@ -1203,7 +1200,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1350,7 +1347,7 @@ private: }; /** - * @brief A collection of references + * @brief A collection of references */ class RDB_PUBLIC References { @@ -1393,7 +1390,7 @@ public: /** * @brief Begin iterator (non-const) */ - iterator begin () + iterator begin () { return m_references.begin (); } @@ -1401,7 +1398,7 @@ public: /** * @brief End iterator (non-const) */ - iterator end () + iterator end () { return m_references.end (); } @@ -1417,7 +1414,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1500,7 +1497,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -1518,7 +1515,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_name (const std::string &d) + void set_name (const std::string &d) { m_name = d; } @@ -1536,7 +1533,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_variant (const std::string &v) + void set_variant (const std::string &v) { m_variant = v; } @@ -1591,7 +1588,7 @@ public: /** * @brief The reference collection (non-const) */ - References &references () + References &references () { return m_references; } @@ -1604,10 +1601,10 @@ public: void import_references (const References &references); /** - * @brief Get one example transformation leading from this cell to a given parent cell + * @brief Get one example transformation leading from this cell to a given parent cell * * This method will try to determine one path from the given cell to the given parent - * cell and return the accumulated transformation for this path. + * cell and return the accumulated transformation for this path. * If no path is found, the first parameter of the returned pair is false, otherwise it's * true. */ @@ -1616,7 +1613,7 @@ public: /** * @brief Get the database reference */ - Database *database () + Database *database () { return mp_database; } @@ -1654,7 +1651,7 @@ private: } /** - * @brief Add an offset to the number of items + * @brief Add an offset to the number of items */ void add_to_num_items (int d) { @@ -1669,7 +1666,7 @@ private: m_num_items = 0; m_num_items_visited = 0; } - + /** * @brief Set the database reference */ @@ -1683,7 +1680,7 @@ private: }; /** - * @brief A collection of cells + * @brief A collection of cells */ class RDB_PUBLIC Cells { @@ -1743,7 +1740,7 @@ public: /** * @brief Begin iterator (non-const) */ - iterator begin () + iterator begin () { return m_cells.begin (); } @@ -1751,7 +1748,7 @@ public: /** * @brief End iterator (non-const) */ - iterator end () + iterator end () { return m_cells.end (); } @@ -1784,7 +1781,7 @@ public: /** * @brief Get the database reference (non-const) */ - Database *database () + Database *database () { return mp_database.get (); } @@ -1848,9 +1845,9 @@ public: /** * @brief Gets a flag indicating whether the tag is a user tag or a system tag - * + * * If this flag is false, the tag is a system tag used for tagging "waived" and - * similar conditions. Otherwise it is a user tag which can be used freely to + * similar conditions. Otherwise it is a user tag which can be used freely to * tag arbitrary conditions. */ bool is_user_tag () const @@ -1863,7 +1860,7 @@ public: * * See \is_user_tag for details. */ - void set_user_tag (bool user) + void set_user_tag (bool user) { m_is_user_tag = user; } @@ -1881,7 +1878,7 @@ public: * * This method must not be used for items in the database to keep the database consistent. */ - void set_id (id_type id) + void set_id (id_type id) { m_id = id; } @@ -1900,7 +1897,7 @@ public: * The name of the tag must not be changed when the tag is already part of a Tags collection. * Otherwise, the tag collection becomes inconsistent. */ - void set_name (const std::string &name) + void set_name (const std::string &name) { m_name = name; } @@ -1921,7 +1918,7 @@ public: typedef tag_list_type::const_iterator const_iterator; /** - * @brief Default constructor for the tags list + * @brief Default constructor for the tags list * * This method is provided for persistency application only. It should not be used otherwise. */ @@ -1972,16 +1969,27 @@ public: * @brief Import a tag * * This method is provided for persistency application only. It should not be used otherwise. - * This will assign a new id to the tag and replace any tag with that + * This will assign a new id to the tag and replace any tag with that * name. */ void import_tag (const Tag &tag); /** * @brief Clear the collection of tags + * + * NOTE: this will not remove the tags from items or values, so the use cases for this method + * are limited. */ void clear (); + /** + * @brief Removes the tag with the given ID + * + * NOTE: this will not remove the tags from items or values, so the use cases for this method + * are limited. + */ + void remove_tag (id_type id); + /** * @brief Gets the name and user flag for a tag ID * @@ -2013,7 +2021,7 @@ private: /** * @brief The database object */ -class RDB_PUBLIC Database +class RDB_PUBLIC Database : public gsi::ObjectBase, public tl::Object { @@ -2123,7 +2131,7 @@ public: } /** - * @brief Set the top cell name + * @brief Set the top cell name */ void set_top_cell_name (const std::string &topcell) { @@ -2179,17 +2187,17 @@ public: void import_categories (Categories *categories); /** - * @brief Create a category and register it + * @brief Create a category and register it */ Category *create_category (const std::string &name); /** - * @brief Create a category as a subcategory and register it + * @brief Create a category as a subcategory and register it */ Category *create_category (Category *parent, const std::string &name); /** - * @brief Create a category as a subcategory in the container and register it + * @brief Create a category as a subcategory in the container and register it * * Hint: this method does not set the parent properly and must not be used * under normal circumstances. It is provided as a internal method and @@ -2268,7 +2276,7 @@ public: * @brief Create a cell variant and register it * * A cell with name name/variant combination must not exist already. - * If the variant string is empty, this method behaves the same as the + * If the variant string is empty, this method behaves the same as the * method without variant. * * "layout_name" is the name of the cell in the layout. If empty, the layout @@ -2417,7 +2425,7 @@ public: /** * @brief Get an iterator pair that delivers the const items (ItemRef) for a given cell */ - std::pair items_by_cell (id_type cell_id) const; + std::pair items_by_cell (id_type cell_id) const; /** * @brief Get an iterator pair that delivers the non-const items (ItemRef) for a given cell @@ -2427,7 +2435,7 @@ public: /** * @brief Get an iterator that delivers the const items (ItemRef) for a given category */ - std::pair items_by_category (id_type category_id) const; + std::pair items_by_category (id_type category_id) const; /** * @brief Get an iterator that delivers the non-const items (ItemRef) for a given category @@ -2437,7 +2445,7 @@ public: /** * @brief Get an iterator that delivers the const items (ItemRef) for a given cell and category */ - std::pair items_by_cell_and_category (id_type cell_id, id_type category_id) const; + std::pair items_by_cell_and_category (id_type cell_id, id_type category_id) const; /** * @brief Get an iterator that delivers the non-const items (ItemRef) for a given cell and category @@ -2455,7 +2463,7 @@ public: /** * @brief Reset the modified file */ - void reset_modified () + void reset_modified () { m_modified = false; } @@ -2526,7 +2534,7 @@ private: void clear (); - void set_modified () + void set_modified () { m_modified = true; } diff --git a/src/rdb/unit_tests/rdbTests.cc b/src/rdb/unit_tests/rdbTests.cc index 88b48676e..247b9c1c0 100644 --- a/src/rdb/unit_tests/rdbTests.cc +++ b/src/rdb/unit_tests/rdbTests.cc @@ -816,7 +816,18 @@ TEST(13_ApplyIgnoreUnknownTag) i2->values ().clear (); i2->add_value (std::string ("xyz"), vtag22); - // values with incompatible tags are ignored -> tag2 is applied + // values with incompatible tags are ignored, but vtag1 is a common tag. + // So far, nothing is applied as the we match abc[vtag1] vs. xyz[vtag2] which is both different value and tag- + db1.apply (db2); + + EXPECT_EQ (i1->tag_str (), ""); + + // NOTE: don't do this at home + const_cast (db2.tags ()).remove_tag (vtag21); + + // vtag1 is no longer a common tag -> we match abc[vtag1] vs. xyz[vtag2] where vtag1 is not known on the + // other side and vtag2 is not known on the self side. + // Hence, the values are ignored and the tag is applied. db1.apply (db2); EXPECT_EQ (i1->tag_str (), "tag2"); From f9a1f730c114353984fc703de685a1ed1bdab5b0 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sat, 17 Aug 2024 22:53:59 +0200 Subject: [PATCH 22/22] Binary RDB format. Missing: progress bar on reading/writing, it is somewhat slower than necessary because of the counting passes. --- src/layui/layui/rdbMarkerBrowserDialog.cc | 10 +- src/rdb/rdb/gsiDeclRdb.cc | 6 +- src/rdb/rdb/rdb.cc | 2 + src/rdb/rdb/rdb.h | 140 ++++++++++--------- src/rdb/rdb/rdbFile.cc | 158 +++++++++++++++------- 5 files changed, 202 insertions(+), 114 deletions(-) diff --git a/src/layui/layui/rdbMarkerBrowserDialog.cc b/src/layui/layui/rdbMarkerBrowserDialog.cc index 56d32c2cf..f31e1233a 100644 --- a/src/layui/layui/rdbMarkerBrowserDialog.cc +++ b/src/layui/layui/rdbMarkerBrowserDialog.cc @@ -387,7 +387,7 @@ BEGIN_PROTECTED } else { - rdb->save (rdb->filename ()); + rdb->save (rdb->filename (), rdb->binary ()); rdb->reset_modified (); } @@ -459,7 +459,7 @@ BEGIN_PROTECTED throw tl::Exception (tl::to_string (tr ("The current report database is not saved.\nSave it to some file with 'Save As', before saving it as waiver DB."))); } - rdb->write (rdb->filename () + ".w"); + rdb->write (rdb->filename () + ".w", rdb->binary ()); END_PROTECTED } @@ -475,11 +475,13 @@ BEGIN_PROTECTED if (rdb) { // prepare and open the file dialog - lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), "KLayout RDB files (*.lyrdb)"); + lay::FileDialog save_dialog (this, tl::to_string (QObject::tr ("Save Marker Database File")), tl::to_string (tr ("KLayout RDB files (*.lyrdb);;KLayout binary RDB files (*.rdb);;All files (*)"))); std::string fn (rdb->filename ()); if (save_dialog.get_save (fn)) { - rdb->save (fn); + bool binary = (tl::extension (fn) == "rdb"); + + rdb->save (fn, binary); rdb->reset_modified (); // update the RDB title strings diff --git a/src/rdb/rdb/gsiDeclRdb.cc b/src/rdb/rdb/gsiDeclRdb.cc index 920334d43..41d6355e2 100644 --- a/src/rdb/rdb/gsiDeclRdb.cc +++ b/src/rdb/rdb/gsiDeclRdb.cc @@ -1677,10 +1677,12 @@ Class decl_ReportDatabase ("rdb", "ReportDatabase", "The reader recognizes the format automatically and will choose the appropriate decoder. 'gzip' compressed files are uncompressed " "automatically.\n" ) + - gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"), + gsi::method ("save", &rdb::Database::save, gsi::arg ("filename"), gsi::arg ("binary", false), "@brief Saves the database to the given file\n" "@param filename The file to which to save the database\n" - "The database is always saved in KLayout's XML-based format.\n" + "The database is saved in KLayout's XML-based format if 'binary' is false or a ProtocolBuffer-based binary format if 'binary' is true.\n" + "\n" + "The 'binary' argument has been added in version 0.29.7." ), "@brief The report database object\n" "A report database is organized around a set of items which are associated with cells and categories. " diff --git a/src/rdb/rdb/rdb.cc b/src/rdb/rdb/rdb.cc index 289c21d68..cbdfbe880 100644 --- a/src/rdb/rdb/rdb.cc +++ b/src/rdb/rdb/rdb.cc @@ -1767,6 +1767,7 @@ Database::clear () m_generator = ""; m_filename = ""; + m_binary = false; m_description = ""; m_name = ""; m_topcell = ""; @@ -1819,6 +1820,7 @@ Database::load (std::string fn) tl::log << "Loading RDB from " << fn; clear (); + set_binary (false); tl::InputStream stream (fn); diff --git a/src/rdb/rdb/rdb.h b/src/rdb/rdb/rdb.h index 3747ada65..b5f4f4180 100644 --- a/src/rdb/rdb/rdb.h +++ b/src/rdb/rdb/rdb.h @@ -2044,7 +2044,7 @@ public: ~Database (); /** - * @brief Get the database description + * @brief Gets the database description */ const std::string &description () const { @@ -2052,7 +2052,7 @@ public: } /** - * @brief Set the database description + * @brief Sets the database description */ void set_description (const std::string &description) { @@ -2061,7 +2061,7 @@ public: } /** - * @brief Get the database original file + * @brief Gets the database original file * * The original file describes what original file the marker database * was derived from. @@ -2072,7 +2072,7 @@ public: } /** - * @brief Set the database original file + * @brief Sets the database original file */ void set_original_file (const std::string &original_file) { @@ -2081,7 +2081,7 @@ public: } /** - * @brief Get the database name + * @brief Gets the database name */ const std::string &name () const { @@ -2089,7 +2089,7 @@ public: } /** - * @brief Set the database name + * @brief Sets the database name */ void set_name (const std::string &name) { @@ -2097,7 +2097,7 @@ public: } /** - * @brief Get the file name + * @brief Gets the file name */ const std::string &filename () const { @@ -2105,7 +2105,7 @@ public: } /** - * @brief Set the file name + * @brief Sets the file name */ void set_filename (const std::string &filename) { @@ -2114,7 +2114,24 @@ public: } /** - * @brief Get the generator name + * @brief Gets a value indicating whether the database was saved to binary format + */ + bool binary () const + { + return m_binary; + } + + /** + * @brief Sets a value indicating whether the database was saved to binary format + */ + void set_binary (bool bin) + { + set_modified (); + m_binary = bin; + } + + /** + * @brief Gets the generator name */ const std::string &generator () const { @@ -2122,7 +2139,7 @@ public: } /** - * @brief Set the generator name + * @brief Sets the generator name */ void set_generator (const std::string &generator) { @@ -2131,7 +2148,7 @@ public: } /** - * @brief Set the top cell name + * @brief Sets the top cell name */ void set_top_cell_name (const std::string &topcell) { @@ -2140,7 +2157,7 @@ public: } /** - * @brief Return the top cell name + * @brief Returns the top cell name */ const std::string &top_cell_name () const { @@ -2148,7 +2165,7 @@ public: } /* - * @brief Get the reference to the tags collection (const version) + * @brief Gets the reference to the tags collection (const version) */ const Tags &tags () const { @@ -2156,14 +2173,14 @@ public: } /** - * @brief Import tags + * @brief Imports tags * * This method is provided for persistency application only. It should not be used otherwise. */ void import_tags (const Tags &tags); /** - * @brief Get the reference to the categories collection (const version) + * @brief Gets the reference to the categories collection (const version) */ const Categories &categories () const { @@ -2171,7 +2188,7 @@ public: } /** - * @brief Get the reference to the categories collection (non-const version) + * @brief Gets the reference to the categories collection (non-const version) */ Categories &categories_non_const () { @@ -2179,7 +2196,7 @@ public: } /** - * @brief Import categories + * @brief Imports categories * * This method is provided for persistency application only. It should not be used otherwise. * This will take over the ownership over the Categories object. @@ -2187,17 +2204,17 @@ public: void import_categories (Categories *categories); /** - * @brief Create a category and register it + * @brief Creates a category and register it */ Category *create_category (const std::string &name); /** - * @brief Create a category as a subcategory and register it + * @brief Creates a category as a subcategory and register it */ Category *create_category (Category *parent, const std::string &name); /** - * @brief Create a category as a subcategory in the container and register it + * @brief Creates a category as a subcategory in the container and register it * * Hint: this method does not set the parent properly and must not be used * under normal circumstances. It is provided as a internal method and @@ -2206,7 +2223,7 @@ public: Category *create_category (Categories *container, const std::string &name); /** - * @brief Get the category pointer for a category name (const version) + * @brief Gets the category pointer for a category name (const version) * * This method returns 0 if the category name is invalid. */ @@ -2216,7 +2233,7 @@ public: } /** - * @brief Get the category pointer for a category id (const version) + * @brief Gets the category pointer for a category id (const version) * * This method returns 0 if the category is invalid. */ @@ -2226,21 +2243,21 @@ public: } /** - * @brief Get the category pointer for a category name (non-const version) + * @brief Gets the category pointer for a category name (non-const version) * * This method returns 0 if the category name is invalid. */ Category *category_by_name_non_const (const std::string &name); /** - * @brief Get the category pointer for a category id (non-const version) + * @brief Gets the category pointer for a category id (non-const version) * * This method returns 0 if the category is invalid. */ Category *category_by_id_non_const (id_type id); /** - * @brief Access to the cell collection (const) + * @brief Gets the cell collection (const) */ const Cells &cells () const { @@ -2248,7 +2265,7 @@ public: } /** - * @brief Access to the cell collection + * @brief Gets the cell collection */ Cells &cells_non_const () { @@ -2256,14 +2273,14 @@ public: } /** - * @brief Import cells + * @brief Imports cells * * This method is provided for persistency application only. It should not be used otherwise. */ void import_cells (const Cells &cells); /** - * @brief Create a cell and register it + * @brief Creates a cell and registers it * * If a cell with that name already exists, a variant is created */ @@ -2273,7 +2290,7 @@ public: } /** - * @brief Create a cell variant and register it + * @brief Creates a cell variant and registers it * * A cell with name name/variant combination must not exist already. * If the variant string is empty, this method behaves the same as the @@ -2285,14 +2302,14 @@ public: Cell *create_cell (const std::string &name, const std::string &variant, const std::string &layout_name); /** - * @brief Get all variants registered for a given cell name (not qname!) + * @brief Gets all variants registered for a given cell name (not qname!) * * @return a vector of id's corresponding to the given variants or an empty vector if the name is not valid or the cell has no variants */ const std::vector &variants (const std::string &name); /** - * @brief Get the cell pointer for a cell name or name:variant combination (const version) + * @brief Gets the cell pointer for a cell name or name:variant combination (const version) * * This method returns 0 if the cell name or name:variant combination is invalid. */ @@ -2302,7 +2319,7 @@ public: } /** - * @brief Get the cell pointer for a cell id (const version) + * @brief Gets the cell pointer for a cell id (const version) * * This method returns 0 if the cell id is invalid. */ @@ -2312,21 +2329,21 @@ public: } /** - * @brief Get the cell pointer for a cell name or name:variant combination (non-const version) + * @brief Gets the cell pointer for a cell name or name:variant combination (non-const version) * * This method returns 0 if the cell name or name:variant combination is invalid. */ Cell *cell_by_qname_non_const (const std::string &qname); /** - * @brief Get the cell pointer for a cell id (non-const version) + * @brief Gets the cell pointer for a cell id (non-const version) * * This method returns 0 if the cell id is invalid. */ Cell *cell_by_id_non_const (id_type id); /** - * @brief Report the number of items in total + * @brief Reports the number of items in total */ size_t num_items () const { @@ -2334,7 +2351,7 @@ public: } /** - * @brief Report the number of items visited + * @brief Reports the number of items visited */ size_t num_items_visited () const { @@ -2342,37 +2359,37 @@ public: } /** - * @brief Report the number of items for a given cell and category id + * @brief Reports the number of items for a given cell and category id */ size_t num_items (id_type cell_id, id_type category_id) const; /** - * @brief Report the number of items visited + * @brief Reports the number of items visited */ size_t num_items_visited (id_type cell_id, id_type category_id) const; /** - * @brief Create a new item for the given cell and category (both given by id) + * @brief Creates a new item for the given cell and category (both given by id) */ Item *create_item (id_type cell_id, id_type category_id); /** - * @brief Set a tag's description + * @brief Sets a tag's description */ void set_tag_description (id_type tag_id, const std::string &description); /** - * @brief Set an item to visited state or reset the state + * @brief Sets an item to visited state or reset the state */ void set_item_visited (const Item *item, bool visited); /** - * @brief Add a tag to the given item + * @brief Adds a tag to the given item */ void add_item_tag (const Item *item, id_type tag); /** - * @brief Remove a tag from the given item + * @brief Removes a tag from the given item */ void remove_item_tag (const Item *item, id_type tag); @@ -2383,23 +2400,23 @@ public: #if defined(HAVE_QT) /** - * @brief Set the image of an item + * @brief Sets the image of an item */ void set_item_image (const Item *item, const QImage &image); #endif /** - * @brief Set the image string of an item + * @brief Sets the image string of an item */ void set_item_image_str (const Item *item, const std::string &image_str); /** - * @brief Set the multiplicity of an item + * @brief Sets the multiplicity of an item */ void set_item_multiplicity (const Item *item, size_t n); /** - * @brief Get the items collection (const version) + * @brief Gets the items collection (const version) */ const Items &items () const { @@ -2407,7 +2424,7 @@ public: } /** - * @brief Get the items collection (non-const version) + * @brief Gets the items collection (non-const version) */ Items &items_non_const () { @@ -2415,7 +2432,7 @@ public: } /** - * @brief Set the items collection + * @brief Sets the items collection * * This method is provided for persistency application only. It should not be used otherwise. * This will take ownership over the items collection. @@ -2423,32 +2440,32 @@ public: void set_items (Items *items); /** - * @brief Get an iterator pair that delivers the const items (ItemRef) for a given cell + * @brief Gets an iterator pair that delivers the const items (ItemRef) for a given cell */ std::pair items_by_cell (id_type cell_id) const; /** - * @brief Get an iterator pair that delivers the non-const items (ItemRef) for a given cell + * @brief Gets an iterator pair that delivers the non-const items (ItemRef) for a given cell */ std::pair items_by_cell (id_type cell_id); /** - * @brief Get an iterator that delivers the const items (ItemRef) for a given category + * @brief Gets an iterator that delivers the const items (ItemRef) for a given category */ std::pair items_by_category (id_type category_id) const; /** - * @brief Get an iterator that delivers the non-const items (ItemRef) for a given category + * @brief Gets an iterator that delivers the non-const items (ItemRef) for a given category */ std::pair items_by_category (id_type category_id); /** - * @brief Get an iterator that delivers the const items (ItemRef) for a given cell and category + * @brief Gets an iterator that delivers the const items (ItemRef) for a given cell and category */ std::pair items_by_cell_and_category (id_type cell_id, id_type category_id) const; /** - * @brief Get an iterator that delivers the non-const items (ItemRef) for a given cell and category + * @brief Gets an iterator that delivers the non-const items (ItemRef) for a given cell and category */ std::pair items_by_cell_and_category (id_type cell_id, id_type category_id); @@ -2461,7 +2478,7 @@ public: } /** - * @brief Reset the modified file + * @brief Resets the modified file */ void reset_modified () { @@ -2469,19 +2486,19 @@ public: } /** - * @brief Save the database to a file + * @brief Saves the database to a file */ - void save (const std::string &filename); + void save (const std::string &filename, bool binary = false); /** - * @brief Write the database to a file + * @brief Writes the database to a file * * This function is like "save", but does not update the file name attribute. */ - void write (const std::string &filename); + void write (const std::string &filename, bool binary = false); /** - * @brief Load the database from a file + * @brief Loads the database from a file * * Note: This clears the existing database. * The argument intentionally is a copy, so we can call @@ -2510,6 +2527,7 @@ public: private: std::string m_generator; std::string m_filename; + bool m_binary; std::string m_description; std::string m_original_file; std::string m_name; diff --git a/src/rdb/rdb/rdbFile.cc b/src/rdb/rdb/rdbFile.cc index 1544ffcaf..35995c872 100644 --- a/src/rdb/rdb/rdbFile.cc +++ b/src/rdb/rdb/rdbFile.cc @@ -71,81 +71,96 @@ make_rdb_structure () { static tl::XMLElementList categories_format = - tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category", - tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name") + - tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description") + - tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories", + tl::make_element_with_parent_ref (&rdb::Categories::begin, &rdb::Categories::end, &rdb::Categories::import_category, "category#1", + tl::make_member (&rdb::Category::name, &rdb::Category::set_name, "name#1") + + tl::make_member (&rdb::Category::description, &rdb::Category::set_description, "description#2") + + tl::make_element_with_parent_ref (&rdb::Category::sub_categories, &rdb::Category::import_sub_categories, "categories#3", &categories_format ) ) ; - return tl::XMLStruct ("report-database", - tl::make_member (&rdb::Database::description, &rdb::Database::set_description, "description") + - tl::make_member (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file") + - tl::make_member (&rdb::Database::generator, &rdb::Database::set_generator, "generator") + - tl::make_member (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell") + - tl::make_element (&rdb::Database::tags, &rdb::Database::import_tags, "tags", - tl::make_element (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag", - tl::make_member (&rdb::Tag::name, &rdb::Tag::set_name, "name") + - tl::make_member (&rdb::Tag::description, &rdb::Tag::set_description, "description") + return tl::XMLStruct ("report-database#0", + tl::make_member (&rdb::Database::description, &rdb::Database::set_description, "description#1") + + tl::make_member (&rdb::Database::original_file, &rdb::Database::set_original_file, "original-file#2") + + tl::make_member (&rdb::Database::generator, &rdb::Database::set_generator, "generator#3") + + tl::make_member (&rdb::Database::top_cell_name, &rdb::Database::set_top_cell_name, "top-cell#4") + + tl::make_element (&rdb::Database::tags, &rdb::Database::import_tags, "tags#5", + tl::make_element (&rdb::Tags::begin_tags, &rdb::Tags::end_tags, &rdb::Tags::import_tag, "tag#1", + tl::make_member (&rdb::Tag::name, &rdb::Tag::set_name, "name#1") + + tl::make_member (&rdb::Tag::description, &rdb::Tag::set_description, "description#2") ) ) + - tl::make_element_with_parent_ref (&rdb::Database::categories, &rdb::Database::import_categories, "categories", + tl::make_element_with_parent_ref (&rdb::Database::categories, &rdb::Database::import_categories, "categories#6", &categories_format ) + - tl::make_element_with_parent_ref (&rdb::Database::cells, &rdb::Database::import_cells, "cells", + tl::make_element_with_parent_ref (&rdb::Database::cells, &rdb::Database::import_cells, "cells#7", // must be sorted cells (children after parents)! - tl::make_element_with_parent_ref (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell", - tl::make_member (&rdb::Cell::name, &rdb::Cell::set_name, "name") + - tl::make_member (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant") + - tl::make_member (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name") + - tl::make_element_with_parent_ref (&rdb::Cell::references, &rdb::Cell::import_references, "references", - tl::make_element_with_parent_ref (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref", - tl::make_member (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent") + - tl::make_member (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans") + tl::make_element_with_parent_ref (&rdb::Cells::begin, &rdb::Cells::end, &rdb::Cells::import_cell, "cell#1", + tl::make_member (&rdb::Cell::name, &rdb::Cell::set_name, "name#1") + + tl::make_member (&rdb::Cell::variant, &rdb::Cell::set_variant, "variant#2") + + tl::make_member (&rdb::Cell::layout_name, &rdb::Cell::set_layout_name, "layout-name#3") + + tl::make_element_with_parent_ref (&rdb::Cell::references, &rdb::Cell::import_references, "references#4", + tl::make_element_with_parent_ref (&rdb::References::begin, &rdb::References::end, &rdb::References::insert, "ref#1", + tl::make_member (&rdb::Reference::parent_cell_qname, &rdb::Reference::set_parent_cell_qname, "parent#1") + + tl::make_member (&rdb::Reference::trans_str, &rdb::Reference::set_trans_str, "trans#2") ) ) ) ) + - tl::make_element_with_parent_ref (&rdb::Database::items, &rdb::Database::set_items, "items", - tl::make_element_with_parent_ref (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item", - tl::make_member (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags") + - tl::make_member (&rdb::Item::category_name, &rdb::Item::set_category_name, "category") + - tl::make_member (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell") + - tl::make_member (&rdb::Item::visited, &rdb::Item::set_visited, "visited") + - tl::make_member (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity") + - tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment") + - tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image") + - tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values", - tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value", ValueConverter ()) - ) + tl::make_element_with_parent_ref (&rdb::Database::items, &rdb::Database::set_items, "items#8", + tl::make_element_with_parent_ref (&rdb::Items::begin, &rdb::Items::end, &rdb::Items::add_item, "item#1", + tl::make_member (&rdb::Item::tag_str, &rdb::Item::set_tag_str, "tags#1") + + tl::make_member (&rdb::Item::category_name, &rdb::Item::set_category_name, "category#2") + + tl::make_member (&rdb::Item::cell_qname, &rdb::Item::set_cell_qname, "cell#3") + + tl::make_member (&rdb::Item::visited, &rdb::Item::set_visited, "visited#4") + + tl::make_member (&rdb::Item::multiplicity, &rdb::Item::set_multiplicity, "multiplicity#5") + + tl::make_member (&rdb::Item::comment, &rdb::Item::set_comment, "comment#6") + + tl::make_member (&rdb::Item::image_str, &rdb::Item::set_image_str, "image#7") + + tl::make_element (&rdb::Item::values, &rdb::Item::set_values, "values#8", + tl::make_member (&rdb::Values::begin, &rdb::Values::end, &rdb::Values::add, "value#1", ValueConverter ()) ) ) ) ); } static tl::XMLStruct s_rdb_struct = make_rdb_structure (); +static tl::RegisteredClass rdb_struct_def (&s_rdb_struct, 0, "KLayout-RDB", false); // ------------------------------------------------------------- // Implementation of rdb::Database::save and write // TODO: move this somewhere else - with generalized functionality void -rdb::Database::save (const std::string &fn) +rdb::Database::save (const std::string &fn, bool binary) { - write (fn); + write (fn, binary); set_filename (fn); + set_binary (binary); } void -rdb::Database::write (const std::string &fn) +rdb::Database::write (const std::string &fn, bool binary) { tl::OutputStream os (fn, tl::OutputStream::OM_Auto); - s_rdb_struct.write (os, *this); - if (tl::verbosity () >= 10) { - tl::log << "Saved RDB to " << fn; + if (binary) { + + tl::ProtocolBufferWriter writer (os); + s_rdb_struct.write (writer, *this); + + if (tl::verbosity () >= 10) { + tl::log << tl::to_string (tr ("Saved binary RDB to ")) << fn; + } + + } else { + + s_rdb_struct.write (os, *this); + + if (tl::verbosity () >= 10) { + tl::log << tl::to_string (tr ("Saved RDB to ")) << fn; + } + } } @@ -156,8 +171,8 @@ class StandardReader : public ReaderBase { public: - StandardReader (tl::InputStream &stream) - : m_input_stream (stream) + StandardReader (tl::InputStream &stream, bool binary) + : m_input_stream (stream), m_binary (binary) { // .. nothing yet .. } @@ -165,17 +180,30 @@ public: virtual void read (Database &db) { tl::SelfTimer timer (tl::verbosity () >= 11, "Reading marker database file"); - tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB"))); - s_rdb_struct.parse (in, db); + + if (m_binary) { + + tl::ProtocolBufferReader reader (m_input_stream); + s_rdb_struct.parse (reader, db); + db.set_binary (true); + + } else { + + tl::XMLStreamSource in (m_input_stream, tl::to_string (tr ("Reading RDB"))); + s_rdb_struct.parse (in, db); + db.set_binary (false); + + } } virtual const char *format () const { - return "KLayout-RDB"; + return m_binary ? "KLayout-RDB-PB" : "KLayout-RDB"; } private: tl::InputStream &m_input_stream; + bool m_binary; }; class StandardFormatDeclaration @@ -203,10 +231,46 @@ class StandardFormatDeclaration virtual ReaderBase *create_reader (tl::InputStream &s) const { - return new StandardReader (s); + return new StandardReader (s, false); } }; static tl::RegisteredClass format_decl (new StandardFormatDeclaration (), 0, "KLayout-RDB"); +class BinaryFormatDeclaration + : public FormatDeclaration +{ + virtual std::string format_name () const { return "KLayout-RDB-PB"; } + virtual std::string format_desc () const { return "KLayout binary report database format"; } + virtual std::string file_format () const { return "KLayout binary RDB files (*.rdb *.rdb.gz)"; } + + virtual bool detect (tl::InputStream &stream) const + { + static const char header[] = { + // ProtocolBuffer wire format, LEN record with ID 0 and string "report-database". + 0x02, 0x0f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2d, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65 + }; + + const char *h = stream.get (sizeof (header)); + if (! h) { + return false; + } + + for (size_t i = 0; i < sizeof (header); ++i) { + if (h[i] != header[i]) { + return false; + } + } + + return true; + } + + virtual ReaderBase *create_reader (tl::InputStream &s) const + { + return new StandardReader (s, true); + } +}; + +static tl::RegisteredClass pb_format_decl (new BinaryFormatDeclaration (), 1, "KLayout-RDB-PB"); + }