diff --git a/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc b/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc index e44086c74..e73fe153b 100644 --- a/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc +++ b/src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc @@ -25,6 +25,7 @@ #include "dbOASISWriter.h" #include "dbTextWriter.h" #include "dbTestSupport.h" +#include "dbLayoutDiff.h" #include "tlLog.h" #include "tlUnitTest.h" #include "tlStream.h" @@ -696,3 +697,49 @@ TEST(BlendCrash) std::string fn_au (tl::testdata () + "/oasis/blend_crash_au.gds.gz"); db::compare_layouts (_this, layout, fn_au, db::WriteGDS2, 1); } + +TEST(CBlockLargePropertyString) +{ + // NOTE: we try different blob sizes for triggering various reallocation szenarios + for (size_t f = 0; f < 50; ++f) { + + size_t blob_size = size_t (12345) * f; + + tl::info << "Trying blob size of " << blob_size; + + db::Layout layout_org (false); + + unsigned int layer = layout_org.insert_layer (db::LayerProperties (1, 0)); + db::Cell &top = layout_org.cell (layout_org.add_cell ("TOP")); + + std::string large_value (blob_size, 'a'); + + db::PropertiesSet ps; + ps.insert (db::property_names_id (tl::Variant ("blob")), tl::Variant (large_value)); + + top.shapes (layer).insert (db::BoxWithProperties (db::Box (0, 0, 100, 100), db::properties_id (ps))); + + std::string tmp_file = tl::TestBase::tmp_file ("tmp_OASISReaderLargeString.oas"); + + { + tl::OutputStream out (tmp_file); + db::SaveLayoutOptions options; + db::OASISWriterOptions &oasis_options = options.get_options (); + oasis_options.write_cblocks = true; + oasis_options.strict_mode = false; + db::OASISWriter writer; + writer.write (layout_org, out, options); + } + + db::Layout layout_read; + + { + tl::InputStream in (tmp_file); + db::OASISReader reader (in); + reader.read (layout_read); + } + + EXPECT_EQ (db::compare_layouts (layout_org, layout_read, db::layout_diff::f_verbose, 0), true); + + } +} diff --git a/src/tl/tl/tlDeflate.cc b/src/tl/tl/tlDeflate.cc index e2a2b2cde..cd31b8544 100644 --- a/src/tl/tl/tlDeflate.cc +++ b/src/tl/tl/tlDeflate.cc @@ -26,6 +26,7 @@ #include "tlAssert.h" #include +#include #include @@ -227,9 +228,12 @@ InflateFilter::InflateFilter (tl::InputStream &input) m_last_block (false), m_uncompressed_length (0) // this forces a new block on "process()" { - for (size_t i = 0; i < sizeof (m_buffer) / sizeof (m_buffer [0]); ++i) { - m_buffer[i] = 0; - } + // NOTE: the minimum buffer size of 65536 corresponds to the maximum block size + // of the block repetition decoder + m_blen = 65536; // initially + + m_buffer = new char [m_blen]; + std::memset (m_buffer, 0, m_blen); mp_dist_decoder = new HuffmannDecoder (); mp_lit_decoder = new HuffmannDecoder (); @@ -237,6 +241,9 @@ InflateFilter::InflateFilter (tl::InputStream &input) InflateFilter::~InflateFilter () { + delete[] m_buffer; + m_buffer = 0; + m_blen = 0; delete mp_dist_decoder; mp_dist_decoder = 0; delete mp_lit_decoder; @@ -246,9 +253,36 @@ InflateFilter::~InflateFilter () const char * InflateFilter::get (size_t n) { - tl_assert (n < sizeof (m_buffer) / 2); + size_t blen = m_blen; + while (n >= blen / 2) { + blen *= 2; + } - while ((m_b_insert + sizeof (m_buffer) - m_b_read) % sizeof (m_buffer) < n) { + // buffer needs to be enlarged - reallocate + if (blen != m_blen) { + + // NOTE: the deflate implementation actually looks back past the read pointer + // (in put_byte_dist), so we have to maintain the bytes between m_b_read + // and m_b_insert too. + + char *new_buffer = new char[blen]; + + // place the current block twice at start and end of the block + std::memcpy (new_buffer, m_buffer, m_blen); + std::memcpy (new_buffer + blen - m_blen, m_buffer, m_blen); + + // adjust read pointer if the stored byte array wrapped around the buffer + if (m_b_insert < m_b_read) { + m_b_read += blen - m_blen; + } + + delete[] m_buffer; + m_buffer = new_buffer; + m_blen = blen; + + } + + while ((m_b_insert + m_blen - m_b_read) % m_blen < n) { if (! process ()) { throw tl::Exception (tl::to_string (tr ("Unexpected end of file (DEFLATE implementation)"))); } @@ -257,14 +291,14 @@ InflateFilter::get (size_t n) tl_assert (m_b_read != m_b_insert); // ensure the block is accessible as a coherent chunk: - if (m_b_read + n >= sizeof (m_buffer)) { - std::rotate (m_buffer, m_buffer + m_b_read, m_buffer + sizeof (m_buffer)); - m_b_insert = (m_b_insert - m_b_read + sizeof (m_buffer)) % sizeof (m_buffer); + if (m_b_read + n >= m_blen) { + std::rotate (m_buffer, m_buffer + m_b_read, m_buffer + m_blen); + m_b_insert = (m_b_insert - m_b_read + m_blen) % m_blen; m_b_read = 0; } const char *r = m_buffer + m_b_read; - m_b_read = (m_b_read + n) % sizeof (m_buffer); + m_b_read = (m_b_read + n) % m_blen; return r; } @@ -290,13 +324,16 @@ void InflateFilter::put_byte (char b) { m_buffer [m_b_insert] = b; - m_b_insert = (m_b_insert + 1) % sizeof (m_buffer); + m_b_insert = (m_b_insert + 1) % m_blen; + // buffer overrun + tl_assert (m_b_insert != m_b_read); } void InflateFilter::put_byte_dist (unsigned int d) { - put_byte (m_buffer [(m_b_insert - d) % sizeof (m_buffer)]); + tl_assert (d < m_blen); + put_byte (m_buffer [(m_b_insert + m_blen - d) % m_blen]); } bool diff --git a/src/tl/tl/tlDeflate.h b/src/tl/tl/tlDeflate.h index 712921d62..f54049feb 100644 --- a/src/tl/tl/tlDeflate.h +++ b/src/tl/tl/tlDeflate.h @@ -232,7 +232,8 @@ public: private: BitStream m_input; - char m_buffer[65536]; + char *m_buffer; + size_t m_blen; unsigned int m_b_insert; unsigned int m_b_read; bool m_at_end;