Merge pull request #2321 from KLayout/bugfix/issue-2320

Fixing issue #2320

src/plugins/streamers/oasis/unit_tests/dbOASISReaderTests.cc

@@ -25,6 +25,7 @@
 #include "dbOASISWriter.h"
 #include "dbTextWriter.h"
 #include "dbTestSupport.h"
+#include "dbLayoutDiff.h"
 #include "tlLog.h"
 #include "tlUnitTest.h"
 #include "tlStream.h"
@@ -696,3 +697,49 @@ TEST(BlendCrash)
   std::string fn_au (tl::testdata () + "/oasis/blend_crash_au.gds.gz");
   db::compare_layouts (_this, layout, fn_au, db::WriteGDS2, 1);
 }
+
+TEST(CBlockLargePropertyString)
+{
+  //  NOTE: we try different blob sizes for triggering various reallocation szenarios
+  for (size_t f = 0; f < 50; ++f) {
+
+    size_t blob_size = size_t (12345) * f;
+
+    tl::info << "Trying blob size of " << blob_size;
+
+    db::Layout layout_org (false);
+
+    unsigned int layer = layout_org.insert_layer (db::LayerProperties (1, 0));
+    db::Cell &top = layout_org.cell (layout_org.add_cell ("TOP"));
+
+    std::string large_value (blob_size, 'a');
+
+    db::PropertiesSet ps;
+    ps.insert (db::property_names_id (tl::Variant ("blob")), tl::Variant (large_value));
+
+    top.shapes (layer).insert (db::BoxWithProperties (db::Box (0, 0, 100, 100), db::properties_id (ps)));
+
+    std::string tmp_file = tl::TestBase::tmp_file ("tmp_OASISReaderLargeString.oas");
+
+    {
+      tl::OutputStream out (tmp_file);
+      db::SaveLayoutOptions options;
+      db::OASISWriterOptions &oasis_options = options.get_options<db::OASISWriterOptions> ();
+      oasis_options.write_cblocks = true;
+      oasis_options.strict_mode = false;
+      db::OASISWriter writer;
+      writer.write (layout_org, out, options);
+    }
+
+    db::Layout layout_read;
+
+    {
+      tl::InputStream in (tmp_file);
+      db::OASISReader reader (in);
+      reader.read (layout_read);
+    }
+
+    EXPECT_EQ (db::compare_layouts (layout_org, layout_read, db::layout_diff::f_verbose, 0), true);
+
+  }
+}

src/tl/tl/tlDeflate.cc

@@ -26,6 +26,7 @@
 #include "tlAssert.h"
 
 #include <algorithm>
+#include <cstring>
 
 #include <zlib.h>
 
@@ -227,9 +228,12 @@ InflateFilter::InflateFilter (tl::InputStream &input)
     m_last_block (false), 
     m_uncompressed_length (0)  //  this forces a new block on "process()"
 {
-  for (size_t i = 0; i < sizeof (m_buffer) / sizeof (m_buffer [0]); ++i) {
-    m_buffer[i] = 0;
-  }
+  //  NOTE: the minimum buffer size of 65536 corresponds to the maximum block size
+  //  of the block repetition decoder
+  m_blen = 65536; //  initially
+
+  m_buffer = new char [m_blen];
+  std::memset (m_buffer, 0, m_blen);
 
   mp_dist_decoder = new HuffmannDecoder ();
   mp_lit_decoder = new HuffmannDecoder ();
@@ -237,6 +241,9 @@ InflateFilter::InflateFilter (tl::InputStream &input)
 
 InflateFilter::~InflateFilter ()
 {
+  delete[] m_buffer;
+  m_buffer = 0;
+  m_blen = 0;
   delete mp_dist_decoder;
   mp_dist_decoder = 0;
   delete mp_lit_decoder;
@@ -246,9 +253,36 @@ InflateFilter::~InflateFilter ()
 const char * 
 InflateFilter::get (size_t n)
 {
-  tl_assert (n < sizeof (m_buffer) / 2);
+  size_t blen = m_blen;
+  while (n >= blen / 2)  {
+    blen *= 2;
+  }
 
-  while ((m_b_insert + sizeof (m_buffer) - m_b_read) % sizeof (m_buffer) < n) {
+  //  buffer needs to be enlarged - reallocate
+  if (blen != m_blen) {
+
+    //  NOTE: the deflate implementation actually looks back past the read pointer
+    //  (in put_byte_dist), so we have to maintain the bytes between m_b_read
+    //  and m_b_insert too.
+
+    char *new_buffer = new char[blen];
+
+    //  place the current block twice at start and end of the block
+    std::memcpy (new_buffer, m_buffer, m_blen);
+    std::memcpy (new_buffer + blen - m_blen, m_buffer, m_blen);
+
+    //  adjust read pointer if the stored byte array wrapped around the buffer
+    if (m_b_insert < m_b_read) {
+      m_b_read += blen - m_blen;
+    }
+
+    delete[] m_buffer;
+    m_buffer = new_buffer;
+    m_blen = blen;
+
+  }
+
+  while ((m_b_insert + m_blen - m_b_read) % m_blen < n) {
     if (! process ()) {
       throw tl::Exception (tl::to_string (tr ("Unexpected end of file (DEFLATE implementation)")));
     }
@@ -257,14 +291,14 @@ InflateFilter::get (size_t n)
   tl_assert (m_b_read != m_b_insert);
 
   //  ensure the block is accessible as a coherent chunk:
-  if (m_b_read + n >= sizeof (m_buffer)) {
-    std::rotate (m_buffer, m_buffer + m_b_read, m_buffer + sizeof (m_buffer));
-    m_b_insert = (m_b_insert - m_b_read + sizeof (m_buffer)) % sizeof (m_buffer);
+  if (m_b_read + n >= m_blen) {
+    std::rotate (m_buffer, m_buffer + m_b_read, m_buffer + m_blen);
+    m_b_insert = (m_b_insert - m_b_read + m_blen) % m_blen;
     m_b_read = 0;
   }
 
   const char *r = m_buffer + m_b_read;
-  m_b_read = (m_b_read + n) % sizeof (m_buffer);
+  m_b_read = (m_b_read + n) % m_blen;
   return r;
 }
 
@@ -290,13 +324,16 @@ void
 InflateFilter::put_byte (char b) 
 {
   m_buffer [m_b_insert] = b;
-  m_b_insert = (m_b_insert + 1) % sizeof (m_buffer);
+  m_b_insert = (m_b_insert + 1) % m_blen;
+  //  buffer overrun
+  tl_assert (m_b_insert != m_b_read);
 }
 
 void 
 InflateFilter::put_byte_dist (unsigned int d) 
 {
-  put_byte (m_buffer [(m_b_insert - d) % sizeof (m_buffer)]);
+  tl_assert (d < m_blen);
+  put_byte (m_buffer [(m_b_insert + m_blen - d) % m_blen]);
 }
 
 bool 

src/tl/tl/tlDeflate.h

@@ -232,7 +232,8 @@ public:
 private:
   BitStream m_input;
 
-  char m_buffer[65536];
+  char *m_buffer;
+  size_t m_blen;
   unsigned int m_b_insert;
   unsigned int m_b_read;
   bool m_at_end;