More progress reporting, performance enhancements

Main performance enhancement: don't update layouts
between make_layer in DeepRegion

src/db/db/dbDeepShapeStore.cc

@@ -552,6 +552,7 @@ DeepLayer DeepShapeStore::create_polygon_layer (const db::RecursiveShapeIterator
   try {
 
     tl::SelfTimer timer (tl::verbosity () >= 41, tl::to_string (tr ("Building working hierarchy")));
+    db::LayoutLocker ll (&layout, true /*no update*/);
 
     builder.set_shape_receiver (&clip);
     db::RecursiveShapeIterator (si).push (& builder);
@@ -590,6 +591,7 @@ DeepLayer DeepShapeStore::create_custom_layer (const db::RecursiveShapeIterator
   try {
 
     tl::SelfTimer timer (tl::verbosity () >= 41, tl::to_string (tr ("Building working hierarchy")));
+    db::LayoutLocker ll (&layout, true /*no update*/);
 
     builder.set_shape_receiver (pipe);
     db::RecursiveShapeIterator (si).push (& builder);
@@ -646,6 +648,7 @@ DeepLayer DeepShapeStore::create_edge_layer (const db::RecursiveShapeIterator &s
   try {
 
     tl::SelfTimer timer (tl::verbosity () >= 41, tl::to_string (tr ("Building working hierarchy")));
+    db::LayoutLocker ll (&layout, true /*no update*/);
 
     builder.set_shape_receiver (&refs);
     db::RecursiveShapeIterator (si).push (& builder);
@@ -676,6 +679,7 @@ DeepLayer DeepShapeStore::create_edge_pair_layer (const db::RecursiveShapeIterat
   try {
 
     tl::SelfTimer timer (tl::verbosity () >= 41, tl::to_string (tr ("Building working hierarchy")));
+    db::LayoutLocker ll (&layout, true /*no update*/);
 
     builder.set_shape_receiver (&refs);
     db::RecursiveShapeIterator (si).push (& builder);

src/db/db/dbHierNetworkProcessor.cc

@@ -1821,7 +1821,7 @@ hier_clusters<T>::build_local_cluster (const db::Layout &layout, const db::Cell
   tl::SelfTimer timer (tl::verbosity () > m_base_verbosity + 20, msg);
 
   connected_clusters<T> &local = m_per_cell_clusters [cell.cell_index ()];
-  local.build_clusters (cell, shape_flags, conn, attr_equivalence, tl::verbosity () >= m_base_verbosity + 30);
+  local.build_clusters (cell, shape_flags, conn, attr_equivalence, true);
 }
 
 template <class T>
@@ -1950,8 +1950,7 @@ hier_clusters<T>::build_hier_connections (cell_clusters_box_converter<T> &cbc, c
     static std::string desc = tl::to_string (tr ("Instance to instance treatment"));
     tl::SelfTimer timer (tl::verbosity () > m_base_verbosity + 30, desc);
 
-    bool report_progress = tl::verbosity () >= m_base_verbosity + 30;
+    db::box_scanner<db::Instance, unsigned int> bs (true, desc);
-    db::box_scanner<db::Instance, unsigned int> bs (report_progress, desc);
 
     for (std::vector<db::Instance>::const_iterator inst = inst_storage.begin (); inst != inst_storage.end (); ++inst) {
       bs.insert (inst.operator-> (), 0);
@@ -1969,8 +1968,7 @@ hier_clusters<T>::build_hier_connections (cell_clusters_box_converter<T> &cbc, c
     static std::string desc = tl::to_string (tr ("Local to instance treatment"));
     tl::SelfTimer timer (tl::verbosity () > m_base_verbosity + 30, desc);
 
-    bool report_progress = tl::verbosity () >= m_base_verbosity + 30;
+    db::box_scanner2<db::local_cluster<T>, unsigned int, db::Instance, unsigned int> bs2 (true, desc);
-    db::box_scanner2<db::local_cluster<T>, unsigned int, db::Instance, unsigned int> bs2 (report_progress, desc);
 
     for (typename connected_clusters<T>::const_iterator c = local.begin (); c != local.end (); ++c) {
 

src/db/db/dbLayout.h

@@ -1562,6 +1562,17 @@ public:
     }
   }
 
+  /**
+   *  @brief Cancel the "in changes" state (see "start_changes")
+   *  This version does not force an update
+   */
+  void end_changes_no_update ()
+  {
+    if (m_invalid > 0) {
+      --m_invalid;
+    }
+  }
+
   /**
    *  @brief Tell if the layout object is under construction
    *
@@ -1763,8 +1774,8 @@ mem_stat (MemStatistics *stat, MemStatistics::purpose_t purpose, int cat, const
 class DB_PUBLIC LayoutLocker
 {
 public:
-  explicit LayoutLocker (db::Layout *layout = 0)
+  explicit LayoutLocker (db::Layout *layout = 0, bool no_update = false)
-    : mp_layout (layout)
+    : mp_layout (layout), m_no_update (no_update)
   {
     if (mp_layout) {
       mp_layout->start_changes ();
@@ -1773,13 +1784,11 @@ public:
 
   ~LayoutLocker ()
   {
-    if (mp_layout) {
+    set (0, false);
-      mp_layout->end_changes ();
-    }
   }
 
   LayoutLocker (const LayoutLocker &other)
-    : mp_layout (other.mp_layout)
+    : mp_layout (other.mp_layout), m_no_update (other.m_no_update)
   {
     if (mp_layout) {
       mp_layout->start_changes ();
@@ -1792,20 +1801,30 @@ public:
       return *this;
     }
 
-    if (mp_layout) {
+    set (other.mp_layout, other.m_no_update);
-      mp_layout->end_changes ();
-    }
-    mp_layout = other.mp_layout;
-    if (mp_layout) {
-      mp_layout->start_changes ();
-    }
-
     return *this;
   }
 
 private:
-
   db::Layout *mp_layout;
+  bool m_no_update;
+
+  void set (db::Layout *layout, bool no_update)
+  {
+    if (mp_layout) {
+      if (m_no_update) {
+        mp_layout->end_changes_no_update ();
+      } else {
+        mp_layout->end_changes ();
+      }
+    }
+    mp_layout = layout;
+    m_no_update = no_update;
+    if (mp_layout) {
+      mp_layout->start_changes ();
+    }
+  }
+
 };
 
 }

src/db/db/dbLayoutToNetlistReader.cc

@@ -60,8 +60,13 @@ typedef l2n_std_format::keys<true> skeys;
 typedef l2n_std_format::keys<false> lkeys;
 
 LayoutToNetlistStandardReader::LayoutToNetlistStandardReader (tl::InputStream &stream)
-  : m_stream (stream), m_path (stream.absolute_path ()), m_dbu (0.0)
+  : m_stream (stream), m_path (stream.absolute_path ()), m_dbu (0.0),
+    m_progress (tl::to_string (tr ("Reading L2N database")), 1000)
 {
+  m_progress.set_format (tl::to_string (tr ("%.0fk lines")));
+  m_progress.set_format_unit (1000.0);
+  m_progress.set_unit (100000.0);
+
   skip ();
 }
 
@@ -122,6 +127,7 @@ LayoutToNetlistStandardReader::skip ()
     if (m_stream.at_end ()) {
       return;
     }
+    m_progress.set (m_stream.line_number ());
     m_line = m_stream.get_line ();
     m_ex = tl::Extractor (m_line.c_str ());
   }
@@ -129,6 +135,8 @@ LayoutToNetlistStandardReader::skip ()
 
 void LayoutToNetlistStandardReader::do_read (db::LayoutToNetlist *l2n)
 {
+  tl::SelfTimer timer (tl::verbosity () >= 21, tl::to_string (QObject::tr ("File read")));
+
   try {
     read_netlist (0, l2n);
   } catch (tl::Exception &ex) {

src/db/db/dbLayoutToNetlistReader.h

@@ -28,6 +28,7 @@
 #include "dbCell.h"
 #include "dbLayoutToNetlist.h"
 #include "tlStream.h"
+#include "tlProgress.h"
 
 namespace db {
 
@@ -149,6 +150,7 @@ private:
   double m_dbu;
   tl::Extractor m_ex;
   db::Point m_ref;
+  tl::AbsoluteProgress m_progress;
 };
 
 }

src/db/db/dbLayoutToNetlistWriter.cc

@@ -56,9 +56,11 @@ static const std::string indent2 ("  ");
 
 template <class Keys>
 std_writer_impl<Keys>::std_writer_impl (tl::OutputStream &stream, double dbu)
-  : mp_stream (&stream), m_dbu (dbu)
+  : mp_stream (&stream), m_dbu (dbu),
+    m_progress (tl::to_string (tr ("Writing L2N database")), 10000)
 {
-  //  .. nothing yet ..
+  m_progress.set_format (tl::to_string (tr ("%.0f MB")));
+  m_progress.set_unit (1024 * 1024);
 }
 
 static std::string name_for_layer (const db::LayoutToNetlist *l2n, unsigned int l)
@@ -115,6 +117,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
         *mp_stream << " " << tl::to_word_or_quoted_string (lp.to_string ());
       }
       *mp_stream << ")" << endl;
+      m_progress.set (mp_stream->pos ());
     }
 
     if (! Keys::is_short ()) {
@@ -130,6 +133,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
           *mp_stream << " " << name_for_layer (l2n, *c);
         }
         *mp_stream << ")" << endl;
+        m_progress.set (mp_stream->pos ());
       }
 
     }
@@ -151,6 +155,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
           *mp_stream << " " << tl::to_word_or_quoted_string (l2n->connectivity ().global_net_name (*g));
         }
         *mp_stream << ")" << endl;
+        m_progress.set (mp_stream->pos ());
       }
 
     }
@@ -163,6 +168,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
       db::DeviceClassTemplateBase *temp = db::DeviceClassTemplateBase::is_a (c.operator-> ());
       if (temp) {
         *mp_stream << indent << Keys::class_key << "(" << tl::to_word_or_quoted_string (c->name ()) << " " << tl::to_word_or_quoted_string (temp->name ()) << ")" << endl;
+        m_progress.set (mp_stream->pos ());
       }
     }
   }
@@ -176,6 +182,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
       *mp_stream << indent << Keys::device_key << "(" << tl::to_word_or_quoted_string (m->name ()) << " " << tl::to_word_or_quoted_string (m->device_class ()->name ()) << endl;
       write (l2n, *m, indent);
       *mp_stream << indent << ")" << endl;
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -188,6 +195,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *nl, const db::LayoutToNetl
     *mp_stream << indent << Keys::circuit_key << "(" << tl::to_word_or_quoted_string (x->name ()) << endl;
     write (nl, l2n, *x, indent, net2id_per_circuit);
     *mp_stream << indent << ")" << endl;
+    m_progress.set (mp_stream->pos ());
   }
 }
 
@@ -215,6 +223,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *netlist, const db::LayoutT
     }
     for (db::Circuit::const_net_iterator n = circuit.begin_nets (); n != circuit.end_nets (); ++n) {
       write (netlist, l2n, *n, (*net2id) [n.operator-> ()], indent);
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -235,6 +244,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *netlist, const db::LayoutT
         *mp_stream << Keys::name_key << "(" << tl::to_word_or_quoted_string (p->name ()) << ")";
       }
       *mp_stream << ")" << endl;
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -244,6 +254,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *netlist, const db::LayoutT
     }
     for (db::Circuit::const_device_iterator d = circuit.begin_devices (); d != circuit.end_devices (); ++d) {
       write (l2n, *d, *net2id, indent);
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -253,6 +264,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *netlist, const db::LayoutT
     }
     for (db::Circuit::const_subcircuit_iterator x = circuit.begin_subcircuits (); x != circuit.end_subcircuits (); ++x) {
       write (l2n, *x, *net2id, indent);
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -381,6 +393,7 @@ void std_writer_impl<Keys>::write (const db::Netlist *netlist, const db::LayoutT
           *mp_stream << indent << indent2;
           write (si.operator-> (), si.trans (), name_for_layer (l2n, *l), true);
           *mp_stream << endl;
+          m_progress.set (mp_stream->pos ());
 
           prev_ci = ci;
 
@@ -441,6 +454,7 @@ void std_writer_impl<Keys>::write (const db::LayoutToNetlist *l2n, const db::Sub
       if (separate_lines) {
         *mp_stream << endl;
       }
+      m_progress.set (mp_stream->pos ());
     }
   }
 
@@ -479,12 +493,14 @@ void std_writer_impl<Keys>::write (const db::LayoutToNetlist *l2n, const db::Dev
         *mp_stream << indent << indent2;
         write (s.operator-> (), db::ICplxTrans (), name_for_layer (l2n, *l), true);
         *mp_stream << endl;
+        m_progress.set (mp_stream->pos ());
 
       }
 
     }
 
     *mp_stream << indent << indent1 << ")" << endl;
+    m_progress.set (mp_stream->pos ());
 
   }
 }

src/db/db/dbLayoutToNetlistWriter.h

@@ -28,6 +28,7 @@
 #include "dbTrans.h"
 #include "dbPolygon.h"
 #include "tlStream.h"
+#include "tlProgress.h"
 
 namespace db
 {
@@ -71,6 +72,7 @@ private:
   tl::OutputStream *mp_stream;
   db::Point m_ref;
   double m_dbu;
+  tl::AbsoluteProgress m_progress;
 };
 
 }

src/laybasic/laybasic/layNetlistBrowserDialog.cc

@@ -404,6 +404,9 @@ BEGIN_PROTECTED
       std::string fn (l2ndb->filename ());
       if (save_dialog.get_save (fn)) {
 
+        tl::log << tl::to_string (QObject::tr ("Saving file: ")) << fn;
+        tl::SelfTimer timer (tl::verbosity () >= 11, tl::to_string (QObject::tr ("Saving")));
+
         l2ndb->save (fn, true);
 
       }
@@ -425,6 +428,9 @@ BEGIN_PROTECTED
     db::LayoutToNetlist *l2ndb = view ()->get_l2ndb (m_l2n_index);
     if (l2ndb && ! l2ndb->filename ().empty ()) {
 
+      tl::log << tl::to_string (QObject::tr ("Loading file: ")) << l2ndb->filename ();
+      tl::SelfTimer timer (tl::verbosity () >= 11, tl::to_string (QObject::tr ("Loading")));
+
       browser_frame->set_l2ndb (0);
       l2ndb->load (l2ndb->filename ());
       browser_frame->set_l2ndb (l2ndb);
@@ -456,6 +462,9 @@ BEGIN_PROTECTED
   lay::FileDialog open_dialog (this, tl::to_string (QObject::tr ("Netlist/LVS Database File")), fmts);
   if (open_dialog.get_open (m_open_filename)) {
 
+    tl::log << tl::to_string (QObject::tr ("Loading file: ")) << m_open_filename;
+    tl::SelfTimer timer (tl::verbosity () >= 11, tl::to_string (QObject::tr ("Loading")));
+
     int l2n_index = view ()->add_l2ndb (db::LayoutToNetlist::create_from_file (m_open_filename));
     l2ndb_cb->setCurrentIndex (l2n_index);
     //  it looks like the setCurrentIndex does not issue this signal:

src/plugins/tools/net_tracer/db_plugin/dbNetTracer.cc

@@ -356,11 +356,14 @@ NetTracerData::configure_l2n (db::LayoutToNetlist &l2n)
 
   std::map <unsigned int, tl::shared_ptr<NetTracerLayerExpression::RegionHolder> > regions_per_org_layer;
 
+  tl::RelativeProgress progress (tl::to_string (tr ("Computing input layers")), m_log_layers.size ());
+
   //  first fetch all the alias expressions
   for (std::map <unsigned int, NetTracerLayerExpression *>::const_iterator l = m_log_layers.begin (); l != m_log_layers.end (); ++l) {
     if (l->second->is_alias ()) {
       tl::shared_ptr<NetTracerLayerExpression::RegionHolder> rh = l->second->make_l2n_region (l2n, regions_per_org_layer, layer_to_symbol [l->first]);
       m_l2n_regions [l->first] = rh;
+      ++progress;
     }
   }
 
@@ -369,6 +372,7 @@ NetTracerData::configure_l2n (db::LayoutToNetlist &l2n)
     if (! l->second->is_alias ()) {
       tl::shared_ptr<NetTracerLayerExpression::RegionHolder> rh = l->second->make_l2n_region (l2n, regions_per_org_layer, layer_to_symbol [l->first]);
       m_l2n_regions [l->first] = rh;
+      ++progress;
     }
   }