Merge pull request #313 from KLayout/query-performance

Query performance

src/db/db/dbLayoutQuery.cc

@@ -167,6 +167,26 @@ public:
     return m_pattern.match (s, mp_eval->match_substrings ());
   }
 
+  bool is_catchall () const
+  {
+    return ! m_needs_eval && m_pattern.is_catchall ();
+  }
+
+  bool is_const () const
+  {
+    return ! m_needs_eval && m_pattern.is_const ();
+  }
+
+  bool needs_eval () const
+  {
+    return m_needs_eval;
+  }
+
+  const std::string &pattern () const
+  {
+    return m_pattern.pattern ();
+  }
+
 private:
   tl::GlobPattern m_pattern;
   tl::Expression m_expression;
@@ -484,11 +504,87 @@ public:
   ChildCellFilterState (const FilterBase *filter, const NameFilterArgument &pattern, ChildCellFilterInstanceMode instance_mode, tl::Eval &eval, db::Layout *layout, bool reading, const ChildCellFilterPropertyIDs &pids)
     : FilterStateBase (filter, layout, eval),
       m_pattern (pattern, eval), m_instance_mode (instance_mode), mp_parent (0), m_pids (pids),
-      m_weight (0), m_references (0), m_weight_set (false), m_references_set (false), m_reading (reading)
+      m_weight (0), m_references (0), m_weight_set (false), m_references_set (false), m_reading (reading),
+      m_cell_index (std::numeric_limits<db::cell_index_type>::max ())
   {
     //  .. nothing yet ..
   }
 
+  bool cell_matches (db::cell_index_type ci)
+  {
+    //  prefilter with the cell objectives
+    if (! objectives ().wants_cell (ci)) {
+      return false;
+    }
+
+    if (m_pattern.is_catchall ()) {
+      return true;
+    } else if (m_cell_index != std::numeric_limits<db::cell_index_type>::max ()) {
+      return ci == db::cell_index_type (m_cell_index);
+    } else if (m_pattern.is_const ()) {
+      if (m_pattern.match (layout ()->cell (ci).get_qualified_name ())) {
+        m_cell_index = ci;
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      return (m_pattern.match (layout ()->cell (ci).get_qualified_name ()));
+    }
+  }
+
+  virtual void do_init ()
+  {
+    if (m_pattern.is_catchall () || m_pattern.needs_eval ()) {
+
+      if (! objectives ().wants_all_cells ()) {
+
+        //  wildcard or unknown filter: include the parent cells if specific child cells are looked for
+
+        int levels = 1;
+        for (size_t i = 0; i < followers ().size (); ++i) {
+          if (followers ()[i] == 0) {
+            //  this is a sign of recursion - collect caller cells from all levels.
+            levels = -1;
+          }
+        }
+
+        //  this means, one follower wants only certain cells. We can optimize by only checking for potential parents
+        std::set<db::cell_index_type> callers;
+        for (FilterStateObjectives::cell_iterator c = objectives ().begin_cells (); c != objectives ().end_cells (); ++c) {
+          layout ()->cell (*c).collect_caller_cells (callers, levels);
+        }
+
+        for (std::set<db::cell_index_type>::const_iterator c = callers.begin (); c != callers.end (); ++c) {
+          objectives ().request_cell (*c);
+        }
+
+      }
+
+    } else if (m_pattern.is_const ()) {
+
+      objectives ().set_wants_all_cells (false);
+
+      //  include the cell with the name we look for into the objectives
+      std::pair<bool, db::cell_index_type> cell_by_name = layout ()->cell_by_name (m_pattern.pattern ().c_str ());
+      if (cell_by_name.first) {
+        objectives ().request_cell (cell_by_name.second);
+      }
+
+    } else {
+
+      objectives ().set_wants_all_cells (false);
+
+      //  include all matching cells into the objectives
+      for (db::Layout::const_iterator c = layout ()->begin (); c != layout ()->end(); ++c) {
+        if (m_pattern.match (layout ()->cell_name (c->cell_index()))) {
+          objectives ().request_cell (c->cell_index ());
+        }
+      }
+
+    }
+  }
+
   virtual void reset (FilterStateBase *previous) 
   {
     FilterStateBase::reset (previous);
@@ -511,7 +607,7 @@ public:
 
       m_top_cell = layout ()->begin_top_down ();
       m_top_cell_end = layout ()->end_top_cells ();
-      while (m_top_cell != m_top_cell_end && (!layout ()->is_valid_cell_index (*m_top_cell) || !m_pattern.match (layout ()->cell (*m_top_cell).get_qualified_name ()))) {
+      while (m_top_cell != m_top_cell_end && (!layout ()->is_valid_cell_index (*m_top_cell) || !cell_matches (*m_top_cell))) {
         ++m_top_cell;
       }
 
@@ -524,7 +620,7 @@ public:
       if (m_instance_mode == NoInstances) {
 
         m_child_cell = mp_parent->begin_child_cells ();
-        while (! m_child_cell.at_end () && (!layout ()->is_valid_cell_index (*m_child_cell) || !m_pattern.match (layout ()->cell (*m_child_cell).get_qualified_name ()))) {
+        while (! m_child_cell.at_end () && (!layout ()->is_valid_cell_index (*m_child_cell) || !cell_matches (*m_child_cell))) {
           ++m_child_cell;
         } 
 
@@ -536,7 +632,7 @@ public:
         while (m_inst != m_inst_end) {
 
           db::cell_index_type cid = (*m_inst)->object ().cell_index ();
-          if (layout ()->is_valid_cell_index (cid) && m_pattern.match (layout ()->cell (cid).get_qualified_name ())) {
+          if (layout ()->is_valid_cell_index (cid) && cell_matches (cid)) {
             break;
           }
 
@@ -576,7 +672,7 @@ public:
 
         do {
           ++m_child_cell;
-        } while (! m_child_cell.at_end () && (!layout ()->is_valid_cell_index (*m_child_cell) || !m_pattern.match (layout ()->cell (*m_child_cell).get_qualified_name ())));
+        } while (! m_child_cell.at_end () && (!layout ()->is_valid_cell_index (*m_child_cell) || !cell_matches (*m_child_cell)));
 
       } else {
 
@@ -600,7 +696,7 @@ public:
               while (m_inst != m_inst_end) {
 
                 cid = (*m_inst)->object ().cell_index ();
-                if (layout ()->is_valid_cell_index (cid) && m_pattern.match (layout ()->cell (cid).get_qualified_name ())) {
+                if (layout ()->is_valid_cell_index (cid) && cell_matches (cid)) {
                   break;
                 }
 
@@ -636,7 +732,7 @@ public:
 
       do {
         ++m_top_cell;
-      } while (m_top_cell != m_top_cell_end && (!layout ()->is_valid_cell_index (*m_top_cell) || !m_pattern.match (layout ()->cell (*m_top_cell).get_qualified_name ())));
+      } while (m_top_cell != m_top_cell_end && (!layout ()->is_valid_cell_index (*m_top_cell) || !cell_matches (*m_top_cell)));
 
     }
   }
@@ -1028,6 +1124,7 @@ private:
   bool m_reading;
   std::set<db::Instance> m_ignored;
   db::Instance m_i;
+  db::cell_index_type m_cell_index;
 };
 
 class DB_PUBLIC ChildCellFilter
@@ -1126,11 +1223,30 @@ public:
       m_pids (pids),
       m_pattern (pattern, eval),
       mp_parent (0),
-      m_reading (reading)
+      m_reading (reading),
+      m_cell_index (std::numeric_limits<db::cell_index_type>::max ())
   {
     //  .. nothing yet ..
   }
 
+  bool cell_matches (db::cell_index_type ci)
+  {
+    if (m_pattern.is_catchall ()) {
+      return true;
+    } else if (m_cell_index != std::numeric_limits<db::cell_index_type>::max ()) {
+      return ci == db::cell_index_type (m_cell_index);
+    } else if (m_pattern.is_const ()) {
+      if (m_pattern.match (layout ()->cell (ci).get_qualified_name ())) {
+        m_cell_index = ci;
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      return (m_pattern.match (layout ()->cell (ci).get_qualified_name ()));
+    }
+  }
+
   virtual void reset (FilterStateBase *previous) 
   {
     FilterStateBase::reset (previous);
@@ -1140,7 +1256,7 @@ public:
     m_cell = layout ()->begin_top_down ();
     m_cell_end = layout ()->end_top_down ();
 
-    while (m_cell != m_cell_end && !m_pattern.match (layout ()->cell (*m_cell).get_qualified_name ())) {
+    while (m_cell != m_cell_end && !cell_matches (*m_cell)) {
       ++m_cell;
     }
 
@@ -1158,7 +1274,7 @@ public:
   {
     do {
       ++m_cell;
-    } while (m_cell != m_cell_end && !m_pattern.match (layout ()->cell (*m_cell).get_qualified_name ()));
+    } while (m_cell != m_cell_end && !cell_matches (*m_cell));
   }
 
   virtual bool at_end () 
@@ -1275,6 +1391,7 @@ private:
   db::Layout::top_down_const_iterator m_cell, m_cell_end;
   std::auto_ptr<db::CellCounter> m_cell_counter;
   bool m_reading;
+  db::cell_index_type m_cell_index;
 };
 
 class DB_PUBLIC CellFilter
@@ -1949,6 +2066,7 @@ LayoutQueryIterator::init ()
 {
   std::vector<FilterStateBase *> f;
   mp_root_state = mp_q->root ().create_state (f, mp_layout, m_eval, false);
+  mp_root_state->init ();
   mp_root_state->reset (0);
   m_state.push_back (mp_root_state);
 
@@ -2816,6 +2934,60 @@ FilterBracket::optimize ()
   // TODO: implement
 }
 
+// --------------------------------------------------------------------------------
+//  FilterStateObjectives implementation
+
+FilterStateObjectives::FilterStateObjectives ()
+  : m_wants_all_cells (false)
+{
+  //  .. nothing yet ..
+}
+
+FilterStateObjectives FilterStateObjectives::everything ()
+{
+  FilterStateObjectives all;
+  all.set_wants_all_cells (true);
+  return all;
+}
+
+FilterStateObjectives &
+FilterStateObjectives::operator+= (const FilterStateObjectives &other)
+{
+  if (! m_wants_all_cells) {
+    m_wants_all_cells = other.m_wants_all_cells;
+    if (! m_wants_all_cells) {
+      m_wants_cells.insert (other.m_wants_cells.begin (), other.m_wants_cells.end ());
+    }
+  }
+
+  if (m_wants_all_cells) {
+    m_wants_cells.clear ();
+  }
+
+  return *this;
+}
+
+void
+FilterStateObjectives::set_wants_all_cells (bool f)
+{
+  m_wants_cells.clear ();
+  m_wants_all_cells = f;
+}
+
+void
+FilterStateObjectives::request_cell (db::cell_index_type ci)
+{
+  if (! m_wants_all_cells) {
+    m_wants_cells.insert (ci);
+  }
+}
+
+bool
+FilterStateObjectives::wants_cell (db::cell_index_type ci) const
+{
+  return m_wants_all_cells || m_wants_cells.find (ci) != m_wants_cells.end ();
+}
+
 // --------------------------------------------------------------------------------
 //  FilterStateBase implementation
 
@@ -2824,6 +2996,35 @@ FilterStateBase::FilterStateBase (const FilterBase *filter, db::Layout *layout,
 {
 }
 
+void
+FilterStateBase::init (bool recursive)
+{
+  if (m_followers.empty ()) {
+
+    m_objectives = FilterStateObjectives::everything ();
+
+  } else {
+
+    for (std::vector<FilterStateBase *>::const_iterator f = m_followers.begin (); f != m_followers.end (); ++f) {
+      if (*f) {
+        if (recursive) {
+          (*f)->init ();
+        }
+        m_objectives += (*f)->objectives ();
+      }
+    }
+
+  }
+
+  do_init ();
+}
+
+void
+FilterStateBase::do_init ()
+{
+  //  .. nothing yet ..
+}
+
 void
 FilterStateBase::dump () const
 {
@@ -2853,6 +3054,7 @@ FilterStateBase::child () const
     if (! b && mp_filter && mp_layout) {
       //  dynamically create a new recursive state execution graph snippet if required
       b = mp_filter->create_state (m_followers, mp_layout, *mp_eval, true);
+      b->init (false);
       m_followers [m_follower] = b;
     }
 

src/db/db/dbLayoutQuery.h

@@ -259,6 +259,44 @@ private:
   FilterStateBase *create_state_helper (std::map<const FilterBase *, FilterStateBase *> &fmap, const FilterBase *child, FilterStateBase *closure_state, db::Layout *layout, tl::Eval &eval) const;
 };
 
+/**
+ *  @brief A structure representing optimization hints for the states
+ */
+struct DB_PUBLIC FilterStateObjectives
+{
+public:
+  typedef std::set<db::cell_index_type>::const_iterator cell_iterator;
+
+  FilterStateObjectives ();
+
+  FilterStateObjectives &operator+= (const FilterStateObjectives &other);
+
+  void set_wants_all_cells (bool f);
+  bool wants_all_cells () const
+  {
+    return m_wants_all_cells;
+  }
+
+  void request_cell (db::cell_index_type ci);
+  bool wants_cell (db::cell_index_type ci) const;
+
+  cell_iterator begin_cells () const
+  {
+    return m_wants_cells.begin ();
+  }
+
+  cell_iterator end_cells () const
+  {
+    return m_wants_cells.end ();
+  }
+
+  static FilterStateObjectives everything ();
+
+private:
+  bool m_wants_all_cells;
+  std::set<db::cell_index_type> m_wants_cells;
+};
+
 /**
  *  @brief A base class for the state objects
  *
@@ -280,6 +318,14 @@ public:
    */
   virtual ~FilterStateBase () { }
 
+  /**
+   *  @brief Initializes the filter state object
+   *
+   *  This method is called after the state graph has been created. It will initialize all followers
+   *  and the call the virtual "do_init" method.
+   */
+  void init (bool recursive = true);
+
   /**
    *  @brief Reset the iterator
    *
@@ -378,6 +424,21 @@ public:
    */
   virtual void dump () const;
 
+protected:
+  /**
+   *  @brief Performs the actual initialization
+   *
+   *  The main intention of this method is to specify optimization hints
+   *  for the objectives. Before this method is called the objectives are
+   *  initialized as the common objectives of all followers.
+   */
+  virtual void do_init ();
+
+  FilterStateObjectives &objectives ()
+  {
+    return m_objectives;
+  }
+
 private:
   friend class LayoutQueryIterator;
 
@@ -389,6 +450,7 @@ private:
   db::Layout *mp_layout;
   size_t m_follower;
   tl::Eval *mp_eval;
+  FilterStateObjectives m_objectives;
 
   void proceed (bool skip);
 };

src/db/unit_tests/dbLayoutQueryTests.cc

@@ -117,7 +117,52 @@ static std::string q2s_cell (db::LayoutQueryIterator &iq, const std::string &pna
   return res;
 }
 
-TEST(1) 
+TEST(0)
+{
+  //  FilterStateObjectives tests
+  db::FilterStateObjectives o1;
+
+  EXPECT_EQ (o1.wants_all_cells (), false);
+  o1.set_wants_all_cells (true);
+  EXPECT_EQ (o1.wants_cell (db::cell_index_type (17)), true);
+  EXPECT_EQ (o1.wants_all_cells (), true);
+
+  o1.set_wants_all_cells (false);
+  o1.request_cell (db::cell_index_type (17));
+  EXPECT_EQ (o1.wants_all_cells (), false);
+  EXPECT_EQ (o1.wants_cell (db::cell_index_type (17)), true);
+  EXPECT_EQ (o1.wants_cell (db::cell_index_type (16)), false);
+
+  db::FilterStateObjectives o2 = o1;
+
+  o1.set_wants_all_cells (false);
+  EXPECT_EQ (o1.wants_cell (db::cell_index_type (17)), false);
+
+  EXPECT_EQ (o2.wants_cell (db::cell_index_type (17)), true);
+
+  db::FilterStateObjectives o3 = o2;
+
+  EXPECT_EQ (o3.wants_cell (db::cell_index_type (17)), true);
+  o3 += db::FilterStateObjectives::everything ();
+  EXPECT_EQ (o3.wants_all_cells (), true);
+
+  o3 = db::FilterStateObjectives::everything ();
+  EXPECT_EQ (o3.wants_all_cells (), true);
+  o3 += o2;
+  EXPECT_EQ (o3.wants_all_cells (), true);
+
+  o3 = db::FilterStateObjectives ();
+  EXPECT_EQ (o3.wants_all_cells (), false);
+  o3.request_cell (db::cell_index_type (16));
+  EXPECT_EQ (o3.wants_cell (db::cell_index_type (17)), false);
+  EXPECT_EQ (o3.wants_cell (db::cell_index_type (16)), true);
+  o3 += o2;
+  EXPECT_EQ (o3.wants_all_cells (), false);
+  EXPECT_EQ (o3.wants_cell (db::cell_index_type (17)), true);
+  EXPECT_EQ (o3.wants_cell (db::cell_index_type (16)), true);
+}
+
+TEST(1)
 {
   db::Layout g;
   g.insert_layer (0);

src/db/unit_tests/unit_tests.pro

@@ -29,7 +29,6 @@ SOURCES = \
   dbLayout.cc \
   dbLayoutDiff.cc \
   dbLayoutUtils.cc \
-  dbLayoutQuery.cc \
   dbLibraries.cc \
   dbMatrix.cc \
   dbObject.cc \
@@ -72,7 +71,8 @@ SOURCES = \
     dbDeepEdgePairsTests.cc \
     dbNetlistCompareTests.cc \
     dbNetlistReaderTests.cc \
-    dbLayoutVsSchematicTests.cc
+    dbLayoutVsSchematicTests.cc \
+    dbLayoutQueryTests.cc
 
 INCLUDEPATH += $$TL_INC $$DB_INC $$GSI_INC
 DEPENDPATH += $$TL_INC $$DB_INC $$GSI_INC

src/tl/tl/tlGlobPattern.cc

@@ -108,6 +108,16 @@ public:
     return op;
   }
 
+  virtual bool is_const () const
+  {
+    return false;
+  }
+
+  virtual bool is_catchall () const
+  {
+    return false;
+  }
+
   virtual bool match (const char *s, std::vector<std::string> *e) const
   {
     size_t n = e ? e->size () : 0;
@@ -186,6 +196,11 @@ public:
     return op;
   }
 
+  virtual bool is_const () const
+  {
+    return next () == 0;
+  }
+
   virtual bool match (const char *s, std::vector<std::string> *e) const
   {
     if (! m_cs) {
@@ -240,6 +255,11 @@ public:
     return op;
   }
 
+  virtual bool is_catchall () const
+  {
+    return true;
+  }
+
   virtual bool match (const char *, std::vector<std::string> *) const
   {
     return true;
@@ -811,6 +831,16 @@ GlobPatternOp *GlobPattern::op () const
   return mp_op;
 }
 
+bool GlobPattern::is_catchall () const
+{
+  return op ()->is_catchall ();
+}
+
+bool GlobPattern::is_const () const
+{
+  return op ()->is_const ();
+}
+
 bool GlobPattern::match (const char *s) const
 {
   return op ()->match (s, 0);

src/tl/tl/tlGlobPattern.h

@@ -106,6 +106,16 @@ public:
     return m_p;
   }
 
+  /**
+   *  @brief Returns true, if the pattern is a catchall expression ("*")
+   */
+  bool is_catchall () const;
+
+  /**
+   *  @brief Returns true, if the pattern is a const string
+   */
+  bool is_const () const;
+
   /**
    *  @brief Match the given string 
    *

src/tl/unit_tests/tlGlobPatternTests.cc

@@ -30,6 +30,15 @@ TEST(1)
   tl::GlobPattern a ("*");
   tl::GlobPattern b ("a");
 
+  EXPECT_EQ (a.is_catchall (), true);
+  EXPECT_EQ (a.is_const (), false);
+  EXPECT_EQ (tl::GlobPattern (a).is_catchall (), true);
+  EXPECT_EQ (tl::GlobPattern (a).is_const (), false);
+  EXPECT_EQ (b.is_catchall (), false);
+  EXPECT_EQ (b.is_const (), true);
+  EXPECT_EQ (tl::GlobPattern (b).is_catchall (), false);
+  EXPECT_EQ (tl::GlobPattern (b).is_const (), true);
+
   EXPECT_EQ (a.match ("abc"), true);
   EXPECT_EQ (a.match ("a"), true);
   EXPECT_EQ (a.match (""), true);
@@ -44,6 +53,13 @@ TEST(2)
   tl::GlobPattern b ("*a?");
   tl::GlobPattern c ("*a\\?");
 
+  EXPECT_EQ (a.is_catchall (), false);
+  EXPECT_EQ (a.is_const (), false);
+  EXPECT_EQ (b.is_catchall (), false);
+  EXPECT_EQ (b.is_const (), false);
+  EXPECT_EQ (c.is_catchall (), false);
+  EXPECT_EQ (c.is_const (), false);
+
   EXPECT_EQ (a.match ("abc"), true);
   EXPECT_EQ (a.match ("a"), true);
   EXPECT_EQ (a.match (""), false);

src/tl/unit_tests/unit_tests.pro

@@ -15,7 +15,6 @@ SOURCES = \
   tlEvents.cc \
   tlExpression.cc \
   tlFileUtils.cc \
-  tlGlobPattern.cc \
   tlIntervalMap.cc \
   tlIntervalSet.cc \
   tlKDTree.cc \
@@ -38,7 +37,8 @@ SOURCES = \
   tlUniqueIdTests.cc \
   tlListTests.cc \
   tlEquivalenceClustersTests.cc \
-    tlUniqueNameTests.cc
+    tlUniqueNameTests.cc \
+    tlGlobPatternTests.cc
 
 !equals(HAVE_QT, "0") {