From 4dbc10352328420fb844895760dc355873da340f Mon Sep 17 00:00:00 2001
From: Matthias Koefferlein <matthias@klayout.de>
Date: Mon, 4 Aug 2025 18:47:39 +0200
Subject: [PATCH] Bugfix: File watcher was dropping files when they were
 temporarily unavailable

---
 src/lay/lay/layMacroEditorDialog.cc | 18 ++++++++++++++++
 src/tl/tl/tlFileSystemWatcher.cc    | 33 ++++++++++++++++++++++++++++-
 src/tl/tl/tlFileSystemWatcher.h     |  1 +
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/src/lay/lay/layMacroEditorDialog.cc b/src/lay/lay/layMacroEditorDialog.cc
index 6f7cdba1a..58b7c8c91 100644
--- a/src/lay/lay/layMacroEditorDialog.cc
+++ b/src/lay/lay/layMacroEditorDialog.cc
@@ -2741,6 +2741,15 @@ MacroEditorDialog::file_changed (const QString &path)
 {
   m_changed_files.push_back (path);
 
+  //  files that have changed are available again
+  auto rw = m_removed_files.begin ();
+  for (auto i = m_removed_files.begin (); i != m_removed_files.end (); ++i) {
+    if (*i != path) {
+      *rw++ = *i;
+    }
+  }
+  m_removed_files.erase (rw, m_removed_files.end ());
+
   //  Wait a little to allow for more reload requests to collect
   m_file_changed_timer->setInterval (300);
   m_file_changed_timer->start ();
@@ -2751,6 +2760,15 @@ MacroEditorDialog::file_removed (const QString &path)
 {
   m_removed_files.push_back (path);
 
+  //  files that are removed are not changed
+  auto rw = m_changed_files.begin ();
+  for (auto i = m_changed_files.begin (); i != m_changed_files.end (); ++i) {
+    if (*i != path) {
+      *rw++ = *i;
+    }
+  }
+  m_changed_files.erase (rw, m_changed_files.end ());
+
   //  Wait a little to let more to allow for more reload requests to collect
   m_file_changed_timer->setInterval (300);
   m_file_changed_timer->start ();
diff --git a/src/tl/tl/tlFileSystemWatcher.cc b/src/tl/tl/tlFileSystemWatcher.cc
index 39d68497f..3c5f45257 100644
--- a/src/tl/tl/tlFileSystemWatcher.cc
+++ b/src/tl/tl/tlFileSystemWatcher.cc
@@ -23,6 +23,7 @@
 #include "tlFileSystemWatcher.h"
 #include "tlString.h"
 #include "tlTimer.h"
+#include "tlLog.h"
 
 #include <QFileInfo>
 #include <QTimer>
@@ -74,6 +75,7 @@ void
 FileSystemWatcher::clear ()
 {
   m_files.clear ();
+  m_files_removed.clear ();
   m_iter = m_files.begin ();
   m_index = 0;
 }
@@ -91,6 +93,8 @@ FileSystemWatcher::add_file (const std::string &path)
     return;
   }
 
+  m_files_removed.erase (path);
+
   size_t size = 0;
   QDateTime time;
 
@@ -109,6 +113,9 @@ FileSystemWatcher::add_file (const std::string &path)
     i->second.time = time;
   } else {
     m_files.insert (std::make_pair (path, FileEntry (1, size, time)));
+    if (tl::verbosity () >= 30) {
+      tl::info << tl::to_string (tr ("Start watching file: ")) << path;
+    }
   }
 
   m_iter = m_files.begin ();
@@ -122,11 +129,16 @@ FileSystemWatcher::remove_file (const std::string &path)
     return;
   }
 
+  m_files_removed.erase (path);
+
   std::map<std::string, FileEntry>::iterator i = m_files.find (path);
   if (i != m_files.end () && --(i->second.refcount) <= 0) {
     m_files.erase (i);
     m_iter = m_files.begin ();
     m_index = 0;
+    if (tl::verbosity () >= 30) {
+      tl::info << tl::to_string (tr ("Stop watching file: ")) << path;
+    }
   }
 }
 
@@ -153,14 +165,26 @@ FileSystemWatcher::timeout ()
     QFileInfo fi (tl::to_qstring (m_iter->first));
     if (! fi.exists ()) {
 
-      files_removed.push_back (m_iter->first);
+      if (m_files_removed.find (m_iter->first) == m_files_removed.end ()) {
+        files_removed.push_back (m_iter->first);
+        m_files_removed.insert (m_iter->first);
+      }
 
+      // don't drop files that temporarily does not exist - keep them monitored.
+      // This way, programs that delete and write the file will not mess with the 
+      // file watcher.
+#if 0 
       std::map<std::string, FileEntry>::iterator i = m_iter;
       ++m_iter;
       m_files.erase (i);
+#else
+      ++m_iter;
+#endif
 
     } else {
 
+      m_files_removed.erase (m_iter->first);
+
       size_t size = size_t (fi.size ());
       QDateTime time = fi.lastModified ();
 
@@ -180,10 +204,17 @@ FileSystemWatcher::timeout ()
   }
 
   for (std::list<std::string>::const_iterator i = files_removed.begin (); i != files_removed.end (); ++i) {
+    if (tl::verbosity () >= 40) {
+      tl::info << tl::to_string (tr ("File removed: ")) << *i;
+    }
     file_removed (*i);
     emit fileRemoved (tl::to_qstring (*i));
   }
+
   for (std::list<std::string>::const_iterator i = files_changed.begin (); i != files_changed.end (); ++i) {
+    if (tl::verbosity () >= 40) {
+      tl::info << tl::to_string (tr ("File changed: ")) << *i;
+    }
     file_changed (*i);
     emit fileChanged (tl::to_qstring (*i));
   }
diff --git a/src/tl/tl/tlFileSystemWatcher.h b/src/tl/tl/tlFileSystemWatcher.h
index 78aba2b14..59c0c0936 100644
--- a/src/tl/tl/tlFileSystemWatcher.h
+++ b/src/tl/tl/tlFileSystemWatcher.h
@@ -136,6 +136,7 @@ private:
   QTimer *m_timer;
   size_t m_batch_size;
   std::map<std::string, FileEntry> m_files;
+  std::set<std::string> m_files_removed;
   size_t m_index;
   std::map<std::string, FileEntry>::iterator m_iter;
 };