diff --git a/kernel/rtlil_bufnorm.cc b/kernel/rtlil_bufnorm.cc
index d26e0d68a..71e502ed0 100644
--- a/kernel/rtlil_bufnorm.cc
+++ b/kernel/rtlil_bufnorm.cc
@@ -953,7 +953,7 @@ void RTLIL::Cell::unsetPort(RTLIL::IdString portname)
 					if (found->second.empty())
 						fanout.erase(found);
 				}
-			} else {
+			} else if (GetSize(conn_it->second)) {
 				Wire *w = conn_it->second.as_wire();
 				log_assert(w->driverCell_ == this);
 				log_assert(w->driverPort_ == portname);
@@ -1094,7 +1094,7 @@ void RTLIL::Cell::setPort(RTLIL::IdString portname, RTLIL::SigSpec signal)
 			int i = 0;
 			for (auto bit : signal)
 				fanout[bit].insert(PortBit(this, portname, i++));
-		} else {
+		} else if (GetSize(signal)) {
 			Wire *w = signal.as_wire();
 			log_assert(w->driverCell_ == nullptr);
 			log_assert(w->driverPort_.empty());