diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index 503fb540f..bcde753fc 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -182,6 +182,7 @@ RAW_OBJS = \ V3Descope.o \ V3Dfg.o \ V3DfgAstToDfg.o \ + V3DfgDecomposition.o \ V3DfgDfgToAst.o \ V3DfgOptimizer.o \ V3DfgPasses.o \ diff --git a/src/V3Dfg.cpp b/src/V3Dfg.cpp index 39983cd5c..b3341ff77 100644 --- a/src/V3Dfg.cpp +++ b/src/V3Dfg.cpp @@ -21,9 +21,6 @@ #include "V3File.h" -#include -#include -#include VL_DEFINE_DEBUG_FUNCTIONS; @@ -47,6 +44,7 @@ void DfgGraph::addGraph(DfgGraph& other) { if (DfgVertex* vtxp = src.begin()) { vtxp->m_verticesEnt.moveAppend(src, dst, vtxp); do { + vtxp->m_userCnt = 0; vtxp->m_graphp = this; vtxp = vtxp->verticesNext(); } while (vtxp); @@ -58,430 +56,6 @@ void DfgGraph::addGraph(DfgGraph& other) { moveVertexList(other.m_opVertices, m_opVertices); } -std::vector> DfgGraph::splitIntoComponents(std::string label) { - size_t componentNumber = 0; - std::unordered_map vertex2component; - - forEachVertex([&](const DfgVertex& vtx) { - // If already assigned this vertex to a component, then continue - if (vertex2component.count(&vtx)) return; - - // Work queue for depth first traversal starting from this vertex - std::vector queue{&vtx}; - - // Depth first traversal - while (!queue.empty()) { - // Pop next work item - const DfgVertex& item = *queue.back(); - queue.pop_back(); - - // Mark vertex as belonging to current component (if it's not marked yet) - const bool isFirstEncounter = vertex2component.emplace(&item, componentNumber).second; - - // If we have already visited this vertex during the traversal, then move on. - if (!isFirstEncounter) continue; - - // Enqueue all sources and sinks of this vertex. - item.forEachSource([&](const DfgVertex& src) { queue.push_back(&src); }); - item.forEachSink([&](const DfgVertex& dst) { queue.push_back(&dst); }); - } - - // Done with this component - ++componentNumber; - }); - - // Create the component graphs - std::vector> results{componentNumber}; - - const std::string prefix{name() + (label.empty() ? "" : "-") + label + "-component-"}; - - for (size_t i = 0; i < componentNumber; ++i) { - results[i].reset(new DfgGraph{*m_modulep, prefix + cvtToStr(i)}); - } - - // Move all vertices under the corresponding component graphs - forEachVertex([&](DfgVertex& vtx) { - this->removeVertex(vtx); - results[vertex2component[&vtx]]->addVertex(vtx); - }); - - UASSERT(size() == 0, "'this' DfgGraph should have been emptied"); - - return results; -} - -class ExtractCyclicComponents final { - static constexpr size_t UNASSIGNED = std::numeric_limits::max(); - - // TYPES - struct VertexState { - size_t index; // Used by Pearce's algorithm for detecting SCCs - size_t component = UNASSIGNED; // Result component number (0 stays in input graph) - VertexState(size_t index) - : index{index} {} - }; - - // STATE - - //========================================================================== - // Shared state - - DfgGraph& m_dfg; // The input graph - const std::string m_prefix; // Component name prefix - std::unordered_map m_state; // Vertex state - size_t m_nonTrivialSCCs = 0; // Number of non-trivial SCCs in the graph - const bool m_doExpensiveChecks = v3Global.opt.debugCheck(); - - //========================================================================== - // State for Pearce's algorithm for detecting SCCs - - size_t m_index = 0; // Visitation index counter - std::vector m_stack; // The stack used by the algorithm - - //========================================================================== - // State for merging - - std::unordered_set m_merged; // Marks visited vertices - - //========================================================================== - // State for extraction - - // The extracted cyclic components - std::vector> m_components; - // Map from 'variable vertex' -> 'component index' -> 'clone in that component' - std::unordered_map> m_clones; - - // METHODS - - //========================================================================== - // Methods for Pearce's algorithm to detect strongly connected components - - void visitColorSCCs(DfgVertex& vtx) { - const auto pair = m_state.emplace(std::piecewise_construct, // - std::forward_as_tuple(&vtx), // - std::forward_as_tuple(m_index)); - - // If already visited, then nothing to do - if (!pair.second) return; - - // Visiting node - const size_t rootIndex = m_index++; - - vtx.forEachSink([&](DfgVertex& child) { - // Visit child - visitColorSCCs(child); - auto& childSatate = m_state.at(&child); - // If the child is not in an SCC - if (childSatate.component == UNASSIGNED) { - auto& vtxState = m_state.at(&vtx); - if (vtxState.index > childSatate.index) vtxState.index = childSatate.index; - } - }); - - auto& vtxState = m_state.at(&vtx); - if (vtxState.index == rootIndex) { - // This is the 'root' of an SCC - - // A trivial SCC contains only a single vertex - const bool isTrivial = m_stack.empty() || m_state.at(m_stack.back()).index < rootIndex; - // We also need a separate component for vertices that drive themselves (which can - // happen for input like 'assign a = a'), as we want to extract them (they are cyclic). - const bool drivesSelf = vtx.findSink([&vtx](const DfgVertex& sink) { // - return &vtx == &sink; - }); - - if (!isTrivial || drivesSelf) { - // Allocate new component - ++m_nonTrivialSCCs; - vtxState.component = m_nonTrivialSCCs; - while (!m_stack.empty()) { - DfgVertex* const topp = m_stack.back(); - auto& topState = m_state.at(topp); - // Only higher nodes belong to the same SCC - if (topState.index < rootIndex) break; - m_stack.pop_back(); - topState.component = m_nonTrivialSCCs; - } - } else { - // Trivial SCC (and does not drive itself), so acyclic. Keep it in original graph. - vtxState.component = 0; - } - } else { - // Not the root of an SCC - m_stack.push_back(&vtx); - } - } - - void colorSCCs() { - // Implements Pearce's algorithm to color the strongly connected components. For reference - // see "An Improved Algorithm for Finding the Strongly Connected Components of a Directed - // Graph", David J.Pearce, 2005 - m_state.reserve(m_dfg.size()); - m_dfg.forEachVertex([&](DfgVertex& vtx) { visitColorSCCs(vtx); }); - } - - //========================================================================== - // Methods for merging - - void visitMergeSCCs(const DfgVertex& vtx, size_t targetComponent) { - // Mark visited/move on if already visited - if (!m_merged.insert(&vtx).second) return; - - // Assign vertex to the target component - m_state.at(&vtx).component = targetComponent; - - // Visit all neighbours. We stop at variable boundaries, - // which is where we will split the graphs - vtx.forEachSource([=](const DfgVertex& other) { - if (other.is()) return; - visitMergeSCCs(other, targetComponent); - }); - vtx.forEachSink([=](const DfgVertex& other) { - if (other.is()) return; - visitMergeSCCs(other, targetComponent); - }); - } - - void mergeSCCs() { - // Ensure that component boundaries are always at variables, by merging SCCs - m_merged.reserve(m_dfg.size()); - // Merging stops at variable boundaries, so we don't need to iterate variables. Constants - // are reachable from their sinks, or ar unused, so we don't need to iterate them either. - for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - // Start DFS from each vertex that is in a non-trivial SCC, and merge everything - // that is reachable from it into this component. - if (const size_t target = m_state.at(vtxp).component) visitMergeSCCs(*vtxp, target); - } - } - - //========================================================================== - // Methods for extraction - - // Retrieve clone of vertex in the given component - DfgVertexVar& getClone(DfgVertexVar& vtx, size_t component) { - UASSERT_OBJ(m_state.at(&vtx).component != component, &vtx, "Vertex is in that component"); - DfgVertexVar*& clonep = m_clones[&vtx][component]; - if (!clonep) { - DfgGraph& dfg = component == 0 ? m_dfg : *m_components[component - 1]; - if (DfgVarPacked* const pVtxp = vtx.cast()) { - clonep = new DfgVarPacked{dfg, pVtxp->varp()}; - } else if (DfgVarArray* const aVtxp = vtx.cast()) { - clonep = new DfgVarArray{dfg, aVtxp->varp()}; - } - UASSERT_OBJ(clonep, &vtx, "Unhandled 'DfgVertexVar' sub-type"); - if (VL_UNLIKELY(m_doExpensiveChecks)) { - // Assign component number of clone for later checks - m_state - .emplace(std::piecewise_construct, std::forward_as_tuple(clonep), - std::forward_as_tuple(0)) - .first->second.component - = component; - } - // We need to mark both the original and the clone as having additional references - vtx.setHasModRefs(); - clonep->setHasModRefs(); - } - return *clonep; - } - - // Fix up non-variable sources of a DfgVertexVar that are in a different component, - // using the provided 'relink' callback - template - void fixSources(T_Vertex& vtx, std::function relink) { - static_assert(std::is_base_of::value, - "'Vertex' must be a 'DfgVertexVar'"); - const size_t component = m_state.at(&vtx).component; - vtx.forEachSourceEdge([&](DfgEdge& edge, size_t idx) { - DfgVertex& source = *edge.sourcep(); - // DfgVertexVar sources are fixed up by `fixSinks` on those sources - if (source.is()) return; - const size_t sourceComponent = m_state.at(&source).component; - // Same component is OK - if (sourceComponent == component) return; - // Unlink the source edge (source is reconnected by 'relink' - edge.unlinkSource(); - // Apply the fixup - DfgVertexVar& clone = getClone(vtx, sourceComponent); - relink(*(clone.as()), source, idx); - }); - } - - // Fix up sinks of given variable vertex that are in a different component - void fixSinks(DfgVertexVar& vtx) { - const size_t component = m_state.at(&vtx).component; - vtx.forEachSinkEdge([&](DfgEdge& edge) { - const size_t sinkComponent = m_state.at(edge.sinkp()).component; - // Same component is OK - if (sinkComponent == component) return; - // Relink the sink to read the clone - edge.relinkSource(&getClone(vtx, sinkComponent)); - }); - } - - // Fix edges that cross components - void fixEdges(DfgVertexVar& vtx) { - if (DfgVarPacked* const vvtxp = vtx.cast()) { - fixSources( - *vvtxp, [&](DfgVarPacked& clone, DfgVertex& driver, size_t driverIdx) { - clone.addDriver(vvtxp->driverFileLine(driverIdx), // - vvtxp->driverLsb(driverIdx), &driver); - }); - fixSinks(*vvtxp); - return; - } - - if (DfgVarArray* const vvtxp = vtx.cast()) { - fixSources( // - *vvtxp, [&](DfgVarArray& clone, DfgVertex& driver, size_t driverIdx) { - clone.addDriver(vvtxp->driverFileLine(driverIdx), // - vvtxp->driverIndex(driverIdx), &driver); - }); - fixSinks(*vvtxp); - return; - } - } - - static void packSources(DfgGraph& dfg) { - // Remove undriven variable sources - for (DfgVertexVar *vtxp = dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - if (DfgVarPacked* const varp = vtxp->cast()) { - varp->packSources(); - if (!varp->hasSinks() && varp->arity() == 0) { - VL_DO_DANGLING(varp->unlinkDelete(dfg), varp); - } - return; - } - if (DfgVarArray* const varp = vtxp->cast()) { - varp->packSources(); - if (!varp->hasSinks() && varp->arity() == 0) { - VL_DO_DANGLING(varp->unlinkDelete(dfg), varp); - } - return; - } - } - } - - void checkGraph(DfgGraph& dfg) const { - // Build set of vertices - std::unordered_set vertices{dfg.size()}; - dfg.forEachVertex([&](const DfgVertex& vtx) { vertices.insert(&vtx); }); - - // Check that: - // - Edges only cross components at variable boundaries - // - Each edge connects to a vertex that is within the same graph - // - Variable vertex sources are all connected. - dfg.forEachVertex([&](const DfgVertex& vtx) { - const size_t component = m_state.at(&vtx).component; - vtx.forEachSource([&](const DfgVertex& src) { - if (!src.is()) { // OK to cross at variables - UASSERT_OBJ(component == m_state.at(&src).component, &vtx, - "Edge crossing components without variable involvement"); - } - UASSERT_OBJ(vertices.count(&src), &vtx, "Source vertex not in graph"); - }); - vtx.forEachSink([&](const DfgVertex& snk) { - if (!snk.is()) { // OK to cross at variables - UASSERT_OBJ(component == m_state.at(&snk).component, &vtx, - "Edge crossing components without variable involvement"); - } - UASSERT_OBJ(vertices.count(&snk), &snk, "Sink vertex not in graph"); - }); - if (const DfgVertexVar* const vtxp = vtx.cast()) { - vtxp->forEachSourceEdge([](const DfgEdge& edge, size_t) { - UASSERT_OBJ(edge.sourcep(), edge.sinkp(), "Missing source on variable vertex"); - }); - return; - } - }); - } - - void extractComponents() { - // If the graph was acyclic (which should be the common case), there will be no non-trivial - // SCCs, so we are done. - if (!m_nonTrivialSCCs) return; - - // Allocate result graphs - m_components.resize(m_nonTrivialSCCs); - for (size_t i = 0; i < m_nonTrivialSCCs; ++i) { - m_components[i].reset(new DfgGraph{*m_dfg.modulep(), m_prefix + cvtToStr(i)}); - } - - // Fix up edges crossing components (we can only do this at variable boundaries, and the - // earlier merging of components ensured crossing in fact only happen at variable - // boundaries). Note that fixing up the edges can create clones of variables. Clones are - // added to the correct component, which also means that they might be added to the - // original DFG. Clones do not need fixing up, but also are not necessarily in the m_state - // map (in fact they are only there in debug mode), so we need to check this. - // Also move vertices into their correct component while we are at it. - for (DfgVertexVar *vtxp = m_dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - // It is possible the last vertex (with a nullptr for 'nextp') gets cloned, and hence - // it's 'nextp' would become none nullptr as the clone is added. However, we don't need - // to iterate clones anyway, so it's ok to get the 'nextp' early in the loop. - nextp = vtxp->verticesNext(); - // Clones need not be fixed up - if (!m_state.count(vtxp)) return; - // Fix up the edges crossing components - fixEdges(*vtxp); - // Move the vertex to the component graph (leave component 0, which is the - // originally acyclic sub-graph, in the original graph) - if (const size_t component = m_state.at(vtxp).component) { - m_dfg.removeVertex(*vtxp); - m_components[component - 1]->addVertex(*vtxp); - } - } - - // Move other vertices to their component graphs - for (DfgConst *vtxp = m_dfg.constVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - if (const size_t component = m_state.at(vtxp).component) { - m_dfg.removeVertex(*vtxp); - m_components[component - 1]->addVertex(*vtxp); - } - } - for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { - nextp = vtxp->verticesNext(); - if (const size_t component = m_state.at(vtxp).component) { - m_dfg.removeVertex(*vtxp); - m_components[component - 1]->addVertex(*vtxp); - } - } - - // Pack sources of variables to remove the now undriven inputs - // (cloning might have unlinked some of the inputs), - packSources(m_dfg); - for (const auto& dfgp : m_components) packSources(*dfgp); - - if (VL_UNLIKELY(m_doExpensiveChecks)) { - // Check results for consistency - checkGraph(m_dfg); - for (const auto& dfgp : m_components) checkGraph(*dfgp); - } - } - - // CONSTRUCTOR - entry point - explicit ExtractCyclicComponents(DfgGraph& dfg, std::string label) - : m_dfg{dfg} - , m_prefix{dfg.name() + (label.empty() ? "" : "-") + label + "-component-"} { - // Find all the non-trivial SCCs (and trivial cycles) in the graph - colorSCCs(); - // Ensure that component boundaries are always at variables, by merging SCCs - mergeSCCs(); - // Extract the components - extractComponents(); - } - -public: - static std::vector> apply(DfgGraph& dfg, const std::string& label) { - return std::move(ExtractCyclicComponents{dfg, label}.m_components); - } -}; - -std::vector> DfgGraph::extractCyclicComponents(std::string label) { - return ExtractCyclicComponents::apply(*this, label); -} - static const string toDotId(const DfgVertex& vtx) { return '"' + cvtToHex(&vtx) + '"'; } // Dump one DfgVertex in Graphviz format diff --git a/src/V3Dfg.h b/src/V3Dfg.h index 3720e09e4..a81f0cea5 100644 --- a/src/V3Dfg.h +++ b/src/V3Dfg.h @@ -184,6 +184,7 @@ public: void addGraph(DfgGraph& other); // Split this graph into individual components (unique sub-graphs with no edges between them). + // Also removes any vertices that are not weakly connected to any variable. // Leaves 'this' graph empty. std::vector> splitIntoComponents(std::string label); @@ -340,6 +341,7 @@ public: // The type of this vertex VDfgType type() const { return m_type; } + // Retrieve user data, constructing it fresh on first try. template T& user() { static_assert(sizeof(T) <= sizeof(UserDataStorage), @@ -357,6 +359,22 @@ public: return *storagep; } + // Retrieve user data, must be current. + template + T& getUser() { + static_assert(sizeof(T) <= sizeof(UserDataStorage), + "Size of user data type 'T' is too large for allocated storage"); + static_assert(alignof(T) <= alignof(UserDataStorage), + "Alignment of user data type 'T' is larger than allocated storage"); + T* const storagep = reinterpret_cast(&m_userDataStorage); +#if VL_DEBUG + const uint32_t userCurrent = m_graphp->m_userCurrent; + UASSERT_OBJ(userCurrent, this, "DfgVertex user data used without reserving"); + UASSERT_OBJ(m_userCnt == userCurrent, this, "DfgVertex user data is stale"); +#endif + return *storagep; + } + // Width of result uint32_t width() const { // This is a hot enough function that this is an expensive check, so in debug build only. @@ -412,6 +430,10 @@ public: DfgVertex* verticesNext() const { return m_verticesEnt.nextp(); } DfgVertex* verticesPrev() const { return m_verticesEnt.prevp(); } + // Calls given function 'f' for each source vertex of this vertex + // Unconnected source edges are not iterated. + inline void forEachSource(std::function f); + // Calls given function 'f' for each source vertex of this vertex // Unconnected source edges are not iterated. inline void forEachSource(std::function f) const; @@ -545,6 +567,7 @@ void DfgGraph::addVertex(DfgVertex& vtx) { } else { vtx.m_verticesEnt.pushBack(m_opVertices, &vtx); } + vtx.m_userCnt = 0; vtx.m_graphp = this; } @@ -558,6 +581,7 @@ void DfgGraph::removeVertex(DfgVertex& vtx) { } else { vtx.m_verticesEnt.unlink(m_opVertices, &vtx); } + vtx.m_userCnt = 0; vtx.m_graphp = nullptr; } @@ -588,6 +612,15 @@ void DfgGraph::forEachVertex(std::function f) const { } } +void DfgVertex::forEachSource(std::function f) { + const auto pair = sourceEdges(); + const DfgEdge* const edgesp = pair.first; + const size_t arity = pair.second; + for (size_t i = 0; i < arity; ++i) { + if (DfgVertex* const sourcep = edgesp[i].m_sourcep) f(*sourcep); + } +} + void DfgVertex::forEachSource(std::function f) const { const auto pair = sourceEdges(); const DfgEdge* const edgesp = pair.first; diff --git a/src/V3DfgDecomposition.cpp b/src/V3DfgDecomposition.cpp new file mode 100644 index 000000000..86b6cb46f --- /dev/null +++ b/src/V3DfgDecomposition.cpp @@ -0,0 +1,544 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: DfgGraph decomposition algorithms +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2022 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// Algorithms that take a DfgGraph and decompose it into multiple DfgGraphs. +// +//************************************************************************* + +#include "config_build.h" +#include "verilatedos.h" + +#include "V3Dfg.h" +#include "V3File.h" + +#include +#include +#include + +VL_DEFINE_DEBUG_FUNCTIONS; + +class SplitIntoComponents final { + + // STATE + DfgGraph& m_dfg; // The input graph + const std::string m_prefix; // Component name prefix + std::vector> m_components; // The extracted components + // Component counter - starting from 1 as 0 is the default value used as a marker + size_t m_componentCounter = 1; + + void colorComponents() { + // Work queue for depth first traversal starting from this vertex + std::vector queue; + queue.reserve(m_dfg.size()); + + // any sort of interesting logic must involve a variable, so we only need to iterate them + for (DfgVertexVar *vtxp = m_dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + // If already assigned this vertex to a component, then continue + if (vtxp->user()) continue; + + // Start depth first traversal at this vertex + queue.push_back(vtxp); + + // Depth first traversal + do { + // Pop next work item + DfgVertex& item = *queue.back(); + queue.pop_back(); + + // Move on if already visited + if (item.user()) continue; + + // Assign to current component + item.user() = m_componentCounter; + + // Enqueue all sources and sinks of this vertex. + item.forEachSource([&](DfgVertex& src) { queue.push_back(&src); }); + item.forEachSink([&](DfgVertex& dst) { queue.push_back(&dst); }); + } while (!queue.empty()); + + // Done with this component + ++m_componentCounter; + } + } + + void moveVertices(DfgVertex* headp) { + for (DfgVertex *vtxp = headp, *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + DfgVertex& vtx = *vtxp; + if (const size_t component = vtx.user()) { + m_dfg.removeVertex(vtx); + m_components[component - 1]->addVertex(vtx); + } else { + // This vertex is not connected to a variable and is hence unused, remove here + vtx.unlinkDelete(m_dfg); + } + } + } + + SplitIntoComponents(DfgGraph& dfg, std::string label) + : m_dfg{dfg} + , m_prefix{dfg.name() + (label.empty() ? "" : "-") + label + "-component-"} { + // Component number is stored as DfgVertex::user() + const auto userDataInUse = m_dfg.userDataInUse(); + // Color each component of the graph + colorComponents(); + // Allocate the component graphs + m_components.resize(m_componentCounter - 1); + for (size_t i = 1; i < m_componentCounter; ++i) { + m_components[i - 1].reset(new DfgGraph{*m_dfg.modulep(), m_prefix + cvtToStr(i - 1)}); + } + // Move the vertices to the component graphs + moveVertices(m_dfg.varVerticesBeginp()); + moveVertices(m_dfg.constVerticesBeginp()); + moveVertices(m_dfg.opVerticesBeginp()); + // + UASSERT(m_dfg.size() == 0, "'this' DfgGraph should have been emptied"); + } + +public: + static std::vector> apply(DfgGraph& dfg, const std::string& label) { + return std::move(SplitIntoComponents{dfg, label}.m_components); + } +}; + +std::vector> DfgGraph::splitIntoComponents(std::string label) { + return SplitIntoComponents::apply(*this, label); +} + +class ExtractCyclicComponents final { + static constexpr size_t UNASSIGNED = std::numeric_limits::max(); + + // TYPES + struct VertexState { + size_t index = UNASSIGNED; // Used by Pearce's algorithm for detecting SCCs + size_t component = UNASSIGNED; // Result component number (0 stays in input graph) + bool merged = false; // Visited in the merging pass + VertexState(){}; + }; + + // STATE + + //========================================================================== + // Shared state + + DfgGraph& m_dfg; // The input graph + std::deque m_stateStorage; // Container for VertexState instances + const std::string m_prefix; // Component name prefix + size_t m_nonTrivialSCCs = 0; // Number of non-trivial SCCs in the graph + const bool m_doExpensiveChecks = v3Global.opt.debugCheck(); + + //========================================================================== + // State for Pearce's algorithm for detecting SCCs + + size_t m_index = 0; // Visitation index counter + std::vector m_stack; // The stack used by the algorithm + + //========================================================================== + // State for extraction + + // The extracted cyclic components + std::vector> m_components; + // Map from 'variable vertex' -> 'component index' -> 'clone in that component' + std::unordered_map> m_clones; + + // METHODS + + //========================================================================== + // Shared methods + + VertexState& state(DfgVertex& vtx) const { return *vtx.getUser(); } + + VertexState& allocState(DfgVertex& vtx) { + VertexState*& statep = vtx.user(); + UASSERT_OBJ(!statep, &vtx, "Vertex state already allocated " << cvtToHex(statep)); + m_stateStorage.emplace_back(); + statep = &m_stateStorage.back(); + return *statep; + } + + VertexState& getOrAllocState(DfgVertex& vtx) { + VertexState*& statep = vtx.user(); + if (!statep) { + m_stateStorage.emplace_back(); + statep = &m_stateStorage.back(); + } + return *statep; + } + + //========================================================================== + // Methods for Pearce's algorithm to detect strongly connected components + + void visitColorSCCs(DfgVertex& vtx, VertexState& vtxState) { + UDEBUGONLY(UASSERT_OBJ(vtxState.index == UNASSIGNED, &vtx, "Already visited vertex");); + + // Visiting vertex + const size_t rootIndex = vtxState.index = ++m_index; + + // Visit children + vtx.forEachSink([&](DfgVertex& child) { + VertexState& childSatate = getOrAllocState(child); + // If the child has not yet been visited, then continue traversal + if (childSatate.index == UNASSIGNED) visitColorSCCs(child, childSatate); + // If the child is not in an SCC + if (childSatate.component == UNASSIGNED) { + if (vtxState.index > childSatate.index) vtxState.index = childSatate.index; + } + }); + + if (vtxState.index == rootIndex) { + // This is the 'root' of an SCC + + // A trivial SCC contains only a single vertex + const bool isTrivial = m_stack.empty() || state(*m_stack.back()).index < rootIndex; + // We also need a separate component for vertices that drive themselves (which can + // happen for input like 'assign a = a'), as we want to extract them (they are cyclic). + const bool drivesSelf = vtx.findSink([&vtx](const DfgVertex& sink) { // + return &vtx == &sink; + }); + + if (!isTrivial || drivesSelf) { + // Allocate new component + ++m_nonTrivialSCCs; + vtxState.component = m_nonTrivialSCCs; + while (!m_stack.empty()) { + VertexState& topState = state(*m_stack.back()); + // Only higher nodes belong to the same SCC + if (topState.index < rootIndex) break; + m_stack.pop_back(); + topState.component = m_nonTrivialSCCs; + } + } else { + // Trivial SCC (and does not drive itself), so acyclic. Keep it in original graph. + vtxState.component = 0; + } + } else { + // Not the root of an SCC + m_stack.push_back(&vtx); + } + } + + void colorSCCs() { + // Implements Pearce's algorithm to color the strongly connected components. For reference + // see "An Improved Algorithm for Finding the Strongly Connected Components of a Directed + // Graph", David J.Pearce, 2005. + + // We can leverage some properties of the input graph to gain a bit of speed. Firstly, we + // know constant nodes have no in edges, so they cannot be part of a non-trivial SCC. Mark + // them as such without starting a whole traversal. + for (DfgConst *vtxp = m_dfg.constVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + VertexState& vtxState = allocState(*vtxp); + vtxState.index = 0; + vtxState.component = 0; + } + + // Next, we know that all SCCs must include a variable (as the input graph was converted + // from an AST, we can only have a cycle by going through a variable), so we only start + // traversals through them, and only if we know they have both in and out edges. + for (DfgVertexVar *vtxp = m_dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + if (vtxp->arity() > 0 && vtxp->hasSinks()) { + VertexState& vtxState = getOrAllocState(*vtxp); + // If not yet visited, start a traversal + if (vtxState.index == UNASSIGNED) visitColorSCCs(*vtxp, vtxState); + } else { + VertexState& vtxState = getOrAllocState(*vtxp); + UDEBUGONLY(UASSERT_OBJ(vtxState.index == UNASSIGNED || vtxState.component == 0, + vtxp, "Non circular variable must be in a trivial SCC");); + vtxState.index = 0; + vtxState.component = 0; + } + } + + // Finally, everything we did not visit through the traversal of a variable cannot be in an + // SCC, (otherwise we would have found it from a variable). + for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + VertexState& vtxState = getOrAllocState(*vtxp); + if (vtxState.index == UNASSIGNED) { + vtxState.index = 0; + vtxState.component = 0; + } + } + } + + //========================================================================== + // Methods for merging + + void visitMergeSCCs(DfgVertex& vtx, size_t targetComponent) { + VertexState& vtxState = state(vtx); + + // Move on if already visited + if (vtxState.merged) return; + + // Visiting vertex + vtxState.merged = true; + + // Assign vertex to the target component + vtxState.component = targetComponent; + + // Visit all neighbours. We stop at variable boundaries, + // which is where we will split the graphs + vtx.forEachSource([=](DfgVertex& other) { + if (other.is()) return; + visitMergeSCCs(other, targetComponent); + }); + vtx.forEachSink([=](DfgVertex& other) { + if (other.is()) return; + visitMergeSCCs(other, targetComponent); + }); + } + + void mergeSCCs() { + // Ensure that component boundaries are always at variables, by merging SCCs. Merging stops + // at variable boundaries, so we don't need to iterate variables. Constants are reachable + // from their sinks, or ar unused, so we don't need to iterate them either. + for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + DfgVertex& vtx = *vtxp; + // Start DFS from each vertex that is in a non-trivial SCC, and merge everything + // that is reachable from it into this component. + if (const size_t target = state(vtx).component) visitMergeSCCs(vtx, target); + } + } + + //========================================================================== + // Methods for extraction + + // Retrieve clone of vertex in the given component + DfgVertexVar& getClone(DfgVertexVar& vtx, size_t component) { + UASSERT_OBJ(state(vtx).component != component, &vtx, "Vertex is in that component"); + DfgVertexVar*& clonep = m_clones[&vtx][component]; + if (!clonep) { + if (DfgVarPacked* const pVtxp = vtx.cast()) { + clonep = new DfgVarPacked{m_dfg, pVtxp->varp()}; + } else if (DfgVarArray* const aVtxp = vtx.cast()) { + clonep = new DfgVarArray{m_dfg, aVtxp->varp()}; + } + UASSERT_OBJ(clonep, &vtx, "Unhandled 'DfgVertexVar' sub-type"); + VertexState& cloneStatep = allocState(*clonep); + cloneStatep.component = component; + // We need to mark both the original and the clone as having additional references + vtx.setHasModRefs(); + clonep->setHasModRefs(); + } + return *clonep; + } + + // Fix up non-variable sources of a DfgVertexVar that are in a different component, + // using the provided 'relink' callback + template + void fixSources(T_Vertex& vtx, std::function relink) { + static_assert(std::is_base_of::value, + "'Vertex' must be a 'DfgVertexVar'"); + const size_t component = state(vtx).component; + vtx.forEachSourceEdge([&](DfgEdge& edge, size_t idx) { + DfgVertex& source = *edge.sourcep(); + // DfgVertexVar sources are fixed up by `fixSinks` on those sources + if (source.is()) return; + const size_t sourceComponent = state(source).component; + // Same component is OK + if (sourceComponent == component) return; + // Unlink the source edge (source is reconnected by 'relink' + edge.unlinkSource(); + // Apply the fixup + DfgVertexVar& clone = getClone(vtx, sourceComponent); + relink(*(clone.as()), source, idx); + }); + } + + // Fix up sinks of given variable vertex that are in a different component + void fixSinks(DfgVertexVar& vtx) { + const size_t component = state(vtx).component; + vtx.forEachSinkEdge([&](DfgEdge& edge) { + const size_t sinkComponent = state(*edge.sinkp()).component; + // Same component is OK + if (sinkComponent == component) return; + // Relink the sink to read the clone + edge.relinkSource(&getClone(vtx, sinkComponent)); + }); + } + + // Fix edges that cross components + void fixEdges(DfgVertexVar& vtx) { + if (DfgVarPacked* const vvtxp = vtx.cast()) { + fixSources( + *vvtxp, [&](DfgVarPacked& clone, DfgVertex& driver, size_t driverIdx) { + clone.addDriver(vvtxp->driverFileLine(driverIdx), // + vvtxp->driverLsb(driverIdx), &driver); + }); + fixSinks(*vvtxp); + return; + } + + if (DfgVarArray* const vvtxp = vtx.cast()) { + fixSources( // + *vvtxp, [&](DfgVarArray& clone, DfgVertex& driver, size_t driverIdx) { + clone.addDriver(vvtxp->driverFileLine(driverIdx), // + vvtxp->driverIndex(driverIdx), &driver); + }); + fixSinks(*vvtxp); + return; + } + } + + static void packSources(DfgGraph& dfg) { + // Remove undriven variable sources + for (DfgVertexVar *vtxp = dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + if (DfgVarPacked* const varp = vtxp->cast()) { + varp->packSources(); + if (!varp->hasSinks() && varp->arity() == 0) { + VL_DO_DANGLING(varp->unlinkDelete(dfg), varp); + } + return; + } + if (DfgVarArray* const varp = vtxp->cast()) { + varp->packSources(); + if (!varp->hasSinks() && varp->arity() == 0) { + VL_DO_DANGLING(varp->unlinkDelete(dfg), varp); + } + return; + } + } + } + + void moveVertices(DfgVertex* headp) { + for (DfgVertex *vtxp = headp, *nextp; vtxp; vtxp = nextp) { + nextp = vtxp->verticesNext(); + DfgVertex& vtx = *vtxp; + if (const size_t component = state(vtx).component) { + m_dfg.removeVertex(vtx); + m_components[component - 1]->addVertex(vtx); + } + } + } + + void checkEdges(DfgGraph& dfg) const { + // Check that: + // - Edges only cross components at variable boundaries + // - Variable vertex sources are all connected. + dfg.forEachVertex([&](DfgVertex& vtx) { + const size_t component = state(vtx).component; + vtx.forEachSource([&](DfgVertex& src) { + if (src.is()) return; // OK to cross at variables + UASSERT_OBJ(component == state(src).component, &vtx, + "Edge crossing components without variable involvement"); + }); + vtx.forEachSink([&](DfgVertex& snk) { + if (snk.is()) return; // OK to cross at variables + UASSERT_OBJ(component == state(snk).component, &vtx, + "Edge crossing components without variable involvement"); + }); + if (const DfgVertexVar* const vtxp = vtx.cast()) { + vtxp->forEachSourceEdge([](const DfgEdge& edge, size_t) { + UASSERT_OBJ(edge.sourcep(), edge.sinkp(), "Missing source on variable vertex"); + }); + } + }); + } + + void checkGraph(DfgGraph& dfg) const { + // Build set of vertices + std::unordered_set vertices{dfg.size()}; + dfg.forEachVertex([&](const DfgVertex& vtx) { vertices.insert(&vtx); }); + + // Check that each edge connects to a vertex that is within the same graph + dfg.forEachVertex([&](DfgVertex& vtx) { + vtx.forEachSource([&](DfgVertex& src) { + UASSERT_OBJ(vertices.count(&src), &vtx, "Source vertex not in graph"); + }); + vtx.forEachSink([&](DfgVertex& snk) { + UASSERT_OBJ(vertices.count(&snk), &snk, "Sink vertex not in graph"); + }); + }); + } + + void extractComponents() { + // Allocate result graphs + m_components.resize(m_nonTrivialSCCs); + for (size_t i = 0; i < m_nonTrivialSCCs; ++i) { + m_components[i].reset(new DfgGraph{*m_dfg.modulep(), m_prefix + cvtToStr(i)}); + } + + // Fix up edges crossing components (we can only do this at variable boundaries, and the + // earlier merging of components ensured crossing in fact only happen at variable + // boundaries). Note that fixing up the edges can create clones of variables. Clones do + // not need fixing up, so we do not need to iterate them. + DfgVertex* const lastp = m_dfg.varVerticesRbeginp(); + for (DfgVertexVar *vtxp = m_dfg.varVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { + // It is possible the last vertex (with a nullptr for 'nextp') gets cloned, and hence + // it's 'nextp' would become none nullptr as the clone is added. However, we don't need + // to iterate clones anyway, so it's ok to get the 'nextp' early in the loop. + nextp = vtxp->verticesNext(); + DfgVertexVar& vtx = *vtxp; + // Fix up the edges crossing components + fixEdges(vtx); + // Don't iterate clones added during this loop + if (vtxp == lastp) break; + } + + // Pack sources of variables to remove the now undriven inputs + // (cloning might have unlinked some of the inputs), + packSources(m_dfg); + for (const auto& dfgp : m_components) packSources(*dfgp); + + // Check results for consistency + if (VL_UNLIKELY(m_doExpensiveChecks)) { + checkEdges(m_dfg); + for (const auto& dfgp : m_components) checkEdges(*dfgp); + } + + // Move other vertices to their component graphs + // After this, vertex states are invalid as we moved the vertices + moveVertices(m_dfg.varVerticesBeginp()); + moveVertices(m_dfg.constVerticesBeginp()); + moveVertices(m_dfg.opVerticesBeginp()); + + // Check results for consistency + if (VL_UNLIKELY(m_doExpensiveChecks)) { + checkGraph(m_dfg); + for (const auto& dfgp : m_components) checkGraph(*dfgp); + } + } + + // CONSTRUCTOR - entry point + explicit ExtractCyclicComponents(DfgGraph& dfg, std::string label) + : m_dfg{dfg} + , m_prefix{dfg.name() + (label.empty() ? "" : "-") + label + "-component-"} { + // VertexState is stored as user data + const auto userDataInUse = dfg.userDataInUse(); + // Find all the non-trivial SCCs (and trivial cycles) in the graph + colorSCCs(); + // If the graph was acyclic (which should be the common case), + // there will be no non-trivial SCCs, so we are done. + if (!m_nonTrivialSCCs) return; + // Ensure that component boundaries are always at variables, by merging SCCs + mergeSCCs(); + // Extract the components + extractComponents(); + } + +public: + static std::vector> apply(DfgGraph& dfg, const std::string& label) { + return std::move(ExtractCyclicComponents{dfg, label}.m_components); + } +}; + +std::vector> DfgGraph::extractCyclicComponents(std::string label) { + return ExtractCyclicComponents::apply(*this, label); +}