// -*- mode: C++; c-file-style: "cc-mode" -*-
//*************************************************************************
// DESCRIPTION: Verilator: Threading's logic to mtask partitioner
//
// Code available from: http://www.veripool.org/verilator
//
//*************************************************************************
//
// Copyright 2003-2018 by Wilson Snyder.  This program is free software; you can
// redistribute it and/or modify it under the terms of either the GNU
// Lesser General Public License Version 3 or the Perl Artistic License
// Version 2.0.
//
// Verilator is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
//*************************************************************************

#include "config_build.h"
#include "verilatedos.h"
#include <list>
#include <memory>
#include <cstring>

#include "V3Os.h"
#include "V3File.h"
#include "V3GraphAlg.h"
#include "V3GraphPathChecker.h"
#include "V3GraphStream.h"
#include "V3InstrCount.h"
#include "V3Partition.h"
#include "V3PartitionGraph.h"
#include "V3Scoreboard.h"
#include "V3Stats.h"
#include VL_INCLUDE_UNORDERED_SET

class MergeCandidate;

//######################################################################
// Partitioner tunable settings:
//
// Before describing these settings, a bit of background:
//
// Early during the development of the partitioner, V3Split was failing to
// split large always blocks (with ~100K assignments) so we had to handle
// very large vertices with ~100K incoming and outgoing edges.
//
// The partitioner attempts to deal with such densely connected
// graphs. Some of the tuning parameters below reference "huge vertices",
// that's what they're talking about, vertices with tens of thousands of
// edges in and out. Whereas most graphs have only tens of edges in and out
// of most vertices.
//
// V3Split has since been fixed to more reliably split large always
// blocks. It's kind of an open question whether the partitioner must
// handle huge nodes gracefully. Maybe not!  But it still can, given
// appropriate tuning.


//   PART_SIBLING_EDGE_LIMIT (integer)
//
// Arbitrarily limit the number of edges on a single vertex that will be
// considered when enumerating siblings, to the given value.  This protects
// the partitioner runtime in the presence of huge vertices.
//
// The sibling-merge is less important than the edge merge.  (You can
// totally disable the sibling merge and get halfway decent partitions; you
// can't disable edge merges, those are fundamental to the process.) So,
// skipping the enumeration of some siblings on a few vertices does not
// have a large impact on the result of the partitioner.
//
// If your vertices are small, the limit (at 25) approaches a no-op.  Hence
// there's basically no cost to applying this limit even when we don't
// expect huge vertices.
//
// If you don't care about partitioner runtime and you want the most
// aggressive partition, set the limit very high.  If you have huge
// vertices, leave this as is.
#define PART_SIBLING_EDGE_LIMIT 25


//   PART_STEPPED_COST (boolean)
//
// When computing critical path costs, use a step function on the actual
// underlying vertex cost.
//
// If there are huge vertices, when a tiny vertex merges into a huge
// vertex, we can often avoid increasing the huge vertex's stepped cost.
// If the stepped cost hasn't increased, and the critical path into the huge
// vertex hasn't increased, we can avoid propagating a new critical path to
// vertices past the huge vertex. Since huge vertices tend to have huge lists
// of children and parents, this can be a substantial savings.
//
// Does not seem to reduce the quality of the partitioner's output.
//
// If you have huge vertices, leave this 'true', it is the major setting
// that allows the partitioner to handle such difficult graphs on anything
// like a human time scale.
//
// If you don't have huge vertices, the 'true' value doesn't help much but
// should cost almost nothing in terms of partitioner quality.
//
// If you want the most aggressive possible partition, set it "false" and
// be prepared to be dissappointed when the improvement in the partition is
// negligible / in the noise.
//
// Q) Why retain the control, if there is really no downside?
//
// A) Cost stepping can lead to corner cases. A developer may wish to
//    disable cost stepping to rule it out as the cause of unexpected
//    behavior.
#define PART_STEPPED_COST true


//   PART_STEPPED_RESCORE_LIMIT (boolean)
//
// If false, we always try to merge the absolute lowest (best) scoring
// mtask pair among all candidates.
//
// If true, we're willing to merge mtask pairs with scores up to 5% higher
// (worse) than the best, in exchange for doing a Rescore() operation
// somewhat less often.
//
// A true setting can result in a much faster compile in the presence of
// huge vertices, eg. 45 minutes versus 4.5 minutes for one particular
// model. HOWEVER, a true setting usually results in modestly worse
// partitions, often around 10% more MTasks and 10% longer cycle times.
//
// (TODO: Why does this setting save time with huge vertices?
// Is there a way to get best of both worlds without the trade off?)
//
// If you have huge vertices, you may wish to set this true.  If you don't
// have huge vertices (which should be everyone, we think, now that V3Split
// is fixed) leave it set false for the most aggressive partition.
#define PART_STEPPED_RESCORE_LIMIT false


// Don't produce more than a certain maximum number of MTasks.  This helps
// the TSP variable sort not to blow up (a concern for some of the tests)
// and we probably don't want a huge number of mtasks in practice anyway
// (50 to 100 is typical.)
//
// If the user doesn't give one with '--threads-max-mtasks', we'll set the
// maximum # of MTasks to
//  (# of threads * PART_DEFAULT_MAX_MTASKS_PER_THREAD)
#define PART_DEFAULT_MAX_MTASKS_PER_THREAD 50

//   end tunables.

//######################################################################
// Misc graph and assertion utilities

static void partCheckCachedScoreVsActual(uint32_t cached, uint32_t actual) {
#if PART_STEPPED_COST
    // Cached CP might be a little bigger than actual, due to stepped CPs.
    // Example:
    // Let's say we have a parent with stepped_cost 40 and a grandparent
    // with stepped_cost 27. Our forward-cp is 67. Then our parent and
    // grandparent get merged, the merged node has stepped cost 66.  We
    // won't propagate that new CP to children as it hasn't grown.  So,
    // children may continue to think that the CP coming through this path
    // is a little higher than it really is; permit that.
    UASSERT((((cached * 10) <= (actual * 11))
             && (cached * 11) >= (actual * 10)),
            "Calculation error in scoring (approximate, may need tweak)");
#else
    UASSERT(cached == actual, "Calculation error in scoring");
#endif
}

//######################################################################
// PartPropagateCp

// Propagate increasing critical path (CP) costs through a graph.
//
// Usage:
//  * Client increases the cost and/or CP at a node or small set of nodes
//    (often a pair in practice, eg. edge contraction.)
//  * Client instances a PartPropagateCp object
//  * Client calls PartPropagateCp::cpHasIncreased() one or more times.
//    Each call indicates that the inclusive CP of some "seed" vertex
//    has increased to a given value.
//    * NOTE: PartPropagateCp will neither read nor modify the cost
//      or CPs at the seed vertices, it only accesses and modifies
//      vertices wayward from the seeds.
//  * Client calls PartPropagateCp::go(). Internally, this iteratively
//    propagates the new CPs wayward through the graph.
//
template <class T_CostAccessor> class PartPropagateCp : GraphAlg<> {
private:
    // MEMBERS
    GraphWay m_way;  // CPs oriented in this direction: either FORWARD
    //               // from graph-start to current node, or REVERSE
    //               // from graph-end to current node.
    T_CostAccessor* m_accessp;  // Access cost and CPs on V3GraphVertex's.
    vluint64_t m_generation;  // Mark each vertex with this number;
    //                        // confirm we only process each vertex once.
    bool m_slowAsserts;  // Enable nontrivial asserts
    typedef SortByValueMap<V3GraphVertex*, uint32_t> PropCpPendSet;
    PropCpPendSet m_pending;  // Pending rescores

public:
    // CONSTRUCTORS
    PartPropagateCp(V3Graph* graphp, GraphWay way, T_CostAccessor* accessp,
                    bool slowAsserts,
                    V3EdgeFuncP edgeFuncp = &V3GraphEdge::followAlwaysTrue)
        : GraphAlg<>(graphp, edgeFuncp)
        , m_way(way)
        , m_accessp(accessp)
        , m_generation(0)
        , m_slowAsserts(slowAsserts) {}

    // METHODS
    void cpHasIncreased(V3GraphVertex* vxp, uint32_t newInclusiveCp) {
        // For *vxp, whose CP-inclusive has just increased to
        // newInclusiveCp, iterate to all wayward nodes, update the edges
        // of each, and add each to m_pending if its overall CP has grown.
        for (V3GraphEdge* edgep = vxp->beginp(m_way);
             edgep; edgep = edgep->nextp(m_way)) {
            if (!m_edgeFuncp(edgep)) continue;
            V3GraphVertex* relativep = edgep->furtherp(m_way);
            m_accessp->notifyEdgeCp(relativep, m_way, vxp, newInclusiveCp);

            if (m_accessp->critPathCost(relativep, m_way) < newInclusiveCp) {
                // relativep's critPathCost() is out of step with its
                // longest !wayward edge. Schedule that to be resolved.
                uint32_t newPendingVal =
                    newInclusiveCp - m_accessp->critPathCost(relativep, m_way);
                if (m_pending.has(relativep)) {
                    if (newPendingVal > m_pending.at(relativep)) {
                        m_pending.set(relativep, newPendingVal);
                    }
                } else {
                    m_pending.set(relativep, newPendingVal);
                }
            }
        }
    }

    void go() {
        // m_pending maps each pending vertex to the amount that it wayward
        // CP will grow.
        //
        // We can iterate over the pending set in reverse order, always
        // choosing the nodes with the largest pending CP-growth.
        //
        // The intuition is: if the original seed node had its CP grow by
        // 50, the most any wayward node can possibly grow is also 50.  So
        // for anything pending to grow by 50, we know we can process it
        // once and we won't have to grow its CP again on the current pass.
        // After we're done with all the grow-by-50s, nothing else will
        // grow by 50 again on the current pass, and we can process the
        // grow-by-49s and we know we'll only have to process each one
        // once.  And so on.
        //
        // This generalizes to multiple seed nodes also.
        while (!m_pending.empty()) {
            PropCpPendSet::reverse_iterator it = m_pending.rbegin();
            V3GraphVertex* updateMep = (*it).key();
            uint32_t cpGrowBy = (*it).value();
            m_pending.erase(it);

            // For *updateMep, whose critPathCost was out-of-date with respect
            // to its edges, update the critPathCost.
            uint32_t startCp = m_accessp->critPathCost(updateMep, m_way);
            uint32_t newCp = startCp + cpGrowBy;
            if (m_slowAsserts) {
                m_accessp->checkNewCpVersusEdges(updateMep, m_way, newCp);
            }

            m_accessp->setCritPathCost(updateMep, m_way, newCp);
            cpHasIncreased(updateMep, newCp + m_accessp->cost(updateMep));
        }
    }

private:
    VL_DEBUG_FUNC;
    VL_UNCOPYABLE(PartPropagateCp);
};

class PartPropagateCpSelfTest {
private:
    // MEMBERS
    V3Graph m_graph;  // A graph
    V3GraphVertex* m_vx[50];  // All vertices within the graph
    typedef vl_unordered_map<V3GraphVertex*, uint32_t> CpMap;
    CpMap m_cp;  // Vertex-to-CP map
    CpMap m_seen;  // Set of vertices we've seen

    // CONSTRUCTORS
    PartPropagateCpSelfTest() {}
    ~PartPropagateCpSelfTest() {}

    // METHODS
protected:
    friend class PartPropagateCp<PartPropagateCpSelfTest>;
    void notifyEdgeCp(V3GraphVertex* vxp, GraphWay way,
                      V3GraphVertex* throughp, uint32_t cp) const {
        uint32_t throughCost = critPathCost(throughp, way);
        UASSERT_SELFTEST(uint32_t, cp, (1 + throughCost));
    }
private:
    void checkNewCpVersusEdges(V3GraphVertex* vxp,
                               GraphWay way, uint32_t cp) const {
        // Don't need to check this in the self test; it supports an assert
        // that runs in production code.
    }
    void setCritPathCost(V3GraphVertex* vxp,
                         GraphWay way, uint32_t cost) {
        m_cp[vxp] = cost;
        // Confirm that we only set each node's CP once.  That's an
        // important property of PartPropagateCp which allows it to be far
        // faster than a recursive algorithm on some graphs.
        CpMap::iterator it = m_seen.find(vxp);
        if (it != m_seen.end()) vxp->v3fatalSrc("Set CP on node twice");
        m_seen[vxp] = cost;
    }
    uint32_t critPathCost(V3GraphVertex* vxp, GraphWay way) const {
        CpMap::const_iterator it = m_cp.find(vxp);
        if (it != m_cp.end()) return it->second;
        return 0;
    }
    uint32_t cost(const V3GraphVertex*) const { return 1; }
    void partInitCriticalPaths(bool checkOnly) {
        // Set up the FORWARD cp's only.  This test only looks in one
        // direction, it assumes REVERSE is symmetrical and would be
        // redundant to test.
        GraphStreamUnordered order(&m_graph);
        while (const V3GraphVertex* cvxp = order.nextp()) {
            V3GraphVertex* vxp = const_cast<V3GraphVertex*>(cvxp);
            uint32_t cpCost = 0;
            for (V3GraphEdge* edgep = vxp->inBeginp();
                 edgep; edgep = edgep->inNextp()) {
                V3GraphVertex* parentp = edgep->fromp();
                cpCost = std::max(cpCost,
                                  critPathCost(parentp, GraphWay::FORWARD) + 1);
            }
            if (checkOnly) {
                UASSERT_SELFTEST(uint32_t, cpCost,
                                 critPathCost(vxp, GraphWay::FORWARD));
            } else {
                setCritPathCost(vxp, GraphWay::FORWARD, cpCost);
            }
        }
    }
    void go() {
        // Generate a pseudo-random graph
        uint16_t rngState[3] = { 0xdead, 0xbeef, 0xf000 };
        // Create 50 vertices
        for (unsigned i = 0; i < 50; ++i) {
            m_vx[i] = new V3GraphVertex(&m_graph);
        }
        // Create 250 edges at random. Edges must go from
        // lower-to-higher index vertices, so we get a DAG.
        for (unsigned i = 0; i < 250; ++i) {
            unsigned idx1 = nrand48(rngState) % 50;
            unsigned idx2 = nrand48(rngState) % 50;
            if (idx1 > idx2) {
                new V3GraphEdge(&m_graph, m_vx[idx2], m_vx[idx1], 1);
            } else if (idx2 > idx1) {
                new V3GraphEdge(&m_graph, m_vx[idx1], m_vx[idx2], 1);
            }
        }

        partInitCriticalPaths(false);

        // This SelfTest class is also the T_CostAccessor
        PartPropagateCp<PartPropagateCpSelfTest>
            prop(&m_graph, GraphWay::FORWARD, this, true);

        // Seed the propagator with every input node;
        // This should result in the complete graph getting all CP's assigned.
        for (unsigned i = 0; i < 50; ++i) {
            if (!m_vx[i]->inBeginp()) {
                prop.cpHasIncreased(m_vx[i], 1 /* inclusive CP starts at 1 */);
            }
        }

        // Run the propagator.
        //  * The setCritPathCost() routine checks that each node's CP changes
        //    at most once.
        //  * The notifyEdgeCp routine is also self checking.
        m_seen.clear();
        prop.go();

        // Finally, confirm that the entire graph appears to have correct CPs.
        partInitCriticalPaths(true);
    }
public:
    static void selfTest() {
        PartPropagateCpSelfTest().go();
    }
};

//######################################################################
// LogicMTask

class LogicMTask : public AbstractLogicMTask {
public:
    // TYPES
    typedef std::list<MTaskMoveVertex*> VxList;

    struct CmpLogicMTask {
        bool operator() (const LogicMTask* ap, const LogicMTask* bp) const {
            return ap->id() < bp->id();
        }
    };

    // This adaptor class allows the PartPropagateCp class to be somewhat
    // independent of the LogicMTask class
    //  - PartPropagateCp can thus be declared before LogicMTask
    //  - PartPropagateCp could be reused with graphs of other node types
    //    in the future, using another Accessor adaptor.
    class CpCostAccessor {
    public:
        CpCostAccessor() {}
        ~CpCostAccessor() {}
        // Return cost of this node
        uint32_t cost(const V3GraphVertex* vxp) const {
            const LogicMTask* mtaskp = dynamic_cast<const LogicMTask*>(vxp);
            return mtaskp->stepCost();
        }
        // Return stored CP to this node
        uint32_t critPathCost(const V3GraphVertex* vxp, GraphWay way) const {
            const LogicMTask* mtaskp = dynamic_cast<const LogicMTask*>(vxp);
            return mtaskp->critPathCost(way);
        }
        // Store a new CP to this node
        void setCritPathCost(V3GraphVertex* vxp,
                             GraphWay way, uint32_t cost) const {
            LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(vxp);
            mtaskp->setCritPathCost(way, cost);
        }
        // Notify vxp that the wayward CP at the throughp-->vxp edge
        // has increased to 'cp'. (vxp is wayward from throughp.)
        // This is our cue to update vxp's m_edges[!way][throughp].
        void notifyEdgeCp(V3GraphVertex* vxp, GraphWay way,
                          V3GraphVertex* throuvhVxp, uint32_t cp) const {
            LogicMTask* updateVxp = dynamic_cast<LogicMTask*>(vxp);
            LogicMTask* lthrouvhVxp = dynamic_cast<LogicMTask*>(throuvhVxp);
            EdgeSet& edges = updateVxp->m_edges[way.invert()];
            uint32_t edgeCp = edges.at(lthrouvhVxp);
            if (cp > edgeCp) edges.set(lthrouvhVxp, cp);
        }
        // Check that CP matches that of the longest edge wayward of vxp.
        void checkNewCpVersusEdges(V3GraphVertex* vxp,
                                   GraphWay way, uint32_t cp) const {
            LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(vxp);
            EdgeSet& edges = mtaskp->m_edges[way.invert()];
            // This is mtaskp's relative with longest !wayward inclusive CP:
            EdgeSet::reverse_iterator edgeIt = edges.rbegin();
            uint32_t edgeCp = (*edgeIt).value();
            if (edgeCp != cp) vxp->v3fatalSrc("CP doesn't match longest wayward edge");
        }
    private:
        VL_UNCOPYABLE(CpCostAccessor);
    };

private:
    // MEMBERS

    // Set of MTaskMoveVertex's assigned to this mtask. LogicMTask does not
    // own the MTaskMoveVertex objects, we merely keep pointers to them
    // here.
    VxList m_vertices;

    // Cost estimate for this LogicMTask, derived from V3InstrCount.
    // In abstract time units.
    uint32_t m_cost;

    // Cost of critical paths going FORWARD from graph-start to the start
    // of this vertex, and also going REVERSE from the end of the graph to
    // the end of the vertex. Same units as m_cost.
    uint32_t m_critPathCost[GraphWay::NUM_WAYS];

    uint32_t m_serialId;  // Unique MTask ID number

    // Count "generations" which are just operations that scan through the
    // graph. We'll mark each node with the last generation that scanned
    // it. We can use this to avoid recursing through the same node twice
    // while searching for a path.
    vluint64_t m_generation;

    // Redundant with the V3GraphEdge's, store a map of relatives so we can
    // quickly check if we have a given parent or child.
    //
    // 'm_edges[way]' maps a wayward relative to the !way critical path at
    // our edge with them. The SortByValueMap supports iterating over
    // relatives in longest-to-shortest CP order.  We rely on this ordering
    // in more than one place.
    typedef SortByValueMap<LogicMTask*, uint32_t, CmpLogicMTask> EdgeSet;
    EdgeSet m_edges[GraphWay::NUM_WAYS];

public:
    // CONSTRUCTORS
    LogicMTask(V3Graph* graphp, MTaskMoveVertex* mtmvVxp)
        : AbstractLogicMTask(graphp)
        , m_cost(0)
        , m_generation(0) {
        for (int i=0; i<GraphWay::NUM_WAYS; ++i) m_critPathCost[i] = 0;
        if (mtmvVxp) {  // Else null for test
            m_vertices.push_back(mtmvVxp);
            if (OrderLogicVertex* olvp = mtmvVxp->logicp()) {
                m_cost += V3InstrCount::count(olvp->nodep(), true);
            }
        }
        // Start at 1, so that 0 indicates no mtask ID.
        static uint32_t s_nextId = 1;
        m_serialId = s_nextId++;
        UASSERT(s_nextId < 0xFFFFFFFFUL, "Too many mtasks");
    }

    // METHODS
    void moveAllVerticesFrom(LogicMTask* otherp) {
        // splice() is constant time
        m_vertices.splice(m_vertices.end(), otherp->m_vertices);
        m_cost += otherp->m_cost;
    }
    virtual const VxList* vertexListp() const {
        return &m_vertices;
    }
    static vluint64_t incGeneration() {
        static vluint64_t s_generation = 0;
        ++s_generation;
        return s_generation;
    }

    // Use this instead of pointer-compares to compare LogicMTasks. Avoids
    // nondeterministic output.  Also name mtasks based on this number in
    // the final C++ output.
    virtual uint32_t id() const { return m_serialId; }
    void id(uint32_t id) { m_serialId = id; }
    // Abstract cost of every logic mtask
    virtual uint32_t cost() const { return m_cost; }
    void setCost(uint32_t cost) { m_cost = cost; }  // For tests only
    uint32_t stepCost() const { return stepCost(m_cost); }
    static uint32_t stepCost(uint32_t cost) {
#if PART_STEPPED_COST
        // Round cost up to the nearest 5%. Use this when computing all
        // critical paths. The idea is that critical path changes don't
        // need to propagate when they don't exceed the next step, saving a
        // lot of recursion.
        if (cost == 0) return 0;

        double logcost = log(cost);
        // log(1.05) is about 0.05
        // So, round logcost up to the next 0.05 boundary
        logcost *= 20.0;
        logcost = ceil(logcost);
        logcost = logcost / 20.0;

        uint32_t stepCost = (uint32_t)(exp(logcost));
        UASSERT_STATIC(stepCost >= cost, "stepped cost error exceeded");
        UASSERT_STATIC(stepCost <= ((cost * 11 / 10)), "stepped cost error exceeded");
        return stepCost;
#else
        return cost;
#endif
    }

    void addRelative(GraphWay way, LogicMTask* relativep) {
        EdgeSet& edges = m_edges[way];
        UASSERT(!edges.has(relativep), "Adding existing edge");
        // value is !way cp to this edge
        edges.set(relativep,
                  relativep->stepCost()
                  + relativep->critPathCost(way.invert()));
    }
    void removeRelative(GraphWay way, LogicMTask* relativep) {
        EdgeSet& edges = m_edges[way];
        edges.erase(relativep);
    }
    bool hasRelative(GraphWay way, LogicMTask* relativep) {
        EdgeSet& edges = m_edges[way];
        return edges.has(relativep);
    }
    void checkRelativesCp(GraphWay way) const {
        const EdgeSet& edges = m_edges[way];
        for (EdgeSet::const_reverse_iterator it = edges.rbegin();
             it != edges.rend(); ++it) {
            LogicMTask* relativep = (*it).key();
            uint32_t cachedCp = (*it).value();
            partCheckCachedScoreVsActual
                (cachedCp,
                 relativep->critPathCost(way.invert()) + relativep->stepCost());
        }
    }

    virtual string name() const {
        // Display forward and reverse critical path costs. This gives a quick
        // read on whether graph partitioning looks reasonable or bad.
        std::ostringstream out;
        out <<"mt"<<m_serialId<<"."<<this
            <<" [b"<<m_critPathCost[GraphWay::FORWARD]
            <<" a"<<m_critPathCost[GraphWay::REVERSE]
            <<" c"<<cost();
        return out.str();
    }

    void setCritPathCost(GraphWay way, uint32_t cost) { m_critPathCost[way] = cost; }
    uint32_t critPathCost(GraphWay way) const { return m_critPathCost[way]; }
    uint32_t critPathCostWithout(GraphWay way,
                                 const V3GraphEdge* withoutp) const {
        // Compute the critical path cost wayward to this node, without
        // considering edge 'withoutp'
        UASSERT(this == withoutp->furtherp(way),
                "In critPathCostWithout(), edge 'withoutp' must "
                "further to 'this'");

        // Iterate through edges until we get a relative other than
        // wayEdgeEndp(way, withoutp). This should take 2 iterations max.
        const EdgeSet& edges = m_edges[way.invert()];
        uint32_t result = 0;
        for (EdgeSet::const_reverse_iterator it = edges.rbegin();
             it != edges.rend(); ++it) {
            if ((*it).key() != withoutp->furtherp(way.invert())) {
                // Use the cached cost. It could be a small overestimate
                // due to stepping. This is consistent with critPathCost()
                // which also returns the cached cost.
                result = (*it).value();
                break;
            }
        }
        return result;
    }

private:
    static bool pathExistsFromInternal(LogicMTask* fromp,
                                       LogicMTask* top,
                                       const V3GraphEdge* excludedEdgep,
                                       vluint64_t generation) {
        // Q) Why does this take LogicMTask instead of generic V3GraphVertex?
        // A) We'll use the critical paths known to LogicMTask to prune the
        //    recursion for speed. Also store 'generation' in
        //    LogicMTask::m_generation so we can prune the search and avoid
        //    recursing through the same node more than once in a single
        //    search.

        if (fromp->m_generation == generation) {
            // Already looked at this node in the current search.
            // Since we're back again, we must not have found a path on the
            // first go.
            return false;
        }
        fromp->m_generation = generation;

        // Base case: we found a path.
        if (fromp == top) return true;

        // Base case: fromp is too late, cannot possibly be a prereq for top.
        if (fromp->critPathCost(GraphWay::REVERSE)
            < (top->critPathCost(GraphWay::REVERSE) + top->stepCost())) return false;
        if ((fromp->critPathCost(GraphWay::FORWARD) + fromp->stepCost())
            > top->critPathCost(GraphWay::FORWARD)) return false;

        // Recursively look for a path
        for (const V3GraphEdge* followp = fromp->outBeginp();
             followp; followp = followp->outNextp()) {
            if (followp == excludedEdgep) continue;
            LogicMTask* nextp = dynamic_cast<LogicMTask*>(followp->top());
            if (pathExistsFromInternal(nextp, top, NULL, generation))
                return true;
        }
        return false;
    }

    // True if there's a path from 'fromp' to 'top' excluding
    // 'excludedEdgep', false otherwise.
    //
    // 'excludedEdgep' may be NULL in which case no edge is excluded.  If
    // 'excludedEdgep' is non-NULL it must connect fromp and top.
    //
    // TODO: consider changing this API to the 'isTransitiveEdge' API
    // used by GraphPathChecker
public:
    static bool pathExistsFrom(LogicMTask* fromp,
                               LogicMTask* top,
                               const V3GraphEdge* excludedEdgep) {
        return pathExistsFromInternal(fromp, top, excludedEdgep,
                                      incGeneration());
    }

    static void dumpCpFilePrefixed(const V3Graph* graphp,
                                   const string& nameComment) {
        string filename = v3Global.debugFilename(nameComment)+".txt";
        UINFO(1,"Writing "<<filename<<endl);
        vl_unique_ptr<std::ofstream> ofp(V3File::new_ofstream(filename));
        std::ostream* osp = &(*ofp);  // &* needed to deref unique_ptr
        if (osp->fail()) v3fatalStatic("Can't write "<<filename);

        // Find start vertex with longest CP
        const LogicMTask* startp = NULL;
        for (const V3GraphVertex* vxp = graphp->verticesBeginp();
             vxp; vxp = vxp->verticesNextp()) {
            const LogicMTask* mtaskp = dynamic_cast<const LogicMTask*>(vxp);
            if (!startp) {
                startp = mtaskp;
                continue;
            }
            if (mtaskp->cost() + mtaskp->critPathCost(GraphWay::REVERSE)
                > startp->cost() + startp->critPathCost(GraphWay::REVERSE)) {
                startp = mtaskp;
            }
        }

        // Follow the entire critical path
        std::vector<const LogicMTask*> path;
        uint32_t totalCost = 0;
        for (const LogicMTask* nextp = startp; nextp;) {
            path.push_back(nextp);
            totalCost += nextp->cost();

            const EdgeSet& children = nextp->m_edges[GraphWay::FORWARD];
            EdgeSet::const_reverse_iterator it = children.rbegin();
            if (it == children.rend()) nextp = NULL;
            else nextp = (*it).key();
        }

        *osp<<"totalCost = "<<totalCost
            <<" (should match the computed critical path cost (CP) for the graph)\n";

        // Dump
        for (std::vector<const LogicMTask*>::iterator it = path.begin();
             it != path.end(); ++it) {
            const LogicMTask* mtaskp = *it;
            *osp<<"begin mtask with cost "<<mtaskp->cost()<<endl;
            for (VxList::const_iterator lit = mtaskp->vertexListp()->begin();
                 lit != mtaskp->vertexListp()->end(); ++lit) {
                const OrderLogicVertex* logicp = (*lit)->logicp();
                if (!logicp) continue;
                if (0) {
                    // Show nodes only
                    *osp<<"> "; logicp->nodep()->dumpTree(*osp);
                } else {
                    // Show nodes with hierarchical costs
                    V3InstrCount::count(logicp->nodep(), false, osp);
                }
            }
        }
    }

private:
    VL_DEBUG_FUNC;  // Declare debug()
    VL_UNCOPYABLE(LogicMTask);
};

//######################################################################
// MTask utility classes

// Sort AbstractMTask objects into deterministic order by calling id()
// which is a unique and stable serial number.
class MTaskIdLessThan {
public:
    MTaskIdLessThan() {}
    virtual ~MTaskIdLessThan() {}
    virtual bool operator() (const AbstractMTask* lhsp,
                             const AbstractMTask* rhsp) const {
        return lhsp->id() < rhsp->id();
    }
};

// Information associated with scoreboarding an MTask
class MergeCandidate {
private:
    bool m_removedFromSb;  // Not on scoreboard, generally ignore
    vluint64_t m_id;  // Serial number for ordering
public:
    // CONSTRUCTORS
    MergeCandidate() : m_removedFromSb(false) {
        static vluint64_t serial = 0;
        ++serial;
        m_id = serial;
    }
    virtual bool mergeWouldCreateCycle() const = 0;
    // METHODS
    bool removedFromSb() const { return m_removedFromSb; }
    void removedFromSb(bool removed) { m_removedFromSb = removed; }
    bool operator<(const MergeCandidate& other) const {
        return m_id < other.m_id;
    }
};

// A pair of associated LogicMTask's that are merge candidates for sibling
// contraction
class SiblingMC : public MergeCandidate {
private:
    LogicMTask* m_ap;
    LogicMTask* m_bp;
    // CONSTRUCTORS
    SiblingMC() VL_EQ_DELETE;
public:
    SiblingMC(LogicMTask* ap, LogicMTask* bp) {
        // Assign 'ap' and 'bp' in a canonical order, so we can more easily
        // compare pairs of SiblingMCs
        if (ap->id() > bp->id()) {
            m_ap = ap;
            m_bp = bp;
        } else {
            m_ap = bp;
            m_bp = ap;
        }
    }
    virtual ~SiblingMC() {}
    // METHODS
    LogicMTask* ap() const { return m_ap; }
    LogicMTask* bp() const { return m_bp; }
    bool mergeWouldCreateCycle() const {
        return (LogicMTask::pathExistsFrom(m_ap, m_bp, NULL)
                || LogicMTask::pathExistsFrom(m_bp, m_ap, NULL));
    }
    bool operator<(const SiblingMC& other) const {
        if (m_ap->id() < other.m_ap->id()) { return true; }
        if (m_ap->id() > other.m_ap->id()) { return false; }
        return m_bp->id() < other.m_bp->id();
    }
};

// GraphEdge for the MTask graph
class MTaskEdge : public V3GraphEdge, public MergeCandidate {
public:
    // CONSTRUCTORS
    MTaskEdge(V3Graph* graphp, LogicMTask* fromp, LogicMTask* top, int weight)
        : V3GraphEdge(graphp, fromp, top, weight),
          MergeCandidate() {
        fromp->addRelative(GraphWay::FORWARD, top);
        top->addRelative(GraphWay::REVERSE, fromp);
    }
    virtual ~MTaskEdge() {
        fromMTaskp()->removeRelative(GraphWay::FORWARD, toMTaskp());
        toMTaskp()->removeRelative(GraphWay::REVERSE, fromMTaskp());
    }
    // METHODS
    LogicMTask* furtherMTaskp(GraphWay way) const {
        return dynamic_cast<LogicMTask*>(this->furtherp(way));
    }
    LogicMTask* fromMTaskp() const {
        return dynamic_cast<LogicMTask*>(fromp());
    }
    LogicMTask* toMTaskp() const {
        return dynamic_cast<LogicMTask*>(top());
    }
    virtual bool mergeWouldCreateCycle() const {
        return LogicMTask::pathExistsFrom(fromMTaskp(), toMTaskp(), this);
    }
    static MTaskEdge* cast(V3GraphEdge* edgep) {
        if (!edgep) return NULL;
        MTaskEdge* resultp = dynamic_cast<MTaskEdge*>(edgep);
        UASSERT(resultp, "Failed to cast in MTaskEdge::cast");
        return resultp;
    }
    // Following initial assignment of critical paths, clear this MTaskEdge
    // out of the edge-map for each node and reinsert at a new location
    // with updated critical path.
    void resetCriticalPaths() {
        LogicMTask* fromp = fromMTaskp();
        LogicMTask* top = toMTaskp();
        fromp->removeRelative(GraphWay::FORWARD, top);
        top->removeRelative(GraphWay::REVERSE, fromp);
        fromp->addRelative(GraphWay::FORWARD, top);
        top->addRelative(GraphWay::REVERSE, fromp);
    }
private:
    VL_UNCOPYABLE(MTaskEdge);
};

//######################################################################
// Vertex utility classes

class OrderByPtrId {
    PartPtrIdMap m_ids;
public:
    virtual bool operator() (const OrderVarStdVertex* lhsp,
                             const OrderVarStdVertex* rhsp) const {
        vluint64_t l_id = m_ids.findId(lhsp);
        vluint64_t r_id = m_ids.findId(rhsp);
        return l_id < r_id;
    }
};

//######################################################################
// PartParallelismEst - Estimate parallelism of graph

class PartParallelismEst {
    // MEMBERS
    const V3Graph* m_graphp;  // Mtask-containing graph

    // Total cost of evaluating the whole graph.
    // The ratio of m_totalGraphCost to longestCpCost gives us an estimate
    // of the parallelizability of this graph which is only as good as the
    // guess returned by LogicMTask::cost().
    uint32_t m_totalGraphCost;

    // Cost of the longest critical path, in abstract units (the same units
    // returned by the vertexCost)
    uint32_t m_longestCpCost;

    size_t m_vertexCount;  // Number of vertexes calculated
    size_t m_edgeCount;  // Number of edges calculated

public:
    // CONSTRUCTORS
    explicit PartParallelismEst(const V3Graph* graphp)
        : m_graphp(graphp),
          m_totalGraphCost(0),
          m_longestCpCost(0),
          m_vertexCount(0),
          m_edgeCount(0) {}

    // METHODS
    uint32_t totalGraphCost() const { return m_totalGraphCost; }
    uint32_t longestCritPathCost() const { return m_longestCpCost; }
    size_t vertexCount() const { return m_vertexCount; }
    size_t edgeCount() const { return m_edgeCount; }
    double parallelismFactor() const {
        return (static_cast<double>(m_totalGraphCost) / m_longestCpCost);
    }
    void traverse() {
        // For each node, record the critical path cost from the start
        // of the graph through the end of the node.
        vl_unordered_map<const V3GraphVertex*, uint32_t> critPaths;
        GraphStreamUnordered serialize(m_graphp);
        for (const V3GraphVertex* vertexp;
             (vertexp = serialize.nextp());) {
            m_vertexCount++;
            uint32_t cpCostToHere = 0;
            for (V3GraphEdge* edgep = vertexp->inBeginp(); edgep;
                 edgep = edgep->inNextp()) {
                ++m_edgeCount;
                // For each upstream item, add its critical path cost to
                // the cost of this edge, to form a new candidate critical
                // path cost to the current node. Whichever is largest is
                // the critical path to reach the start of this node.
                cpCostToHere = std::max(cpCostToHere, critPaths[edgep->fromp()]);
            }
            // Include the cost of the current vertex in the critical
            // path, so it represents the critical path to the end of
            // this vertex.
            cpCostToHere += vertexCost(vertexp);
            critPaths[vertexp] = cpCostToHere;
            m_longestCpCost = std::max(m_longestCpCost, cpCostToHere);
            // Tally the total cost contributed by vertices.
            m_totalGraphCost += vertexCost(vertexp);
        }
    }
    void statsReport(const string& stage) {
        V3Stats::addStat("MTask graph, "+stage+", critical path cost",
                         m_longestCpCost);
        V3Stats::addStat("MTask graph, "+stage+", total graph cost",
                         m_totalGraphCost);
        V3Stats::addStat("MTask graph, "+stage+", mtask count",
                         m_vertexCount);
        V3Stats::addStat("MTask graph, "+stage+", edge count",
                         m_edgeCount);
        V3Stats::addStat("MTask graph, "+stage+", parallelism factor",
                         parallelismFactor());
    }
    void debugReport() {
        UINFO(0, "    Critical path cost = "<<m_longestCpCost<<endl);
        UINFO(0, "    Total graph cost = "<<m_totalGraphCost<<endl);
        UINFO(0, "    MTask vertex count = "<<m_vertexCount<<endl);
        UINFO(0, "    Edge count = "<<m_edgeCount<<endl);
        UINFO(0, "    Parallelism factor = "<<parallelismFactor()<<endl);
    }
    static uint32_t vertexCost(const V3GraphVertex* vertexp) {
        return dynamic_cast<const AbstractMTask*>(vertexp)->cost();
    }

private:
    VL_DEBUG_FUNC;  // Declare debug()
    VL_UNCOPYABLE(PartParallelismEst);
};

//######################################################################

// Look at vertex costs (in one way) to form critical paths for each
// vertex.
static void partInitHalfCriticalPaths(GraphWay way, V3Graph* mtasksp, bool checkOnly) {
    GraphStreamUnordered order(mtasksp, way);
    GraphWay rev = way.invert();
    for (const V3GraphVertex* vertexp;
         (vertexp = order.nextp());) {
        const LogicMTask* mtaskcp = dynamic_cast<const LogicMTask*>(vertexp);
        LogicMTask* mtaskp = const_cast<LogicMTask*>(mtaskcp);
        uint32_t cpCost = 0;
        vl_unordered_set<V3GraphVertex*> relatives;
        for (V3GraphEdge* edgep = vertexp->beginp(rev);
             edgep; edgep = edgep->nextp(rev)) {
            // Run a few asserts on the initial mtask graph,
            // while we're iterating through...
            if (edgep->weight() == 0) {
                mtaskp->v3fatalSrc("Should be no cut edges in mtasks graph");
            }
            if (relatives.find(edgep->furtherp(rev)) != relatives.end()) {
                mtaskp->v3fatalSrc("Should be no redundant edges in mtasks graph");
            }
            relatives.insert(edgep->furtherp(rev));

            LogicMTask* relativep
                = dynamic_cast<LogicMTask*>(edgep->furtherp(rev));
            cpCost = std::max(cpCost,
                              (relativep->critPathCost(way)
                               + static_cast<uint32_t>(relativep->stepCost())));
        }
        if (checkOnly) {
            partCheckCachedScoreVsActual(mtaskp->critPathCost(way), cpCost);
        } else {
            mtaskp->setCritPathCost(way, cpCost);
        }
    }
}

// Look at vertex costs to form critical paths for each vertex.
static void partInitCriticalPaths(V3Graph* mtasksp) {
    partInitHalfCriticalPaths(GraphWay::FORWARD, mtasksp, false);
    partInitHalfCriticalPaths(GraphWay::REVERSE, mtasksp, false);

    // Reset all MTaskEdges so that 'm_edges' will show correct CP numbers.
    // They would have been all zeroes on initial creation of the MTaskEdges.
    std::vector<V3GraphEdge*> edges;
    for (V3GraphVertex* vxp = mtasksp->verticesBeginp();
         vxp; vxp = vxp->verticesNextp()) {
        for (V3GraphEdge* edgep = vxp->outBeginp();
             edgep; edgep = edgep->outNextp()) {
            MTaskEdge* mtedgep = dynamic_cast<MTaskEdge*>(edgep);
            mtedgep->resetCriticalPaths();
        }
    }
}

// Do an EXPENSIVE check to make sure that all incremental CP updates have
// gone correctly.
static void partCheckCriticalPaths(V3Graph* mtasksp) {
    partInitHalfCriticalPaths(GraphWay::FORWARD, mtasksp, true);
    partInitHalfCriticalPaths(GraphWay::REVERSE, mtasksp, true);
    for (V3GraphVertex* vxp = mtasksp->verticesBeginp();
         vxp; vxp = vxp->verticesNextp()) {
        LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(vxp);
        mtaskp->checkRelativesCp(GraphWay::FORWARD);
        mtaskp->checkRelativesCp(GraphWay::REVERSE);
    }
}

// Advance to nextp(way) and delete edge
static V3GraphEdge* partBlastEdgep(GraphWay way, V3GraphEdge* edgep) {
    V3GraphEdge* nextp = edgep->nextp(way);
    edgep->unlinkDelete(); VL_DANGLING(edgep);
    return nextp;
}

// Merge edges from a LogicMtask.
//
// This code removes 'hasRelative' edges. When this occurs, mark it in need
// of a rescore, in case its score has fallen and we need to move it up
// toward the front of the scoreboard.
//
// Wait, whaaat? Shouldn't the scores only increase as we merge nodes? Well
// that's almost true. But there is one exception.
//
// Suppose we have A->B, B->C, and A->C.
//
// The A->C edge is a "transitive" edge. It's ineligible to be merged, as
// the merge would create a cycle. We score it on the scoreboard like any
// other edge.
//
// However, our "score" estimate for A->C is bogus, because the forward
// critical path to C and the reverse critical path to A both contain the
// same node (B) so we overestimate the score of A->C. At first this
// doesn't matter, since transitive edges aren't eligible to merge anyway.
//
// Later, suppose the edge contractor decides to merge the B->C edge, with
// B donating all its incoming edges into C, say.  (So we reach this
// function.)
//
// With B going away, the A->C edge will no longer be transitive and it
// will become eligible to merge. But if we don't mark it for rescore,
// it'll stay in the scoreboard with its old (overestimate) score. We'll
// merge it too late due to the bogus score. When we finally merge it, we
// fail the assert in the main edge contraction loop which checks that the
// actual score did not fall below the scoreboard's score.
//
// Another way of stating this: this code ensures that scores of
// non-transitive edges only ever increase.
static void partMergeEdgesFrom(V3Graph* mtasksp, LogicMTask* recipientp,
                               LogicMTask* donorp,
                               V3Scoreboard<MergeCandidate, uint32_t>* sbp) {
    for (unsigned wi = 0; wi < 2; ++wi) {
        GraphWay way = wi ? GraphWay::REVERSE : GraphWay::FORWARD;
        for (V3GraphEdge* edgep = donorp->beginp(way);
             edgep; edgep = partBlastEdgep(way, edgep)) {
            MTaskEdge* tedgep = MTaskEdge::cast(edgep);
            if (sbp && !tedgep->removedFromSb())
                sbp->removeElem(tedgep);
            // Existing edge; mark it in need of a rescore
            if (recipientp->hasRelative(way, tedgep->furtherMTaskp(way))) {
                if (sbp) {
                    MTaskEdge* existMTaskEdgep =
                        MTaskEdge::cast(recipientp->findConnectingEdgep
                                        (way, tedgep->furtherMTaskp(way)));
                    UASSERT(existMTaskEdgep, "findConnectingEdge didn't find edge");
                    if (!existMTaskEdgep->removedFromSb()) {
                        sbp->hintScoreChanged(existMTaskEdgep);
                    }
                }
            } else {
                // No existing edge into *this, make one.
                MTaskEdge* newEdgep;
                if (way == GraphWay::REVERSE) {
                    newEdgep = new MTaskEdge(mtasksp, tedgep->fromMTaskp(),
                                             recipientp, 1);
                } else {
                    newEdgep = new MTaskEdge(mtasksp, recipientp,
                                             tedgep->toMTaskp(), 1);
                }
                if (sbp) sbp->addElem(newEdgep);
            }
        }
    }
}

//######################################################################
// PartContraction

// Perform edge or sibling contraction on the partition graph
class PartContraction {
private:
    // TYPES

    // TODO: might get a little more speed by making this a
    // vl_unordered_set and defining hash and equal_to functors for the
    // SiblingMC:
    typedef std::set<SiblingMC> SibSet;
    typedef vl_unordered_set<const SiblingMC*> SibpSet;
    typedef vl_unordered_map<const LogicMTask*, SibpSet> MTask2Sibs;

    // New CP information for mtaskp reflecting an upcoming merge
    struct NewCp {
        uint32_t cp;
        uint32_t propagateCp;
        bool propagate;
    };

    // MEMBERS
    V3Graph* m_mtasksp;  // Mtask graph
    uint32_t m_scoreLimit;  // Sloppy score allowed when picking merges
    uint32_t m_scoreLimitBeforeRescore;  // Next score rescore at
    unsigned m_mergesSinceRescore;  // Merges since last rescore
    bool m_slowAsserts;  // Take extra time to validate algorithm
    V3Scoreboard<MergeCandidate, uint32_t> m_sb;  // Scoreboard
    SibSet m_pairs;  // Storage for each SiblingMC
    MTask2Sibs m_mtask2sibs;  // SiblingMC set for each mtask

public:
    // CONSTRUCTORS
    PartContraction(V3Graph* mtasksp, uint32_t scoreLimit, bool slowAsserts)
        : m_mtasksp(mtasksp)
        , m_scoreLimit(scoreLimit)
        , m_scoreLimitBeforeRescore(0xffffffff)
        , m_mergesSinceRescore(0)
        , m_slowAsserts(slowAsserts)
        , m_sb(&mergeCandidateScore, slowAsserts) { }

    // METHODS
    void go() {
        unsigned maxMTasks = v3Global.opt.threadsMaxMTasks();
        if (maxMTasks == 0) {  // Unspecified so estimate
            if (v3Global.opt.threads() > 1) {
                maxMTasks = (PART_DEFAULT_MAX_MTASKS_PER_THREAD
                             * v3Global.opt.threads());
            } else {
                // Running PartContraction with --threads <= 1 means self-test
                maxMTasks = 500;
            }
        }

        // OPTIMIZATION PASS: Edge contraction and sibling contraction.
        //  - Score each pair of mtasks which is a candidate to merge.
        //    * Each edge defines such a candidate pair
        //    * Two mtasks that are prereqs or postreqs of a common third
        //      vertex are "siblings", these are also a candidate pair.
        //  - Build a list of MergeCandidates, sorted by score.
        //  - Merge the best pair.
        //  - Incrementally recompute critical paths near the merged mtask.

        for (V3GraphVertex* itp = m_mtasksp->verticesBeginp(); itp;
             itp = itp->verticesNextp()) {
            vl_unordered_set<const V3GraphVertex*> neighbors;
            for (V3GraphEdge* edgep = itp->outBeginp(); edgep;
                 edgep=edgep->outNextp()) {
                m_sb.addElem(MTaskEdge::cast(edgep));
                if (neighbors.find(edgep->top()) != neighbors.end()) {
                    itp->v3fatalSrc("Redundant edge found in input to PartContraction()");
                }
                neighbors.insert(edgep->top());
            }
            siblingPairFromRelatives(GraphWay::REVERSE, itp, true);
            siblingPairFromRelatives(GraphWay::FORWARD, itp, true);
        }

        doRescore();  // Set initial scores in scoreboard

        while (1) {
            // This is the best edge to merge, with the lowest
            // score (shortest local critical path)
            MergeCandidate* mergeCanp = const_cast<MergeCandidate*>(m_sb.bestp());
            if (!mergeCanp) {
                // Scoreboard found no eligible merges. Maybe a rescore
                // will produce some merge-able pairs?
                if (m_sb.needsRescore()) {
                    doRescore();
                    continue;
                }
                break;
            }

            if (m_slowAsserts) {
                UASSERT(!m_sb.needsRescore(mergeCanp),
                        "Need-rescore items should not be returned by bestp");
            }
            uint32_t cachedScore = m_sb.cachedScore(mergeCanp);
            uint32_t actualScore = mergeCandidateScore(mergeCanp);

            if (actualScore > cachedScore) {
                // Cached score is out-of-date.
                // Mark this elem as in need of a rescore and continue.
                m_sb.hintScoreChanged(mergeCanp);
                continue;
            }
            // ... we'll also confirm that actualScore hasn't shrunk relative
            // to cached score, after the mergeWouldCreateCycle() check.

            if (actualScore > m_scoreLimit) {
                // Our best option isn't good enough
                if (m_sb.needsRescore()) {
                    // Some pairs need a rescore, maybe those will be
                    // eligible to merge afterward.
                    doRescore();
                    continue;
                } else {
                    // We've exhausted everything below m_scoreLimit; stop.

                    // Except, if we have too many mtasks, raise the score
                    // limit and keep going...
                    unsigned mtaskCount = 0;
                    for (V3GraphVertex* vxp = m_mtasksp->verticesBeginp();
                         vxp; vxp = vxp->verticesNextp()) {
                        ++mtaskCount;
                    }
                    if (mtaskCount > maxMTasks) {
                        uint32_t oldLimit = m_scoreLimit;
                        m_scoreLimit = (m_scoreLimit * 120) / 100;

                        // Line must be >0 otherwise FileLine doesn't check
                        // if the warning is suppressed with -Wno-UNOPTTHREADS
                        FileLine dummyFl("AstRoot", 1);
                        dummyFl.v3warn(UNOPTTHREADS, "Thread scheduler is unable to provide requested parallelism; consider asking for fewer threads.");
                        UINFO(1,"Critical path limit was="<<oldLimit
                              <<" now="<<m_scoreLimit<<endl);
                        continue;
                    }
                    // Really stop
                    break;
                }
            }
            if (actualScore > m_scoreLimitBeforeRescore) {
                // Time to rescore, that will result in a higher
                // scoreLimitBeforeRescore, and possibly lower-scoring
                // elements returned from bestp().
                doRescore();
                continue;
            }

            // Avoid merging any edge that would create a cycle.
            //
            // For example suppose we begin with vertices A, B, C and edges
            // A->B, B->C, A->C.
            //
            // Suppose we want to merge A->C into a single vertex.
            // New edges would be AC->B and B->AC which is not a DAG.
            // Do not allow this.
            if (mergeCanp->mergeWouldCreateCycle()) {
                // Remove this edge from scoreboard so we don't keep
                // reconsidering it on every loop.
                m_sb.removeElem(mergeCanp);
                mergeCanp->removedFromSb(true);
                continue;
            }

            partCheckCachedScoreVsActual(cachedScore, actualScore);

            // Finally there's no cycle risk, no need to rescore, we're
            // within m_scoreLimit and m_scoreLimitBeforeRescore.
            // This is the edge to merge.
            //
            // Bookkeeping: if this is the first edge we'll merge since
            // the last rescore, compute the new m_scoreLimitBeforeRescore
            // to be somewhat higher than this edge's score.
            if (m_mergesSinceRescore == 0) {
#if PART_STEPPED_RESCORELIMIT
                m_scoreLimitBeforeRescore = (actualScore * 105) / 100;
#else
                m_scoreLimitBeforeRescore = actualScore;
#endif

                // This print can serve as a progress indicator, as it
                // increases from low numbers up toward cpLimit. It may be
                // helpful to see progress during slow partitions. Maybe
                // display something by default even?
                UINFO(6, "New scoreLimitBeforeRescore: "
                      <<m_scoreLimitBeforeRescore<<endl);
            }

            // Finally merge this candidate.
            contract(mergeCanp);
        }
    }

private:
    NewCp newCp(GraphWay way, LogicMTask* mtaskp, LogicMTask* otherp,
                MTaskEdge* mergeEdgep) {
        // Return new wayward-CP for mtaskp reflecting its upcoming merge
        // with otherp. Set 'result.propagate' if mtaskp's wayward
        // relatives will see a new wayward CP from this merge.
        uint32_t newCp;
        if (mergeEdgep) {
            if (mtaskp == mergeEdgep->furtherp(way)) {
                newCp = std::max(otherp->critPathCost(way),
                                 mtaskp->critPathCostWithout(way, mergeEdgep));
            } else {
                newCp = std::max(mtaskp->critPathCost(way),
                                 otherp->critPathCostWithout(way, mergeEdgep));
            }
        } else {
            newCp = std::max(otherp->critPathCost(way), mtaskp->critPathCost(way));
        }

        uint32_t origRelativesCp
            = mtaskp->critPathCost(way) + mtaskp->stepCost();
        uint32_t newRelativesCp
            = newCp + LogicMTask::stepCost(mtaskp->cost() + otherp->cost());

        NewCp result;
        result.cp = newCp;
        result.propagate = (newRelativesCp > origRelativesCp);
        result.propagateCp = newRelativesCp;
        return result;
    }

    void removeSiblingMCsWith(LogicMTask* mtaskp) {
        for (SibpSet::iterator it = m_mtask2sibs[mtaskp].begin();
             it != m_mtask2sibs[mtaskp].end(); ++it) {
            const SiblingMC* pairp = *it;
            if (!pairp->removedFromSb()) {
                m_sb.removeElem(pairp);
            }
            LogicMTask* otherp = (pairp->bp() == mtaskp) ?
                pairp->ap() : pairp->bp();
            size_t erased = m_mtask2sibs[otherp].erase(pairp);
            if (erased <= 0) otherp->v3fatalSrc("Expected existing mtask");
            erased = m_pairs.erase(*pairp);
            if (erased <= 0) mtaskp->v3fatalSrc("Expected existing mtask");
        }
        size_t erased = m_mtask2sibs.erase(mtaskp);
        if (erased <= 0) mtaskp->v3fatalSrc("Expected existing mtask");
    }

    void contract(MergeCandidate* mergeCanp) {
        LogicMTask *top = NULL;
        LogicMTask *fromp = NULL;
        MTaskEdge* mergeEdgep = dynamic_cast<MTaskEdge*>(mergeCanp);
        SiblingMC* mergeSibsp = NULL;
        if (mergeEdgep) {
            top = dynamic_cast<LogicMTask*>(mergeEdgep->top());
            fromp = dynamic_cast<LogicMTask*>(mergeEdgep->fromp());
        } else {
            mergeSibsp = dynamic_cast<SiblingMC*>(mergeCanp);
            UASSERT(mergeSibsp,
                    "Failed to cast mergeCanp to either MTaskEdge or SiblingMC");
            top = mergeSibsp->ap();
            fromp = mergeSibsp->bp();
        }

        // Merge the smaller mtask into the larger mtask.  If one of them
        // is much larger, this will save time in partMergeEdgesFrom().
        // Assume the more costly mtask has more edges.
        //
        // [TODO: now that we have edge maps, we could count the edges
        //  exactly without a linear search.]
        LogicMTask* recipientp;
        LogicMTask* donorp;
        if (fromp->cost() > top->cost()) {
            recipientp = fromp;
            donorp = top;
        } else {
            donorp = fromp;
            recipientp = top;
        }
        fromp = top = NULL;  // Use donorp and recipientp now instead

        // Recursively update forward and reverse CP numbers.
        //
        // Doing this before merging the mtasks lets us often avoid
        // recursing through either incoming or outgoing edges on one or
        // both mtasks.
        //
        // These 'NewCp' objects carry a bit indicating whether we must
        // propagate CP for each of the four cases:
        NewCp recipientNewCpFwd
            = newCp(GraphWay::FORWARD, recipientp, donorp, mergeEdgep);
        NewCp donorNewCpFwd
            = newCp(GraphWay::FORWARD, donorp, recipientp, mergeEdgep);
        NewCp recipientNewCpRev
            = newCp(GraphWay::REVERSE, recipientp, donorp, mergeEdgep);
        NewCp donorNewCpRev
            = newCp(GraphWay::REVERSE, donorp, recipientp, mergeEdgep);

        if (mergeEdgep) {
            // Remove and free the connecting edge. Must do this before
            // propagating CP's below.
            m_sb.removeElem(mergeCanp);
            mergeEdgep->unlinkDelete(); mergeEdgep=NULL;
        }

        // This also updates cost and stepCost on recipientp
        recipientp->moveAllVerticesFrom(donorp);

        UINFO(9, "recipient = "<<recipientp->id()
              << ", donor = "<<donorp->id()
              << ", mergeEdgep = "<<mergeEdgep
              << "\n"
              << "recipientNewCpFwd = "<<recipientNewCpFwd.cp
              << (recipientNewCpFwd.propagate ? " true " : " false ")
              << recipientNewCpFwd.propagateCp
              << "\n"
              << "donorNewCpFwd = "<<donorNewCpFwd.cp
              << (donorNewCpFwd.propagate ? " true " : " false ")
              << donorNewCpFwd.propagateCp
              << endl);

        LogicMTask::CpCostAccessor cpAccess;
        PartPropagateCp<LogicMTask::CpCostAccessor>
            forwardPropagator(m_mtasksp, GraphWay::FORWARD, &cpAccess, m_slowAsserts);
        PartPropagateCp<LogicMTask::CpCostAccessor>
            reversePropagator(m_mtasksp, GraphWay::REVERSE, &cpAccess, m_slowAsserts);

        recipientp->setCritPathCost(GraphWay::FORWARD,
                                    recipientNewCpFwd.cp);
        if (recipientNewCpFwd.propagate) {
            forwardPropagator.cpHasIncreased(recipientp, recipientNewCpFwd.propagateCp);
        }
        recipientp->setCritPathCost(GraphWay::REVERSE,
                                    recipientNewCpRev.cp);
        if (recipientNewCpRev.propagate) {
            reversePropagator.cpHasIncreased(recipientp, recipientNewCpRev.propagateCp);
        }
        if (donorNewCpFwd.propagate) {
            forwardPropagator.cpHasIncreased(donorp, donorNewCpFwd.propagateCp);
        }
        if (donorNewCpRev.propagate) {
            reversePropagator.cpHasIncreased(donorp, donorNewCpRev.propagateCp);
        }
        forwardPropagator.go();
        reversePropagator.go();

        // Remove all SiblingMCs that include donorp. This Includes the one
        // we're merging, if we're merging a SiblingMC.
        removeSiblingMCsWith(donorp);
        // Remove all SiblingMCs that include recipientp also, so we can't
        // get huge numbers of SiblingMCs.  We'll recreate them below, up
        // to a bounded number.
        removeSiblingMCsWith(recipientp);

        // Merge all edges
        partMergeEdgesFrom(m_mtasksp, recipientp, donorp, &m_sb);

        // Delete the donorp mtask from the graph
        donorp->unlinkDelete(m_mtasksp); donorp = NULL;

        m_mergesSinceRescore++;

        // Do an expensive check, confirm we haven't botched the CP
        // updates.
        if (m_slowAsserts) partCheckCriticalPaths(m_mtasksp);

        // Finally, make new sibling pairs as needed:
        //  - prereqs and postreqs of recipientp
        //  - prereqs of recipientp's postreqs
        //  - postreqs of recipientp's prereqs
        // Note that this depends on the updated critical paths (above).
        siblingPairFromRelatives(GraphWay::REVERSE, recipientp, true);
        siblingPairFromRelatives(GraphWay::FORWARD, recipientp, true);
        unsigned edges = 0;
        for (V3GraphEdge* edgep = recipientp->outBeginp();
             edgep; edgep = edgep->outNextp()) {
            LogicMTask* postreqp = dynamic_cast<LogicMTask*>(edgep->top());
            siblingPairFromRelatives(GraphWay::REVERSE, postreqp, false);
            edges++;
            if (edges > PART_SIBLING_EDGE_LIMIT) break;
        }
        edges = 0;
        for (V3GraphEdge* edgep = recipientp->inBeginp();
             edgep; edgep = edgep->inNextp()) {
            LogicMTask* prereqp = dynamic_cast<LogicMTask*>(edgep->fromp());
            siblingPairFromRelatives(GraphWay::FORWARD, prereqp, false);
            edges++;
            if (edges > PART_SIBLING_EDGE_LIMIT) break;
        }
    }

    void doRescore() {
        // During rescore, we know that graph isn't changing, so allow
        // the critPathCost*Without() routines to cache some data in
        // each LogicMTask. This is just an optimization, things should
        // behave identically without the caching (just slower)

        m_sb.rescore();
        UINFO(6, "Did rescore. Merges since previous = "
              << m_mergesSinceRescore << endl);

        m_mergesSinceRescore = 0;
        m_scoreLimitBeforeRescore = 0xffffffff;
    }

    static uint32_t mergeCandidateScore(const MergeCandidate* pairp) {
        const MTaskEdge* edgep = dynamic_cast<const MTaskEdge*>(pairp);
        if (edgep) {
            // The '1 +' favors merging a SiblingMC over an otherwise-
            // equal-scoring MTaskEdge. The comment on selfTest() talks
            // about why.
            return 1 + edgeScore(edgep);
        }
        const SiblingMC* sibsp = dynamic_cast<const SiblingMC*>(pairp);
        if (sibsp) {
            return siblingScore(sibsp);
        }
        v3fatalSrc("Failed to cast pairp to either MTaskEdge or SiblingMC in mergeCandidateScore");
        return 0;
    }

    static uint32_t siblingScore(const SiblingMC* sibsp) {
        LogicMTask* ap = sibsp->ap();
        LogicMTask* bp = sibsp->bp();
        uint32_t mergedCpCostFwd = std::max(ap->critPathCost(GraphWay::FORWARD),
                                            bp->critPathCost(GraphWay::FORWARD));
        uint32_t mergedCpCostRev = std::max(ap->critPathCost(GraphWay::REVERSE),
                                            bp->critPathCost(GraphWay::REVERSE));
        return mergedCpCostRev + mergedCpCostFwd
            + LogicMTask::stepCost(ap->cost() + bp->cost());
    }

    static uint32_t edgeScore(const V3GraphEdge* edgep) {
        // Score this edge. Lower is better. The score is the new local CP
        // length if we merge these mtasks.  ("Local" means the longest
        // critical path running through the merged node.)
        LogicMTask* top = dynamic_cast<LogicMTask*>(edgep->top());
        LogicMTask* fromp = dynamic_cast<LogicMTask*>(edgep->fromp());
        uint32_t mergedCpCostFwd = std::max
            (fromp->critPathCost(GraphWay::FORWARD),
             top->critPathCostWithout(GraphWay::FORWARD, edgep));
        uint32_t mergedCpCostRev = std::max
            (fromp->critPathCostWithout(GraphWay::REVERSE, edgep),
             top->critPathCost(GraphWay::REVERSE));
        return mergedCpCostRev + mergedCpCostFwd
            + LogicMTask::stepCost(fromp->cost() + top->cost());
    }

    void makeSiblingMC(LogicMTask* ap, LogicMTask *bp) {
        SiblingMC newSibs(ap, bp);
        std::pair<SibSet::iterator, bool> insertResult = m_pairs.insert(newSibs);
        if (insertResult.second) {
            const SiblingMC* newSibsp = &(*insertResult.first);
            m_mtask2sibs[ap].insert(newSibsp);
            m_mtask2sibs[bp].insert(newSibsp);
            m_sb.addElem(newSibsp);
        } else if (m_slowAsserts) {
            // It's fine if we already have this SiblingMC, we may have
            // created it earlier. Just confirm that we have associated data.
            if (m_mtask2sibs.find(ap) == m_mtask2sibs.end()) {
                ap->v3fatalSrc("Sibling not found");
            }
            if (m_mtask2sibs.find(bp) == m_mtask2sibs.end()) {
                bp->v3fatalSrc("Sibling not found");
            }
            bool found = false;
            for (SibpSet::iterator it = m_mtask2sibs[ap].begin();
                 it != m_mtask2sibs[ap].end(); ++it) {
                const SiblingMC* sibsp = *it;
                if (!sibsp->removedFromSb() && !m_sb.contains(sibsp)) {
                    ap->v3fatalSrc("One sibling must be the one we collided with");
                }
                if (   (sibsp->ap() == ap && sibsp->bp() == bp)
                    || (sibsp->bp() == ap && sibsp->ap() == bp))
                    found = true;
            }
            if (!found) ap->v3fatalSrc("Sibling not found");
        }
    };

    static const GraphWay* s_shortestWaywardCpInclusiveWay;
    static int shortestWaywardCpInclusive(const void* vap, const void* vbp) {
        const GraphWay* wp = s_shortestWaywardCpInclusiveWay;
        const LogicMTask* ap = *reinterpret_cast<const LogicMTask* const *>(vap);
        const LogicMTask* bp = *reinterpret_cast<const LogicMTask* const *>(vbp);
        uint32_t aCp = ap->critPathCost(*wp) + ap->stepCost();
        uint32_t bCp = bp->critPathCost(*wp) + bp->stepCost();
        if (aCp < bCp) { return -1; }
        if (aCp > bCp) { return 1; }
        if (ap->id() < bp->id()) { return -1; }
        if (ap->id() > bp->id()) { return 1; }
        return 0;
    }

    void siblingPairFromRelatives(GraphWay way, V3GraphVertex* mtaskp,
                                  bool exhaustive) {
        std::vector<LogicMTask*> shortestPrereqs;

        for (V3GraphEdge* edgep = mtaskp->beginp(way);
             edgep; edgep = edgep->nextp(way)) {
            LogicMTask* prereqp = dynamic_cast<LogicMTask*>(edgep->furtherp(way));
            shortestPrereqs.push_back(prereqp);
            // Prevent nodes with huge numbers of edges from massively
            // slowing down the partitioner:
            if (shortestPrereqs.size() > PART_SIBLING_EDGE_LIMIT) break;
        }

        if (shortestPrereqs.empty()) return;

        // qsort_r would be nice here, but it isn't portable
        s_shortestWaywardCpInclusiveWay = &way;
        qsort(&shortestPrereqs[0], shortestPrereqs.size(),
              sizeof(LogicMTask*), &shortestWaywardCpInclusive);

        // Don't make all NxN/2 possible pairs of prereqs, that's a lot
        // to cart around. Just make a few pairs.
        std::vector<LogicMTask*>::iterator it = shortestPrereqs.begin();
        for (unsigned i = 0; exhaustive || (i < 3); ++i) {
            if (it == shortestPrereqs.end()) break;
            LogicMTask* ap = *(it++);
            if (it == shortestPrereqs.end()) break;
            LogicMTask* bp = *(it++);
            makeSiblingMC(ap, bp);
        }
    }

    // SELF TESTS

    // This is a performance test, its intent is to demonstrate that the
    // partitioner doesn't run on this chain in N^2 time or worse. Overall
    // runtime should be N*log(N) for a chain-shaped graph.
    //
    static void selfTestChain() {
        vluint64_t usecsSmall = partitionChainUsecs(5);
        vluint64_t usecsLarge = partitionChainUsecs(500);
        // Large input is 50x bigger than small input.
        // Its runtime should be about 10x longer -- not about 2500x longer
        // or worse which would suggest N^2 scaling or worse.
        UASSERT(usecsLarge < (usecsSmall * 1500),
                "selfTestChain() took longer than expected. Small input runtime = "
                <<usecsSmall<<", large input runtime = "<<usecsLarge);
    }

    static vluint64_t partitionChainUsecs(unsigned chain_len) {
        // NOTE: To get a dot file run with --debugi-V3Partition 4 or more.
        vluint64_t startUsecs = V3Os::timeUsecs();
        V3Graph mtasks;
        LogicMTask* lastp = NULL;
        for (unsigned i=0; i<chain_len; ++i) {
            LogicMTask* mtp = new LogicMTask(&mtasks, NULL);
            mtp->setCost(1);
            if (lastp) {
                new MTaskEdge(&mtasks, lastp, mtp, 1);
            }
            lastp = mtp;
        }
        partInitCriticalPaths(&mtasks);

        // Since slowAsserts mode is *expected* to cause N^2 runtime, and the
        // intent of this test is to demonstrate better-than-N^2 runtime, disable
        // slowAsserts.
        PartContraction ec(&mtasks,
                           // Any CP limit >chain_len should work:
                           chain_len * 2,
                           false /* slowAsserts */);
        ec.go();

        PartParallelismEst check(&mtasks);
        check.traverse();

        vluint64_t endUsecs = V3Os::timeUsecs();
        vluint64_t elapsedUsecs = endUsecs - startUsecs;

        if (debug()>=6) {
            UINFO(0, "Chain self test stats:\n");
            check.debugReport();
            UINFO(0, "Elapsed usecs = " << elapsedUsecs << "\n");
        }

        // All vertices should merge into one
        UASSERT_SELFTEST(size_t, check.vertexCount(), 1);
        return elapsedUsecs;
    }

    // This test defends against a particular failure mode that the
    // partitioner exhibited during development:
    //
    // At one time, the partitioner consistently favored edge-merges over
    // equal-scoring sibling merges. Every edge and sibling merge in this
    // test starts out with an equal score. If you only do edge-merges, all
    // possible merges will continue to have equal score as the center node
    // grows and grows. Soon the critical path budget is exhausted by a
    // large center node, and we still have many small leaf nodes -- it's
    // literally the worst partition possible.
    //
    // Now, instead, the partitioner gives slight favoritism to sibling
    // merges in the event that scores are tied. This is better for the
    // test and also real designs.
    static void selfTestX() {
        // NOTE: To get a dot file run with --debugi-V3Partition 4 or more.
        V3Graph mtasks;
        LogicMTask* center = new LogicMTask(&mtasks, NULL);
        center->setCost(1);
        unsigned i;
        for (i=0; i<50; ++i) {
            LogicMTask* mtp = new LogicMTask(&mtasks, NULL);
            mtp->setCost(1);
            // Edge from every input -> center
            new MTaskEdge(&mtasks, mtp, center, 1);
        }
        for (i=0; i<50; ++i) {
            LogicMTask* mtp = new LogicMTask(&mtasks, NULL);
            mtp->setCost(1);
            // Edge from center -> every output
            new MTaskEdge(&mtasks, center, mtp, 1);
        }

        partInitCriticalPaths(&mtasks);
        PartContraction(&mtasks, 20, true).go();

        PartParallelismEst check(&mtasks);
        check.traverse();

        // Checking exact values here is maybe overly precise.  What we're
        // mostly looking for is a healthy reduction in the number of
        // mtasks.
        if (debug()>=5) {
            UINFO(0, "X self test stats:\n");
            check.debugReport();
        }
        UASSERT_SELFTEST(uint32_t, check.longestCritPathCost(), 19);
        UASSERT_SELFTEST(uint32_t, check.totalGraphCost(), 101);
        UASSERT_SELFTEST(uint32_t, check.vertexCount(), 14);
        UASSERT_SELFTEST(uint32_t, check.edgeCount(), 13);
    }
public:
    static void selfTest() {
        selfTestX();
        selfTestChain();
    }

private:
    VL_DEBUG_FUNC;  // Declare debug()
    VL_UNCOPYABLE(PartContraction);
};

const GraphWay* PartContraction::s_shortestWaywardCpInclusiveWay = NULL;

//######################################################################
// DpiImportCallVisitor

// Scan node, indicate whether it contains a call to a DPI imported
// routine.
class DpiImportCallVisitor : public AstNVisitor {
private:
    bool m_hasDpiHazard;  // Found a DPI import call.
    bool m_tracingCall;  // Iterating into a CCall to a CFunc
    // METHODS
    VL_DEBUG_FUNC;

    virtual void visit(AstCFunc* nodep) {
        if (!m_tracingCall) return;
        m_tracingCall = false;
        if (nodep->dpiImportWrapper()) {
            if (nodep->pure() ? !v3Global.opt.threadsDpiPure()
                : !v3Global.opt.threadsDpiUnpure()) {
                m_hasDpiHazard = true;
            }
        }
        iterateChildren(nodep);
    }
    virtual void visit(AstCCall* nodep) {
        iterateChildren(nodep);
        // Enter the function and trace it
        m_tracingCall = true;
        iterate(nodep->funcp());
    }
    virtual void visit(AstNode* nodep) {
        iterateChildren(nodep);
    }

public:
    // CONSTUCTORS
    explicit DpiImportCallVisitor(AstNode* nodep)
        : m_hasDpiHazard(false)
        , m_tracingCall(false) {
        iterate(nodep);
    }
    bool hasDpiHazard() const { return m_hasDpiHazard; }
    virtual ~DpiImportCallVisitor() {}

private:
    VL_UNCOPYABLE(DpiImportCallVisitor);
};

//######################################################################
// PartFixDataHazards

// Fix data hazards in the partition graph.
//
// The fine-grained graph from V3Order may contain data hazards which are
// not a problem for serial mode, but which would be a problem in parallel
// mode.
//
// There are basically two classes: unordered pairs of writes, and
// unordered write-read pairs. We fix both here, with a combination of
// MTask-merges and new edges to ensure no such unordered pairs remain.
//
// ABOUT UNORDERED WRITE-WRITE PAIRS
//
//   The V3Order dependency graph treats these as unordered events:
//
//    a)  sig[15:8] = stuff;
//          ...
//    b)  sig[7:0]  = other_stuff;
//
//   Seems OK right? They are writes to disjoint bits of the same
//   signal. They can run in either order, in serial mode, and the result
//   will be the same.
//
//   The resulting C code for each of this isn't a pure write, it's
//   actually an R-M-W sequence:
//
//    a)  sig = (sig & 0xff)   | (0xff00 & (stuff << 8));
//          ...
//    b)  sig = (sig & 0xff00) | (0xff & other_stuff);
//
//   In serial mode, order doesn't matter so long as these run serially.
//   In parallel mode, we must serialize these RMW's to avoid a race.
//
//   We don't actually check here if each write would involve an R-M-W, we
//   just assume that it would. If this routine ever causes a drastic
//   increase in critical path, it could be optimized to make a better
//   prediction (with all the risk that word implies!) about whether a
//   given write is likely to turn into an R-M-W.
//
// ABOUT UNORDERED WRITE-READ PAIRS
//
//   If we don't put unordered write-read pairs into some order at verilation
//   time, we risk a runtime race.
//
//   How do such unordered writer/reader pairs happen? Here's a partial list
//   of scenarios:
//
//   Case 1: Circular logic
//
//     If the design has circular logic, V3Order has by now generated some
//     dependency cycles, and also cut some of the edges to make it
//     acyclic.
//
//     For serial mode, that was fine. We can break logic circles at an
//     arbitrary point. At runtime, we'll repeat the _eval() until no
//     changes are detected, which papers over the discarded dependency.
//
//     For parallel mode, this situation can lead to unordered reads and
//     writes of the same variable, causing a data race. For example if the
//     original code is this:
//
//       assign b = b | a << 2;
//       assign out = b;
//
//     ... there's originally a dependency edge which records that 'b'
//     depends on the first assign. V3Order may cut this edge, making the
//     statements unordered. In serial mode that's fine, they can run in
//     either order. In parallel mode it's a reader/writer race.
//
//   Case 2: Race Condition in Verilog Sources
//
//     If the input has races, eg. blocking assignments in always blocks
//     that share variables, the graph at this point will contain unordered
//     writes and reads (or unordered write-write pairs) reflecting that.
//
//   Case 3: Interesting V3Order Behavior
//
//     There's code in V3Order that explicitly avoids making a dependency
//     edge from a clock-gater signal to the logic node that produces the
//     clock signal. This leads to unordered reader/writer pairs in
//     parallel mode.
//
class PartFixDataHazards {
private:
    // TYPES
    typedef std::set<LogicMTask*, MTaskIdLessThan> LogicMTaskSet;
    typedef std::map<uint32_t/*rank*/, LogicMTaskSet> TasksByRank;
    typedef std::set<const OrderVarStdVertex*, OrderByPtrId&> OvvSet;
    typedef vl_unordered_map<const OrderLogicVertex*, LogicMTask*> Olv2MTaskMap;

    // MEMBERS
    V3Graph* m_mtasksp;  // Mtask graph
    Olv2MTaskMap m_olv2mtask;  // Map OrderLogicVertex to LogicMTask who wraps it
    unsigned m_mergesDone;  // Number of MTasks merged. For stats only.
public:
    // CONSTRUCTORs
    explicit PartFixDataHazards(V3Graph* mtasksp)
        : m_mtasksp(mtasksp), m_mergesDone(0) {}
    // METHODS
private:
    void findAdjacentTasks(OvvSet::iterator ovvIt, TasksByRank* tasksByRankp) {
        // Find all writer tasks for this variable, group by rank.
        for (V3GraphEdge* edgep = (*ovvIt)->inBeginp();
             edgep; edgep = edgep->inNextp()) {
            OrderLogicVertex* logicp = dynamic_cast<OrderLogicVertex*>(edgep->fromp());
            if (!logicp) continue;
            if (logicp->domainp()->hasInitial()
                || logicp->domainp()->hasSettle()) continue;
            LogicMTask* writerMtaskp = m_olv2mtask.at(logicp);
            (*tasksByRankp)[writerMtaskp->rank()].insert(writerMtaskp);
        }
        // Find all reader tasks for this variable, group by rank.
        for (V3GraphEdge* edgep = (*ovvIt)->outBeginp();
             edgep; edgep = edgep->outNextp()) {
            OrderLogicVertex* logicp = dynamic_cast<OrderLogicVertex*>(edgep->fromp());
            if (!logicp) continue;
            if (logicp->domainp()->hasInitial()
                || logicp->domainp()->hasSettle()) continue;
            LogicMTask* readerMtaskp = m_olv2mtask.at(logicp);
            (*tasksByRankp)[readerMtaskp->rank()].insert(readerMtaskp);
        }
    }
    void mergeSameRankTasks(TasksByRank* tasksByRankp) {
        LogicMTask* lastMergedp = NULL;
        for (TasksByRank::iterator rankIt = tasksByRankp->begin();
             rankIt != tasksByRankp->end(); ++rankIt) {
            // Find the largest node at this rank, merge into it.  (If we
            // happen to find a huge node, this saves time in
            // partMergeEdgesFrom() versus merging into an arbitrary node.)
            LogicMTask* mergedp = NULL;
            for (LogicMTaskSet::iterator it = rankIt->second.begin();
                 it != rankIt->second.end(); ++it) {
                LogicMTask* mtaskp = *it;
                if (mergedp) {
                    if (mergedp->cost() < mtaskp->cost()) {
                        mergedp = mtaskp;
                    }
                } else {
                    mergedp = mtaskp;
                }
            }
            rankIt->second.erase(mergedp);

            while (!rankIt->second.empty()) {
                LogicMTaskSet::iterator begin = rankIt->second.begin();
                LogicMTask* donorp = *begin;
                if (donorp == mergedp) donorp->v3fatalSrc("Donor can't be merged edge");
                rankIt->second.erase(begin);
                // Merge donorp into mergedp.
                // Fix up the map, so donor's OLVs map to mergedp
                for (LogicMTask::VxList::const_iterator tmvit =
                         donorp->vertexListp()->begin();
                     tmvit != donorp->vertexListp()->end(); ++tmvit) {
                    MTaskMoveVertex* tmvp = *tmvit;
                    OrderLogicVertex* logicp = tmvp->logicp();
                    if (logicp) m_olv2mtask[logicp] = mergedp;
                }
                // Move all vertices from donorp to mergedp
                mergedp->moveAllVerticesFrom(donorp);
                // Move edges from donorp to recipientp
                partMergeEdgesFrom(m_mtasksp, mergedp, donorp, NULL);
                // Remove donorp from the graph
                donorp->unlinkDelete(m_mtasksp); VL_DANGLING(donorp);
                m_mergesDone++;
            }

            if (lastMergedp) {
                if (lastMergedp->rank() >= mergedp->rank()) {
                    mergedp->v3fatalSrc("Merging must be on lower rank");
                }
                if (!lastMergedp->hasRelative(GraphWay::FORWARD, mergedp)) {
                    new MTaskEdge(m_mtasksp, lastMergedp, mergedp, 1);
                }
            }
            lastMergedp = mergedp;
        }
    }
    bool hasDpiHazard(LogicMTask* mtaskp) {
        for (LogicMTask::VxList::const_iterator it = mtaskp->vertexListp()->begin();
             it != mtaskp->vertexListp()->end(); ++it) {
            if (!(*it)->logicp()) continue;
            AstNode* nodep = (*it)->logicp()->nodep();
            // NOTE: We don't handle DPI exports. If testbench code calls a
            // DPI-exported function at any time during eval() we may have
            // a data hazard. (Likewise in non-threaded mode if an export
            // messes with an ordered variable we're broken.)

            // Find all calls to DPI-imported functions, we can put those
            // into a serial order at least. That should solve the most
            // likely DPI-related data hazards.
            if (DpiImportCallVisitor(nodep).hasDpiHazard()) {
                return true;
            }
        }
        return false;
    }
public:
    void go() {
        vluint64_t startUsecs = 0;
        if (debug() >= 3) startUsecs = V3Os::timeUsecs();

        // Build an OLV->mtask map and a set of OVVs
        OrderByPtrId ovvOrder;
        OvvSet ovvSet(ovvOrder);
        // OVV's which wrap systemC vars will be handled slightly specially
        OvvSet ovvSetSystemC(ovvOrder);

        for (V3GraphVertex* vxp = m_mtasksp->verticesBeginp();
             vxp; vxp = vxp->verticesNextp()) {
            LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(vxp);
            // Should be only one MTaskMoveVertex in each mtask at this
            // stage, but whatever, write it as a loop:
            for (LogicMTask::VxList::const_iterator it
                     = mtaskp->vertexListp()->begin();
                 it != mtaskp->vertexListp()->end(); ++it) {
                MTaskMoveVertex* tmvp = *it;
                if (OrderLogicVertex* logicp = tmvp->logicp()) {
                    m_olv2mtask[logicp] = mtaskp;
                    // Look at downstream vars.
                    for (V3GraphEdge *edgep = logicp->outBeginp();
                         edgep; edgep = edgep->outNextp()) {
                        // Only consider OrderVarStdVertex which reflects
                        // an actual lvalue assignment; the others do not.
                        OrderVarStdVertex* ovvp
                            = dynamic_cast<OrderVarStdVertex*>(edgep->top());
                        if (!ovvp) continue;
                        if (ovvp->varScp()->varp()->isSc()) {
                            ovvSetSystemC.insert(ovvp);
                        } else {
                            ovvSet.insert(ovvp);
                        }
                    }
                }
            }
        }

        // Rank the graph.
        // DGS is faster than V3GraphAlg's recursive rank, in the worst
        // cases where the recursive rank must pass through the same node
        // many times. (We saw 22s for DGS vs. 500s for recursive rank on
        // one large design.)
        {
            GraphStreamUnordered serialize(m_mtasksp);
            const V3GraphVertex* vertexp;
            while ((vertexp = serialize.nextp())) {
                uint32_t rank = 0;
                for (V3GraphEdge* edgep = vertexp->inBeginp(); edgep;
                     edgep = edgep->inNextp()) {
                    rank = std::max(edgep->fromp()->rank() + 1, rank);
                }
                const_cast<V3GraphVertex*>(vertexp)->rank(rank);
            }
        }

        // For each OrderVarVertex, look at its writer and reader mtasks.
        //
        // If there's a set of writers and readers at the same rank, we
        // know these are unordered with respect to one another, so merge
        // those mtasks all together.
        //
        // At this point, we have at most one merged mtask per rank (for a
        // given OVV.) Create edges across these remaining mtasks to ensure
        // they run in serial order (going along with the existing ranks.)
        //
        // NOTE: we don't update the CP's stored in the LogicMTasks to
        // reflect the changes we make to the graph. That's OK, as we
        // haven't yet initialized CPs when we call this routine.
        for (OvvSet::iterator ovvit = ovvSet.begin();
             ovvit != ovvSet.end(); ++ovvit) {
            // Build a set of mtasks, per rank, which access this var.
            // Within a rank, sort by MTaskID to avoid nondeterminism.
            TasksByRank tasksByRank;

            // Find all reader and writer tasks for this variable, add to
            // tasksByRank.
            findAdjacentTasks(ovvit, &tasksByRank);

            // Merge all writer and reader tasks from same rank together.
            //
            // NOTE: Strictly speaking, we don't need to merge all the
            // readers together. That may lead to extra serialization. The
            // least amount of ordering we could impose here would be to
            // merge all writers at a given rank together; then make edges
            // from the merged writer node to each reader node at the same
            // rank; and then from each reader node to the merged writer at
            // the next rank.
            //
            // Whereas, merging all readers and writers at the same rank
            // together is "the simplest thing that could possibly work"
            // and it seems to.  It also creates fairly few edges. We don't
            // want to create tons of edges here, doing so is not nice to
            // the main edge contraction pass.
            mergeSameRankTasks(&tasksByRank);
        }

        // Handle SystemC vars just a little differently. Instead of
        // treating each var as an independent entity, and serializing
        // writes to that one var, we treat ALL systemC vars as a single
        // entity and serialize writes (and, conservatively, reads) across
        // all of them.
        //
        // Reasoning: writing a systemC var actually turns into a call to a
        // var.write() method, which under the hood is accessing some data
        // structure that's shared by many SC vars. It's not thread safe.
        //
        // Hopefully we only have a few SC vars -- top level ports, probably.
        {
            TasksByRank tasksByRank;
            for (OvvSet::iterator ovvit = ovvSetSystemC.begin();
                 ovvit != ovvSetSystemC.end(); ++ovvit) {
                findAdjacentTasks(ovvit, &tasksByRank);
            }
            mergeSameRankTasks(&tasksByRank);
        }

        // Handle nodes containing DPI calls, we want to serialize those
        // by default unless user gave --threads-dpi-concurrent.
        // Same basic strategy as above to serialize access to SC vars.
        if (!v3Global.opt.threadsDpiPure() || !v3Global.opt.threadsDpiUnpure()) {
            TasksByRank tasksByRank;
            for (V3GraphVertex* vxp = m_mtasksp->verticesBeginp();
                 vxp; vxp = vxp->verticesNextp()) {
                LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(vxp);
                if (hasDpiHazard(mtaskp)) {
                    tasksByRank[vxp->rank()].insert(mtaskp);
                }
            }
            mergeSameRankTasks(&tasksByRank);
        }

        UINFO(4, "PartFixDataHazards() merged "<<m_mergesDone
              <<" pairs of nodes in "<<(V3Os::timeUsecs() - startUsecs)
              <<" usecs.\n");
    }

private:
    VL_UNCOPYABLE(PartFixDataHazards);
    VL_DEBUG_FUNC;
};


//######################################################################
// PartPackMTasks

// Statically pack tasks into threads.
//
// The simplest thing that could possibly work would be to assume that our
// predictions of task runtimes are precise, and that every thread will
// make progress at an equal rate. Simulate a single "clock", pack the the
// highest priority ready task into whatever thread becomes ready earliest,
// repeating until no tasks remain.
//
// That doesn't work well, as our predictions of task runtimes have wide
// error bars (+/- 60% is typical.)
//
// So be a little more clever: let each task have a different end time,
// depending on which thread is looking. Be a little bit pessimistic when
// thread A checks the end time of an mtask running on thread B. This extra
// "padding" avoids tight "layovers" at cross-thread dependencies.
class PartPackMTasks {
private:
    // TYPES
    struct MTaskState {
        uint32_t completionTime;  // Estimated time this mtask will complete
    };
    struct MTaskCmp {
        bool operator() (const ExecMTask* ap, ExecMTask* bp) const {
            return ap->id() < bp->id();
        }
    };

    // MEMBERS
    V3Graph* m_mtasksp;  // Mtask graph
    uint32_t m_nThreads;  // Number of threads
    uint32_t m_sandbagNumerator;  // Numerator padding for est runtime
    uint32_t m_sandbagDenom;  // Denomerator padding for est runtime

    typedef vl_unordered_map<const ExecMTask*, MTaskState> MTaskStateMap;
    MTaskStateMap m_mtaskState;  // State for each mtask.

    MTaskCmp m_mtaskCmp;  // Comparison functor
    typedef std::set<ExecMTask*, MTaskCmp&> ReadyMTasks;
    ReadyMTasks m_ready;  // MTasks ready to be assigned next; all their
    //                    // dependencies are already assigned.

    typedef std::vector<ExecMTask*> MTaskVec;
    MTaskVec m_prevMTask;  // Previous mtask scheduled to each thread.
    std::vector<uint32_t> m_busyUntil;  // Time each thread is occupied until

public:
    // CONSTRUCTORS
    PartPackMTasks(V3Graph* mtasksp,
                   uint32_t nThreads = v3Global.opt.threads(),
                   unsigned sandbagNumerator = 30,
                   unsigned sandbagDenom = 100)
        : m_mtasksp(mtasksp)
        , m_nThreads(nThreads)
        , m_sandbagNumerator(sandbagNumerator)
        , m_sandbagDenom(sandbagDenom)
        , m_ready(m_mtaskCmp) {}
    ~PartPackMTasks() {}

    // METHOS
    uint32_t completionTime(const ExecMTask* mtaskp, uint32_t thread) {
        const MTaskState& state = m_mtaskState[mtaskp];
        UASSERT(mtaskp->thread() != 0xffffffff, "Mtask should have assigned thread");
        if (thread == mtaskp->thread()) {
            // No overhead on native thread
            return state.completionTime;
        }

        // Add some padding to the estimated runtime when looking from
        // another thread
        uint32_t sandbaggedEndTime = state.completionTime
            + (m_sandbagNumerator * mtaskp->cost()) / m_sandbagDenom;

        // If task B is packed after task A on thread 0, don't let thread 1
        // think that A finishes later than thread 0 thinks that B
        // finishes, otherwise we get priority inversions and fail the self
        // test.
        if (mtaskp->packNextp()) {
            uint32_t successorEndTime
                = completionTime(mtaskp->packNextp(), mtaskp->thread());
            if ((sandbaggedEndTime >= successorEndTime)
                && (successorEndTime > 1)) {
                sandbaggedEndTime = successorEndTime - 1;
            }
        }

        UINFO(6, "Sandbagged end time for "<<mtaskp->name()
              <<" on th "<<thread<<" = "<<sandbaggedEndTime<<endl);
        return sandbaggedEndTime;
    }

    void setCompletionTime(ExecMTask* mtaskp, uint32_t time) {
        MTaskState& state = m_mtaskState[mtaskp];
        state.completionTime = time;
    }

    void go() {
        // Build initial ready list
        for (V3GraphVertex* vxp = m_mtasksp->verticesBeginp();
             vxp; vxp = vxp->verticesNextp()) {
            ExecMTask* mtaskp = dynamic_cast<ExecMTask*>(vxp);
            if (vxp->inEmpty()) m_ready.insert(mtaskp);
        }

        m_prevMTask.clear();
        m_prevMTask.resize(m_nThreads);
        m_busyUntil.clear();
        m_busyUntil.resize(m_nThreads);

        while (!m_ready.empty()) {
            // For each task in the ready set, compute when it might start
            // on each thread (in that thread's local time frame.)
            uint32_t bestTime = 0xffffffff;
            uint32_t bestTh = 0;
            ExecMTask* bestMtaskp = NULL;
            for (uint32_t th = 0; th < m_nThreads; ++th) {
                for (ReadyMTasks::iterator taskIt = m_ready.begin();
                     taskIt != m_ready.end(); ++taskIt) {
                    uint32_t timeBegin = m_busyUntil[th];
                    if (timeBegin > bestTime) {
                        UINFO(6, "th "<<th<<" busy until "<<timeBegin
                              <<", later than bestTime "<<bestTime
                              <<", skipping thread.\n");
                        break;
                    }
                    ExecMTask* taskp = *taskIt;
                    for (V3GraphEdge* edgep = taskp->inBeginp();
                         edgep; edgep = edgep->inNextp()) {
                        ExecMTask* priorp
                            = dynamic_cast<ExecMTask*>(edgep->fromp());
                        uint32_t priorEndTime = completionTime(priorp, th);
                        if (priorEndTime > timeBegin) {
                            timeBegin = priorEndTime;
                        }
                    }
                    UINFO(6, "Task "<<taskp->name()
                          <<" start at "<<timeBegin
                          <<" on thread "<<th<<endl);
                    if ((timeBegin < bestTime)
                        || ((timeBegin == bestTime)
                            && (taskp->priority() > bestMtaskp->priority()))) {
                        bestTime = timeBegin;
                        bestTh = th;
                        bestMtaskp = taskp;
                    }
                }
            }

            UINFO(6, "Will schedule "<<bestMtaskp->name()
                  <<" onto thread "<<bestTh<<endl);
            uint32_t bestEndTime = bestTime + bestMtaskp->cost();
            setCompletionTime(bestMtaskp, bestEndTime);

            // Update the ready list
            size_t erased = m_ready.erase(bestMtaskp);
            if (erased <= 0) bestMtaskp->v3fatalSrc("Should have erased something?");
            for (V3GraphEdge* edgeOutp = bestMtaskp->outBeginp();
                 edgeOutp; edgeOutp = edgeOutp->outNextp()) {
                ExecMTask* nextp = dynamic_cast<ExecMTask*>(edgeOutp->top());

                UASSERT(nextp->thread() == 0xffffffff,
                        "Tasks after one being assigned should not be assigned yet");
                // They also should not be ready yet, since they only now
                // may become ready
                if (m_ready.find(nextp) != m_ready.end()) {
                    nextp->v3fatalSrc("Tasks after one being assigned should not be ready");
                }
                bool isReady = true;
                for (V3GraphEdge* edgeInp = nextp->inBeginp();
                     edgeInp; edgeInp = edgeInp->inNextp()) {
                    ExecMTask* priorp = dynamic_cast<ExecMTask*>(edgeInp->fromp());
                    if (priorp == bestMtaskp) continue;
                    if (priorp->thread() == 0xffffffff) {
                        // This prior is not assigned yet
                        isReady = false;
                    }
                }
                if (isReady) {
                    m_ready.insert(nextp);
                    UINFO(6, "Inserted "<<nextp->name()<<" into ready\n");
                }
            }

            // Update the ExecMTask itself
            if (m_prevMTask[bestTh]) {
                m_prevMTask[bestTh]->packNextp(bestMtaskp);
                UINFO(6, "Packing "<<bestMtaskp->name()
                      <<" after "<<m_prevMTask[bestTh]->name()<<endl);
            } else {
                UINFO(6, "Marking "<<bestMtaskp->name()<<" as thread root\n");
                bestMtaskp->threadRoot(true);
            }
            bestMtaskp->thread(bestTh);

            // Update the thread state
            m_prevMTask[bestTh] = bestMtaskp;
            m_busyUntil[bestTh] = bestEndTime;
        }
    }

    // SELF TEST
    static void selfTest() {
        V3Graph graph;
        ExecMTask* t0 = new ExecMTask(&graph, NULL, 0);
        t0->cost(1000);
        t0->priority(1100);
        ExecMTask* t1 = new ExecMTask(&graph, NULL, 1);
        t1->cost(100);
        t1->priority(100);
        ExecMTask* t2 = new ExecMTask(&graph, NULL, 2);
        t2->cost(100);
        t2->priority(100);

        new V3GraphEdge(&graph, t0, t1, 1);
        new V3GraphEdge(&graph, t0, t2, 1);

        PartPackMTasks packer(&graph,
                              2,  // Threads
                              3,  // Sandbag numerator
                              10);  // Sandbag denom
        packer.go();

        UASSERT_SELFTEST(bool, t0->threadRoot(), true);
        UASSERT_SELFTEST(uint32_t, t0->thread(), 0);
        UASSERT_SELFTEST(const void*, t0->packNextp(), t1);

        UASSERT_SELFTEST(uint32_t, t1->thread(), 0);
        UASSERT_SELFTEST(bool, t1->threadRoot(), false);
        UASSERT_SELFTEST(const void*, t1->packNextp(), NULL);

        UASSERT_SELFTEST(uint32_t, t2->thread(), 1);
        UASSERT_SELFTEST(bool, t2->threadRoot(), true);
        UASSERT_SELFTEST(const void*, t2->packNextp(), NULL);

        // On its native thread, we see the actual end time for t0:
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t0, 0), 1000);
        // On the other thread, we see a sandbagged end time which does not
        // exceed the t1 end time:
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t0, 1), 1099);

        // Actual end time on native thread:
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t1, 0), 1100);
        // Sandbagged end time seen on thread 1.  Note it does not compound
        // with t0's sandbagged time; compounding caused trouble in
        // practice.
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t1, 1), 1130);
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t2, 0), 1229);
        UASSERT_SELFTEST(uint32_t, packer.completionTime(t2, 1), 1199);
    }

private:
    VL_DEBUG_FUNC;  // Declare debug()
    VL_UNCOPYABLE(PartPackMTasks);
};

//######################################################################
// V3Partition implementation

void V3Partition::debugMTaskGraphStats(const V3Graph* graphp, const string& stage) {
    if (!debug()) return;

    UINFO(4, "\n");
    UINFO(4, " Stats for "<<stage<<endl);
    uint32_t mtaskCount = 0;
    uint32_t totalCost = 0;
    uint32_t mtaskCostHist[32]; memset(mtaskCostHist, 0, sizeof(mtaskCostHist));

    for (const V3GraphVertex* mtaskp = graphp->verticesBeginp(); mtaskp;
         mtaskp = mtaskp->verticesNextp()) {
        ++mtaskCount;
        uint32_t mtaskCost = dynamic_cast<const AbstractMTask*>(mtaskp)->cost();
        totalCost += mtaskCost;

        unsigned log2Cost = 0;
        while (mtaskCost >>= 1) ++log2Cost;
        UASSERT(log2Cost < 32, "log2Cost overflow in debugMTaskGraphStats");
        ++mtaskCostHist[log2Cost];
    }
    UINFO(4, "  Total mtask cost = "<<totalCost<<"\n");
    UINFO(4, "  Mtask count = "<<mtaskCount<<"\n");
    UINFO(4, "  Avg cost / mtask = "
          << ((mtaskCount > 0)
              ? cvtToStr(totalCost / mtaskCount)
              : "INF!") << "\n");
    UINFO(4, "  Histogram of mtask costs:\n");
    for (unsigned i = 0; i < 32; ++i) {
        if (mtaskCostHist[i]) {
            UINFO(4, "    2^"<<i<<": "<<mtaskCostHist[i]<<endl);
            V3Stats::addStat("MTask graph, "+stage+", mtask cost 2^"
                             +(i<10 ? " ":"")
                             +cvtToStr(i), mtaskCostHist[i]);
        }
    }

    if (mtaskCount < 1000) {
        string filePrefix("ordermv_");
        filePrefix += stage;
        if (debug() >= 4) graphp->dumpDotFilePrefixedAlways(filePrefix);
    }

    // Look only at the cost of each mtask, neglect communication cost.
    // This will show us how much parallelism we expect, assuming cache-miss
    // costs are minor and the cost of running logic is the dominant cost.
    PartParallelismEst vertexParEst(graphp);
    vertexParEst.traverse();
    vertexParEst.statsReport(stage);
    if (debug()>=4) {
        UINFO(0, "\n");
        UINFO(0, "  Parallelism estimate for based on mtask costs:\n");
        vertexParEst.debugReport();
    }
}

// Print a hash of the shape of graphp.  If you are battling
// nondeterminism, this can help to pinpoint where in the pipeline it's
// creeping in.
void V3Partition::hashGraphDebug(const V3Graph* graphp, const char* debugName) {
    // Disabled when there are no nondeterminism issues in flight.
    if (!v3Global.opt.debugNondeterminism()) return;

    vl_unordered_map<const V3GraphVertex*, uint32_t> vx2Id;
    unsigned id = 0;
    for (const V3GraphVertex* vxp = graphp->verticesBeginp();
         vxp; vxp = vxp->verticesNextp()) {
        vx2Id[vxp] = id++;
    }
    unsigned hash = 0;
    for (const V3GraphVertex* vxp = graphp->verticesBeginp();
         vxp; vxp = vxp->verticesNextp()) {
        for (const V3GraphEdge* edgep = vxp->outBeginp();
             edgep; edgep= edgep->outNextp()) {
            const V3GraphVertex* top = edgep->top();
            hash = vx2Id[top] + 31u * hash;  // The K&R hash function
        }
    }
    UINFO(0, "Hash of shape (not contents) of "<<debugName
          <<" = "<<cvtToStr(hash)<<endl);
}

void V3Partition::setupMTaskDeps(V3Graph* mtasksp, const Vx2MTaskMap* vx2mtaskp) {
    // Look at each mtask
    for (V3GraphVertex* itp = mtasksp->verticesBeginp(); itp;
         itp=itp->verticesNextp()) {
        LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(itp);
        const LogicMTask::VxList* vertexListp = mtaskp->vertexListp();

        // For each logic vertex in this mtask, create an mtask-to-mtask
        // edge based on the logic-to-logic edge.
        for (LogicMTask::VxList::const_iterator vit = vertexListp->begin();
             vit != vertexListp->end(); ++vit) {
            for (V3GraphEdge* outp = (*vit)->outBeginp(); outp;
                 outp = outp->outNextp()) {
                UASSERT(outp->weight() > 0, "Mtask not assigned weight");
                const MTaskMoveVertex* top
                    = dynamic_cast<MTaskMoveVertex*>(outp->top());
                UASSERT(top, "MoveVertex not associated to mtask");
                Vx2MTaskMap::const_iterator it = vx2mtaskp->find(top);
                UASSERT(it != vx2mtaskp->end(), "MTask map can't find id");
                LogicMTask* otherMTaskp = it->second;
                UASSERT(otherMTaskp, "NULL other Mtask");
                if (otherMTaskp == mtaskp) mtaskp->v3fatalSrc("Would create a cycle edge");

                // Don't create redundant edges.
                if (mtaskp->hasRelative(GraphWay::FORWARD, otherMTaskp)) {
                    continue;
                }
                new MTaskEdge(mtasksp, mtaskp, otherMTaskp, 1);
            }
        }
    }
}

void V3Partition::go(V3Graph* mtasksp) {
    // Called by V3Order
    hashGraphDebug(m_fineDepsGraphp, "v3partition initial fine-grained deps");

    // Create the first MTasks. Initially, each MTask just wraps one
    // MTaskMoveVertex. Over time, we'll merge MTasks together and
    // eventually each MTask will wrap a large number of MTaskMoveVertices
    // (and the logic nodes therein.)
    uint32_t totalGraphCost = 0;
    {
        // The V3InstrCount within LogicMTask will set user5 on each AST
        // node, to assert that we never count any node twice.
        AstUser5InUse inUser5;
        Vx2MTaskMap vx2mtask;
        for (V3GraphVertex* vxp = m_fineDepsGraphp->verticesBeginp();
             vxp; vxp = vxp->verticesNextp()) {
            MTaskMoveVertex* mtmvVxp = dynamic_cast<MTaskMoveVertex*>(vxp);
            if (!mtmvVxp) vxp->v3fatalSrc("Every vertex here should be an MTaskMoveVertex");

            LogicMTask* mtaskp = new LogicMTask(mtasksp, mtmvVxp);
            vx2mtask[mtmvVxp] = mtaskp;

            totalGraphCost += mtaskp->cost();
        }

        // Create the mtask->mtask dep edges based on vertex deps
        setupMTaskDeps(mtasksp, &vx2mtask);
    }

    V3Partition::debugMTaskGraphStats(mtasksp, "initial");

    // For debug: print out the longest critical path.  This allows us to
    // verify that the costs look reasonable, that we aren't combining
    // nodes that should probably be split, etc.
    if (v3Global.opt.dumpTreeLevel(__FILE__) >= 3) {
        LogicMTask::dumpCpFilePrefixed(mtasksp, "cp");
    }

    // Merge nodes that could present data hazards; see comment within.
    {
        PartFixDataHazards(mtasksp).go();
        V3Partition::debugMTaskGraphStats(mtasksp, "hazards");
        hashGraphDebug(mtasksp, "mtasksp after fixDataHazards()");
    }

    // Setup the critical path into and out of each node.
    partInitCriticalPaths(mtasksp);
    hashGraphDebug(mtasksp, "after partInitCriticalPaths()");

    // Order the graph. We know it's already ranked from fixDataHazards()
    // so we don't need to rank it again.
    //
    // On at least some models, ordering the graph here seems to help
    // performance. (Why? Is it just triggering noise in a lucky direction?
    // Is it just as likely to harm results?)
    //
    // More diversity of models that can build with --threads will
    // eventually tell us. For now keep the order() so we don't forget
    // about it, in case it actually helps.  TODO: get more data and maybe
    // remove this later if it doesn't really help.
    mtasksp->orderPreRanked();

    int targetParFactor = v3Global.opt.threads();
    if (targetParFactor < 2) {
        v3fatalSrc("We should not reach V3Partition when --threads <= 1");
    }

    // Set cpLimit to roughly totalGraphCost / nThreads
    //
    // Actually set it a bit lower, by a hardcoded fudge factor. This
    // results in more smaller mtasks, which helps reduce fragmentation
    // when scheduling them.
    unsigned fudgeNumerator = 3;
    unsigned fudgeDenominator = 5;
    uint32_t cpLimit = ((totalGraphCost * fudgeNumerator)
                        / (targetParFactor * fudgeDenominator));
    UINFO(4, "V3Partition set cpLimit = "<<cpLimit<<endl);

    // Merge MTask nodes together, repeatedly, until the CP budget is
    // reached.  Coarsens the graph, usually by several orders of
    // magnitude.
    //
    // Some tests disable this, hence the test on threadsCoarsen().
    // Coarsening is always enabled in production.
    if (v3Global.opt.threadsCoarsen()) {
        PartContraction(mtasksp, cpLimit,
                        // --debugPartition is used by tests
                        // to enable slow assertions.
                        v3Global.opt.debugPartition()).go();
        V3Partition::debugMTaskGraphStats(mtasksp, "contraction");
    }
    {
        mtasksp->removeTransitiveEdges();
        V3Partition::debugMTaskGraphStats(mtasksp, "transitive1");
    }

    // Reassign MTask IDs onto smaller numbers, which should be more stable
    // across small logic changes.  Keep MTask IDs in the same relative
    // order though, otherwise we break CmpLogicMTask for still-existing
    // EdgeSet's that haven't destructed yet.
    {
        typedef std::set<LogicMTask*, LogicMTask::CmpLogicMTask> SortedMTaskSet;
        SortedMTaskSet sorted;
        for (V3GraphVertex* itp = mtasksp->verticesBeginp(); itp;
             itp = itp->verticesNextp()) {
            LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(itp);
            sorted.insert(mtaskp);
        }
        uint32_t nextId = 1;
        for (SortedMTaskSet::iterator it = sorted.begin();
             it != sorted.end(); ++it) {
            // We shouldn't perturb the sort order of the set, despite
            // changing the IDs, they should all just remain in the same
            // relative order. Confirm that:
            UASSERT(nextId <= (*it)->id(), "Should only shrink MTaskIDs here");
            UINFO(4, "Reassigning MTask id " << (*it)->id()
                  << " to id " << nextId << "\n");
            (*it)->id(nextId);
            nextId++;
        }
    }

    // Set color to indicate an mtaskId on every underlying MTaskMoveVertex.
    for (V3GraphVertex* itp = mtasksp->verticesBeginp(); itp;
         itp = itp->verticesNextp()) {
        LogicMTask* mtaskp = dynamic_cast<LogicMTask*>(itp);
        for (LogicMTask::VxList::const_iterator it
                 = mtaskp->vertexListp()->begin();
             it != mtaskp->vertexListp()->end(); ++it) {
            MTaskMoveVertex* mvertexp = *it;
            mvertexp->color(mtaskp->id());
        }
    }
}

void V3Partition::finalizeCosts(V3Graph* execMTaskGraphp) {
    GraphStreamUnordered ser(execMTaskGraphp, GraphWay::REVERSE);

    while (const V3GraphVertex* vxp = ser.nextp()) {
        ExecMTask* mtp = dynamic_cast<ExecMTask*>(const_cast<V3GraphVertex*>(vxp));
        uint32_t costCount = V3InstrCount::count(mtp->bodyp(), false);
        mtp->cost(costCount);
        mtp->priority(costCount);

        // "Priority" is the critical path from the start of the mtask, to
        // the end of the graph reachable from this mtask.  Given the
        // choice among several ready mtasks, we'll want to start the
        // highest priority one first, so we're always working on the "long
        // pole"
        for (V3GraphEdge* edgep = mtp->outBeginp();
             edgep; edgep = edgep->outNextp()) {
            ExecMTask* followp = dynamic_cast<ExecMTask*>(edgep->top());
            if ((followp->priority() + mtp->cost()) > mtp->priority()) {
                mtp->priority(followp->priority() + mtp->cost());
            }
        }
    }

    // Some MTasks may now have zero cost, eliminate those.
    // (It's common for tasks to shrink to nothing when V3LifePost
    // removes dly assignments.)
    for (V3GraphVertex* vxp = execMTaskGraphp->verticesBeginp(); vxp; ) {
        ExecMTask* mtp = dynamic_cast<ExecMTask*>(vxp);
        vxp = vxp->verticesNextp();  // Advance before delete

        // Don't rely on checking mtp->cost() == 0 to detect an empty task.
        // Our cost-estimating logic is just an estimate. Instead, check
        // the MTaskBody to see if it's empty. That's the source of truth.
        AstMTaskBody* bodyp = mtp->bodyp();
        if (!bodyp->stmtsp()) {  // Kill this empty mtask
            UINFO(6, "Removing zero-cost "<<mtp->name()<<endl);
            for (V3GraphEdge* inp = mtp->inBeginp();
                 inp; inp = inp->inNextp()) {
                for (V3GraphEdge* outp = mtp->outBeginp();
                     outp; outp = outp->outNextp()) {
                    new V3GraphEdge(execMTaskGraphp, inp->fromp(),
                                    outp->top(), 1);
                }
            }
            mtp->unlinkDelete(execMTaskGraphp); VL_DANGLING(mtp);
            // Also remove and delete the AstMTaskBody, otherwise it would
            // keep a dangling pointer to the ExecMTask.
            bodyp->unlinkFrBack()->deleteTree(); VL_DANGLING(bodyp);
        }
    }

    // Removing tasks may cause edges that were formerly non-transitive to
    // become transitive. Also we just created new edges around the removed
    // tasks, which could be transitive. Prune out all transitive edges.
    {
        execMTaskGraphp->removeTransitiveEdges();
        V3Partition::debugMTaskGraphStats(execMTaskGraphp,
                                          "transitive2");
    }

    // Record summary stats for final m_tasks graph.
    // (More verbose stats are available with --debugi-V3Partition >= 3.)
    PartParallelismEst parEst(execMTaskGraphp);
    parEst.traverse();
    parEst.statsReport("final");
    if (debug() >= 3) {
        UINFO(0,"  Final mtask parallelism report:\n");
        parEst.debugReport();
    }
}

void V3Partition::finalize() {
    // Called by Verilator top stage
    AstExecGraph* execGraphp = v3Global.rootp()->execGraphp();
    UASSERT(execGraphp, "Couldn't find AstExecGraph singleton.");

    // Back in V3Order, we partitioned mtasks using provisional cost
    // estimates. However, V3Order precedes some optimizations (notably
    // V3LifePost) that can change the cost of logic within each mtask.
    // Now that logic is final, recompute the cost and priority of each
    // ExecMTask.
    finalizeCosts(execGraphp->mutableDepGraphp());

    // "Pack" the mtasks: statically associate each mtask with a thread,
    // and determine the order in which each thread will runs its mtasks.
    PartPackMTasks(execGraphp->mutableDepGraphp()).go();
}

void V3Partition::selfTest() {
    PartPropagateCpSelfTest::selfTest();
    PartPackMTasks::selfTest();
    PartContraction::selfTest();
}