diff --git a/src/V3Ast.h b/src/V3Ast.h index 1302c1e76..4d1880ffd 100644 --- a/src/V3Ast.h +++ b/src/V3Ast.h @@ -1334,25 +1334,27 @@ inline std::ostream& operator<<(std::ostream& os, const VNRelinker& rhs) { return os; } -//###################################################################### -// Callback base class to determine if node matches some formula +// ###################################################################### +// Callback base class to determine if node matches some formula class VNodeMatcher VL_NOT_FINAL { public: virtual bool nodeMatch(const AstNode* nodep) const { return true; } }; -//###################################################################### -// AstNode -- Base type of all Ast types +// ###################################################################### +// AstNode -- Base type of all Ast types // Prefetch a node. +#define ASTNODE_PREFETCH_NON_NULL(nodep) \ + do { \ + VL_PREFETCH_RD(&((nodep)->m_nextp)); \ + VL_PREFETCH_RD(&((nodep)->m_type)); \ + } while (false) // The if() makes it faster, even though prefetch won't fault on null pointers #define ASTNODE_PREFETCH(nodep) \ do { \ - if (nodep) { \ - VL_PREFETCH_RD(&((nodep)->m_nextp)); \ - VL_PREFETCH_RD(&((nodep)->m_type)); \ - } \ + if (nodep) ASTNODE_PREFETCH_NON_NULL(nodep); \ } while (false) class AstNode VL_NOT_FINAL { @@ -1859,12 +1861,6 @@ private: // Note: specializations for particular node types are provided by 'astgen' template inline static bool privateTypeTest(const AstNode* nodep); - // For internal use only. - // Note: specializations for particular node types are provided below - template inline static bool privateMayBeUnder(const AstNode* nodep) { - return true; - } - // For internal use only. template constexpr static bool uselessCast() { using NonRef = typename std::remove_reference::type; @@ -1923,102 +1919,39 @@ public: // Predicate that returns true if the given 'nodep' might have a descendant of type 'T_Node'. // This is conservative and is used to speed up traversals. - template inline static bool mayBeUnder(const AstNode* nodep) { + // Note: specializations for particular node types are provided below + template static bool mayBeUnder(const AstNode* nodep) { static_assert(!std::is_const::value, "Type parameter 'T_Node' should not be const qualified"); static_assert(std::is_base_of::value, "Type parameter 'T_Node' must be a subtype of AstNode"); - return privateMayBeUnder(nodep); + return true; + } + + // Predicate that is true for node subtypes 'T_Node' that do not have any children + // This is conservative and is used to speed up traversals. + // Note: specializations for particular node types are provided below + template static constexpr bool isLeaf() { + static_assert(!std::is_const::value, + "Type parameter 'T_Node' should not be const qualified"); + static_assert(std::is_base_of::value, + "Type parameter 'T_Node' must be a subtype of AstNode"); + return false; } private: - template - static void foreachImpl( - // Using std::conditional for const correctness in the public 'foreach' functions - typename std::conditional::value, const AstNode*, AstNode*>::type - nodep, - std::function f) { + // Using std::conditional for const correctness in the public 'foreach' functions + template + using ConstCorrectAstNode = + typename std::conditional::value, const AstNode, AstNode>::type; - // Note: Using a loop to iterate the nextp() chain, instead of tail recursion, because - // debug builds don't eliminate tail calls, causing stack overflow on long lists of nodes. - do { - // Prefetch children and next - ASTNODE_PREFETCH(nodep->op1p()); - ASTNODE_PREFETCH(nodep->op2p()); - ASTNODE_PREFETCH(nodep->op3p()); - ASTNODE_PREFETCH(nodep->op4p()); - if VL_CONSTEXPR_CXX17 (VisitNext) ASTNODE_PREFETCH(nodep->nextp()); + template + inline static void foreachImpl(ConstCorrectAstNode* nodep, + const std::function& f, bool visitNext); - // Apply function in pre-order - if (privateTypeTest::type>(nodep)) { - f(static_cast(nodep)); - } - - // Traverse children (including their 'nextp()' chains), unless futile - if (mayBeUnder::type>(nodep)) { - if (AstNode* const op1p = nodep->op1p()) foreachImpl(op1p, f); - if (AstNode* const op2p = nodep->op2p()) foreachImpl(op2p, f); - if (AstNode* const op3p = nodep->op3p()) foreachImpl(op3p, f); - if (AstNode* const op4p = nodep->op4p()) foreachImpl(op4p, f); - } - - // Traverse 'nextp()' chain if requested - if VL_CONSTEXPR_CXX17 (VisitNext) { - nodep = nodep->nextp(); - } else { - break; - } - } while (nodep); - } - - template - static bool predicateImpl( - // Using std::conditional for const correctness in the public 'foreach' functions - typename std::conditional::value, const AstNode*, AstNode*>::type - nodep, - std::function p) { - - // Note: Using a loop to iterate the nextp() chain, instead of tail recursion, because - // debug builds don't eliminate tail calls, causing stack overflow on long lists of nodes. - do { - // Prefetch children and next - ASTNODE_PREFETCH(nodep->op1p()); - ASTNODE_PREFETCH(nodep->op2p()); - ASTNODE_PREFETCH(nodep->op3p()); - ASTNODE_PREFETCH(nodep->op4p()); - if VL_CONSTEXPR_CXX17 (VisitNext) ASTNODE_PREFETCH(nodep->nextp()); - - // Apply function in pre-order - if (privateTypeTest::type>(nodep)) { - if (p(static_cast(nodep)) != Default) return !Default; - } - - // Traverse children (including their 'nextp()' chains), unless futile - if (mayBeUnder::type>(nodep)) { - if (AstNode* const op1p = nodep->op1p()) { - if (predicateImpl(op1p, p) != Default) return !Default; - } - if (AstNode* const op2p = nodep->op2p()) { - if (predicateImpl(op2p, p) != Default) return !Default; - } - if (AstNode* const op3p = nodep->op3p()) { - if (predicateImpl(op3p, p) != Default) return !Default; - } - if (AstNode* const op4p = nodep->op4p()) { - if (predicateImpl(op4p, p) != Default) return !Default; - } - } - - // Traverse 'nextp()' chain if requested - if VL_CONSTEXPR_CXX17 (VisitNext) { - nodep = nodep->nextp(); - } else { - break; - } - } while (nodep); - - return Default; - } + template + inline static bool predicateImpl(ConstCorrectAstNode* nodep, + const std::function& p); template constexpr static bool checkTypeParameter() { static_assert(!std::is_const::value, @@ -2030,31 +1963,32 @@ private: public: // Traverse subtree and call given function 'f' in pre-order on each node that has type - // 'T_Node'. Prefer 'foreach' over simple VNVisitor that only needs to handle a single (or a - // few) node types, as it's easier to write, but more importantly, the dispatch to the - // operation function in 'foreach' should be completely predictable by branch target caches in - // modern CPUs, while it is basically unpredictable for VNVisitor. + // 'T_Node'. The node passd to the function 'f' can be removed or replaced, but other editing + // of the iterated tree is not safe. Prefer 'foreach' over simple VNVisitor that only needs to + // handle a single (or a few) node types, as it's easier to write, but more importantly, the + // dispatch to the operation function in 'foreach' should be completely predictable by branch + // target caches in modern CPUs, while it is basically unpredictable for VNVisitor. template void foreach (std::function f) { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - foreachImpl(this, f); + foreachImpl(this, f, /* visitNext: */ false); } // Same as above, but for 'const' nodes template void foreach (std::function f) const { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - foreachImpl(this, f); + foreachImpl(this, f, /* visitNext: */ false); } // Same as 'foreach' but also follows 'this->nextp()' template void foreachAndNext(std::function f) { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - foreachImpl(this, f); + foreachImpl(this, f, /* visitNext: */ true); } // Same as 'foreach' but also follows 'this->nextp()' template void foreachAndNext(std::function f) const { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - foreachImpl(this, f); + foreachImpl(this, f, /* visitNext: */ true); } // Given a predicate function 'p' return true if and only if there exists a node of type @@ -2063,13 +1997,13 @@ public: // result can be determined. template bool exists(std::function p) { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - return predicateImpl(this, p); + return predicateImpl(this, p); } // Same as above, but for 'const' nodes template void exists(std::function p) const { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - return predicateImpl(this, p); + return predicateImpl(this, p); } // Given a predicate function 'p' return true if and only if all nodes of type @@ -2078,13 +2012,13 @@ public: // result can be determined. template bool forall(std::function p) { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - return predicateImpl(this, p); + return predicateImpl(this, p); } // Same as above, but for 'const' nodes template void forall(std::function p) const { static_assert(checkTypeParameter(), "Invalid type parameter 'T_Node'"); - return predicateImpl(this, p); + return predicateImpl(this, p); } int nodeCount() const { @@ -2098,22 +2032,196 @@ public: // Specialisations of privateTypeTest #include "V3Ast__gen_impl.h" // From ./astgen -// Specializations of privateMayBeUnder -template <> inline bool AstNode::privateMayBeUnder(const AstNode* nodep) { +// Specializations of AstNode::mayBeUnder +template <> inline bool AstNode::mayBeUnder(const AstNode* nodep) { return !VN_IS(nodep, NodeStmt) && !VN_IS(nodep, NodeMath); } -template <> inline bool AstNode::privateMayBeUnder(const AstNode* nodep) { +template <> inline bool AstNode::mayBeUnder(const AstNode* nodep) { return !VN_IS(nodep, NodeMath); } -template <> inline bool AstNode::privateMayBeUnder(const AstNode* nodep) { +template <> inline bool AstNode::mayBeUnder(const AstNode* nodep) { return !VN_IS(nodep, NodeStmt) && !VN_IS(nodep, NodeMath); } -template <> inline bool AstNode::privateMayBeUnder(const AstNode* nodep) { +template <> inline bool AstNode::mayBeUnder(const AstNode* nodep) { if (VN_IS(nodep, ExecGraph)) return false; // Should not nest if (VN_IS(nodep, NodeStmt)) return false; // Should be directly under CFunc return true; } +// Specializations of AstNode::isLeaf +template <> constexpr bool AstNode::isLeaf() { return true; } +template <> constexpr bool AstNode::isLeaf() { return true; } +template <> constexpr bool AstNode::isLeaf() { return true; } + +// foreach implementation +template +void AstNode::foreachImpl(ConstCorrectAstNode* nodep, const std::function& f, + bool visitNext) { + // Checking the function is bound up front eliminates this check from the loop at invocation + if (!f) { + nodep->v3fatal("AstNode::foreach called with unbound function"); // LCOV_EXCL_LINE + } else { + // Pre-order traversal implemented directly (without recursion) for speed reasons. The very + // first iteration (the one that operates on the input nodep) is special, as we might or + // might not need to enqueue nodep->nextp() depending on VisitNext, while in all other + // iterations, we do want to enqueue nodep->nextp(). Duplicating code (via + // 'foreachImplVisit') for the initial iteration here to avoid an extra branch in the loop + + using T_Arg_NonConst = typename std::remove_const::type; + using Node = ConstCorrectAstNode; + + // Traversal stack + std::vector stack; // Kept as a vector for easy resizing + Node** basep = nullptr; // Pointer to base of stack + Node** topp = nullptr; // Pointer to top of stack + Node** limp = nullptr; // Pointer to stack limit (when need growing) + + // We prefetch this far into the stack + constexpr int prefetchDistance = 2; + + // Grow stack to given size + const auto grow = [&](size_t size) VL_ATTR_ALWINLINE { + const ptrdiff_t occupancy = topp - basep; + stack.resize(size); + basep = stack.data() + prefetchDistance; + topp = basep + occupancy; + limp = basep + size - 5; // We push max 5 items per iteration + }; + + // Initial stack size + grow(32); + + // We want some non-null pointers at the beginning. These will be prefetched, but not + // visited, so the root node will suffice. This eliminates needing branches in the loop. + for (int i = -prefetchDistance; i; ++i) basep[i] = nodep; + + // Visit given node, enqueue children for traversal + const auto visit = [&](Node* currp) VL_ATTR_ALWINLINE { + // Type test this node + if (AstNode::privateTypeTest(currp)) { + // Call the client function + f(static_cast(currp)); + // Short circuit if iterating leaf nodes + if VL_CONSTEXPR_CXX17 (isLeaf()) return; + } + + // Enqueue children for traversal, unless futile + if (mayBeUnder(currp)) { + if (AstNode* const op4p = currp->op4p()) *topp++ = op4p; + if (AstNode* const op3p = currp->op3p()) *topp++ = op3p; + if (AstNode* const op2p = currp->op2p()) *topp++ = op2p; + if (AstNode* const op1p = currp->op1p()) *topp++ = op1p; + } + }; + + // Enqueue the next of the root node, if required + if (visitNext && nodep->nextp()) *topp++ = nodep->nextp(); + + // Visit the root node + visit(nodep); + + // Visit the rest of the tree + while (VL_LIKELY(topp > basep)) { + // Pop next node in the traversal + Node* const headp = *--topp; + + // Prefetch in case we are ascending the tree + ASTNODE_PREFETCH_NON_NULL(topp[-prefetchDistance]); + + // Ensure we have stack space for nextp and the 4 children + if (VL_UNLIKELY(topp >= limp)) grow(stack.size() * 2); + + // Enqueue the next node + if (headp->nextp()) *topp++ = headp->nextp(); + + // Visit the head node + visit(headp); + } + } +} + +// predicate implementation +template +bool AstNode::predicateImpl(ConstCorrectAstNode* nodep, + const std::function& p) { + // Implementation similar to foreach, but abort traversal as soon as result is determined. + if (!p) { + nodep->v3fatal("AstNode::foreach called with unbound function"); // LCOV_EXCL_LINE + } else { + using T_Arg_NonConst = typename std::remove_const::type; + using Node = ConstCorrectAstNode; + + // Traversal stack + std::vector stack; // Kept as a vector for easy resizing + Node** basep = nullptr; // Pointer to base of stack + Node** topp = nullptr; // Pointer to top of stack + Node** limp = nullptr; // Pointer to stack limit (when need growing) + + // We prefetch this far into the stack + constexpr int prefetchDistance = 2; + + // Grow stack to given size + const auto grow = [&](size_t size) VL_ATTR_ALWINLINE { + const ptrdiff_t occupancy = topp - basep; + stack.resize(size); + basep = stack.data() + prefetchDistance; + topp = basep + occupancy; + limp = basep + size - 5; // We push max 5 items per iteration + }; + + // Initial stack size + grow(32); + + // We want some non-null pointers at the beginning. These will be prefetched, but not + // visited, so the root node will suffice. This eliminates needing branches in the loop. + for (int i = -prefetchDistance; i; ++i) basep[i] = nodep; + + // Visit given node, enqueue children for traversal, return true if result determined. + const auto visit = [&](Node* currp) VL_ATTR_ALWINLINE { + // Type test this node + if (AstNode::privateTypeTest(currp)) { + // Call the client function + if (p(static_cast(currp)) != Default) return true; + // Short circuit if iterating leaf nodes + if VL_CONSTEXPR_CXX17 (isLeaf()) return false; + } + + // Enqueue children for traversal, unless futile + if (mayBeUnder(currp)) { + if (AstNode* const op4p = currp->op4p()) *topp++ = op4p; + if (AstNode* const op3p = currp->op3p()) *topp++ = op3p; + if (AstNode* const op2p = currp->op2p()) *topp++ = op2p; + if (AstNode* const op1p = currp->op1p()) *topp++ = op1p; + } + + return false; + }; + + // Visit the root node + if (visit(nodep)) return !Default; + + // Visit the rest of the tree + while (VL_LIKELY(topp > basep)) { + // Pop next node in the traversal + Node* const headp = *--topp; + + // Prefetch in case we are ascending the tree + ASTNODE_PREFETCH_NON_NULL(topp[-prefetchDistance]); + + // Ensure we have stack space for nextp and the 4 children + if (VL_UNLIKELY(topp >= limp)) grow(stack.size() * 2); + + // Enqueue the next node + if (headp->nextp()) *topp++ = headp->nextp(); + + // Visit the head node + if (visit(headp)) return !Default; + } + + return Default; + } +} + inline std::ostream& operator<<(std::ostream& os, const AstNode* rhs) { if (!rhs) { os << "nullptr"; diff --git a/src/V3Force.cpp b/src/V3Force.cpp index 137f74fde..2c21b842c 100644 --- a/src/V3Force.cpp +++ b/src/V3Force.cpp @@ -151,14 +151,14 @@ class ForceConvertVisitor final : public VNVisitor { // referenced AstVarScope with the given function. void transformWritenVarScopes(AstNode* nodep, std::function f) { UASSERT_OBJ(nodep->backp(), nodep, "Must have backp, otherwise will be lost if replaced"); - nodep->foreach([this, &f](AstNodeVarRef* refp) { + nodep->foreach([&f](AstNodeVarRef* refp) { if (refp->access() != VAccess::WRITE) return; // TODO: this is not strictly speaking safe for some complicated lvalues, eg.: // 'force foo[a(cnt)] = 1;', where 'cnt' is an out parameter, but it will // do for now... refp->replaceWith( new AstVarRef{refp->fileline(), f(refp->varScopep()), VAccess::WRITE}); - pushDeletep(refp); + VL_DO_DANGLING(refp->deleteTree(), refp); }); } @@ -238,7 +238,7 @@ class ForceConvertVisitor final : public VNVisitor { flp->warnOff(V3ErrorCode::BLKANDNBLK, true); AstVarRef* const newpRefp = new AstVarRef{flp, newVscp, VAccess::WRITE}; refp->replaceWith(newpRefp); - pushDeletep(refp); + VL_DO_DANGLING(refp->deleteTree(), refp); }); // Replace write refs on RHS resetRdp->rhsp()->foreach([this](AstNodeVarRef* refp) { @@ -249,7 +249,7 @@ class ForceConvertVisitor final : public VNVisitor { AstVarRef* const newpRefp = new AstVarRef{refp->fileline(), newVscp, VAccess::READ}; newpRefp->user2(1); // Don't replace this read ref with the read signal refp->replaceWith(newpRefp); - pushDeletep(refp); + VL_DO_DANGLING(refp->deleteTree(), refp); }); resetEnp->addNext(resetRdp);