From 78e659a142ca0a1fe77d78e936c78c6bd3a52a9f Mon Sep 17 00:00:00 2001 From: Geza Lore Date: Sat, 24 Sep 2022 16:57:42 +0100 Subject: [PATCH] Reduce size of FileLine Multiple tricks to reduce the size of class FileLine from 72 to 40 bytes: - Reduce file name index from 32 to 16 bits. This still allows 64K unique input files, which is hopefully enough. - Intern message/warning enable bitset and use a 16-bit index, again allowing 64K unique sets which is hopefully enough. - Put the m_waive flag into the sign bit of one of the line numbers. - Use explicit reference counting to avoid overhead of shared_ptr. Added assertions to ensure interned data fits within it's index space. This saves ~5-10% peak memory consumption at no measurable run-time cost on various designs. --- src/V3FileLine.cpp | 100 ++++++++++++++++++++----------- src/V3FileLine.h | 145 +++++++++++++++++++++++++++++++-------------- src/V3Param.cpp | 2 +- 3 files changed, 167 insertions(+), 80 deletions(-) diff --git a/src/V3FileLine.cpp b/src/V3FileLine.cpp index b313b2063..2e1c76afe 100644 --- a/src/V3FileLine.cpp +++ b/src/V3FileLine.cpp @@ -31,6 +31,7 @@ #include #include +#include #include VL_DEFINE_DEBUG_FUNCTIONS; @@ -38,7 +39,7 @@ VL_DEFINE_DEBUG_FUNCTIONS; //###################################################################### // FileLineSingleton class functions -string FileLineSingleton::filenameLetters(int fileno) { +string FileLineSingleton::filenameLetters(fileNameIdx_t fileno) { constexpr int size = 1 + (64 / 4); // Each letter retires more than 4 bits of a > 64 bit number char out[size]; @@ -60,14 +61,18 @@ string FileLineSingleton::filenameLetters(int fileno) { //! We associate a language with each source file, so we also set the default //! for this. -int FileLineSingleton::nameToNumber(const string& filename) { - const auto it = vlstd::as_const(m_namemap).find(filename); - if (VL_LIKELY(it != m_namemap.end())) return it->second; - const int num = m_names.size(); - m_names.push_back(filename); - m_languages.push_back(V3LangCode::mostRecent()); - m_namemap.emplace(filename, num); - return num; +FileLineSingleton::fileNameIdx_t FileLineSingleton::nameToNumber(const string& filename) { + const auto pair = m_namemap.emplace(filename, 0); + fileNameIdx_t& idx = pair.first->second; + if (pair.second) { + const size_t nextIdx = m_names.size(); + UASSERT(nextIdx <= std::numeric_limits::max(), + "Too many input files (" + cvtToStr(nextIdx) + "+)."); + idx = static_cast(nextIdx); + m_names.push_back(filename); + m_languages.push_back(V3LangCode::mostRecent()); + } + return idx; } //! Support XML output @@ -82,8 +87,46 @@ void FileLineSingleton::fileNameNumMapDumpXml(std::ostream& os) { os << "\n"; } -//###################################################################### -// VFileContents class functions +FileLineSingleton::msgEnSetIdx_t FileLineSingleton::addMsgEnBitSet(const MsgEnBitSet& bitSet) { + const auto pair = m_internedMsgEnIdxs.emplace(bitSet, 0); + msgEnSetIdx_t& idx = pair.first->second; + if (pair.second) { + const size_t nextIdx = m_internedMsgEns.size(); + UASSERT(nextIdx <= std::numeric_limits::max(), + "Too many unique message enable sets (" + cvtToStr(nextIdx) + "+)."); + idx = static_cast(nextIdx); + m_internedMsgEns.push_back(bitSet); + } + return idx; +} + +FileLineSingleton::msgEnSetIdx_t FileLineSingleton::defaultMsgEnIndex() { + MsgEnBitSet msgEnBitSet; + for (int i = V3ErrorCode::EC_MIN; i < V3ErrorCode::_ENUM_MAX; ++i) { + msgEnBitSet.set(i, !V3ErrorCode{i}.defaultsOff()); + } + return addMsgEnBitSet(msgEnBitSet); +} + +FileLineSingleton::msgEnSetIdx_t FileLineSingleton::msgEnSetBit(msgEnSetIdx_t setIdx, + size_t bitIdx, bool value) { + if (msgEn(setIdx).test(bitIdx) == value) return setIdx; + MsgEnBitSet msgEnBitSet{msgEn(setIdx)}; + msgEnBitSet.set(bitIdx, value); + return addMsgEnBitSet(msgEnBitSet); +} + +FileLineSingleton::msgEnSetIdx_t FileLineSingleton::msgEnAnd(msgEnSetIdx_t lhsIdx, + msgEnSetIdx_t rhsIdx) { + MsgEnBitSet msgEnBitSet{msgEn(lhsIdx)}; + msgEnBitSet &= msgEn(rhsIdx); + if (msgEnBitSet == msgEn(lhsIdx)) return lhsIdx; + if (msgEnBitSet == msgEn(rhsIdx)) return rhsIdx; + return addMsgEnBitSet(msgEnBitSet); +} + +// ###################################################################### +// VFileContents class functions void VFileContent::pushText(const string& text) { if (m_lines.size() == 0) { @@ -136,22 +179,17 @@ std::ostream& operator<<(std::ostream& os, VFileContent* contentp) { return os; } -//###################################################################### -// FileLine class functions +// ###################################################################### +// FileLine class functions -// Sort of a singleton -FileLine::FileLine(FileLine::EmptySecret) { - m_filenameno = singleton().nameToNumber(FileLine::builtInFilename()); - - m_warnOn = 0; - for (int codei = V3ErrorCode::EC_MIN; codei < V3ErrorCode::_ENUM_MAX; codei++) { - const V3ErrorCode code{codei}; - warnOff(code, code.defaultsOff()); - } +FileLine::~FileLine() { + if (m_contentp) VL_DO_DANGLING(m_contentp->refDec(), m_contentp); } void FileLine::newContent() { - m_contentp = std::make_shared(); + if (m_contentp) VL_DO_DANGLING(m_contentp->refDec(), m_contentp); + m_contentp = new VFileContent; + m_contentp->refInc(); m_contentLineno = 1; } @@ -318,24 +356,16 @@ void FileLine::warnStyleOff(bool flag) { } bool FileLine::warnIsOff(V3ErrorCode code) const { - if (!m_warnOn.test(code)) return true; - if (!defaultFileLine().m_warnOn.test(code)) return true; // Global overrides local + if (!msgEn().test(code)) return true; + if (!defaultFileLine().msgEn().test(code)) return true; // Global overrides local // UNOPTFLAT implies UNOPT - if (code == V3ErrorCode::UNOPT && !m_warnOn.test(V3ErrorCode::UNOPTFLAT)) return true; - if ((code.lintError() || code.styleError()) && !m_warnOn.test(V3ErrorCode::I_LINT)) { + if (code == V3ErrorCode::UNOPT && !msgEn().test(V3ErrorCode::UNOPTFLAT)) return true; + if ((code.lintError() || code.styleError()) && !msgEn().test(V3ErrorCode::I_LINT)) { return true; } return false; } -void FileLine::modifyStateInherit(const FileLine* fromp) { - // Any warnings that are off in "from", become off in "this". - for (int codei = V3ErrorCode::EC_MIN; codei < V3ErrorCode::_ENUM_MAX; codei++) { - const V3ErrorCode code{codei}; - if (fromp->warnIsOff(code)) warnOff(code, true); - } -} - void FileLine::v3errorEnd(std::ostringstream& sstr, const string& extra) { std::ostringstream nsstr; if (lastLineno()) nsstr << this; diff --git a/src/V3FileLine.h b/src/V3FileLine.h index e3cd40fc4..c8a44a0bf 100644 --- a/src/V3FileLine.h +++ b/src/V3FileLine.h @@ -23,14 +23,17 @@ #include "V3Error.h" #include "V3LangCode.h" +#include #include #include #include #include #include #include +#include +#include -//###################################################################### +// ###################################################################### class FileLine; @@ -39,68 +42,109 @@ class FileLine; //! This singleton class contains tables of data that are unchanging in each //! source file (each with its own unique filename number). class FileLineSingleton final { + friend class FileLine; + + // TYPES + using fileNameIdx_t = uint16_t; // Increase width if 64K input files are not enough + using msgEnSetIdx_t = uint16_t; // Increase width if 64K unique message sets are not enough + using MsgEnBitSet = std::bitset; + // MEMBERS - std::map m_namemap; // filenameno for each filename + std::map m_namemap; // filenameno for each filename std::deque m_names; // filename text for each filenameno std::deque m_languages; // language for each filenameno + + // Map from flag set to the index in m_internedMsgEns for interning + std::unordered_map m_internedMsgEnIdxs; + // Interned message enablement flag sets + std::vector m_internedMsgEns; + // CONSTRUCTORS FileLineSingleton() = default; ~FileLineSingleton() = default; -protected: - friend class FileLine; - int nameToNumber(const string& filename); - string numberToName(int filenameno) const { return m_names[filenameno]; } - V3LangCode numberToLang(int filenameno) const { return m_languages[filenameno]; } - void numberToLang(int filenameno, const V3LangCode& l) { m_languages[filenameno] = l; } + fileNameIdx_t nameToNumber(const string& filename); + string numberToName(fileNameIdx_t filenameno) const { return m_names[filenameno]; } + V3LangCode numberToLang(fileNameIdx_t filenameno) const { return m_languages[filenameno]; } + void numberToLang(fileNameIdx_t filenameno, const V3LangCode& l) { + m_languages[filenameno] = l; + } void clear() { m_namemap.clear(); m_names.clear(); m_languages.clear(); } void fileNameNumMapDumpXml(std::ostream& os); - static string filenameLetters(int fileno); + static string filenameLetters(fileNameIdx_t fileno); + + // Add given bitset to the interned bitsets, return interned index + msgEnSetIdx_t addMsgEnBitSet(const MsgEnBitSet& bitSet); + // Add index of default bitset + msgEnSetIdx_t defaultMsgEnIndex(); + // Set bitIdx to value in bitset at interned idnex setIdx, return interned index of result + msgEnSetIdx_t msgEnSetBit(msgEnSetIdx_t setIdx, size_t bitIdx, bool value); + // Return index to intersection set + msgEnSetIdx_t msgEnAnd(msgEnSetIdx_t lhsIdx, msgEnSetIdx_t rhsIdx); + // Retrieve interned bitset at given interned index. The returned reference is not persistent. + const MsgEnBitSet& msgEn(msgEnSetIdx_t idx) const { return m_internedMsgEns.at(idx); } }; -//! All source lines from a file/stream, to enable errors to show sources +// All source lines from a file/stream, to enable errors to show sources class VFileContent final { + friend class FileLine; // MEMBERS int m_id; // Content ID number + // Reference count for sharing (shared_ptr has size overhead that we don't want) + std::atomic m_refCount{0}; std::deque m_lines; // Source text lines -public: VFileContent() { static int s_id = 0; m_id = ++s_id; } ~VFileContent() = default; // METHODS + void refInc() { ++m_refCount; } + void refDec() { + if (!--m_refCount) delete this; + } + +public: void pushText(const string& text); // Add arbitrary text (need not be line-by-line) string getLine(int lineno) const; string ascii() const { return "ct" + cvtToStr(m_id); } }; std::ostream& operator<<(std::ostream& os, VFileContent* contentp); -//! File and line number of an object, mostly for error reporting +// File and line number of an object, mostly for error reporting -//! This class is instantiated for every source code line (potentially -//! millions). To save space, per-file information (e.g. filename, source -//! language is held in tables in the FileLineSingleton class. +// This class is instantiated for every source code line (potentially millions), and instances +// created at any point usually persist until the end of the program. To save space, per-file +// information (e.g. filename, source language) is held in tables in the FileLineSingleton class. +// Similarly, message enablement flags are interned in FileLineSingleton. + +// WARNING: Avoid increasing the size of this class as much as possible. class FileLine final { + // CONSTANTS static constexpr unsigned SHOW_SOURCE_MAX_LENGTH = 400; // Don't show source lines > this long + // TYPES + using fileNameIdx_t = FileLineSingleton::fileNameIdx_t; + using msgEnSetIdx_t = FileLineSingleton::msgEnSetIdx_t; + using MsgEnBitSet = FileLineSingleton::MsgEnBitSet; + // MEMBERS // Columns here means number of chars from beginning (i.e. tabs count as one) + msgEnSetIdx_t m_msgEnIdx = 0; // Message enable bit set (index into interned array) + fileNameIdx_t m_filenameno = 0; // `line corrected filename number + bool m_waive : 1; // Waive warning - pack next to the line number to save 8 bytes of storage + unsigned m_contentLineno : 31; // Line number within source stream int m_firstLineno = 0; // `line corrected token's first line number int m_firstColumn = 0; // `line corrected token's first column number int m_lastLineno = 0; // `line corrected token's last line number int m_lastColumn = 0; // `line corrected token's last column number - int m_filenameno; // `line corrected filename number - int m_contentLineno = 0; // Line number within source stream - std::shared_ptr m_contentp = nullptr; // Source text contents line is within + VFileContent* m_contentp = nullptr; // Source text contents line is within FileLine* m_parent = nullptr; // Parent line that included this line - std::bitset m_warnOn; - bool m_waive = false; // Waive warning protected: // User routines should never need to change line numbers @@ -118,30 +162,38 @@ private: return s; } static FileLine& defaultFileLine() { - static FileLine* defFilelinep = new FileLine(FileLine::EmptySecret()); - return *defFilelinep; + static FileLine s; + return s; } + FileLine() // Only used for defaultFileLine above + : m_msgEnIdx{singleton().defaultMsgEnIndex()} + , m_filenameno{singleton().nameToNumber(FileLine::builtInFilename())} + , m_waive{false} + , m_contentLineno{0} {} + public: explicit FileLine(const string& filename) - : m_filenameno{singleton().nameToNumber(filename)} - , m_warnOn{defaultFileLine().m_warnOn} {} + : m_msgEnIdx{defaultFileLine().m_msgEnIdx} + , m_filenameno{singleton().nameToNumber(filename)} + , m_waive{false} + , m_contentLineno{0} {} explicit FileLine(FileLine* fromp) - : m_firstLineno{fromp->m_firstLineno} + : m_msgEnIdx{fromp->m_msgEnIdx} + , m_filenameno{fromp->m_filenameno} + , m_waive{fromp->m_waive} + , m_contentLineno{fromp->m_contentLineno} + , m_firstLineno{fromp->m_firstLineno} , m_firstColumn{fromp->m_firstColumn} , m_lastLineno{fromp->m_lastLineno} , m_lastColumn{fromp->m_lastColumn} - , m_filenameno{fromp->m_filenameno} - , m_contentLineno{fromp->m_contentLineno} , m_contentp{fromp->m_contentp} - , m_parent{fromp->m_parent} - , m_warnOn{fromp->m_warnOn} - , m_waive{fromp->m_waive} {} - struct EmptySecret {}; // Constructor selection - explicit FileLine(EmptySecret); + , m_parent{fromp->m_parent} { + if (m_contentp) m_contentp->refInc(); + } FileLine* copyOrSameFileLine(); static void deleteAllRemaining(); - ~FileLine() = default; + ~FileLine(); #ifdef VL_LEAK_CHECKS static void* operator new(size_t size); static void operator delete(void* obj, size_t size); @@ -150,7 +202,7 @@ public: void newContent(); void contentLineno(int num) { lineno(num); - m_contentLineno = num; + m_contentLineno = static_cast(num); } void lineno(int num) { m_firstLineno = num; @@ -180,7 +232,7 @@ public: int firstColumn() const { return m_firstColumn; } int lastLineno() const { return m_lastLineno; } int lastColumn() const { return m_lastColumn; } - std::shared_ptr contentp() const { return m_contentp; } + VFileContent* contentp() const { return m_contentp; } // If not otherwise more specific, use last lineno for errors etc, // as the parser errors etc generally make more sense pointing at the last parse point int lineno() const { return m_lastLineno; } @@ -204,22 +256,24 @@ public: string lineDirectiveStrg(int enterExit) const; // Turn on/off warning messages on this line. - void warnOn(V3ErrorCode code, bool flag) { m_warnOn.set(code, flag); } + void warnOn(V3ErrorCode code, bool flag) { + m_msgEnIdx = singleton().msgEnSetBit(m_msgEnIdx, code, flag); + } void warnOff(V3ErrorCode code, bool flag) { warnOn(code, !flag); } bool warnOff(const string& msg, bool flag); // Returns 1 if ok bool warnIsOff(V3ErrorCode code) const; void warnLintOff(bool flag); void warnStyleOff(bool flag); - void warnStateFrom(const FileLine& from) { m_warnOn = from.m_warnOn; } + void warnStateFrom(const FileLine& from) { m_msgEnIdx = from.m_msgEnIdx; } void warnResetDefault() { warnStateFrom(defaultFileLine()); } bool lastWarnWaived() const { return m_waive; } // Specific flag ACCESSORS/METHODS - bool celldefineOn() const { return m_warnOn.test(V3ErrorCode::I_CELLDEFINE); } + bool celldefineOn() const { return msgEn().test(V3ErrorCode::I_CELLDEFINE); } void celldefineOn(bool flag) { warnOn(V3ErrorCode::I_CELLDEFINE, flag); } - bool coverageOn() const { return m_warnOn.test(V3ErrorCode::I_COVERAGE); } + bool coverageOn() const { return msgEn().test(V3ErrorCode::I_COVERAGE); } void coverageOn(bool flag) { warnOn(V3ErrorCode::I_COVERAGE, flag); } - bool tracingOn() const { return m_warnOn.test(V3ErrorCode::I_TRACING); } + bool tracingOn() const { return msgEn().test(V3ErrorCode::I_TRACING); } void tracingOn(bool flag) { warnOn(V3ErrorCode::I_TRACING, flag); } // METHODS - Global @@ -238,7 +292,9 @@ public: // METHODS - Called from netlist // Merge warning disables from another fileline - void modifyStateInherit(const FileLine* fromp); + void modifyStateInherit(const FileLine* fromp) { + m_msgEnIdx = singleton().msgEnAnd(m_msgEnIdx, fromp->m_msgEnIdx); + } // Change the current fileline due to actions discovered after parsing // and may have side effects on other nodes sharing this FileLine. // Use only when this is intended @@ -266,7 +322,7 @@ public: bool operator==(const FileLine& rhs) const { return (m_firstLineno == rhs.m_firstLineno && m_firstColumn == rhs.m_firstColumn && m_lastLineno == rhs.m_lastLineno && m_lastColumn == rhs.m_lastColumn - && m_filenameno == rhs.m_filenameno && m_warnOn == rhs.m_warnOn); + && m_filenameno == rhs.m_filenameno && m_msgEnIdx == rhs.m_msgEnIdx); } // Returns -1 if (*this) should come before rhs after sorted. 1 for the opposite case. 0 for // equivalent. @@ -278,14 +334,15 @@ public: return (m_firstColumn < rhs.m_firstColumn) ? -1 : 1; if (m_lastLineno != rhs.m_lastLineno) return (m_lastLineno < rhs.m_lastLineno) ? -1 : 1; if (m_lastColumn != rhs.m_lastColumn) return (m_lastColumn < rhs.m_lastColumn) ? -1 : 1; - for (size_t i = 0; i < m_warnOn.size(); ++i) { - if (m_warnOn[i] != rhs.m_warnOn[i]) return (m_warnOn[i] < rhs.m_warnOn[i]) ? -1 : 1; + for (size_t i = 0; i < msgEn().size(); ++i) { + if (msgEn().test(i) != rhs.msgEn().test(i)) return rhs.msgEn().test(i) ? -1 : 1; } return 0; // (*this) and rhs are equivalent } private: string warnContext(bool secondary) const; + const MsgEnBitSet& msgEn() const { return singleton().msgEn(m_msgEnIdx); } }; std::ostream& operator<<(std::ostream& os, FileLine* fileline); diff --git a/src/V3Param.cpp b/src/V3Param.cpp index e4719d3f3..18dfc06b6 100644 --- a/src/V3Param.cpp +++ b/src/V3Param.cpp @@ -101,7 +101,7 @@ public: for (V3HierarchicalBlockOption::ParamStrMap::const_iterator pIt = params.begin(); pIt != params.end(); ++pIt) { std::unique_ptr constp{AstConst::parseParamLiteral( - new FileLine(FileLine::EmptySecret()), pIt->second)}; + new FileLine{FileLine::builtInFilename()}, pIt->second)}; UASSERT(constp, pIt->second << " is not a valid parameter literal"); const bool inserted = consts.emplace(pIt->first, std::move(constp)).second; UASSERT(inserted, pIt->first << " is already added");