From 4a1d18f0e92430380ffa539c9f1b452f6a73fc6a Mon Sep 17 00:00:00 2001 From: Wilson Snyder Date: Sun, 7 Jun 2020 15:35:36 -0400 Subject: [PATCH] Internals: Refactor parser pipeline to use deque. No functional change intended. --- src/V3ParseImp.cpp | 65 +++++++++++++++++++++++++++------------------- src/V3ParseImp.h | 14 +++++----- src/V3ParseLex.cpp | 4 +-- src/verilog.y | 2 +- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/src/V3ParseImp.cpp b/src/V3ParseImp.cpp index 42504226c..16e1e1d01 100644 --- a/src/V3ParseImp.cpp +++ b/src/V3ParseImp.cpp @@ -336,20 +336,27 @@ bool V3ParseImp::bisonValIdThenColon() const { return bisonValPrev().token == yaID__ETC && bisonValCur().token == yP_COLONCOLON; } -void V3ParseImp::lexToken() { - // called from lexToBison, has a "this" +void V3ParseImp::tokenPull() { + // Pull token from lex into the pipeline + // This corrupts yylval, must save/restore if required // Fetch next token from prefetch or real lexer - int token; - if (m_ahead) { - // We prefetched an extra token, give it back - m_ahead = false; - token = m_aheadVal.token; - yylval = m_aheadVal; - } else { - // Parse new token - token = yylexReadTok(); - // yylval // Set by yylexReadTok() - } + yylexReadTok(); // sets yylval + m_tokensAhead.push_back(yylval); +} + +const V3ParseBisonYYSType* V3ParseImp::tokenPeekp(size_t depth) { + // Look ahead "depth" number of tokens in the input stream + // Returns pointer to token, which is no longer valid after changing m_tokensAhead + while (m_tokensAhead.size() <= depth) tokenPull(); + return &m_tokensAhead.at(depth); +} + +void V3ParseImp::tokenPipeline() { + // called from bison's "yylex", has a "this" + if (m_tokensAhead.empty()) tokenPull(); // corrupts yylval + yylval = m_tokensAhead.front(); + m_tokensAhead.pop_front(); + int token = yylval.token; // If a paren, read another if (token == '(' // || token == yCONST__LEX // @@ -359,17 +366,13 @@ void V3ParseImp::lexToken() { || token == ySTATIC__LEX // || token == yVIRTUAL__LEX // || token == yWITH__LEX // - // Never put yID_* here; below symbol table resolution would break; - // parser will change symbol table affecting how next ID maybe interpreted ) { if (debugFlex() >= 6) { - cout << " lexToken: reading ahead to find possible strength" << endl; + cout << " tokenPipeline: reading ahead to find possible strength" << endl; } V3ParseBisonYYSType curValue = yylval; // Remember value, as about to read ahead - int nexttok = yylexReadTok(); - m_ahead = true; - m_aheadVal = yylval; - m_aheadVal.token = nexttok; + const V3ParseBisonYYSType* nexttokp = tokenPeekp(0); + int nexttok = nexttokp->token; yylval = curValue; // Now potentially munge the current token if (token == '(' @@ -435,18 +438,26 @@ void V3ParseImp::lexToken() { } // If add to above "else if", also add to "if (token" further above } + yylval.token = token; + // effectively returns yylval +} + +void V3ParseImp::tokenPipelineSym() { // If an id, change the type based on symbol table // Note above sometimes converts yGLOBAL to a yaID__LEX + tokenPipeline(); // sets yylval + int token = yylval.token; if (token == yaID__LEX) { VSymEnt* foundp; if (VSymEnt* look_underp = V3ParseImp::parsep()->symp()->nextId()) { - UINFO(7, " lexToken: next id lookup forced under " << look_underp << endl); + UINFO(7, " tokenPipelineSym: next id lookup forced under " << look_underp << endl); foundp = look_underp->findIdFallback(*(yylval.strp)); // "consume" it. Must set again if want another token under temp scope V3ParseImp::parsep()->symp()->nextId(NULL); } else { - UINFO(7, " lexToken: find upward " << V3ParseImp::parsep()->symp()->symCurrentp() - << " for '" << *(yylval.strp) << "'" << endl); + UINFO(7, " tokenPipelineSym: find upward " + << V3ParseImp::parsep()->symp()->symCurrentp() << " for '" + << *(yylval.strp) << "'" << endl); // if (debug()>=9) V3ParseImp::parsep()->symp()->symCurrentp()->dump(cout," -findtree: // ", true); foundp = V3ParseImp::parsep()->symp()->symCurrentp()->findIdFallback(*(yylval.strp)); @@ -454,7 +465,7 @@ void V3ParseImp::lexToken() { if (foundp) { AstNode* scp = foundp->nodep(); yylval.scp = scp; - UINFO(7, " lexToken: Found " << scp << endl); + UINFO(7, " tokenPipelineSym: Found " << scp << endl); if (VN_IS(scp, Typedef)) { token = yaID__aTYPE; } else if (VN_IS(scp, TypedefFwd)) { @@ -480,16 +491,16 @@ void V3ParseImp::lexToken() { // effectively returns yylval } -int V3ParseImp::lexToBison() { +int V3ParseImp::tokenToBison() { // Called as global since bison doesn't have our pointer - lexToken(); // sets yylval + tokenPipelineSym(); // sets yylval m_bisonValPrev = m_bisonValCur; m_bisonValCur = yylval; m_bisonLastFileline = yylval.fl; // yylval.scp = NULL; // Symbol table not yet needed - no packages if (debugFlex() >= 6 || debugBison() >= 6) { // --debugi-flex and --debugi-bison - cout << "lexToBison " << yylval << endl; + cout << "tokenToBison " << yylval << endl; } return yylval.token; } diff --git a/src/V3ParseImp.h b/src/V3ParseImp.h index 9447dc8b4..2603c4e96 100644 --- a/src/V3ParseImp.h +++ b/src/V3ParseImp.h @@ -116,10 +116,9 @@ class V3ParseImp { VOptionBool m_unconnectedDrive; // Last unconnected drive int m_lexPrevToken; // previous parsed token (for lexer) - bool m_ahead; // aheadval is valid - V3ParseBisonYYSType m_aheadVal; // ahead token value V3ParseBisonYYSType m_bisonValCur; // current token for error reporting V3ParseBisonYYSType m_bisonValPrev; // previous token for error reporting + std::deque m_tokensAhead; // Tokens we parsed ahead of parser std::deque m_stringps; // Created strings for later cleanup std::deque m_numberps; // Created numbers for later cleanup @@ -253,10 +252,8 @@ public: m_lexKwdDepth = 0; m_lexKwdLast = stateVerilogRecent(); m_lexPrevToken = 0; - m_ahead = false; m_bisonValCur.token = 0; m_bisonValPrev.token = 0; - // m_aheadVal not used as m_ahead = false, and not all compilers support initing it m_tagNodep = NULL; m_timeLastUnit = v3Global.opt.timeDefaultUnit(); } @@ -266,15 +263,18 @@ public: // METHODS // Preprocess and read the Verilog file specified into the netlist database - int lexToBison(); // Pass token to bison + int tokenToBison(); // Pass token to bison void parseFile(FileLine* fileline, const string& modfilename, bool inLibrary, const string& errmsg); private: void lexFile(const string& modname); - int yylexReadTok(); - void lexToken(); // Internal; called from lexToBison + void yylexReadTok(); + void tokenPull(); + void tokenPipeline(); // Internal; called from tokenToBison + void tokenPipelineSym(); + const V3ParseBisonYYSType* tokenPeekp(size_t depth); void preprocDumps(std::ostream& os); }; diff --git a/src/V3ParseLex.cpp b/src/V3ParseLex.cpp index 4a24b51b9..06b2a4024 100644 --- a/src/V3ParseLex.cpp +++ b/src/V3ParseLex.cpp @@ -50,12 +50,12 @@ void V3ParseImp::lexUnputString(const char* textp, size_t length) { parsep()->m_lexerp->unputString(textp, length); } -int V3ParseImp::yylexReadTok() { +void V3ParseImp::yylexReadTok() { // Call yylex() remembering last non-whitespace token parsep()->lexFileline()->startToken(); int token = parsep()->m_lexerp->yylex(); m_lexPrevToken = token; // Save so can find '#' to parse following number - return token; + yylval.token = token; } //###################################################################### diff --git a/src/verilog.y b/src/verilog.y index 8d07359d3..9049e1b79 100644 --- a/src/verilog.y +++ b/src/verilog.y @@ -36,7 +36,7 @@ #define YYMAXDEPTH 10000 // Pick up new lexer -#define yylex PARSEP->lexToBison +#define yylex PARSEP->tokenToBison #define BBUNSUP(fl, msg) \ { \ if (!v3Global.opt.bboxUnsup()) { (fl)->v3error(msg); } \