Add experimental --pipe-filter to filter all Verilog input.

Changes

@@ -33,6 +33,8 @@ indicates the contributor was also the author of the fix; Thanks!
 
 ***  Add experimental clock domain crossing checks.
 
+***  Add experimental --pipe-filter to filter all Verilog input.
+
 ***  Speed compiles by avoiding including the STL iostream header.
      Application programs may need to include it themselves to avoid errors.
 

Makefile.in

@@ -125,6 +125,7 @@ DISTFILES_INC = $(INFOS) .gitignore Artistic COPYING COPYING.LESSER \
 	test_regress/t/*.mem \
 	test_regress/t/*.out \
 	test_regress/t/*.pl \
+	test_regress/t/*.pf \
 	test_regress/t/*.v* \
 	verilator.* \
 

bin/verilator

@@ -231,6 +231,7 @@ descriptions in the next sections for more information.
     --output-split-cfuncs <statements>   Split .ccp functions
     --pins-bv <bits>            Specify types for top level ports
     --pins-uint8                Specify types for top level ports
+    --pipe-filter <command>	Filter all input through a script
     --prefix <topname>          Name of top level class
     --profile-cfuncs            Name functions for profiling
     --private                   Debugging; see docs
@@ -573,6 +574,23 @@ Specifies SystemC inputs/outputs that are smaller than the --pins-bv
 setting and 8 bits or less should use uint8_t instead of uint32_t.
 Likewise pins of width 9-16 will use uint16_t instead of uint32_t.
 
+=item --pipe-filter I<command>
+
+Rarely needed and experimental.  Verilator will spawn the specified command
+as a subprocess pipe, to allow the command to perform custom edits on the
+Verilog code before it reaches Verilator.
+
+Before reading each Verilog file, Verilator will pass the file name to the
+subprocess' stdin with 'read_verilog "<filename>"'.  The filter may then
+read the file and perform any filtering it desires, and feeds the new file
+contents back to Verilator on stdout with 'Content-Length'.  Output to
+stderr from the filter feeds through to Verilator's stdout and if the
+filter exits with non-zero status Verilator terminates.  See the
+t/t_pipe_filter test for an example.
+
+To debug the output of the filter, try using the -E option to see
+preprocessed output.
+
 =item --prefix I<topname>
 
 Specifies the name of the top level class and makefile.  Defaults to V

src/V3File.cpp

@@ -25,14 +25,28 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <iomanip>
 #include <memory>
+#include <map>
+
+#if defined(__unix__)
+# define INFILTER_PIPE  // Allow pipe filtering.  Needs fork()
+#endif
+
+#ifdef INFILTER_PIPE
+# include <sys/wait.h>
+#endif
 
 #include "V3Global.h"
 #include "V3File.h"
 #include "V3PreShell.h"
 #include "V3Ast.h"
 
+#define INFILTER_IPC_BUFSIZ 64*1024  // For debug, try this as a small number
+#define INFILTER_CACHE_MAX  64*1024  // Maximum bytes to cache if same file read twice
+
 //######################################################################
 // V3File Internal state
 
@@ -252,6 +266,249 @@ void V3File::createMakeDir() {
     }
 }
 
+//######################################################################
+// V3InFilterImp
+
+class V3InFilterImp {
+    typedef map<string,string> FileContentsMap;
+
+    FileContentsMap	m_contentsMap;	// Cache of file contents
+    bool		m_readEof;	// Received EOF on read
+#ifdef INFILTER_PIPE
+    pid_t		m_pid;		// fork() process id
+#else
+    int			m_pid;		// fork() process id - always zero as disabled
+#endif
+    bool		m_pidExited;
+    int			m_pidStatus;
+    int			m_writeFd;	// File descriptor TO filter
+    int			m_readFd;	// File descriptor FROM filter
+
+private:
+    // METHODS
+    static int debug() {
+	static int level = -1;
+	if (VL_UNLIKELY(level < 0)) level = v3Global.opt.debugSrcLevel(__FILE__);
+	return level;
+    }
+
+    bool readContents(const string& filename, string& out) {
+	if (m_pid) return readContentsFilter(filename,out);
+	else return readContentsFile(filename,out);
+    }
+    bool readContentsFile(const string& filename, string& out) {
+	int fd = open (filename.c_str(), O_RDONLY);
+	if (!fd) return false;
+	m_readEof = false;
+	out = readBlocks(fd, -1);
+	close(fd);
+	return true;
+    }
+    bool readContentsFilter(const string& filename, string& out) {
+	if (filename!="" || out!="") {}  // Prevent unused
+#ifdef INFILTER_PIPE
+	writeFilter("read \""+filename+"\"\n");
+	string line = readFilterLine();
+	if (line.find("Content-Length") != string::npos) {
+	    int len = 0;
+	    sscanf(line.c_str(), "Content-Length: %d\n", &len);
+	    out = readBlocks(m_readFd, len);
+	    return true;
+	} else {
+	    if (line!="") v3error("--pipe-filter protocol error, unexpected: "<<line);
+	    return false;
+	}
+#else
+	v3fatalSrc("--pipe-filter not implemented on this platform");
+	return false;
+#endif
+    }
+
+    void checkFilter(bool hang) {
+#ifdef INFILTER_PIPE
+	if (!m_pidExited && waitpid(m_pid, &m_pidStatus, hang?0:WNOHANG)) {
+	    UINFO(1,"--pipe-filter: Exited, status "<<m_pidStatus<<" exit="<<WEXITSTATUS(m_pidStatus)<<" err"<<strerror(errno)<<endl);
+	    m_readEof = true;
+	    m_pidExited = true;
+	}
+#endif
+    }
+
+    string readBlocks(int fd, int size=-1) {
+	string out;
+	char buf[INFILTER_IPC_BUFSIZ];
+	while (!m_readEof && (size<0 || size>(int)out.length())) {
+	    int todo = INFILTER_IPC_BUFSIZ;
+	    if (size>0 && size<INFILTER_IPC_BUFSIZ) todo = size;
+	    int got = read (fd, buf, todo);
+	    //UINFO(9,"RD GOT g "<< got<<" e "<<errno<<" "<<strerror(errno)<<endl);  usleep(50*1000);
+	    if (got>0) out.append(buf, got);
+	    else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+		checkFilter(false); usleep(1000); continue;
+	    } else { m_readEof = true; break; }
+	}
+	return out;
+    }
+    string readFilterLine() {
+	// Slow, but we don't need it much
+	UINFO(9,"readFilterLine\n");
+	string line;
+	while (!m_readEof) {
+	    string c = readBlocks(m_readFd, 1);
+	    line += c;
+	    if (c == "\n") {
+		if (line == "\n") { line=""; continue; }
+		else break;
+	    }
+	}
+	UINFO(6,"filter-line-in: "<<line);
+	return line;
+    }
+    void writeFilter(const string& out) {
+	if (debug()>=6) { UINFO(6,"filter-out: "<<out); if (out[out.length()-1]!='\n') cout<<endl; }
+	if (!m_pid) { v3error("--pipe-filter: write to closed file\n"); m_readEof = true; stop(); }
+	unsigned offset = 0;
+	while (!m_readEof && out.length()>offset) {
+	    int got = write (m_writeFd, (out.c_str())+offset, out.length()-offset);
+	    //UINFO(9,"WR GOT g "<< got<<" e "<<errno<<" "<<strerror(errno)<<endl);  usleep(50*1000);
+	    if (got>0) offset += got;
+	    else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+		checkFilter(false); usleep(1000); continue;
+	    }
+	    else break;
+	}
+    }
+
+    // Start the filter
+    void start(const string& command) {
+	if (command=="") {
+	    m_pid = 0;  // Disabled
+	} else {
+	    startFilter(command);
+	}
+    }
+    void startFilter(const string& command) {
+	if (command=="") {} // Prevent Unused
+#ifdef INFILTER_PIPE
+	int fd_stdin[2], fd_stdout[2];
+	static const int P_RD = 0;
+	static const int P_WR = 1;
+
+	if (pipe(fd_stdin) != 0 || pipe(fd_stdout) != 0) {
+	    v3fatal("--pipe-filter: Can't pipe: "<<strerror(errno));
+	}
+	if (fd_stdin[P_RD]<=2 || fd_stdin[P_WR]<=2
+	    || fd_stdout[P_RD]<=2 || fd_stdout[P_WR]<=2) {
+	    // We'd have to rearrange all of the FD usages in this case.
+	    // Too unlikely; verilator isn't a daemon.
+	    v3fatal("--pipe-filter: stdin/stdout closed before pipe opened\n");
+	}
+
+	UINFO(1,"--pipe-filter: /bin/sh -c "<<command<<endl);
+
+	pid_t pid = fork();
+	if (pid < 0) v3fatal("--pipe-filter: fork failed: "<<strerror(errno));
+	if (pid == 0) {  // Child
+	    UINFO(6,"In child\n");
+	    close(fd_stdin[P_WR]);
+	    dup2(fd_stdin[P_RD], 0);
+	    close(fd_stdout[P_RD]);
+	    dup2(fd_stdout[P_WR], 1);
+	    // And stderr comes from parent
+
+	    execl("/bin/sh", "sh", "-c", command.c_str(), NULL);
+	    // Don't use v3fatal, we don't share the common structures any more
+	    fprintf(stderr,"--pipe-filter: exec failed: %s\n",strerror(errno));
+	    _exit(10);
+	}
+	else {  // Parent
+	    UINFO(6,"In parent, child pid "<<pid
+		  <<" stdin "<<fd_stdin[P_WR]<<"->"<<fd_stdin[P_RD]
+		  <<" stdout "<<fd_stdout[P_WR]<<"->"<<fd_stdout[P_RD]<<endl);
+	    m_pid = pid;
+	    m_pidExited = false;
+	    m_pidStatus = 0;
+	    m_writeFd = fd_stdin[P_WR];
+	    m_readFd = fd_stdout[P_RD];
+	    m_readEof = false;
+
+	    close(fd_stdin[P_RD]);
+	    close(fd_stdout[P_WR]);
+
+	    int flags = fcntl(m_readFd,F_GETFL,0);
+	    fcntl(m_readFd, F_SETFL, flags | O_NONBLOCK);
+
+	    flags = fcntl(m_writeFd,F_GETFL,0);
+	    fcntl(m_writeFd, F_SETFL, flags | O_NONBLOCK);
+	}
+	UINFO(6,"startFilter complete\n");
+#else
+	v3fatalSrc("--pipe-filter not implemented on this platform");
+#endif
+    }
+
+    void stop() {
+	if (m_pid) stopFilter();
+    }
+    void stopFilter() {
+	UINFO(6,"Stopping filter process\n");
+#ifdef INFILTER_PIPE
+	close(m_writeFd);
+	checkFilter(true);
+	if (!WIFEXITED(m_pidStatus) || WEXITSTATUS(m_pidStatus)!=0) {
+	    v3fatal("--pipe-filter returned bad status");
+	}
+	m_pid = 0;
+	close(m_readFd);
+	UINFO(6,"Closed\n");
+#else
+	v3fatalSrc("--pipe-filter not implemented on this platform");
+#endif
+    }
+
+protected:
+    friend class V3InFilter;
+    // Read file contents and return it
+    bool readWholefile(const string& filename, string& out) {
+	FileContentsMap::iterator it = m_contentsMap.find(filename);
+	if (it != m_contentsMap.end()) {
+	    out = it->second;
+	    return true;
+	}
+	if (!readContents(filename, out)) return false;
+	if (out.length() < INFILTER_CACHE_MAX) {
+	    // Cache small files (only to save space)
+	    // It's quite common to `include "timescale" thousands of times
+	    // This isn't so important if it's just a open(), but filtering can be slow
+	    m_contentsMap.insert(make_pair(filename,out));
+	}
+	return true;
+    }
+    // CONSTRUCTORS
+    V3InFilterImp(const string& command) {
+	m_readEof = false;
+	m_pid = 0;
+	m_pidExited = false;
+	m_pidStatus = 0;
+	m_writeFd = 0;
+	m_readFd = 0;
+	start(command);
+    }
+    ~V3InFilterImp() { stop(); }
+};
+
+//######################################################################
+// V3InFilter
+// Just dispatch to the implementation
+
+V3InFilter::V3InFilter(const string& command) { m_impp = new V3InFilterImp(command); }
+V3InFilter::~V3InFilter() { if (m_impp) delete m_impp; m_impp=NULL; }
+
+bool V3InFilter::readWholefile(const string& filename, string& out) {
+    if (!m_impp) v3fatalSrc("readWholefile on invalid filter");
+    return m_impp->readWholefile(filename, out);
+}
+
 //######################################################################
 // V3OutFormatter: A class for printing to a file, with automatic indentation of C++ code.
 

src/V3File.h

@@ -71,6 +71,23 @@ public:
     static void createMakeDir();
 };
 
+//============================================================================
+// V3InFilter: Read a input file, possibly filtering it, and caching contents
+
+class V3InFilterImp;
+
+class V3InFilter {
+    V3InFilterImp* m_impp;
+public:
+    // METHODS
+    // Read file contents and return it.  Return true on success.   
+    bool readWholefile(const string& filename, string& out);
+
+    // CONSTRUCTORS
+    V3InFilter(const string& command);
+    ~V3InFilter();
+};
+
 //============================================================================
 // V3OutFormatter: A class for automatic indentation of C++ or Verilog code.
 

src/V3LinkCells.cpp

@@ -96,6 +96,8 @@ private:
     AstUser1InUse	m_inuser1;
 
     // STATE
+    V3InFilter*		m_filterp;	// Parser filter
+
     // Below state needs to be preserved between each module call.
     AstNodeModule*	m_modp;		// Current module
     V3SymTable  	m_mods;		// Symbol table of all module names
@@ -174,7 +176,7 @@ private:
 	    AstNodeModule* modp = m_mods.findIdUpward(nodep->modName())->castNodeModule();
 	    if (!modp) {
 		// Read-subfile
-		V3Parse parser (v3Global.rootp());
+		V3Parse parser (v3Global.rootp(), m_filterp);
 		parser.parseFile(nodep->fileline(), nodep->modName(), false);
 		V3Error::abortIfErrors();
 		// We've read new modules, grab new pointers to their names
@@ -269,7 +271,8 @@ private:
 
 public:
     // CONSTUCTORS
-    LinkCellsVisitor(AstNetlist* rootp) {
+    LinkCellsVisitor(AstNetlist* rootp, V3InFilter* filterp) {
+	m_filterp = filterp;
 	m_modp = NULL;
 	m_libVertexp = NULL;
 	m_topVertexp = NULL;
@@ -281,7 +284,7 @@ public:
 //######################################################################
 // Link class functions
 
-void V3LinkCells::link(AstNetlist* rootp) {
+void V3LinkCells::link(AstNetlist* rootp, V3InFilter* filterp) {
     UINFO(4,__FUNCTION__<<": "<<endl);
-    LinkCellsVisitor visitor (rootp);
+    LinkCellsVisitor visitor (rootp, filterp);
 }

src/V3LinkCells.h

@@ -27,11 +27,13 @@
 #include "V3Error.h"
 #include "V3Ast.h"
 
+class V3InFilter;
+
 //============================================================================
 
 class V3LinkCells {
 public:
-    static void link(AstNetlist* nodep);
+    static void link(AstNetlist* nodep, V3InFilter* filterp);
 };
 
 #endif // Guard

src/V3Options.cpp

@@ -798,6 +798,9 @@ void V3Options::parseOptsList(FileLine* fl, int argc, char** argv) {
 	    else if ( !strcmp (sw, "-pins-bv") && (i+1)<argc ) {
 		shift; m_pinsBv = atoi(argv[i]);
 	    }
+	    else if ( !strcmp (sw, "-pipe-filter") && (i+1)<argc ) {
+		shift; m_pipeFilter = argv[i];
+	    }
 	    else if ( !strcmp (sw, "-prefix") && (i+1)<argc ) {
 		shift; m_prefix = argv[i];
 		if (m_modPrefix=="") m_modPrefix = m_prefix;

src/V3Options.h

@@ -136,8 +136,9 @@ class V3Options {
     string	m_bin;		// main switch: --bin {binary}
     string	m_flags;	// main switch: -f {name}
     string	m_makeDir;	// main switch: -Mdir
-    string	m_prefix;	// main switch: --prefix
     string	m_modPrefix;	// main switch: --mod-prefix
+    string	m_pipeFilter;	// main switch: --pipe-filter
+    string	m_prefix;	// main switch: --prefix
     string	m_xAssign;	// main switch: --x-assign
     string	m_topModule;	// main switch: --top-module
 
@@ -244,8 +245,9 @@ class V3Options {
     int    compLimitParens() const { return m_compLimitParens; }
 
     string makeDir() const { return m_makeDir; }
-    string prefix() const { return m_prefix; }
     string modPrefix() const { return m_modPrefix; }
+    string pipeFilter() const { return m_pipeFilter; }
+    string prefix() const { return m_prefix; }
     string topModule() const { return m_topModule; }
     string xAssign() const { return m_xAssign; }
 

src/V3Parse.h

@@ -28,6 +28,7 @@
 #include "V3Global.h"
 
 class AstNetlist;
+class V3InFilter;
 class V3ParseImp;
 
 //============================================================================
@@ -39,7 +40,7 @@ private:
 public:
     // CONSTRUCTORS
     // We must allow reading multiple files into one parser
-    V3Parse(AstNetlist* rootp);
+    V3Parse(AstNetlist* rootp, V3InFilter* filterp);
     ~V3Parse();
 
     // METHODS

src/V3ParseImp.cpp

@@ -104,7 +104,7 @@ void V3ParseImp::parseFile(FileLine* fileline, const string& modfilename, bool i
     }
 
     // Preprocess into m_ppBuffer
-    V3PreShell::preproc(fileline, modfilename, this);
+    V3PreShell::preproc(fileline, modfilename, m_filterp, this);
 
     if (v3Global.opt.preprocOnly() || v3Global.opt.keepTempFiles()) {
 	// Create output file with all the preprocessor output we buffered up
@@ -151,8 +151,8 @@ void V3ParseImp::lexFile(const string& modname) {
 //======================================================================
 // V3Parse functions
 
-V3Parse::V3Parse(AstNetlist* rootp) {
+V3Parse::V3Parse(AstNetlist* rootp, V3InFilter* filterp) {
-    m_impp = new V3ParseImp (rootp);
+    m_impp = new V3ParseImp (rootp, filterp);
 }
 V3Parse::~V3Parse() {
     delete m_impp; m_impp = NULL;

src/V3ParseImp.h

@@ -207,10 +207,11 @@ public:
 
 class V3ParseImp {
     // MEMBERS
-    AstNetlist* m_rootp;	// Root of the design
+    AstNetlist* 	m_rootp;	// Root of the design
-    V3Lexer*	m_lexerp;	// Current FlexLexer
+    V3InFilter*		m_filterp;	// Reading filter
+    V3Lexer*		m_lexerp;	// Current FlexLexer
     static V3ParseImp*	s_parsep;	// Current THIS, bison() isn't class based
-    FileLine*	m_fileline;		// Filename/linenumber currently active
+    FileLine*		m_fileline;	// Filename/linenumber currently active
 
     V3ParseSym	m_sym;			// Symbol table
     bool	m_inCellDefine;		// Inside a `celldefine
@@ -308,7 +309,8 @@ public:
 
 public:
     // CREATORS
-    V3ParseImp(AstNetlist* rootp) : m_sym(rootp) {
+    V3ParseImp(AstNetlist* rootp, V3InFilter* filterp)
+	: m_sym(rootp), m_filterp(filterp) {
 	m_rootp = rootp; m_lexerp = NULL;
 	m_inCellDefine = false;
 	m_inLibrary = false;

src/V3PreLex.h

@@ -107,7 +107,6 @@ class V3PreLex {
     FileLine*	m_curFilelinep;	// Current processing point
 
     // Parse state
-    FILE*	m_fp;		// File state is for
     stack<YY_BUFFER_STATE> m_bufferStack; // Stack of inserted text above current point
 
     // State to lexer
@@ -123,19 +122,15 @@ class V3PreLex {
     string	m_defValue;	// Definition value being built.
 
     // CONSTRUCTORS
-    V3PreLex(FILE* fp) {
+    V3PreLex() {
-	m_fp = fp;
 	m_keepComments = 0;
 	m_pedantic = false;
 	m_formalLevel = 0;
 	m_parenLevel = 0;
 	m_pslParenLevel = 0;
 	m_pslMoreNeeded = false;
-	m_bufferStack.push(yy_create_buffer (fp, YY_BUF_SIZE));
-	yy_switch_to_buffer(m_bufferStack.top());
     }
     ~V3PreLex() {
-	fclose(m_fp);
 	while (!m_bufferStack.empty()) { yy_delete_buffer(m_bufferStack.top()); m_bufferStack.pop(); }
     }
 

src/V3PreProc.cpp

@@ -162,7 +162,7 @@ private:
 
     bool commentTokenMatch(string& cmdr, const char* strg);
     string trimWhitespace(const string& strg, bool trailing);
-    void unputString(const string& strg);
+    void unputString(const string& strg, bool first=false);
 
     void parsingOn() {
 	m_off--;
@@ -173,7 +173,7 @@ private:
 
 public:
     // METHODS, called from upper level shell
-    virtual void openFile(FileLine* fl, const string& filename);
+    virtual void openFile(FileLine* fl, V3InFilter* filterp, const string& filename);
     virtual bool isEof() const { return (m_lexp==NULL); }
     virtual string getline();
     virtual void insertUnreadback(const string& text) { m_lineCmt += text; }
@@ -402,14 +402,16 @@ const char* V3PreProcImp::tokenName(int tok) {
     }
 }
 
-void V3PreProcImp::unputString(const string& strg) {
+void V3PreProcImp::unputString(const string& strg, bool first) {
     // We used to just m_lexp->unputString(strg.c_str());
     // However this can lead to "flex scanner push-back overflow"
     // so instead we scan from a temporary buffer, then on EOF return.
     // This is also faster than the old scheme, amazingly.
-    if (m_lexp->m_bufferStack.empty() || m_lexp->m_bufferStack.top()!=m_lexp->currentBuffer()) {
+    if (!first) {  // Else the initial creation
-	fileline()->v3fatalSrc("bufferStack missing current buffer; will return incorrectly");
+	if (m_lexp->m_bufferStack.empty() || m_lexp->m_bufferStack.top()!=m_lexp->currentBuffer()) {
-	// Hard to debug lost text as won't know till much later
+	    fileline()->v3fatalSrc("bufferStack missing current buffer; will return incorrectly");
+	    // Hard to debug lost text as won't know till much later
+	}
     }
     m_lexp->scanBytes(strg);
 }
@@ -576,15 +578,17 @@ string V3PreProcImp::defineSubst(V3DefineRef* refp) {
 //**********************************************************************
 // Parser routines
 
-void V3PreProcImp::openFile(FileLine* fl, const string& filename) {
+void V3PreProcImp::openFile(FileLine* fl, V3InFilter* filterp, const string& filename) {
     // Open a new file, possibly overriding the current one which is active.
     if (fl) {
 	m_fileline = new FileLine(fl);
     }
 
     V3File::addSrcDepend(filename);
-    FILE* fp = fopen (filename.c_str(), "r");
+
-    if (!fp) {
+    string wholefile;
+    bool ok = filterp->readWholefile(filename, wholefile/*ref*/);
+    if (!ok) {
 	fileline()->v3error("File not found: "+filename+"\n");
 	return;
     }
@@ -601,7 +605,7 @@ void V3PreProcImp::openFile(FileLine* fl, const string& filename) {
 	addLineComment(0);
     }
 
-    m_lexp = new V3PreLex (fp);
+    m_lexp = new V3PreLex();
     m_lexp->m_keepComments = keepComments();
     m_lexp->m_pedantic = pedantic();
     m_lexp->m_curFilelinep = new FileLine(filename, 1);
@@ -609,6 +613,7 @@ void V3PreProcImp::openFile(FileLine* fl, const string& filename) {
     addLineComment(1); // Enter
 
     yy_flex_debug = (debug()>4)?1:0;
+    unputString(wholefile,true);
 }
 
 void V3PreProcImp::insertUnreadbackAtBol(const string& text) {

src/V3PreProc.h

@@ -30,6 +30,8 @@
 #include <map>
 #include <iostream>
 
+class V3InFilter;
+
 class V3PreProc {
     // This defines a preprocessor.  Functions are virtual so implementation can be hidden.
     // After creating, call open(), then getline() in a loop.  The class will to the rest...
@@ -48,7 +50,7 @@ public:
 
     // ACCESSORS
     // Insert given file into this point in input stream
-    virtual void openFile(FileLine* fileline, const string& filename)=0;
+    virtual void openFile(FileLine* fileline, V3InFilter* filterp, const string& filename)=0;
     virtual string getline()=0;		// Return next line/lines. (Null if done.)
     virtual bool isEof() const =0;		// Return true on EOF.
     virtual void insertUnreadback(const string& text) = 0;

src/V3PreShell.cpp

@@ -43,6 +43,7 @@ protected:
 
     static V3PreShellImp s_preImp;
     static V3PreProc*	s_preprocp;
+    static V3InFilter*	s_filterp;
 
     //---------------------------------------
     // METHODS
@@ -70,12 +71,13 @@ protected:
 	}
     }
 
-    void preproc (FileLine* fl, const string& modname, V3ParseImp* parsep) {
+    void preproc (FileLine* fl, const string& modname, V3InFilter* filterp, V3ParseImp* parsep) {
 	// Preprocess the given module, putting output in vppFilename
 	UINFONL(1,"  Preprocessing "<<modname<<endl);
 
 	// Preprocess
-	preprocOpen(fl, modname, "Cannot find file containing module: ");
+	s_filterp = filterp;
+	preprocOpen(fl, s_filterp, modname, "Cannot find file containing module: ");
 	while (!s_preprocp->isEof()) {
 	    string line = s_preprocp->getline();
 	    V3Parse::ppPushText(parsep, line);
@@ -83,10 +85,10 @@ protected:
     }
 
     void preprocInclude (FileLine* fl, const string& modname) {
-	preprocOpen(fl, modname, "Cannot find include file: ");
+	preprocOpen(fl, s_filterp, modname, "Cannot find include file: ");
     }
 
-    void preprocOpen (FileLine* fl, const string& modname, const string& errmsg) {
+    void preprocOpen (FileLine* fl, V3InFilter* filterp, const string& modname, const string& errmsg) {
 	// Allow user to put `defined names on the command line instead of filenames,
 	// then convert them properly.
 	string ppmodname = s_preprocp->removeDefines (modname);
@@ -96,15 +98,17 @@ protected:
 	if (filename=="") return;  // Not found
 
 	UINFO(2,"    Reading "<<filename<<endl);
-	s_preprocp->openFile(fl, filename);
+	s_preprocp->openFile(fl, filterp, filename);
     }
 
     // CONSTRUCTORS
     V3PreShellImp() {}
+    ~V3PreShellImp() {}
 };
 
 V3PreShellImp V3PreShellImp::s_preImp;
 V3PreProc* V3PreShellImp::s_preprocp = NULL;
+V3InFilter* V3PreShellImp::s_filterp = NULL;
 
 //######################################################################
 // Perl class functions
@@ -112,8 +116,8 @@ V3PreProc* V3PreShellImp::s_preprocp = NULL;
 void V3PreShell::boot(char** env) {
     V3PreShellImp::s_preImp.boot(env);
 }
-void V3PreShell::preproc(FileLine* fl, const string& modname, V3ParseImp* parsep) {
+void V3PreShell::preproc(FileLine* fl, const string& modname, V3InFilter* filterp, V3ParseImp* parsep) {
-    V3PreShellImp::s_preImp.preproc(fl, modname, parsep);
+    V3PreShellImp::s_preImp.preproc(fl, modname, filterp, parsep);
 }
 void V3PreShell::preprocInclude(FileLine* fl, const string& modname) {
     V3PreShellImp::s_preImp.preprocInclude(fl, modname);

src/V3PreShell.h

@@ -28,6 +28,7 @@
 #include "V3Error.h"
 
 class V3ParseImp;
+class V3InFilter;
 
 //============================================================================
 
@@ -35,7 +36,7 @@ class V3PreShell {
     // Static class for calling preprocessor
 public:
     static void boot(char** env);
-    static void preproc(FileLine* fileline, const string& module, V3ParseImp* parsep);
+    static void preproc(FileLine* fileline, const string& module, V3InFilter* filterp, V3ParseImp* parsep);
     static void preprocInclude(FileLine* fileline, const string& module);
     static string dependFiles() { return ""; }   // Perl only
     static void defineCmdLine(const string& name, const string& value);

src/Verilator.cpp

@@ -98,7 +98,9 @@ void V3Global::readFiles() {
     //   AstNode::user4p()	// V3SymTable*    Package and typedef symbol names
     AstUser4InUse	inuser4;
 
-    V3Parse parser (v3Global.rootp());
+    V3InFilter filter (v3Global.opt.pipeFilter());
+
+    V3Parse parser (v3Global.rootp(), &filter);
     // Read top module
     for (V3StringList::const_iterator it = v3Global.opt.vFiles().begin();
 	 it != v3Global.opt.vFiles().end(); ++it) {
@@ -119,7 +121,7 @@ void V3Global::readFiles() {
 
     if (!v3Global.opt.preprocOnly()) {
 	// Resolve all modules cells refer to
-	V3LinkCells::link(v3Global.rootp());
+	V3LinkCells::link(v3Global.rootp(), &filter);
     }
 }
 

test_regress/t/t_pipe_exit_bad.pf

@@ -0,0 +1,9 @@
+#!/usr/bin/perl -w
+# DESCRIPTION: Verilator: Verilog Test example --pipe-filter script
+#
+# Copyright 2010 by Wilson Snyder. This program is free software; you can
+# redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+
+die "%Error: t_pipe_exit_bad.pf: Intentional bad exit status...\n";

test_regress/t/t_pipe_exit_bad.pl

@@ -0,0 +1,28 @@
+#!/usr/bin/perl
+if (!$::Driver) { use FindBin; exec("$FindBin::Bin/bootstrap.pl", @ARGV, $0); die; }
+# DESCRIPTION: Verilator: Verilog Test driver/expect definition
+#
+# Copyright 2010-2010 by Wilson Snyder. This program is free software; you can
+# redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+
+top_filename("t/t_pipe_filter.v");
+
+if (!$Self->{v3}) {
+    ok(1);
+} else {
+    compile (
+	v_flags2 => ['-E --pipe-filter \'perl t/t_pipe_exit_bad.pf\' '],
+	verilator_make_gcc=>0,
+	stdout_filename => $stdout_filename,
+	fails=>1,
+	expect=>
+'%Error: t_pipe_exit_bad.pf: Intentional bad exit status...
+%Error: File not found: t/t_pipe_filter.v
+%Error: Exiting due to.*',
+	);
+    ok(1);
+}
+
+1;

test_regress/t/t_pipe_filter.out

@@ -0,0 +1,52 @@
+`line 1 "t/t_pipe_filter.v" 1
+
+
+
+
+
+
+
+
+
+example line 10;
+example line 11;
+
+ 
+`line 13 "t/t_pipe_filter.v" 0
+`line 1 "t/t_pipe_filter_inc.v" 1
+int lint_off_line_7 = 1;
+int lint_off_line_8 = 1;
+
+
+
+
+
+inc line 6;
+inc line 7;  
+inc line 8;  
+inc line 9;
+`line 12 "t/t_pipe_filter_inc.v" 2
+`line 13 "t/t_pipe_filter.v" 0
+
+
+ 
+`line 15 "t/t_pipe_filter.v" 0
+`line 1 "t/t_pipe_filter_inc.v" 1
+int lint_off_line_7 = 1;
+int lint_off_line_8 = 1;
+
+
+
+
+
+inc line 6;
+inc line 7;  
+inc line 8;  
+inc line 9;
+`line 12 "t/t_pipe_filter_inc.v" 2
+`line 15 "t/t_pipe_filter.v" 0
+
+
+example line 15;
+example line 16;
+`line 19 "t/t_pipe_filter.v" 2

test_regress/t/t_pipe_filter.pf

@@ -0,0 +1,56 @@
+#!/usr/bin/perl -w
+# DESCRIPTION: Verilator: Verilog Test example --pipe-filter script
+#
+# Copyright 2010 by Wilson Snyder. This program is free software; you can
+# redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+
+use IO::File;
+use strict;
+
+our $Debug = 0;
+
+autoflush STDOUT;
+
+print STDERR "t_pipe_filter.pf: Hello in Perl\n" if $Debug;
+while (defined(my $cmd = <STDIN>)) {
+    print STDERR "t_pipe_filter.pf: gotcmd: $cmd" if $Debug;
+    if ($cmd =~ /^read "(.*)"/) {
+	my $filename = $1;
+	open(my $fh, "<$filename")
+	    or die "t_pipe_filter.pf: %Error: $! $filename\n";
+
+	my $wholefile="";
+	# It's faster to slurp the whole file then scan (if needed),
+	# then to read it into an array with getlines
+	{ local $/; undef $/; $wholefile = <$fh>; }
+	close $fh;
+
+	if ($wholefile =~ /example_lint/) {  # else short circuit
+	    my $lineno = 1; my $pos=0;
+	    my @prefixes;
+	    while (1) {
+		my $newpos=index($wholefile,"\n",$pos);
+		last if $newpos<$pos;
+		my $line = substr($wholefile,$pos,$newpos-$pos);
+		if ($line =~ /example_lint/) {
+		    # We don't have a way to specify this yet, so just for now
+		    #print STDERR $line;
+		    push @prefixes, "int lint_off_line_${lineno} = 1;\n";
+		}
+		$lineno++;
+		$pos = $newpos+1;
+	    }
+	    #print STDERR "Line count: $lineno\n";
+	    $wholefile = join('',@prefixes) . $wholefile;
+	}
+
+	print STDOUT "Content-Length: ".length($wholefile)."\n".$wholefile."\n";
+    } else {
+	die "t_pipe_filter.pf: %Error: Unknown command: $cmd\n";
+    }
+}
+
+print STDOUT "t_pipe_filter.pf: Fin\n" if $Debug;
+exit(0);

test_regress/t/t_pipe_filter.pl

@@ -0,0 +1,24 @@
+#!/usr/bin/perl
+if (!$::Driver) { use FindBin; exec("$FindBin::Bin/bootstrap.pl", @ARGV, $0); die; }
+# DESCRIPTION: Verilator: Verilog Test driver/expect definition
+#
+# Copyright 2010-2010 by Wilson Snyder. This program is free software; you can
+# redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+
+$Self->{golden_out} ||= "t/$Self->{name}.out";
+my $stdout_filename = "$Self->{obj_dir}/$Self->{name}__test.vpp";
+
+if (!$Self->{v3}) {
+    ok(1);
+} else {
+    compile (
+	v_flags2 => ['-E --pipe-filter \'perl t/t_pipe_filter.pf\' '],
+	verilator_make_gcc=>0,
+	stdout_filename => $stdout_filename,
+	);
+    ok(files_identical($stdout_filename, $Self->{golden_out}));
+}
+
+1;

test_regress/t/t_pipe_filter.v

@@ -0,0 +1,18 @@
+// DESCRIPTION: Verilator: Verilog Test module
+//
+// This file ONLY is placed into the Public Domain, for any use,
+// without warranty, 2010 by Wilson Snyder.
+
+//===========================================================================
+// Includes
+
+
+example line 10;
+example line 11;
+
+`include "t_pipe_filter_inc.v"
+// Twice to check caching of includes
+`include "t_pipe_filter_inc.v"
+
+example line 15;
+example line 16;

test_regress/t/t_pipe_filter_inc.v

@@ -0,0 +1,9 @@
+// DESCRIPTION: Verilator: Verilog Test module
+//
+// This file ONLY is placed into the Public Domain, for any use,
+// without warranty, 2010 by Wilson Snyder.
+
+inc line 6;
+inc line 7;  // example_lint_off_line FOO
+inc line 8;  // example_lint_off_line BAR
+inc line 9;