diff --git a/docs/conf.py b/docs/conf.py index 9d14b725..e4fbfd18 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,16 +12,19 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os import re +# Markdown support +import recommonmark + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('.')) +from markdown_code_symlinks import MarkdownCodeSymlinks # -- General configuration ------------------------------------------------ @@ -42,9 +45,10 @@ templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] +source_parsers = { + '.md': 'recommonmark.parser.CommonMarkParser', +} # The master toctree document. master_doc = 'index' @@ -177,3 +181,12 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} + + +def setup(app): + MarkdownCodeSymlinks.find_links() + app.add_config_value( + 'recommonmark_config', { + 'github_code_repo': 'https://github.com/SymbiFlow/prjxray', + }, True) + app.add_transform(MarkdownCodeSymlinks) diff --git a/docs/db_dev_process/fuzzers/005-tilegrid.md b/docs/db_dev_process/fuzzers/005-tilegrid.md new file mode 120000 index 00000000..6e65a4da --- /dev/null +++ b/docs/db_dev_process/fuzzers/005-tilegrid.md @@ -0,0 +1 @@ +../../../fuzzers/005-tilegrid/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/011-ffconfig.md b/docs/db_dev_process/fuzzers/011-ffconfig.md new file mode 120000 index 00000000..157999c5 --- /dev/null +++ b/docs/db_dev_process/fuzzers/011-ffconfig.md @@ -0,0 +1 @@ +../../../fuzzers/011-ffconfig/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/012-clbn5ffmux.md b/docs/db_dev_process/fuzzers/012-clbn5ffmux.md new file mode 120000 index 00000000..4c9865de --- /dev/null +++ b/docs/db_dev_process/fuzzers/012-clbn5ffmux.md @@ -0,0 +1 @@ +../../../fuzzers/012-clbn5ffmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/013-clbncy0.md b/docs/db_dev_process/fuzzers/013-clbncy0.md new file mode 120000 index 00000000..ffd80bf3 --- /dev/null +++ b/docs/db_dev_process/fuzzers/013-clbncy0.md @@ -0,0 +1 @@ +../../../fuzzers/013-clbncy0/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/014-ffsrcemux.md b/docs/db_dev_process/fuzzers/014-ffsrcemux.md new file mode 120000 index 00000000..20c90e11 --- /dev/null +++ b/docs/db_dev_process/fuzzers/014-ffsrcemux.md @@ -0,0 +1 @@ +../../../fuzzers/014-ffsrcemux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/015-clbnffmux.md b/docs/db_dev_process/fuzzers/015-clbnffmux.md new file mode 120000 index 00000000..2e65f3f3 --- /dev/null +++ b/docs/db_dev_process/fuzzers/015-clbnffmux.md @@ -0,0 +1 @@ +../../../fuzzers/015-clbnffmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/016-clbnoutmux.md b/docs/db_dev_process/fuzzers/016-clbnoutmux.md new file mode 120000 index 00000000..efc775b6 --- /dev/null +++ b/docs/db_dev_process/fuzzers/016-clbnoutmux.md @@ -0,0 +1 @@ +../../../fuzzers/016-clbnoutmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/017-clbprecyinit.md b/docs/db_dev_process/fuzzers/017-clbprecyinit.md new file mode 120000 index 00000000..1080e6f4 --- /dev/null +++ b/docs/db_dev_process/fuzzers/017-clbprecyinit.md @@ -0,0 +1 @@ +../../../fuzzers/017-clbprecyinit/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/018-clbram.md b/docs/db_dev_process/fuzzers/018-clbram.md new file mode 120000 index 00000000..50454ea9 --- /dev/null +++ b/docs/db_dev_process/fuzzers/018-clbram.md @@ -0,0 +1 @@ +../../../fuzzers/018-clbram/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/019-ndi1mux.md b/docs/db_dev_process/fuzzers/019-ndi1mux.md new file mode 120000 index 00000000..347d074a --- /dev/null +++ b/docs/db_dev_process/fuzzers/019-ndi1mux.md @@ -0,0 +1 @@ +../../../fuzzers/019-ndi1mux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/050-intpips.md b/docs/db_dev_process/fuzzers/050-intpips.md new file mode 120000 index 00000000..b097a73e --- /dev/null +++ b/docs/db_dev_process/fuzzers/050-intpips.md @@ -0,0 +1 @@ +../../../fuzzers/050-intpips/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/051-imuxlout.md b/docs/db_dev_process/fuzzers/051-imuxlout.md new file mode 120000 index 00000000..36da9404 --- /dev/null +++ b/docs/db_dev_process/fuzzers/051-imuxlout.md @@ -0,0 +1 @@ +../../../fuzzers/051-imuxlout/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/052-clkin.md b/docs/db_dev_process/fuzzers/052-clkin.md new file mode 120000 index 00000000..815c7cac --- /dev/null +++ b/docs/db_dev_process/fuzzers/052-clkin.md @@ -0,0 +1 @@ +../../../fuzzers/052-clkin/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/053-ctrlin.md b/docs/db_dev_process/fuzzers/053-ctrlin.md new file mode 120000 index 00000000..60fc07aa --- /dev/null +++ b/docs/db_dev_process/fuzzers/053-ctrlin.md @@ -0,0 +1 @@ +../../../fuzzers/053-ctrlin/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/054-gfan.md b/docs/db_dev_process/fuzzers/054-gfan.md new file mode 120000 index 00000000..fe1c03ae --- /dev/null +++ b/docs/db_dev_process/fuzzers/054-gfan.md @@ -0,0 +1 @@ +../../../fuzzers/054-gfan/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/055-gnd.md b/docs/db_dev_process/fuzzers/055-gnd.md new file mode 120000 index 00000000..f028afd9 --- /dev/null +++ b/docs/db_dev_process/fuzzers/055-gnd.md @@ -0,0 +1 @@ +../../../fuzzers/055-gnd/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/056-rempips.md b/docs/db_dev_process/fuzzers/056-rempips.md new file mode 120000 index 00000000..e79879c4 --- /dev/null +++ b/docs/db_dev_process/fuzzers/056-rempips.md @@ -0,0 +1 @@ +../../../fuzzers/056-rempips/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/057-bipips.md b/docs/db_dev_process/fuzzers/057-bipips.md new file mode 120000 index 00000000..040214eb --- /dev/null +++ b/docs/db_dev_process/fuzzers/057-bipips.md @@ -0,0 +1 @@ +../../../fuzzers/057-bipips/README.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/058-hclkpips.md b/docs/db_dev_process/fuzzers/058-hclkpips.md new file mode 120000 index 00000000..071ce3dd --- /dev/null +++ b/docs/db_dev_process/fuzzers/058-hclkpips.md @@ -0,0 +1 @@ +../../../fuzzers/058-hclkpips/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/bram.md b/docs/db_dev_process/minitests/bram.md new file mode 120000 index 00000000..527cf3e3 --- /dev/null +++ b/docs/db_dev_process/minitests/bram.md @@ -0,0 +1 @@ +../../../minitests/bram/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_bused.md b/docs/db_dev_process/minitests/clb_bused.md new file mode 120000 index 00000000..031bc25e --- /dev/null +++ b/docs/db_dev_process/minitests/clb_bused.md @@ -0,0 +1 @@ +../../../minitests/clb_bused/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_ffcfg.md b/docs/db_dev_process/minitests/clb_ffcfg.md new file mode 120000 index 00000000..d560c280 --- /dev/null +++ b/docs/db_dev_process/minitests/clb_ffcfg.md @@ -0,0 +1 @@ +../../../minitests/clb_ffcfg/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_muxf8.md b/docs/db_dev_process/minitests/clb_muxf8.md new file mode 120000 index 00000000..53b0db4a --- /dev/null +++ b/docs/db_dev_process/minitests/clb_muxf8.md @@ -0,0 +1 @@ +../../../minitests/clb_muxf8/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_n5ffmux.md b/docs/db_dev_process/minitests/clb_n5ffmux.md new file mode 120000 index 00000000..ec617644 --- /dev/null +++ b/docs/db_dev_process/minitests/clb_n5ffmux.md @@ -0,0 +1 @@ +../../../minitests/clb_n5ffmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_ncy0.md b/docs/db_dev_process/minitests/clb_ncy0.md new file mode 120000 index 00000000..22d88244 --- /dev/null +++ b/docs/db_dev_process/minitests/clb_ncy0.md @@ -0,0 +1 @@ +../../../minitests/clb_ncy0/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_ndi1mux.md b/docs/db_dev_process/minitests/clb_ndi1mux.md new file mode 120000 index 00000000..c4b7aa29 --- /dev/null +++ b/docs/db_dev_process/minitests/clb_ndi1mux.md @@ -0,0 +1 @@ +../../../minitests/clb_ndi1mux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_nffmux.md b/docs/db_dev_process/minitests/clb_nffmux.md new file mode 120000 index 00000000..3b41da5b --- /dev/null +++ b/docs/db_dev_process/minitests/clb_nffmux.md @@ -0,0 +1 @@ +../../../minitests/clb_nffmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_noutmux.md b/docs/db_dev_process/minitests/clb_noutmux.md new file mode 120000 index 00000000..ace55c8a --- /dev/null +++ b/docs/db_dev_process/minitests/clb_noutmux.md @@ -0,0 +1 @@ +../../../minitests/clb_noutmux/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/clb_ram.md b/docs/db_dev_process/minitests/clb_ram.md new file mode 120000 index 00000000..1f946093 --- /dev/null +++ b/docs/db_dev_process/minitests/clb_ram.md @@ -0,0 +1 @@ +../../../minitests/clb_ram/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/fixedpnr.md b/docs/db_dev_process/minitests/fixedpnr.md new file mode 120000 index 00000000..406424c0 --- /dev/null +++ b/docs/db_dev_process/minitests/fixedpnr.md @@ -0,0 +1 @@ +../../../minitests/fixedpnr/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/partial_reconfig_flow.md b/docs/db_dev_process/minitests/partial_reconfig_flow.md new file mode 120000 index 00000000..1a9a42e7 --- /dev/null +++ b/docs/db_dev_process/minitests/partial_reconfig_flow.md @@ -0,0 +1 @@ +../../../minitests/partial_reconfig_flow/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/picorv32-v.md b/docs/db_dev_process/minitests/picorv32-v.md new file mode 120000 index 00000000..d0d80c60 --- /dev/null +++ b/docs/db_dev_process/minitests/picorv32-v.md @@ -0,0 +1 @@ +../../../minitests/picorv32-v/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/picorv32-y.md b/docs/db_dev_process/minitests/picorv32-y.md new file mode 120000 index 00000000..38591c50 --- /dev/null +++ b/docs/db_dev_process/minitests/picorv32-y.md @@ -0,0 +1 @@ +../../../minitests/picorv32-y/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/roi_harness.md b/docs/db_dev_process/minitests/roi_harness.md new file mode 120000 index 00000000..5a587711 --- /dev/null +++ b/docs/db_dev_process/minitests/roi_harness.md @@ -0,0 +1 @@ +../../../minitests/roi_harness/README.md \ No newline at end of file diff --git a/docs/db_dev_process/overview.rst b/docs/db_dev_process/overview.rst index 08e26a8d..049b6efb 100644 --- a/docs/db_dev_process/overview.rst +++ b/docs/db_dev_process/overview.rst @@ -10,8 +10,8 @@ This is where we describe the logical components in a device to VPR. * VtR also has its own synthesis tool called ODIN-II, but we are using `Yosys `_ instead of that. -SymbiFlow/prjxray/fuzzers/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Fuzzers +^^^^^^^ Fuzzers are things that generate a design, feed it to Vivado, and look at the resulting bitstream to make some conclusion. This is how the contents of the database are generated. @@ -25,11 +25,32 @@ By looking at all the resulting specimens, you can correlate which bits in which Looking at the implemented design in Vivado with "Show Routing Resources" turned on is quite helpful in understanding what all choices exist. -SymbiFlow/prjxray/tools/ -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. toctree:: + :maxdepth: 1 + :caption: Current Fuzzers + :glob: + + fuzzers/* + +Minitests +^^^^^^^^^ + +Minitests are experiments to figure out how things work. They allow us to understand how to better write new fuzzers. + +.. toctree:: + :maxdepth: 1 + :caption: Current Minitests + :glob: + + minitests/* + +Tools +^^^^^ + +`SymbiFlow/prjxray/tools/` + Here, you can find various programs to work with bitstreams, mainly to assist building fuzzers. SymbiFlow/prjxray/minitests/roi_harness ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Shows how to use a bunch of tools together to patch an existing bitstream with hand-crafted FASM (FPGA assembler). - diff --git a/docs/index.rst b/docs/index.rst index 0a607e3c..e9963287 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,5 +25,5 @@ to develop a free and open Verilog to bitstream toolchain for these devices. .. toctree:: :maxdepth: 2 :caption: Database Development Process - + db_dev_process/overview diff --git a/docs/markdown_code_symlinks.py b/docs/markdown_code_symlinks.py new file mode 100644 index 00000000..1955493e --- /dev/null +++ b/docs/markdown_code_symlinks.py @@ -0,0 +1,179 @@ +import logging +import os + +from recommonmark import transform +""" +Allow linking of Markdown documentation from the source code tree into the Sphinx +documentation tree. + +The Markdown documents will have links relative to the source code root, rather +than the place they are now linked too - this code fixes these paths up. + +We also want links from two Markdown documents found in the Sphinx docs to +work, so that is also fixed up. +""" + + +def path_contains(parent_path, child_path): + """Check a path contains another path. + + >>> path_contains("a/b", "a/b") + True + >>> path_contains("a/b", "a/b/") + True + >>> path_contains("a/b", "a/b/c") + True + >>> path_contains("a/b", "c") + False + >>> path_contains("a/b", "c/b") + False + >>> path_contains("a/b", "c/../a/b/d") + True + >>> path_contains("../a/b", "../a/b/d") + True + >>> path_contains("../a/b", "../a/c") + False + >>> path_contains("a", "abc") + False + >>> path_contains("aa", "abc") + False + """ + # Append a separator to the end of both paths to work around the fact that + # os.path.commonprefix does character by character comparisons rather than + # path segment by path segment. + parent_path = os.path.join(os.path.normpath(parent_path), '') + child_path = os.path.join(os.path.normpath(child_path), '') + common_path = os.path.commonprefix((parent_path, child_path)) + return common_path == parent_path + + +def relative(parent_dir, child_path): + """Get the relative between a path that contains another path.""" + child_dir = os.path.dirname(child_path) + assert path_contains(parent_dir, child_dir), "{} not inside {}".format( + child_path, parent_dir) + return os.path.relpath(child_path, start=parent_dir) + + +class MarkdownCodeSymlinks(transform.AutoStructify, object): + docs_root_dir = os.path.realpath(os.path.dirname(__file__)) + code_root_dir = os.path.realpath(os.path.join(docs_root_dir, "..", "..")) + + mapping = { + 'docs2code': {}, + 'code2docs': {}, + } + + @classmethod + def relative_code(cls, url): + """Get a value relative to the code directory.""" + return relative(cls.code_root_dir, url) + + @classmethod + def relative_docs(cls, url): + """Get a value relative to the docs directory.""" + return relative(cls.docs_root_dir, url) + + @classmethod + def add_mapping(cls, docs_rel, code_rel): + assert docs_rel not in cls.mapping['docs2code'], """\ +Assertion error! Document already in mapping! + New Value: {} +Current Value: {} +""".format(docs_rel, cls.mapping['docs2code'][docs_rel]) + assert code_rel not in cls.mapping['code2docs'], """\ +Assertion error! Document already in mapping! + New Value: {} +Current Value: {} +""".format(docs_rel, cls.mapping['code2docs'][code_rel]) + + cls.mapping['docs2code'][docs_rel] = code_rel + cls.mapping['code2docs'][code_rel] = docs_rel + + @classmethod + def find_links(cls): + """Walk the docs dir and find links to docs in the code dir.""" + for root, dirs, files in os.walk(cls.docs_root_dir): + for fname in files: + fpath = os.path.abspath(os.path.join(root, fname)) + + if not os.path.islink(fpath): + continue + + link_path = os.path.join(root, os.readlink(fpath)) + # Is link outside the code directory? + if not path_contains(cls.code_root_dir, link_path): + continue + + # Is link internal to the docs directory? + if path_contains(cls.docs_root_dir, link_path): + continue + + docs_rel = cls.relative_docs(fpath) + code_rel = cls.relative_code(link_path) + + cls.add_mapping(docs_rel, code_rel) + import pprint + pprint.pprint(cls.mapping) + + @property + def url_resolver(self): + return self._url_resolver + + @url_resolver.setter + def url_resolver(self, value): + print(self, value) + + # Resolve a link from one markdown to another document. + def _url_resolver(self, ourl): + """Resolve a URL found in a markdown file.""" + assert self.docs_root_dir == os.path.realpath(self.root_dir), """\ +Configuration error! Document Root != Current Root +Document Root: {} + Current Root: {} +""".format(self.docs_root_dir, self.root_dir) + + src_path = os.path.abspath(self.document['source']) + src_dir = os.path.dirname(src_path) + dst_path = os.path.abspath(os.path.join(self.docs_root_dir, ourl)) + dst_rsrc = os.path.relpath(dst_path, start=src_dir) + + src_rdoc = self.relative_docs(src_path) + + print + print("url_resolver") + print(src_path) + print(dst_path) + print(dst_rsrc) + print(src_rdoc) + + # Is the source document a linked one? + if src_rdoc not in self.mapping['docs2code']: + # Don't do any rewriting on non-linked markdown. + url = ourl + + # Is the destination also inside docs? + elif dst_rsrc not in self.mapping['code2docs']: + # Return a path to the GitHub repo. + url = "{}/blob/master/{}".format( + self.config['github_code_repo'], dst_rsrc) + else: + url = os.path.relpath( + os.path.join( + self.docs_root_dir, self.mapping['code2docs'][dst_rsrc]), + start=src_dir) + base_url, ext = os.path.splitext(url) + assert ext in (".md", + ".markdown"), ("Unknown extension {}".format(ext)) + url = "{}.html".format(base_url) + + print("---") + print(ourl) + print(url) + print + return url + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..56c1f78d --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,8 @@ +docutils +sphinx +sphinx-autobuild + +breathe +recommonmark +sphinx_rtd_theme +sphinxcontrib-napoleon diff --git a/requirements.txt b/requirements.txt index f16b1bd9..ab61583a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ futures -sphinx -sphinx_rtd_theme -sphinx-autobuild yapf