Merge pull request #104 from mithro/add-docs

docs: Adding the current fuzzers + minitests into documentation.

docs/conf.py

@@ -12,16 +12,19 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import os
 import re
 
+# Markdown support
+import recommonmark
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
+import os
-# import sys
+import sys
-# sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('.'))
+from markdown_code_symlinks import MarkdownCodeSymlinks
 
 # -- General configuration ------------------------------------------------
 
@@ -42,9 +45,10 @@ templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
-#
+source_suffix = ['.rst', '.md']
-# source_suffix = ['.rst', '.md']
+source_parsers = {
-source_suffix = '.rst'
+    '.md': 'recommonmark.parser.CommonMarkParser',
+}
 
 # The master toctree document.
 master_doc = 'index'
@@ -177,3 +181,12 @@ texinfo_documents = [
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'https://docs.python.org/': None}
+
+
+def setup(app):
+    MarkdownCodeSymlinks.find_links()
+    app.add_config_value(
+        'recommonmark_config', {
+            'github_code_repo': 'https://github.com/SymbiFlow/prjxray',
+        }, True)
+    app.add_transform(MarkdownCodeSymlinks)

docs/db_dev_process/fuzzers/005-tilegrid.md

@@ -0,0 +1 @@
+../../../fuzzers/005-tilegrid/README.md

docs/db_dev_process/fuzzers/011-ffconfig.md

@@ -0,0 +1 @@
+../../../fuzzers/011-ffconfig/README.md

docs/db_dev_process/fuzzers/012-clbn5ffmux.md

@@ -0,0 +1 @@
+../../../fuzzers/012-clbn5ffmux/README.md

docs/db_dev_process/fuzzers/013-clbncy0.md

@@ -0,0 +1 @@
+../../../fuzzers/013-clbncy0/README.md

docs/db_dev_process/fuzzers/014-ffsrcemux.md

@@ -0,0 +1 @@
+../../../fuzzers/014-ffsrcemux/README.md

docs/db_dev_process/fuzzers/015-clbnffmux.md

@@ -0,0 +1 @@
+../../../fuzzers/015-clbnffmux/README.md

docs/db_dev_process/fuzzers/016-clbnoutmux.md

@@ -0,0 +1 @@
+../../../fuzzers/016-clbnoutmux/README.md

docs/db_dev_process/fuzzers/017-clbprecyinit.md

@@ -0,0 +1 @@
+../../../fuzzers/017-clbprecyinit/README.md

docs/db_dev_process/fuzzers/018-clbram.md

@@ -0,0 +1 @@
+../../../fuzzers/018-clbram/README.md

docs/db_dev_process/fuzzers/019-ndi1mux.md

@@ -0,0 +1 @@
+../../../fuzzers/019-ndi1mux/README.md

docs/db_dev_process/fuzzers/050-intpips.md

@@ -0,0 +1 @@
+../../../fuzzers/050-intpips/README.md

docs/db_dev_process/fuzzers/051-imuxlout.md

@@ -0,0 +1 @@
+../../../fuzzers/051-imuxlout/README.md

docs/db_dev_process/fuzzers/052-clkin.md

@@ -0,0 +1 @@
+../../../fuzzers/052-clkin/README.md

docs/db_dev_process/fuzzers/053-ctrlin.md

@@ -0,0 +1 @@
+../../../fuzzers/053-ctrlin/README.md

docs/db_dev_process/fuzzers/054-gfan.md

@@ -0,0 +1 @@
+../../../fuzzers/054-gfan/README.md

docs/db_dev_process/fuzzers/055-gnd.md

@@ -0,0 +1 @@
+../../../fuzzers/055-gnd/README.md

docs/db_dev_process/fuzzers/056-rempips.md

@@ -0,0 +1 @@
+../../../fuzzers/056-rempips/README.md

docs/db_dev_process/fuzzers/057-bipips.md

@@ -0,0 +1 @@
+../../../fuzzers/057-bipips/README.md

docs/db_dev_process/fuzzers/058-hclkpips.md

@@ -0,0 +1 @@
+../../../fuzzers/058-hclkpips/README.md

docs/db_dev_process/minitests/bram.md

@@ -0,0 +1 @@
+../../../minitests/bram/README.md

docs/db_dev_process/minitests/clb_bused.md

@@ -0,0 +1 @@
+../../../minitests/clb_bused/README.md

docs/db_dev_process/minitests/clb_ffcfg.md

@@ -0,0 +1 @@
+../../../minitests/clb_ffcfg/README.md

docs/db_dev_process/minitests/clb_muxf8.md

@@ -0,0 +1 @@
+../../../minitests/clb_muxf8/README.md

docs/db_dev_process/minitests/clb_n5ffmux.md

@@ -0,0 +1 @@
+../../../minitests/clb_n5ffmux/README.md

docs/db_dev_process/minitests/clb_ncy0.md

@@ -0,0 +1 @@
+../../../minitests/clb_ncy0/README.md

docs/db_dev_process/minitests/clb_ndi1mux.md

@@ -0,0 +1 @@
+../../../minitests/clb_ndi1mux/README.md

docs/db_dev_process/minitests/clb_nffmux.md

@@ -0,0 +1 @@
+../../../minitests/clb_nffmux/README.md

docs/db_dev_process/minitests/clb_noutmux.md

@@ -0,0 +1 @@
+../../../minitests/clb_noutmux/README.md

docs/db_dev_process/minitests/clb_ram.md

@@ -0,0 +1 @@
+../../../minitests/clb_ram/README.md

docs/db_dev_process/minitests/fixedpnr.md

@@ -0,0 +1 @@
+../../../minitests/fixedpnr/README.md

docs/db_dev_process/minitests/partial_reconfig_flow.md

@@ -0,0 +1 @@
+../../../minitests/partial_reconfig_flow/README.md

docs/db_dev_process/minitests/picorv32-v.md

@@ -0,0 +1 @@
+../../../minitests/picorv32-v/README.md

docs/db_dev_process/minitests/picorv32-y.md

@@ -0,0 +1 @@
+../../../minitests/picorv32-y/README.md

docs/db_dev_process/minitests/roi_harness.md

@@ -0,0 +1 @@
+../../../minitests/roi_harness/README.md

docs/db_dev_process/overview.rst

@@ -10,8 +10,8 @@ This is where we describe the logical components in a device to VPR.
 * VtR also has its own synthesis tool called ODIN-II, but we are using `Yosys <https://github.com/YosysHQ/yosys>`_ instead of that.
   
 
-SymbiFlow/prjxray/fuzzers/
+Fuzzers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^
 Fuzzers are things that generate a design, feed it to Vivado, and look at the resulting bitstream to make some conclusion.
 This is how the contents of the database are generated.
 
@@ -25,11 +25,32 @@ By looking at all the resulting specimens, you can correlate which bits in which
 
 Looking at the implemented design in Vivado with "Show Routing Resources" turned on is quite helpful in understanding what all choices exist.
 
-SymbiFlow/prjxray/tools/
+.. toctree::
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   :maxdepth: 1
+   :caption: Current Fuzzers
+   :glob:
+
+   fuzzers/*
+
+Minitests
+^^^^^^^^^
+
+Minitests are experiments to figure out how things work. They allow us to understand how to better write new fuzzers.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Current Minitests
+   :glob:
+
+   minitests/*
+
+Tools
+^^^^^
+
+`SymbiFlow/prjxray/tools/`
+
 Here, you can find various programs to work with bitstreams, mainly to assist building fuzzers.
 
 SymbiFlow/prjxray/minitests/roi_harness
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Shows how to use a bunch of tools together to patch an existing bitstream with hand-crafted FASM (FPGA assembler).
-

docs/index.rst

@@ -25,5 +25,5 @@ to develop a free and open Verilog to bitstream toolchain for these devices.
 .. toctree::
    :maxdepth: 2
    :caption: Database Development Process
-   
+
    db_dev_process/overview

docs/markdown_code_symlinks.py

@@ -0,0 +1,179 @@
+import logging
+import os
+
+from recommonmark import transform
+"""
+Allow linking of Markdown documentation from the source code tree into the Sphinx
+documentation tree.
+
+The Markdown documents will have links relative to the source code root, rather
+than the place they are now linked too - this code fixes these paths up.
+
+We also want links from two Markdown documents found in the Sphinx docs to
+work, so that is also fixed up.
+"""
+
+
+def path_contains(parent_path, child_path):
+    """Check a path contains another path.
+
+    >>> path_contains("a/b", "a/b")
+    True
+    >>> path_contains("a/b", "a/b/")
+    True
+    >>> path_contains("a/b", "a/b/c")
+    True
+    >>> path_contains("a/b", "c")
+    False
+    >>> path_contains("a/b", "c/b")
+    False
+    >>> path_contains("a/b", "c/../a/b/d")
+    True
+    >>> path_contains("../a/b", "../a/b/d")
+    True
+    >>> path_contains("../a/b", "../a/c")
+    False
+    >>> path_contains("a", "abc")
+    False
+    >>> path_contains("aa", "abc")
+    False
+    """
+    # Append a separator to the end of both paths to work around the fact that
+    # os.path.commonprefix does character by character comparisons rather than
+    # path segment by path segment.
+    parent_path = os.path.join(os.path.normpath(parent_path), '')
+    child_path = os.path.join(os.path.normpath(child_path), '')
+    common_path = os.path.commonprefix((parent_path, child_path))
+    return common_path == parent_path
+
+
+def relative(parent_dir, child_path):
+    """Get the relative between a path that contains another path."""
+    child_dir = os.path.dirname(child_path)
+    assert path_contains(parent_dir, child_dir), "{} not inside {}".format(
+        child_path, parent_dir)
+    return os.path.relpath(child_path, start=parent_dir)
+
+
+class MarkdownCodeSymlinks(transform.AutoStructify, object):
+    docs_root_dir = os.path.realpath(os.path.dirname(__file__))
+    code_root_dir = os.path.realpath(os.path.join(docs_root_dir, "..", ".."))
+
+    mapping = {
+        'docs2code': {},
+        'code2docs': {},
+    }
+
+    @classmethod
+    def relative_code(cls, url):
+        """Get a value relative to the code directory."""
+        return relative(cls.code_root_dir, url)
+
+    @classmethod
+    def relative_docs(cls, url):
+        """Get a value relative to the docs directory."""
+        return relative(cls.docs_root_dir, url)
+
+    @classmethod
+    def add_mapping(cls, docs_rel, code_rel):
+        assert docs_rel not in cls.mapping['docs2code'], """\
+Assertion error! Document already in mapping!
+    New Value: {}
+Current Value: {}
+""".format(docs_rel, cls.mapping['docs2code'][docs_rel])
+        assert code_rel not in cls.mapping['code2docs'], """\
+Assertion error! Document already in mapping!
+    New Value: {}
+Current Value: {}
+""".format(docs_rel, cls.mapping['code2docs'][code_rel])
+
+        cls.mapping['docs2code'][docs_rel] = code_rel
+        cls.mapping['code2docs'][code_rel] = docs_rel
+
+    @classmethod
+    def find_links(cls):
+        """Walk the docs dir and find links to docs in the code dir."""
+        for root, dirs, files in os.walk(cls.docs_root_dir):
+            for fname in files:
+                fpath = os.path.abspath(os.path.join(root, fname))
+
+                if not os.path.islink(fpath):
+                    continue
+
+                link_path = os.path.join(root, os.readlink(fpath))
+                # Is link outside the code directory?
+                if not path_contains(cls.code_root_dir, link_path):
+                    continue
+
+                # Is link internal to the docs directory?
+                if path_contains(cls.docs_root_dir, link_path):
+                    continue
+
+                docs_rel = cls.relative_docs(fpath)
+                code_rel = cls.relative_code(link_path)
+
+                cls.add_mapping(docs_rel, code_rel)
+        import pprint
+        pprint.pprint(cls.mapping)
+
+    @property
+    def url_resolver(self):
+        return self._url_resolver
+
+    @url_resolver.setter
+    def url_resolver(self, value):
+        print(self, value)
+
+    # Resolve a link from one markdown to another document.
+    def _url_resolver(self, ourl):
+        """Resolve a URL found in a markdown file."""
+        assert self.docs_root_dir == os.path.realpath(self.root_dir), """\
+Configuration error! Document Root != Current Root
+Document Root: {}
+ Current Root: {}
+""".format(self.docs_root_dir, self.root_dir)
+
+        src_path = os.path.abspath(self.document['source'])
+        src_dir = os.path.dirname(src_path)
+        dst_path = os.path.abspath(os.path.join(self.docs_root_dir, ourl))
+        dst_rsrc = os.path.relpath(dst_path, start=src_dir)
+
+        src_rdoc = self.relative_docs(src_path)
+
+        print
+        print("url_resolver")
+        print(src_path)
+        print(dst_path)
+        print(dst_rsrc)
+        print(src_rdoc)
+
+        # Is the source document a linked one?
+        if src_rdoc not in self.mapping['docs2code']:
+            # Don't do any rewriting on non-linked markdown.
+            url = ourl
+
+        # Is the destination also inside docs?
+        elif dst_rsrc not in self.mapping['code2docs']:
+            # Return a path to the GitHub repo.
+            url = "{}/blob/master/{}".format(
+                self.config['github_code_repo'], dst_rsrc)
+        else:
+            url = os.path.relpath(
+                os.path.join(
+                    self.docs_root_dir, self.mapping['code2docs'][dst_rsrc]),
+                start=src_dir)
+            base_url, ext = os.path.splitext(url)
+            assert ext in (".md",
+                           ".markdown"), ("Unknown extension {}".format(ext))
+            url = "{}.html".format(base_url)
+
+        print("---")
+        print(ourl)
+        print(url)
+        print
+        return url
+
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod()

docs/requirements.txt

@@ -0,0 +1,8 @@
+docutils
+sphinx
+sphinx-autobuild
+
+breathe
+recommonmark
+sphinx_rtd_theme
+sphinxcontrib-napoleon

requirements.txt

@@ -1,5 +1,2 @@
 futures
-sphinx
-sphinx_rtd_theme
-sphinx-autobuild
 yapf