diff --git a/docs/.gitignore b/docs/.gitignore index 9d7ce536..c8d939ef 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,3 @@ _build -db_dev_process/fuzzers/*.md -db_dev_process/minitests/*.md +src/db_dev_process/fuzzers/*.md +src/db_dev_process/minitests/*.md diff --git a/docs/Makefile b/docs/Makefile index d845e77b..02bebc6c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXAUTOBUILD = sphinx-autobuild SPHINXPROJ = ProjectX-Ray -SOURCEDIR = . +SOURCEDIR = src BUILDDIR = _build # Put it first so that "make" without argument is like "make help". @@ -22,10 +22,10 @@ livehtml: # Update fuzzer / minitest markdown links. fuzzers-links: - @mkdir -p $(MAKEDIR)/db_dev_process/fuzzers - @cd $(MAKEDIR)/db_dev_process/fuzzers && rm -f *.md - @cd $(MAKEDIR)/db_dev_process/fuzzers && \ - for i in ../../../fuzzers/*; do \ + @mkdir -p $(MAKEDIR)/${SOURCEDIR}/db_dev_process/fuzzers + @cd $(MAKEDIR)/${SOURCEDIR}/db_dev_process/fuzzers && rm -f *.md + @cd $(MAKEDIR)/${SOURCEDIR}/db_dev_process/fuzzers && \ + for i in ../../../../fuzzers/*; do \ n=$$(basename $$i | sed -e's/^[0-9][0-9][0-9]-//'); \ if [ ! -d $$i ]; then \ continue; \ @@ -42,10 +42,10 @@ fuzzers-links: done minitests-links: - @mkdir -p $(MAKEDIR)/db_dev_process/minitests - @cd $(MAKEDIR)/db_dev_process/minitests && rm -f *.md - @cd $(MAKEDIR)/db_dev_process/minitests && \ - for i in ../../../minitests/*; do \ + @mkdir -p $(MAKEDIR)/${SOURCEDIR}/db_dev_process/minitests + @cd $(MAKEDIR)/${SOURCEDIR}/db_dev_process/minitests && rm -f *.md + @cd $(MAKEDIR)/${SOURCEDIR}/db_dev_process/minitests && \ + for i in ../../../../minitests/*; do \ n=$$(basename $$i | sed -e's/^[0-9][0-9][0-9]-//'); \ if [ ! -d $$i ]; then \ continue; \ diff --git a/docs/architecture/code_of_conduct.md b/docs/architecture/code_of_conduct.md deleted file mode 120000 index a3613c99..00000000 --- a/docs/architecture/code_of_conduct.md +++ /dev/null @@ -1 +0,0 @@ -../../CODE_OF_CONDUCT.md \ No newline at end of file diff --git a/docs/architecture/copying.md b/docs/architecture/copying.md deleted file mode 120000 index 7d29222e..00000000 --- a/docs/architecture/copying.md +++ /dev/null @@ -1 +0,0 @@ -../../COPYING \ No newline at end of file diff --git a/docs/architecture/updating_the_docs.md b/docs/architecture/updating_the_docs.md deleted file mode 120000 index 32e89853..00000000 --- a/docs/architecture/updating_the_docs.md +++ /dev/null @@ -1 +0,0 @@ -../../UPDATING-THE-DOCS.md \ No newline at end of file diff --git a/docs/db_dev_process/contributing.md b/docs/db_dev_process/contributing.md deleted file mode 120000 index f939e75f..00000000 --- a/docs/db_dev_process/contributing.md +++ /dev/null @@ -1 +0,0 @@ -../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/index b/docs/db_dev_process/fuzzers/index deleted file mode 120000 index 8e8d4720..00000000 --- a/docs/db_dev_process/fuzzers/index +++ /dev/null @@ -1 +0,0 @@ -../../../fuzzers \ No newline at end of file diff --git a/docs/db_dev_process/minitests/fixedpnr.md b/docs/db_dev_process/minitests/fixedpnr.md deleted file mode 120000 index 406424c0..00000000 --- a/docs/db_dev_process/minitests/fixedpnr.md +++ /dev/null @@ -1 +0,0 @@ -../../../minitests/fixedpnr/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/index b/docs/db_dev_process/minitests/index deleted file mode 120000 index 9f989966..00000000 --- a/docs/db_dev_process/minitests/index +++ /dev/null @@ -1 +0,0 @@ -../../../minitests \ No newline at end of file diff --git a/docs/db_dev_process/minitests/partial_reconfig_flow.md b/docs/db_dev_process/minitests/partial_reconfig_flow.md deleted file mode 120000 index 1a9a42e7..00000000 --- a/docs/db_dev_process/minitests/partial_reconfig_flow.md +++ /dev/null @@ -1 +0,0 @@ -../../../minitests/partial_reconfig_flow/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/picorv32-v.md b/docs/db_dev_process/minitests/picorv32-v.md deleted file mode 120000 index d0d80c60..00000000 --- a/docs/db_dev_process/minitests/picorv32-v.md +++ /dev/null @@ -1 +0,0 @@ -../../../minitests/picorv32-v/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/picorv32-y.md b/docs/db_dev_process/minitests/picorv32-y.md deleted file mode 120000 index 38591c50..00000000 --- a/docs/db_dev_process/minitests/picorv32-y.md +++ /dev/null @@ -1 +0,0 @@ -../../../minitests/picorv32-y/README.md \ No newline at end of file diff --git a/docs/db_dev_process/minitests/roi_harness.md b/docs/db_dev_process/minitests/roi_harness.md deleted file mode 120000 index 5a587711..00000000 --- a/docs/db_dev_process/minitests/roi_harness.md +++ /dev/null @@ -1 +0,0 @@ -../../../minitests/roi_harness/README.md \ No newline at end of file diff --git a/docs/db_dev_process/readme.md b/docs/db_dev_process/readme.md deleted file mode 120000 index fe840054..00000000 --- a/docs/db_dev_process/readme.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/docs/markdown_code_symlinks.py b/docs/markdown_code_symlinks.py deleted file mode 100644 index f43bfcaa..00000000 --- a/docs/markdown_code_symlinks.py +++ /dev/null @@ -1,250 +0,0 @@ -import logging -import os, sys -from os.path import splitext -from docutils import nodes -from sphinx import addnodes -from sphinx.roles import XRefRole -from sphinx.domains import Domain -from sphinx.util.nodes import make_refnode - -if sys.version_info < (3, 0): - from urlparse import urlparse, unquote -else: - from urllib.parse import urlparse, unquote - -from recommonmark import parser -""" -Allow linking of Markdown documentation from the source code tree into the Sphinx -documentation tree. - -The Markdown documents will have links relative to the source code root, rather -than the place they are now linked too - this code fixes these paths up. - -We also want links from two Markdown documents found in the Sphinx docs to -work, so that is also fixed up. -""" - - -def path_contains(parent_path, child_path): - """Check a path contains another path. - - >>> path_contains("a/b", "a/b") - True - >>> path_contains("a/b", "a/b/") - True - >>> path_contains("a/b", "a/b/c") - True - >>> path_contains("a/b", "c") - False - >>> path_contains("a/b", "c/b") - False - >>> path_contains("a/b", "c/../a/b/d") - True - >>> path_contains("../a/b", "../a/b/d") - True - >>> path_contains("../a/b", "../a/c") - False - >>> path_contains("a", "abc") - False - >>> path_contains("aa", "abc") - False - """ - # Append a separator to the end of both paths to work around the fact that - # os.path.commonprefix does character by character comparisons rather than - # path segment by path segment. - parent_path = os.path.join(os.path.normpath(parent_path), '') - child_path = os.path.join(os.path.normpath(child_path), '') - common_path = os.path.commonprefix((parent_path, child_path)) - return common_path == parent_path - - -def relative(parent_dir, child_path): - """Get the relative between a path that contains another path.""" - child_dir = os.path.dirname(child_path) - assert path_contains(parent_dir, child_dir), "{} not inside {}".format( - child_path, parent_dir) - return os.path.relpath(child_path, start=parent_dir) - - -class MarkdownSymlinksDomain(Domain): - """ - Extension of the Domain class to implement custom cross-reference links - solve methodology - """ - - name = 'markdown_symlinks' - label = 'MarkdownSymlinks' - - roles = { - 'xref': XRefRole(), - } - - docs_root_dir = os.path.realpath(os.path.dirname(__file__)) - code_root_dir = os.path.realpath(os.path.join(docs_root_dir, "..")) - - mapping = { - 'docs2code': {}, - 'code2docs': {}, - } - - github_repo_url = "" - github_repo_branch = "" - - @classmethod - def relative_code(cls, url): - """Get a value relative to the code directory.""" - return relative(cls.code_root_dir, url) - - @classmethod - def relative_docs(cls, url): - """Get a value relative to the docs directory.""" - return relative(cls.docs_root_dir, url) - - @classmethod - def add_mapping(cls, docs_rel, code_rel): - assert docs_rel not in cls.mapping['docs2code'], """\ -Assertion error! Document already in mapping! - New Value: {} -Current Value: {} -""".format(docs_rel, cls.mapping['docs2code'][docs_rel]) - - assert code_rel not in cls.mapping['code2docs'], """\ -Assertion error! Document already in mapping! - New Value: {} -Current Value: {} -""".format(docs_rel, cls.mapping['code2docs'][code_rel]) - - cls.mapping['docs2code'][docs_rel] = code_rel - cls.mapping['code2docs'][code_rel] = docs_rel - - @classmethod - def find_links(cls): - """Walk the docs dir and find links to docs in the code dir.""" - for root, dirs, files in os.walk(cls.docs_root_dir): - for dname in dirs: - dpath = os.path.abspath(os.path.join(root, dname)) - - if not os.path.islink(dpath): - continue - - link_path = os.path.join(root, os.readlink(dpath)) - # Is link outside the code directory? - if not path_contains(cls.code_root_dir, link_path): - continue - - # Is link internal to the docs directory? - if path_contains(cls.docs_root_dir, link_path): - continue - - docs_rel = cls.relative_docs(dpath) - code_rel = cls.relative_code(link_path) - - cls.add_mapping(docs_rel, code_rel) - - for fname in files: - fpath = os.path.abspath(os.path.join(root, fname)) - - if not os.path.islink(fpath): - continue - - link_path = os.path.join(root, os.readlink(fpath)) - # Is link outside the code directory? - if not path_contains(cls.code_root_dir, link_path): - continue - - # Is link internal to the docs directory? - if path_contains(cls.docs_root_dir, link_path): - continue - - docs_rel = cls.relative_docs(fpath) - code_rel = cls.relative_code(link_path) - - cls.add_mapping(docs_rel, code_rel) - - import pprint - pprint.pprint(cls.mapping) - - @classmethod - def add_github_repo(cls, github_repo_url, github_repo_branch): - """Initialize the github repository to update links correctly.""" - cls.github_repo_url = github_repo_url - cls.github_repo_branch = github_repo_branch - - # Overriden method to solve the cross-reference link - def resolve_xref( - self, env, fromdocname, builder, typ, target, node, contnode): - if '#' in target: - todocname, targetid = target.split('#') - else: - todocname = target - targetid = '' - - if todocname not in self.mapping['code2docs']: - # Could be a link to a repository's code tree directory/file - todocname = '{}{}{}'.format(self.github_repo_url, self.github_repo_branch, todocname) - - newnode = nodes.reference('', '', internal=True, refuri=todocname) - newnode.append(contnode[0]) - else: - # Removing filename extension (e.g. contributing.md -> contributing) - todocname, _ = os.path.splitext(self.mapping['code2docs'][todocname]) - - newnode = make_refnode( - builder, fromdocname, todocname, targetid, contnode[0]) - - return newnode - - def resolve_any_xref( - self, env, fromdocname, builder, target, node, contnode): - res = self.resolve_xref( - env, fromdocname, builder, 'xref', target, node, contnode) - return [('markdown_symlinks:xref', res)] - - -class LinkParser(parser.CommonMarkParser, object): - def visit_link(self, mdnode): - ref_node = nodes.reference() - - destination = mdnode.destination - - ref_node.line = self._get_line(mdnode) - if mdnode.title: - ref_node['title'] = mdnode.title - - url_check = urlparse(destination) - # If there's not a url scheme (e.g. 'https' for 'https:...' links), - # or there is a scheme but it's not in the list of known_url_schemes, - # then assume it's a cross-reference and pass it to Sphinx as an `:any:` ref. - known_url_schemes = self.config.get('known_url_schemes') - if known_url_schemes: - scheme_known = url_check.scheme in known_url_schemes - else: - scheme_known = bool(url_check.scheme) - - is_dest_xref = url_check.fragment and destination[:1] != '#' - - ref_node['refuri'] = destination - next_node = ref_node - - # Adds pending-xref wrapper to unsolvable cross-references - if (is_dest_xref or not url_check.fragment) and not scheme_known: - wrap_node = addnodes.pending_xref( - reftarget=unquote(destination), - reftype='xref', - refdomain='markdown_symlinks', # Added to enable cross-linking - refexplicit=True, - refwarn=True) - # TODO also not correct sourcepos - wrap_node.line = self._get_line(mdnode) - if mdnode.title: - wrap_node['title'] = mdnode.title - wrap_node.append(ref_node) - next_node = wrap_node - - self.current_node.append(next_node) - self.current_node = ref_node - - -if __name__ == "__main__": - import doctest - doctest.testmod() diff --git a/docs/requirements.txt b/docs/requirements.txt index 6edb8579..26dbdc3b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -7,3 +7,6 @@ recommonmark sphinx-markdown-tables git+https://github.com/SymbiFlow/sphinx_symbiflow_theme sphinxcontrib-napoleon + +# Markdown cross-reference solver library +git+https://github.com/SymbiFlow/markdown-code-symlinks diff --git a/docs/architecture/bitstream_format.rst b/docs/src/architecture/bitstream_format.rst similarity index 100% rename from docs/architecture/bitstream_format.rst rename to docs/src/architecture/bitstream_format.rst diff --git a/docs/src/architecture/code_of_conduct.md b/docs/src/architecture/code_of_conduct.md new file mode 120000 index 00000000..5d525cd1 --- /dev/null +++ b/docs/src/architecture/code_of_conduct.md @@ -0,0 +1 @@ +../../../CODE_OF_CONDUCT.md \ No newline at end of file diff --git a/docs/architecture/configuration.rst b/docs/src/architecture/configuration.rst similarity index 100% rename from docs/architecture/configuration.rst rename to docs/src/architecture/configuration.rst diff --git a/docs/src/architecture/copying.md b/docs/src/architecture/copying.md new file mode 120000 index 00000000..28b80bcc --- /dev/null +++ b/docs/src/architecture/copying.md @@ -0,0 +1 @@ +../../../COPYING \ No newline at end of file diff --git a/docs/architecture/dram_configuration.rst b/docs/src/architecture/dram_configuration.rst similarity index 100% rename from docs/architecture/dram_configuration.rst rename to docs/src/architecture/dram_configuration.rst diff --git a/docs/architecture/glossary.rst b/docs/src/architecture/glossary.rst similarity index 100% rename from docs/architecture/glossary.rst rename to docs/src/architecture/glossary.rst diff --git a/docs/architecture/interconnect.rst b/docs/src/architecture/interconnect.rst similarity index 100% rename from docs/architecture/interconnect.rst rename to docs/src/architecture/interconnect.rst diff --git a/docs/architecture/overview.rst b/docs/src/architecture/overview.rst similarity index 100% rename from docs/architecture/overview.rst rename to docs/src/architecture/overview.rst diff --git a/docs/architecture/reference.rst b/docs/src/architecture/reference.rst similarity index 100% rename from docs/architecture/reference.rst rename to docs/src/architecture/reference.rst diff --git a/docs/src/architecture/updating_the_docs.md b/docs/src/architecture/updating_the_docs.md new file mode 120000 index 00000000..22d86c5d --- /dev/null +++ b/docs/src/architecture/updating_the_docs.md @@ -0,0 +1 @@ +../../../UPDATING-THE-DOCS.md \ No newline at end of file diff --git a/docs/conf.py b/docs/src/conf.py similarity index 95% rename from docs/conf.py rename to docs/src/conf.py index 833cb06b..ac2647ff 100644 --- a/docs/conf.py +++ b/docs/src/conf.py @@ -197,9 +197,14 @@ intersphinx_mapping = {'https://docs.python.org/': None} def setup(app): github_code_repo = 'https://github.com/SymbiFlow/prjxray/' + github_code_branch = 'blob/master/' + docs_root_dir = os.path.realpath(os.path.dirname(__file__)) + code_root_dir = os.path.realpath(os.path.join(docs_root_dir, "..", "..")) + + MarkdownSymlinksDomain.init_domain(github_code_repo, github_code_branch, + docs_root_dir, code_root_dir) MarkdownSymlinksDomain.find_links() - MarkdownSymlinksDomain.add_github_repo(github_code_repo, 'blob/master/') app.add_domain(MarkdownSymlinksDomain) app.add_config_value( 'recommonmark_config', { diff --git a/docs/src/db_dev_process/contributing.md b/docs/src/db_dev_process/contributing.md new file mode 120000 index 00000000..c97564d9 --- /dev/null +++ b/docs/src/db_dev_process/contributing.md @@ -0,0 +1 @@ +../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/src/db_dev_process/fuzzers/index b/docs/src/db_dev_process/fuzzers/index new file mode 120000 index 00000000..d78a6674 --- /dev/null +++ b/docs/src/db_dev_process/fuzzers/index @@ -0,0 +1 @@ +../../../../fuzzers/ \ No newline at end of file diff --git a/docs/db_dev_process/fuzzers/index.rst b/docs/src/db_dev_process/fuzzers/index.rst similarity index 100% rename from docs/db_dev_process/fuzzers/index.rst rename to docs/src/db_dev_process/fuzzers/index.rst diff --git a/docs/src/db_dev_process/minitests/index b/docs/src/db_dev_process/minitests/index new file mode 120000 index 00000000..353e56d0 --- /dev/null +++ b/docs/src/db_dev_process/minitests/index @@ -0,0 +1 @@ +../../../../minitests/ \ No newline at end of file diff --git a/docs/db_dev_process/minitests/index.rst b/docs/src/db_dev_process/minitests/index.rst similarity index 100% rename from docs/db_dev_process/minitests/index.rst rename to docs/src/db_dev_process/minitests/index.rst diff --git a/docs/db_dev_process/parts.rst b/docs/src/db_dev_process/parts.rst similarity index 100% rename from docs/db_dev_process/parts.rst rename to docs/src/db_dev_process/parts.rst diff --git a/docs/src/db_dev_process/readme.md b/docs/src/db_dev_process/readme.md new file mode 120000 index 00000000..8a33348c --- /dev/null +++ b/docs/src/db_dev_process/readme.md @@ -0,0 +1 @@ +../../../README.md \ No newline at end of file diff --git a/docs/format/db.rst b/docs/src/format/db.rst similarity index 100% rename from docs/format/db.rst rename to docs/src/format/db.rst diff --git a/docs/format/tile.rst b/docs/src/format/tile.rst similarity index 100% rename from docs/format/tile.rst rename to docs/src/format/tile.rst diff --git a/docs/index.rst b/docs/src/index.rst similarity index 100% rename from docs/index.rst rename to docs/src/index.rst