Merge pull request #661 from robtaylor/markdown-index

Add markdown index support

.coveragerc

@@ -7,6 +7,8 @@ omit =
     */tests/*
     */__main__.py
 
+dynamic_context = test_function
+
 [report]
 
 omit =
@@ -20,4 +22,7 @@ exclude_lines =
     raise NotImplementedError
     except DistributionNotFound
 
-skip_covered = true
+skip_covered = false
+
+[html]
+show_contexts = True

.readthedocs.yaml

@@ -1,13 +1,20 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: "ubuntu-22.04"
   tools:
     python: "3.9"
+  jobs:
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry
+    post_install:
+      # Install dependencies with 'docs' dependency group
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      # VIRTUAL_ENV needs to be set manually for now.
+      # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install
 
 mkdocs:
   configuration: mkdocs.yml
-
-python:
-  install:
-    - requirements: docs/requirements.txt

Makefile

@@ -177,7 +177,7 @@ docs/gen/*.tex: $(YAML)
 .PHONY: reqs-md
 reqs-md: install docs/gen/*.md
 docs/gen/*.md: $(YAML)
-	$(DOORSTOP) publish all docs/gen --markdown
+	$(DOORSTOP) publish all docs/gen --markdown --index
 
 .PHONY: reqs-pdf
 reqs-pdf: reqs-latex

docs/gen_reqs.py

@@ -0,0 +1,15 @@
+import os
+
+from doorstop.core import publisher, builder
+from doorstop.cli import utilities
+
+
+def on_pre_build(config):
+    cwd = os.getcwd()
+    path = os.path.abspath(os.path.join(cwd, "docs/gen"))
+    tree = builder.build(cwd=cwd)
+
+    published_path = publisher.publish(tree, path, ".md", index=True)
+
+    if published_path:
+        utilities.show("published: {}".format(published_path))

doorstop/cli/commands.py

@@ -564,6 +564,9 @@ def run_publish(args, cwd, error, catch=True):
     if args.width:
         kwargs["width"] = args.width
 
+    if args.index:
+        kwargs["index"] = True
+
     # Write to output file(s)
     if args.path:
         path = os.path.abspath(os.path.join(cwd, args.path))

doorstop/cli/main.py

@@ -76,7 +76,9 @@ def main(args=None):  # pylint: disable=R0915
 
     # Build main parser
     parser = argparse.ArgumentParser(
-        prog=CLI, description=DESCRIPTION, **shared  # type: ignore
+        prog=CLI,
+        description=DESCRIPTION,
+        **shared,  # type: ignore
     )
     parser.add_argument(
         "-F",
@@ -536,6 +538,11 @@ def _publish(subs, shared):
         help="do not include levels on heading and non-heading or non-heading items",
     )
     sub.add_argument("--template", help="template file", default=None)
+    sub.add_argument(
+        "--index",
+        help="Generate top level index (when producing markdown).",
+        action="store_true",
+    )
 
 
 if __name__ == "__main__":

doorstop/cli/tests/test_all.py

@@ -810,10 +810,38 @@ def test_publish_tree_text(self):
         self.assertTrue(os.path.isdir(path))
         self.assertFalse(os.path.isfile(os.path.join(path, "index.html")))
 
+    def test_publish_tree_md(self):
+        """Verify 'doorstop publish' can create a Markdown directory."""
+        path = os.path.join(self.temp, "all")
+        self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"]))
+        self.assertTrue(os.path.isdir(path))
+        self.assertTrue(os.path.isfile(os.path.join(path, "index.md")))
+
     def test_publish_tree_no_path(self):
         """Verify 'doorstop publish' returns an error with no path."""
         self.assertRaises(SystemExit, main, ["publish", "all"])
 
+    def test_publish_tree_markdown_with_index(self):
+        """Verify 'doorstop publish' can create Markdown output for a tree,
+        with an index."""
+        path = os.path.join(self.temp, "all")
+        self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"]))
+        self.assertTrue(os.path.isdir(path))
+        self.assertTrue(os.path.isfile(os.path.join(path, "index.md")))
+
+    def test_publish_markdown_tree_no_path(self):
+        """Verify 'doorstop publish' returns an error with no path."""
+        self.assertRaises(
+            SystemExit,
+            main,
+            [
+                "publish",
+                "-m",
+                "--index",
+                "all",
+            ],
+        )
+
 
 class TestPublishCommand(TempTestCase):
     """Tests 'doorstop publish' options toc and template"""

doorstop/core/publishers/markdown.py

@@ -2,20 +2,85 @@
 
 """Functions to publish documents and items."""
 
+import os
 from re import sub
 
 from doorstop import common, settings
-from doorstop.core.publishers.base import BasePublisher, format_level
+from doorstop.core.publishers.base import (
+    BasePublisher,
+    extract_prefix,
+    format_level,
+    get_document_attributes,
+)
 from doorstop.core.types import is_item, iter_items
 
 log = common.logger(__name__)
+INDEX = "index.md"
 
 
 class MarkdownPublisher(BasePublisher):
     """Markdown publisher."""
 
-    def create_index(self, directory, index=None, extensions=(".md",), tree=None):
-        """No index for Markdown."""
+    def create_index(self, directory, index=INDEX, extensions=(".md",), tree=None):
+        """Create an markdown index of all files in a directory.
+
+        :param directory: directory for index
+        :param index: filename for index
+        :param extensions: file extensions to include
+        :param tree: optional tree to determine index structure
+
+        """
+        # Get paths for the index index
+        filenames = []
+        for filename in os.listdir(directory):
+            if filename.endswith(extensions) and filename != INDEX:
+                filenames.append(os.path.join(filename))
+
+        # Create the index
+        if filenames:
+            path = os.path.join(directory, index)
+            log.info("creating an {}...".format(index))
+            lines = self.lines_index(sorted(filenames), tree=tree)
+            common.write_text(" # Requirements index", path)
+            common.write_text("\n".join(lines), path)
+        else:
+            log.warning("no files for {}".format(index))
+
+    def _index_tree(self, tree, depth):
+        """Recursively generate markdown index.
+
+        :param tree: optional tree to determine index structure
+        :param depth: depth recursed into tree
+        """
+
+        depth = depth + 1
+
+        title = get_document_attributes(tree.document)["title"]
+        prefix = extract_prefix(tree.document)
+        filename = f"{prefix}.md"
+
+        # Tree structure
+        yield " " * (depth * 2 - 1) + f"* [{prefix}]({filename}) - {title}"
+        # yield self.table_of_contents(linkify=True, obj=tree.document, depth=depth, heading=False)
+        for child in tree.children:
+            yield from self._index_tree(tree=child, depth=depth)
+
+    def lines_index(self, filenames, tree=None):
+        """Yield lines of Markdown for index.md.
+
+        :param filenames: list of filenames to add to the index
+        :param tree: optional tree to determine index structure
+        """
+        if tree:
+            yield from self._index_tree(tree, depth=0)
+
+        # Additional files
+        if filenames:
+            yield ""
+            yield "### Published Documents:"
+            for filename in filenames:
+                name = os.path.splitext(filename)[0]
+                yield " * [{n}]({f})".format(f=filename, n=name)
 
     def create_matrix(self, directory):
         """No traceability matrix for Markdown."""

doorstop/core/publishers/tests/test_publisher_markdown.py

@@ -5,15 +5,15 @@
 # pylint: disable=unused-argument,protected-access
 
 import os
-import stat
 import unittest
 from secrets import token_hex
 from shutil import rmtree
-from unittest.mock import Mock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 from doorstop.core import publisher
 from doorstop.core.publishers.tests.helpers import YAML_CUSTOM_ATTRIBUTES, getLines
 from doorstop.core.tests import (
+    EMPTY,
     FILES,
     ROOT,
     MockDataMixIn,
@@ -22,6 +22,7 @@
     MockItemAndVCS,
 )
 from doorstop.core.tests.helpers import on_error_with_retry
+from doorstop.core.types import UID
 
 
 class TestModule(MockDataMixIn, unittest.TestCase):
@@ -263,3 +264,51 @@ def test_toc(self):
         md_publisher = publisher.check(".md", self.document)
         toc = md_publisher.table_of_contents(linkify=True, obj=self.document)
         self.assertEqual(expected, toc)
+
+    def test_index(self):
+        """Verify an Markdown index can be created."""
+        # Arrange
+        path = os.path.join(FILES, "index.md")
+        md_publisher = publisher.check(".md")
+        # Act
+        md_publisher.create_index(FILES)
+        # Assert
+        self.assertTrue(os.path.isfile(path))
+
+    def test_index_no_files(self):
+        """Verify an Markdown index is only created when files exist."""
+        path = os.path.join(EMPTY, "index.md")
+        md_publisher = publisher.check(".md")
+        # Act
+        md_publisher.create_index(EMPTY)
+        # Assert
+        self.assertFalse(os.path.isfile(path))
+
+    def test_index_tree(self):
+        """Verify an Markdown index can be created with a tree."""
+        path = os.path.join(FILES, "index2.md")
+        mock_tree = MagicMock()
+        mock_tree.documents = []
+        for prefix in ("SYS", "HLR", "LLR", "HLT", "LLT"):
+            mock_document = MagicMock()
+            mock_document.prefix = prefix
+            mock_tree.documents.append(mock_document)
+        mock_tree.draw = lambda: "(mock tree structure)"
+        mock_item = Mock()
+        mock_item.uid = "KNOWN-001"
+        mock_item.document = Mock()
+        mock_item.document.prefix = "KNOWN"
+        mock_item.header = None
+        mock_item_unknown = Mock(spec=["uid"])
+        mock_item_unknown.uid = "UNKNOWN-002"
+        mock_trace = [
+            (None, mock_item, None, None, None),
+            (None, None, None, mock_item_unknown, None),
+            (None, None, None, None, None),
+        ]
+        mock_tree.get_traceability = lambda: mock_trace
+        md_publisher = publisher.check(".md")
+        # Act
+        md_publisher.create_index(FILES, index="index2.md", tree=mock_tree)
+        # Assert
+        self.assertTrue(os.path.isfile(path))

doorstop/core/tests/test_publisher.py

@@ -128,6 +128,22 @@ def test_index_none_for_md(self):
         # Assert
         self.assertEqual(result, False)
 
+    def test_index_true_md(self):
+        """Verify that index = true forces true."""
+        tmp_publisher = publisher.check(".md", self.mock_tree)
+        tmp_publisher.setup(None, True, None)
+        do_index = tmp_publisher.getIndex()
+        # Assert
+        self.assertEqual(do_index, True)
+
+    def test_index_false_md(self):
+        """Verify that index = false forces false."""
+        tmp_publisher = publisher.check(".md", self.mock_tree)
+        tmp_publisher.setup(None, False, None)
+        do_index = tmp_publisher.getIndex()
+        # Assert
+        self.assertEqual(do_index, False)
+
     def test_index_none_for_txt(self):
         """Verify that index = None works correctly."""
         tmp_publisher = publisher.check(".txt", self.mock_tree)
@@ -136,15 +152,15 @@ def test_index_none_for_txt(self):
         # Assert
         self.assertEqual(result, False)
 
-    def test_index_true(self):
+    def test_index_true_html(self):
         """Verify that index = true forces true."""
         tmp_publisher = publisher.check(".html", self.mock_tree)
         tmp_publisher.setup(None, True, None)
         do_index = tmp_publisher.getIndex()
         # Assert
         self.assertEqual(do_index, True)
 
-    def test_index_false(self):
+    def test_index_false_html(self):
         """Verify that index = false forces false."""
         tmp_publisher = publisher.check(".html", self.mock_tree)
         tmp_publisher.setup(None, False, None)

mkdocs.yml

@@ -9,6 +9,9 @@ theme: readthedocs
 
 markdown_extensions:
   - admonition
+use_directory_urls: false
+hooks: 
+ - docs/gen_reqs.py
 
 nav:
 - Home: index.md
@@ -25,6 +28,7 @@ nav:
 - Desktop Interface: gui/overview.md
 - Web Interface: web.md
 - Scripting Interface: api/scripting.md
+- Doorstop's requirements: gen/index.md
 - Reference:
   - Tree: reference/tree.md
   - Document: reference/document.md