tests: docs: consoletest: Do not use Sphinx builder interface

Signed-off-by: John Andersen <johnandersenpdx@gmail.com>

Author: pdxjohnny

docs/tutorials/models/package.rst

@@ -236,6 +236,7 @@ Let's add some training data to a CSV file.
 **train.csv**
 
 .. code-block::
+    :test:
     :filepath: train.csv
 
     Years,Salary

docs/tutorials/models/slr.rst

@@ -183,6 +183,7 @@ Let's first create our training, test, and prediction data CSV files.
 **train.csv**
 
 .. code-block::
+    :test:
     :filepath: train.csv
 
     Years,Salary
@@ -195,6 +196,7 @@ Let's first create our training, test, and prediction data CSV files.
 **test.csv**
 
 .. code-block::
+    :test:
     :filepath: test.csv
 
     Years,Salary
@@ -204,6 +206,7 @@ Let's first create our training, test, and prediction data CSV files.
 **predict.csv**
 
 .. code-block::
+    :test:
     :filepath: predict.csv
 
     Years

docs/tutorials/sources/file.rst

@@ -53,7 +53,7 @@ Add configuration
 
 **dffml_source_ini/misc.py**
 
-.. literalinclude:: ../../../dffml/source/ini.py
+.. literalinclude:: /../dffml/source/ini.py
     :test:
     :filepath: dffml_source_ini/misc.py
     :lines: 11-15
@@ -67,7 +67,7 @@ Create Source
 
 **dffml_source_ini/misc.py**
 
-.. literalinclude:: ../../../dffml/source/ini.py
+.. literalinclude:: /../dffml/source/ini.py
     :test:
     :filepath: dffml_source_ini/misc.py
     :lines: 18-24
@@ -92,7 +92,7 @@ Add load method
 
 **dffml_source_ini/misc.py**
 
-.. literalinclude:: ../../../dffml/source/ini.py
+.. literalinclude:: /../dffml/source/ini.py
     :test:
     :filepath: dffml_source_ini/misc.py
     :lines: 26-47
@@ -116,7 +116,7 @@ Add dump method
 
 **dffml_source_ini/misc.py**
 
-.. literalinclude:: ../../../dffml/source/ini.py
+.. literalinclude:: /../dffml/source/ini.py
     :test:
     :filepath: dffml_source_ini/misc.py
     :lines: 49-66
@@ -153,7 +153,7 @@ test methods as coroutines in default event loop.
 
 **tests/test_source.py**
 
-.. literalinclude:: ../../../tests/source/test_ini.py
+.. literalinclude:: /../tests/source/test_ini.py
     :test:
     :filepath: tests/test_source.py
     :lines: 10-30

tests/docs/test_consoletest.py

@@ -5,6 +5,7 @@
 import tempfile
 import unittest
 import contextlib
+import dataclasses
 import unittest.mock
 import importlib.util
 
@@ -27,7 +28,7 @@
     setattr(sys.modules[__name__], module_name, module)
 
 
-class TestDocs(unittest.TestCase):
+class TestDocs(AsyncTestCase):
     """
     A testcase for each doc will be added to this class
     """
@@ -67,9 +68,20 @@ def test__all_docs_being_tested(self):
         self.assertListEqual(should_have, docs)
 
 
-def mktestcase(filepath: pathlib.Path, relative: pathlib.Path):
-    # The test case itself, assigned to test_doctest of each class
-    def testcase(self):
+def mktestcase(filepath: pathlib.Path, relative: pathlib.Path, builder: bool):
+    async def testcase(self):
+        return await consoletest_doc(
+            [
+                str(filepath.resolve()),
+                "--root",
+                str(ROOT_PATH),
+                "--docs",
+                str(DOCS_PATH),
+                *args,
+            ]
+        )
+
+    def builder_testcase(self):
         from sphinx.cmd.build import (
             get_parser,
             Tee,
@@ -137,6 +149,8 @@ def pickle_load(fileobj):
             app.build(False, [])
         self.assertFalse(app.statuscode)
 
+    if builder:
+        return builder_testcase
     return testcase
 
 
@@ -149,23 +163,26 @@ def pickle_load(fileobj):
     relative = filepath.relative_to(DOCS_PATH).with_suffix("")
     if str(relative) in SKIP_DOCS:
         continue
-    TestDocs.TESTABLE_DOCS.append(str(relative))
-    name = "test_" + str(relative).replace(os.sep, "_")
-    # Do not add the tests if we are running with GitHub Actions for the main
-    # package. This is because there are seperate jobs for each tutorial test
-    # and the TestDocs.test__all_docs_being_tested ensures that we are running a
-    # job for each tutorial
-    if (
-        "GITHUB_ACTIONS" in os.environ
-        and "PLUGIN" in os.environ
-        and os.environ["PLUGIN"] == "."
-    ):
-        continue
-    setattr(
-        TestDocs,
-        name,
-        unittest.skipIf(
+    # Create the testcase
+    testcase = mktestcase(filepath, relative, True)
+    if True:
+        # Don't check for a long test entry in the workflow if doc in NO_SETUP
+        TestDocs.TESTABLE_DOCS.append(str(relative))
+        # Skip if not in NO_SETUP and RUN_CONSOLETESTS not set
+        testcase = unittest.skipIf(
             "RUN_CONSOLETESTS" not in os.environ,
             "RUN_CONSOLETESTS environment variable not set",
-        )(mktestcase(filepath, relative)),
-    )
+        )(testcase)
+        # Do not add the tests if we are running with GitHub Actions for the main
+        # package. This is because there are seperate jobs for each tutorial test
+        # and the TestDocs.test__all_docs_being_tested ensures that we are running a
+        # job for each tutorial
+        if (
+            "GITHUB_ACTIONS" in os.environ
+            and "PLUGIN" in os.environ
+            and os.environ["PLUGIN"] == "."
+        ):
+            continue
+    # Add the testcase
+    name = "test_" + str(relative).replace(os.sep, "_")
+    setattr(TestDocs, name, testcase)