Added config allow unsafe filter for filter_func (#949)

* Added config allow unsafe filter for filter_func

* Fixed review

* Updated sphinx-immaterial to fix CI

* Fix linkcheck anchor issue

---------

Co-authored-by: Daniel Woste <daniel.woste@useblocks.com>

docs/changelog.rst

@@ -27,6 +27,9 @@ Released: under development
 * Removed configuration **needs_ide_directive_snippets** to support custom directive snippets for IDE features.
 * Provided new IDE support option: VsCode extension
   `Sphinx-Needs-VsCode <https://marketplace.visualstudio.com/items?itemName=useblocks.sphinx-needs-vscode>`_.
+* Improvement: Configuration option :ref:`needs_allow_unsafe_filters` added, which allows unsafe filter for
+  :ref:`filter_func`.
+  (`#831 <https://github.com/useblocks/sphinx-needs/issues/831>`_)
 
 1.2.2
 -----

docs/configuration.rst

@@ -520,6 +520,27 @@ The defined extra filter data can also be used like:
       :style: green_border
 
 
+.. _needs_allow_unsafe_filters:
+
+needs_allow_unsafe_filters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Allow unsafe filter for :ref:`filter_func`. Default is ``False``.
+
+If set to True, the filtered results will keep all fields as they are returned by the dynamic functions.
+Fields can be added or existing fields can even be manipulated.
+
+.. note::
+
+   Keep in mind this only affects the filter results, original needs as displayed somewhere else are not modified.
+
+If set to False, the filter results contains the original need fields and any manipulations of need fields are lost.
+
+|ex|
+
+.. code-block:: python
+
+   needs_allow_unsafe_filters = True
 
 .. _needs_flow_show_links:
 

docs/requirements.txt

@@ -7,4 +7,4 @@ sphinxcontrib-programoutput
 sphinx-design
 click
 tabulate
-sphinx-immaterial==0.11.6
+sphinx-immaterial==0.11.6

sphinx_needs/filter_common.py

@@ -173,11 +173,15 @@ def process_filters(app: Sphinx, all_needs, current_needlist, include_external:
         found_dirty_needs = context["results"]
         found_needs = []
 
-        # Just take the ids from search result and use the related, but original need
-        found_need_ids = [x["id_complete"] for x in found_dirty_needs]
-        for need in all_needs_incl_parts:
-            if need["id_complete"] in found_need_ids:
-                found_needs.append(need)
+        # Check if config allow unsafe filters
+        if app.config.needs_allow_unsafe_filters:
+            found_needs = found_dirty_needs
+        else:
+            # Just take the ids from search result and use the related, but original need
+            found_need_ids = [x["id_complete"] for x in found_dirty_needs]
+            for need in all_needs_incl_parts:
+                if need["id_complete"] in found_need_ids:
+                    found_needs.append(need)
 
     # Store basic filter configuration and result global list.
     # Needed mainly for exporting the result to needs.json (if builder "needs" is used).

sphinx_needs/needs.py

@@ -222,6 +222,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("needs_extra_links", [], "html")
 
     app.add_config_value("needs_filter_data", {}, "html")
+    app.add_config_value("needs_allow_unsafe_filters", False, "html")
 
     app.add_config_value("needs_flow_show_links", False, "html")
     app.add_config_value("needs_flow_link_types", ["links"], "html")

tests/doc_test/doc_needs_filter_func_allow_dirty_filter/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = needstestdocs
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

tests/doc_test/doc_needs_filter_func_allow_dirty_filter/conf.py

@@ -0,0 +1,57 @@
+# Configuration file for Sphinx-Needs Documentation.
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+
+# -- General configuration ------------------------------------------------
+
+project = "Useblocks Sphinx-Needs Test"
+copyright = "2023, Useblocks GmbH"
+author = "Teams Useblocks"
+version = "1.0"
+
+extensions = ["sphinx_needs", "sphinxcontrib.plantuml"]
+
+needs_id_regex = "^[A-Za-z0-9_]*"
+
+needs_types = [
+    {"directive": "feature", "title": "Feature", "prefix": "FE_", "color": "#FEDCD2", "style": "node"},
+    {"directive": "usecase", "title": "Use Case", "prefix": "USE_", "color": "#DF744A", "style": "node"},
+]
+
+needs_extra_options = ["ti", "tcl"]
+
+needs_extra_links = [
+    {
+        "option": "features",
+        "incoming": "featured by",
+        "outgoing": "features",
+        "copy": False,
+        "style": "#Gold",
+        "style_part": "#Gold",
+    },
+]
+
+needs_allow_unsafe_filters = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# Doc info
+source_suffix = ".rst"
+master_doc = "index"
+language = "en"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "alabaster"

tests/doc_test/doc_needs_filter_func_allow_dirty_filter/filter_func.py

@@ -0,0 +1,14 @@
+def my_own_filter(needs, results, **kwargs):
+    needs_dict = {x["id"]: x for x in needs}
+    curr_need_id = kwargs["arg1"]
+    link_type = kwargs["arg2"]
+
+    for link_id in needs_dict[curr_need_id][link_type]:
+        if needs_dict[curr_need_id]["ti"] == "1":
+            needs_dict[link_id]["tcl"] = "10"
+        elif needs_dict[curr_need_id]["ti"] == "3":
+            needs_dict[link_id]["tcl"] = "30"
+        else:
+            needs_dict[link_id]["tcl"] = "unknown"
+
+        results.append(needs_dict[link_id])

tests/doc_test/doc_needs_filter_func_allow_dirty_filter/index.rst

@@ -0,0 +1,31 @@
+Needs Data to be filtered
+=========================
+
+.. feature:: Feature 001
+   :id: FE_001
+
+   Example feature 001 content.
+
+.. usecase:: Usecase 001
+   :id: USE_001
+   :ti: 1
+   :features: FE_001
+
+   Example tesusecaset 001 content.
+
+   .. needtable:: Filter func table 001
+      :style: table
+      :columns: id, title, tcl
+      :filter-func: filter_func.my_own_filter(USE_001,features)
+
+.. usecase:: Usecase 002
+   :id: USE_002
+   :ti: 3
+   :features: FE_001
+
+   Example usecase 002 content.
+
+   .. needtable:: Filter func table 002
+      :style: table
+      :columns: id, title, tcl
+      :filter-func: filter_func.my_own_filter(USE_002,features)

tests/doc_test/doc_needs_filter_func_allow_dirty_filter/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=needstestdocs
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

tests/test_unsafe_filter_for_filter_func.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "test_app", [{"buildername": "html", "srcdir": "doc_test/doc_needs_filter_func_allow_dirty_filter"}], indirect=True
+)
+def test_doc_allow_unsafe_filter_for_filter_func(test_app):
+    app = test_app
+    app.build()
+    index_html = Path(app.outdir, "index.html").read_text()
+
+    assert '<span class="caption-text">Filter func table 001' in index_html
+    assert '<td class="needs_title"><p>Feature 001</p></td>' in index_html
+    assert '<td class="needs_tcl"><p>10</p></td>' in index_html
+
+    assert '<span class="caption-text">Filter func table 002' in index_html
+    assert '<td class="needs_title"><p>Feature 001</p></td>' in index_html
+    assert '<td class="needs_tcl"><p>30</p></td>' in index_html