Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Namnn/need per id json #960

Merged
merged 91 commits into from Sep 6, 2023
Merged
Show file tree
Hide file tree
Changes from 89 commits
Commits
Show all changes
91 commits
Select commit Hold shift + click to select a range
eb546d4
gen needs.json per needs.id and command some service git to bypass test
nhatnamnguyengtvthcm Jul 7, 2023
600ad3f
update review 11/07
nhatnamnguyengtvthcm Jul 12, 2023
0228b24
update review 11/07
nhatnamnguyengtvthcm Jul 12, 2023
9a698b0
update conf
nhatnamnguyengtvthcm Jul 14, 2023
3de0647
config variable for needs_id forder path
nhatnamnguyengtvthcm Jul 19, 2023
dbb52c0
update env config for needs per id
nhatnamnguyengtvthcm Jul 21, 2023
01320d4
add configuration for needs per id~
nhatnamnguyengtvthcm Jul 21, 2023
177d3e3
add configuration for needs per id~
nhatnamnguyengtvthcm Jul 21, 2023
efbe1fa
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
04f5019
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
be61481
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
103cc2f
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
274842f
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
1c2af5b
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
9136af8
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
26a18b8
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
ac7c5fe
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
618e1b0
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
8802a04
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
55e0cd3
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
6462295
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
dbb9f3f
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
13c1ca7
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
86bcfef
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
f42e9d7
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
f0ec457
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 10, 2023
724a367
change version build
nhatnamnguyengtvthcm Aug 21, 2023
7e5b7e4
change changelog file
nhatnamnguyengtvthcm Aug 21, 2023
86e343a
Removed esbonio for IDE support
haiyangToAI Feb 17, 2023
30cae4d
enable docker arm builds
cpolzer May 31, 2023
41c0368
🔧 Add .nox to gitignore
chrisjsewell Aug 2, 2023
bb83ac1
Update pyproject.toml
chrisjsewell Aug 3, 2023
c3ff8b6
Update pyproject.toml
chrisjsewell Aug 3, 2023
7ca8d6e
Update requirements.txt
chrisjsewell Aug 3, 2023
7680146
Update api.rst
chrisjsewell Aug 3, 2023
e969347
Added config allow unsafe filter for filter_func (#949)
haiyangToAI Aug 3, 2023
3935da6
🔧 Update pre-commit hooks (#956)
chrisjsewell Aug 4, 2023
e518929
Bump actions/checkout from 3.3.0 to 3.5.3
dependabot[bot] Jun 12, 2023
8a500e6
needs: Remove some of the extra IDs in the output
tim-nordell-nimbelink Nov 21, 2022
786db24
Added config option needs_report_dead_links (#937)
haiyangToAI Aug 4, 2023
624fd84
Raises version to 1.3.0
danwos Aug 16, 2023
ee77802
👌 Performance: Memoize Inline Parser
chrisjsewell Aug 11, 2023
cbc7bc0
fix linting
chrisjsewell Aug 16, 2023
ceae80e
Add to changelog
chrisjsewell Aug 16, 2023
4ee996e
Easier Sphinx-Needs docs builds
danwos Aug 18, 2023
b8fe439
Docker fix
danwos Aug 18, 2023
1b79d17
Removing docker context: ci
danwos Aug 18, 2023
ab99745
gen needs.json per needs.id and command some service git to bypass test
nhatnamnguyengtvthcm Jul 7, 2023
b9a0f99
update conf
nhatnamnguyengtvthcm Jul 14, 2023
f868ffe
update env config for needs per id
nhatnamnguyengtvthcm Jul 21, 2023
4985e9f
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
0c8e80f
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
18cbd4e
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
305ad77
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 10, 2023
d9a967d
change version build
nhatnamnguyengtvthcm Aug 21, 2023
a9edc4e
update change log
nhatnamnguyengtvthcm Aug 21, 2023
bd18a2b
change release version
nhatnamnguyengtvthcm Aug 21, 2023
6d50c27
♻️ Change `NeedsBuilder` format to `needs`
chrisjsewell Aug 21, 2023
b6819f4
Update changelog.rst
chrisjsewell Aug 21, 2023
588e4cf
gen needs.json per needs.id and command some service git to bypass test
nhatnamnguyengtvthcm Jul 7, 2023
5c30c12
update review 11/07
nhatnamnguyengtvthcm Jul 12, 2023
6fd7eba
fix-ci-lint-err
nhatnamnguyengtvthcm Aug 4, 2023
f85bb09
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 7, 2023
d474b40
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
aaf3869
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 9, 2023
1a288e0
add test-case and fix some setting for need per id builder
nhatnamnguyengtvthcm Aug 10, 2023
d8013c9
change version build
nhatnamnguyengtvthcm Aug 21, 2023
819e716
change changelog file
nhatnamnguyengtvthcm Aug 21, 2023
7aac5d3
Raises version to 1.3.0
danwos Aug 16, 2023
41609ff
Add to changelog
chrisjsewell Aug 16, 2023
4d88520
Easier Sphinx-Needs docs builds
danwos Aug 18, 2023
6f5e62e
update conf
nhatnamnguyengtvthcm Jul 14, 2023
6ff9b71
change version build
nhatnamnguyengtvthcm Aug 21, 2023
d1c46ac
rebase and merge version 1.3.0
nhatnamnguyengtvthcm Aug 21, 2023
c2ef673
merge version 1.3.0
nhatnamnguyengtvthcm Aug 21, 2023
5e8fc85
fix merge version 1.3.0
nhatnamnguyengtvthcm Aug 21, 2023
71988c4
fix merge version 1.3.0
nhatnamnguyengtvthcm Aug 21, 2023
a0c771b
move kogic in needs id builder into needsfile
nhatnamnguyengtvthcm Aug 25, 2023
4d06008
move kogic in needs id builder into needsfile
nhatnamnguyengtvthcm Aug 25, 2023
a5af64f
move kogic in needs id builder into needsfile
nhatnamnguyengtvthcm Aug 25, 2023
9077dca
using doc_needs_builder for test needs_id builder
nhatnamnguyengtvthcm Aug 28, 2023
025cfc5
mofify configuration.rst
nhatnamnguyengtvthcm Aug 28, 2023
dafb701
merge version 1.3.0
nhatnamnguyengtvthcm Aug 28, 2023
cf47fd4
change version needs_id builder
nhatnamnguyengtvthcm Aug 28, 2023
b84e4cc
fix test case for needs_id_builder
nhatnamnguyengtvthcm Aug 28, 2023
ef926b1
update version 1.3.0
nhatnamnguyengtvthcm Aug 30, 2023
0ac8a56
update version 1.3.0
nhatnamnguyengtvthcm Aug 30, 2023
3a10072
update version 1.3.0
nhatnamnguyengtvthcm Aug 30, 2023
49e91cb
update version 1.3.0
nhatnamnguyengtvthcm Aug 30, 2023
43689e5
refactor confiuration file and builder
nhatnamnguyengtvthcm Aug 31, 2023
ba544e7
add version 1.4.0 in needs_build_json_per_id_path configurations.rst
nhatnamnguyengtvthcm Sep 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/builders.rst
Expand Up @@ -163,3 +163,21 @@ or
.. hint::

As an alternative, you can set the config option :ref:`needs_build_needumls` to export the needumls files during each build.


.. _needs_id_builder:

needs_id
--------
.. versionadded:: 1.4.0

The **needs_id** builder exports all found needs and selected filter results to a set json files of each need with the name is ``id`` of need.
danwos marked this conversation as resolved.
Show resolved Hide resolved

danwos marked this conversation as resolved.
Show resolved Hide resolved
The build creates a folder called :ref:``needs_build_json_per_id_path`` and all file json of each need inside the given build-folder.
danwos marked this conversation as resolved.
Show resolved Hide resolved

Usage
+++++

.. code-block:: bash

sphinx-build -b needs_id source_dir build_dir
6 changes: 4 additions & 2 deletions docs/changelog.rst
Expand Up @@ -7,14 +7,15 @@ License
-------

.. include:: ../LICENSE

2.0.0
-----
Released: under development

1.4.0
-----
Released: under development
* Improvement: Added Builder :ref:`needs_id_Builder` added and config option :ref:`needs_build_json_per_id` in `conf.py`.
nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved

* Improvement: Reduce document build time, by memoizing the inline parse in ``build_need`` (`#968 <https://github.com/useblocks/sphinx-needs/pull/968>`_)

Expand All @@ -29,7 +30,6 @@ Released: 16.08.2023
* Improvement: Configuration option :ref:`needs_debug_measurement` added, which creates a runtime report
for debugging purposes.
(`#917 <https://github.com/useblocks/sphinx-needs/pull/917>`_)

* Bugfix: Replace hardcoded `index` with config value `root_doc`.
(`#877 <https://github.com/useblocks/sphinx-needs/pull/877>`_)
* Bugfix: Fix unbounded memory usage in pickle environment.
Expand Down Expand Up @@ -914,3 +914,5 @@ custom css definitions you need to update them.
* Integrated interaction with the activated plantuml sphinx extension

* Added role **need** to create a reference to a need by giving the id


3 changes: 3 additions & 0 deletions docs/conf.py
Expand Up @@ -361,6 +361,9 @@ def custom_defined_func():
# build needs.json to make permalinks work
needs_build_json = True

# build needs_json for every needs-id to make detail panel
needs_build_json_per_id = True
nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved
nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

Expand Down
44 changes: 44 additions & 0 deletions docs/configuration.rst
Expand Up @@ -2289,3 +2289,47 @@ If true, need options like status, tags or links are collapsed and shown only af
Default value: True

Can be overwritten for each single need by setting :ref:`need_collapse`.

.. _needs_build_json_per_id:

needs_build_json_per_id
~~~~~~~~~~~~~~~~~~~~~~~
danwos marked this conversation as resolved.
Show resolved Hide resolved

.. versionadded:: 1.4.0

Builds list json file for each need, which have name is ``ID`` of need.
nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved
This option works like :ref:`needs_build_json`.

Default: False

Example:

.. code-block:: python

needs_build_json_per_id = False

.. hint::
The created single json file per need, located in :ref:`needs_build_json_per_id_path` folder. This is e.g ``_build/needs_id/abc_432.json``
nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved

.. _needs_build_json_per_id_path:

needs_build_json_per_id_path
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved
This option sets the location of the set of ``needs.json`` for every needs-id.

Default value: ``needs_id``

Example:

.. code-block:: python

needs_build_json_per_id_path = "needs_id"

.. hint::

Default: False
The created ``needs_id`` folder gets stored in the ``outdir`` of the current builder. The final location is e.g. ``_build/needs_id``



72 changes: 72 additions & 0 deletions sphinx_needs/builder.py
Expand Up @@ -160,3 +160,75 @@ def build_needumls_pumls(app: Sphinx, _exception: Exception) -> None:
needs_builder.set_environment(env)

needs_builder.finish()


class NeedsIdBuilder(Builder):
danwos marked this conversation as resolved.
Show resolved Hide resolved

nhatnamnguyengtvthcm marked this conversation as resolved.
Show resolved Hide resolved
"""Json builder for needs, which creates separate json-files per need"""

name = "needs_id"
danwos marked this conversation as resolved.
Show resolved Hide resolved
format = "needs"
file_suffix = ".txt"
links_suffix = None

def write_doc(self, docname: str, doctree: nodes.document) -> None:
pass

def finish(self) -> None:
env = unwrap(self.env)
data = SphinxNeedsData(env)
needs = data.get_or_create_needs().values() # We need a list of needs for later filter checks
version = getattr(env.config, "version", "unset")
needs_config = NeedsSphinxConfig(env.config)
filter_string = needs_config.builder_filter
from sphinx_needs.filter_common import filter_needs

filtered_needs = filter_needs(self.app, needs, filter_string)
needs_build_json_per_id_path = needs_config.build_json_per_id_path
needs_dir = os.path.join(self.outdir, needs_build_json_per_id_path)
if not os.path.exists(needs_dir):
os.makedirs(needs_dir, exist_ok=True)
for need in filtered_needs:
needs_list = NeedsList(env.config, self.outdir, self.srcdir)
needs_list.wipe_version(version)
needs_list.add_need(version, need)
id = need["id"]
try:
file_name = f"{id}.json"
needs_list.write_json(file_name, needs_dir)
except Exception as e:
danwos marked this conversation as resolved.
Show resolved Hide resolved
log.error(f"Needs-ID Builder {id} error: {e}")
log.info("Needs_id successfully exported")

def get_outdated_docs(self) -> Iterable[str]:
return []

def prepare_writing(self, _docnames: Set[str]) -> None:
pass

def write_doc_serialized(self, _docname: str, _doctree: nodes.document) -> None:
pass

def cleanup(self) -> None:
pass

def get_target_uri(self, _docname: str, _typ: Optional[str] = None) -> str:
return ""


def build_needs_id_json(app: Sphinx, _exception: Exception) -> None:
env = unwrap(app.env)

if not NeedsSphinxConfig(env.config).build_json_per_id:
return

# Do not create an additional needs_json for every needs_id, if builder is already "needs_id".
if isinstance(app.builder, NeedsIdBuilder):
return
try:
needs_id_builder = NeedsIdBuilder(app, env)
except TypeError:
needs_id_builder = NeedsIdBuilder(app)
needs_id_builder.set_environment(env)

needs_id_builder.finish()
3 changes: 3 additions & 0 deletions sphinx_needs/config.py
Expand Up @@ -230,6 +230,9 @@ def __setattr__(self, name: str, value: Any) -> None:
"""Jinja context for rendering templates"""

debug_measurement: bool = field(default=False, metadata={"rebuild": "html", "types": (bool,)})
# add config for needs_id_builder
build_json_per_id: bool = field(default=False, metadata={"rebuild": "html", "types": (bool,)})
build_json_per_id_path: str = field(default="needs_id", metadata={"rebuild": "html", "types": (str,)})

@classmethod
def add_config_values(cls, app: Sphinx) -> None:
Expand Down
5 changes: 5 additions & 0 deletions sphinx_needs/needs.py
Expand Up @@ -11,7 +11,9 @@
import sphinx_needs.debug as debug # Need to set global var in it for timeing measurements
from sphinx_needs.builder import (
NeedsBuilder,
NeedsIdBuilder,
NeedumlsBuilder,
build_needs_id_json,
build_needs_json,
build_needumls_pumls,
)
Expand Down Expand Up @@ -139,6 +141,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:

app.add_builder(NeedsBuilder)
app.add_builder(NeedumlsBuilder)
app.add_builder(NeedsIdBuilder)

NeedsSphinxConfig.add_config_values(app)

Expand Down Expand Up @@ -237,6 +240,8 @@ def setup(app: Sphinx) -> Dict[str, Any]:
app.connect("env-updated", install_lib_static_files)
app.connect("env-updated", install_permalink_file)

#
app.connect("build-finished", build_needs_id_json)
# This should be called last, so that need-styles can override styles from used libraries
app.connect("env-updated", install_styles_static_files)

Expand Down
8 changes: 6 additions & 2 deletions sphinx_needs/needsfile.py
Expand Up @@ -93,13 +93,17 @@ def wipe_version(self, version: str) -> None:
if version in self.needs_list["versions"]:
del self.needs_list["versions"][version]

def write_json(self, needs_file: str = "needs.json") -> None:
def write_json(self, needs_file: str = "needs.json", needs_path: str = "") -> None:
# We need to rewrite some data, because this kind of data gets overwritten during needs.json import.
self.needs_list["created"] = datetime.now().isoformat()
self.needs_list["current_version"] = self.current_version
self.needs_list["project"] = self.project
if needs_path:
needs_dir = needs_path
else:
needs_dir = self.outdir

with open(os.path.join(self.outdir, needs_file), "w") as f:
with open(os.path.join(needs_dir, needs_file), "w") as f:
json.dump(self.needs_list, f, indent=4, sort_keys=True)

def load_json(self, file: str) -> None:
Expand Down
36 changes: 36 additions & 0 deletions tests/test_needs_id_builder.py
@@ -0,0 +1,36 @@
import json
from pathlib import Path

import pytest

from sphinx_needs.config import NeedsSphinxConfig
from sphinx_needs.data import SphinxNeedsData


@pytest.mark.parametrize(
"test_app", [{"buildername": "needs_id", "srcdir": "doc_test/doc_needs_builder"}], indirect=True
)
def test_doc_needs_id_builder(test_app):
import os

from sphinx_needs.utils import unwrap

app = test_app
app.build()
out_dir = app.outdir
env = unwrap(app.env)
data = SphinxNeedsData(env)
needs_config = NeedsSphinxConfig(env.config)
needs = data.get_or_create_needs().values() # We need a list of needs for later filter checks
needs_build_json_per_id_path = needs_config.build_json_per_id_path
needs_id_path = os.path.join(out_dir, needs_build_json_per_id_path)
assert os.path.exists(needs_id_path)
for need in needs:
need_id = need["id"]
need_file_name = f"{need_id}.json"
needs_json = Path(needs_id_path, need_file_name)
assert os.path.exists(needs_json)
with open(needs_json) as needs_file:
needs_file_content = needs_file.read()
needs_list = json.loads(needs_file_content)
assert needs_list["versions"]["1.0"]["needs"][need_id]["docname"]