diff --git a/Makefile b/Makefile index 9f2a35cb0d..7af3491f5f 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,8 @@ upload: tags: find tmt -name '*.py' | xargs ctags --python-kinds=-i clean: - rm -rf $(TMP) build docs/{_build,stories} dist .cache .pytest_cache + rm -rf $(TMP) build dist .cache .pytest_cache + rm -rf docs/_build docs/stories docs/specs find . -type f -name "*.py[co]" -delete find . -type f -name "*,cover" -delete find . -type d -name "__pycache__" -delete diff --git a/docs/conf.py b/docs/conf.py index 7d911723e6..8ed2176c9b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -246,17 +246,19 @@ tree = tmt.Tree('.') areas = { - 'docs': 'Documentation', - 'cli': 'Command Line', - 'api': 'API', - 'core': 'Core', - 'install': 'Installation', + '/stories/docs': 'Documentation', + '/stories/cli': 'Command Line', + '/stories/api': 'API', + '/stories/core': 'Core', + '/stories/install': 'Installation', + '/specs/l1': 'L1 Metadata', + '/specs/l2': 'L2 Metadata', } os.makedirs('stories', exist_ok=True) +os.makedirs('specs', exist_ok=True) for area in areas: - with open('stories/{}.rst'.format(area), 'w') as doc: + with open('{}.rst'.format(area.lstrip('/')), 'w') as doc: doc.write('{}\n{}\n'.format(areas[area], '=' * len(areas[area]))) - root = '/stories/{}'.format(area) - for story in tree.stories(names=[root], whole=True): - doc.write(story.export(title=story.name != root)) + for story in tree.stories(names=[area], whole=True): + doc.write(story.export(title=story.name != area)) diff --git a/docs/index.rst b/docs/index.rst index 2fb97131b0..d2ed9a3e7f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ Table of Contents Overview Dreams Stories + Specifications Commands Examples Modules diff --git a/docs/specs.rst b/docs/specs.rst new file mode 100644 index 0000000000..79e2bdda2f --- /dev/null +++ b/docs/specs.rst @@ -0,0 +1,34 @@ + +Metadata Specification +====================== + +This is Metadata Specification which defines that all data needed +for test execution in the CI system are stored as a plain text +information in Flexible Metadata Format under version control in +the git repository close to the test code or source code. FMF uses +YAML to store data in a concise human and machine readable way +plus adds a few nice features like virtual hierarchy, inheritance +and elasticity to minimize data duplication and maintenance. There +are two levels of metadata defined: + +**L1 Metadata** + +These are test metadata closely related to individual test cases +for which it make sense to store them directly with the test code. +Examples of such metadata are summary, description, duration or +tags. See Level 1 Metadata section for more detailed information +and list of already defined attributes. + +**L2 Metadata** + +These metadata contain information for execution of multiple test +cases such as how the environment for testing should be prepared, +which set of test cases is relevant for testing specific artifact +or which frameworks should be used for execution. See Level 2 +Metadata section for detailed specification. + +.. toctree:: + :maxdepth: 2 + + specs/l1 + specs/l2 diff --git a/specs/l1/main.fmf b/specs/l1/main.fmf new file mode 100644 index 0000000000..be5e0670cb --- /dev/null +++ b/specs/l1/main.fmf @@ -0,0 +1,8 @@ +story: + As a tester I want to store test metadata close to the test + source code. +description: + These are test metadata closely related to individual test + cases for which it make sense to store them directly with the + test code. Examples of such metadata are summary, description, + duration or tags. diff --git a/specs/l1/test.fmf b/specs/l1/test.fmf new file mode 100644 index 0000000000..90287cd79b --- /dev/null +++ b/specs/l1/test.fmf @@ -0,0 +1,16 @@ +summary: Shell command which executes the test + +story: + As a test writer I want to run a single test script in + multiple ways (e.g. by providing different parameters). + +description: + This attribute defines how the test is to be executed. + +examples: | + test: ./test.sh + test: ./test.sh --depth 1000 + test: pytest -k performance + test: make run + +type: string diff --git a/specs/l2/main.fmf b/specs/l2/main.fmf new file mode 100644 index 0000000000..dcb89c25ff --- /dev/null +++ b/specs/l2/main.fmf @@ -0,0 +1,10 @@ +story: + As a tester I want to group tests and specify environment for + testing. + +description: + These metadata contain information for execution of multiple + test cases such as how the environment for testing should be + prepared, which set of test cases is relevant for testing + specific artifact or which frameworks should be used for + execution.