Skip to content

Commit

Permalink
Added script to generate documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
joachimmetz committed Apr 1, 2024
1 parent c4d6665 commit fbf96b0
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 24 deletions.
35 changes: 12 additions & 23 deletions .github/workflows/test_docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
version: ['39']
version: ['39', '40']
container:
image: registry.fedoraproject.org/fedora:${{ matrix.version }}
steps:
Expand All @@ -18,7 +18,7 @@ jobs:
- name: Install dependencies
run: |
dnf copr -y enable @gift/dev
dnf install -y @development-tools python3 python3-devel libbde-python3 libcreg-python3 libewf-python3 libexe-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-artifacts python3-cffi python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml python3-setuptools python3-xattr
dnf install -y @development-tools libbde-python3 libcreg-python3 libewf-python3 libexe-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3 python3-artifacts python3-build python3-cffi python3-cryptography python3-devel python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml python3-setuptools python3-wheel python3-xattr
- name: Run tests
env:
LANG: C.utf8
Expand All @@ -27,21 +27,17 @@ jobs:
- name: Run end-to-end tests
run: |
if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
- name: Build source distribution
- name: Build source distribution (sdist)
run: |
python3 ./setup.py sdist
- name: Build binary distribution
python3 -m build --no-isolation --sdist
- name: Build binary distribution (wheel)
run: |
python3 ./setup.py bdist
- name: Run build and install test
run: |
python3 ./setup.py build
python3 ./setup.py install
python3 -m build --no-isolation --wheel
test_ubuntu:
runs-on: ubuntu-latest
strategy:
matrix:
version: ['22.04']
version: ['24.04']
container:
image: ubuntu:${{ matrix.version }}
steps:
Expand All @@ -58,7 +54,7 @@ jobs:
run: |
add-apt-repository -y ppa:gift/dev
apt-get update -q
apt-get install -y build-essential python3 python3-dev libbde-python3 libcreg-python3 libewf-python3 libexe-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3-artifacts python3-cffi-backend python3-cryptography python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-wheel python3-xattr python3-yaml
apt-get install -y build-essential libbde-python3 libcreg-python3 libewf-python3 libexe-python3 libfsapfs-python3 libfsext-python3 libfsfat-python3 libfshfs-python3 libfsntfs-python3 libfsxfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libmodi-python3 libphdi-python3 libqcow-python3 libregf-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvsapm-python3 libvsgpt-python3 libvshadow-python3 libvslvm-python3 libwrc-python3 python3 python3-artifacts python3-build python3-cffi-backend python3-cryptography python3-dev python3-dfdatetime python3-dfimagetools python3-dfvfs python3-dfwinreg python3-distutils python3-dtfabric python3-idna python3-pip python3-pytsk3 python3-setuptools python3-wheel python3-xattr python3-yaml
- name: Run tests
env:
LANG: en_US.UTF-8
Expand All @@ -69,16 +65,9 @@ jobs:
LANG: en_US.UTF-8
run: |
if test -f tests/end-to-end.py; then PYTHONPATH=. python3 ./tests/end-to-end.py --debug -c config/end-to-end.ini; fi
- name: Update setuptools
run: |
python3 -m pip install -U setuptools
- name: Build source distribution
run: |
python3 ./setup.py sdist
- name: Build binary distribution
- name: Build source distribution (sdist)
run: |
python3 ./setup.py bdist
- name: Run build and install test
python3 -m build --no-isolation --sdist
- name: Build binary distribution (wheel)
run: |
python3 ./setup.py build
python3 ./setup.py install
python3 -m build --no-isolation --wheel
1 change: 0 additions & 1 deletion artifactsrc/resource_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ class MessageResourceFile(object):
windows_path (str): Windows path of the message resource file.
"""

_STRING_TABLE_RESOURCE_IDENTIFIER = 0x06
_MESSAGE_TABLE_RESOURCE_IDENTIFIER = 0x0b
_VERSION_INFORMATION_RESOURCE_IDENTIFIER = 0x10

Expand Down
1 change: 1 addition & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ python_requires = >=3.8
scripts =
tools/check_artifacts.py
tools/export.py
tools/generate_docs.py

[options.package_data]
artifactsrc =
Expand Down
205 changes: 205 additions & 0 deletions tools/generate_docs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Script to generate artifact definition documentation."""

import argparse
import logging
import os
import sys

import artifacts

from artifacts import definitions as artifacts_definitions
from artifacts import reader as artifacts_reader
from artifacts import registry as artifacts_registry


class IndexRstOutputWriter(object):
"""Index.rst output writer."""

def __init__(self, path):
"""Initializes an index.rst output writer."""
super(IndexRstOutputWriter, self).__init__()
self._file_object = None
self._path = path

def __enter__(self):
"""Make this work with the 'with' statement."""
self._file_object = open(self._path, 'w', encoding='utf-8')

text = '\n'.join([
'#####################################',
'Digital Forensics Artifact definition',
'#####################################',
'',
'.. toctree::',
' :maxdepth: 1',
'',
''])
self._file_object.write(text)

return self

def __exit__(self, exception_type, value, traceback):
"""Make this work with the 'with' statement."""
self._file_object.close()
self._file_object = None

def WriteArtifactDefinition(self, artifact_name):
"""Writes an artifact definition to the index.rst file.
Args:
artifact_name (str): artifact name.
"""
self._file_object.write(
f' {artifact_name:s} <{artifact_name:s}>\n')


class MarkdownOutputWriter(object):
"""Markdown output writer."""

_URL_PREFIX = 'https://artifacts-kb.readthedocs.io/en/latest/sources/'

def __init__(self, path):
"""Initializes a Markdown output writer."""
super(MarkdownOutputWriter, self).__init__()
self._file_object = None
self._path = path

def __enter__(self):
"""Make this work with the 'with' statement."""
self._file_object = open(self._path, 'w', encoding='utf-8')
return self

def __exit__(self, exception_type, value, traceback):
"""Make this work with the 'with' statement."""
self._file_object.close()
self._file_object = None

def WriteArtifactDefinition(self, artifact_definition):
"""Writes an artifact definition to a Markdown file.
Args:
artifact_definition (list[artifacs.ArtifactDefinition]): artifact
definition.
"""
lines = [
f'## {artifact_definition.name:s}',
'',
artifact_definition.description,
'']

for source in sorted(
artifact_definition.sources, key=lambda source: source.supported_os):
if source.type_indicator in (
artifacts_definitions.TYPE_INDICATOR_FILE,
artifacts_definitions.TYPE_INDICATOR_PATH):

supported_os = ', '.join(
source.supported_os or artifact_definition.supported_os)

lines.extend([
f'Paths on: {supported_os:s}',
'',
'```'])

for path in sorted(source.paths):
lines.append(path)

lines.extend([
'```',
''])

if source.type_indicator == (
artifacts_definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY):
lines.append('```')
for key_path in sorted(source.keys):
lines.append(key_path)

lines.extend([
'```',
''])

if artifact_definition.urls:
lines.extend([
'### References',
''])

for url in artifact_definition.urls:
if url.startswith(self._URL_PREFIX) and url.endswith('.html'):
url = url[len(self._URL_PREFIX):-5]
url = f'../{url:s}.md'

lines.append(f'* {url:s}')

lines.extend([
'',
''])

text = '\n'.join(lines)
self._file_object.write(text)


def Main():
"""Entry point of console script to generate documentation.
Returns:
int: exit code that is provided to sys.exit().
"""
argument_parser = argparse.ArgumentParser(description=(
'Generated artifact definition documentation.'))

argument_parser.add_argument(
'--artifact_definitions', '--artifact-definitions',
dest='artifact_definitions', type=str, metavar='PATH', action='store',
help=('Path to a directory or file containing the artifact definition '
'.yaml files.'))

options = argument_parser.parse_args()

artifact_definitions = options.artifact_definitions
if not artifact_definitions:
artifact_definitions = os.path.join(
os.path.dirname(artifacts.__file__), 'data')
if not os.path.exists(artifact_definitions):
artifact_definitions = os.path.join('/', 'usr', 'share', 'artifacts')
if not os.path.exists(artifact_definitions):
artifact_definitions = None

if not artifact_definitions:
print('Missing artifact definitions.')
print('')
argument_parser.print_help()
print('')
return 1

logging.basicConfig(
level=logging.INFO, format='[%(levelname)s] %(message)s')

registry = artifacts_registry.ArtifactDefinitionsRegistry()
reader = artifacts_reader.YamlArtifactsReader()

if os.path.isdir(artifact_definitions):
registry.ReadFromDirectory(reader, artifact_definitions)
elif os.path.isfile(artifact_definitions):
registry.ReadFromFile(reader, artifact_definitions)

output_directory = os.path.join('docs', 'sources', 'definitions')
os.makedirs(output_directory, exist_ok=True)

index_rst_file_path = os.path.join(output_directory, 'index.rst')
with IndexRstOutputWriter(index_rst_file_path) as index_rst_writer:
for artifact_definition in sorted(
registry.GetDefinitions(), key=lambda definition: definition.name):
markdown_file_path = os.path.join(
output_directory, f'{artifact_definition.name:s}.md')
with MarkdownOutputWriter(markdown_file_path) as markdown_writer:
markdown_writer.WriteArtifactDefinition(artifact_definition)

index_rst_writer.WriteArtifactDefinition(artifact_definition.name)

return 0


if __name__ == '__main__':
sys.exit(Main())

0 comments on commit fbf96b0

Please sign in to comment.