feat(integration tests): Utilities for testing ansible-trace

.github/workflows/CI.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ansible:
+          - stable-2.11
+          - stable-2.12
+          - stable-2.13
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          path: ansible_collections/mhansen/ansible-trace
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+
+      - name: Install ansible-base (${{ matrix.ansible }})
+        run: pip3 install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check
+
+      - name: Install test dependencies
+        run: pip3 install pytest mypy pytest-ansible-playbook
+
+      - name: Run integration tests
+        run: pytest -v  $(ls tests/*.py)
+        working-directory: ansible_collections/mhansen/ansible-trace/tests/integration
+

.gitignore

@@ -1 +1,5 @@
 __pycache__
+tests/integration/inventory.ini
+tests/integration/trace
+tests/integration/tests/__pycache__
+tests/integration/tests/.pytest_cache

plugins/callback/trace.py

@@ -10,6 +10,15 @@
 # GNU General Public License for more details.
 
 
+from ansible.plugins.callback import CallbackBase
+from typing import Dict, TextIO
+from datetime import datetime
+from dataclasses import dataclass
+import time
+import os
+import json
+import atexit
+
 DOCUMENTATION = '''
     name: trace
     type: aggregate
@@ -34,16 +43,6 @@
       - enable in configuration
 '''
 
-import atexit
-import json
-import os
-import time
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Dict, TextIO
-
-from ansible.plugins.callback import CallbackBase
-
 
 class CallbackModule(CallbackBase):
     """
@@ -64,8 +63,10 @@ class CallbackModule(CallbackBase):
     def __init__(self):
         super(CallbackModule, self).__init__()
 
-        self._output_dir: str = os.getenv('TRACE_OUTPUT_DIR', os.path.join(os.path.expanduser('.'), 'trace'))
-        self._hide_task_arguments: str = os.getenv('TRACE_HIDE_TASK_ARGUMENTS', 'False').lower()
+        self._output_dir: str = os.getenv(
+            'TRACE_OUTPUT_DIR', os.path.join(os.path.expanduser('.'), 'trace'))
+        self._hide_task_arguments: str = os.getenv(
+            'TRACE_HIDE_TASK_ARGUMENTS', 'False').lower()
         self._hosts: Dict[Host] = {}
         self._next_pid: int = 1
         self._first: bool = True
@@ -84,7 +85,8 @@ def _write_event(self, e: Dict):
         if not self._first:
             self._f.write(",\n")
         self._first = False
-        json.dump(e, self._f, sort_keys=True, indent=2) # sort for reproducibility
+        # sort for reproducibility
+        json.dump(e, self._f, sort_keys=True, indent=2)
         self._f.flush()
 
     def v2_runner_on_start(self, host, task):
@@ -128,7 +130,6 @@ def v2_runner_on_start(self, host, task):
             },
         })
 
-
     def _end_span(self, result, status: str):
         task = result._task
         uuid = task._uuid
@@ -162,6 +163,7 @@ def _end(self):
         self._f.write("\n]")
         self._f.close()
 
+
 @dataclass
 class Host:
     name: str

tests/integration/ansible.cfg

@@ -0,0 +1,89 @@
+[defaults]
+# (boolean) By default Ansible will issue a warning when received from a task action (module or action plugin)
+# These warnings can be silenced by adjusting this setting to False.
+;action_warnings=True
+
+# (list) Accept list of cowsay templates that are 'safe' to use, set to empty list if you want to enable all installed templates.
+;cowsay_enabled_stencils=bud-frogs, bunny, cheese, daemon, default, dragon, elephant-in-snake, elephant, eyes, hellokitty, kitty, luke-koala, meow, milk, moofasa, moose, ren, sheep, small, stegosaurus, stimpy, supermilker, three-eyes, turkey, turtle, tux, udder, vader-koala, vader, www
+
+# (string) Specify a custom cowsay path or swap in your cowsay implementation of choice
+;cowpath=
+
+# (string) This allows you to chose a specific cowsay stencil for the banners or use 'random' to cycle through them.
+;cow_selection=default
+
+# (boolean) This option forces color mode even when running without a TTY or the "nocolor" setting is True.
+;force_color=False
+
+# (boolean) This setting allows suppressing colorizing output, which is used to give a better indication of failure and status information.
+;nocolor=False
+
+# (boolean) If you have cowsay installed but want to avoid the 'cows' (why????), use this.
+;nocows=False
+
+# (boolean) Sets the default value for the any_errors_fatal keyword, if True, Task failures will be considered fatal errors.
+;any_errors_fatal=False
+
+# (path) The password file to use for the become plugin. --become-password-file.
+# If executable, it will be run and the resulting stdout will be used as the password.
+;become_password_file=
+
+# (pathspec) Colon separated paths in which Ansible will search for Become Plugins.
+;become_plugins=~/.ansible/plugins/become:/usr/share/ansible/plugins/become
+
+# (string) Chooses which cache plugin to use, the default 'memory' is ephemeral.
+;fact_caching=memory
+
+# (string) Defines connection or path information for the cache plugin
+;fact_caching_connection=
+
+# (string) Prefix to use for cache plugin files/tables
+;fact_caching_prefix=ansible_facts
+
+# (integer) Expiration timeout for the cache plugin data
+;fact_caching_timeout=86400
+
+# (list) Whitelist of callable methods to be made available to template evaluation
+;callable_enabled=
+
+# (list) List of enabled callbacks, not all callbacks need enabling, but many of those shipped with Ansible do as we don't want them activated by default.
+;callbacks_enabled=
+
+# (string) When a collection is loaded that does not support the running Ansible version (via the collection metadata key `requires_ansible`), the default behavior is to issue a warning and continue anyway. Setting this value to `ignore` skips the warning entirely, while setting it to `fatal` will immediately halt Ansible execution.
+;collections_on_ansible_version_mismatch=warning
+
+# (pathspec) Colon separated paths in which Ansible will search for collections content. Collections must be in nested *subdirectories*, not directly in these directories. For example, if ``COLLECTIONS_PATHS`` includes ``~/.ansible/collections``, and you want to add ``my.collection`` to that directory, it must be saved as ``~/.ansible/collections/ansible_collections/my/collection``.
+
+;collections_path=~/.ansible/collections:/usr/share/ansible/collections
+
+# (boolean) A boolean to enable or disable scanning the sys.path for installed collections
+;collections_scan_sys_path=True
+
+# (boolean) Ansible can issue a warning when the shell or command module is used and the command appears to be similar to an existing Ansible module.
+# These warnings can be silenced by adjusting this setting to False. You can also control this at the task level with the module option ``warn``.
+# As of version 2.11, this is disabled by default.
+;command_warnings=False
+
+# (path) The password file to use for the connection plugin. --connection-password-file.
+;connection_password_file=
+
+# (pathspec) Colon separated paths in which Ansible will search for Action Plugins.
+;action_plugins=~/.ansible/plugins/action:/usr/share/ansible/plugins/action
+
+# (boolean) When enabled, this option allows lookup plugins (whether used in variables as ``{{lookup('foo')}}`` or as a loop as with_foo) to return data that is not marked 'unsafe'.
+# By default, such data is marked as unsafe to prevent the templating engine from evaluating any jinja2 templating language, as this could represent a security risk. This option is provided to allow for backward compatibility, however users should first consider adding allow_unsafe=True to any lookups which may be expected to contain data which may be run through the templating engine late
+;allow_unsafe_lookups=False
+
+# (boolean) This controls whether an Ansible playbook should prompt for a login password. If using SSH keys for authentication, you probably do not needed to change this setting.
+;ask_pass=False
+
+# (boolean) This controls whether an Ansible playbook should prompt for a vault password.
+;ask_vault_pass=False
+
+# (pathspec) Colon separated paths in which Ansible will search for Cache Plugins.
+;cache_plugins=~/.ansible/plugins/cache:/usr/share/ansible/plugins/cache
+
+# (pathspec) Colon separated paths in which Ansible will search for Callback Plugins.
+callback_plugins=../../plugins/callback
+callbacks_enabled = trace
+

tests/integration/basic/basic.yml

@@ -0,0 +1,14 @@
+---
+
+- hosts: all
+  environment:
+    CALLBACKS_ENABLED: trace
+    TRACE_OUTPUT_DIR: /ansible_collections/mhansen/ansible-trace
+    TRACE_HIDE_TASK_ARGUMENTS: True
+  tasks:
+    - name: Ping self
+      ansible.builtin.ping:
+
+    - name: Hello world
+      shell: "echo 'hello world!'"
+

tests/integration/inventories/multiple_hosts.ini

@@ -0,0 +1,11 @@
+[all]
+127.0.0.1 ansible_connection=local
+127.0.0.2 ansible_connection=local
+127.0.0.3 ansible_connection=local
+127.0.0.4 ansible_connection=local
+127.0.0.5 ansible_connection=local
+127.0.0.6 ansible_connection=local
+127.0.0.7 ansible_connection=local
+127.0.0.8 ansible_connection=local
+127.0.0.9 ansible_connection=local
+127.0.0.10 ansible_connection=local

tests/integration/inventories/one_host.ini

@@ -0,0 +1,2 @@
+[all]
+localhost ansible_connection=local

tests/integration/tests/basic.py

@@ -0,0 +1,37 @@
+# Handle integration tests
+from typing import Union, Dict, List, Any
+from utils import get_last_trace, parse_and_validate_trace
+from event import HostEvent
+import pytest
+
+JSONTYPE = Union[None, int, str, bool, List[Any], Dict[str, Any]]
+
+
+@pytest.mark.ansible_playbook('basic/basic.yml')
+@pytest.mark.ansible_inventory('inventories/multiple_hosts.ini')
+@pytest.mark.ansible_strategy('free')
+def test_basic_multiple_free(ansible_play):
+    trace_hosts: Dict[int, HostEvent]
+    trace_events: Dict[int, Any]
+    trace_json: JSONTYPE = get_last_trace()
+    trace_hosts, trace_events = parse_and_validate_trace(trace_json)
+
+
+@pytest.mark.ansible_playbook('basic/basic.yml')
+@pytest.mark.ansible_inventory('inventories/multiple_hosts.ini')
+@pytest.mark.ansible_strategy('linear')
+def test_basic_multiple_linear(ansible_play):
+    trace_hosts: Dict[int, HostEvent]
+    trace_events: Dict[int, Any]
+    trace_json: JSONTYPE = get_last_trace()
+    trace_hosts, trace_events = parse_and_validate_trace(trace_json)
+
+
+@pytest.mark.ansible_playbook('basic/basic.yml')
+@pytest.mark.ansible_inventory('inventories/one_host.ini')
+@pytest.mark.ansible_strategy('linear')
+def test_basic_single_linear(ansible_play):
+    trace_hosts: Dict[int, HostEvent]
+    trace_events: Dict[int, Any]
+    trace_json: JSONTYPE = get_last_trace()
+    trace_hosts, trace_events = parse_and_validate_trace(trace_json)

tests/integration/tests/conftest.py

@@ -0,0 +1,3 @@
+import pytest
+from fixtures import ansible_play
+

tests/integration/tests/event.py

@@ -0,0 +1,46 @@
+import re
+
+class Event:
+
+    pid: int
+    name: str
+    ph: str
+
+    def __init__(self, dict):
+        if not 'pid' in dict:
+            raise ValueError('Event must be linked to pid')
+        self.pid = dict['pid']
+        self.ph = dict['ph']
+
+class DurationEvent(Event):
+
+    id: int
+    ts: float
+
+    def __init__(self, dict):
+        if not 'id' in dict:
+            raise ValueError('Duration event needs to have id')
+        self.id = dict['id']
+        if not 'ts' in dict:
+            raise ValueError('Duration event {} needs to have a timestamp'.format(self.id))
+        if not 'name' in dict:
+            raise ValueError('Duration event {} needs to have a name'.format(self.id))
+        if not 'ph' in dict or not re.search("^(B|E)$", dict['ph']):
+            raise ValueError('Duration event {} needs to have a ph either set to B or E'.format(self.id))
+        self.ts = dict['ts']
+        self.name = dict['name']
+
+        super().__init__(dict)
+
+class HostEvent(Event):
+
+    def __init__(self, dict):
+        if not 'ph' in dict or dict['ph'] != 'M':
+            raise ValueError('Host event needs to have a ph set to M')
+        if not 'args' in dict or not 'name' in dict['args']:
+            raise ValueError('Host events needs to have name')
+
+        self.name = dict['args']['name']
+
+        super().__init__(dict)
+

tests/integration/tests/fixtures.py

@@ -0,0 +1,23 @@
+import os
+import pytest
+from pytest_ansible_playbook import runner
+
+
+@pytest.fixture
+def ansible_play(request):
+
+    # Get required marks, raise exception if not defined or invalid params
+    inventory = request.node.get_closest_marker('ansible_inventory').args[0]
+    strategy = request.node.get_closest_marker('ansible_strategy').args[0]
+    playbook = request.node.get_closest_marker('ansible_playbook').args[0]
+
+    # Set params for ansible runner
+    request.config.option.ansible_playbook_directory = "."
+    request.config.option.ansible_playbook_inventory = inventory
+
+    # Assign strategy
+    os.environ["ANSIBLE_STRATEGY"] = strategy
+
+    # Test required playbook
+    with runner(request, [playbook]):
+        yield

tests/integration/tests/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+markers = 
+    ansible_strategy
+    ansible_playbook
+    ansible_inventory