Moving callbacks to validations-libs

Callback plugins were transferred from validations-common
repository to validations-libs.

Necessary adjustments were made to the module structure,
requierements, as well as installation and documentation generator config.

Associated tests were moved as well, with removal of superfluous inheritance
and imports included.

Demonstration http server module for communication with `http_json`
callback was moved with directory structure preserved.

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
Change-Id: I31768375430a2f29da71aae8f3db3882c373ced5

doc/source/conf.py

@@ -53,6 +53,7 @@
 # autodoc generation is a bit aggressive and a nuisance when doing heavy
 # text edit cycles.
 # execute "export SPHINX_DEBUG=1" in your terminal to disable
+autodoc_mock_imports = ['oslotest', 'ansible']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

setup.cfg

@@ -25,6 +25,7 @@ packages = validations_libs
 data_files =
     etc =
         validation.cfg
+    share/ansible/callback_plugins = validations_libs/callback_plugins/*
 
 [compile_catalog]
 directory = validations-libs/locale

test-requirements.txt

@@ -13,4 +13,5 @@ python-subunit>=1.0.0 # Apache-2.0/BSD
 stestr>=2.0.0 # Apache-2.0
 testscenarios>=0.4 # Apache-2.0/BSD
 testtools>=2.2.0 # MIT
+oslotest>=3.2.0 # Apache-2.0
 pre-commit # MIT

tools/http_server.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import logging
+
+
+class SimpleHandler(BaseHTTPRequestHandler):
+    def _set_headers(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_GET(self):
+        logging.info("Received GET request:\n"
+                     "Headers: {}\n".format(str(self.headers)))
+        self._set_headers()
+        self.wfile.write("GET request: {}".format(self.path).encode('utf-8'))
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length'])
+        data = self.rfile.read(content_length)
+        logging.info("Received POST request:\n"
+                     "Headers: {}\n"
+                     "Body: \n{}\n".format(self.headers, data.decode('utf-8')))
+        self._set_headers()
+        self.wfile.write("POST request: {}".format(self.path).encode('utf-8'))
+
+
+def run(host='localhost', port=8989):
+    logging.basicConfig(level=logging.INFO)
+    http_server = HTTPServer((host, port), SimpleHandler)
+    logging.info("Starting http server...\n")
+    try:
+        http_server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    http_server.server_close()
+    logging.info('Stopping http server...\n')
+
+
+if __name__ == '__main__':
+    run()

validations_libs/ansible.py

@@ -116,10 +116,13 @@ def _callbacks(self, callback_whitelist, output_callback, envvars={},
         if isinstance(envvars, dict):
             env.update(envvars)
         output_callback = env.get('ANSIBLE_STDOUT_CALLBACK', output_callback)
+        # TODO(jpodivin) Whitelist was extended with new callback names
+        # to prevent issues during transition period.
+        # The entries with 'vf_' prefix should be removed afterwards.
         callback_whitelist = ','.join(filter(None, [callback_whitelist,
                                                     output_callback,
-                                                    'validation_json',
-                                                    'profile_tasks']))
+                                                    'profile_tasks',
+                                                    'vf_validation_json']))
         return callback_whitelist, output_callback
 
     def _ansible_env_var(self, output_callback, ssh_user, workdir, connection,

validations_libs/callback_plugins/__init__.py

@@ -0,0 +1,13 @@
+"""
+This module contains various callbacks developed to facilitate functions
+of the Validation Framework.
+
+Somewhat unorthodox naming of the callback classes is a direct result of how
+ansible handles loading plugins.
+The ansible determines the purpose of each plugin by looking at its class name.
+As you can see in the 'https://github.com/ansible/ansible/blob/devel/lib/ansible/plugins/loader.py'
+from the ansible repo, the loader uses the class names to categorize plugins.
+This means that every callback plugin has to have the same class name,
+and the unfortunate coder has to discern their purpose by checking
+their module names.
+"""

validations_libs/callback_plugins/vf_fail_if_no_hosts.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from ansible.plugins.callback import CallbackBase
+
+
+class CallbackModule(CallbackBase):
+    CALLBACK_VERSION = 2.0
+    CALLBACK_NAME = 'fail_if_no_hosts'
+
+    def __init__(self, display=None):
+        super(CallbackModule, self).__init__(display)
+
+    def v2_playbook_on_stats(self, stats):
+        if len(stats.processed.keys()) == 0:
+            sys.exit(10)

validations_libs/callback_plugins/vf_http_json.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    requirements:
+      - whitelist in configuration
+    short_description: sends JSON events to a HTTP server
+    description:
+      - This plugin logs ansible-playbook and ansible runs to an HTTP server in JSON format
+    options:
+      server:
+        description: remote server that will receive the event
+        env:
+        - name: HTTP_JSON_SERVER
+        default: http://localhost
+        ini:
+          - section: callback_http_json
+            key: http_json_server
+      port:
+        description: port on which the remote server is listening
+        env:
+          - name: HTTP_JSON_PORT
+        default: 8989
+        ini:
+          - section: callback_http_json
+            key: http_json_port
+'''
+import datetime
+import json
+import os
+
+from urllib import request
+
+from validations_libs.callback_plugins import vf_validation_json
+
+url = '{}:{}'.format(os.getenv('HTTP_JSON_SERVER', 'http://localhost'),
+                     os.getenv('HTTP_JSON_PORT', '8989'))
+
+
+def http_post(data):
+    req = request.Request(url)
+    req.add_header('Content-Type', 'application/json; charset=utf-8')
+    json_data = json.dumps(data)
+    json_bytes = json_data.encode('utf-8')
+    req.add_header('Content-Length', len(json_bytes))
+    response = request.urlopen(req, json_bytes)
+
+
+def current_time():
+    return '%sZ' % datetime.datetime.utcnow().isoformat()
+
+
+class CallbackModule(vf_validation_json.CallbackModule):
+
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'http_json'
+    CALLBACK_NEEDS_WHITELIST = True
+
+    def __init__(self):
+        super(vf_validation_json.CallbackModule, self).__init__()
+        self.results = []
+        self.simple_results = []
+        self.env = {}
+        self.t0 = None
+        self.current_time = current_time()
+
+    def v2_playbook_on_stats(self, stats):
+        """Display info about playbook statistics"""
+
+        hosts = sorted(stats.processed.keys())
+
+        summary = {}
+        for h in hosts:
+            s = stats.summarize(h)
+            summary[h] = s
+
+        http_post({
+            'plays': self.results,
+            'stats': summary,
+            'validation_output': self.simple_results
+        })