Merge pull request #1 from ashcrow/host-endpoint

Added host endpoint and tests.

conf/logger.json

@@ -19,6 +19,11 @@
             "level": "DEBUG",
             "propagate": false
         },
+        "investigator": {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": false
+        },
         "resources": {
             "handlers": ["console"],
             "level": "DEBUG",

doc/apidoc/commissaire.queues.rst

@@ -0,0 +1,7 @@
+commissaire.queues module
+=========================
+
+.. automodule:: commissaire.queues
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/apidoc/commissaire.rst

@@ -14,6 +14,7 @@ Subpackages
     commissaire.authentication
     commissaire.compat
     commissaire.handlers
+    commissaire.transport
 
 Submodules
 ----------
@@ -22,6 +23,7 @@ Submodules
 
    commissaire.middleware
    commissaire.model
+   commissaire.queues
    commissaire.resource
    commissaire.script
 

doc/apidoc/commissaire.transport.ansibleapi.rst

@@ -0,0 +1,7 @@
+commissaire.transport.ansibleapi module
+=======================================
+
+.. automodule:: commissaire.transport.ansibleapi
+    :members:
+    :undoc-members:
+    :show-inheritance:

doc/apidoc/commissaire.transport.rst

@@ -0,0 +1,15 @@
+commissaire.transport package
+=============================
+
+.. automodule:: commissaire.transport
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+
+   commissaire.transport.ansibleapi
+

doc/endpoints.rst

@@ -8,12 +8,69 @@ Host
 
 GET
 ```
+Retrieve a specific host record.
+
+.. code-block:: javascript
+
+   {
+       "address": string,       // The IP address of the cluster host
+       "status":  enum(string), // The status of the cluster host
+       "os": enum(string),      // The OS name
+       "cpus": int,             // The number of CPUs on the cluster host
+       "memory": int,           // The memory of the cluster host in kilobytes
+       "space": int,            // The diskspace on the cluster host
+       "last_check": string     // ISO date format the cluster host was last checked
+   }
+
+.. note::
+   See :ref:`host-statuses` for a list and description of host statuses.
+
+.. note::
+   See :ref:`host-os` for a list and description of host statuses.
+
+Example
+~~~~~~~
+
+.. code-block:: javascript
+
+   {
+       "address": "192.168.100.50",
+       "status": "active",
+       "os": "atomic",
+       "cpus": 4,
+       "memory": 11989228,
+       "space": 487652,
+       "last_check": "2015-12-17T15:48:18.710454"
+   }
 
 PUT
 ```
+Creates a new host record.
+
+.. code-block:: javascript
+
+   {
+       "address": string      // The IP address of the cluster host
+       "ssh_priv_key": string // base64 encoded ssh private key
+   }
+
+.. note::
+   The rest of the host record will be filled out once the data has been pulled from the cluster host.
+
+Example
+~~~~~~~
+
+.. code-block:: javascript
+
+   {
+       "address": "192.168.100.50",
+       "ssh_priv_key": "dGVzdAo..."
+   }
 
 DELETE
 ``````
+Deletes a host record.
+
 
 Hosts
 -----

requirements.txt

@@ -2,3 +2,4 @@ gevent==1.1rc3
 falcon>=0.3
 bcrypt
 python-etcd==0.4.2
+ansible>=2.0.0.1

src/commissaire/handlers/hosts.py

@@ -13,11 +13,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import falcon
 import etcd
 import json
 
 from commissaire.model import Model
+from commissaire.queues import INVESTIGATE_QUEUE
 from commissaire.resource import Resource
 
 
@@ -27,7 +29,9 @@ class Host(Model):
     """
     _json_type = dict
     _attributes = (
-        'address', 'status', 'os', 'cpus', 'memory', 'space', 'last_check')
+        'address', 'status', 'os', 'cpus', 'memory',
+        'space', 'last_check', 'ssh_priv_key')
+    _hidden_attributes = ('ssh_priv_key', )
 
 
 class Hosts(Model):
@@ -73,8 +77,7 @@ def on_get(self, req, resp):
             resp.status = falcon.HTTP_200
             req.context['model'] = None
 
-# TODO
-'''
+
 class HostResource(Resource):
     """
     Resource for working with a single Host.
@@ -119,11 +122,22 @@ def on_put(self, req, resp, address):
             resp.status = falcon.HTTP_409
             return
         except etcd.EtcdKeyNotFound:
-            data = req.stream.read()
-            print(data.decode())
-            host = Host(**json.loads(data.decode()))
+            data = req.stream.read().decode()
+            host_creation = json.loads(data)
+            ssh_priv_key = host_creation['ssh_priv_key']
+            INVESTIGATE_QUEUE.put((host_creation, ssh_priv_key))
+            host_creation['address'] = address
+            host_creation['os'] = ''
+            host_creation['status'] = 'investigating'
+            host_creation['cpus'] = -1
+            host_creation['memory'] = -1
+            host_creation['space'] = -1
+            host_creation['ssh_priv_key'] = ssh_priv_key
+            host_creation['last_check'] = datetime.datetime.min.isoformat()
+            host = Host(**host_creation)
             new_host = self.store.set(
-                '/commissaire/hosts/{0}'.format(address), host.to_json())
+                '/commissaire/hosts/{0}'.format(
+                    address), host.to_json(secure=True))
             resp.status = falcon.HTTP_201
             req.context['model'] = Host(**json.loads(new_host.value))
 
@@ -142,7 +156,6 @@ def on_delete(self, req, resp, address):
         try:
             host = self.store.delete(
                 '/commissaire/hosts/{0}'.format(address))
-            falcon.status = falcon.HTTP_410
+            resp.status = falcon.HTTP_410
         except etcd.EtcdKeyNotFound:
             resp.status = falcon.HTTP_404
-'''

src/commissaire/model.py

@@ -23,6 +23,7 @@ class Model:
     """
     _json_type = None
     _attributes = ()
+    _hidden_attributes = ()
 
     def __init__(self, **kwargs):
         for key in self._attributes:
@@ -33,24 +34,27 @@ def __init__(self, **kwargs):
                         ', '.join(self._attributes)))
             setattr(self, key, kwargs[key])
 
-    def _struct_for_json(self):
+    def _struct_for_json(self, secure=False):
         if self._json_type is dict:
-            return self._dict_for_json()
+            return self._dict_for_json(secure)
         elif self._json_type is list:
-            return self._list_for_json()
+            return self._list_for_json(secure)
 
-    def _list_for_json(self):
+    def _list_for_json(self, secure):
         if len(self._attributes) == 1:
             data = getattr(self, self._attributes[0])
         return data
 
-    def _dict_for_json(self):
+    def _dict_for_json(self, secure):
         data = {}
         for key in self._attributes:
-            data[key] = getattr(self, key)
+            if secure:
+                data[key] = getattr(self, key)
+            elif key not in self._hidden_attributes:
+                data[key] = getattr(self, key)
         return data
 
-    def to_json(self):
+    def to_json(self, secure=False):
         return json.dumps(
-            self._struct_for_json(),
-            default=lambda o: o._struct_for_json())
+            self._struct_for_json(secure=secure),
+            default=lambda o: o._struct_for_json(secure=secure))

src/commissaire/queues.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2016  Red Hat, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://
+from gevent.queue import Queue, Empty
+
+
+INVESTIGATE_QUEUE = Queue()
+'''
+ROUTER_QUEUE = Queue()
+QUEUES = {
+    "ALL": [Queue(), Queue()],
+    "10.2.0.2": [Queue()],
+}
+'''

src/commissaire/script.py

@@ -18,6 +18,8 @@
 from gevent.monkey import patch_all
 patch_all()
 
+import datetime
+import base64
 import json
 import logging
 import logging.config
@@ -26,21 +28,61 @@
 import falcon
 import gevent
 
-from gevent.queue import Queue, Empty
 from gevent.pywsgi import WSGIServer
 
-from commissaire.handlers.hosts import HostsResource  # , HostResource
+from commissaire.handlers.hosts import HostsResource, HostResource
+from commissaire.queues import *
 from commissaire.authentication import httpauth
 from commissaire.middleware import JSONify
 
 
-ROUTER_QUEUE = Queue()
-QUEUES = {
-    "ALL": [Queue(), Queue()],
-    "10.2.0.2": [Queue()],
-}
+# TODO: Move greenlet funcs to their own module
+def investigator(q, c):  # pragma: no cover
+    import tempfile
+    from commissaire.transport import ansibleapi
+    logger = logging.getLogger('investigator')
+    logger.info('Investigator started')
 
+    get_info = ansibleapi.Transport().get_info
 
+    while True:
+        to_investigate, ssh_priv_key = q.get()
+        address = to_investigate['address']
+        logger.info('Investigating {0}...'.format(address))
+        logger.debug('Investigation details: key={0}, data={1}'.format(
+            to_investigate, ssh_priv_key))
+
+        f = tempfile.NamedTemporaryFile(prefix='key', delete=False)
+        key_file = f.name
+        logger.debug(
+            'Using {0} as the temporary key location for {1}'.format(
+                key_file, address))
+        f.write(base64.decodestring(ssh_priv_key))
+        logger.debug('Wrote key for {0}'.format(address))
+        f.close()
+
+        result, facts = get_info(address, key_file)
+        try:
+            f.unlink(key_file)
+            logger.debug('Removed temporary key file {0}'.format(key_file))
+        except:
+            logger.warn(
+                'Unable to remove the temporary key file: {0}'.format(
+                    key_file))
+        uri = '/commissaire/hosts/{0}'.format(address)
+        data = json.loads(c.get(uri).value)
+        data.update(facts)
+        data['last_check'] = datetime.datetime.utcnow().isoformat()
+        data['status'] = 'bootstrapping'
+        logger.info('Facts for {0} retrieved'.format(address))
+
+        c.set(uri, json.dumps(data))
+        logging.debug('Investigation update for {0}: {1}'.format(
+            address, data))
+        logger.info(
+            'Finished and stored investigation for {0}'.format(address))
+
+'''
 def host_watcher(q, c):  # pragma: no cover
     logger = logging.getLogger('watcher')
     next_idx = None
@@ -91,6 +133,8 @@ def router(q):  # pragma: no cover
         logger.info('Sent change for {0} to {1} queues.'.format(
             change.etcd_index, sent_to))
 
+'''
+
 
 def create_app(ds):  # pragma: no cover
     # TODO: Make this configurable
@@ -102,8 +146,8 @@ def create_app(ds):  # pragma: no cover
 
     app = falcon.API(middleware=[http_auth, JSONify()])
 
-    # app.add_route('/api/v0/hosts/{address}', HostResource(ds))
-    app.add_route('/api/v0/hosts', HostsResource(ds, ROUTER_QUEUE))
+    app.add_route('/api/v0/host/{address}', HostResource(ds, None))
+    app.add_route('/api/v0/hosts', HostsResource(ds, None))
     return app
 
 
@@ -135,6 +179,7 @@ def main():  # pragma: no cover
         sys.stderr.write('{0}\n'.format(err))
         raise SystemExit(1)
 
+    investigator_thread = gevent.spawn(investigator, INVESTIGATE_QUEUE, ds)
     # watch_thread = gevent.spawn(host_watcher, ROUTER_QUEUE, ds)
     # router_thread = gevent.spawn(router, ROUTER_QUEUE)
 
@@ -144,6 +189,7 @@ def main():  # pragma: no cover
     except KeyboardInterrupt:
         pass
 
+    investigator_thread.kill()
     # watch_thread.kill()
     # router_thread.kill()