web.py -> falcon; progress on python3

.coveragerc

@@ -1,4 +1,4 @@
 [report]
 omit = setup.py
-omit = */conftest.py
-omit = */suplemon/*
+       */conftest.py
+       */suplemon/*

.gitignore

@@ -8,6 +8,7 @@
 .cache
 .coverage
 .DS_Store
+.pytest_cache
 .vent
 /build/
 dist

Dockerfile.test

@@ -1,7 +1,7 @@
 FROM elasticsearch:2-alpine
 FROM rabbitmq:3-management
 FROM redis:alpine
-FROM ubuntu:latest
+FROM ubuntu:16.04
 LABEL maintainer="Charlie Lewis <clewis@iqt.org>"
 
 RUN apt-get update && apt-get install -y \
@@ -10,8 +10,8 @@ RUN apt-get update && apt-get install -y \
     curl \
     git \
     make \
-    python \
-    python-pip \
+    python3 \
+    python3-pip \
     software-properties-common
 
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
@@ -25,8 +25,8 @@ RUN apt-get update && apt-get install -y docker-ce
 RUN echo "https://github.com/cyberreboot/vent:\n  rabbitmq:" >> ~/.vent_startup.yml
 ADD . /vent
 WORKDIR /vent
-RUN pip install -r tests/requirements.txt
-RUN python2.7 setup.py install
+RUN pip3 install -r tests/requirements.txt
+RUN python3 setup.py install
 
 ENTRYPOINT ["pytest"]
 CMD ["-l", "-s", "-v", "--cov=.", "-k", "'not vendor'", "--cov-report", "term-missing"]

tests/requirements.txt

@@ -1,14 +1,13 @@
 docker==3.3.0
 elasticsearch==6.2.0
+falcon==1.4.1
+falcon-cors==1.1.7
 npyscreen==4.10.5
-paste==2.0.3
 pika==0.11.2
 pytest==3.6.0
 pytest-cov==2.5.1
 python-magic==0.4.15
 redis==2.10.6
 rq==0.10.0
 sphinx==1.7.5
-urllib3==1.22
 watchdog==0.8.3
-web.py==0.39

tests/test_api_actions.py

@@ -11,8 +11,10 @@ def test_startup():
     status = instance.startup()
     assert isinstance(status, tuple)
     assert status[0]
+    txt = None
     with open(instance.plugin.manifest) as man:
-        assert 'rabbitmq' in man.read()
+        txt = man.read()
+    assert 'rabbitmq' in txt
 
 def test_add():
     """ Test the add function """

vent/api/actions.py

@@ -1,14 +1,13 @@
-import Queue
-
 import ast
 import docker
 import getpass
 import json
 import os
+import queue
 import re
 import shutil
 import tempfile
-import urllib2
+import urllib
 import yaml
 
 from vent.api.plugins import Plugin
@@ -30,7 +29,7 @@ def __init__(self, **kargs):
         self.vent_config = self.plugin.path_dirs.cfg_file
         self.startup_file = self.plugin.path_dirs.startup_file
         self.p_helper = self.plugin.p_helper
-        self.queue = Queue.Queue()
+        self.queue = queue.Queue()
         self.logger = Logger(__name__)
 
     def add(self, repo, tools=None, overrides=None, version="HEAD",
@@ -1453,9 +1452,9 @@ def post_request(url, json_data):
             data = json.dumps(data)
 
             # create the post request and send it off
-            req = urllib2.Request(url, data)
+            req = urllib.request(url, data)
             req.add_header('Content-Type', 'application/json')
-            response = urllib2.urlopen(req, data)
+            response = urllib.request.urlopen(req, data)
 
             # return whatever the webpage returned
             return (True, response.read())
@@ -1476,7 +1475,7 @@ def get_request(url):
             after a GET request or a failure message
         """
         try:
-            response = urllib2.urlopen(url)
+            response = urllib.request.urlopen(url)
             return (True, response.read())
         except Exception as e:  # pragma no cover
             return (False, "failed get request to " + url + " " + str(e))

vent/core/file_drop/test_file_drop.py

@@ -1,13 +1,12 @@
-import file_drop
-
+from .file_drop import GZHandler
 from redis import Redis
 from redis import StrictRedis
 from rq import Queue
 
 
 def test_file_drop_GZHandler():
     """ Tests the GZZHandler for file drop """
-    a = file_drop.GZHandler()
+    a = GZHandler()
 
     class Event:
         """ Creates a mock event object for tests """

vent/core/network_tap/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.7
+FROM python:3-alpine3.7
 LABEL maintainer="Charlie Lewis <clewis@iqt.org>" \
       vent="" \
       vent.name="network_tap" \
@@ -7,17 +7,18 @@ LABEL maintainer="Charlie Lewis <clewis@iqt.org>" \
       vent.repo="https://github.com/cyberreboot/vent" \
       vent.type="repository"
 
-
 RUN apk add --update \
-    python \
-    py-pip \
+    gcc \
+    git \
+    linux-headers \
+    musl-dev \
+    python3-dev \
     && rm -rf /var/cache/apk/*
 
 COPY . /network-tap
-WORKDIR /network-tap/ncontrol
-RUN pip install -r requirements.txt
+WORKDIR /network-tap
+RUN pip3 install -r ncontrol/requirements.txt
 
 EXPOSE 8080
 
-ENTRYPOINT ["python", "ncontrol.py"]
-CMD [""]
+CMD ["/network-tap/startup.sh"]

vent/core/network_tap/ncontrol/ncontrol.py

@@ -1,58 +1,16 @@
-#!/usr/bin/env python
 import docker
-import logging
-import sys
-import web
+import falcon
 
-from rest.create import CreateR
-from rest.delete import DeleteR
-from rest.nics import NICsR
-from rest.nlist import ListR
-from rest.start import StartR
-from rest.stop import StopR
-from rest.update import UpdateR
+from falcon_cors import CORS
+from .routes import routes
 
 
-module_logger = logging.getLogger(__name__)
+cors = CORS(allow_all_origins=True)
+api = application = falcon.API(middleware=[cors.middleware])
 
+r = routes()
+for route in r:
+    api.add_route(route, r[route])
 
-class NControlServer(object):
-    """
-    This class is responsible for initializing the urls and web server.
-    """
-    # need __new__ for tests, but fails to call __init__ when actually running
-    def __new__(*args, **kw):
-        if hasattr(sys, '_called_from_test'):
-            module_logger.info("don't call __init__")
-        else:  # pragma: no cover
-            return object.__new__(*args, **kw)
-
-    def __init__(self, port=8080, host='0.0.0.0'):  # pragma: no cover
-        d_client = docker.from_env()
-        d_client.images.pull('cyberreboot/vent-ncapture', tag='master')
-        nf_inst = NControl()
-        urls = nf_inst.urls()
-        app = web.application(urls, globals())
-        web.httpserver.runsimple(app.wsgifunc(), (host, port))
-
-
-class NControl:
-    """
-    This class is for defining things needed to start up.
-    """
-
-    @staticmethod
-    def urls():
-        urls = (
-            '/create', CreateR,
-            '/delete', DeleteR,
-            '/list', ListR,
-            '/nics', NICsR,
-            '/start', StartR,
-            '/stop', StopR,
-            '/update', UpdateR
-        )
-        return urls
-
-if __name__ == '__main__':
-    NControlServer().app.run()
+d_client = docker.from_env()
+d_client.images.pull('cyberreboot/vent-ncapture', tag='master')