Merge branch 'dev-gitlab-ci' into 'master'

Resolve "Migrate testing to gitlab-ci"

Closes ESRA-119, #4, #5, and #6

See merge request geon/redhawk-ui/rest-python!8

Author: B. Thomas Goodwin

.gitlab-ci.yml

@@ -0,0 +1,112 @@
+# This CI script builds Docker-REDHAWK's webserver image and deploys it to
+# dockerhub upon successful execution of its tests.
+variables:
+  CONTAINER_TEST_IMAGE:    $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+  CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
+  DOCKERHUB_IMAGE:         geontech/redhawk-webserver
+
+stages:
+  - build
+  - test
+  - release
+  - deploy
+  - cleanup
+
+.dind: &dind
+  image: docker:latest
+
+.container_registry: &container_registry
+  <<: *dind
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  after_script:
+    - docker logout $CI_REGISTRY
+
+# Build the test image and push it to the registry, delete the runner-local
+# image to avoid cluttering the runner.
+build:
+  stage: build
+  <<: *container_registry
+  script:
+    - sed -i -E "s/(geontech.+\:)(.+)/\1"$REDHAWK_VERSION"/g" Dockerfile
+    - docker build --pull --rm -t $CONTAINER_TEST_IMAGE .
+    - docker push $CONTAINER_TEST_IMAGE
+    - docker rmi $CONTAINER_TEST_IMAGE
+
+test:
+  stage: test
+  <<: *container_registry
+  script:
+    # Install docker-compose
+    - apk add --no-cache py2-pip && pip install docker-compose
+
+    # Pull the test image and start the stack without rebuilding it.
+    - cd tests
+    - docker pull ${CONTAINER_TEST_IMAGE}
+    - docker-compose -p ${CI_COMMIT_REF_SLUG} up -d --no-build
+    - docker-compose -p ${CI_COMMIT_REF_SLUG} exec -T rest
+      bash -l -c 'yum install -y rh.SigGen rh.FileWriter'
+    - docker-compose -p ${CI_COMMIT_REF_SLUG} exec -T rest
+      bash -l -c './test.sh' || RESULT=$?
+
+    # Clean up and exit with the result
+    - docker-compose -p ${CI_COMMIT_REF_SLUG} down
+    - docker rmi ${CONTAINER_TEST_IMAGE}
+    - exit ${RESULT}
+
+# Move the test image to internal 'latest' and clean up the worker's images
+release:
+  stage: release
+  only:
+    - master
+  <<: *container_registry
+  script:
+    - docker pull $CONTAINER_TEST_IMAGE
+    - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
+    - docker push $CONTAINER_RELEASE_IMAGE
+    - docker rmi $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
+
+# Clean up the test image from the internal registry.
+cleanup-ci:
+  stage: cleanup
+  when: always
+  image: alpine:latest
+  script:
+    - apk add --no-cache git bash curl
+    - git clone ${DOCKER_UTILS} docker-util
+    - docker-util/delete-image.sh $CONTAINER_TEST_IMAGE
+
+# Push the released image to DockerHub on tags
+deploy-image:
+  stage: deploy
+  only:
+    - tags
+  <<: *container_registry
+  variables:
+    dockerhub_image_tag:    geontech/redhawk-webserver:$CI_COMMIT_TAG
+    dockerhub_image_latest: geontech/redhawk-webserver:latest
+  script:
+    - docker pull $CONTAINER_RELEASE_IMAGE
+    - docker logout
+    - docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_TOKEN}
+    - docker tag $CONTAINER_RELEASE_IMAGE ${dockerhub_image_tag}
+    - docker tag $CONTAINER_RELEASE_IMAGE ${dockerhub_image_latest}
+    - docker push ${dockerhub_image_tag} ${dockerhub_image_latest}
+    - docker rmi ${CONTAINER_RELEASE_IMAGE} ${dockerhub_image_tag} ${dockerhub_image_latest}
+
+# Push to GitHub
+deploy-github:
+  stage: deploy
+  only:
+    - tags
+  image: alpine/git
+  script:
+    - eval $(ssh-agent -s)
+    - echo "${GITHUB_TOKEN}" | ssh-add - 
+    - mkdir -p ~/.ssh
+    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+    # Reconfigure git and push w/ tag(s)
+    - git config user.name "${GITHUB_USER_NAME}"
+    - git config user.email "${GITHUB_USER_EMAIL}"
+    - git remote set-url origin ${GITHUB_REPO}
+    - git push origin ${GITHUB_BRANCH} --follow-tags

Dockerfile

@@ -0,0 +1,60 @@
+# This file is protected by Copyright. Please refer to the COPYRIGHT file
+# distributed with this source distribution.
+#
+# This file is part of Geon's REST-Python.
+#
+# REST-Python is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# REST-Python is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see http://www.gnu.org/licenses/.
+#
+
+FROM geontech/redhawk-runtime:2.0.8
+LABEL name="REST-Python Web Server" \
+    description="Geon's Fork of REST-Python" \
+    maintainer="Thomas Goodwin <btgoodwin@geontech.com>"
+
+# Build-time configurable variables
+ARG REST_PYTHON_PORT=8080
+
+# Runtime variables
+ENV REST_PYTHON_PORT=${REST_PYTHON_PORT}
+
+# Expose the configured default port.
+EXPOSE ${REST_PYTHON_PORT}
+
+RUN yum install -y \
+    git \
+    gcc \
+    python-dev \
+    curl \
+    python-virtualenv && \
+    yum clean all -y
+
+# Install and update pip
+RUN curl https://bootstrap.pypa.io/get-pip.py | python && \
+    pip install -U pip
+
+# Install the rest-python server requirements
+COPY ./setup.sh ./pyvenv ./requirements.txt /opt/rest-python/
+WORKDIR /opt/rest-python
+RUN ./setup.sh install && pip install -r requirements.txt
+
+# Install the rest of the rest-python server and launcher
+COPY . /opt/rest-python
+RUN cp docker/rest-python.conf /etc/supervisor.d/rest-python.conf && \
+    cp docker/kill_supervisor.py /usr/bin/kill_supervisor.py && \
+    chmod u+x /usr/bin/kill_supervisor.py
+
+# Mount point for end-user apps
+VOLUME /opt/rest-python/apps
+
+CMD ["/usr/bin/supervisord"]

README.md

@@ -39,3 +39,32 @@ Once running the REST Interface can be tested at `http://localhost:<desired_port
 ## Deploying Applications
 
 You can either install your application in `apps` for REST-Python to serve them, or deploy them with a separate server (e.g., NodeJS).  REST-Python supports cross-domain responses to REST and Websocket requests to facilitate dual- or multi-server configurations to completely decouple the REDHAWK environment from the web application environment.  (See [Docker-REDHAWK's](http://github.com/GeonTech/docker-redhawk) `geontech/redhawk-webserver` image.)
+
+## Testing
+
+If you would like to test REST-Python changes in a functional domain, download Docker-CE and Docker Compose for your operating system.  The process takes 3 parts:
+
+ 1. Your locally-built instance of the webserver image
+ 2. Running the `tests/docker-compose.yml` stack with your webserver image
+ 3. Using `docker-compose exec` to start the tests script.
+
+ > Note: Verify the Dockerfile version of the parent image (`FROM...`) and the compose file vs. the version of REDHAWK you want to test against.  You can export `REDHAWK_VERSION` set to that version prior to running `docker-compose` to simplify changing versions of the infrastructure.
+
+The process is shown here:
+
+```
+cd tests
+export TEST_IMAGE=test-webserver
+docker-compose up -d --build
+docker-compose exec -T rest \
+    bash -l -c 'yum install -y rh.FileWriter rh.SigGen && ./test.sh'
+```
+
+If you make further changes and want to update the webserver image and container (the service is `rest` in the compose file), you can recreate the container without tearing down the entire compose by doing the following:
+
+```
+docker-compose stop rest
+docker-compose up --build -d rest
+```
+
+You would then repeat the `exec` command from above.

docker-compose.yml

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# Special thanks to the example at:
+# https://blog.zhaw.ch/icclab/process-management-in-docker-containers/
+import sys
+import os
+import signal
+
+
+def write_stdout(s):
+   sys.stdout.write(s)
+   sys.stdout.flush()
+
+
+def write_stderr(s):
+   sys.stderr.write(s)
+   sys.stderr.flush()
+
+
+def main():
+   while 1:
+       write_stdout('READY\n')
+       line = sys.stdin.readline()
+       write_stdout('This line kills supervisor: ' + line)
+       try:
+               pidfile = open('/var/run/supervisord.pid', 'r')
+               pid = int(pidfile.readline())
+               os.kill(pid, signal.SIGQUIT)
+       except Exception as e:
+               write_stdout('Could not kill supervisor: ' + e.strerror + '\n')
+       write_stdout('RESULT 2\nOK')
+
+
+if __name__ == '__main__':
+   main()
+   import sys

docker/kill_supervisor.py

@@ -0,0 +1,11 @@
+[program:rest-python]
+autostart=true
+redirect_stderr=true
+command=/bin/bash -lc "/opt/rest-python/pyrest.py --port=%(ENV_REST_PYTHON_PORT)s"
+startsecs=5
+priority=10
+
+[eventlistener:rest-python_exit]
+command=/usr/bin/kill_supervisor.py
+process_name=rest-python
+events=PROCESS_STATE_EXITED

docker/rest-python.conf

@@ -37,7 +37,7 @@
 import tornado.httpserver
 import tornado.web
 import tornado.websocket
-from tornado import ioloop, gen
+from tornado import ioloop, gen, concurrent
 
 from model.redhawk import Redhawk
 
@@ -74,6 +74,7 @@ class Application(tornado.web.Application):
     def __init__(self, *args, **kwargs):
         # explicit _ioloop for unit testing
         _ioloop = kwargs.get('_ioloop', None)
+        _ws_close_future = kwargs.get('close_future', concurrent.Future())
         cwd = os.path.abspath(os.path.dirname(__import__(__name__).__file__))
 
         # REDHAWK Service
@@ -118,7 +119,7 @@ def __init__(self, *args, **kwargs):
             (_APPLICATION_PATH + _ID + _PORT_PATH + _ID, PortHandler, 
                 dict(redhawk=redhawk, kind='application')),
             (_APPLICATION_PATH + _ID + _BULKIO_PATH, BulkIOWebsocketHandler, 
-                dict(redhawk=redhawk, kind='application', _ioloop=_ioloop)),
+                dict(redhawk=redhawk, kind='application', _ioloop=_ioloop, close_future=_ws_close_future)),
             (_APPLICATION_PATH + _ID + _PROPERTIES_PATH + _LIST, ApplicationProperties,
                 dict(redhawk=redhawk)),
             (_APPLICATION_PATH + _ID + _PROPERTIES_PATH + _ID, ApplicationProperties,
@@ -136,7 +137,7 @@ def __init__(self, *args, **kwargs):
             (_COMPONENT_PATH + _ID + _PORT_PATH + _ID, PortHandler,
                 dict(redhawk=redhawk, kind='component')),
             (_COMPONENT_PATH + _ID + _BULKIO_PATH, BulkIOWebsocketHandler,
-                dict(redhawk=redhawk, kind='component', _ioloop=_ioloop)),
+                dict(redhawk=redhawk, kind='component', _ioloop=_ioloop, close_future=_ws_close_future)),
 
             # Device Managers
             (_DEVICE_MGR_PATH + _LIST, DeviceManagers, dict(redhawk=redhawk)),
@@ -160,7 +161,8 @@ def __init__(self, *args, **kwargs):
             (_DEVICE_PATH + _ID + _PORT_PATH + _FEI_NAVDATA_ID + _LIST, FEINavDataHandler, dict(redhawk=redhawk)),
             (_DEVICE_PATH + _ID + _PORT_PATH + _FEI_NAVDATA_ID + _ID, FEINavDataHandler, dict(redhawk=redhawk)),  
             (_DEVICE_PATH + _ID + _PORT_PATH + _ID, PortHandler, dict(redhawk=redhawk, kind='device')), # Default port handler
-            (_DEVICE_PATH + _ID + _BULKIO_PATH, BulkIOWebsocketHandler, dict(redhawk=redhawk, kind='device', _ioloop=_ioloop)),
+            (_DEVICE_PATH + _ID + _BULKIO_PATH, BulkIOWebsocketHandler,
+                dict(redhawk=redhawk, kind='device', _ioloop=_ioloop, close_future=_ws_close_future)),
 
             # Filesystem
             (_FS_PATH, FileSystem, dict(redhawk=redhawk))

pyrest.py

@@ -26,8 +26,7 @@
 from omniORB import CORBA
 
 # third party imports
-from tornado import ioloop, gen
-from tornado import websocket
+from tornado import ioloop, gen, websocket
 
 import time
 import json
@@ -48,7 +47,8 @@ def enum(**enums):
 
 
 class BulkIOWebsocketHandler(CrossDomainSockets):
-    def initialize(self, kind, redhawk=None, _ioloop=None):
+    def initialize(self, close_future, kind, redhawk=None, _ioloop=None):
+        self.close_future = close_future
         self.kind = kind
         self.redhawk = redhawk
         if not _ioloop:
@@ -240,6 +240,7 @@ def on_message(self, message):
 
     def on_close(self):
         logging.debug('Stream CLOSE')
+        self.close_future.set_result((self.close_code, self.close_reason))
         try:
             self.port.ref.disconnectPort(self._connectionId)
             logging.info("Closed websocket to %s, %s", self.port, self._connectionId)