RedhawkSDR · Jan 14, 2015 · Jan 15, 2015 · Jan 15, 2015 · Jan 27, 2015 · Jan 28, 2015
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
-apps/*
+*.pyc
+apps/*/
 .virtualenv/
-*pyc
+bower_components
+*.*~
+local-runner-env
+.vscode/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,137 @@
+# This CI script builds Docker-REDHAWK's webserver image and deploys it to
+# dockerhub upon successful execution of its tests.
+variables:
+  CONTAINER_TEST_IMAGE:    $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
+  CONTAINER_TAGGED_IMAGE:  $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
+  DOCKERHUB_IMAGE:         geontech/redhawk-webserver
+  REDHAWK_VERSION:         2.0.8
+
+stages:
+  - build
+  - test
+  - release
+  - deploy
+  - cleanup
+
+.dind: &dind
+  image: docker:latest
+
+.container_registry: &container_registry
+  <<: *dind
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  after_script:
+    - docker logout $CI_REGISTRY
+
+# Build the test image and push it to the registry, delete the runner-local
+# image to avoid cluttering the runner.
+build:
+  stage: build
+  except:
+    - tags
+  <<: *container_registry
+  script:
+    - sed -i -E "s/(geontech.+\:)(.+)/\1"$REDHAWK_VERSION"/g" Dockerfile
+    - docker build
+      --pull
+      --tag $CONTAINER_TEST_IMAGE .
+    - docker push $CONTAINER_TEST_IMAGE
+    - docker rmi $CONTAINER_TEST_IMAGE
+
+test:
+  stage: test
+  <<: *container_registry
+  retry: 2
+  except:
+    - tags
+  script:
+    # Install docker-compose
+    - apk add --no-cache py2-pip && pip install docker-compose
+    # Pull the test image and start the stack without rebuilding it.
+    - cd tests
+    - docker pull ${CONTAINER_TEST_IMAGE}
+    - export REDHAWK_VERSION=$REDHAWK_VERSION
+    - export CONTAINER_TEST_IMAGE=$CONTAINER_TEST_IMAGE
+    - docker-compose -p ${CI_COMMIT_SHA} up -d --no-build
+    - docker-compose -p ${CI_COMMIT_SHA} images
+    - docker-compose -p ${CI_COMMIT_SHA} exec -T rest
+      bash -l -c 'yum install -y rh.SigGen rh.FileWriter'
+    - docker-compose -p ${CI_COMMIT_SHA} exec -T rest
+      bash -l -c './test.sh' || RESULT=$?
+
+    # Clean up and exit with the result
+    - docker-compose -p ${CI_COMMIT_SHA} down
+    - docker rmi ${CONTAINER_TEST_IMAGE}
+    - exit ${RESULT}
+
+# Move the test image to internal 'latest' and clean up the worker's images
+release:
+  stage: release
+  only:
+    - master
+  except:
+    - tags
+  <<: *container_registry
+  script:
+    - docker pull $CONTAINER_TEST_IMAGE
+    - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
+    - docker push $CONTAINER_RELEASE_IMAGE
+    - docker rmi $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
+
+# Deploy internally, tagged.
+deploy-image:
+  stage: deploy
+  only:
+    - tags
+  <<: *container_registry
+  script:
+    # Pull the release image, re-tag it to the dockerhub name and push it.
+    - docker pull $CONTAINER_RELEASE_IMAGE
+    - docker tag $CONTAINER_RELEASE_IMAGE $CONTAINER_TAGGED_IMAGE
+    - docker push $CONTAINER_TAGGED_IMAGE
+    - docker rmi $CONTAINER_RELEASE_IMAGE $CONTAINER_TAGGED_IMAGE
+
+# Push to GitHub
+deploy-github:
+  stage: deploy
+  only:
+    - tags
+  image: alpine
+  script:
+    - apk add --no-cache openssh-client git bash
+    - eval $(ssh-agent -s)
+    - bash -c 'ssh-add <(echo "$GITHUB_TOKEN")'
+    - mkdir -p ~/.ssh
+    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+    # Configure git and push w/ tag(s)
+    - git config user.name "${GITHUB_USER_NAME}"
+    - git config user.email "${GITHUB_USER_EMAIL}"
+    - git remote set-url origin ${GITHUB_REPO}
+    - git push origin ${GITHUB_BRANCH} --follow-tags
+
+# Clean dev/test images not on master branch
+clean-dev:
+  stage: cleanup
+  <<: *container_registry
+  when: always
+  except:
+    - master
+    - tags
+  script:
+    - apk add --no-cache git bash curl
+    - git clone ${DOCKER_UTILS} docker-util
+    - docker-util/delete-image.sh $CONTAINER_TEST_IMAGE
+
+# Cleanup the locally stored image after tagging and pushing to public
+# This has to be manual since deploying to Dockerhub is manual too.
+clean-internal:
+  stage: cleanup
+  <<: *container_registry
+  when: manual
+  only:
+    - tags
+  script:
+    - apk add --no-cache git bash curl
+    - git clone ${DOCKER_UTILS} docker-util
+    - docker-util/delete-image.sh $CONTAINER_RELEASE_IMAGE
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,60 @@
+# This file is protected by Copyright. Please refer to the COPYRIGHT file
+# distributed with this source distribution.
+#
+# This file is part of Geon's REST-Python.
+#
+# REST-Python is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# REST-Python is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see http://www.gnu.org/licenses/.
+#
+
+FROM geontech/redhawk-runtime:2.0.8
+LABEL name="REST-Python Web Server" \
+    description="Geon's Fork of REST-Python" \
+    maintainer="Thomas Goodwin <btgoodwin@geontech.com>"
+
+# Build-time configurable variables
+ARG REST_PYTHON_PORT=8080
+
+# Runtime variables
+ENV REST_PYTHON_PORT=${REST_PYTHON_PORT}
+
+# Expose the configured default port.
+EXPOSE ${REST_PYTHON_PORT}
+
+RUN yum install -y \
+    git \
+    gcc \
+    python-dev \
+    curl \
+    python-virtualenv && \
+    yum clean all -y
+
+# Install and update pip
+RUN curl https://bootstrap.pypa.io/get-pip.py | python && \
+    pip install -U pip
+
+# Install the rest-python server requirements
+COPY ./setup.sh ./pyvenv ./requirements.txt /opt/rest-python/
+WORKDIR /opt/rest-python
+RUN ./setup.sh install && pip install -r requirements.txt
+
+# Install the rest of the rest-python server and launcher
+COPY . /opt/rest-python
+RUN cp docker/rest-python.conf /etc/supervisor.d/rest-python.conf && \
+    cp docker/kill_supervisor.py /usr/bin/kill_supervisor.py && \
+    chmod u+x /usr/bin/kill_supervisor.py
+
+# Mount point for end-user apps
+VOLUME /opt/rest-python/apps
+
+CMD ["/usr/bin/supervisord"]
diff --git a/README.md b/README.md
@@ -2,7 +2,9 @@
 
 ## Description
 
-Contains the REDHAWK python implementation of the generic REST API
+Contains the REDHAWK python implementation of the generic [REST API](http://geontech.github.io/rest-python).
+
+Please visit [the Wiki](https://github.com/Geontech/rest-python/wiki) for more detailed instructions on the changes made to this repository and how it can be used for development of external control interfaces, data bridges, and graphical user interfaces without having to install REDHAWK on the target system.
 
 ## REDHAWK Documentation
 
@@ -18,7 +20,7 @@ REDHAWK REST Python is licensed under the GNU Lesser General Public License (LGP
 
 ## Running
 
-For Development/Test environments there are scripts to automatically create a local environment and run the server.
+For Development/Test environments there are scripts to automatically create a local environment and run the server. You will need to install the `virtualenv` python package before using the commands below.
 
     ./setup.sh install
     ./start.sh --port=<desired_port>
@@ -33,3 +35,101 @@ For a more permanent solution, consult the `requirements.txt` and run the follow
 can be found at `deploy/rest-python-supervisor.conf`.
 
 Once running the REST Interface can be tested at `http://localhost:<desired_port>/rh/rest/domains`.
+
+## Deploying Applications
+
+You can either install your application in `apps` for REST-Python to serve them, or deploy them with a separate server (e.g., NodeJS).  REST-Python supports cross-domain responses to REST and Websocket requests to facilitate dual- or multi-server configurations to completely decouple the REDHAWK environment from the web application environment.  (See [Docker-REDHAWK's](http://github.com/GeonTech/docker-redhawk) `geontech/redhawk-webserver` image.)
+
+## Testing
+
+If you would like to test REST-Python changes in a functional domain, download Docker-CE and Docker Compose for your operating system.  The process takes 3 parts:
+
+ 1. Your locally-built instance of the webserver image
+ 2. Running the `tests/docker-compose.yml` stack with your webserver image
+ 3. Using `docker-compose exec` to start the tests script.
+
+ > Note: Verify the Dockerfile version of the parent image (`FROM...`) and the compose file vs. the version of REDHAWK you want to test against.  You can export `REDHAWK_VERSION` set to that version prior to running `docker-compose` to simplify changing versions of the infrastructure.
+
+The process is shown here:
+
+```
+cd tests
+export TEST_IMAGE=test-webserver
+docker-compose up -d --build
+docker-compose exec -T rest \
+    bash -l -c 'yum install -y rh.FileWriter rh.SigGen && ./test.sh'
+```
+
+If you make further changes and want to update the webserver image and container (the service is `rest` in the compose file), you can recreate the container without tearing down the entire compose by doing the following:
+
+```
+docker-compose stop rest
+docker-compose up --build -d rest
+```
+
+You would then repeat the `exec` command from above.
+
+### Using GitLab Runner Locally
+
+You can run individual jobs defined in the `.gitlab-ci.yml` file.  To do this, you will need to install the `gitlab-runner` for your OS and, preferably, Docker-CE.  You can then use the Docker executor to run the jobs.
+
+Any environment variables that would normally be set by GitLab's CI/CD process must be passed into the build environment.  Please read the job in question to verify you have all possible keys/secrets already configured and take special care to **NOT COMMIT SECRETS** to the repository.  For convenience, the file name `local-runner-env` has been added to the `.gitignore`.  Feel free to use that file name for a script like this, which stores the variables as strings in an array and then prints each as a `--env VAR=val` argument:
+
+```
+# local-runner-env
+ENV_VARS=(
+SPECIAL_USER=somebody
+SPECIAL_PASSWORD=password
+)
+printf " --env %s" "${ENV_VARS[@]}"
+```
+
+And then if we had a `printenv` job, we could execute it like this:
+
+```
+[~/project]$ gitlab-runner exec docker $(. ./local-runner-env) printenv
+
+Running with gitlab-runner 11.0.0 (5396d320)
+Using Docker executor with image alpine:latest ...
+Pulling docker image alpine:latest ...
+Using docker image sha256:3fd9065eaf02feaf94d68376da52541925650b81698c53c6824d92ff63f98353 for alpine:latest ...
+Running on runner--project-0-concurrent-0 via some.host.com...
+Cloning repository...
+Cloning into '/builds/project-0'...
+done.
+Checking out 355f773c as master...
+Skipping Git submodules setup
+$ printenv
+CI_SERVER_REVISION=
+CI=true
+CI_RUNNER_REVISION=5396d320
+HOSTNAME=runner--project-0-concurrent-0
+CI_JOB_STAGE=build
+CI_SERVER_VERSION=
+SHLVL=2
+OLDPWD=/
+SPECIAL_USER=somebody
+SPECIAL_PASSWORD=password
+HOME=/root
+CI_JOB_ID=1
+CI_COMMIT_REF_NAME=master
+CI_RUNNER_VERSION=11.0.0
+CI_PROJECT_ID=0
+GITLAB_CI=true
+CI_COMMIT_SHA=355f773c471add104ab17f984ce5016001813b62
+CI_PROJECT_DIR=/builds/project-0
+CI_SERVER_NAME=GitLab CI
+CI_JOB_TOKEN=
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+CI_SERVER=yes
+CI_REPOSITORY_URL=/home/user/project
+CI_RUNNER_EXECUTABLE_ARCH=linux/amd64
+CI_DISPOSABLE_ENVIRONMENT=true
+PWD=/builds/project-0
+```
+
+ > **Important 1:** THe CI feature that obfuscates secrets within the build logs _does not exist_ when running locally.  Any exposure of those variables to the logging environment when running locally will appear in plain text!
+
+ > **Important 2:** The above method will not work for multi-line variables like SSH keys.
+
+ > Note: Running a job locally has the drawback that your changes must be committed (pushing is not required).  You may find it helpful to `git commit --amend` as you work through changes and then `git reset HEAD^` to begin splitting the commit into real changes that you plan to push (`git add --patch`, for example).
diff --git a/apps/INSTALL b/apps/INSTALL
diff --git a/docker/kill_supervisor.py b/docker/kill_supervisor.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# Special thanks to the example at:
+# https://blog.zhaw.ch/icclab/process-management-in-docker-containers/
+import sys
+import os
+import signal
+
+
+def write_stdout(s):
+   sys.stdout.write(s)
+   sys.stdout.flush()
+
+
+def write_stderr(s):
+   sys.stderr.write(s)
+   sys.stderr.flush()
+
+
+def main():
+   while 1:
+       write_stdout('READY\n')
+       line = sys.stdin.readline()
+       write_stdout('This line kills supervisor: ' + line)
+       try:
+               pidfile = open('/var/run/supervisord.pid', 'r')
+               pid = int(pidfile.readline())
+               os.kill(pid, signal.SIGQUIT)
+       except Exception as e:
+               write_stdout('Could not kill supervisor: ' + e.strerror + '\n')
+       write_stdout('RESULT 2\nOK')
+
+
+if __name__ == '__main__':
+   main()
+   import sys
diff --git a/docker/rest-python.conf b/docker/rest-python.conf
@@ -0,0 +1,11 @@
+[program:rest-python]
+autostart=true
+redirect_stderr=true
+command=/bin/bash -lc "/opt/rest-python/pyrest.py --port=%(ENV_REST_PYTHON_PORT)s"
+startsecs=5
+priority=10
+
+[eventlistener:rest-python_exit]
+command=/usr/bin/kill_supervisor.py
+process_name=rest-python
+events=PROCESS_STATE_EXITED