From e7a28afe44c25d8a9846910ec4456bc16d787445 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Fri, 30 Sep 2016 09:52:23 -0400 Subject: [PATCH 01/48] update: coveralls --- Dockerfile | 59 +++++++++--------------------------------------------- README.rst | 6 +++++- 2 files changed, 14 insertions(+), 51 deletions(-) diff --git a/Dockerfile b/Dockerfile index 049058ff..11d3f132 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,8 +145,6 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config \ && pip3 install virtualenv virtualenvwrapper ipython numpy tox coveralls -# RUN pip3 install virtualenv virtualenvwrapper ipython numpy tox coveralls - ENV PYTHON_VERSION_MAJOR '3' ENV GSTREAMER '1.0' ENV USER 'pi' @@ -175,21 +173,24 @@ ENV PIP_DOWNLOAD_CACHE '${USER_HOME}/.pip/cache' ENV WORKON_HOME '${VIRT_ROOT}' RUN set -xe \ - && useradd -U -d ${PI_HOME} -m -r -G adm,sudo,dip,plugdev,tty,audio ${USER} \ - && usermod -a -G ${USER} ${USER} \ && mkdir -p ${MAIN_DIR} \ - && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ + && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github + +RUN set -xe \ && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ && chown -hR ${USER}:${USER} ${MAIN_DIR} \ + +RUN set -xe \ + && useradd -U -d ${PI_HOME} -m -r -G adm,sudo,dip,plugdev,tty,audio ${USER} \ + && usermod -a -G ${USER} ${USER} \ && echo '${USER} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ - && echo '%${USER} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ - && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages + && echo '%${USER} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers + # && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages COPY ./dotfiles/.bashrc /home/pi/.bashrc COPY ./dotfiles/.profile /home/pi/.profile RUN set -xe \ - && chown -hR ${USER}:${USER} ${MAIN_DIR} \ && apt-get autoclean -y \ && apt-get autoremove -y \ && rm -rf /tmp/* /var/tmp/* \ @@ -198,60 +199,18 @@ RUN set -xe \ # Layer customizations over existing structure COPY ./container/root / -# USER ${USER} - # Ensure application code makes it into the /app directory COPY ./ /home/pi/dev/bossjones-github/scarlett_os/ RUN ls -lta /home/pi/dev/bossjones-github/scarlett_os/ && \ bash -c "source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv --system-site-packages ${GITHUB_REPO_NAME}" -# RUN bash -c "source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv --system-site-packages ${GITHUB_REPO_NAME}" - COPY ./postactivate /home/pi/.virtualenvs/scarlett_os/postactivate -# RUN bash -c "/usr/sbin/sudo chown -hR ${USER}:${USER} /home/pi/.virtualenvs/scarlett_os/postactivate" - -# RUN bash -c "source /usr/local/bin/virtualenvwrapper.sh && workon ${GITHUB_REPO_NAME} && pip install -r /home/pi/home/pi/dev/bossjones-github/scarlett_os/requirements.txt" - RUN set -xe \ && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ && chown -hR ${USER}:${USER} ${MAIN_DIR} \ && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages \ && chown -R ${USER}:${USER} /home/pi/.virtualenvs -# -# RUN sudo su - ${USER} -c "source /usr/local/bin/virtualenvwrapper.sh && workon ${GITHUB_REPO_NAME} && pip install -r /home/pi/home/pi/dev/bossjones-github/scarlett_os/requirements_dev.txt" - -# # virtualenv python runtime -# RUN sudo su - rdash -c "echo 'export WORKON_HOME=${WORKON_HOME}' >> /home/rdash/.profile" -# RUN sudo su - rdash -c "echo 'export PROJECT_HOME=${PROJECT_HOME}' >> /home/rdash/.profile" -# RUN sudo su - rdash -c "echo 'source /usr/local/bin/virtualenvwrapper.sh' >> /home/rdash/.profile" -# RUN sudo su - rdash -c "mkvirtualenv rdash" -# # RUN sudo su - rdash -c "mkproject rdash" -# RUN sudo su - rdash -c "echo 'workon rdash' >> /home/rdash/.profile" -# -# # Add source code -# -# #ADD . ${PROJECT_HOME} -# RUN git clone https://github.com/adysuciu/rdash-django.git ${PROJECT_HOME} -# -# # Environment info -# ADD rdash/.env ${PROJECT_HOME}/rdash/.env -# -# RUN chown -R rdash:users ${PROJECT_HOME} -# RUN chmod 755 ${PROJECT_HOME}/manage.py -# -# # Install django and dependencies -# RUN sudo su - rdash -c "${WORKON_HOME}/rdash/bin/pip install -r ${PROJECT_HOME}/requirements.txt" -# RUN sudo su - rdash -c 'echo {\"registry\":\"http://bower.herokuapp.com\"} > /home/rdash/.bowerrc' -# RUN sudo su - rdash -c "python ${PROJECT_HOME}/manage.py bower_install" -# RUN sudo su - rdash -c "python ${PROJECT_HOME}/manage.py collectstatic --noinput" -# #RUN sudo su - rdash -c "python ${PROJECT_HOME}/manage.py compilemessages" -# RUN sudo su - rdash -c "python ${PROJECT_HOME}/manage.py makemigrations" -# RUN sudo su - rdash -c "python ${PROJECT_HOME}/manage.py migrate" -# -# ENTRYPOINT ["sudo","su","-","rdash"] -# CMD ["/bin/bash", "/run.sh"] -# ENTRYPOINT ["ls", "-lta", "/home/pi/dev/bossjones-github/scarlett_os/"] ENTRYPOINT ["file","/bin/bash"] diff --git a/README.rst b/README.rst index 068057fe..83c9fa66 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,11 @@ ScarlettOS :target: https://pyup.io/repos/github/bossjones/scarlett_os/ :alt: Updates +.. image:: https://coveralls.io/repos/github/bossjones/scarlett_os/badge.svg?branch=master + :target: https://coveralls.io/github/bossjones/scarlett_os?branch=master + :alt: Coverage + + S.C.A.R.L.E.T.T is Tony Darks artificially programmed intelligent computer. She is programmed to speak with a female voice in a British accent. @@ -37,4 +42,3 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack .. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage - From b4d9b8e736bcdf251dc491c2f70776fb5ac9c634 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 12:21:39 -0400 Subject: [PATCH 02/48] Chg: adding coveralls and logger.py --- .coveralls.yml | 2 ++ .travis.yml | 56 +++++++++++++++++++++---------------------- scarlett_os/logger.py | 37 ++++++++++++++++++++++++++++ setup.py | 3 ++- 4 files changed, 69 insertions(+), 29 deletions(-) create mode 100644 .coveralls.yml create mode 100644 scarlett_os/logger.py diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 00000000..85f68247 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1,2 @@ +repo_token: sV16DIhoxZ1MQ5REfJYJlzpPSLfYu5HBG +service-name: travis-ci diff --git a/.travis.yml b/.travis.yml index d90ab58f..80476348 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,51 +52,51 @@ before_install: - if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:gnome3-team/gnome3-staging; fi - sudo add-apt-repository -y ppa:pitti/systemd-semaphore -- sudo apt-get update -qq -- sudo apt-get update -qq -- sudo apt-get install -y libz-dev libbz2-dev gstreamer$GSTREAMER-tools libgstreamer$GSTREAMER-dev +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get install -y libz-dev libbz2-dev gstreamer$GSTREAMER-tools libgstreamer$GSTREAMER-dev libgstreamer-plugins-base$GSTREAMER-dev libgstreamer-plugins-bad$GSTREAMER-dev -- if [ '$GSTREAMER' = '1.0' ]; then sudo apt-get install -y libgstreamer-plugins-good$GSTREAMER-dev; +- if [ '$GSTREAMER' = '1.0' ]; then travis_retry sudo apt-get install -y libgstreamer-plugins-good$GSTREAMER-dev; fi -- sudo apt-get update -qq -- sudo apt-get install -y gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 graphviz-dev +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get install -y gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 graphviz-dev gstreamer1.0-plugins-good gstreamer1.0-plugins-bad python-gst-1.0 -- sudo apt-get install -qq python3-gi -- sudo apt-get install -qq python-gst-1.0 gstreamer1.0-plugins-base gstreamer1.0-plugins-bad +- travis_retry sudo apt-get install -qq python3-gi +- travis_retry sudo apt-get install -qq python-gst-1.0 gstreamer1.0-plugins-base gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-libav libsndfile1-dev libasound2-dev libgstreamer-plugins-base1.0-dev python-numpy python-scipy -- sudo apt-get -y install automake gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 +- travis_retry sudo apt-get -y install automake gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-tools libasound2-dev libgstreamer-plugins-base1.0-dev libsndfile1-dev python python-dev python-gi python-gst-1.0 python-gst-1.0 python-imaging python-lxml python-numpy python-scipy python-virtualenv python3-gi -- sudo apt-get -y install libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev +- travis_retry sudo apt-get -y install libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev gnome-common; -- sudo apt-get -y install libgstreamer1.0-dev gstreamer1.0-alsa gstreamer1.0-plugins-base; -- sudo apt-get -y install python-dev libsmpeg-dev libswscale-dev libavformat-dev libavcodec-dev +- travis_retry sudo apt-get -y install libgstreamer1.0-dev gstreamer1.0-alsa gstreamer1.0-plugins-base; +- travis_retry sudo apt-get -y install python-dev libsmpeg-dev libswscale-dev libavformat-dev libavcodec-dev libjpeg-dev libtiff4-dev libX11-dev libmtdev-dev; -- sudo apt-get -y install python-setuptools build-essential libgl1-mesa-dev libgles2-mesa-dev; -- sudo apt-get -y install xvfb pulseaudio; +- travis_retry sudo apt-get -y install python-setuptools build-essential libgl1-mesa-dev libgles2-mesa-dev; +- travis_retry sudo apt-get -y install xvfb pulseaudio; - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted universe multiverse'; fi - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe multiverse'; fi -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then travis_retry sudo apt-get update -qq; travis_retry sudo apt-get install -qq pkg-config; fi -- sudo apt-get -y update -- sudo apt-get -y install ubuntu-restricted-extras -- sudo apt-get -y install libfftw3-dev -- sudo apt-get install -qq python3-dev python3-cairo-dev python3-gi gnome-common gtk-doc-tools +- travis_retry sudo apt-get -y update +- travis_retry sudo apt-get -y install ubuntu-restricted-extras +- travis_retry sudo apt-get -y install libfftw3-dev +- travis_retry sudo apt-get install -qq python3-dev python3-cairo-dev python3-gi gnome-common gtk-doc-tools libgtk-3-dev libgirepository1.0-dev libmount-dev cvs; -- sudo apt-get update -q -- sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) -- sudo apt-get install -qq build-essential git -- sudo apt-get install -qq gettext xsltproc docbook-xml docbook-xsl -- sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 +- travis_retry sudo apt-get update -q +- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) +- travis_retry sudo apt-get install -qq build-essential git +- travis_retry sudo apt-get install -qq gettext xsltproc docbook-xml docbook-xsl +- travis_retry sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 byacc flex apt-file autopoint -- sudo apt-get install -qq apt-file +- travis_retry sudo apt-get install -qq apt-file - sudo apt-file update -- sudo apt-get install wget -qq +- travis_retry sudo apt-get install wget -qq - pip install virtualenv virtualenvwrapper - sudo pip install --upgrade virtualenvwrapper - export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh` @@ -120,8 +120,8 @@ before_install: - jhbuild run python3 setup.py install install: -- sudo apt-get update -q -- sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) +- travis_retry sudo apt-get update -q +- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: jhbuild run python3 setup.py test diff --git a/scarlett_os/logger.py b/scarlett_os/logger.py new file mode 100644 index 00000000..7ea27949 --- /dev/null +++ b/scarlett_os/logger.py @@ -0,0 +1,37 @@ +import logging +logger = logging.getLogger('scarlettlogger') + +try: + from colorlog import ColoredFormatter + from gettext import gettext as _ + """Return a logger with a default ColoredFormatter.""" + formatter = ColoredFormatter( + "(%(threadName)-9s) %(log_color)s%(levelname)-8s%(reset)s (%(funcName)-5s) %(message_log_color)s%(message)s", + datefmt=None, + reset=True, + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red', + }, + secondary_log_colors={ + 'message': { + 'ERROR': 'red', + 'CRITICAL': 'red', + 'DEBUG': 'yellow', + 'INFO': 'yellow,bg_blue' + } + }, + style='%' + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) +except ImportError: + # No color available, use default config + logging.basicConfig(format='%(levelname)s: %(message)s') + logger.warn("Disabling color, you really want to install colorlog.") diff --git a/setup.py b/setup.py index 29263e43..419777f6 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,8 @@ ] test_requirements = [ - 'pytest' + 'pytest', + 'coverage' ] From c95d5c6005378bae3b0fbca6ff0e06161e380231 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 13:27:25 -0400 Subject: [PATCH 03/48] chg: update Makefile with new coverage steps --- .coveragerc | 3 +++ .travis.yml | 1 + Makefile | 8 +++++++- tests/test_scarlett_os.py | 9 ++++++--- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 04204bd1..f1f41ba5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,8 @@ omit = */python?.?/* */site-packages/nose/* */test/* + scarlett_os/__main__.py + scarlett_os/scripts/*.py [report] # Regexes for lines to exclude from consideration @@ -22,4 +24,5 @@ exclude_lines = # From old Scarlett repo assert False raise NotImplementedError + raise AssertionError if __name__ == .__main__.: diff --git a/.travis.yml b/.travis.yml index 80476348..2ad7f1ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ cache: directories: - "$HOME/.cache/pip" - "$HOME/virtualenv/python3.5.2/lib/python3.5/site-packages" + - "$HOME/.ccache" before_install: - export SKIP_ON_TRAVIS=yes diff --git a/Makefile b/Makefile index 83a92ef3..e8531417 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ test: ## run tests quickly with the default Python python setup.py test test-all: ## run tests on every Python version with tox - tox + python setup.py test coverage: ## check code coverage quickly with the default Python @@ -68,6 +68,12 @@ coverage: ## check code coverage quickly with the default Python coverage html $(BROWSER) htmlcov/index.html +coverage-no-html: ## check code coverage quickly with the default Python + + coverage run --source scarlett_os setup.py test + + coverage report -m + docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/scarlett_os.rst rm -f docs/modules.rst diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 49dbd911..6bf68490 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -18,7 +18,8 @@ from scarlett_os import scarlett_os from scarlett_os import cli - +import lsb_release +ubuntu_version = lsb_release.get_lsb_information() class TestScarlett_os(unittest.TestCase): @@ -39,8 +40,10 @@ def test_gstreamer_versions(self): Gst.debug_set_active(True) Gst.debug_set_default_threshold(1) - assert GObject.pygobject_version == (3, 22, 0) - # assert Gst.version_string() == 'GStreamer 1.8.2' + if ubuntu_version['CODENAME'] is 'trusty': + assert GObject.pygobject_version == (3, 22, 0) + else: + assert GObject.pygobject_version == (3, 20, 0) assert Gst.version_string() == 'GStreamer 1.8.2' def test_command_line_interface(self): From 35dff708ded86372b3524c03478bfc99e015fe09 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 15:03:48 -0400 Subject: [PATCH 04/48] refactor: small refactoring --- ci/travis.sh | 10 ++++++---- scarlett_os/__init__.py | 3 +++ scarlett_os/tools/verify.py | 8 ++++++++ tests/test_scarlett_os.py | 6 +++--- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 scarlett_os/tools/verify.py diff --git a/ci/travis.sh b/ci/travis.sh index f7905f5f..e55f1284 100755 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -102,8 +102,7 @@ sudo apt-get install yasm nasm -qq # Get JHBuild # source: https://github.com/GNOME/jhbuild/commit/86d958b6778da649b559815c0a0dbe6a5d1a8cd4 (cd "${JHBUILD}" && - git clone https://github.com/GNOME/jhbuild.git && - cd jhbuild && + if test ! -d "${JHBUILD}/jhbuild"; then git clone https://github.com/GNOME/jhbuild.git && cd jhbuild; else echo "exists" && cd jhbuild; fi && git checkout 86d958b6778da649b559815c0a0dbe6a5d1a8cd4 && ./autogen.sh --prefix=/usr/local > /dev/null && make > /dev/null && @@ -133,7 +132,8 @@ fi # Need LGI from git master if [ "x${ENABLE_LUA51}" == "xyes" ]; then (cd "${JHBUILD}" && - git clone https://github.com/pavouk/lgi.git && cd lgi && + git clone https://github.com/pavouk/lgi.git && + cd lgi && jhbuild run make PREFIX="${PREFIX}" CFLAGS="${CFLAGS} -I/usr/include/lua5.1" && jhbuild run make install PREFIX="${PREFIX}"); @@ -155,7 +155,9 @@ fi # Need PyGObject built for Python 3 if [ "x${ENABLE_PYTHON3}" == "xyes" ]; then - (cd "${JHBUILD}" && cd pygobject && git checkout fb1b8fa8a67f2c7ea7ad4b53076496a8f2b4afdb && + (cd "${JHBUILD}" && + cd pygobject && + git checkout fb1b8fa8a67f2c7ea7ad4b53076496a8f2b4afdb && jhbuild run ./autogen.sh --prefix="${PREFIX}" --with-python=python3 > /dev/null && jhbuild run make install > /dev/null); fi diff --git a/scarlett_os/__init__.py b/scarlett_os/__init__.py index 2f4a418e..16e8666c 100644 --- a/scarlett_os/__init__.py +++ b/scarlett_os/__init__.py @@ -17,6 +17,9 @@ # source: timeside from .tools import package as ts_package +from .tools import verify as ts_verify + +ts_verify.check_python3_installed() # Check Availability of Gstreamer python bindings ts_package.check_gi() diff --git a/scarlett_os/tools/verify.py b/scarlett_os/tools/verify.py new file mode 100644 index 00000000..9e0ee47c --- /dev/null +++ b/scarlett_os/tools/verify.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + + +def check_python3_installed(): + import sys + if sys.version_info < (3, 4) <= sys.version_info < (3, 5): + print('ScarlettOS requires at least Python 3.4 or 3.3 to run.') + sys.exit(1) diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 6bf68490..aa49d938 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -18,8 +18,8 @@ from scarlett_os import scarlett_os from scarlett_os import cli -import lsb_release -ubuntu_version = lsb_release.get_lsb_information() +import platform +ubuntu_version = platform.platform().split('-') class TestScarlett_os(unittest.TestCase): @@ -40,7 +40,7 @@ def test_gstreamer_versions(self): Gst.debug_set_active(True) Gst.debug_set_default_threshold(1) - if ubuntu_version['CODENAME'] is 'trusty': + if ubuntu_version[8] is 'trusty': assert GObject.pygobject_version == (3, 22, 0) else: assert GObject.pygobject_version == (3, 20, 0) From 08e4b31226d1405f8332091094d096389bee9f1b Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 17:37:45 -0400 Subject: [PATCH 05/48] tests: small fix --- requirements.txt | 1 + scarlett_os/__init__.py | 51 +++++++++++++++++++------ scarlett_os/__main__.py | 13 +++++++ scarlett_os/cli.py | 79 ++++++++++++++++++++++++++++++++++++++- scarlett_os/main.py | 16 ++++++++ tests/test_scarlett_os.py | 26 ++++++++++++- 6 files changed, 171 insertions(+), 15 deletions(-) create mode 100644 scarlett_os/__main__.py create mode 100644 scarlett_os/main.py diff --git a/requirements.txt b/requirements.txt index 6c99a389..23e9d564 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ pydbus==0.5.1 colorlog click==6.6 +psutil==4.3.0 diff --git a/scarlett_os/__init__.py b/scarlett_os/__init__.py index 16e8666c..445dfd28 100644 --- a/scarlett_os/__init__.py +++ b/scarlett_os/__init__.py @@ -1,24 +1,14 @@ # -*- coding: utf-8 -*- - from __future__ import absolute_import - -# from . import api -# from . import processor -# -# from .processor import Processor, get_processor, list_processors -# from .component import implements, interfacedoc, abstract +from .tools import package as ts_package +from .tools import verify as ts_verify __author__ = 'Malcolm Jones' __email__ = 'bossjones@theblacktonystark.com' __version__ = '0.1.0' - # source: timeside - -from .tools import package as ts_package -from .tools import verify as ts_verify - ts_verify.check_python3_installed() # Check Availability of Gstreamer python bindings @@ -36,3 +26,40 @@ # Clean-up del ts_package del absolute_import + +# Import psutil +try: + from psutil import __version__ as psutil_version +except ImportError: + print('PSutil library not found. ScarlettOS cannot start.') + sys.exit(1) + + +def main(): + """Main entry point for ScarlettOS. + + Select the mode (standalone, client or server) + Run it... + """ + + # TODO: Grabe values from cli, setup correct logging. + # TODO: For now we'll default to scarlett_os.logger + + import platform + import scarlett_os.logger + import logging + logger = logging.getLogger('scarlettlogger') + + # Log ScarlettOS and PSutil version + logger.info('Start ScarlettOS {}'.format(__version__)) + logger.info('{} {} and PSutil {} detected'.format( + platform.python_implementation(), + platform.python_version(), + psutil_version)) + + # Share global var + global core, dbus_server, listener, tasker, check_all_services + + # Create the ScarlettOS main instance + # source: Glances + # core = ScarlettOSMain() diff --git a/scarlett_os/__main__.py b/scarlett_os/__main__.py new file mode 100644 index 00000000..5b5024c1 --- /dev/null +++ b/scarlett_os/__main__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +"""Allow user to run ScarlettOS as a module.""" + +# Execute with: +# $ python -m scarlett_os + +import scarlett_os + +if __name__ == '__main__': + scarlett_os.main() diff --git a/scarlett_os/cli.py b/scarlett_os/cli.py index 17f1930b..aea46b0d 100644 --- a/scarlett_os/cli.py +++ b/scarlett_os/cli.py @@ -1,12 +1,87 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function +# import logging import click +from . import __version__ + +# source: Doc2Dash +# class ClickEchoHandler(logging.Handler): +# """ +# Use click.echo() for logging. Has the advantage of stripping color codes +# if output is redirected. Also is generally more predictable. +# """ +# _level_to_fg = { +# logging.ERROR: "red", +# logging.WARN: "yellow", +# } +# +# def emit(self, record): +# click.echo(click.style( +# record.getMessage(), +# fg=self._level_to_fg.get(record.levelno, "reset") +# ), err=record.levelno >= logging.WARN) + @click.command() -def main(args=None): - """Console script for scarlett_os""" +@click.option( + "--name", + "-n", + help="Name ScarlettOS process explicitly.", + metavar="NAME", + default="scarlett_system" +) +@click.option( + "--daemon", + "-d", + is_flag=True, + help="Daemon mode, background process.", + default=False +) +@click.option( + "--slave", + "-s", + is_flag=True, + help="Run ScarlettOS process as a Slave", + default=False +) +@click.option( + "--master", + "-m", + is_flag=True, + help="Run ScarlettOS process as a Master", + default=False +) +@click.option( + '--type', + type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), + help="ScarlettOS type", + default='check_all_services' +) +@click.option( + '--etcd-host', + help="Etcd Host for distributed mode.", + default=False +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Limit output to errors and warnings.", + default=False +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Be verbose.", + default=False +) +@click.version_option(version=__version__) +def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): + """Console script for scarlett_os.""" click.echo("Replace this message by putting your code into " "scarlett_os.cli.main") click.echo("See click documentation at http://click.pocoo.org/") diff --git a/scarlett_os/main.py b/scarlett_os/main.py new file mode 100644 index 00000000..0d6db3c3 --- /dev/null +++ b/scarlett_os/main.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +"""ScarlettOS main class.""" + +# from scarlett_os import package as ts_package + +# import argparse +# import os +# import sys +# import tempfile +# +# from glances import __appname__, __version__, psutil_version +# from glances.compat import input +# from glances.config import Config +# from glances.globals import LINUX, WINDOWS +# from glances.logger import logger diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index aa49d938..9a86817b 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -11,6 +11,7 @@ import sys import unittest +import pytest import click from contextlib import contextmanager from click.testing import CliRunner @@ -21,6 +22,28 @@ import platform ubuntu_version = platform.platform().split('-') + +# from IPython.core.debugger import Tracer # NOQA +# from IPython.core import ultratb +# import traceback +# +# import logging +# logger = logging.getLogger('scarlettlogger') +# # from pydbus import SessionBus +# # from pydbus.green import sleep +# +# sys.excepthook = ultratb.FormattedTB(mode='Verbose', +# color_scheme='Linux', +# call_pdb=True, +# ostream=sys.__stdout__) + +# @pytest.fixture +# def runner(): +# """ +# Click's test helper. +# """ +# return CliRunner() + class TestScarlett_os(unittest.TestCase): def setUp(self): @@ -53,7 +76,8 @@ def test_command_line_interface(self): assert 'scarlett_os.cli.main' in result.output help_result = runner.invoke(cli.main, ['--help']) assert help_result.exit_code == 0 - assert '--help Show this message and exit.' in help_result.output + print(help_result.output) + assert 'dbus_server|listener|tasker|check_all_services' in help_result.output if __name__ == '__main__': From 9dbb1d23975555643d3812823f3b1d757f796d65 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 18:20:59 -0400 Subject: [PATCH 06/48] chg: requirements.txt --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2ad7f1ee..17ca66fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -118,6 +118,8 @@ before_install: - workon scarlett_os - pip install -U coveralls sphinx numpy ipython - bash ./ci/travis.sh +- jhbuild run pip install -r requirements.txt +- jhbuild run pip install -r requirements_dev.txt - jhbuild run python3 setup.py install install: From 2fce59f99333ddd23ec87cb07b1b7412dbd4a267 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 18:58:50 -0400 Subject: [PATCH 07/48] chg: setup.py install commands --- .travis.yml | 1 + setup.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17ca66fe..0e82c2d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,6 +95,7 @@ before_install: - travis_retry sudo apt-get install -qq gettext xsltproc docbook-xml docbook-xsl - travis_retry sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 byacc flex apt-file autopoint +- travis_retry sudo apt-get install gcc automake autoconf libtool bison swig python-dev libpulse-dev -y - travis_retry sudo apt-get install -qq apt-file - sudo apt-file update - travis_retry sudo apt-get install wget -qq diff --git a/setup.py b/setup.py index 419777f6..b3ef4a15 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,9 @@ import platform import re import warnings +import pprint +pp = pprint.PrettyPrinter(indent=4) + # Don't force people to install setuptools unless # we have to. @@ -26,15 +29,42 @@ with open('HISTORY.rst') as history_file: history = history_file.read() -requirements = [ - 'Click>=6.0', - # TODO: put package requirements here -] +static = {} + +for root, dirs, files in os.walk('static'): + for filename in files: + filepath = os.path.join(root, filename) + + if root not in static: + static[root] = [] + + static[root].append(filepath) + +# Might use this later +try: + here = os.path.abspath(os.path.dirname(__file__)) +except: + pass + + +def read_requirements(filename): + content = open(os.path.join(here, filename)).read() + requirements = map(lambda r: r.strip(), content.splitlines()) + return requirements + + +requirements = read_requirements('requirements.txt') +test_requirements = read_requirements('requirements_dev.txt') -test_requirements = [ - 'pytest', - 'coverage' -] +# requirements = [ +# 'Click>=6.0', +# # TODO: put package requirements here +# ] +# +# test_requirements = [ +# 'pytest', +# 'coverage' +# ] # Pytest From 3afd5c66e0c5df707d8dd78a42d524051507c3a1 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 19:29:49 -0400 Subject: [PATCH 08/48] chg: travis.yml --- .travis.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e82c2d9..e5375076 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,14 +114,12 @@ before_install: - export PYTHONSTARTUP=$HOME/.pythonrc - export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache - mkdir -p $WORKON_HOME -- mkvirtualenv scarlett_os -- bash ./ci/set_postactivate.sh +- travis_retry mkvirtualenv scarlett_os +- travis_retry bash ./ci/set_postactivate.sh - workon scarlett_os -- pip install -U coveralls sphinx numpy ipython -- bash ./ci/travis.sh -- jhbuild run pip install -r requirements.txt -- jhbuild run pip install -r requirements_dev.txt -- jhbuild run python3 setup.py install +- travis_retry pip install -U coveralls sphinx numpy ipython +- travis_retry bash ./ci/travis.sh +- travis_retry jhbuild run python3 setup.py install install: - travis_retry sudo apt-get update -q From 463edae5311ce1cdbe8634257db65606c7ad7def Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 1 Oct 2016 19:34:53 -0400 Subject: [PATCH 09/48] chg: travis.yml cache jhbuild and prefix dirs --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index e5375076..dffc4e10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,9 @@ cache: - "$HOME/.cache/pip" - "$HOME/virtualenv/python3.5.2/lib/python3.5/site-packages" - "$HOME/.ccache" + - "${PREFIX}" + - "${JHBUILD}" + before_install: - export SKIP_ON_TRAVIS=yes From fc92e79555ba5e57a37303e322001712f244b41f Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 08:19:55 -0400 Subject: [PATCH 10/48] chg: pytest==3.0.3 --- requirements.txt | 1 + requirements_dev.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23e9d564..879f5646 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pydbus==0.5.1 colorlog click==6.6 psutil==4.3.0 +pytest==3.0.3 diff --git a/requirements_dev.txt b/requirements_dev.txt index 5ae33f2a..7dfb6ed1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -19,4 +19,3 @@ coveralls>=1.1 # data science ipython==5.1.0 gnureadline -pytest==3.0.3 From d8ea2974718263701a49a078a50243a46a19a5b6 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 12:29:30 -0400 Subject: [PATCH 11/48] chg: setup.py --- requirements.txt | 2 +- setup.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 879f5646..0a059bd8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ pydbus==0.5.1 colorlog click==6.6 psutil==4.3.0 -pytest==3.0.3 +wheel diff --git a/setup.py b/setup.py index b3ef4a15..3ddc7c51 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,15 @@ from setuptools.command.test import test as TestCommand from setuptools.command import install_lib +PACKAGE_NAME = 'scarlett_os' +MINIMUM_PYTHON_VERSION = 3, 5 + + +def check_python_version(): + """Exit when the Python version is too low.""" + if sys.version_info < MINIMUM_PYTHON_VERSION: + sys.exit("Python {}.{}+ is required.".format(*MINIMUM_PYTHON_VERSION)) + with open('README.rst') as readme_file: readme = readme_file.read() @@ -54,17 +63,18 @@ def read_requirements(filename): requirements = read_requirements('requirements.txt') -test_requirements = read_requirements('requirements_dev.txt') +# test_requirements = read_requirements('requirements_dev.txt') # requirements = [ # 'Click>=6.0', # # TODO: put package requirements here # ] # -# test_requirements = [ -# 'pytest', -# 'coverage' -# ] + +test_requirements = [ + 'pytest', + 'coverage' +] # Pytest @@ -80,6 +90,10 @@ def run_tests(self): errno = pytest.main(self.test_args) sys.exit(errno) + +check_python_version() + + setup( name='scarlett_os', version='0.1.0', From 0f419def220491d026a3e75af06af462a920d6f5 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 12:46:00 -0400 Subject: [PATCH 12/48] fix: fs-watch fix --- fs-scarlett.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/fs-scarlett.sh b/fs-scarlett.sh index c452a7e9..87e5391c 100755 --- a/fs-scarlett.sh +++ b/fs-scarlett.sh @@ -1,3 +1,18 @@ #!/usr/bin/env bash -fswatch -o . -e .git | xargs -n1 -I{} rsync -avz -e "ssh -i /Users/malcolm/dev/bossjones/scarlett_os/keys/vagrant_id_rsa -p 2222 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" --port 2222 --exclude *.pyc --exclude *.git --exclude *.vagrant --exclude *.vendor . pi@127.0.0.1:/home/pi/dev/bossjones-github/scarlett_os/ +fswatch -o . -e .git | xargs -n1 -I{} rsync -avz -e \ + "ssh -i /Users/malcolm/dev/bossjones/scarlett_os/keys/vagrant_id_rsa \ + -p 2222 -o UserKnownHostsFile=/dev/null \ + -o StrictHostKeyChecking=no" \ + --port 2222 \ + --exclude *.pyc --exclude *.git --exclude *.vagrant \ + --exclude *.vendor \ + --exclude .Python --exclude env/ \ + --exclude build/ --exclude develop-eggs/ \ + --exclude dist/ --exclude downloads/ \ + --exclude eggs/ --exclude .eggs/ \ + --exclude lib/ --exclude lib64/ \ + --exclude parts/ --exclude sdist/ \ + --exclude var/ --exclude *.egg-info/ \ + --exclude .installed.cfg --exclude *.egg \ + . pi@127.0.0.1:/home/pi/dev/bossjones-github/scarlett_os/ From c313a2e2daf6d575d28fb0a9f3b5202f22612220 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 13:51:27 -0400 Subject: [PATCH 13/48] update: setup.py requirements update --- requirements.txt | 9 ++++----- requirements_dev.txt | 6 +++--- setup.py | 32 +++++++++++++++++++++++++------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0a059bd8..6b884013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -pydbus==0.5.1 -colorlog -click==6.6 -psutil==4.3.0 -wheel +pydbus>=0.5.0 +colorlog>=2.7 +click>=6.0 +psutil>=4.3.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 7dfb6ed1..23e34ed3 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -pip==8.1.2 +pip>=7.0 bumpversion>=0.5.3 wheel>=0.29.0 watchdog>=0.8.3 @@ -17,5 +17,5 @@ pylint>=1.5.6 coveralls>=1.1 # data science -ipython==5.1.0 -gnureadline +ipython>=5.1.0 +gnureadline>=6.3.0 diff --git a/setup.py b/setup.py index 3ddc7c51..15eeafae 100644 --- a/setup.py +++ b/setup.py @@ -65,15 +65,33 @@ def read_requirements(filename): requirements = read_requirements('requirements.txt') # test_requirements = read_requirements('requirements_dev.txt') -# requirements = [ -# 'Click>=6.0', -# # TODO: put package requirements here -# ] -# +requirements = [ + 'Click>=6.0', + 'pydbus>=0.5.0', + 'colorlog>=2.7', + 'psutil>=4.3.0' +] + test_requirements = [ - 'pytest', - 'coverage' + 'pytest>=3.0', + 'pip>=7.0', + 'bumpversion>=0.5.3', + 'wheel>=0.29.0', + 'watchdog>=0.8.3', + 'flake8>=2.6.2', + 'flake8-docstrings>=0.2.8', + 'coverage>=4.1', + 'Sphinx>=1.4.5', + 'cryptography==1.5.2', + 'PyYAML>=3.11', + 'pydocstyle>=1.0.0', + 'mypy-lang>=0.4', + 'pylint>=1.5.6', + 'coveralls>=1.1', + 'ipython>=5.1.0', + 'gnureadline>=6.3.0' + ] From 078e3be9c27d70eecd7dfa0741a92c2e6ad89e89 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 15:30:11 -0400 Subject: [PATCH 14/48] chg: commenting out apt-file update --- .travis.yml | 10 ++++++---- setup.py | 3 --- tests/test_scarlett_os.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index dffc4e10..c05422f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,8 @@ cache: - "$HOME/.cache/pip" - "$HOME/virtualenv/python3.5.2/lib/python3.5/site-packages" - "$HOME/.ccache" - - "${PREFIX}" - - "${JHBUILD}" + - "${HOME}/jhbuild" + - "${HOME}/gnome" before_install: @@ -99,8 +99,10 @@ before_install: - travis_retry sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 byacc flex apt-file autopoint - travis_retry sudo apt-get install gcc automake autoconf libtool bison swig python-dev libpulse-dev -y -- travis_retry sudo apt-get install -qq apt-file -- sudo apt-file update +# NOTE: We only need this for sysdeps +# - travis_retry sudo apt-get install -qq apt-file +# - sudo apt-file update +- python -c "import platform;print(platform.platform().split('-'))" - travis_retry sudo apt-get install wget -qq - pip install virtualenv virtualenvwrapper - sudo pip install --upgrade virtualenvwrapper diff --git a/setup.py b/setup.py index 15eeafae..9ee49ee4 100644 --- a/setup.py +++ b/setup.py @@ -62,9 +62,6 @@ def read_requirements(filename): return requirements -requirements = read_requirements('requirements.txt') -# test_requirements = read_requirements('requirements_dev.txt') - requirements = [ 'Click>=6.0', 'pydbus>=0.5.0', diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 9a86817b..30a2021a 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -21,6 +21,8 @@ import platform ubuntu_version = platform.platform().split('-') +import pprint +pp = pprint.PrettyPrinter(indent=4) # from IPython.core.debugger import Tracer # NOQA @@ -44,6 +46,13 @@ # """ # return CliRunner() + +# @pytest.fixture +# def ubuntu_version(): +# """ubuntu_version.""" +# return ubuntu_version + + class TestScarlett_os(unittest.TestCase): def setUp(self): @@ -63,6 +72,8 @@ def test_gstreamer_versions(self): Gst.debug_set_active(True) Gst.debug_set_default_threshold(1) + pp.pprint(ubuntu_version) + if ubuntu_version[8] is 'trusty': assert GObject.pygobject_version == (3, 22, 0) else: From bf679e936e0ec5198dd800ba371714676227c2ee Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 2 Oct 2016 16:05:54 -0400 Subject: [PATCH 15/48] chg: different conditional to make this work --- tests/test_scarlett_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 30a2021a..ec919ba0 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -74,7 +74,7 @@ def test_gstreamer_versions(self): pp.pprint(ubuntu_version) - if ubuntu_version[8] is 'trusty': + if'trusty' in ubuntu_version or 'jessie' in ubuntu_version: assert GObject.pygobject_version == (3, 22, 0) else: assert GObject.pygobject_version == (3, 20, 0) From fe03e8ebba69d1b7c2f9bd301f45e4298c9a78a8 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 13:54:09 -0400 Subject: [PATCH 16/48] wip: need to begin deleting files --- scarlett_os/__init__.py | 56 ++++----- scarlett_os/__main__.py | 4 +- scarlett_os/cli.py | 214 +++++++++++++++++++++++--------- scarlett_os/compat.py | 9 ++ scarlett_os/scripts/__init__.py | 1 + scarlett_os/scripts/cli.py | 133 ++++++++++++++++++++ scarlett_os/scripts/config.py | 37 ++++++ scarlett_os/tools/package.py | 3 + scarlett_os/tools/verify.py | 6 + setup.py | 36 +++++- tests/test_scarlett_os.py | 27 +++- 11 files changed, 424 insertions(+), 102 deletions(-) create mode 100644 scarlett_os/compat.py create mode 100644 scarlett_os/scripts/__init__.py create mode 100644 scarlett_os/scripts/cli.py create mode 100644 scarlett_os/scripts/config.py diff --git a/scarlett_os/__init__.py b/scarlett_os/__init__.py index 445dfd28..8c85fc84 100644 --- a/scarlett_os/__init__.py +++ b/scarlett_os/__init__.py @@ -35,31 +35,31 @@ sys.exit(1) -def main(): - """Main entry point for ScarlettOS. - - Select the mode (standalone, client or server) - Run it... - """ - - # TODO: Grabe values from cli, setup correct logging. - # TODO: For now we'll default to scarlett_os.logger - - import platform - import scarlett_os.logger - import logging - logger = logging.getLogger('scarlettlogger') - - # Log ScarlettOS and PSutil version - logger.info('Start ScarlettOS {}'.format(__version__)) - logger.info('{} {} and PSutil {} detected'.format( - platform.python_implementation(), - platform.python_version(), - psutil_version)) - - # Share global var - global core, dbus_server, listener, tasker, check_all_services - - # Create the ScarlettOS main instance - # source: Glances - # core = ScarlettOSMain() +# def main(): +# """Main entry point for ScarlettOS. +# +# Select the mode (standalone, client or server) +# Run it... +# """ +# +# # TODO: Grabe values from cli, setup correct logging. +# # TODO: For now we'll default to scarlett_os.logger +# +# import platform +# import scarlett_os.logger +# import logging +# logger = logging.getLogger('scarlettlogger') +# +# # Log ScarlettOS and PSutil version +# logger.info('Start ScarlettOS {}'.format(__version__)) +# logger.info('{} {} and PSutil {} detected'.format( +# platform.python_implementation(), +# platform.python_version(), +# psutil_version)) +# +# # Share global var +# global core, dbus_server, listener, tasker, check_all_services +# +# # Create the ScarlettOS main instance +# # source: Glances +# # core = ScarlettOSMain() diff --git a/scarlett_os/__main__.py b/scarlett_os/__main__.py index 5b5024c1..0ed44399 100644 --- a/scarlett_os/__main__.py +++ b/scarlett_os/__main__.py @@ -8,6 +8,8 @@ # $ python -m scarlett_os import scarlett_os +import scarlett_os.cli + if __name__ == '__main__': - scarlett_os.main() + scarlett_os.cli.main() diff --git a/scarlett_os/cli.py b/scarlett_os/cli.py index aea46b0d..18ae7a5a 100644 --- a/scarlett_os/cli.py +++ b/scarlett_os/cli.py @@ -25,67 +25,157 @@ # ), err=record.levelno >= logging.WARN) -@click.command() -@click.option( - "--name", - "-n", - help="Name ScarlettOS process explicitly.", - metavar="NAME", - default="scarlett_system" -) -@click.option( - "--daemon", - "-d", - is_flag=True, - help="Daemon mode, background process.", - default=False -) -@click.option( - "--slave", - "-s", - is_flag=True, - help="Run ScarlettOS process as a Slave", - default=False -) -@click.option( - "--master", - "-m", - is_flag=True, - help="Run ScarlettOS process as a Master", - default=False -) -@click.option( - '--type', - type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), - help="ScarlettOS type", - default='check_all_services' -) -@click.option( - '--etcd-host', - help="Etcd Host for distributed mode.", - default=False -) -@click.option( - "--quiet", - "-q", - is_flag=True, - help="Limit output to errors and warnings.", - default=False -) -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Be verbose.", - default=False -) -@click.version_option(version=__version__) -def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): - """Console script for scarlett_os.""" - click.echo("Replace this message by putting your code into " - "scarlett_os.cli.main") - click.echo("See click documentation at http://click.pocoo.org/") +# from docopt import docopt +# import logging +# +# import localtomatoes +# +# log = logging.getLogger(__name__) +# +# +# def main(): +# arguments = docopt(__doc__, version=localtomatoes.__version__) +# debug = arguments['--debug'] +# logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) +# log.debug('arguments: %s', arguments) + + +class ScarlettMain(object): + + """"Main class to manages Scarlett instance.""" + def __init__(self): + """Manage the command line arguments.""" + self.args = self.parse_args() -if __name__ == "__main__": - main() + @click.command() + @click.option( + "--name", + "-n", + help="Name ScarlettOS process explicitly.", + metavar="NAME", + default="scarlett_system" + ) + @click.option( + "--daemon", + "-d", + is_flag=True, + help="Daemon mode, background process.", + default=False + ) + @click.option( + "--slave", + "-s", + is_flag=True, + help="Run ScarlettOS process as a Slave", + default=False + ) + @click.option( + "--master", + "-m", + is_flag=True, + help="Run ScarlettOS process as a Master", + default=False + ) + @click.option( + '--type', + type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), + help="ScarlettOS type", + default='check_all_services' + ) + @click.option( + '--etcd-host', + help="Etcd Host for distributed mode.", + default=False + ) + @click.option( + "--quiet", + "-q", + is_flag=True, + help="Limit output to errors and warnings.", + default=False + ) + @click.option( + "--verbose", + "-V", + is_flag=True, + help="Be verbose.", + default=False + ) + @click.version_option(version=__version__) + def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): + """Console script for scarlett_os.""" + click.echo("Replace this message by putting your code into " + "scarlett_os.cli.main") + click.echo("See click documentation at http://click.pocoo.org/") + + + if __name__ == "__main__": + main() + + +# +# @click.command() +# @click.option( +# "--name", +# "-n", +# help="Name ScarlettOS process explicitly.", +# metavar="NAME", +# default="scarlett_system" +# ) +# @click.option( +# "--daemon", +# "-d", +# is_flag=True, +# help="Daemon mode, background process.", +# default=False +# ) +# @click.option( +# "--slave", +# "-s", +# is_flag=True, +# help="Run ScarlettOS process as a Slave", +# default=False +# ) +# @click.option( +# "--master", +# "-m", +# is_flag=True, +# help="Run ScarlettOS process as a Master", +# default=False +# ) +# @click.option( +# '--type', +# type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), +# help="ScarlettOS type", +# default='check_all_services' +# ) +# @click.option( +# '--etcd-host', +# help="Etcd Host for distributed mode.", +# default=False +# ) +# @click.option( +# "--quiet", +# "-q", +# is_flag=True, +# help="Limit output to errors and warnings.", +# default=False +# ) +# @click.option( +# "--verbose", +# "-v", +# is_flag=True, +# help="Be verbose.", +# default=False +# ) +# @click.version_option(version=__version__) +# def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): +# """Console script for scarlett_os.""" +# click.echo("Replace this message by putting your code into " +# "scarlett_os.cli.main") +# click.echo("See click documentation at http://click.pocoo.org/") +# +# +# if __name__ == "__main__": +# main() diff --git a/scarlett_os/compat.py b/scarlett_os/compat.py new file mode 100644 index 00000000..88c98076 --- /dev/null +++ b/scarlett_os/compat.py @@ -0,0 +1,9 @@ +# compatibility module. + +import itertools +import sys + +from six.moves import configparser + + +map = itertools.imap if sys.version_info < (3,) else map diff --git a/scarlett_os/scripts/__init__.py b/scarlett_os/scripts/__init__.py new file mode 100644 index 00000000..b3ab5bb3 --- /dev/null +++ b/scarlett_os/scripts/__init__.py @@ -0,0 +1 @@ +# module. diff --git a/scarlett_os/scripts/cli.py b/scarlett_os/scripts/cli.py new file mode 100644 index 00000000..896151f2 --- /dev/null +++ b/scarlett_os/scripts/cli.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" +Main click group for CLI +""" + +import logging +import os +import sys + +import click +from click_plugins import with_plugins +from pkg_resources import iter_entry_points + +import scarlett_os +from scarlett_os.compat import configparser + + +def configure_logging(verbosity): + log_level = max(10, 30 - 10 * verbosity) + logging.basicConfig(stream=sys.stderr, level=log_level) + + +def read_config(cfg): + parser = configparser.ConfigParser() + parser.read(cfg) + rv = {} + for section in parser.sections(): + for key, value in parser.items(section): + rv['{0}.{1}'.format(section, key)] = value + return rv + + +@with_plugins( + ep for ep in list(iter_entry_points('scarlett_os.scarlett_os_commands'))) +@click.group() +@click.version_option( + version=scarlett_os.__version__, + message='%(version)s' +) +@click.option( + "--name", + "-n", + help="Name ScarlettOS process explicitly.", + metavar="NAME", + default="scarlett_system" +) +@click.option( + "--daemon", + "-d", + is_flag=True, + help="Daemon mode, background process.", + default=False +) +@click.option( + '--mode', + '-m' + type=click.Choice( + ['dbus_server', 'listener', 'tasker', 'check_all_services'] + ), + help="ScarlettOS type", + default='check_all_services' +) +@click.option( + "--master", + "-m", + is_flag=True, + help="Run ScarlettOS process as a Master", + default=False +) +@click.option( + "--slave", + "-s", + is_flag=True, + help="Run ScarlettOS process as a Slave", + default=False +) +@click.option( + '--etcd-host', + help="Etcd Host for distributed mode.", + default=False +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Limit output to errors and warnings.", + default=False +) +@click.option( + "--verbose", + "-V", + is_flag=True, + help="Be verbose.", + default=False +) +@click.option( + '--config', + '-c', + type=click.Path(exists=True, resolve_path=True), + help="Config file" +) +@click.pass_context +def main_group(ctx, name, daemon, mode, master, slave, etcd_host, quiet, verbose, config): + """This is the command line interface to ScarlettOS. + """ + # NOTE: ctx + # Most public functions are actually methods of a 'context' object which + # is passed as the first parameter (ctx). The context object stores the + # precision, cached data, and a few other things. It also defines + # conversions so that the same high-level code can be used for several + # different base types (mpf, mpfs in Sage, intervals, Python floats) by + # switching contexts. + # + # The default context is called 'mp'. You can call most functions as + # mpmath.mp.foo(). The top-level function mpmath.foo() is just an alias + # for this. + + ctx.obj = {} + config = config or os.path.join(click.get_app_dir('scarlett_os'), 'scarlett_os.ini') + cfg = read_config(config) + if cfg: + ctx.obj['config_file'] = config + ctx.obj['cfg'] = cfg + ctx.default_map = cfg + + verbosity = (os.environ.get('SCARLETTOS_VERBOSE') or + ctx.lookup_default('scarlett_os.verbosity') or 0) + if verbose or quiet: + verbosity = verbose - quiet + verbosity = int(verbosity) + configure_logging(verbosity) + + ctx.obj['verbosity'] = verbosity diff --git a/scarlett_os/scripts/config.py b/scarlett_os/scripts/config.py new file mode 100644 index 00000000..447237db --- /dev/null +++ b/scarlett_os/scripts/config.py @@ -0,0 +1,37 @@ +import os + +import click + + +@click.command(short_help="Show all config settings.") +@click.pass_context +def config(ctx): + """Show access token and other configuration settings. + + The access token and command verbosity level can be set on the + command line, as environment variables, and in mapbox.ini config + files. + """ + ctx.default_map = ctx.obj['cfg'] + click.echo("CLI:") + # click.echo("access-token = {0}".format(ctx.obj['access_token'])) + # click.echo("verbosity = {0}".format(ctx.obj['verbosity'])) + click.echo("") + + click.echo("Environment:") + # if 'MAPBOX_ACCESS_TOKEN' in os.environ: + # click.echo("MAPBOX_ACCESS_TOKEN = {0}".format( + # os.environ['MAPBOX_ACCESS_TOKEN'])) + # if 'MapboxAccessToken' in os.environ: + # click.echo("MapboxAccessToken = {0}".format( + # os.environ['MapboxAccessToken'])) + # if 'MAPBOX_VERBOSE' in os.environ: + # click.echo("MAPBOX_VERBOSE = {0}".format( + # os.environ['MAPBOX_VERBOSE'])) + click.echo("") + + if 'config_file' in ctx.obj: + click.echo("Config file {0}:".format(ctx.obj['config_file'])) + for key, value in ctx.default_map.items(): + click.echo("{0} = {1}".format(key, value)) + click.echo("") diff --git a/scarlett_os/tools/package.py b/scarlett_os/tools/package.py index df38cb98..f2975f0e 100644 --- a/scarlett_os/tools/package.py +++ b/scarlett_os/tools/package.py @@ -63,6 +63,9 @@ def add_gi_packages(): python_version = sys.version[:3] global_path = os.path.join('/usr/lib', 'python' + python_version) + # import sys + # sys.path + if os.environ.get('PYTHONPATH'): py_path = os.environ.get('PYTHONPATH') py_paths = py_path.split(':') diff --git a/scarlett_os/tools/verify.py b/scarlett_os/tools/verify.py index 9e0ee47c..b0c8491d 100644 --- a/scarlett_os/tools/verify.py +++ b/scarlett_os/tools/verify.py @@ -1,8 +1,14 @@ # -*- coding: utf-8 -*- + def check_python3_installed(): import sys if sys.version_info < (3, 4) <= sys.version_info < (3, 5): print('ScarlettOS requires at least Python 3.4 or 3.3 to run.') sys.exit(1) + + +def get_current_os(): + import platform + return platform.platform().split('-') diff --git a/setup.py b/setup.py index 9ee49ee4..464947ae 100644 --- a/setup.py +++ b/setup.py @@ -64,9 +64,11 @@ def read_requirements(filename): requirements = [ 'Click>=6.0', + 'click-plugins', 'pydbus>=0.5.0', 'colorlog>=2.7', - 'psutil>=4.3.0' + 'psutil>=4.3.0', + 'six' ] @@ -122,11 +124,33 @@ def run_tests(self): ], package_dir={'scarlett_os': 'scarlett_os'}, - entry_points={ - 'console_scripts': [ - 'scarlett_os=scarlett_os.cli:main' - ] - }, + # , + # entry_points={ + # 'console_scripts': [ + # 'hass = homeassistant.__main__:main' + # ] + # }, + # entry_points={ + # 'console_scripts': [ + # 'scarlett_os=scarlett_os.cli:main' + # ] + # }, + # source: mapbox-cli-py + entry_points=""" + [console_scripts] + scarlett_os=scarlett_os.scripts.cli:main_group + + [scarlett_os.scarlett_os_commands] + config=scarlett_os.scripts.config:config + """, + # geocoding=mapboxcli.scripts.geocoding:geocoding + # directions=mapboxcli.scripts.directions:directions + # distance=mapboxcli.scripts.distance:distance + # mapmatching=mapboxcli.scripts.mapmatching:match + # upload=mapboxcli.scripts.uploads:upload + # staticmap=mapboxcli.scripts.static:staticmap + # surface=mapboxcli.scripts.surface:surface + # dataset=mapboxcli.scripts.datasets:datasets include_package_data=True, install_requires=requirements, license="MIT license", diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index ec919ba0..7bdd9e5b 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -13,15 +13,17 @@ import unittest import pytest import click +import importlib from contextlib import contextmanager from click.testing import CliRunner from scarlett_os import scarlett_os from scarlett_os import cli +from scarlett_os.tools import verify -import platform -ubuntu_version = platform.platform().split('-') import pprint + +ubuntu_version = verify.get_current_os() pp = pprint.PrettyPrinter(indent=4) @@ -61,8 +63,23 @@ def setUp(self): def tearDown(self): pass - def test_000_something(self): - pass + def test_imports_something(self): + assert importlib.util.find_spec("platform") is not None + assert importlib.util.find_spec("scarlett_os.logger") is not None + assert importlib.util.find_spec("logging") is not None + pp.pprint(dir(scarlett_os)) + # [ '__builtins__', + # '__cached__', + # '__doc__', + # '__file__', + # '__loader__', + # '__name__', + # '__package__', + # '__spec__'] + print(scarlett_os.__name__) + print(scarlett_os.__package__) + print(scarlett_os.__file__) + assert scarlett_os.main.platform is not None def test_gstreamer_versions(self): import gi @@ -74,7 +91,7 @@ def test_gstreamer_versions(self): pp.pprint(ubuntu_version) - if'trusty' in ubuntu_version or 'jessie' in ubuntu_version: + if 'trusty' in ubuntu_version or 'jessie' in ubuntu_version: assert GObject.pygobject_version == (3, 22, 0) else: assert GObject.pygobject_version == (3, 20, 0) From fb7abc246d537fcea29e73687947d3e7655a31ea Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 14:41:38 -0400 Subject: [PATCH 17/48] cli: fixed cli runner --- scarlett_os/__main__.py | 23 +++-- scarlett_os/cli.py | 181 ------------------------------------ scarlett_os/main.py | 16 ---- scarlett_os/scarlett_os.py | 2 - scarlett_os/scripts/cli.py | 6 +- scarlett_os/tools/logger.py | 40 -------- scarlett_os/tools/verify.py | 2 - tests/test_cli.py | 51 ++++++++++ tests/test_scarlett_os.py | 23 +---- 9 files changed, 72 insertions(+), 272 deletions(-) delete mode 100644 scarlett_os/cli.py delete mode 100644 scarlett_os/main.py delete mode 100644 scarlett_os/scarlett_os.py delete mode 100644 scarlett_os/tools/logger.py create mode 100644 tests/test_cli.py diff --git a/scarlett_os/__main__.py b/scarlett_os/__main__.py index 0ed44399..c5783fd3 100644 --- a/scarlett_os/__main__.py +++ b/scarlett_os/__main__.py @@ -2,14 +2,25 @@ # -*- coding: utf-8 -*- # +# ##################################################### +# # NOTE: THIS USE TO BE IN cli.py in ROOT folder +# ##################################################### +# from __future__ import absolute_import, division, print_function +# +# # import logging +# import click +# +# from . import __version__ +# ##################################################### + """Allow user to run ScarlettOS as a module.""" # Execute with: # $ python -m scarlett_os -import scarlett_os -import scarlett_os.cli - - -if __name__ == '__main__': - scarlett_os.cli.main() +# import scarlett_os +# import scarlett_os.cli +# +# +# if __name__ == '__main__': +# scarlett_os.cli.main() diff --git a/scarlett_os/cli.py b/scarlett_os/cli.py deleted file mode 100644 index 18ae7a5a..00000000 --- a/scarlett_os/cli.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from __future__ import absolute_import, division, print_function - -# import logging -import click - -from . import __version__ - -# source: Doc2Dash -# class ClickEchoHandler(logging.Handler): -# """ -# Use click.echo() for logging. Has the advantage of stripping color codes -# if output is redirected. Also is generally more predictable. -# """ -# _level_to_fg = { -# logging.ERROR: "red", -# logging.WARN: "yellow", -# } -# -# def emit(self, record): -# click.echo(click.style( -# record.getMessage(), -# fg=self._level_to_fg.get(record.levelno, "reset") -# ), err=record.levelno >= logging.WARN) - - -# from docopt import docopt -# import logging -# -# import localtomatoes -# -# log = logging.getLogger(__name__) -# -# -# def main(): -# arguments = docopt(__doc__, version=localtomatoes.__version__) -# debug = arguments['--debug'] -# logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) -# log.debug('arguments: %s', arguments) - - -class ScarlettMain(object): - - """"Main class to manages Scarlett instance.""" - - def __init__(self): - """Manage the command line arguments.""" - self.args = self.parse_args() - - @click.command() - @click.option( - "--name", - "-n", - help="Name ScarlettOS process explicitly.", - metavar="NAME", - default="scarlett_system" - ) - @click.option( - "--daemon", - "-d", - is_flag=True, - help="Daemon mode, background process.", - default=False - ) - @click.option( - "--slave", - "-s", - is_flag=True, - help="Run ScarlettOS process as a Slave", - default=False - ) - @click.option( - "--master", - "-m", - is_flag=True, - help="Run ScarlettOS process as a Master", - default=False - ) - @click.option( - '--type', - type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), - help="ScarlettOS type", - default='check_all_services' - ) - @click.option( - '--etcd-host', - help="Etcd Host for distributed mode.", - default=False - ) - @click.option( - "--quiet", - "-q", - is_flag=True, - help="Limit output to errors and warnings.", - default=False - ) - @click.option( - "--verbose", - "-V", - is_flag=True, - help="Be verbose.", - default=False - ) - @click.version_option(version=__version__) - def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): - """Console script for scarlett_os.""" - click.echo("Replace this message by putting your code into " - "scarlett_os.cli.main") - click.echo("See click documentation at http://click.pocoo.org/") - - - if __name__ == "__main__": - main() - - -# -# @click.command() -# @click.option( -# "--name", -# "-n", -# help="Name ScarlettOS process explicitly.", -# metavar="NAME", -# default="scarlett_system" -# ) -# @click.option( -# "--daemon", -# "-d", -# is_flag=True, -# help="Daemon mode, background process.", -# default=False -# ) -# @click.option( -# "--slave", -# "-s", -# is_flag=True, -# help="Run ScarlettOS process as a Slave", -# default=False -# ) -# @click.option( -# "--master", -# "-m", -# is_flag=True, -# help="Run ScarlettOS process as a Master", -# default=False -# ) -# @click.option( -# '--type', -# type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), -# help="ScarlettOS type", -# default='check_all_services' -# ) -# @click.option( -# '--etcd-host', -# help="Etcd Host for distributed mode.", -# default=False -# ) -# @click.option( -# "--quiet", -# "-q", -# is_flag=True, -# help="Limit output to errors and warnings.", -# default=False -# ) -# @click.option( -# "--verbose", -# "-v", -# is_flag=True, -# help="Be verbose.", -# default=False -# ) -# @click.version_option(version=__version__) -# def main(name, daemon, slave, quiet, verbose, master, type, etcd_host): -# """Console script for scarlett_os.""" -# click.echo("Replace this message by putting your code into " -# "scarlett_os.cli.main") -# click.echo("See click documentation at http://click.pocoo.org/") -# -# -# if __name__ == "__main__": -# main() diff --git a/scarlett_os/main.py b/scarlett_os/main.py deleted file mode 100644 index 0d6db3c3..00000000 --- a/scarlett_os/main.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- - -"""ScarlettOS main class.""" - -# from scarlett_os import package as ts_package - -# import argparse -# import os -# import sys -# import tempfile -# -# from glances import __appname__, __version__, psutil_version -# from glances.compat import input -# from glances.config import Config -# from glances.globals import LINUX, WINDOWS -# from glances.logger import logger diff --git a/scarlett_os/scarlett_os.py b/scarlett_os/scarlett_os.py deleted file mode 100644 index faa18be5..00000000 --- a/scarlett_os/scarlett_os.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/scarlett_os/scripts/cli.py b/scarlett_os/scripts/cli.py index 896151f2..d35d4d3d 100644 --- a/scarlett_os/scripts/cli.py +++ b/scarlett_os/scripts/cli.py @@ -53,10 +53,8 @@ def read_config(cfg): ) @click.option( '--mode', - '-m' - type=click.Choice( - ['dbus_server', 'listener', 'tasker', 'check_all_services'] - ), + '-m', + type=click.Choice(['dbus_server', 'listener', 'tasker', 'check_all_services']), help="ScarlettOS type", default='check_all_services' ) diff --git a/scarlett_os/tools/logger.py b/scarlett_os/tools/logger.py deleted file mode 100644 index 6b0fec3d..00000000 --- a/scarlett_os/tools/logger.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import logging -logger = logging.getLogger('scarlett_os') - -try: - from colorlog import ColoredFormatter - from gettext import gettext as _ - """Return a logger with a default ColoredFormatter.""" - formatter = ColoredFormatter( - "(%(threadName)-9s) %(log_color)s%(levelname)-8s%(reset)s (%(funcName)-5s) %(message_log_color)s%(message)s", - datefmt=None, - reset=True, - log_colors={ - 'DEBUG': 'cyan', - 'INFO': 'green', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'red', - }, - secondary_log_colors={ - 'message': { - 'ERROR': 'red', - 'CRITICAL': 'red', - 'DEBUG': 'yellow', - 'INFO': 'yellow,bg_blue' - } - }, - style='%' - ) - - handler = logging.StreamHandler() - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) -except ImportError: - # No color available, use default config - logging.basicConfig(format='%(levelname)s: %(message)s') - logger.warn("Disabling color, you really want to install colorlog.") diff --git a/scarlett_os/tools/verify.py b/scarlett_os/tools/verify.py index b0c8491d..63ca025c 100644 --- a/scarlett_os/tools/verify.py +++ b/scarlett_os/tools/verify.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- - - def check_python3_installed(): import sys if sys.version_info < (3, 4) <= sys.version_info < (3, 5): diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..20d82efa --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +test_scarlett_os +---------------------------------- + +Tests for `scarlett_os` module. +""" + + +import sys +import unittest +import pytest +import click +import importlib +from contextlib import contextmanager +from click.testing import CliRunner + +import scarlett_os +from scarlett_os.scripts.cli import main_group +from scarlett_os.tools import verify + +import pprint + +ubuntu_version = verify.get_current_os() +pp = pprint.PrettyPrinter(indent=4) + + +class TestScarlettCli(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_command_line_interface_help(self): + runner = CliRunner() + result = runner.invoke(main_group) + assert result.exit_code == 0 + assert 'main_group' in result.output + assert 'This is the command line interface to ScarlettOS' in result.output + help_result = runner.invoke(main_group, ['--help']) + assert help_result.exit_code == 0 + print(help_result.output) + assert 'dbus_server|listener|tasker|check_all_services' in help_result.output + + +if __name__ == '__main__': + sys.exit(unittest.main()) diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 7bdd9e5b..57b87611 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -17,8 +17,8 @@ from contextlib import contextmanager from click.testing import CliRunner -from scarlett_os import scarlett_os -from scarlett_os import cli +import scarlett_os +from scarlett_os.scripts.cli import main_group from scarlett_os.tools import verify import pprint @@ -68,18 +68,9 @@ def test_imports_something(self): assert importlib.util.find_spec("scarlett_os.logger") is not None assert importlib.util.find_spec("logging") is not None pp.pprint(dir(scarlett_os)) - # [ '__builtins__', - # '__cached__', - # '__doc__', - # '__file__', - # '__loader__', - # '__name__', - # '__package__', - # '__spec__'] print(scarlett_os.__name__) print(scarlett_os.__package__) print(scarlett_os.__file__) - assert scarlett_os.main.platform is not None def test_gstreamer_versions(self): import gi @@ -97,16 +88,6 @@ def test_gstreamer_versions(self): assert GObject.pygobject_version == (3, 20, 0) assert Gst.version_string() == 'GStreamer 1.8.2' - def test_command_line_interface(self): - runner = CliRunner() - result = runner.invoke(cli.main) - assert result.exit_code == 0 - assert 'scarlett_os.cli.main' in result.output - help_result = runner.invoke(cli.main, ['--help']) - assert help_result.exit_code == 0 - print(help_result.output) - assert 'dbus_server|listener|tasker|check_all_services' in help_result.output - if __name__ == '__main__': sys.exit(unittest.main()) From 8bc8b5f563e4921ebcd2271736901c380c288dab Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 15:35:09 -0400 Subject: [PATCH 18/48] tests: update --- .coveralls.yml | 2 -- .travis.yml | 11 +++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 85f68247..00000000 --- a/.coveralls.yml +++ /dev/null @@ -1,2 +0,0 @@ -repo_token: sV16DIhoxZ1MQ5REfJYJlzpPSLfYu5HBG -service-name: travis-ci diff --git a/.travis.yml b/.travis.yml index c05422f5..d8f1847f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,6 +102,10 @@ before_install: # NOTE: We only need this for sysdeps # - travis_retry sudo apt-get install -qq apt-file # - sudo apt-file update +# ubuntu 16.04 says: +# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'Ubuntu', '16.04', 'xenial'] +# travis says +# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'debian', 'jessie', 'sid'] - python -c "import platform;print(platform.platform().split('-'))" - travis_retry sudo apt-get install wget -qq - pip install virtualenv virtualenvwrapper @@ -124,15 +128,18 @@ before_install: - workon scarlett_os - travis_retry pip install -U coveralls sphinx numpy ipython - travis_retry bash ./ci/travis.sh +- travis_retry jhbuild run pip install -U coveralls sphinx numpy - travis_retry jhbuild run python3 setup.py install install: - travis_retry sudo apt-get update -q - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) -script: jhbuild run python3 setup.py test +script: +- jhbuild run coverage run --source=scarlett_os setup.py test -after_success: coveralls +after_success: +- coveralls deploy: distributions: sdist bdist_wheel From 1bf966c1ddff86f1dfbcbeab97baf9819f3d2444 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 16:17:53 -0400 Subject: [PATCH 19/48] chg: more pytest changes --- .coveragerc | 2 ++ .travis.yml | 3 +-- Makefile | 7 +++++++ setup.cfg | 3 +++ setup.py | 3 +++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index f1f41ba5..f3737016 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,8 @@ omit = */python?.?/* */site-packages/nose/* */test/* + */pygobject-3.20.1/* + */site-packages/* scarlett_os/__main__.py scarlett_os/scripts/*.py diff --git a/.travis.yml b/.travis.yml index d8f1847f..8f9f26c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -128,7 +128,6 @@ before_install: - workon scarlett_os - travis_retry pip install -U coveralls sphinx numpy ipython - travis_retry bash ./ci/travis.sh -- travis_retry jhbuild run pip install -U coveralls sphinx numpy - travis_retry jhbuild run python3 setup.py install install: @@ -136,7 +135,7 @@ install: - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: -- jhbuild run coverage run --source=scarlett_os setup.py test +- jhbuild run python3 setup.py install after_success: - coveralls diff --git a/Makefile b/Makefile index e8531417..767a52a1 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,13 @@ clean-test: ## remove test and coverage artifacts lint: ## check style with flake8 flake8 scarlett_os tests +pytest-install-test-deps: clean + pip install -e .[test] + python setup.py install + +pytest-run: + py.test + test: ## run tests quickly with the default Python python setup.py test diff --git a/setup.cfg b/setup.cfg index d837c4e8..fd3d5ab0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,6 @@ universal = 1 [flake8] exclude = docs + +[tool:pytest] +addopts = --cov=scarlett_os --cov-report term-missing tests/ diff --git a/setup.py b/setup.py index 464947ae..fee0244c 100644 --- a/setup.py +++ b/setup.py @@ -151,6 +151,9 @@ def run_tests(self): # staticmap=mapboxcli.scripts.static:staticmap # surface=mapboxcli.scripts.surface:surface # dataset=mapboxcli.scripts.datasets:datasets + extras_require={ + 'test': ['coveralls', 'pytest>=3.0', 'pytest-cov'], + }, include_package_data=True, install_requires=requirements, license="MIT license", From b06f2e75b48cb9ee6b37280c53ba264e5d19f421 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 19:45:16 -0400 Subject: [PATCH 20/48] fix: travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8f9f26c9..c2ff71ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -131,11 +131,12 @@ before_install: - travis_retry jhbuild run python3 setup.py install install: +- travis_retry pip install -U coveralls sphinx numpy ipython - travis_retry sudo apt-get update -q - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: -- jhbuild run python3 setup.py install +- jhbuild run coverage run ./setup.py test after_success: - coveralls From 924ba6e77b7af28b92be17d2f29f1e4cb3328c81 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 20:14:12 -0400 Subject: [PATCH 21/48] fix: fix coverage test command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c2ff71ca..744b2448 100644 --- a/.travis.yml +++ b/.travis.yml @@ -136,7 +136,7 @@ install: - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: -- jhbuild run coverage run ./setup.py test +- jhbuild run python3 coverage run ./setup.py test after_success: - coveralls From 37fe2742c27e39e689ee056ab78d22afa16523bb Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 15 Oct 2016 20:35:17 -0400 Subject: [PATCH 22/48] chg: pip install -e .[test] --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 744b2448..0799e857 100644 --- a/.travis.yml +++ b/.travis.yml @@ -136,6 +136,7 @@ install: - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: +- jhbuild run pip install -e .[test] - jhbuild run python3 coverage run ./setup.py test after_success: From e316658aed82b9c80c87c91d50bd1279970618b1 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sun, 16 Oct 2016 17:11:30 -0400 Subject: [PATCH 23/48] chg: fabfile task runner --- .travis.yml | 6 +-- fabfile.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + setup.py | 3 +- 4 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 fabfile.py diff --git a/.travis.yml b/.travis.yml index 0799e857..0679d859 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,8 @@ cache: - "$HOME/.cache/pip" - "$HOME/virtualenv/python3.5.2/lib/python3.5/site-packages" - "$HOME/.ccache" - - "${HOME}/jhbuild" - - "${HOME}/gnome" + - "$HOME/jhbuild" + - "$HOME/gnome" before_install: @@ -136,7 +136,7 @@ install: - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: -- jhbuild run pip install -e .[test] +# - jhbuild run pip install -e .[test] - jhbuild run python3 coverage run ./setup.py test after_success: diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 00000000..72c6f26a --- /dev/null +++ b/fabfile.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# NOTE: fab --show=debug if you need to debug stuff + +import sys +import os +import pprint +import re +# import string +# import collections +import time +from fabric.api import abort, cd, env, get, hide, hosts, local, prompt, \ + put, require, roles, run, runs_once, settings, show, sudo, warn +from fabric.contrib.project import rsync_project +from textwrap import dedent +# from fabric.operations import run, put +# from fabric.contrib.files import exists +# from fabric.operations import reboot +# from fabric.colors import red, green, yellow, blue +# from fabric.utils import puts +# from fabric.tasks import execute +# from fabric.contrib.files import exists, sed + +# env.remote_interrupt = True +# env.use_ssh_config = True +# #env.gateway = FABRIC_JUMPSERVER +# #env.disable_known_hosts = True +# #env.reject_unknown_hosts = False +# env.forward_agent = False +# env.keepalive = 60 +# env.key_filename = FABRIC_KEY_FILENAME +# env.parallel = True +# env.sudo_user = FABRIC_USER +# env.user = FABRIC_USER +# env.abort_on_prompts = True + +pp = pprint.PrettyPrinter(indent=4, width=80) +VM_PATTERN = 'travis-ci/ubuntu1404' + + +def vagrant(): + """USAGE: + fab vagrant uname + + Note that the command to run Fabric might be different on different + platforms. + """ + + global VM_PATTERN + # change from the default user to 'vagrant' + vagrant_ssh_config = local("cd $HOME/dev/{0} && vagrant ssh-config".format(VM_PATTERN), capture=True) + + hostname = re.findall(r'HostName\s+([^\n]+)', vagrant_ssh_config)[0] + port = re.findall(r'Port\s+([^\n]+)', vagrant_ssh_config)[0] + env.hosts = ['%s:%s' % (hostname, port)] + env.user = re.findall(r'User\s+([^\n]+)', vagrant_ssh_config)[0] + + identity_file = re.findall(r'IdentityFile\s+([^\n]+)', vagrant_ssh_config)[0] + + env.key_filename = identity_file[1:-1] + env.disable_known_hosts = True + env.forward_agent = True + env.colorize_errors = True + env.skip_bad_hosts = True + env.warn_only = True + + print(""" + hostname = {} + port = {} + env.hosts = {} + env.user = {} + env.key_filename = {} + env.disable_known_hosts = {} + env.forward_agent = {} +""".format(hostname, + port, + env.hosts, + env.user, + env.key_filename, + env.disable_known_hosts, + env.forward_agent)) + + result = local("cd $HOME/dev/{0} && vagrant global-status | grep running | grep '{0}'".format(VM_PATTERN), capture=True) + print("result: {}".format(result)) + machineId = result.split()[0] + print("machineId: {}".format(machineId)) + + +def uname(): + result = run('uname -a') + + +# def copy(): +# +# +# def pack(loc): +# +# rsync_project() diff --git a/requirements.txt b/requirements.txt index 6b884013..3d4f678f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pydbus>=0.5.0 colorlog>=2.7 click>=6.0 psutil>=4.3.0 +Fabric3==1.12.post1 diff --git a/setup.py b/setup.py index fee0244c..a7baca36 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,8 @@ def read_requirements(filename): 'pydbus>=0.5.0', 'colorlog>=2.7', 'psutil>=4.3.0', - 'six' + 'six', + 'Fabric3==1.12.post1' ] From b330da65cb6e668f4a812c804542e59c76b66868 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 20:48:43 -0400 Subject: [PATCH 24/48] path: fabfile patch commands --- fabfile.py | 167 +++++++++++++++++++++++++++++++++++++++-------- requirements.txt | 1 + setup.py | 3 +- 3 files changed, 143 insertions(+), 28 deletions(-) diff --git a/fabfile.py b/fabfile.py index 72c6f26a..2776a0d7 100644 --- a/fabfile.py +++ b/fabfile.py @@ -7,13 +7,20 @@ import os import pprint import re -# import string +import string # import collections import time +from fabric.operations import sudo, os from fabric.api import abort, cd, env, get, hide, hosts, local, prompt, \ - put, require, roles, run, runs_once, settings, show, sudo, warn -from fabric.contrib.project import rsync_project + put, require, roles, run, runs_once, settings, show, sudo, warn, prefix +from fabric.contrib.project import rsync_project, upload_project +from fabric.contrib.files import append, sed from textwrap import dedent +import yaml +import csv + +pp = pprint.PrettyPrinter(indent=4, width=80) + # from fabric.operations import run, put # from fabric.contrib.files import exists # from fabric.operations import reboot @@ -22,21 +29,15 @@ # from fabric.tasks import execute # from fabric.contrib.files import exists, sed -# env.remote_interrupt = True -# env.use_ssh_config = True -# #env.gateway = FABRIC_JUMPSERVER -# #env.disable_known_hosts = True -# #env.reject_unknown_hosts = False -# env.forward_agent = False -# env.keepalive = 60 -# env.key_filename = FABRIC_KEY_FILENAME -# env.parallel = True -# env.sudo_user = FABRIC_USER -# env.user = FABRIC_USER -# env.abort_on_prompts = True - -pp = pprint.PrettyPrinter(indent=4, width=80) VM_PATTERN = 'travis-ci/ubuntu1404' +USER = "vagrant" +HOME = "/home/{}".format(USER) +WORKON_HOME = "{}/.virtualenvs".format(HOME) +PROJECT_HOME = "{}/dev".format(HOME) +REPO_NAME = "scarlett_os" +REPO_ORG = "bossjones" +VENV_NAME = "{}".format(REPO_NAME) +PATH_TO_PROJECT = "{}/{}-github/{}".format(PROJECT_HOME, REPO_ORG, REPO_NAME) def vagrant(): @@ -47,7 +48,7 @@ def vagrant(): platforms. """ - global VM_PATTERN + # global VM_PATTERN # change from the default user to 'vagrant' vagrant_ssh_config = local("cd $HOME/dev/{0} && vagrant ssh-config".format(VM_PATTERN), capture=True) @@ -88,12 +89,124 @@ def vagrant(): def uname(): - result = run('uname -a') - - -# def copy(): -# -# -# def pack(loc): -# -# rsync_project() + """Testing vagrant connections with uname -a.""" + run('uname -a') + + +def deploy(): + rsync_project(local_dir='.', + remote_dir='/home/vagrant/dev/bossjones-github/scarlett_os/', + exclude=['*.git', + '*.pyc', + '*.vagrant', + '*.vendor', + '.Python', + 'env/', + 'build/', + 'develop-eggs/', + 'dist/', + 'downloads/', + 'eggs/', + '.eggs/', + 'lib/', + 'lib64/', + 'parts/', + 'sdist/', + 'var/', + '*.egg-info/', + '.installed.cfg', + '*.egg', + '.tox/', + '.bundle/', + '.cache/', + '__pycache__/' + ], + ssh_opts='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' + ) + + +def clean_build(): + """Nuke all testing directories before we get started.""" + with prefix('workon scarlett_os'), cd('/home/vagrant'): + run('rm -rfv /home/vagrant/dev/bossjones-github/scarlett_os') + run('rm -rfv /home/vagrant/gnome') + run('rm -rfv /home/vagrant/jhbuild') + run('rm -rfv /home/vagrant/jhbuild') + + +def bootstrap_travisci(): + with prefix('mkvirtualenv --python=`which python3` scarlett_os'): + run('mkdir -p /home/vagrant/dev/bossjones-github/scarlett_os') + with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): + # run: travis-build step + pass + + +def read_yaml(): + with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): + with open('.travis.yml', 'r') as f: + travis_config = yaml.safe_load(f) + print(yaml.dump(travis_config)) + travis_lines = ['#!/bin/bash', + 'set -e', + 'set -x' + ] + + matrix = travis_config['matrix']['include'] + print('****************************matrix****************************') + for line in matrix: + print(line) + split_line = line['env'].split(" ") + print(split_line) + for l in split_line: + travis_lines.append("export {}".format(l)) + + before_install = travis_config['before_install'] + print(before_install) + print('****************************before_install****************************') + for line in before_install: + line = re.sub('travis_retry ', '', line) + line = re.sub('pip install', 'pip3.5 install', line) + line = re.sub('pip install -I path.py==7.7.1', '', line) + line = re.sub('which python3', 'which python3.5', line) + travis_lines.append(line) + print(line) + + install = travis_config['install'] + print('****************************install****************************') + for line in install: + line = re.sub('travis_retry ', '', line) + travis_lines.append(line) + print(line) + + print('****************************travis_lines****************************') + print(travis_lines) + + run('rm /home/vagrant/test_travis.sh') + append('/home/vagrant/test_travis.sh', travis_lines) + print('****************************patch test_travis****************************') + sed('/home/vagrant/test_travis.sh', 'travis_retry ', '') + sed('/home/vagrant/test_travis.sh', 'pip install', 'pip3.5 install') + sed('/home/vagrant/test_travis.sh', 'pip install -I path.py==7.7.1', '') + sed('/home/vagrant/test_travis.sh', 'which python3', 'which python3.5') + sed('/home/vagrant/test_travis.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + print('****************************patch test_travis****************************') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'travis_retry ', '') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'pip install', 'pip3.5 install') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'pip install -I path.py==7.7.1', '') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'which python3', 'which python3.5') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + print('****************************patch test_travis****************************') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'travis_retry ', '') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'pip install', 'pip3.5 install') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'pip install -I path.py==7.7.1', '') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'which python3', 'which python3.5') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + print('****************************final-travis-script****************************') + run('cat /home/vagrant/test_travis.sh') + run('chmod +x /home/vagrant/test_travis.sh') + + +def run_travis(): + with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): + run('/home/vagrant/test_travis.sh') diff --git a/requirements.txt b/requirements.txt index 3d4f678f..7db786b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ colorlog>=2.7 click>=6.0 psutil>=4.3.0 Fabric3==1.12.post1 +PyYAML>=3.0 diff --git a/setup.py b/setup.py index a7baca36..db624e9d 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,8 @@ def read_requirements(filename): 'colorlog>=2.7', 'psutil>=4.3.0', 'six', - 'Fabric3==1.12.post1' + 'Fabric3==1.12.post1', + 'PyYAML>=3.0' ] From ee0ccdb169c299ee4cc98145ca80714074f84aab Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 21:37:27 -0400 Subject: [PATCH 25/48] chg: fix context management prefix with environment variables --- fabfile.py | 60 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/fabfile.py b/fabfile.py index 2776a0d7..a4b77a0c 100644 --- a/fabfile.py +++ b/fabfile.py @@ -127,19 +127,40 @@ def deploy(): def clean_build(): """Nuke all testing directories before we get started.""" - with prefix('workon scarlett_os'), cd('/home/vagrant'): - run('rm -rfv /home/vagrant/dev/bossjones-github/scarlett_os') - run('rm -rfv /home/vagrant/gnome') - run('rm -rfv /home/vagrant/jhbuild') - run('rm -rfv /home/vagrant/jhbuild') + # export WORKON_HOME=${HOME}/.virtualenvs + # export PROJECT_HOME=${HOME}/dev + # export VIRTUALENVWRAPPER_PYTHON=/usr/bin/python3 + # export VIRTUALENVWRAPPER_VIRTUALENV=/usr/local/bin/virtualenv + # source /usr/local/bin/virtualenvwrapper.sh + # export PYTHONSTARTUP=$HOME/.pythonrc + # export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache + ################################################################################ + + with prefix('export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh`'): + with prefix('export VIRTUALENVWRAPPER_PYTHON=`which python3.5`'): + with prefix('export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv`'): + with prefix('export WORKON_HOME=${HOME}/.virtualenvs'): + with prefix('export PROJECT_HOME=${HOME}/dev'): + with prefix('source $VIRTUALENV_WRAPPER_SH'): + with prefix('export PYTHONSTARTUP=$HOME/.pythonrc'): + with prefix('export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache'): + with prefix('workon scarlett_os'): + with cd('/home/vagrant'): + sudo('pip3.5 install virtualenv virtualenvwrapper') + run('rm -rf /home/vagrant/dev/bossjones-github/scarlett_os') + run('rm -rf /home/vagrant/gnome') + run('rm -rf /home/vagrant/jhbuild') + with prefix('deactivate'): + run('rmvirtualenv scarlett_os') def bootstrap_travisci(): - with prefix('mkvirtualenv --python=`which python3` scarlett_os'): + with prefix('mkvirtualenv --python=`which python3.5` scarlett_os'): run('mkdir -p /home/vagrant/dev/bossjones-github/scarlett_os') - with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): - # run: travis-build step - pass + run('which python') + # with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): + # # run: travis-build step + # pass def read_yaml(): @@ -149,7 +170,8 @@ def read_yaml(): print(yaml.dump(travis_config)) travis_lines = ['#!/bin/bash', 'set -e', - 'set -x' + 'set -x', + 'export DEBIAN_FRONTEND=noninteractive' ] matrix = travis_config['matrix']['include'] @@ -165,10 +187,10 @@ def read_yaml(): print(before_install) print('****************************before_install****************************') for line in before_install: - line = re.sub('travis_retry ', '', line) - line = re.sub('pip install', 'pip3.5 install', line) - line = re.sub('pip install -I path.py==7.7.1', '', line) - line = re.sub('which python3', 'which python3.5', line) + # line = re.sub('travis_retry ', '', line) + # line = re.sub('pip install', 'pip3.5 install', line) + # line = re.sub('pip install -I path.py==7.7.1', '', line) + # line = re.sub('which python3', 'which python3.5', line) travis_lines.append(line) print(line) @@ -190,18 +212,21 @@ def read_yaml(): sed('/home/vagrant/test_travis.sh', 'pip install -I path.py==7.7.1', '') sed('/home/vagrant/test_travis.sh', 'which python3', 'which python3.5') sed('/home/vagrant/test_travis.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + sed('/home/vagrant/test_travis.sh', 'PYTHON="python3"', 'PYTHON="python3.5"') print('****************************patch test_travis****************************') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'travis_retry ', '') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'pip install', 'pip3.5 install') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'pip install -I path.py==7.7.1', '') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'which python3', 'which python3.5') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'PYTHON="python3"', 'PYTHON="python3.5"') print('****************************patch test_travis****************************') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'travis_retry ', '') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'pip install', 'pip3.5 install') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'pip install -I path.py==7.7.1', '') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'which python3', 'which python3.5') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') + sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/travis.sh', 'PYTHON="python3"', 'PYTHON="python3.5"') print('****************************final-travis-script****************************') run('cat /home/vagrant/test_travis.sh') run('chmod +x /home/vagrant/test_travis.sh') @@ -210,3 +235,10 @@ def read_yaml(): def run_travis(): with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): run('/home/vagrant/test_travis.sh') + + +# fab vagrant clean_build +# fab vagrant bootstrap_travisci +# fab vagrant deploy +# fab vagrant read_yaml +# fab vagrant run_travis From fffadc4382c41c1b8d10effd691036aec80a1459 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 21:40:53 -0400 Subject: [PATCH 26/48] update: bootstrap_travisci --- fabfile.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/fabfile.py b/fabfile.py index a4b77a0c..7f5ef65b 100644 --- a/fabfile.py +++ b/fabfile.py @@ -155,12 +155,18 @@ def clean_build(): def bootstrap_travisci(): - with prefix('mkvirtualenv --python=`which python3.5` scarlett_os'): - run('mkdir -p /home/vagrant/dev/bossjones-github/scarlett_os') - run('which python') - # with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): - # # run: travis-build step - # pass + with prefix('export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh`'): + with prefix('export VIRTUALENVWRAPPER_PYTHON=`which python3.5`'): + with prefix('export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv`'): + with prefix('export WORKON_HOME=${HOME}/.virtualenvs'): + with prefix('export PROJECT_HOME=${HOME}/dev'): + with prefix('source $VIRTUALENV_WRAPPER_SH'): + with prefix('export PYTHONSTARTUP=$HOME/.pythonrc'): + with prefix('export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache'): + with prefix('mkvirtualenv --python=`which python3.5` scarlett_os'): + run('mkdir -p /home/vagrant/dev/bossjones-github/scarlett_os') + with prefix('workon scarlett_os'): + run('which python') def read_yaml(): From bfdd3137fe1fda602adafaecd1d557f723e2d6a7 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 22:31:02 -0400 Subject: [PATCH 27/48] chg: more patches to fabfile --- fabfile.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 7f5ef65b..1fbba382 100644 --- a/fabfile.py +++ b/fabfile.py @@ -175,7 +175,6 @@ def read_yaml(): travis_config = yaml.safe_load(f) print(yaml.dump(travis_config)) travis_lines = ['#!/bin/bash', - 'set -e', 'set -x', 'export DEBIAN_FRONTEND=noninteractive' ] @@ -193,10 +192,38 @@ def read_yaml(): print(before_install) print('****************************before_install****************************') for line in before_install: + line = re.sub('mkvirtualenv scarlett_os', '', line) # line = re.sub('travis_retry ', '', line) # line = re.sub('pip install', 'pip3.5 install', line) # line = re.sub('pip install -I path.py==7.7.1', '', line) # line = re.sub('which python3', 'which python3.5', line) + ############################################################################# + # export TOXENV=typing + # export PYTHON="python3.5" + # export PACKAGES="python3-gi + # export python3-gi-cairo" + # export TOXENV=py35 + # export SKIP_ON_TRAVIS=yes + # export CURRENT_DIR=$(pwd) + # export GSTREAMER=1.0 + # export ENABLE_PYTHON2=yes + # export ENABLE_PYTHON3=yes + # export ENABLE_GTK=yes + # export PYTHON_VERSION_MAJOR=3 + # export PYTHON_VERSION=3.5 + # export CFLAGS="-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer" + # export MAKEFLAGS="-j4" + # export PREFIX="${HOME}/jhbuild" + # export JHBUILD="${HOME}/gnome" + # export PATH=${PREFIX}/bin:${PREFIX}/sbin:${PATH} + # export LD_LIBRARY_PATH=${PREFIX}/lib:${LD_LIBRARY_PATH} + # export PYTHONPATH=${PREFIX}/lib/python${PYTHON_VERSION}/site-packages:/usr/lib/python${PYTHON_VERSION}/site-packages + # export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig:${PREFIX}/share/pkgconfig:/usr/lib/pkgconfig + # export XDG_DATA_DIRS=${PREFIX}/share:/usr/share + # export XDG_CONFIG_DIRS=${PREFIX}/etc/xdg + # export PACKAGES="python3-gi python3-gi-cairo" + # export CC=gcc + ############################################################################# travis_lines.append(line) print(line) @@ -219,6 +246,7 @@ def read_yaml(): sed('/home/vagrant/test_travis.sh', 'which python3', 'which python3.5') sed('/home/vagrant/test_travis.sh', '/usr/bin/python3', '/usr/local/bin/python3.5') sed('/home/vagrant/test_travis.sh', 'PYTHON="python3"', 'PYTHON="python3.5"') + sed('/home/vagrant/test_travis.sh', 'pip3.5 install -I path.py==7.7.1', '') print('****************************patch test_travis****************************') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'travis_retry ', '') sed('/home/vagrant/dev/bossjones-github/scarlett_os/ci/set_postactivate.sh', 'pip install', 'pip3.5 install') From f59c0389cdd0b25ed04221f0a2444a5dc2ab8ac9 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 23:11:05 -0400 Subject: [PATCH 28/48] fix: coverage and tests --- .travis.yml | 4 ++-- fabfile.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0679d859..b58d2690 100644 --- a/.travis.yml +++ b/.travis.yml @@ -129,6 +129,7 @@ before_install: - travis_retry pip install -U coveralls sphinx numpy ipython - travis_retry bash ./ci/travis.sh - travis_retry jhbuild run python3 setup.py install +- travis_retry jhbuild run -- pip install -e .[test] install: - travis_retry pip install -U coveralls sphinx numpy ipython @@ -136,8 +137,7 @@ install: - travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) script: -# - jhbuild run pip install -e .[test] -- jhbuild run python3 coverage run ./setup.py test +- jhbuild run -- coverage run -- setup.py test after_success: - coveralls diff --git a/fabfile.py b/fabfile.py index 1fbba382..0d0330fa 100644 --- a/fabfile.py +++ b/fabfile.py @@ -276,3 +276,19 @@ def run_travis(): # fab vagrant deploy # fab vagrant read_yaml # fab vagrant run_travis +# fab vagrant retest_travis + +def retest_travis(): + with prefix('export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh`'): + with prefix('export VIRTUALENVWRAPPER_PYTHON=`which python3.5`'): + with prefix('export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv`'): + with prefix('export WORKON_HOME=${HOME}/.virtualenvs'): + with prefix('export PROJECT_HOME=${HOME}/dev'): + with prefix('source $VIRTUALENV_WRAPPER_SH'): + with prefix('export PYTHONSTARTUP=$HOME/.pythonrc'): + with prefix('export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache'): + with prefix('workon scarlett_os'): + with cd('/home/vagrant/dev/bossjones-github/scarlett_os'): + run('jhbuild run python setup.py install') + run('jhbuild run -- pip install -e .[test]') + run('jhbuild run -- coverage run -- setup.py test') From 2a4758a8ab9f738ca69ef6034ef12aa961a4fb66 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 19 Oct 2016 23:34:13 -0400 Subject: [PATCH 29/48] chg: fix python 3.4 --- Makefile | 17 +++++++++++------ setup.py | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 767a52a1..71e4c7c3 100644 --- a/Makefile +++ b/Makefile @@ -60,17 +60,22 @@ pytest-install-test-deps: clean pytest-run: py.test -test: ## run tests quickly with the default Python - - python setup.py test +jhbuild-run-test: + jhbuild run python setup.py install + jhbuild run -- pip install -e .[test] + jhbuild run -- coverage run -- setup.py test + jhbuild run -- coverage report -m -test-all: ## run tests on every Python version with tox +test: ## run tests quickly with the default Python python setup.py test -coverage: ## check code coverage quickly with the default Python +test-clean-all: ## run tests on every Python version with tox + pip install -e .[test] + python setup.py install + coverage run setup.py test +coverage: ## check code coverage quickly with the default Python coverage run --source scarlett_os setup.py test - coverage report -m coverage html $(BROWSER) htmlcov/index.html diff --git a/setup.py b/setup.py index db624e9d..acbee636 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from setuptools.command import install_lib PACKAGE_NAME = 'scarlett_os' -MINIMUM_PYTHON_VERSION = 3, 5 +MINIMUM_PYTHON_VERSION = 3, 4 def check_python_version(): @@ -167,6 +167,7 @@ def run_tests(self): 'License :: OSI Approved :: MIT License', 'Natural Language :: English', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', ], test_suite='tests', From 05139c3a01870e1bb345f46f57f817fd69925b1e Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 20 Oct 2016 00:28:05 -0400 Subject: [PATCH 30/48] chg: python version --- scarlett_os/tools/verify.py | 3 ++- setup.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scarlett_os/tools/verify.py b/scarlett_os/tools/verify.py index 63ca025c..ef6644da 100644 --- a/scarlett_os/tools/verify.py +++ b/scarlett_os/tools/verify.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- + def check_python3_installed(): import sys if sys.version_info < (3, 4) <= sys.version_info < (3, 5): - print('ScarlettOS requires at least Python 3.4 or 3.3 to run.') + print('ScarlettOS requires at least Python 3.5 or 3.4 to run.') sys.exit(1) diff --git a/setup.py b/setup.py index acbee636..afdac158 100644 --- a/setup.py +++ b/setup.py @@ -24,13 +24,19 @@ from setuptools.command import install_lib PACKAGE_NAME = 'scarlett_os' -MINIMUM_PYTHON_VERSION = 3, 4 +MINIMUM_PYTHON_VERSION = 3, 0 -def check_python_version(): - """Exit when the Python version is too low.""" - if sys.version_info < MINIMUM_PYTHON_VERSION: - sys.exit("Python {}.{}+ is required.".format(*MINIMUM_PYTHON_VERSION)) +print('Current Python Version, B: {}'.format(sys.version_info)) +# +# def check_python_version(): +# """Exit when the Python version is too low.""" +# if sys.version_info < (3, 4) <= sys.version_info < (3, 5): +# print('ScarlettOS requires at least Python 3.5 or 3.4 to run.') +# sys.exit(1) +# # if sys.version_info < MINIMUM_PYTHON_VERSION: +# # print("Using version: ") +# # sys.exit("Python {}.{}+ is required. Using {}".format(*MINIMUM_PYTHON_VERSION)) with open('README.rst') as readme_file: readme = readme_file.read() @@ -110,7 +116,7 @@ def run_tests(self): sys.exit(errno) -check_python_version() +# check_python_version() setup( From c0e85364bb58e6d9691ae396ebca90d982ff9a6b Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 20 Oct 2016 08:33:43 -0400 Subject: [PATCH 31/48] chg: pytest-cov --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b58d2690..939a3f51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,10 +126,10 @@ before_install: - travis_retry mkvirtualenv scarlett_os - travis_retry bash ./ci/set_postactivate.sh - workon scarlett_os -- travis_retry pip install -U coveralls sphinx numpy ipython +- travis_retry pip install -U coveralls sphinx numpy ipython pytest-cov - travis_retry bash ./ci/travis.sh +# - travis_retry jhbuild run -- pip install -e .[test] - travis_retry jhbuild run python3 setup.py install -- travis_retry jhbuild run -- pip install -e .[test] install: - travis_retry pip install -U coveralls sphinx numpy ipython From 8e3bcadcc1131a6562f1edd69a609081af2ebe94 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 20 Oct 2016 08:57:38 -0400 Subject: [PATCH 32/48] chg: disable ENABLE_PYTHON2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 939a3f51..d064d16f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_install: - export SKIP_ON_TRAVIS=yes - export CURRENT_DIR=$(pwd) - export GSTREAMER=1.0 -- export ENABLE_PYTHON2=yes +# - export ENABLE_PYTHON2=yes - export ENABLE_PYTHON3=yes - export ENABLE_GTK=yes - export PYTHON_VERSION_MAJOR=3 From 1e8ae52e7a1ba3c435709026bae27ebed537875f Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 22 Oct 2016 10:15:26 -0400 Subject: [PATCH 33/48] chg: updated Dockerfile --- Dockerfile | 603 ++++++++++++------ {dotfiles => container/root/dotfiles}/.bashrc | 0 .../root/dotfiles}/.profile | 0 3 files changed, 418 insertions(+), 185 deletions(-) rename {dotfiles => container/root/dotfiles}/.bashrc (100%) rename {dotfiles => container/root/dotfiles}/.profile (100%) diff --git a/Dockerfile b/Dockerfile index 11d3f132..2c7a723c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,216 +1,449 @@ -FROM behance/docker-base:latest +FROM bossjones/boss-docker-python3:latest MAINTAINER Malcolm Jones # Prepare packaging environment ENV DEBIAN_FRONTEND noninteractive -# Workaround for bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=807948 -RUN chmod 0777 /tmp +# # Ensure UTF-8 lang and locale +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +# ENV + +# ln -fs /usr/share/zoneinfo/UTC /etc/localtime + +# ENV \ +# USER=john \ +# RBENV_MRIS="2.3.1 2.2.5 2.1.10" \ +# DEFAULT_MRI=2.1.10 \ +# TIMEZONE="Europe/Stockholm" \ +# EMAIL=john@insane.se \ +# NAME="John Axel Eriksson" \ +# USER_UID=1337 \ +# USER_GID=1337 \ +# GOLANG_VERSION=1.6.2 # ensure local python is preferred over distribution python ENV PATH /usr/local/bin:$PATH # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV USER "pi" +ENV USER_HOME "/home/${USER}" ENV LANG C.UTF-8 +ENV SKIP_ON_TRAVIS yes +ENV CURRENT_DIR $(pwd) +ENV GSTREAMER 1.0 +ENV ENABLE_PYTHON3 yes +ENV ENABLE_GTK yes +ENV PYTHON_VERSION_MAJOR 3 +ENV PYTHON_VERSION 3.5 +ENV CFLAGS "-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer" +ENV MAKEFLAGS "-j4" +ENV PREFIX "${USER_HOME}/jhbuild" +ENV JHBUILD "${USER_HOME}/gnome" +ENV PATH ${PREFIX}/bin:${PREFIX}/sbin:${PATH} +ENV LD_LIBRARY_PATH ${PREFIX}/lib:${LD_LIBRARY_PATH} +ENV PYTHONPATH ${PREFIX}/lib/python${PYTHON_VERSION}/site-packages:/usr/lib/python${PYTHON_VERSION}/site-packages +ENV PKG_CONFIG_PATH ${PREFIX}/lib/pkgconfig:${PREFIX}/share/pkgconfig:/usr/lib/pkgconfig +ENV XDG_DATA_DIRS ${PREFIX}/share:/usr/share +ENV XDG_CONFIG_DIRS ${PREFIX}/etc/xdg +ENV PYTHON "python3" +ENV PACKAGES "python3-gi python3-gi-cairo" +ENV CC gcc + + +# deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted +# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted +# deb http://us.archive.ubuntu.com/ubuntu/ xenial universe +# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe +# deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse +# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse +# deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse +# deb http://security.ubuntu.com/ubuntu xenial-security main restricted +# deb http://security.ubuntu.com/ubuntu xenial-security universe +# deb http://security.ubuntu.com/ubuntu xenial-security multiverse -RUN apt-get update -q && \ +# # Ensure cleanup script is available for the next command +# ADD ./container/root/clean.sh /clean.sh +# +# RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime +# +# RUN ln -sf /usr/share/zoneinfo/$TIMEZONE /etc/localtime && \ +# dpkg-reconfigure -f noninteractive tzdata && \ +# sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ +# sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ +# sed -i -e 's/# sv_SE.UTF-8 UTF-8/sv_SE.UTF-8 UTF-8/' /etc/locale.gen && \ +# echo 'LANG="en_US.UTF-8"' > /etc/default/locale && \ +# dpkg-reconfigure -f noninteractive locales && \ +# update-locale LANG=en_US.UTF-8 &&\ +# apt-get update &&\ +# apt-get upgrade -y -q &&\ +# apt-get dist-upgrade -y -q &&\ + +RUN apt-get update -yqq && \ apt-get upgrade -yqq && \ - apt-get install -yqq \ - git \ - curl \ - wget \ - curl \ - software-properties-common \ - ca-certificates \ - && \ - locale-gen en_US.UTF-8 && export LANG=en_US.UTF-8 && \ - add-apt-repository ppa:git-core/ppa -y && \ - add-apt-repository ppa:ricotz/testing -y && \ - add-apt-repository ppa:gnome3-team/gnome3 -y && \ - add-apt-repository ppa:gnome3-team/gnome3-staging -y && \ - apt-get update && apt-get install -y --no-install-recommends \ - tcl \ - tk \ - wget \ - curl \ - ca-certificates && \ - apt-get update && apt-get install -y --no-install-recommends \ - bzr \ - git \ - mercurial \ - openssh-client \ - subversion \ - procps && \ - apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - automake \ - bzip2 \ - file \ - g++ \ - gcc \ - imagemagick \ - libbz2-dev \ - libc6-dev \ - libcurl4-openssl-dev \ - libdb-dev \ - libevent-dev \ - libffi-dev \ - libgeoip-dev \ - libglib2.0-dev \ - libjpeg-dev \ - libkrb5-dev \ - liblzma-dev \ - libmagickcore-dev \ - libmagickwand-dev \ - libmysqlclient-dev \ - libncurses-dev \ - libpng-dev \ - libpq-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libwebp-dev \ - libxml2-dev \ - libxslt-dev \ - libyaml-dev \ - make \ - patch \ - xz-utils \ - zlib1g-dev \ - bash \ - && rm -rf /var/lib/apt/lists/* - -ENV GPG_KEY 97FC712E4C024BBEA48A61ED3A5CA953F73C700D -ENV PYTHON_VERSION 3.5.2 - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 8.1.2 - -RUN set -ex \ - && buildDeps=' \ - tcl-dev \ - tk-dev \ - ' \ - && apt-get update && apt-get install -y $buildDeps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ - \ - && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ - && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ - && gpg --batch --verify python.tar.xz.asc python.tar.xz \ - && rm -r "$GNUPGHOME" python.tar.xz.asc \ - && mkdir -p /usr/src/python \ - && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ - && rm python.tar.xz \ - \ - && cd /usr/src/python \ - && ./configure \ - --enable-loadable-sqlite-extensions \ - --enable-shared \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - \ -# explicit path to "pip3" to ensure distribution-provided "pip3" cannot interfere - && if [ ! -e /usr/local/bin/pip3 ]; then : \ - && wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \ - && rm /tmp/get-pip.py \ - ; fi \ -# we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python -# ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.6/site-packages") -# https://github.com/docker-library/python/pull/143#issuecomment-241032683 - && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \ -# then we use "pip list" to ensure we don't have more than one pip version installed -# https://github.com/docker-library/python/pull/100 - && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ] \ - \ - && find /usr/local -depth \ - \( \ - \( -type d -a -name test -o -name tests \) \ - -o \ - \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ - \) -exec rm -rf '{}' + \ - && apt-get purge -y --auto-remove $buildDeps \ - && rm -rf /usr/src/python ~/.cache - -# make some useful symlinks that are expected to exist -RUN cd /usr/local/bin \ - && { [ -e easy_install ] || ln -s easy_install-* easy_install; } \ - && ln -s idle3 idle \ - && ln -s pydoc3 pydoc \ - && ln -s python3 python \ - && ln -s python3-config python-config \ - && pip3 install virtualenv virtualenvwrapper ipython numpy tox coveralls - -ENV PYTHON_VERSION_MAJOR '3' -ENV GSTREAMER '1.0' -ENV USER 'pi' + export LANG=en_US.UTF-8 && \ + apt-get install -qq gnome-common \ + gtk-doc-tools \ + libgtk-3-dev \ + libgirepository1.0-dev \ + yelp-tools \ + libgladeui-dev \ + python3-dev \ + python3-cairo-dev \ + python3-gi \ + automake \ + autopoint \ + bison \ + build-essential \ + byacc \ + flex \ + gcc \ + automake \ + autoconf \ + libtool \ + bison \ + swig \ + python-dev \ + libpulse-dev \ + gettext \ + gnome-common \ + gtk-doc-tools \ + libgtk-3-dev \ + libgirepository1.0-dev \ + python3-gi-cairo \ + yasm \ + nasm \ + bison \ + flex \ + libusb-1.0-0-dev \ + libgudev-1.0-dev \ + libxv-dev \ + build-essential \ + autotools-dev \ + automake \ + autoconf \ + libtool \ + autopoint \ + libxml2-dev \ + zlib1g-dev \ + libglib2.0-dev \ + pkg-config \ + flex \ + python \ + libasound2-dev \ + libgudev-1.0-dev \ + libxt-dev \ + libvorbis-dev \ + libcdparanoia-dev \ + libpango1.0-dev \ + libtheora-dev \ + libvisual-0.4-dev \ + iso-codes \ + libgtk-3-dev \ + libraw1394-dev \ + libiec61883-dev \ + libavc1394-dev \ + libv4l-dev \ + libcairo2-dev \ + libcaca-dev \ + libspeex-dev \ + libpng-dev \ + libshout3-dev \ + libjpeg-dev \ + libaa1-dev \ + libflac-dev \ + libdv4-dev \ + libtag1-dev \ + libwavpack-dev \ + libpulse-dev \ + gstreamer1.0* \ + lame \ + flac \ + libfftw3-dev \ + xvfb \ + gir1.2-gtk-3.0 \ + xsltproc \ + docbook-xml \ + docbook-xsl \ + python-libxml2 \ + sudo \ + && \ + apt-get autoclean -y && \ + apt-get autoremove -y && \ + rm -rf /var/lib/{cache,log}/ && \ + rm -rf /var/lib/apt/lists/*.lz4 + +ENV VIRTUALENVWRAPPER_PYTHON '/usr/local/bin/python3' +ENV VIRTUALENVWRAPPER_VIRTUALENV '/usr/local/bin/virtualenv' +ENV VIRTUALENV_WRAPPER_SH '/usr/local/bin/virtualenvwrapper.sh' + +ENV PYTHON_VERSION_MAJOR "3" +ENV GSTREAMER "1.0" +ENV USER "pi" ENV USER_HOME "/home/${USER}" ENV LANGUAGE_ID 1473 ENV GITHUB_BRANCH "master" -ENV GITHUB_REPO_NAME 'scarlett_os' -ENV GITHUB_REPO_ORG 'bossjones' -ENV PI_HOME '/home/pi' -ENV MAIN_DIR '${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}' -ENV VIRT_ROOT '${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}' -ENV PKG_CONFIG_PATH '${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/pkgconfig' -ENV SCARLETT_CONFIG '${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/.scarlett' -ENV SCARLETT_HMM '${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/.virtualenvs/${GITHUB_REPO_NAME}/share/pocketsphinx/model/en-us/en-us' -ENV SCARLETT_LM '${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/lm/${LANGUAGE_ID}.lm' -ENV SCARLETT_DICT '${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/dict/${LANGUAGE_ID}.dic' -ENV LD_LIBRARY_PATH '${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib' -ENV GST_PLUGIN_PATH '${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/gstreamer-${GSTREAMER}' -ENV PYTHON '/usr/local/bin/python3' -ENV PYTHON_VERSION '3.5' -ENV VIRTUALENVWRAPPER_PYTHON '/usr/local/bin/python3' -ENV VIRTUALENVWRAPPER_VIRTUALENV '/usr/local/bin/virtualenv' -ENV VIRTUALENVWRAPPER_SCRIPT '/usr/local/bin/virtualenvwrapper.sh' -ENV PYTHONSTARTUP '${USER_HOME}/.pythonrc' -ENV PIP_DOWNLOAD_CACHE '${USER_HOME}/.pip/cache' -ENV WORKON_HOME '${VIRT_ROOT}' +ENV GITHUB_REPO_NAME "scarlett_os" +ENV GITHUB_REPO_ORG "bossjones" +ENV PI_HOME "/home/pi" +ENV MAIN_DIR "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}" +ENV VIRT_ROOT "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}" +ENV PKG_CONFIG_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/pkgconfig" +ENV SCARLETT_CONFIG "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/.scarlett" +ENV SCARLETT_HMM "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/.virtualenvs/${GITHUB_REPO_NAME}/share/pocketsphinx/model/en-us/en-us" +ENV SCARLETT_LM "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/lm/${LANGUAGE_ID}.lm" +ENV SCARLETT_DICT "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/dict/${LANGUAGE_ID}.dic" +ENV LD_LIBRARY_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib" +ENV GST_PLUGIN_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/gstreamer-${GSTREAMER}" +ENV PYTHON "/usr/local/bin/python3" +ENV PYTHON_VERSION "3.5" +ENV VIRTUALENVWRAPPER_PYTHON "/usr/local/bin/python3" +ENV VIRTUALENVWRAPPER_VIRTUALENV "/usr/local/bin/virtualenv" +ENV VIRTUALENVWRAPPER_SCRIPT "/usr/local/bin/virtualenvwrapper.sh" +ENV PYTHONSTARTUP "${USER_HOME}/.pythonrc" +ENV PIP_DOWNLOAD_CACHE "${USER_HOME}/.pip/cache" +ENV WORKON_HOME "${VIRT_ROOT}" RUN set -xe \ + && useradd -U -d ${PI_HOME} -m -r -G adm,sudo,dip,plugdev,tty,audio ${USER} \ + && usermod -a -G ${USER} ${USER} \ + && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ + && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME} \ && mkdir -p ${MAIN_DIR} \ - && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github - -RUN set -xe \ - && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ && chown -hR ${USER}:${USER} ${MAIN_DIR} \ + && echo 'pi ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ + && echo '%pi ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ + && cat /etc/sudoers -RUN set -xe \ - && useradd -U -d ${PI_HOME} -m -r -G adm,sudo,dip,plugdev,tty,audio ${USER} \ - && usermod -a -G ${USER} ${USER} \ - && echo '${USER} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ - && echo '%${USER} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers - # && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages +USER pi -COPY ./dotfiles/.bashrc /home/pi/.bashrc -COPY ./dotfiles/.profile /home/pi/.profile +WORKDIR /home/pi -RUN set -xe \ - && apt-get autoclean -y \ - && apt-get autoremove -y \ - && rm -rf /tmp/* /var/tmp/* \ - && rm -rf /var/lib/apt/lists/* +# Create a basic .jhbuildrc +RUN echo "import os" > /home/pi/.jhbuildrc && \ + echo "prefix='$PREFIX'" >> /home/pi/.jhbuildrc && \ + echo "checkoutroot='$JHBUILD'" >> /home/pi/.jhbuildrc && \ + echo "moduleset = 'gnome-world'" >> /home/pi/.jhbuildrc && \ + echo "interact = False" >> /home/pi/.jhbuildrc && \ + echo "makeargs = '$MAKEFLAGS'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['CFLAGS'] = '$CFLAGS'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PYTHON'] = 'python$PYTHON_VERSION_MAJOR'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['GSTREAMER'] = '1.0'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['ENABLE_PYTHON3'] = 'yes'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['ENABLE_GTK'] = 'yes'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PYTHON_VERSION'] = '$PYTHON_VERSION'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['CFLAGS'] = '-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['MAKEFLAGS'] = '-j4'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PREFIX'] = '$USER_HOME/jhbuild'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['JHBUILD'] = '$USER_HOME/gnome'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PATH'] = '$PREFIX/bin:$PREFIX/sbin:$PATH'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['LD_LIBRARY_PATH'] = '$PREFIX/lib:$LD_LIBRARY_PATH'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PYTHONPATH'] = '$PREFIX/lib/python$PYTHON_VERSION/site-packages:/usr/lib/python$PYTHON_VERSION/site-packages'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PKG_CONFIG_PATH'] = '$PREFIX/lib/pkgconfig:$PREFIX/share/pkgconfig:/usr/lib/pkgconfig'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['XDG_DATA_DIRS'] = '$PREFIX/share:/usr/share'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['XDG_CONFIG_DIRS'] = '$PREFIX/etc/xdg'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['CC'] = 'gcc'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['WORKON_HOME'] = '$USER_HOME/.virtualenvs'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PROJECT_HOME'] = '$USER_HOME/dev'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['VIRTUALENVWRAPPER_PYTHON'] = '$VIRTUALENVWRAPPER_PYTHON'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['VIRTUALENVWRAPPER_VIRTUALENV'] = '$VIRTUALENVWRAPPER_VIRTUALENV'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PYTHONSTARTUP'] = '$USER_HOME/.pythonrc'" >> /home/pi/.jhbuildrc && \ + echo "os.environ['PIP_DOWNLOAD_CACHE'] = '$USER_HOME/.pip/cache'" >> /home/pi/.jhbuildrc && \ + cat /home/pi/.jhbuildrc -# Layer customizations over existing structure -COPY ./container/root / + # && \ + # mkdir -p /home/pi/jhbuild && \ + # -# Ensure application code makes it into the /app directory -COPY ./ /home/pi/dev/bossjones-github/scarlett_os/ +RUN cat /home/pi/.jhbuildrc -RUN ls -lta /home/pi/dev/bossjones-github/scarlett_os/ && \ - bash -c "source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv --system-site-packages ${GITHUB_REPO_NAME}" +# jhbuild +RUN mkdir -p /home/pi/gnome && \ -COPY ./postactivate /home/pi/.virtualenvs/scarlett_os/postactivate + echo "****************[JHBUILD]****************" && \ + cd /home/pi && \ + if test ! -d /home/pi/jhbuild; then git clone https://github.com/GNOME/jhbuild.git && \ + cd jhbuild; else echo "exists" && cd jhbuild; fi && \ + git checkout 86d958b6778da649b559815c0a0dbe6a5d1a8cd4 && \ + ./autogen.sh --prefix=/usr/local > /dev/null && \ + make > /dev/null && \ + sudo make install > /dev/null && \ + sudo chown pi:pi -R /usr/local/ && \ + chown pi:pi -R /home/pi/jhbuild && \ -RUN set -xe \ - && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ - && chown -hR ${USER}:${USER} ${MAIN_DIR} \ - && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages \ - && chown -R ${USER}:${USER} /home/pi/.virtualenvs + echo "****************[GLIB]****************" && \ + cd /home/pi/gnome && \ + git clone https://github.com/GNOME/glib.git && \ + cd glib && \ + git checkout eaca4f4116801f99e30e42a857559e19a1e6f4ce && \ + jhbuild buildone -n glib && \ + + echo "****************[GOBJECT-INTROSPECTION]****************" && \ + cd /home/pi/gnome && \ + git clone https://github.com/GNOME/gobject-introspection.git && \ + cd gobject-introspection && \ + git checkout cee2a4f215d5edf2e27b9964d3cfcb28a9d4941c && \ + jhbuild buildone -n gobject-introspection && \ + + echo "****************[PYGOBJECT]****************" && \ + cd /home/pi/gnome && \ + git clone https://github.com/GNOME/pygobject.git && \ + cd /home/pi/gnome && \ + cd pygobject && \ + git checkout fb1b8fa8a67f2c7ea7ad4b53076496a8f2b4afdb && \ + jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild --with-python=python3 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GSTREAMER]****************" && \ + cd /home/pi/gnome && \ + curl -L "https://gstreamer.freedesktop.org/src/gstreamer/gstreamer-1.8.2.tar.xz" > gstreamer-1.8.2.tar.xz && \ + tar -xJf gstreamer-1.8.2.tar.xz && \ + cd gstreamer-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[ORC]****************" && \ + cd /home/pi/gnome && \ + curl -L "https://gstreamer.freedesktop.org/src/orc/orc-0.4.25.tar.xz" > orc-0.4.25.tar.xz && \ + tar -xJf orc-0.4.25.tar.xz && \ + cd orc-0.4.25 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-PLUGINS-BASE]****************" && \ + cd /home/pi/gnome && \ + curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-base/gst-plugins-base-1.8.2.tar.xz" > gst-plugins-base-1.8.2.tar.xz && \ + tar -xJf gst-plugins-base-1.8.2.tar.xz && \ + cd gst-plugins-base-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc --with-x > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-PLUGINS-GOOD]****************" && \ + cd /home/pi/gnome && \ + curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-good/gst-plugins-good-1.8.2.tar.xz" > gst-plugins-good-1.8.2.tar.xz && \ + tar -xJf gst-plugins-good-1.8.2.tar.xz && \ + cd gst-plugins-good-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc --with-libv4l2 --with-x > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-PLUGINS-UGLY]****************" && \ + cd /home/pi/gnome && \ + curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-ugly/gst-plugins-ugly-1.8.2.tar.xz" > gst-plugins-ugly-1.8.2.tar.xz && \ + tar -xJf gst-plugins-ugly-1.8.2.tar.xz && \ + cd gst-plugins-ugly-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-PLUGINS-BAD]****************" && \ + cd /home/pi/gnome && \ + curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-bad/gst-plugins-bad-1.8.2.tar.xz" > gst-plugins-bad-1.8.2.tar.xz && \ + tar -xJf gst-plugins-bad-1.8.2.tar.xz && \ + cd gst-plugins-bad-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-LIBAV]****************" && \ + cd /home/pi/gnome && \ + curl -L "http://gstreamer.freedesktop.org/src/gst-libav/gst-libav-1.8.2.tar.xz" > gst-libav-1.8.2.tar.xz && \ + tar -xJf gst-libav-1.8.2.tar.xz && \ + cd gst-libav-1.8.2 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ + jhbuild run make -j4 > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[GST-PLUGINS-ESPEAK]****************" && \ + cd $JHBUILD && \ + curl -L "https://github.com/bossjones/bossjones-gst-plugins-espeak-0-4-0/archive/v0.4.1.tar.gz" > gst-plugins-espeak-0.4.0.tar.gz && \ + tar xvf gst-plugins-espeak-0.4.0.tar.gz && \ + rm -rfv gst-plugins-espeak-0.4.0 && \ + mv -fv bossjones-gst-plugins-espeak-0-4-0-0.4.1 gst-plugins-espeak-0.4.0 && \ + cd gst-plugins-espeak-0.4.0 && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run make > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[SPHINXBASE]****************" && \ + cd $JHBUILD && \ + git clone https://github.com/cmusphinx/sphinxbase.git && \ + cd sphinxbase && \ + git checkout 74370799d5b53afc5b5b94a22f5eff9cb9907b97 && \ + cd $JHBUILD/sphinxbase && \ + jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run make clean all > /dev/null && \ + jhbuild run make install > /dev/null && \ + + echo "****************[POCKETSPHINX]****************" && \ + cd $JHBUILD && \ + git clone https://github.com/cmusphinx/pocketsphinx.git && \ + cd pocketsphinx && \ + git checkout 68ef5dc6d48d791a747026cd43cc6940a9e19f69 && \ + jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ + jhbuild run make clean all > /dev/null && \ + jhbuild run make install > /dev/null + +# RUN set -xe \ +# && useradd -d /var/lib/mediagoblin -m -r -g www-data mediagoblin \ +# && groupadd -r mediagoblin \ +# && usermod -a -G mediagoblin mediagoblin \ +# && mkdir -p /var/log/mediagoblin \ +# /srv/mediagoblin.example.org/mediagoblin \ +# && chown -hR mediagoblin:www-data /var/log/mediagoblin \ +# /srv/mediagoblin.example.org \ +# && echo 'mediagoblin ALL=(ALL:ALL) NOPASSWD: /usr/sbin/nginx, /bin/chown' >> /etc/sudoers +# +# USER mediagoblin +# +# WORKDIR /srv/mediagoblin.example.org/mediagoblin +# +# RUN set -xe \ +# && git clone http://git.savannah.gnu.org/r/mediagoblin.git . \ +# && git checkout stable \ +# && git submodule sync \ +# && git submodule update --force --init --recursive \ +# && ./bootstrap.sh \ +# && ./configure \ +# && make \ +# && bin/easy_install flup==1.0.3.dev-20110405 \ +# && bin/pip install scikits.audiolab \ +# && ln -s /var/lib/mediagoblin user_dev \ +# && cp -av mediagoblin.ini mediagoblin_local.ini \ +# && cp -av paste.ini paste_local.ini \ +# && sed -i 's@.*sql_engine = .*@sql_engine = sqlite:////var/lib/mediagoblin/mediagoblin.db@' mediagoblin_local.ini \ +# && { \ +# echo '[[mediagoblin.media_types.video]]'; \ +# echo '[[mediagoblin.media_types.audio]]'; \ +# echo '[[mediagoblin.media_types.pdf]]'; \ +# } >> mediagoblin_local.ini + +# COPY ./dotfiles/.bashrc /home/pi/.bashrc +# COPY ./dotfiles/.profile /home/pi/.profile + + +###### # Layer customizations over existing structure +###### COPY ./container/root / +###### +###### # Ensure application code makes it into the /app directory +###### COPY ./ /home/pi/dev/bossjones-github/scarlett_os/ +###### +###### RUN ls -lta /home/pi/dev/bossjones-github/scarlett_os/ && \ +###### bash -c "source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv --system-site-packages ${GITHUB_REPO_NAME}" +###### +###### COPY ./postactivate /home/pi/.virtualenvs/scarlett_os/postactivate +###### +###### RUN set -xe \ +###### && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ +###### && chown -hR ${USER}:${USER} ${MAIN_DIR} \ +###### && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages \ +###### && chown -R ${USER}:${USER} /home/pi/.virtualenvs ENTRYPOINT ["file","/bin/bash"] diff --git a/dotfiles/.bashrc b/container/root/dotfiles/.bashrc similarity index 100% rename from dotfiles/.bashrc rename to container/root/dotfiles/.bashrc diff --git a/dotfiles/.profile b/container/root/dotfiles/.profile similarity index 100% rename from dotfiles/.profile rename to container/root/dotfiles/.profile From 8f8accf7745358673dbd7bf182ba3fbfc1261eb4 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Mon, 24 Oct 2016 12:33:43 -0400 Subject: [PATCH 34/48] WOOT: Dockerfile workssssss --- Dockerfile | 81 ++++++++++++++++++++++++++++++++--------- container/root/clean.sh | 11 ++++++ 2 files changed, 75 insertions(+), 17 deletions(-) create mode 100755 container/root/clean.sh diff --git a/Dockerfile b/Dockerfile index 2c7a723c..f137dd02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ ENV LC_ALL en_US.UTF-8 # GOLANG_VERSION=1.6.2 # ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:$PATH +ENV PATH /usr/local/bin:/usr/local/sbin:$PATH # http://bugs.python.org/issue19846 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. @@ -52,18 +52,6 @@ ENV PYTHON "python3" ENV PACKAGES "python3-gi python3-gi-cairo" ENV CC gcc - -# deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted -# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted -# deb http://us.archive.ubuntu.com/ubuntu/ xenial universe -# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe -# deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse -# deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse -# deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse -# deb http://security.ubuntu.com/ubuntu xenial-security main restricted -# deb http://security.ubuntu.com/ubuntu xenial-security universe -# deb http://security.ubuntu.com/ubuntu xenial-security multiverse - # # Ensure cleanup script is available for the next command # ADD ./container/root/clean.sh /clean.sh # @@ -81,10 +69,45 @@ ENV CC gcc # apt-get upgrade -y -q &&\ # apt-get dist-upgrade -y -q &&\ -RUN apt-get update -yqq && \ +RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata && \ + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + echo 'LANG="en_US.UTF-8"' > /etc/default/locale && \ + dpkg-reconfigure -f noninteractive locales && \ + update-locale LANG=en_US.UTF-8 && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted' | tee /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial universe' | tee -a /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial universe' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe' | tee -a /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-security main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-security main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-security universe' | tee -a /etc/apt/sources.list && \ + echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-security universe' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse' | tee -a /etc/apt/sources.list && \ + echo 'deb http://security.ubuntu.com/ubuntu xenial-security main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb http://security.ubuntu.com/ubuntu xenial-security main restricted' | tee -a /etc/apt/sources.list && \ + echo 'deb http://security.ubuntu.com/ubuntu xenial-security universe' | tee -a /etc/apt/sources.list && \ + echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse' | tee -a /etc/apt/sources.list && \ + echo 'deb http://security.ubuntu.com/ubuntu xenial-security multiverse' | tee -a /etc/apt/sources.list && \ + cat /etc/apt/sources.list | grep -v "^#" | sort -u > /etc/apt/sources.list.bak && \ + mv -fv /etc/apt/sources.list.bak /etc/apt/sources.list && \ + add-apt-repository -y ppa:ricotz/testing && \ + add-apt-repository -y ppa:gnome3-team/gnome3 && \ + add-apt-repository -y ppa:gnome3-team/gnome3-staging && \ + add-apt-repository -y ppa:pitti/systemd-semaphore && \ + apt-get update -yqq && \ apt-get upgrade -yqq && \ export LANG=en_US.UTF-8 && \ - apt-get install -qq gnome-common \ + apt-get install -qqy libpulse-dev espeak && \ + apt-cache search --names-only '^(lib)?gstreamer1.0\S*' | sed 's/\(.*\) -.*/\1 /' | grep -iv "Speech" > dependencies && \ + cat dependencies && \ + apt-get build-dep -y `cat dependencies` && \ + apt-get install -qqy gnome-common \ gtk-doc-tools \ libgtk-3-dev \ libgirepository1.0-dev \ @@ -125,6 +148,7 @@ RUN apt-get update -yqq && \ automake \ autoconf \ libtool \ + binutils \ autopoint \ libxml2-dev \ zlib1g-dev \ @@ -169,11 +193,21 @@ RUN apt-get update -yqq && \ docbook-xsl \ python-libxml2 \ sudo \ - && \ + # begin - gst-plugins-bad req + libqt4-opengl \ + libdvdread4 \ + libdvdnav4 \ + libllvm3.8 \ + libsoundtouch-dev \ + libsoundtouch1 \ + # end gst-plugins-bad req + ubuntu-restricted-extras && \ + apt-get clean && \ apt-get autoclean -y && \ apt-get autoremove -y && \ rm -rf /var/lib/{cache,log}/ && \ - rm -rf /var/lib/apt/lists/*.lz4 + rm -rf /var/lib/apt/lists/*.lz4 /tmp/* /var/tmp/* + ENV VIRTUALENVWRAPPER_PYTHON '/usr/local/bin/python3' ENV VIRTUALENVWRAPPER_VIRTUALENV '/usr/local/bin/virtualenv' @@ -273,6 +307,11 @@ RUN mkdir -p /home/pi/gnome && \ sudo chown pi:pi -R /usr/local/ && \ chown pi:pi -R /home/pi/jhbuild && \ + echo "****************[GTK-DOC]****************" && \ + cd /home/pi/gnome && \ + git clone https://github.com/GNOME/gtk-doc.git && \ + jhbuild buildone -n gtk-doc && \ + echo "****************[GLIB]****************" && \ cd /home/pi/gnome && \ git clone https://github.com/GNOME/glib.git && \ @@ -342,6 +381,9 @@ RUN mkdir -p /home/pi/gnome && \ jhbuild run make install > /dev/null && \ echo "****************[GST-PLUGINS-BAD]****************" && \ + cat /home/pi/jhbuild/bin/gdbus-codegen && \ + sed -i "s,#!python3,#!/usr/bin/python3,g" /home/pi/jhbuild/bin/gdbus-codegen && \ + cat /home/pi/jhbuild/bin/gdbus-codegen && \ cd /home/pi/gnome && \ curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-bad/gst-plugins-bad-1.8.2.tar.xz" > gst-plugins-bad-1.8.2.tar.xz && \ tar -xJf gst-plugins-bad-1.8.2.tar.xz && \ @@ -428,6 +470,11 @@ RUN mkdir -p /home/pi/gnome && \ # COPY ./dotfiles/.bashrc /home/pi/.bashrc # COPY ./dotfiles/.profile /home/pi/.profile +# RUN set -xe \ +# && apt-get autoclean -y \ +# && apt-get autoremove -y \ +# && rm -rf /tmp/* /var/tmp/* \ +# && rm -rf /var/lib/apt/lists/* ###### # Layer customizations over existing structure ###### COPY ./container/root / diff --git a/container/root/clean.sh b/container/root/clean.sh new file mode 100755 index 00000000..e9286e5f --- /dev/null +++ b/container/root/clean.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +######################################################################## +# Performs cleanup, ensure unnecessary packages are removed +######################################################################## +# `apt-mark showauto` for any additional installed packages + +apt-get autoclean -y && \ +apt-get autoremove -y && \ +rm -rf /var/lib/{cache,log}/ && \ +rm -rf /var/lib/apt/lists/*.lz4 From 50c8c94bb49620f301856ac93f0e7cc47b8a5404 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 20:50:56 -0400 Subject: [PATCH 35/48] Update: docker travis build --- .dockerignore | 28 + .travis.yml | 148 ++---- .travis_non_docker.yml | 161 ++++++ Dockerfile | 502 +----------------- container/root/docker_entrypoint.sh | 16 + container/root/etc/services.d/scarlett_os/run | 2 +- docker-compose.test.yml | 132 +++++ docker-compose.yml | 15 +- tests/test_scarlett_os.py | 10 +- 9 files changed, 398 insertions(+), 616 deletions(-) create mode 100644 .dockerignore create mode 100644 .travis_non_docker.yml create mode 100755 container/root/docker_entrypoint.sh create mode 100644 docker-compose.test.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..21d13848 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +.git/* +vendor/* +node_modules/* + +*.git +*.pyc +*.vagrant +*.vendor +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +.tox/ +.bundle/ +.cache/ +__pycache__/ diff --git a/.travis.yml b/.travis.yml index d064d16f..9c5ef6d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,14 @@ matrix: fast_finish: true include: + # - python: '3.5' + # env: TOXENV=typing PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" DOCKER_COMPOSE_VERSION=1.8.0 TEST_TARGET=typing DOCKER_DATA="$HOME/docker_data" DOCKER_VERSION=1.12.0-0~trusty DOCKER_COMPOSE_VERSION=1.8.0 + # sudo: required + # dist: trusty + # language: python + # group: edge - python: '3.5' - env: TOXENV=typing PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" - sudo: required - dist: trusty - language: python - group: edge - - python: '3.5' - env: TOXENV=py35 PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" + env: TOXENV=py35 PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" DOCKER_COMPOSE_VERSION=1.8.0 TEST_TARGET=default DOCKER_DATA="$HOME/docker_data" DOCKER_VERSION=1.12.0-0~trusty DOCKER_COMPOSE_VERSION=1.8.0 sudo: required dist: trusty language: python @@ -17,6 +17,22 @@ matrix: - python: '3.5' env: TOXENV=typing +services: + - docker + +before_install: + - travis_retry pip install coveralls + # List available docker versions. + - apt-cache madison docker-engine + # Update Docker. See: https://graysonkoonce.com/managing-docker-and-docker-compose-versions-on-travis-ci/. + - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y docker-engine=${DOCKER_VERSION} + # Add docker-compose at the version specified in ENV. + - sudo rm -f /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + - docker-compose --version + cache: apt: true directories: @@ -26,121 +42,19 @@ cache: - "$HOME/jhbuild" - "$HOME/gnome" - -before_install: -- export SKIP_ON_TRAVIS=yes -- export CURRENT_DIR=$(pwd) -- export GSTREAMER=1.0 -# - export ENABLE_PYTHON2=yes -- export ENABLE_PYTHON3=yes -- export ENABLE_GTK=yes -- export PYTHON_VERSION_MAJOR=3 -- export PYTHON_VERSION=3.5 -- export CFLAGS="-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer" -- export MAKEFLAGS="-j4" -- export PREFIX="${HOME}/jhbuild" -- export JHBUILD="${HOME}/gnome" -- export PATH=${PREFIX}/bin:${PREFIX}/sbin:${PATH} -- export LD_LIBRARY_PATH=${PREFIX}/lib:${LD_LIBRARY_PATH} -- export PYTHONPATH=${PREFIX}/lib/python${PYTHON_VERSION}/site-packages:/usr/lib/python${PYTHON_VERSION}/site-packages -- export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig:${PREFIX}/share/pkgconfig:/usr/lib/pkgconfig -- export XDG_DATA_DIRS=${PREFIX}/share:/usr/share -- export XDG_CONFIG_DIRS=${PREFIX}/etc/xdg -- export PYTHON="python3" -- export PACKAGES="python3-gi python3-gi-cairo" -- export CC=gcc -- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:ricotz/testing; - fi -- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:gnome3-team/gnome3; - fi -- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:gnome3-team/gnome3-staging; - fi -- sudo add-apt-repository -y ppa:pitti/systemd-semaphore -- travis_retry sudo apt-get update -qq -- travis_retry sudo apt-get update -qq -- travis_retry sudo apt-get install -y libz-dev libbz2-dev gstreamer$GSTREAMER-tools libgstreamer$GSTREAMER-dev - libgstreamer-plugins-base$GSTREAMER-dev libgstreamer-plugins-bad$GSTREAMER-dev -- if [ '$GSTREAMER' = '1.0' ]; then travis_retry sudo apt-get install -y libgstreamer-plugins-good$GSTREAMER-dev; - fi -- travis_retry sudo apt-get update -qq -- travis_retry sudo apt-get install -y gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 graphviz-dev - gstreamer1.0-plugins-good gstreamer1.0-plugins-bad python-gst-1.0 -- travis_retry sudo apt-get install -qq python3-gi -- travis_retry sudo apt-get install -qq python-gst-1.0 gstreamer1.0-plugins-base gstreamer1.0-plugins-bad - gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-libav libsndfile1-dev - libasound2-dev libgstreamer-plugins-base1.0-dev python-numpy python-scipy -- travis_retry sudo apt-get -y install automake gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 - gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good - gstreamer1.0-plugins-ugly gstreamer1.0-tools libasound2-dev libgstreamer-plugins-base1.0-dev - libsndfile1-dev python python-dev python-gi python-gst-1.0 python-gst-1.0 python-imaging - python-lxml python-numpy python-scipy python-virtualenv python3-gi -- travis_retry sudo apt-get -y install libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev - gnome-common; -- travis_retry sudo apt-get -y install libgstreamer1.0-dev gstreamer1.0-alsa gstreamer1.0-plugins-base; -- travis_retry sudo apt-get -y install python-dev libsmpeg-dev libswscale-dev libavformat-dev libavcodec-dev - libjpeg-dev libtiff4-dev libX11-dev libmtdev-dev; -- travis_retry sudo apt-get -y install python-setuptools build-essential libgl1-mesa-dev libgles2-mesa-dev; -- travis_retry sudo apt-get -y install xvfb pulseaudio; -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ - trusty main restricted universe multiverse'; fi -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ - trusty-updates main restricted universe multiverse'; fi -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then travis_retry sudo apt-get update -qq; travis_retry sudo apt-get install - -qq pkg-config; fi -- travis_retry sudo apt-get -y update -- travis_retry sudo apt-get -y install ubuntu-restricted-extras -- travis_retry sudo apt-get -y install libfftw3-dev -- travis_retry sudo apt-get install -qq python3-dev python3-cairo-dev python3-gi gnome-common gtk-doc-tools - libgtk-3-dev libgirepository1.0-dev libmount-dev cvs; -- travis_retry sudo apt-get update -q -- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) -- travis_retry sudo apt-get install -qq build-essential git -- travis_retry sudo apt-get install -qq gettext xsltproc docbook-xml docbook-xsl -- travis_retry sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 - byacc flex apt-file autopoint -- travis_retry sudo apt-get install gcc automake autoconf libtool bison swig python-dev libpulse-dev -y -# NOTE: We only need this for sysdeps -# - travis_retry sudo apt-get install -qq apt-file -# - sudo apt-file update -# ubuntu 16.04 says: -# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'Ubuntu', '16.04', 'xenial'] -# travis says -# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'debian', 'jessie', 'sid'] -- python -c "import platform;print(platform.platform().split('-'))" -- travis_retry sudo apt-get install wget -qq -- pip install virtualenv virtualenvwrapper -- sudo pip install --upgrade virtualenvwrapper -- export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh` -- export VIRTUALENVWRAPPER_PYTHON=`which python3` -- export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv` -- pip install -I path.py==7.7.1 -- export WORKON_HOME=${HOME}/.virtualenvs -- export PROJECT_HOME=${HOME}/dev -- export VIRTUALENVWRAPPER_PYTHON=`which python3` -- export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv` -- env -- travis_retry source $VIRTUALENV_WRAPPER_SH -- export PYTHONSTARTUP=$HOME/.pythonrc -- export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache -- mkdir -p $WORKON_HOME -- travis_retry mkvirtualenv scarlett_os -- travis_retry bash ./ci/set_postactivate.sh -- workon scarlett_os -- travis_retry pip install -U coveralls sphinx numpy ipython pytest-cov -- travis_retry bash ./ci/travis.sh -# - travis_retry jhbuild run -- pip install -e .[test] -- travis_retry jhbuild run python3 setup.py install - install: -- travis_retry pip install -U coveralls sphinx numpy ipython -- travis_retry sudo apt-get update -q -- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) + - travis_retry docker-compose pull + - travis_retry docker-compose -f docker-compose.yml -f ci/build.yml build script: -- jhbuild run -- coverage run -- setup.py test + - "docker-compose -f docker-compose.test.yml run --rm ${TEST_TARGET}" after_success: -- coveralls + - mv .coverage .coverage.tests + - coverage combine + # Coverage report contains Docker paths. We replace them, so that we can run Coveralls. + - sed -i 's@\"/home/pi/dev/bossjones-github/scarlett_os/@'"\"$(pwd)/"'@g' .coverage + - coveralls deploy: distributions: sdist bdist_wheel diff --git a/.travis_non_docker.yml b/.travis_non_docker.yml new file mode 100644 index 00000000..d064d16f --- /dev/null +++ b/.travis_non_docker.yml @@ -0,0 +1,161 @@ +matrix: + fast_finish: true + include: + - python: '3.5' + env: TOXENV=typing PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" + sudo: required + dist: trusty + language: python + group: edge + - python: '3.5' + env: TOXENV=py35 PYTHON="python3" PACKAGES="python3-gi python3-gi-cairo" + sudo: required + dist: trusty + language: python + group: edge + allow_failures: + - python: '3.5' + env: TOXENV=typing + +cache: + apt: true + directories: + - "$HOME/.cache/pip" + - "$HOME/virtualenv/python3.5.2/lib/python3.5/site-packages" + - "$HOME/.ccache" + - "$HOME/jhbuild" + - "$HOME/gnome" + + +before_install: +- export SKIP_ON_TRAVIS=yes +- export CURRENT_DIR=$(pwd) +- export GSTREAMER=1.0 +# - export ENABLE_PYTHON2=yes +- export ENABLE_PYTHON3=yes +- export ENABLE_GTK=yes +- export PYTHON_VERSION_MAJOR=3 +- export PYTHON_VERSION=3.5 +- export CFLAGS="-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer" +- export MAKEFLAGS="-j4" +- export PREFIX="${HOME}/jhbuild" +- export JHBUILD="${HOME}/gnome" +- export PATH=${PREFIX}/bin:${PREFIX}/sbin:${PATH} +- export LD_LIBRARY_PATH=${PREFIX}/lib:${LD_LIBRARY_PATH} +- export PYTHONPATH=${PREFIX}/lib/python${PYTHON_VERSION}/site-packages:/usr/lib/python${PYTHON_VERSION}/site-packages +- export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig:${PREFIX}/share/pkgconfig:/usr/lib/pkgconfig +- export XDG_DATA_DIRS=${PREFIX}/share:/usr/share +- export XDG_CONFIG_DIRS=${PREFIX}/etc/xdg +- export PYTHON="python3" +- export PACKAGES="python3-gi python3-gi-cairo" +- export CC=gcc +- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:ricotz/testing; + fi +- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:gnome3-team/gnome3; + fi +- if [ '$GSTREAMER' = '1.0' ]; then sudo add-apt-repository -y ppa:gnome3-team/gnome3-staging; + fi +- sudo add-apt-repository -y ppa:pitti/systemd-semaphore +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get install -y libz-dev libbz2-dev gstreamer$GSTREAMER-tools libgstreamer$GSTREAMER-dev + libgstreamer-plugins-base$GSTREAMER-dev libgstreamer-plugins-bad$GSTREAMER-dev +- if [ '$GSTREAMER' = '1.0' ]; then travis_retry sudo apt-get install -y libgstreamer-plugins-good$GSTREAMER-dev; + fi +- travis_retry sudo apt-get update -qq +- travis_retry sudo apt-get install -y gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 graphviz-dev + gstreamer1.0-plugins-good gstreamer1.0-plugins-bad python-gst-1.0 +- travis_retry sudo apt-get install -qq python3-gi +- travis_retry sudo apt-get install -qq python-gst-1.0 gstreamer1.0-plugins-base gstreamer1.0-plugins-bad + gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-libav libsndfile1-dev + libasound2-dev libgstreamer-plugins-base1.0-dev python-numpy python-scipy +- travis_retry sudo apt-get -y install automake gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 + gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good + gstreamer1.0-plugins-ugly gstreamer1.0-tools libasound2-dev libgstreamer-plugins-base1.0-dev + libsndfile1-dev python python-dev python-gi python-gst-1.0 python-gst-1.0 python-imaging + python-lxml python-numpy python-scipy python-virtualenv python3-gi +- travis_retry sudo apt-get -y install libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev + gnome-common; +- travis_retry sudo apt-get -y install libgstreamer1.0-dev gstreamer1.0-alsa gstreamer1.0-plugins-base; +- travis_retry sudo apt-get -y install python-dev libsmpeg-dev libswscale-dev libavformat-dev libavcodec-dev + libjpeg-dev libtiff4-dev libX11-dev libmtdev-dev; +- travis_retry sudo apt-get -y install python-setuptools build-essential libgl1-mesa-dev libgles2-mesa-dev; +- travis_retry sudo apt-get -y install xvfb pulseaudio; +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ + trusty main restricted universe multiverse'; fi +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo add-apt-repository 'deb http://us.archive.ubuntu.com/ubuntu/ + trusty-updates main restricted universe multiverse'; fi +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then travis_retry sudo apt-get update -qq; travis_retry sudo apt-get install + -qq pkg-config; fi +- travis_retry sudo apt-get -y update +- travis_retry sudo apt-get -y install ubuntu-restricted-extras +- travis_retry sudo apt-get -y install libfftw3-dev +- travis_retry sudo apt-get install -qq python3-dev python3-cairo-dev python3-gi gnome-common gtk-doc-tools + libgtk-3-dev libgirepository1.0-dev libmount-dev cvs; +- travis_retry sudo apt-get update -q +- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) +- travis_retry sudo apt-get install -qq build-essential git +- travis_retry sudo apt-get install -qq gettext xsltproc docbook-xml docbook-xsl +- travis_retry sudo apt-get install -qq bison docbook-xsl build-essential git-core python-libxml2 + byacc flex apt-file autopoint +- travis_retry sudo apt-get install gcc automake autoconf libtool bison swig python-dev libpulse-dev -y +# NOTE: We only need this for sysdeps +# - travis_retry sudo apt-get install -qq apt-file +# - sudo apt-file update +# ubuntu 16.04 says: +# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'Ubuntu', '16.04', 'xenial'] +# travis says +# ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'debian', 'jessie', 'sid'] +- python -c "import platform;print(platform.platform().split('-'))" +- travis_retry sudo apt-get install wget -qq +- pip install virtualenv virtualenvwrapper +- sudo pip install --upgrade virtualenvwrapper +- export VIRTUALENV_WRAPPER_SH=`which virtualenvwrapper.sh` +- export VIRTUALENVWRAPPER_PYTHON=`which python3` +- export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv` +- pip install -I path.py==7.7.1 +- export WORKON_HOME=${HOME}/.virtualenvs +- export PROJECT_HOME=${HOME}/dev +- export VIRTUALENVWRAPPER_PYTHON=`which python3` +- export VIRTUALENVWRAPPER_VIRTUALENV=`which virtualenv` +- env +- travis_retry source $VIRTUALENV_WRAPPER_SH +- export PYTHONSTARTUP=$HOME/.pythonrc +- export PIP_DOWNLOAD_CACHE=$HOME/.pip/cache +- mkdir -p $WORKON_HOME +- travis_retry mkvirtualenv scarlett_os +- travis_retry bash ./ci/set_postactivate.sh +- workon scarlett_os +- travis_retry pip install -U coveralls sphinx numpy ipython pytest-cov +- travis_retry bash ./ci/travis.sh +# - travis_retry jhbuild run -- pip install -e .[test] +- travis_retry jhbuild run python3 setup.py install + +install: +- travis_retry pip install -U coveralls sphinx numpy ipython +- travis_retry sudo apt-get update -q +- travis_retry sudo apt-get install --no-install-recommends -y xvfb gir1.2-gtk-3.0 $(echo $PACKAGES) + +script: +- jhbuild run -- coverage run -- setup.py test + +after_success: +- coveralls + +deploy: + distributions: sdist bdist_wheel + provider: pypi + user: bossjones + password: + secure: RTwOAJzBeEXy1as+a3sutcSjNTfoSg15ep8tHIjOqK0etkThWXNFs0FvWNuChV3zsFhA40sMpAamHvcGXJ717GAYIPstIrgAusLKM7OIUXpUF6kqvZsRQw9RPS4vBmueBoBo+pLY6q9gIP+Bav+fFwWocSyjwHLGcaOVdOF/m2VbWEg3p+SGwVolguSpWBNobLDmwj8yd/qiZ6b2LvTrOVkUENfa8Zk8BSXerZiTe98I2c/KuGqkVird+3LulK0kDU1FENO9v5eh+wZzElgbVSGfy87gkdc5whYIz4oA794Gc0jt+YiP2k0pNPRGAcrAn5MRjCdvWQtmkbxdns/JZOeK59SrhcQ6BRfDNQC97MPEZAz/E3qhiDPxwbhuK+LsdIiRHnazwv4MuUU27TcqwtTWcYijzT0aifiQUW6hLwpmVMx6GJy8dvhh+Ursr52jaAaFVx31NUB96+Wa204LCZV1CFo8vKgqDxTjZBfEK0zepQUh06dRdMJaeAN/dhsupMjnEfVY5GSv+ySKkJJ2r77mngyVO3snMs6x2sdH+PjfMnOEmQUcpnaNCQF01RxxfUz+7TY0JonM4yb7dnc+T402G+PhEDKGi5/O1YCIHwWQcMxhv2rMiYSlUt3ikcxkW8YGhhRvEUX1rGyAtPWBanyJ+KMb3QWv7huKE7Ezc/E= + on: + condition: "$TOXENV == py27" + repo: bossjones/scarlett_os + tags: true + +notifications: + slack: + on_success: change + on_failure: always + rooms: + secure: arS7e7IDG80ukrrjy4fVfNDh5Yqrz4mxmduWqJU2zHzwxL0QkJWnhncWBpTgXr06MVgWYwq/YnoCGZ+wcWHkYzsqDHWtFI0JguZuJcFAWV5l7PWOxcUUh731hipRcgkZIH03pqnfZTbN3kkNTvFcTL9NG92ZOYd/ZSQeCnNPF9T2PUjn5LnPNwOxJSVAdmvz7CoLJWIR6vF9ojoxZdMj1RhV4b1C9orP3pBh3V3sVc5ExX6cUePXJtkvb51fXwdCsGNA22xZrDfbxCoOXfMmgDuN/7W/DVPPyaZCInYQl2B/oLF+3TOqUZ8STM3BPDIBjxgPRBdULIqeaQJNNJ9rfQ6h+YhvCWDmZD5ugdTQkGCSX2hR4TGMnS9v/nu5KKSalQEkNOfcqZUSTRVHKL800arEwja2avRGu9KBlijg79r/k2EILb7sPfleeS0+eLKIYTBVyQvuiqo31NcZ4Yun9ykQlTJW0n1NiVrLRVHbCIScrHwyH8pI4ts6PbwtUO8HWWDXEUSZRipc14ckZBzqB6ozAAwtF4fStnHC5rkd1sRreZalLKsZxaVpsX0mspZfSLfz6eV34vnDs9Xv0+tPuW0X0P1/FZIykvjDrmoM8s5TD1xzES9zRslhWxK2yv0a5OaryWvicQKcCwg8zwdiCTAwq1wBNkYzrtTnTN9QnRA= diff --git a/Dockerfile b/Dockerfile index f137dd02..c2de57dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,496 +1,18 @@ -FROM bossjones/boss-docker-python3:latest +FROM bossjones/boss-docker-jhbuild-pygobject3:v1 MAINTAINER Malcolm Jones -# Prepare packaging environment -ENV DEBIAN_FRONTEND noninteractive +COPY ./ /home/pi/dev/bossjones-github/scarlett_os -# # Ensure UTF-8 lang and locale -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -# ENV +WORKDIR /home/pi/dev/bossjones-github/scarlett_os -# ln -fs /usr/share/zoneinfo/UTC /etc/localtime +RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ + && pwd \ + && jhbuild run python3 setup.py install \ + && jhbuild run -- pip install -e .[test] \ + && jhbuild run -- coverage run -- setup.py test \ + && jhbuild run -- pip install coveralls -# ENV \ -# USER=john \ -# RBENV_MRIS="2.3.1 2.2.5 2.1.10" \ -# DEFAULT_MRI=2.1.10 \ -# TIMEZONE="Europe/Stockholm" \ -# EMAIL=john@insane.se \ -# NAME="John Axel Eriksson" \ -# USER_UID=1337 \ -# USER_GID=1337 \ -# GOLANG_VERSION=1.6.2 +COPY ./container/root / -# ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:/usr/local/sbin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV USER "pi" -ENV USER_HOME "/home/${USER}" -ENV LANG C.UTF-8 -ENV SKIP_ON_TRAVIS yes -ENV CURRENT_DIR $(pwd) -ENV GSTREAMER 1.0 -ENV ENABLE_PYTHON3 yes -ENV ENABLE_GTK yes -ENV PYTHON_VERSION_MAJOR 3 -ENV PYTHON_VERSION 3.5 -ENV CFLAGS "-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer" -ENV MAKEFLAGS "-j4" -ENV PREFIX "${USER_HOME}/jhbuild" -ENV JHBUILD "${USER_HOME}/gnome" -ENV PATH ${PREFIX}/bin:${PREFIX}/sbin:${PATH} -ENV LD_LIBRARY_PATH ${PREFIX}/lib:${LD_LIBRARY_PATH} -ENV PYTHONPATH ${PREFIX}/lib/python${PYTHON_VERSION}/site-packages:/usr/lib/python${PYTHON_VERSION}/site-packages -ENV PKG_CONFIG_PATH ${PREFIX}/lib/pkgconfig:${PREFIX}/share/pkgconfig:/usr/lib/pkgconfig -ENV XDG_DATA_DIRS ${PREFIX}/share:/usr/share -ENV XDG_CONFIG_DIRS ${PREFIX}/etc/xdg -ENV PYTHON "python3" -ENV PACKAGES "python3-gi python3-gi-cairo" -ENV CC gcc - -# # Ensure cleanup script is available for the next command -# ADD ./container/root/clean.sh /clean.sh -# -# RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime -# -# RUN ln -sf /usr/share/zoneinfo/$TIMEZONE /etc/localtime && \ -# dpkg-reconfigure -f noninteractive tzdata && \ -# sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ -# sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ -# sed -i -e 's/# sv_SE.UTF-8 UTF-8/sv_SE.UTF-8 UTF-8/' /etc/locale.gen && \ -# echo 'LANG="en_US.UTF-8"' > /etc/default/locale && \ -# dpkg-reconfigure -f noninteractive locales && \ -# update-locale LANG=en_US.UTF-8 &&\ -# apt-get update &&\ -# apt-get upgrade -y -q &&\ -# apt-get dist-upgrade -y -q &&\ - -RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata && \ - sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ - echo 'LANG="en_US.UTF-8"' > /etc/default/locale && \ - dpkg-reconfigure -f noninteractive locales && \ - update-locale LANG=en_US.UTF-8 && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial main restricted' | tee /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial universe' | tee -a /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial universe' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe' | tee -a /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-updates universe' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-security main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-security main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-security universe' | tee -a /etc/apt/sources.list && \ - echo 'deb-src http://us.archive.ubuntu.com/ubuntu/ xenial-security universe' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial multiverse' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-updates multiverse' | tee -a /etc/apt/sources.list && \ - echo 'deb http://security.ubuntu.com/ubuntu xenial-security main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb http://security.ubuntu.com/ubuntu xenial-security main restricted' | tee -a /etc/apt/sources.list && \ - echo 'deb http://security.ubuntu.com/ubuntu xenial-security universe' | tee -a /etc/apt/sources.list && \ - echo 'deb http://us.archive.ubuntu.com/ubuntu/ xenial-backports main restricted universe multiverse' | tee -a /etc/apt/sources.list && \ - echo 'deb http://security.ubuntu.com/ubuntu xenial-security multiverse' | tee -a /etc/apt/sources.list && \ - cat /etc/apt/sources.list | grep -v "^#" | sort -u > /etc/apt/sources.list.bak && \ - mv -fv /etc/apt/sources.list.bak /etc/apt/sources.list && \ - add-apt-repository -y ppa:ricotz/testing && \ - add-apt-repository -y ppa:gnome3-team/gnome3 && \ - add-apt-repository -y ppa:gnome3-team/gnome3-staging && \ - add-apt-repository -y ppa:pitti/systemd-semaphore && \ - apt-get update -yqq && \ - apt-get upgrade -yqq && \ - export LANG=en_US.UTF-8 && \ - apt-get install -qqy libpulse-dev espeak && \ - apt-cache search --names-only '^(lib)?gstreamer1.0\S*' | sed 's/\(.*\) -.*/\1 /' | grep -iv "Speech" > dependencies && \ - cat dependencies && \ - apt-get build-dep -y `cat dependencies` && \ - apt-get install -qqy gnome-common \ - gtk-doc-tools \ - libgtk-3-dev \ - libgirepository1.0-dev \ - yelp-tools \ - libgladeui-dev \ - python3-dev \ - python3-cairo-dev \ - python3-gi \ - automake \ - autopoint \ - bison \ - build-essential \ - byacc \ - flex \ - gcc \ - automake \ - autoconf \ - libtool \ - bison \ - swig \ - python-dev \ - libpulse-dev \ - gettext \ - gnome-common \ - gtk-doc-tools \ - libgtk-3-dev \ - libgirepository1.0-dev \ - python3-gi-cairo \ - yasm \ - nasm \ - bison \ - flex \ - libusb-1.0-0-dev \ - libgudev-1.0-dev \ - libxv-dev \ - build-essential \ - autotools-dev \ - automake \ - autoconf \ - libtool \ - binutils \ - autopoint \ - libxml2-dev \ - zlib1g-dev \ - libglib2.0-dev \ - pkg-config \ - flex \ - python \ - libasound2-dev \ - libgudev-1.0-dev \ - libxt-dev \ - libvorbis-dev \ - libcdparanoia-dev \ - libpango1.0-dev \ - libtheora-dev \ - libvisual-0.4-dev \ - iso-codes \ - libgtk-3-dev \ - libraw1394-dev \ - libiec61883-dev \ - libavc1394-dev \ - libv4l-dev \ - libcairo2-dev \ - libcaca-dev \ - libspeex-dev \ - libpng-dev \ - libshout3-dev \ - libjpeg-dev \ - libaa1-dev \ - libflac-dev \ - libdv4-dev \ - libtag1-dev \ - libwavpack-dev \ - libpulse-dev \ - gstreamer1.0* \ - lame \ - flac \ - libfftw3-dev \ - xvfb \ - gir1.2-gtk-3.0 \ - xsltproc \ - docbook-xml \ - docbook-xsl \ - python-libxml2 \ - sudo \ - # begin - gst-plugins-bad req - libqt4-opengl \ - libdvdread4 \ - libdvdnav4 \ - libllvm3.8 \ - libsoundtouch-dev \ - libsoundtouch1 \ - # end gst-plugins-bad req - ubuntu-restricted-extras && \ - apt-get clean && \ - apt-get autoclean -y && \ - apt-get autoremove -y && \ - rm -rf /var/lib/{cache,log}/ && \ - rm -rf /var/lib/apt/lists/*.lz4 /tmp/* /var/tmp/* - - -ENV VIRTUALENVWRAPPER_PYTHON '/usr/local/bin/python3' -ENV VIRTUALENVWRAPPER_VIRTUALENV '/usr/local/bin/virtualenv' -ENV VIRTUALENV_WRAPPER_SH '/usr/local/bin/virtualenvwrapper.sh' - -ENV PYTHON_VERSION_MAJOR "3" -ENV GSTREAMER "1.0" -ENV USER "pi" -ENV USER_HOME "/home/${USER}" -ENV LANGUAGE_ID 1473 -ENV GITHUB_BRANCH "master" -ENV GITHUB_REPO_NAME "scarlett_os" -ENV GITHUB_REPO_ORG "bossjones" -ENV PI_HOME "/home/pi" -ENV MAIN_DIR "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}" -ENV VIRT_ROOT "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}" -ENV PKG_CONFIG_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/pkgconfig" -ENV SCARLETT_CONFIG "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/.scarlett" -ENV SCARLETT_HMM "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/.virtualenvs/${GITHUB_REPO_NAME}/share/pocketsphinx/model/en-us/en-us" -ENV SCARLETT_LM "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/lm/${LANGUAGE_ID}.lm" -ENV SCARLETT_DICT "${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME}/tests/fixtures/dict/${LANGUAGE_ID}.dic" -ENV LD_LIBRARY_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib" -ENV GST_PLUGIN_PATH "${PI_HOME}/.virtualenvs/${GITHUB_REPO_NAME}/lib/gstreamer-${GSTREAMER}" -ENV PYTHON "/usr/local/bin/python3" -ENV PYTHON_VERSION "3.5" -ENV VIRTUALENVWRAPPER_PYTHON "/usr/local/bin/python3" -ENV VIRTUALENVWRAPPER_VIRTUALENV "/usr/local/bin/virtualenv" -ENV VIRTUALENVWRAPPER_SCRIPT "/usr/local/bin/virtualenvwrapper.sh" -ENV PYTHONSTARTUP "${USER_HOME}/.pythonrc" -ENV PIP_DOWNLOAD_CACHE "${USER_HOME}/.pip/cache" -ENV WORKON_HOME "${VIRT_ROOT}" - -RUN set -xe \ - && useradd -U -d ${PI_HOME} -m -r -G adm,sudo,dip,plugdev,tty,audio ${USER} \ - && usermod -a -G ${USER} ${USER} \ - && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ - && mkdir -p ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github/${GITHUB_REPO_NAME} \ - && mkdir -p ${MAIN_DIR} \ - && chown -hR ${USER}:${USER} ${MAIN_DIR} \ - && echo 'pi ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ - && echo '%pi ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers \ - && cat /etc/sudoers - -USER pi - -WORKDIR /home/pi - -# Create a basic .jhbuildrc -RUN echo "import os" > /home/pi/.jhbuildrc && \ - echo "prefix='$PREFIX'" >> /home/pi/.jhbuildrc && \ - echo "checkoutroot='$JHBUILD'" >> /home/pi/.jhbuildrc && \ - echo "moduleset = 'gnome-world'" >> /home/pi/.jhbuildrc && \ - echo "interact = False" >> /home/pi/.jhbuildrc && \ - echo "makeargs = '$MAKEFLAGS'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['CFLAGS'] = '$CFLAGS'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PYTHON'] = 'python$PYTHON_VERSION_MAJOR'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['GSTREAMER'] = '1.0'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['ENABLE_PYTHON3'] = 'yes'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['ENABLE_GTK'] = 'yes'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PYTHON_VERSION'] = '$PYTHON_VERSION'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['CFLAGS'] = '-fPIC -O0 -ggdb -fno-inline -fno-omit-frame-pointer'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['MAKEFLAGS'] = '-j4'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PREFIX'] = '$USER_HOME/jhbuild'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['JHBUILD'] = '$USER_HOME/gnome'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PATH'] = '$PREFIX/bin:$PREFIX/sbin:$PATH'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['LD_LIBRARY_PATH'] = '$PREFIX/lib:$LD_LIBRARY_PATH'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PYTHONPATH'] = '$PREFIX/lib/python$PYTHON_VERSION/site-packages:/usr/lib/python$PYTHON_VERSION/site-packages'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PKG_CONFIG_PATH'] = '$PREFIX/lib/pkgconfig:$PREFIX/share/pkgconfig:/usr/lib/pkgconfig'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['XDG_DATA_DIRS'] = '$PREFIX/share:/usr/share'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['XDG_CONFIG_DIRS'] = '$PREFIX/etc/xdg'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['CC'] = 'gcc'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['WORKON_HOME'] = '$USER_HOME/.virtualenvs'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PROJECT_HOME'] = '$USER_HOME/dev'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['VIRTUALENVWRAPPER_PYTHON'] = '$VIRTUALENVWRAPPER_PYTHON'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['VIRTUALENVWRAPPER_VIRTUALENV'] = '$VIRTUALENVWRAPPER_VIRTUALENV'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PYTHONSTARTUP'] = '$USER_HOME/.pythonrc'" >> /home/pi/.jhbuildrc && \ - echo "os.environ['PIP_DOWNLOAD_CACHE'] = '$USER_HOME/.pip/cache'" >> /home/pi/.jhbuildrc && \ - cat /home/pi/.jhbuildrc - - # && \ - # mkdir -p /home/pi/jhbuild && \ - # - -RUN cat /home/pi/.jhbuildrc - -# jhbuild -RUN mkdir -p /home/pi/gnome && \ - - echo "****************[JHBUILD]****************" && \ - cd /home/pi && \ - if test ! -d /home/pi/jhbuild; then git clone https://github.com/GNOME/jhbuild.git && \ - cd jhbuild; else echo "exists" && cd jhbuild; fi && \ - git checkout 86d958b6778da649b559815c0a0dbe6a5d1a8cd4 && \ - ./autogen.sh --prefix=/usr/local > /dev/null && \ - make > /dev/null && \ - sudo make install > /dev/null && \ - sudo chown pi:pi -R /usr/local/ && \ - chown pi:pi -R /home/pi/jhbuild && \ - - echo "****************[GTK-DOC]****************" && \ - cd /home/pi/gnome && \ - git clone https://github.com/GNOME/gtk-doc.git && \ - jhbuild buildone -n gtk-doc && \ - - echo "****************[GLIB]****************" && \ - cd /home/pi/gnome && \ - git clone https://github.com/GNOME/glib.git && \ - cd glib && \ - git checkout eaca4f4116801f99e30e42a857559e19a1e6f4ce && \ - jhbuild buildone -n glib && \ - - echo "****************[GOBJECT-INTROSPECTION]****************" && \ - cd /home/pi/gnome && \ - git clone https://github.com/GNOME/gobject-introspection.git && \ - cd gobject-introspection && \ - git checkout cee2a4f215d5edf2e27b9964d3cfcb28a9d4941c && \ - jhbuild buildone -n gobject-introspection && \ - - echo "****************[PYGOBJECT]****************" && \ - cd /home/pi/gnome && \ - git clone https://github.com/GNOME/pygobject.git && \ - cd /home/pi/gnome && \ - cd pygobject && \ - git checkout fb1b8fa8a67f2c7ea7ad4b53076496a8f2b4afdb && \ - jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild --with-python=python3 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GSTREAMER]****************" && \ - cd /home/pi/gnome && \ - curl -L "https://gstreamer.freedesktop.org/src/gstreamer/gstreamer-1.8.2.tar.xz" > gstreamer-1.8.2.tar.xz && \ - tar -xJf gstreamer-1.8.2.tar.xz && \ - cd gstreamer-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[ORC]****************" && \ - cd /home/pi/gnome && \ - curl -L "https://gstreamer.freedesktop.org/src/orc/orc-0.4.25.tar.xz" > orc-0.4.25.tar.xz && \ - tar -xJf orc-0.4.25.tar.xz && \ - cd orc-0.4.25 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-PLUGINS-BASE]****************" && \ - cd /home/pi/gnome && \ - curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-base/gst-plugins-base-1.8.2.tar.xz" > gst-plugins-base-1.8.2.tar.xz && \ - tar -xJf gst-plugins-base-1.8.2.tar.xz && \ - cd gst-plugins-base-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc --with-x > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-PLUGINS-GOOD]****************" && \ - cd /home/pi/gnome && \ - curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-good/gst-plugins-good-1.8.2.tar.xz" > gst-plugins-good-1.8.2.tar.xz && \ - tar -xJf gst-plugins-good-1.8.2.tar.xz && \ - cd gst-plugins-good-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc --with-libv4l2 --with-x > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-PLUGINS-UGLY]****************" && \ - cd /home/pi/gnome && \ - curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-ugly/gst-plugins-ugly-1.8.2.tar.xz" > gst-plugins-ugly-1.8.2.tar.xz && \ - tar -xJf gst-plugins-ugly-1.8.2.tar.xz && \ - cd gst-plugins-ugly-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-PLUGINS-BAD]****************" && \ - cat /home/pi/jhbuild/bin/gdbus-codegen && \ - sed -i "s,#!python3,#!/usr/bin/python3,g" /home/pi/jhbuild/bin/gdbus-codegen && \ - cat /home/pi/jhbuild/bin/gdbus-codegen && \ - cd /home/pi/gnome && \ - curl -L "http://gstreamer.freedesktop.org/src/gst-plugins-bad/gst-plugins-bad-1.8.2.tar.xz" > gst-plugins-bad-1.8.2.tar.xz && \ - tar -xJf gst-plugins-bad-1.8.2.tar.xz && \ - cd gst-plugins-bad-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-LIBAV]****************" && \ - cd /home/pi/gnome && \ - curl -L "http://gstreamer.freedesktop.org/src/gst-libav/gst-libav-1.8.2.tar.xz" > gst-libav-1.8.2.tar.xz && \ - tar -xJf gst-libav-1.8.2.tar.xz && \ - cd gst-libav-1.8.2 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild --enable-orc > /dev/null && \ - jhbuild run make -j4 > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[GST-PLUGINS-ESPEAK]****************" && \ - cd $JHBUILD && \ - curl -L "https://github.com/bossjones/bossjones-gst-plugins-espeak-0-4-0/archive/v0.4.1.tar.gz" > gst-plugins-espeak-0.4.0.tar.gz && \ - tar xvf gst-plugins-espeak-0.4.0.tar.gz && \ - rm -rfv gst-plugins-espeak-0.4.0 && \ - mv -fv bossjones-gst-plugins-espeak-0-4-0-0.4.1 gst-plugins-espeak-0.4.0 && \ - cd gst-plugins-espeak-0.4.0 && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run make > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[SPHINXBASE]****************" && \ - cd $JHBUILD && \ - git clone https://github.com/cmusphinx/sphinxbase.git && \ - cd sphinxbase && \ - git checkout 74370799d5b53afc5b5b94a22f5eff9cb9907b97 && \ - cd $JHBUILD/sphinxbase && \ - jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run make clean all > /dev/null && \ - jhbuild run make install > /dev/null && \ - - echo "****************[POCKETSPHINX]****************" && \ - cd $JHBUILD && \ - git clone https://github.com/cmusphinx/pocketsphinx.git && \ - cd pocketsphinx && \ - git checkout 68ef5dc6d48d791a747026cd43cc6940a9e19f69 && \ - jhbuild run ./autogen.sh --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run ./configure --prefix=/home/pi/jhbuild > /dev/null && \ - jhbuild run make clean all > /dev/null && \ - jhbuild run make install > /dev/null - -# RUN set -xe \ -# && useradd -d /var/lib/mediagoblin -m -r -g www-data mediagoblin \ -# && groupadd -r mediagoblin \ -# && usermod -a -G mediagoblin mediagoblin \ -# && mkdir -p /var/log/mediagoblin \ -# /srv/mediagoblin.example.org/mediagoblin \ -# && chown -hR mediagoblin:www-data /var/log/mediagoblin \ -# /srv/mediagoblin.example.org \ -# && echo 'mediagoblin ALL=(ALL:ALL) NOPASSWD: /usr/sbin/nginx, /bin/chown' >> /etc/sudoers -# -# USER mediagoblin -# -# WORKDIR /srv/mediagoblin.example.org/mediagoblin -# -# RUN set -xe \ -# && git clone http://git.savannah.gnu.org/r/mediagoblin.git . \ -# && git checkout stable \ -# && git submodule sync \ -# && git submodule update --force --init --recursive \ -# && ./bootstrap.sh \ -# && ./configure \ -# && make \ -# && bin/easy_install flup==1.0.3.dev-20110405 \ -# && bin/pip install scikits.audiolab \ -# && ln -s /var/lib/mediagoblin user_dev \ -# && cp -av mediagoblin.ini mediagoblin_local.ini \ -# && cp -av paste.ini paste_local.ini \ -# && sed -i 's@.*sql_engine = .*@sql_engine = sqlite:////var/lib/mediagoblin/mediagoblin.db@' mediagoblin_local.ini \ -# && { \ -# echo '[[mediagoblin.media_types.video]]'; \ -# echo '[[mediagoblin.media_types.audio]]'; \ -# echo '[[mediagoblin.media_types.pdf]]'; \ -# } >> mediagoblin_local.ini - -# COPY ./dotfiles/.bashrc /home/pi/.bashrc -# COPY ./dotfiles/.profile /home/pi/.profile - -# RUN set -xe \ -# && apt-get autoclean -y \ -# && apt-get autoremove -y \ -# && rm -rf /tmp/* /var/tmp/* \ -# && rm -rf /var/lib/apt/lists/* - -###### # Layer customizations over existing structure -###### COPY ./container/root / -###### -###### # Ensure application code makes it into the /app directory -###### COPY ./ /home/pi/dev/bossjones-github/scarlett_os/ -###### -###### RUN ls -lta /home/pi/dev/bossjones-github/scarlett_os/ && \ -###### bash -c "source /usr/local/bin/virtualenvwrapper.sh && mkvirtualenv --system-site-packages ${GITHUB_REPO_NAME}" -###### -###### COPY ./postactivate /home/pi/.virtualenvs/scarlett_os/postactivate -###### -###### RUN set -xe \ -###### && chown -hR ${USER}:${USER} ${PI_HOME}/dev/${GITHUB_REPO_ORG}-github \ -###### && chown -hR ${USER}:${USER} ${MAIN_DIR} \ -###### && chown -hR ${USER}:${USER} /usr/local/lib/python3.5/site-packages \ -###### && chown -R ${USER}:${USER} /home/pi/.virtualenvs - -ENTRYPOINT ["file","/bin/bash"] +ENTRYPOINT ["/docker_entrypoint.sh"] +CMD true diff --git a/container/root/docker_entrypoint.sh b/container/root/docker_entrypoint.sh new file mode 100755 index 00000000..242b8666 --- /dev/null +++ b/container/root/docker_entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# if [ ! -f /virtualenv/bin/activate ]; then +# virtualenv /virtualenv -p python${INSPIRE_PYTHON_VERSION} +# source /virtualenv/bin/activate +# pip install --upgrade pip setuptools wheel +# cp -r /src-cache /virtualenv/src +# else +# source /virtualenv/bin/activate +# fi + +# find \( -name __pycache__ -o -name '*.pyc' \) | xargs rm -rf + +exec "$@" diff --git a/container/root/etc/services.d/scarlett_os/run b/container/root/etc/services.d/scarlett_os/run index f1d58c88..b28e40b5 100755 --- a/container/root/etc/services.d/scarlett_os/run +++ b/container/root/etc/services.d/scarlett_os/run @@ -1,3 +1,3 @@ #!/usr/bin/execlineb -P -with-contenv /home/pi/.virtualenvs/scarlett_os/bin/python3 --version +with-contenv jhbuild run python --version diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..c74181f2 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,132 @@ +version: '2' + +services: + scarlett_master: + build: . + environment: + SERVER_LOG_MINIMAL: 1 + SERVER_APP_NAME: jhbuild-scarlett-os + S6_KILL_FINISH_MAXTIME: 1 + S6_KILL_GRACETIME: 1 + SERVER_WORKER_PROCESSES: 1 + volumes: + - ./:/home/pi/dev/bossjones-github/scarlett_os + + default: + extends: + service: scarlett_master + volumes_from: + - scarlett_master + command: "jhbuild run -- py.test --cov scarlett_os --cov-report term-missing" + # command: jhbuild run -- coverage run -- setup.py test + # command: env + # unit: + # extends: + # service: scarlett_master + # volumes_from: + # - scarlett_master + # command: py.test tests/unit + # disambiguation: + # extends: + # service: scarlett_master + # command: py.test tests/integration/disambiguation + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # depends_on: + # - worker + # workflows: + # extends: + # service: scarlett_master + # command: py.test tests/integration/workflows + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # depends_on: + # - worker + # integration: + # extends: + # service: scarlett_master + # command: py.test tests/integration --ignore tests/integration/disambiguation --ignore tests/integration/workflows + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # depends_on: + # - worker + # acceptance: + # extends: + # service: scarlett_master + # command: py.test --driver Remote --host selenium --port 4444 --capability browserName firefox --html=selenium-report.html tests/acceptance + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # - selenium + # depends_on: + # - web + # - worker + # environment: + # - SERVER_NAME=web:5000 + # web: + # extends: + # service: scarlett_master + # command: gunicorn -b 0.0.0.0:5000 -t 3600 --access-logfile "-" inspirehep.wsgi_with_coverage + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # environment: + # - APP_SERVER_NAME=web:5000 + # worker: + # container_name: inspirehep-test-worker + # extends: + # service: scarlett_master + # command: celery worker -E -A inspirehep.celery --loglevel=INFO --purge + # volumes_from: + # - scarlett_master + # links: + # - database + # - indexer + # - rabbitmq + # - redis + # redis: + # container_name: inspirehep-test-redis + # image: redis + # indexer: + # container_name: inspirehep-test-indexer + # extends: + # file: services.yml + # service: indexer + # rabbitmq: + # container_name: inspirehep-test-rabbitmq + # image: rabbitmq + # database: + # container_name: inspirehep-test-database + # extends: + # file: services.yml + # service: database + # selenium: + # image: selenium/standalone-firefox:2.53.1-beryllium + # scarlett_master: + # container_name: inspirehep-test-scarlett_master + # extends: + # file: services.yml + # service: scarlett_master diff --git a/docker-compose.yml b/docker-compose.yml index 4a402098..43c9b0b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,10 @@ scarlett_master: - build: . - environment: - - FAKE_ENV=random - - volumes: - - ./:/home/pi/dev/bossjones-github/scarlett_os + build: . + environment: + SERVER_LOG_MINIMAL: 1 + SERVER_APP_NAME: jhbuild-scarlett-os + S6_KILL_FINISH_MAXTIME: 1 + S6_KILL_GRACETIME: 1 + SERVER_WORKER_PROCESSES: 1 + volumes: + - ./:/home/pi/dev/bossjones-github/scarlett_os diff --git a/tests/test_scarlett_os.py b/tests/test_scarlett_os.py index 57b87611..ebabee00 100644 --- a/tests/test_scarlett_os.py +++ b/tests/test_scarlett_os.py @@ -81,8 +81,14 @@ def test_gstreamer_versions(self): Gst.debug_set_default_threshold(1) pp.pprint(ubuntu_version) - - if 'trusty' in ubuntu_version or 'jessie' in ubuntu_version: + # ubuntu 16.04 says: + # ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'Ubuntu', '16.04', 'xenial'] + # travis says + # ['Linux', '4.4.0', '38', 'generic', 'x86_64', 'with', 'debian', 'jessie', 'sid'] + # docker says: + # ['Linux', '4.4.17', 'boot2docker', 'x86_64', 'with', 'debian', 'stretch', 'sid'] + + if 'trusty' in ubuntu_version or 'jessie' in ubuntu_version or 'stretch' in ubuntu_version: assert GObject.pygobject_version == (3, 22, 0) else: assert GObject.pygobject_version == (3, 20, 0) From c35218c4db2ac1983b3444bd9f891a6391198308 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 20:53:45 -0400 Subject: [PATCH 36/48] ci: build.yml --- ci/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ci/build.yml diff --git a/ci/build.yml b/ci/build.yml new file mode 100644 index 00000000..30966a61 --- /dev/null +++ b/ci/build.yml @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +scarlett_master: + build: . + +# worker: +# build: . From 2c7e931aaed34c8c0d674f9a9991cf92331249ea Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 21:17:15 -0400 Subject: [PATCH 37/48] chg: docker build test again --- .dockerignore | 11 +++++++++++ ci/build.yml | 3 --- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 21d13848..1e12ab75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,3 +26,14 @@ var/ .bundle/ .cache/ __pycache__/ + +.git +*.gitignore + +*.mo +*.pyc +*.swp +*.swo +*.~ + +Procfile* diff --git a/ci/build.yml b/ci/build.yml index 30966a61..1aa8c0ba 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -2,6 +2,3 @@ scarlett_master: build: . - -# worker: -# build: . From 6f419b86441daae22e87d5ed134e09eaf376a732 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 21:46:04 -0400 Subject: [PATCH 38/48] chg: moved .coveralls.yml --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c2de57dd..175a2708 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,12 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run python3 setup.py install \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ - && jhbuild run -- pip install coveralls + && jhbuild run -- pip install coveralls \ + && jhbuild run coveralls COPY ./container/root / +# RUN sudo touch .coverage + ENTRYPOINT ["/docker_entrypoint.sh"] CMD true From e097920f4e9729be76030fddd41f696fec3f1f8e Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 22:20:38 -0400 Subject: [PATCH 39/48] chg: .coverage now instead --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 175a2708..1cd59da8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ && jhbuild run -- pip install coveralls \ - && jhbuild run coveralls + && touch .coverage COPY ./container/root / From a1b2578694d444e4d767dfba9709f07014bf5117 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 22:37:06 -0400 Subject: [PATCH 40/48] chg: the worst commit ever --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1cd59da8..b54d86c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,8 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ && jhbuild run -- pip install coveralls \ - && touch .coverage + && touch .coverage \ + && sudo chmod 777 .coverage COPY ./container/root / From 96263a95f3321539e09d68bcfd0843ae2f22c017 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 23:05:31 -0400 Subject: [PATCH 41/48] chg: another stupid commit --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b54d86c6..4984714b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,13 +10,13 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run python3 setup.py install \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ - && jhbuild run -- pip install coveralls \ - && touch .coverage \ - && sudo chmod 777 .coverage + && jhbuild run -- pip install coveralls COPY ./container/root / # RUN sudo touch .coverage +USER root + ENTRYPOINT ["/docker_entrypoint.sh"] CMD true From e5eb7845184cd7a9bb365437e39b350e03c87613 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Wed, 26 Oct 2016 23:16:57 -0400 Subject: [PATCH 42/48] chg: another attempt --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4984714b..80b70174 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,13 +10,12 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run python3 setup.py install \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ - && jhbuild run -- pip install coveralls + && jhbuild run -- pip install coveralls \ + && sudo chmod 777 /home/pi/dev/bossjones-github/scarlett_os COPY ./container/root / # RUN sudo touch .coverage -USER root - ENTRYPOINT ["/docker_entrypoint.sh"] CMD true From 7d9507bcd14ad474d56158396bb52b6afb5842df Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 27 Oct 2016 08:59:43 -0400 Subject: [PATCH 43/48] chg: docker-compose --- Dockerfile | 3 +-- docker-compose.test.yml | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 80b70174..f56e8f23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,7 @@ RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && jhbuild run python3 setup.py install \ && jhbuild run -- pip install -e .[test] \ && jhbuild run -- coverage run -- setup.py test \ - && jhbuild run -- pip install coveralls \ - && sudo chmod 777 /home/pi/dev/bossjones-github/scarlett_os + && jhbuild run -- pip install coveralls COPY ./container/root / diff --git a/docker-compose.test.yml b/docker-compose.test.yml index c74181f2..2b0754d5 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -16,8 +16,10 @@ services: extends: service: scarlett_master volumes_from: - - scarlett_master + - scarlett_master:rw command: "jhbuild run -- py.test --cov scarlett_os --cov-report term-missing" + # command: "jhbuild run -- python setup.py test" + # command: "ls -lta" # command: jhbuild run -- coverage run -- setup.py test # command: env # unit: From a29cc7310b0758febf91b151adf43c9bbb36edef Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 27 Oct 2016 16:28:33 -0400 Subject: [PATCH 44/48] chg: fix to coveralls maybe --- .travis.yml | 2 ++ Dockerfile | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c5ef6d4..7b477634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ cache: install: - travis_retry docker-compose pull + # set containers as privileged to bypass Travis environment limitations + # - 'sed -i "/build/a \ \ privileged: true" docker-compose.yml' - travis_retry docker-compose -f docker-compose.yml -f ci/build.yml build script: diff --git a/Dockerfile b/Dockerfile index f56e8f23..ff3bcb77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,9 @@ WORKDIR /home/pi/dev/bossjones-github/scarlett_os RUN set -x cd /home/pi/dev/bossjones-github/scarlett_os \ && pwd \ && jhbuild run python3 setup.py install \ - && jhbuild run -- pip install -e .[test] \ - && jhbuild run -- coverage run -- setup.py test \ - && jhbuild run -- pip install coveralls + && jhbuild run -- pip install -e .[test] COPY ./container/root / -# RUN sudo touch .coverage - ENTRYPOINT ["/docker_entrypoint.sh"] CMD true From 2a0c27d7e01dad39957fbc28f060389f58160b22 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 27 Oct 2016 17:33:42 -0400 Subject: [PATCH 45/48] chg: docker run --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7b477634..38b71657 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ install: - travis_retry docker-compose -f docker-compose.yml -f ci/build.yml build script: - - "docker-compose -f docker-compose.test.yml run --rm ${TEST_TARGET}" + #- "docker-compose -f docker-compose.test.yml run --rm ${TEST_TARGET}" + - docker run --privileged -v `pwd`:/home/pi/dev/bossjones-github/scarlett_os -i -t --rm scarlettos_scarlett_master make test after_success: - mv .coverage .coverage.tests From 77e01bad01b491a293f4ab2915351c94b872649b Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Thu, 27 Oct 2016 17:42:51 -0400 Subject: [PATCH 46/48] chg: sed ccmd --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38b71657..2b243536 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ cache: install: - travis_retry docker-compose pull # set containers as privileged to bypass Travis environment limitations - # - 'sed -i "/build/a \ \ privileged: true" docker-compose.yml' + - 'sed -i "/build: ./a \ \ privileged: true" docker-compose.yml' - travis_retry docker-compose -f docker-compose.yml -f ci/build.yml build script: From 86a3827d6764b23a601fb4764eb68fe5a3614cb5 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Fri, 28 Oct 2016 20:41:39 -0400 Subject: [PATCH 47/48] fix: docker permissions hopefully --- .travis.yml | 3 +++ container/root/docker_entrypoint.sh | 1 + docker-compose.test.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b243536..e13bc9d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,9 @@ script: - docker run --privileged -v `pwd`:/home/pi/dev/bossjones-github/scarlett_os -i -t --rm scarlettos_scarlett_master make test after_success: + - _USER=$(ls -lta | awk '{print $3}') + - _GROUP=$(ls -lta | awk '{print $4}') + - sudo chown $_USER:$_GROUP -R . - mv .coverage .coverage.tests - coverage combine # Coverage report contains Docker paths. We replace them, so that we can run Coveralls. diff --git a/container/root/docker_entrypoint.sh b/container/root/docker_entrypoint.sh index 242b8666..c7d5b033 100755 --- a/container/root/docker_entrypoint.sh +++ b/container/root/docker_entrypoint.sh @@ -12,5 +12,6 @@ set -e # fi # find \( -name __pycache__ -o -name '*.pyc' \) | xargs rm -rf +sudo chown pi:pi -R /home/pi/dev exec "$@" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 2b0754d5..80824df8 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -10,7 +10,7 @@ services: S6_KILL_GRACETIME: 1 SERVER_WORKER_PROCESSES: 1 volumes: - - ./:/home/pi/dev/bossjones-github/scarlett_os + - ./:/home/pi/dev/bossjones-github/scarlett_os:rw default: extends: diff --git a/docker-compose.yml b/docker-compose.yml index 43c9b0b8..bd75ea4c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,4 +7,4 @@ scarlett_master: S6_KILL_GRACETIME: 1 SERVER_WORKER_PROCESSES: 1 volumes: - - ./:/home/pi/dev/bossjones-github/scarlett_os + - ./:/home/pi/dev/bossjones-github/scarlett_os:rw From 89a8ba99e3a08e05ca9466ab7749660a83732af0 Mon Sep 17 00:00:00 2001 From: Malcolm Jones Date: Sat, 29 Oct 2016 10:25:34 -0400 Subject: [PATCH 48/48] chg: remove circleci --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 83c9fa66..79204561 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,7 @@ ScarlettOS =============================== + .. image:: https://img.shields.io/pypi/v/scarlett_os.svg :target: https://pypi.python.org/pypi/scarlett_os