Merge pull request #11 from minrk/conda

use conda

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ansible-conda"]
+	path = ansible-conda
+	url = https://github.com/UDST/ansible-conda.git

ansible.cfg

@@ -2,6 +2,7 @@
 remote_user=root
 become_user=root
 hostfile=./hosts
+library = ./ansible-conda
 
 [ssh_connection]
 ssh_args = -o ServerAliveInterval=60

deploy.yml

@@ -5,7 +5,6 @@
   roles:
     - common
     - python
-    - node
     - { role: newrelic, when: newrelic_license_key is defined and newrelic_license_key != ''}
     - nginx
     - supervisor

roles/bash/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: pip install bash_kernel package
-  pip: name={{item}} state=present executable=pip3 editable=false
+  pip: name={{item}} state=present editable=false
   become: true
   with_items:
       - bash_kernel

roles/common/tasks/packages.yml

@@ -4,6 +4,10 @@
   apt: update_cache=yes cache_valid_time=3600
   become: yes
 
+- name: install aptitude
+  apt: pkg=aptitude
+  become: yes
+
 - name: Upgrade all packages
   apt: upgrade=safe
   become: yes
@@ -26,7 +30,7 @@
     - vim
     - emacs
     - tree
-    - htop 
+    - htop
 
 - name: install open-iscsi for cloud block volumes
   apt: pkg=open-iscsi state=present update_cache=yes cache_valid_time=3600

roles/formgrade/tasks/main.yml

@@ -24,3 +24,11 @@
 - name: install supervisor config for nbgrader
   template: "src=nbgrader.conf.j2 dest=/etc/supervisor/conf.d/nbgrader_{{nbgrader_course_id}}.conf owner=root group=root mode='0600' backup=yes"
   become: true
+
+- name: load nbgrader supervisor config
+  supervisorctl: name=nbgrader_{{nbgrader_course_id}} state=present
+  become: true
+
+- name: start nbgrader with supervisor
+  supervisorctl: name=nbgrader_{{nbgrader_course_id}} state=restarted
+  become: true

roles/formgrade/templates/nbgrader.conf.j2

@@ -1,12 +1,12 @@
 # {{ ansible_managed }}
 
 [program:nbgrader_{{nbgrader_course_id}}]
-command=nbgrader formgrade
+command=/opt/conda/bin/nbgrader formgrade
 redirect_stderr=true
 stdout_logfile={{nbgrader_log_dir}}/nbgrader_{{nbgrader_course_id}}.log
 autostart=true
 autorestart=false
 stopasgroup=true
 user={{nbgrader_owner}}
 directory={{nbgrader_base_dir}}/
-environment=HOME="{{home_dir}}/{{nbgrader_owner}}",USER="{{nbgrader_owner}}"
+environment=HOME="{{home_dir}}/{{nbgrader_owner}}",USER="{{nbgrader_owner}}",PATH="/opt/conda/bin:%(ENV_PATH)s"

roles/jupyterhub/tasks/packages.yml

@@ -1,12 +1,14 @@
 ---
 
-# Using the npm ansible module doesn't work!!!
-- name: npm install configurable-http-proxy
-  command: npm install -g jupyter/configurable-http-proxy
+- name: JupyterHub conda dependencies
+  conda: name={{item}} state=present
   become: true
+  with_items:
+    - configurable-http-proxy
+    - sqlalchemy
 
-- name: pip install 
-  pip: name={{item}} state=present executable=pip3 editable=false
+- name: pip install jupyterhub
+  pip: name={{item}} state=present editable=false
   become: true
   with_items:
     - jupyterhub==0.6

roles/jupyterhub/tasks/supervisor.yml

@@ -4,5 +4,12 @@
   template: src=jupyterhub.conf.j2 dest=/etc/supervisor/conf.d/jupyterhub.conf owner=root group=root mode=0600 backup=yes
   become: true
 
-- name: start jupyterhub with supervisor
+- name: install jupyterhub launch script
+  action: template src=start-jupyterhub.sh.j2 dest={{jupyterhub_srv_dir}}/start-jupyterhub.sh mode=0700
+  become: true
+
+- name: load jupyterhub supervisor config
+  action: supervisorctl name=jupyterhub state=present
+
+- name: restart jupyterhub with supervisor
   supervisorctl: name=jupyterhub state=restarted

roles/jupyterhub/templates/jupyterhub.conf.j2

@@ -1,7 +1,7 @@
 # {{ ansible_managed }}
 
 [program:jupyterhub]
-command=jupyterhub --config={{ jupyterhub_config_dir }}/jupyterhub_config.py
+command={{jupyterhub_srv_dir}}/start-jupyterhub.sh --config={{ jupyterhub_config_dir }}/jupyterhub_config.py
 redirect_stderr=true
 stdout_logfile={{ jupyterhub_log_dir }}/jupyterhub.log
 autostart=true

roles/jupyterhub/templates/start-jupyterhub.sh.j2

@@ -0,0 +1,4 @@
+#!/bin/bash -l
+# {{ ansible_managed }}
+env
+exec jupyterhub $@

roles/nbgrader/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
-- name: pip install 
-  pip: name={{item}} state=present executable=pip3 editable=false
+- name: pip install nbgrader
+  pip: name={{item}} state=present
   become: true
   with_items:
     # - nbgrader

roles/python/tasks/conda.yml

@@ -0,0 +1,34 @@
+---
+
+# derived from https://github.com/uchida/ansible-miniconda-role
+
+- name: miniconda installer is downloaded
+  get_url:
+    # disable validate-certs due to Python 2 SSL issues
+    # we use checksum anyway, so it should be fine
+    validate_certs: False
+    url: "https://repo.continuum.io/miniconda/{{conda_installer}}"
+    dest: "/tmp/{{conda_installer}}"
+    checksum: "{{conda_checksum}}"
+    mode: 0755
+
+- name: miniconda is installed
+  shell:
+    '"/tmp/{{ conda_installer }}" -b -p "{{ conda_prefix }}"'
+  args:
+    creates: "{{ conda_prefix }}"
+    executable: /bin/bash
+
+- name: add conda to PATH
+  # add it to the front of bashrc, so it's even on noninteractive paths
+  lineinfile:
+    dest: /etc/bash.bashrc
+    state: present
+    line: "export PATH={{conda_prefix}}/bin:$PATH"
+    insertbefore: BOF
+
+- name: add conda config
+  copy:
+    content: "{{ conda_config | to_yaml }}"
+    dest: "{{ conda_prefix }}/.condarc"
+

roles/python/tasks/jupyter.yml

@@ -1,40 +1,22 @@
 ---
 
-- name: pip install 
-  pip: name={{item}} state=present executable=pip3 editable=false
-  become: true
-  with_items:
-    # temporarily install from master until a release is made
-    # - git+https://github.com/takluyver/terminado.git#egg=terminado
-    - terminado
 
-- name: pip install ipython/jupyter deps
-  pip: name={{item}} state=present executable=pip3
+- name: conda install ipython/jupyter deps
+  conda: name={{item}} state=present
   become: true
   with_items:
-    - pexpect
-    - nose
-    - pygments
-    - Sphinx
-    - numpydoc
-    - requests
-    - jsonpointer
-    - jsonschema
-    - mistune   
-    - tornado
-    - jinja2
-    - pyzmq
+    - notebook==4.2
+    - ipyparallel
+    - ipykernel
+    - nbconvert
+    - pandoc
 
-- name: pip install jupyter
-  pip: name={{item}} state=present executable=pip3 editable=false
+- name: pip install non-conda deps
+  pip: name={{item}} state=present
   become: true
   with_items:
-      - jupyter
-      - notebook
-      - ipywidgets
-      - ipyparallel
+    - ipywidgets==5
 
 - name: install python3 kernelspec
-  command: python3 -m IPython kernelspec install-self
+  command: python3 -m IPython kernel install
   become: true
-  ignore_errors: yes

roles/python/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
 
-- include: base.yml
-- include: python3.yml
+- include: conda.yml
 - include: jupyter.yml
+- include: python3.yml

roles/python/tasks/python3.yml

@@ -1,104 +1,11 @@
 ---
 
-- name: python3 build dependencies
-  command: apt-get build-dep -y {{item}}
+- name: user conda packages
+  conda: name={{item}} state=present
   become: true
-  with_items:
-      - python3
-      - python3-numpy
-      - python3-scipy
-      - python3-matplotlib
-      - cython3
-      - python3-h5py
+  with_items: conda_packages
 
-- name: python3 base
-  apt: pkg={{item}} state=present update_cache=yes cache_valid_time=3600
+- name: user pip packages
+  pip: name={{item}} state=present
   become: true
-  with_items:
-    - python3
-    - python3-dev
-    - python3-pexpect
-    - python3-h5py
-
-- name: remove system python3 packages to be managed by pip
-  apt: pkg={{item}} state=absent
-  become: true
-  with_items:
-    - python3-pip
-    - ipython3
-    - ipython3-notebook
-    - ipython3-qtconsole
-    - python3-numpy
-    - python3-sklearn
-    - python3-skimage
-    - python3-scipy
-    - python3-matplotlib
-    - python3-pandas
-    - python3-sympy
-    - cython3
-    - python3-networkx
-    - python3-nose
-    - python3-sphinx
-    - python3-pygments
-    - python3-tk
-
-- name: pip3 install script
-  get_url: dest=/tmp/get_pip.py url=https://bootstrap.pypa.io/get-pip.py
-
-- name: pip3
-  command: python3 /tmp/get_pip.py creates=/usr/local/bin/pip3
-  become: yes
-
-# Splitting the big compiled packages off into seaprate tasks should help
-
-- name: pip install numpy
-  pip: name=numpy state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 30
-
-- name: pip install scipy
-  pip: name=scipy state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 30
-
-- name: pip install matplotlib
-  pip: name=matplotlib state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 10
-
-- name: pip install cython
-  pip: name=cython state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 10
-
-- name: pip install pandas
-  pip: name=pandas state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 10
-
-- name: pip install other packages
-  pip: name={{item}} state=present executable=pip3 editable=false
-  become: true
-  async: 3600
-  poll: 30
-  with_items:
-    - sympy
-    - scikit-learn
-    - pillow
-    - scikit-image
-    - seaborn
-    - h5py
-    - patsy
-    - ggplot
-    - statsmodels
-    - networkx
-    - ipythonblocks
-    - plotly
-    - dask
-    - blaze
-    - odo
+  with_items: pip_packages