Merge pull request #18 from randy3k/cull

add cull_idle_servers

deploy.yml

@@ -11,4 +11,5 @@
     - saveusers
     - bash
     - jupyterhub
+    - { role: cull_idle, when: use_cull_idle_servers}
     - nbgrader

host_vars/hostname.example

@@ -68,6 +68,17 @@ nginx_public_html: false
 # as root.
 github_usernames: ['instructor', 'grader']
 
+# Optionally cull idle single user servers
+use_cull_idle_servers: false
+# The username and API token for culling idle servers
+# Create using something like `openssl rand -hex 32`
+cull_idle_servers_owner: instructor
+cull_idle_servers_hubapi_token: ''
+# The interval (in seconds) for checking for idle servers to cull
+cull_every: 600
+# The idle timeout (in seconds)
+cull_timeout: 3600
+
 # Optionally set this as your Google Analytics Tracking ID
 ga_tracking_id: ''
 

roles/cull_idle/files/cull_idle_servers.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+"""script to monitor and cull idle single-user servers
+
+Caveats:
+
+last_activity is not updated with high frequency,
+so cull timeout should be greater than the sum of:
+
+- single-user websocket ping interval (default: 30s)
+- JupyterHub.last_activity_interval (default: 5 minutes)
+
+Generate an API token and store it in `JPY_API_TOKEN`:
+
+    export JPY_API_TOKEN=`jupyterhub token`
+    python cull_idle_servers.py [--timeout=900] [--url=http://127.0.0.1:8081/hub]
+"""
+
+import datetime
+import json
+import os
+
+from dateutil.parser import parse as parse_date
+
+from tornado.gen import coroutine
+from tornado.log import app_log
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
+from tornado.ioloop import IOLoop, PeriodicCallback
+from tornado.options import define, options, parse_command_line
+
+
+@coroutine
+def cull_idle(url, api_token, timeout):
+    """cull idle single-user servers"""
+    auth_header = {
+            'Authorization': 'token %s' % api_token
+        }
+    req = HTTPRequest(url=url + '/api/users',
+        headers=auth_header,
+    )
+    now = datetime.datetime.utcnow()
+    cull_limit = now - datetime.timedelta(seconds=timeout)
+    client = AsyncHTTPClient()
+    resp = yield client.fetch(req)
+    users = json.loads(resp.body.decode('utf8', 'replace'))
+    futures = []
+    for user in users:
+        last_activity = parse_date(user['last_activity'])
+        if user['server'] and last_activity < cull_limit:
+            app_log.info("Culling %s (inactive since %s)", user['name'], last_activity)
+            req = HTTPRequest(url=url + '/api/users/%s/server' % user['name'],
+                method='DELETE',
+                headers=auth_header,
+            )
+            futures.append((user['name'], client.fetch(req)))
+        elif user['server'] and last_activity > cull_limit:
+            app_log.debug("Not culling %s (active since %s)", user['name'], last_activity)
+
+    for (name, f) in futures:
+        yield f
+        app_log.debug("Finished culling %s", name)
+
+if __name__ == '__main__':
+    define('url', default='http://127.0.0.1:8081/hub', help="The JupyterHub API URL")
+    define('timeout', default=600, help="The idle timeout (in seconds)")
+    define('cull_every', default=0, help="The interval (in seconds) for checking for idle servers to cull")
+
+    parse_command_line()
+    if not options.cull_every:
+        options.cull_every = options.timeout // 2
+
+    api_token = os.environ['JPY_API_TOKEN']
+
+    loop = IOLoop.current()
+    cull = lambda : cull_idle(options.url, api_token, options.timeout)
+    # run once before scheduling periodic call
+    loop.run_sync(cull)
+    # schedule periodic cull
+    pc = PeriodicCallback(cull, 1e3 * options.cull_every)
+    pc.start()
+    try:
+        loop.start()
+    except KeyboardInterrupt:
+        pass
+

roles/cull_idle/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+
+- name: install cull_idle_servers dependencies
+  pip: name=python-dateutil state=present
+
+- name: install cull_idle_servers.py into {{jupyterhub_srv_dir}}
+  copy: src=cull_idle_servers.py dest={{jupyterhub_srv_dir}} owner=root group=root mode=0700
+
+- name: install supervisor config for cull_idle_servers
+  template: src=cull_idle_servers.conf.j2 dest=/etc/supervisor/conf.d/cull_idle_servers.conf owner=root group=root mode=0600
+
+- name: load cull_idle_servers supervisor config
+  supervisorctl: name=cull_idle_servers state=present
+
+- name: restart cull_idle_servers with supervisor
+  supervisorctl: name=cull_idle_servers state=restarted

roles/cull_idle/templates/cull_idle_servers.conf.j2

@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+
+[program:cull_idle_servers]
+environment=JPY_API_TOKEN='{{ cull_idle_servers_hubapi_token }}'
+command=/opt/conda/bin/python3 {{jupyterhub_srv_dir}}/cull_idle_servers.py --cull-every={{ cull_every }} --timeout={{ cull_timeout}}
+redirect_stderr=true
+stdout_logfile={{ jupyterhub_log_dir }}/cull_idle_servers.log
+autostart=true
+autorestart=false
+stopasgroup=true
+user=root
+directory={{jupyterhub_srv_dir}}

roles/jupyterhub/templates/jupyterhub_config.py.j2

@@ -38,8 +38,14 @@ c.Authenticator.whitelist = {
 c.Authenticator.whitelist = set()
 {% endif %}
 
+c.JupyterHub.api_tokens = {}
 {% if formgrader_hubapi_token %}
-c.JupyterHub.api_tokens = {
-    '{{formgrader_hubapi_token}}': '{{nbgrader_owner}}',
-}
+c.JupyterHub.api_tokens.update({
+    '{{formgrader_hubapi_token}}': '{{nbgrader_owner}}'
+})
+{% endif %}
+{% if cull_idle_servers_hubapi_token %}
+c.JupyterHub.api_tokens.update({
+    '{{cull_idle_servers_hubapi_token}}': '{{cull_idle_servers_owner}}'
+})
 {% endif %}