Merge pull request #186 from astiunov/master

Custom service support and some improvements

Makefile

@@ -56,6 +56,14 @@ install:  ## install everware
 docker-build: ## build docker image
 	docker build --no-cache -t everware/everware:0.10.0 .
 
+clean:  ## clean user base
+	if [ -f ${PIDFILE} ] ; then echo "${PIDFILE} exists, cannot continute" ; exit 1; fi
+	rm -f jupyterhub.sqlite
+
+run-linux: clean  ## run everware server on linux
+	source ./env.sh && \
+		${EXECUTOR} -f etc/local_config.py --no-ssl 2>&1 | tee ${LOG}
+
 test: ## run all tests
 	export UPLOADDIR=${UPLOADDIR}; \
 		py.test everware/ && \

build_tools/frontend_test_normal_config.py

@@ -2,5 +2,5 @@
 load_subconfig('etc/base_config.py')
 
 c.JupyterHub.authenticator_class = 'everware.DummyTokenAuthenticator'
-c.Spawner.start_timeout = 50
+c.Spawner.start_timeout = 120
 c.Spawner.http_timeout = 120 # docker sometimes doesn't show up for a while

build_tools/test_frontend.sh

@@ -18,6 +18,7 @@ function kill_everware {
         pkill -KILL -f everware-server
         sleep $WAIT_FOR_STOP
     fi
+    pkill -KILL node || true
 }
 
 if [ -z "$UPLOADDIR" ] ; then

etc/nginx_config.conf

@@ -0,0 +1,130 @@
+user www-data;
+worker_processes 4;
+pid /run/nginx.pid;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+    upstream custom_service {
+        server 127.0.0.1:8000;
+    }
+    server {
+        listen %PORT%;
+        server_name docker_proxy;
+
+        if ($cookie_everware_custom_service_token != "%TOKEN%") {
+            return 403;
+        }
+
+        location / {
+            proxy_pass http://custom_service;
+            proxy_set_header Host $host;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location = /user/%USERNAME% {
+            return 302 /user/%USERNAME%/;
+        }
+
+        location /user/%USERNAME%/ {
+            proxy_pass http://custom_service/;
+            proxy_set_header Host $host;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+
+
+	##
+	# Basic Settings
+	##
+
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	# server_tokens off;
+
+	# server_names_hash_bucket_size 64;
+	# server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# Logging Settings
+	##
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+	gzip_disable "msie6";
+
+	# gzip_vary on;
+	# gzip_proxied any;
+	# gzip_comp_level 6;
+	# gzip_buffers 16 8k;
+	# gzip_http_version 1.1;
+	# gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+	##
+	# nginx-naxsi config
+	##
+	# Uncomment it if you installed nginx-naxsi
+	##
+
+	#include /etc/nginx/naxsi_core.rules;
+
+	##
+	# nginx-passenger config
+	##
+	# Uncomment it if you installed nginx-passenger
+	##
+	
+	#passenger_root /usr;
+	#passenger_ruby /usr/bin/ruby;
+
+	##
+	# Virtual Host Configs
+	##
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}
+
+
+#mail {
+#	# See sample authentication script at:
+#	# http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
+# 
+#	# auth_http localhost/auth.php;
+#	# pop3_capabilities "TOP" "USER";
+#	# imap_capabilities "IMAP4rev1" "UIDPLUS";
+# 
+#	server {
+#		listen     localhost:110;
+#		protocol   pop3;
+#		proxy      on;
+#	}
+# 
+#	server {
+#		listen     localhost:143;
+#		protocol   imap;
+#		proxy      on;
+#	}
+#}
+

everware/container_handler.py

@@ -0,0 +1,142 @@
+from dockerspawner import DockerSpawner
+from tornado import gen
+import re
+import os.path
+import sys
+import yaml
+
+class ShellCommand:
+    def __init__(self, commands=[]):
+        self.commands = commands
+
+    def add_commands(self, commands_list):
+        self.commands.extend(commands_list)
+
+    def extend(self, command):
+        self.add_commands(command.commands)
+
+    def get_single_command(self):
+        return 'bash -c "{}"' .format(' && '.join(self.commands))
+
+def make_git_command(repourl, commit_sha):
+    return ShellCommand([
+        'apt-get install -y git',
+        'git clone {} /notebooks'.format(repourl),
+        'cd /notebooks',
+        'git reset --hard {}'.format(commit_sha)
+    ])
+
+def make_nginx_start_command(nginx_config):
+    return ShellCommand([
+        'apt-get install -y nginx',
+        "python -c 'print(\\\"{}\\\")' >/etc/nginx/nginx.conf".format(nginx_config),
+        'service nginx restart',
+        'cat /etc/nginx/nginx.conf'
+    ])
+
+def make_default_start_command(env):
+    return ShellCommand([
+        'jupyterhub-singleuser --port=8888 --ip=0.0.0.0 --allow-root --user={} --cookie-name={} --base-url={} '.format(
+            env['JPY_USER'],
+            env['JPY_COOKIE_NAME'],
+            env['JPY_BASE_URL']
+        ) + '--hub-prefix={} --hub-api-url={} --notebook-dir=/notebooks'.format(
+            env['JPY_HUB_PREFIX'],
+            env['JPY_HUB_API_URL']
+        )
+    ])
+
+def make_custom_start_command(command):
+    return ShellCommand([command])
+
+
+class ContainerHandler(DockerSpawner):
+    def parse_config(self, directory):
+        self.everware_config = {
+            'everware_based': True
+        }
+        try:
+            with open(os.path.join(directory, 'everware.yml')) as fin:
+                try:
+                    self.everware_config = yaml.load(fin)
+                except yaml.YAMLError as exc:
+                    self.log.warn('Fail reading everware.yml: {}'.format(exc))
+        except IOError:
+            self.log.info('No everware.yaml in repo')
+
+    @gen.coroutine
+    def prepare_container(self):
+        if self.everware_config.get('everware_based', True):
+            return
+        container = yield self.get_container()
+        was_cloned = yield self._check_for_git_compatibility(container)
+        if not was_cloned:
+            command = make_git_command(self.repo_url_with_token, self.commit_sha)
+            setup = yield self.docker(
+                'exec_create',
+                container=container,
+                cmd=command.get_single_command()
+            )
+            output = yield self.docker('exec_start', exec_id=setup['Id'])
+
+
+    @gen.coroutine
+    def start(self, image=None):
+        self.parse_config(self._repo_dir)
+        start_command = None
+        extra_create_kwargs = {
+            'ports': [self.container_port]
+        }
+        if not self.everware_config.get('everware_based', True):
+            start_command = make_git_command(self.repo_url_with_token, self.commit_sha)
+            if 'start_command' in self.everware_config:
+                nginx_config = self._get_nginx_config(
+                    8888,
+                    self.custom_service_token(),
+                    self.user.name
+                )
+                start_command.extend(make_nginx_start_command(nginx_config))
+                start_command.extend(make_custom_start_command(self.everware_config['start_command']))
+            else:
+                start_command.extend(make_default_start_command(self.get_env()))
+            extra_create_kwargs.update({
+                'command': start_command.get_single_command()
+            })
+
+        extra_host_config = {
+            'port_bindings': {
+                self.container_port: (self.container_ip,)
+            }
+        }
+        ip, port = yield DockerSpawner.start(self, image,
+                            extra_create_kwargs=extra_create_kwargs,
+                            extra_host_config=extra_host_config)
+        return ip, port
+
+    def _encode_conf(self, s):
+        return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s)
+
+    def _get_nginx_config(self, port, token, username):
+        try:
+            result = ''
+            with open('etc/nginx_config.conf') as fin:
+                for line in fin:
+                    result += self._encode_conf(
+                        line.replace('%TOKEN%', token)
+                        .replace('%USERNAME%', username)
+                        .replace('%PORT%', str(port))
+                    )
+            return result
+        except OSError:
+            self.log.warn('No nginx config')
+            raise
+
+    @gen.coroutine
+    def _check_for_git_compatibility(self, container):
+        setup = yield self.docker(
+            'exec_create',
+            container=container,
+            cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\""
+        )
+        output = yield self.docker('exec_start', exec_id=setup['Id'])
+        return output != ""

everware/git_processor.py

@@ -107,17 +107,27 @@ def prepare_local_repo(self):
 
         dockerfile_path = os.path.join(self._repo_dir, 'Dockerfile')
         if not os.path.isfile(dockerfile_path):
-            if not os.environ.get('DEFAULT_DOCKER_IMAGE'):
+            if self.config_files_exist():
+                parent_image = 'everware/base:latest'
+            elif not os.environ.get('DEFAULT_DOCKER_IMAGE'):
                 raise Exception('No dockerfile in repository')
+            else:
+                parent_image = os.environ['DEFAULT_DOCKER_IMAGE']
             with open(dockerfile_path, 'w') as fout:
                 fout.writelines([
-                    'FROM %s\n' % os.environ['DEFAULT_DOCKER_IMAGE'],
-                    'MAINTAINER Alexander Tiunov <astiunov@yandex-team.ru>'
+                    'FROM %s\n' % parent_image
                 ])
             return False
         else:
             return True
 
+    def config_files_exist(self):
+        # uncomment when this functionality will be implemented
+        # for conf in ('everware.yml', 'requirements.txt', 'environment.yml'):
+        #    cur_path = os.path.join(self._repo_dir, conf)
+        #    if os.path.isfile(cur_path):
+        #        return True
+        return False
 
     @property
     def escaped_repo_url(self):
@@ -183,4 +193,4 @@ def get_state(self):
     def load_state(self, state):
         for key in self.STATE_VARS:
             if key in state:
-                setattr(self, key, state[key])
+                setattr(self, key, state[key])