From a51895b41d730d061091744881d3dbb58a55b377 Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 4 Apr 2017 21:38:41 +0300 Subject: [PATCH 01/12] updated configs and makefile to deploy everware as docker image --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 7aa9c99..1ad63b7 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,14 @@ install: ## install everware docker-build: ## build docker image docker build --no-cache -t everware/everware:0.10.0 . +clean: ## clean user base + if [ -f ${PIDFILE} ] ; then echo "${PIDFILE} exists, cannot continute" ; exit 1; fi + rm -f jupyterhub.sqlite + +run-linux: clean ## run everware server on linux + source ./env.sh && \ + ${EXECUTOR} -f etc/local_config.py --no-ssl 2>&1 | tee ${LOG} + test: ## run all tests export UPLOADDIR=${UPLOADDIR}; \ py.test everware/ && \ From c8610add1b6d453ff46e8066c0d9f24e16231441 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Wed, 8 Feb 2017 23:28:13 +0300 Subject: [PATCH 02/12] start implementing custom user service --- etc/local_config.py | 3 +++ everware/spawner.py | 36 +++++++++++++++++++++++++++++++++++- everware/user_spawn_handler.py | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/etc/local_config.py b/etc/local_config.py index 41ef63a..bdb5134 100644 --- a/etc/local_config.py +++ b/etc/local_config.py @@ -4,3 +4,6 @@ c = get_config() load_subconfig('etc/base_config.py') load_subconfig('etc/github_auth.py') + +# c.Spawner.custom_service_command = 'git webui' +# c.Spawner.custom_service_url = 'git' diff --git a/everware/spawner.py b/everware/spawner.py index 27a7504..e0952ca 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -14,6 +14,7 @@ from traitlets import ( Integer, Unicode, + Int ) from tornado import gen @@ -31,6 +32,10 @@ class CustomDockerSpawner(DockerSpawner, GitMixin, EmailNotificator): + custom_service_port = Int(8081, config=True) + custom_service_url = Unicode('', config=True) + custom_service_command = Unicode('', config=True) + def __init__(self, **kwargs): self._user_log = [] self._is_failed = False @@ -214,6 +219,10 @@ def is_failed(self): def is_building(self): return self._is_building + @property + def service_host(self): + return 'http://%s:%d' % (self.user.server.ip, 32001) + def _add_to_log(self, message, level=1): self._user_log.append({ 'text': message, @@ -344,8 +353,24 @@ def start(self, image=None): yield self.remove_old_container() self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') + + extra_create_kwargs = {} + extra_host_config = {} + if self.has_custom_service(): + extra_host_config = { + 'port_bindings': { + self.custom_service_port: (self.container_ip, 32001), + self.container_port: (self.container_ip,) + } + } + extra_create_kwargs = { + 'ports': [self.custom_service_port] + } + yield super(CustomDockerSpawner, self).start( - image=image_name + image=image_name, + extra_host_config=extra_host_config, + extra_create_kwargs=extra_create_kwargs ) self._add_to_log('Adding to proxy') except gen.TimeoutError: @@ -453,6 +478,9 @@ def is_running(self): status = yield self.poll() return status is None + def has_custom_service(self): + return self.custom_service_command and self.custom_service_url + def get_env(self): env = super(CustomDockerSpawner, self).get_env() env.update({ @@ -465,6 +493,12 @@ def get_env(self): 'EVER_VERSION': __version__, }) env.update(self.user_options) + + if self.has_custom_service(): + env.update({ + 'CUSTOM_SERVICE': self.custom_service_command, + 'CUSTOM_SERVICE_PORT': self.custom_service_port + }) return env diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index 0e84d0a..58da93c 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -31,6 +31,7 @@ def _spawn(self, user, form_options): # Or clones from a private repo if user.spawner.token: user.token = user.spawner.token + except Exception as e: self.log.error("Failed to spawn single-user server with form", exc_info=True) From 2dfabe0af703da67ab708f054430096d97bb31da Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Sun, 19 Feb 2017 17:51:24 +0300 Subject: [PATCH 03/12] get custom service port automatically, link to the custom service --- etc/local_config.py | 2 +- everware/home_handler.py | 9 ++++++++- everware/spawner.py | 27 +++++++++++++++++++-------- share/static/html/home.html | 3 +++ share/static/js/home.js | 1 + 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/etc/local_config.py b/etc/local_config.py index bdb5134..f9fb0e9 100644 --- a/etc/local_config.py +++ b/etc/local_config.py @@ -5,5 +5,5 @@ load_subconfig('etc/base_config.py') load_subconfig('etc/github_auth.py') -# c.Spawner.custom_service_command = 'git webui' # c.Spawner.custom_service_url = 'git' +# c.Spawner.custom_service_name = 'Git UI' diff --git a/everware/home_handler.py b/everware/home_handler.py index 0907bac..a4e5fa8 100755 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -54,10 +54,15 @@ def get(self): repo_url = '' fork_exists = False repository_changed = False + custom_service_url = None + custom_service_name = None if user.running: branch_name = user.spawner.branch_name commit_sha = user.spawner.commit_sha repo_url = user.spawner.repo_url + if user.spawner.has_custom_service(): + custom_service_url = yield user.spawner.service_host() + custom_service_name = user.spawner.custom_service_name if user.running and getattr(user, 'login_service', '') == 'github': if do_fork: self.log.info('Will fork %s' % user.spawner.repo_url) @@ -106,7 +111,9 @@ def get(self): notify_message=notify_message, version=__version__, g_analitics_id=g_id, - ya_metrica_id=ya_id + ya_metrica_id=ya_id, + custom_service_url=custom_service_url, + custom_service_name=custom_service_name ) self.finish(html) diff --git a/everware/spawner.py b/everware/spawner.py index e0952ca..a59e854 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -30,11 +30,10 @@ ssl._create_default_https_context = ssl._create_unverified_context - class CustomDockerSpawner(DockerSpawner, GitMixin, EmailNotificator): custom_service_port = Int(8081, config=True) custom_service_url = Unicode('', config=True) - custom_service_command = Unicode('', config=True) + custom_service_name = Unicode('', config=True) def __init__(self, **kwargs): self._user_log = [] @@ -219,9 +218,10 @@ def is_failed(self): def is_building(self): return self._is_building - @property + @gen.coroutine def service_host(self): - return 'http://%s:%d' % (self.user.server.ip, 32001) + ip, port = yield self.custom_service_ip_and_port() + return 'http://%s:%s' % (ip, port) def _add_to_log(self, message, level=1): self._user_log.append({ @@ -359,7 +359,7 @@ def start(self, image=None): if self.has_custom_service(): extra_host_config = { 'port_bindings': { - self.custom_service_port: (self.container_ip, 32001), + self.custom_service_port: (self.container_ip,), self.container_port: (self.container_ip,) } } @@ -398,7 +398,6 @@ def start(self, image=None): yield self.wait_up() return self.user.server.ip, self.user.server.port # jupyterhub 0.7 prefers returning ip, port - @gen.coroutine def stop(self, now=False): """Stop the container @@ -426,6 +425,10 @@ def stop(self, now=False): self.clear_state() + @property + def username(self): + return self.user_options['username'] + @gen.coroutine def notify_about_fail(self, reason): email = os.environ.get('EMAIL_SUPPORT_ADDR') @@ -479,7 +482,7 @@ def is_running(self): return status is None def has_custom_service(self): - return self.custom_service_command and self.custom_service_url + return self.custom_service_url and self.custom_service_name def get_env(self): env = super(CustomDockerSpawner, self).get_env() @@ -496,11 +499,19 @@ def get_env(self): if self.has_custom_service(): env.update({ - 'CUSTOM_SERVICE': self.custom_service_command, 'CUSTOM_SERVICE_PORT': self.custom_service_port }) return env + @gen.coroutine + def custom_service_ip_and_port(self): + resp = yield self.docker('port', self.container_id, self.custom_service_port) + if resp is None: + raise RuntimeError("Failed to get custom service port info for %s" % self.container_id) + ip = resp[0]['HostIp'] + port = resp[0]['HostPort'] + return ip, port + class CustomSwarmSpawner(CustomDockerSpawner): container_ip = '0.0.0.0' diff --git a/share/static/html/home.html b/share/static/html/home.html index 925ec3d..3d8dcf1 100755 --- a/share/static/html/home.html +++ b/share/static/html/home.html @@ -28,6 +28,9 @@
{% if user.running %} Stop My Server + {% if custom_service_url %} + {{custom_service_name}} + {% endif %} {% endif %} {% if not user.stop_pending %} Date: Sun, 19 Feb 2017 17:54:04 +0300 Subject: [PATCH 04/12] remove unnecessary changes --- everware/spawner.py | 4 ---- everware/user_spawn_handler.py | 1 - 2 files changed, 5 deletions(-) diff --git a/everware/spawner.py b/everware/spawner.py index a59e854..e3edd98 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -425,10 +425,6 @@ def stop(self, now=False): self.clear_state() - @property - def username(self): - return self.user_options['username'] - @gen.coroutine def notify_about_fail(self, reason): email = os.environ.get('EMAIL_SUPPORT_ADDR') diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index 58da93c..0e84d0a 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -31,7 +31,6 @@ def _spawn(self, user, form_options): # Or clones from a private repo if user.spawner.token: user.token = user.spawner.token - except Exception as e: self.log.error("Failed to spawn single-user server with form", exc_info=True) From 06b7d8fec95d7456b189b92e50482da7df6e8d4d Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 24 Mar 2017 01:15:43 +0300 Subject: [PATCH 05/12] run nginx inside container --- etc/nginx_config.conf | 126 +++++++++++++++++++++++++++++++++++++++++ everware/container_handler.py | 106 ++++++++++++++++++++++++++++++++++ everware/home_handler.py | 6 +- everware/spawner.py | 72 ++++++++++++++++------- everware/user_spawn_handler.py | 7 +++ 5 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 etc/nginx_config.conf create mode 100644 everware/container_handler.py diff --git a/etc/nginx_config.conf b/etc/nginx_config.conf new file mode 100644 index 0000000..8af31c8 --- /dev/null +++ b/etc/nginx_config.conf @@ -0,0 +1,126 @@ +user www-data; +worker_processes 4; +pid /run/nginx.pid; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + upstream custom_service { + server 127.0.0.1:8081; + } + server { + listen 8080; + server_name docker_proxy; + + if ($cookie_everware_custom_service_token != "%TOKEN%") { + return 403; + } + + location / { + proxy_pass http://custom_service; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + } + + location /service/%USERNAME%/ { + proxy_pass http://custom_service/; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + } + } + + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # nginx-naxsi config + ## + # Uncomment it if you installed nginx-naxsi + ## + + #include /etc/nginx/naxsi_core.rules; + + ## + # nginx-passenger config + ## + # Uncomment it if you installed nginx-passenger + ## + + #passenger_root /usr; + #passenger_ruby /usr/bin/ruby; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} + + +#mail { +# # See sample authentication script at: +# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript +# +# # auth_http localhost/auth.php; +# # pop3_capabilities "TOP" "USER"; +# # imap_capabilities "IMAP4rev1" "UIDPLUS"; +# +# server { +# listen localhost:110; +# protocol pop3; +# proxy on; +# } +# +# server { +# listen localhost:143; +# protocol imap; +# proxy on; +# } +#} + diff --git a/everware/container_handler.py b/everware/container_handler.py new file mode 100644 index 0000000..6401c2f --- /dev/null +++ b/everware/container_handler.py @@ -0,0 +1,106 @@ +from dockerspawner import DockerSpawner +from tornado import gen +import sys + +class ContainerHandler(DockerSpawner): + @gen.coroutine + def prepare_container(self, everware_based, need_service): + self.debug_log = open('debug_log', 'w') + container = yield self.get_container() + if not everware_based or need_service: + yield self._init_container(container, everware_based) + if need_service: + nginx_started = yield self._start_nginx( + container, + self.custom_service_token(), + self.user.name + ) + git_webui_started = False + if nginx_started: + self.log.info('nginx has started in %s' % self.container_id) + git_webui_started = yield self._start_service(container) + if git_webui_started: + self.log.info('git webui has started in %s' % self.container_id) + self._add_to_log('Git Web UI has started') + if not git_webui_started: + self._add_to_log('Failed to start git web ui') + + if everware_based: + return + + yield self._run_all(container) + + def _encode_conf(self, s): + return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) + + @gen.coroutine + def _start_nginx(self, container, token, username): + try: + result = '' + with open('etc/nginx_config.conf') as fin: + for line in fin: + result += self._encode_conf( + line.replace('%TOKEN%', token).replace('%USERNAME%', username) + ) + setup = yield self.docker( + 'exec_create', + container=container, + cmd="bash -c \"apt-get install nginx -y && " +\ + "python -c 'print(\\\"%s\\\")' >/etc/nginx/nginx.conf && " % result +\ + "service nginx restart && cat /etc/nginx/nginx.conf\"" + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + print(output, file=self.debug_log) + except OSError: + self.log.info('No nginx config') + return False + return True + + @gen.coroutine + def _start_service(self, container): + setup = yield self.docker( + 'exec_create', + container=container, + cmd="bash -c '"+\ + "curl https://raw.githubusercontent.com/alberthier/git-webui/master/install/installer.sh >installer.sh" +\ + " && bash installer.sh && cd {} && git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>/dev/null &'".format( + '/notebooks' + ) + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + print(output, file=sys.stderr) + return True + + @gen.coroutine + def _init_container(self, container, everware_based): + cmd = "bash -c 'apt-get update && apt-get install git wget curl -y" + if everware_based: + cmd += "'" + else: + notebook_dir = '/notebooks' + cmd += " && git clone {} {} && cd {} && git reset --hard {}'".format( + self.repo_url_with_token, + notebook_dir, + notebook_dir, + self.commit_sha + ) + + setup = yield self.docker( + 'exec_create', + container=container, + cmd=cmd + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + print(output, file=self.debug_log) + + @gen.coroutine + def _run_all(self, container): + setup = yield self.docker( + 'exec_create', + container=container, + cmd="bash -c 'apt-get install jupyter -y && "+\ + "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ + " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + print(output, file=self.debug_log) diff --git a/everware/home_handler.py b/everware/home_handler.py index a4e5fa8..d92f98c 100755 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -60,9 +60,9 @@ def get(self): branch_name = user.spawner.branch_name commit_sha = user.spawner.commit_sha repo_url = user.spawner.repo_url - if user.spawner.has_custom_service(): - custom_service_url = yield user.spawner.service_host() - custom_service_name = user.spawner.custom_service_name + if user.spawner.need_run_custom_service(): + custom_service_url = user.spawner.custom_service_path + custom_service_name = 'Git Web UI' if user.running and getattr(user, 'login_service', '') == 'github': if do_fork: self.log.info('Will fork %s' % user.spawner.repo_url) diff --git a/everware/spawner.py b/everware/spawner.py index e3edd98..50d3b95 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -17,6 +17,7 @@ Int ) from tornado import gen +from tornado.httpclient import HTTPError import ssl import json @@ -25,15 +26,15 @@ from .image_handler import ImageHandler from .git_processor import GitMixin from .email_notificator import EmailNotificator +from .container_handler import ContainerHandler from . import __version__ ssl._create_default_https_context = ssl._create_unverified_context -class CustomDockerSpawner(DockerSpawner, GitMixin, EmailNotificator): +class CustomDockerSpawner(GitMixin, EmailNotificator, ContainerHandler): + container_proxy_port = Int(8080, config=True) custom_service_port = Int(8081, config=True) - custom_service_url = Unicode('', config=True) - custom_service_name = Unicode('', config=True) def __init__(self, **kwargs): self._user_log = [] @@ -87,6 +88,10 @@ def lister(mm): else: return m(*args, **kwargs) + @property + def proxy(self): + return self.user_options['proxy'] + def clear_state(self): state = super(CustomDockerSpawner, self).clear_state() self.container_id = '' @@ -135,6 +140,21 @@ def _options_form_default(self): checked /> Remove previous container if it exists + + """ def options_from_form(self, formdata): @@ -142,7 +162,11 @@ def options_from_form(self, formdata): options['repo_url'] = formdata.get('repository_url', [''])[0].strip() options.update(formdata) need_remove = formdata.get('need_remove', ['on'])[0].strip() + need_custom_service = formdata.get('custom_service', ['off'])[0].strip() + compatible_repo = formdata.get('everware_based', ['on'])[0].strip() options['need_remove'] = need_remove == 'on' + options['custom_service'] = need_custom_service == 'on' + options['everware_based'] = compatible_repo == 'on' if not options['repo_url']: raise Exception('You have to provide the URL to a git repository.') return options @@ -171,8 +195,6 @@ def is_up(self): @gen.coroutine def get_container(self): - - # self.log.debug("Getting container: %s", self.container_name) try: container = yield self.docker( 'inspect_container', self.container_name @@ -234,10 +256,11 @@ def wait_up(self): # copied from jupyterhub, because if user's server didn't appear, it # means that spawn was unsuccessful, need to set is_failed try: - yield self.user.server.wait_up(http=True, timeout=self.http_timeout) ip, port = yield self.get_ip_and_port() self.user.server.ip = ip self.user.server.port = port + self.log.info('have PORT {}'.format(port)) + yield self.user.server.wait_up(http=True, timeout=self.http_timeout) self._is_up = True except TimeoutError: self._is_failed = True @@ -296,7 +319,7 @@ def build_image(self): 'build', path=tmp_dir, tag=image_name, - pull=True, + pull=False, rm=True, ) self._user_log.extend(self._cur_waiter.building_log) @@ -356,15 +379,15 @@ def start(self, image=None): extra_create_kwargs = {} extra_host_config = {} - if self.has_custom_service(): + if self.need_run_custom_service(): extra_host_config = { 'port_bindings': { - self.custom_service_port: (self.container_ip,), - self.container_port: (self.container_ip,) + self.container_port: (self.container_ip,), + self.container_proxy_port: (self.container_ip,) } } extra_create_kwargs = { - 'ports': [self.custom_service_port] + 'ports': [self.container_proxy_port] } yield super(CustomDockerSpawner, self).start( @@ -372,7 +395,6 @@ def start(self, image=None): extra_host_config=extra_host_config, extra_create_kwargs=extra_create_kwargs ) - self._add_to_log('Adding to proxy') except gen.TimeoutError: self._is_failed = True if self._cur_waiter: @@ -395,7 +417,16 @@ def start(self, image=None): finally: self._is_building = False + yield self.prepare_container( + self.user_options['everware_based'], + self.need_run_custom_service() + ) + self._add_to_log('Adding to proxy') yield self.wait_up() + if self.need_run_custom_service(): + yield self.proxy.api_request(self.custom_service_path, method='POST', body={ + 'target': self.container_proxy_host + }) return self.user.server.ip, self.user.server.port # jupyterhub 0.7 prefers returning ip, port @gen.coroutine @@ -422,6 +453,12 @@ def stop(self, now=False): self.container_name, self.container_id[:7]) # remove the container, as well as any associated volumes yield self.docker('remove_container', self.container_id, v=True) + finally: + if self.need_run_custom_service(): + try: + yield self.proxy.api_request(self.custom_service_path, method='DELETE') + except HTTPError: + self.log.info('failed to erase custom service of %s from proxy' % self.user.name) self.clear_state() @@ -477,8 +514,8 @@ def is_running(self): status = yield self.poll() return status is None - def has_custom_service(self): - return self.custom_service_url and self.custom_service_name + def need_run_custom_service(self): + return self.user_options.get('custom_service', False) def get_env(self): env = super(CustomDockerSpawner, self).get_env() @@ -492,16 +529,11 @@ def get_env(self): 'EVER_VERSION': __version__, }) env.update(self.user_options) - - if self.has_custom_service(): - env.update({ - 'CUSTOM_SERVICE_PORT': self.custom_service_port - }) return env @gen.coroutine def custom_service_ip_and_port(self): - resp = yield self.docker('port', self.container_id, self.custom_service_port) + resp = yield self.docker('port', self.container_id, self.container_proxy_port) if resp is None: raise RuntimeError("Failed to get custom service port info for %s" % self.container_id) ip = resp[0]['HostIp'] diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index 0e84d0a..b3c291d 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -1,6 +1,7 @@ from tornado import web, gen import jupyterhub.handlers.pages as default_handlers import sys +import uuid from . import __version__ from .metrica import MetricaIdsMixin @@ -22,9 +23,15 @@ def _render_form(self, message=''): @gen.coroutine def _spawn(self, user, form_options): + token = uuid.uuid1().hex + self.set_cookie('everware_custom_service_token', token) self.redirect('/user/%s' % user.name) try: options = user.spawner.options_from_form(form_options) + options.update({ + 'proxy': self.proxy, + 'service_token': token + }) yield self.spawn_single_user(user, options=options) # if user set another access token (for example he logged with github # and clones from bitbucket) From a7972270a43512f4e0c8e336c25f068c1c162db3 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 24 Mar 2017 19:56:45 +0300 Subject: [PATCH 06/12] working proxy in container --- etc/local_config.py | 3 -- etc/nginx_config.conf | 12 ++------ everware/container_handler.py | 65 ++++++++++++++++++++++++++++--------------- everware/git_processor.py | 18 +++++++++--- everware/spawner.py | 37 +++++++++++++----------- 5 files changed, 79 insertions(+), 56 deletions(-) diff --git a/etc/local_config.py b/etc/local_config.py index f9fb0e9..41ef63a 100644 --- a/etc/local_config.py +++ b/etc/local_config.py @@ -4,6 +4,3 @@ c = get_config() load_subconfig('etc/base_config.py') load_subconfig('etc/github_auth.py') - -# c.Spawner.custom_service_url = 'git' -# c.Spawner.custom_service_name = 'Git UI' diff --git a/etc/nginx_config.conf b/etc/nginx_config.conf index 8af31c8..e63915d 100644 --- a/etc/nginx_config.conf +++ b/etc/nginx_config.conf @@ -21,20 +21,12 @@ http { location / { proxy_pass http://custom_service; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header Host localhost:8080; } location /service/%USERNAME%/ { proxy_pass http://custom_service/; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header Host localhost:8080; } } diff --git a/everware/container_handler.py b/everware/container_handler.py index 6401c2f..270e659 100644 --- a/everware/container_handler.py +++ b/everware/container_handler.py @@ -1,12 +1,12 @@ from dockerspawner import DockerSpawner from tornado import gen -import sys +import re class ContainerHandler(DockerSpawner): @gen.coroutine - def prepare_container(self, everware_based, need_service): - self.debug_log = open('debug_log', 'w') + def prepare_container(self, need_service): container = yield self.get_container() + everware_based = yield self._is_everware_compatible(container) if not everware_based or need_service: yield self._init_container(container, everware_based) if need_service: @@ -22,13 +22,11 @@ def prepare_container(self, everware_based, need_service): if git_webui_started: self.log.info('git webui has started in %s' % self.container_id) self._add_to_log('Git Web UI has started') + else: + self.log.info('failed to start nginx in %s' % self.container_id) if not git_webui_started: self._add_to_log('Failed to start git web ui') - - if everware_based: - return - - yield self._run_all(container) + self.user_options['custom_service'] = False def _encode_conf(self, s): return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) @@ -50,11 +48,16 @@ def _start_nginx(self, container, token, username): "service nginx restart && cat /etc/nginx/nginx.conf\"" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + # print(output, file=self.debug_log) + # print(str(output), file=sys.stderr) + return re.search( + r'Restarting nginx.+?\.\.\.done\.', + str(output), + flags=re.DOTALL + ) except OSError: self.log.info('No nginx config') return False - return True @gen.coroutine def _start_service(self, container): @@ -62,18 +65,16 @@ def _start_service(self, container): 'exec_create', container=container, cmd="bash -c '"+\ - "curl https://raw.githubusercontent.com/alberthier/git-webui/master/install/installer.sh >installer.sh" +\ - " && bash installer.sh && cd {} && git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>/dev/null &'".format( - '/notebooks' - ) + "curl https://raw.githubusercontent.com/everware/git-webui/master/install/installer.sh >installer.sh" +\ + " && bash installer.sh && cd /notebooks; "+\ + "git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>&1 &'" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=sys.stderr) return True @gen.coroutine def _init_container(self, container, everware_based): - cmd = "bash -c 'apt-get update && apt-get install git wget curl -y" + cmd = "bash -c 'apt-get update && apt-get install git curl net-tools -y" if everware_based: cmd += "'" else: @@ -91,16 +92,36 @@ def _init_container(self, container, everware_based): cmd=cmd ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + + #@gen.coroutine + #def _run_jupyter(self, container): + # setup = yield self.docker( + # 'exec_create', + # container=container, + # cmd="bash -c 'apt-get install jupyter -y && "+\ + # "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ + # " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" + # ) + # output = yield self.docker('exec_start', exec_id=setup['Id']) + # print(output, file=self.debug_log) @gen.coroutine - def _run_all(self, container): + def _is_everware_compatible(self, container): setup = yield self.docker( 'exec_create', container=container, - cmd="bash -c 'apt-get install jupyter -y && "+\ - "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ - " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" + cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\"" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + return output != "" + + #@gen.coroutine + #def _is_jupyter_inside(self, container): + # setup = yield self.docker( + # 'exec_create', + # container=container, + # cmd="bash -c 'netstat -peant | grep \":8888 \"'" + # ) + # output = yield self.docker('exec_start', exec_id=setup['Id']) + # print('jupyter check:', output, file=sys.stderr) + # return output != "" diff --git a/everware/git_processor.py b/everware/git_processor.py index 5731f83..d97c60f 100644 --- a/everware/git_processor.py +++ b/everware/git_processor.py @@ -107,17 +107,27 @@ def prepare_local_repo(self): dockerfile_path = os.path.join(self._repo_dir, 'Dockerfile') if not os.path.isfile(dockerfile_path): - if not os.environ.get('DEFAULT_DOCKER_IMAGE'): + if self.config_files_exist(): + parent_image = 'everware/base:latest' + elif not os.environ.get('DEFAULT_DOCKER_IMAGE'): raise Exception('No dockerfile in repository') + else: + parent_image = os.environ['DEFAULT_DOCKER_IMAGE'] with open(dockerfile_path, 'w') as fout: fout.writelines([ - 'FROM %s\n' % os.environ['DEFAULT_DOCKER_IMAGE'], - 'MAINTAINER Alexander Tiunov ' + 'FROM %s\n' % parent_image ]) return False else: return True + def config_files_exist(self): + # uncomment when this functionality will be implemented + # for conf in ('everware.yml', 'requirements.txt', 'environment.yml'): + # cur_path = os.path.join(self._repo_dir, conf) + # if os.path.isfile(cur_path): + # return True + return False @property def escaped_repo_url(self): @@ -183,4 +193,4 @@ def get_state(self): def load_state(self, state): for key in self.STATE_VARS: if key in state: - setattr(self, key, state[key]) \ No newline at end of file + setattr(self, key, state[key]) diff --git a/everware/spawner.py b/everware/spawner.py index 50d3b95..026f9ab 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -140,14 +140,6 @@ def _options_form_default(self): checked /> Remove previous container if it exists - - """ def options_from_form(self, formdata): @@ -154,9 +140,7 @@ def options_from_form(self, formdata): options['repo_url'] = formdata.get('repository_url', [''])[0].strip() options.update(formdata) need_remove = formdata.get('need_remove', ['on'])[0].strip() - need_custom_service = formdata.get('custom_service', ['off'])[0].strip() options['need_remove'] = need_remove == 'on' - options['custom_service'] = need_custom_service == 'on' if not options['repo_url']: raise Exception('You have to provide the URL to a git repository.') return options @@ -230,11 +214,6 @@ def is_failed(self): def is_building(self): return self._is_building - @gen.coroutine - def service_host(self): - ip, port = yield self.custom_service_ip_and_port() - return 'http://%s:%s' % (ip, port) - def _add_to_log(self, message, level=1): self._user_log.append({ 'text': message, @@ -321,8 +300,6 @@ def build_image(self): raise Exception(full_output) except: raise - finally: - rmtree(tmp_dir, ignore_errors=True) return image_name @@ -368,23 +345,8 @@ def start(self, image=None): self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') - extra_create_kwargs = {} - extra_host_config = {} - if self.need_run_custom_service(): - extra_host_config = { - 'port_bindings': { - self.container_port: (self.container_ip,), - self.container_proxy_port: (self.container_ip,) - } - } - extra_create_kwargs = { - 'ports': [self.container_proxy_port] - } - yield super(CustomDockerSpawner, self).start( image=image_name, - extra_host_config=extra_host_config, - extra_create_kwargs=extra_create_kwargs ) except gen.TimeoutError: self._is_failed = True @@ -407,28 +369,16 @@ def start(self, image=None): yield self.notify_about_fail(message) self._is_building = False raise e + finally: + rmtree(self._repo_dir, ignore_errors=True) try: - yield self.prepare_container( - self.need_run_custom_service() - ) + yield self.prepare_container() except Exception as e: - self.log.warn('exception when preparing container: {}'.format(str(e))) - finally: - self._is_building = False + self.log.warn('Fail to prepare the container: {}'.format(e)) self._add_to_log('Adding to proxy') yield self.wait_up() - if self.need_run_custom_service(): - target_host = yield self.service_host() - self.log.info('Adding service of %s to proxy %s => %s' % ( - self.user.name, - self.custom_service_path, - target_host - )) - yield self.proxy.api_request(self.custom_service_path, method='POST', body={ - 'target': target_host - }) return self.user.server.ip, self.user.server.port # jupyterhub 0.7 prefers returning ip, port @gen.coroutine @@ -455,12 +405,6 @@ def stop(self, now=False): self.container_name, self.container_id[:7]) # remove the container, as well as any associated volumes yield self.docker('remove_container', self.container_id, v=True) - finally: - if self.need_run_custom_service(): - try: - yield self.proxy.api_request(self.custom_service_path, method='DELETE') - except HTTPError: - self.log.info('failed to erase custom service of %s from proxy' % self.user.name) self.clear_state() @@ -517,9 +461,6 @@ def is_running(self): status = yield self.poll() return status is None - def need_run_custom_service(self): - return self.user_options.get('custom_service', False) - def get_env(self): env = super(CustomDockerSpawner, self).get_env() env.update({ @@ -534,15 +475,6 @@ def get_env(self): env.update(self.user_options) return env - @gen.coroutine - def custom_service_ip_and_port(self): - resp = yield self.docker('port', self.container_id, self.container_proxy_port) - if resp is None: - raise RuntimeError("Failed to get custom service port info for %s" % self.container_id) - ip = resp[0]['HostIp'] - port = resp[0]['HostPort'] - return ip, port - class CustomSwarmSpawner(CustomDockerSpawner): container_ip = '0.0.0.0' diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index b3c291d..96d7101 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -29,7 +29,6 @@ def _spawn(self, user, form_options): try: options = user.spawner.options_from_form(form_options) options.update({ - 'proxy': self.proxy, 'service_token': token }) yield self.spawn_single_user(user, options=options) diff --git a/everware/user_wait_handler.py b/everware/user_wait_handler.py index 5fa3655..3d17a40 100755 --- a/everware/user_wait_handler.py +++ b/everware/user_wait_handler.py @@ -47,7 +47,7 @@ def get(self, name, user_path): else: if is_up: self.set_login_cookie(current_user) - target = '/user/%s' % ( + target = '/user/%s/' % ( current_user.name ) self.log.info('redirecting to %s' % target) diff --git a/frontend_tests/normal_scenarios.py b/frontend_tests/normal_scenarios.py index 07b3279..495d46d 100644 --- a/frontend_tests/normal_scenarios.py +++ b/frontend_tests/normal_scenarios.py @@ -96,3 +96,22 @@ def scenario_default_private_repos(user): user.wait_for_pattern_in_page(r"Launch\s+a\s+notebook") driver.find_element_by_id("logout").click() user.log("logout clicked") + +def scenario_r_shiny(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/everware/r-shiny-example") + user.log("spawn clicked") + user.wait_for_pattern_in_page("Iris\s+k-means\s+clustering") + +def scenario_jupyter_only(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/astiunov/qutip-lectures") + user.log("spawn clicked") + user.wait_for_element_present(By.LINK_TEXT, "Control Panel") + driver.find_element_by_link_text("Control Panel").click() + + diff --git a/frontend_tests/test_generator.py b/frontend_tests/test_generator.py index c3857b2..a002f32 100644 --- a/frontend_tests/test_generator.py +++ b/frontend_tests/test_generator.py @@ -17,6 +17,8 @@ normal_scenarios.scenario_no_dockerfile, normal_scenarios.scenario_default_dockerfile, # should go after no_dockerfile normal_scenarios.scenario_default_private_repos, + normal_scenarios.scenario_r_shiny, + normal_scenarios.scenario_jupyter_only, nonstop_scenarios.scenario_simple ] diff --git a/share/static/html/home.html b/share/static/html/home.html index 3d8dcf1..925ec3d 100755 --- a/share/static/html/home.html +++ b/share/static/html/home.html @@ -28,9 +28,6 @@
{% if user.running %} Stop My Server - {% if custom_service_url %} - {{custom_service_name}} - {% endif %} {% endif %} {% if not user.stop_pending %} Date: Fri, 7 Apr 2017 04:47:11 +0300 Subject: [PATCH 08/12] add pyyaml to dependencies --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c80c20..73b3d9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ ipython[notebook] coveralls nose2 +pyyaml pytest>=2.8 GitPython==2.1.1 selenium==2.52.0 jupyterhub==0.7.2 docker-py==1.10.6 --e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner \ No newline at end of file +-e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner From 252a36c79236d584cf9ef779d98367cd91f9f705 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 7 Apr 2017 05:10:00 +0300 Subject: [PATCH 09/12] fix some bugs --- everware/home_handler.py | 2 +- everware/spawner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/everware/home_handler.py b/everware/home_handler.py index 14e3203..0907bac 100755 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -106,7 +106,7 @@ def get(self): notify_message=notify_message, version=__version__, g_analitics_id=g_id, - ya_metrica_id=ya_id, + ya_metrica_id=ya_id ) self.finish(html) diff --git a/everware/spawner.py b/everware/spawner.py index 2a3661c..c9914c4 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -345,8 +345,8 @@ def start(self, image=None): self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') - yield super(CustomDockerSpawner, self).start( - image=image_name, + yield super(ContainerHandler, self).start( + image=image_name ) except gen.TimeoutError: self._is_failed = True From 7e92838acad8056a4f2d18680b9944d71fb92f08 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 14 Apr 2017 00:27:21 +0300 Subject: [PATCH 10/12] minor fix --- everware/spawner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/everware/spawner.py b/everware/spawner.py index c9914c4..f382409 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -145,6 +145,9 @@ def options_from_form(self, formdata): raise Exception('You have to provide the URL to a git repository.') return options + def custom_service_token(self): + return self.user_options['service_token'] + @property def form_repo_url(self): """Repository URL as submitted by the user.""" @@ -345,7 +348,7 @@ def start(self, image=None): self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') - yield super(ContainerHandler, self).start( + yield ContainerHandler.start(self, image=image_name ) except gen.TimeoutError: From 990bb5b4c35e706c6e041e3358c6e2434625b927 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 14 Apr 2017 00:52:07 +0300 Subject: [PATCH 11/12] increase timeout in tests --- build_tools/frontend_test_normal_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/frontend_test_normal_config.py b/build_tools/frontend_test_normal_config.py index 7684372..6a1a2cc 100644 --- a/build_tools/frontend_test_normal_config.py +++ b/build_tools/frontend_test_normal_config.py @@ -2,5 +2,5 @@ load_subconfig('etc/base_config.py') c.JupyterHub.authenticator_class = 'everware.DummyTokenAuthenticator' -c.Spawner.start_timeout = 50 +c.Spawner.start_timeout = 120 c.Spawner.http_timeout = 120 # docker sometimes doesn't show up for a while From 6d4499004c1fff6e77f13f61f4909a68d7a1a449 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 14 Apr 2017 23:54:31 +0300 Subject: [PATCH 12/12] KILL PROXY WITH FIRE --- build_tools/test_frontend.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index 4b839a0..b719a55 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -18,6 +18,7 @@ function kill_everware { pkill -KILL -f everware-server sleep $WAIT_FOR_STOP fi + pkill -KILL node || true } if [ -z "$UPLOADDIR" ] ; then