From e178eb862406710d92397d322a54160c8cd81d13 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 7 Apr 2017 04:45:45 +0300 Subject: [PATCH] support for custom run command and auto git clone inside container --- etc/nginx_config.conf | 22 +++- everware/container_handler.py | 211 ++++++++++++++++++++----------------- everware/home_handler.py | 7 -- everware/spawner.py | 78 +------------- everware/user_spawn_handler.py | 1 - everware/user_wait_handler.py | 2 +- frontend_tests/normal_scenarios.py | 19 ++++ frontend_tests/test_generator.py | 2 + share/static/html/home.html | 3 - share/static/js/home.js | 1 - 10 files changed, 157 insertions(+), 189 deletions(-) diff --git a/etc/nginx_config.conf b/etc/nginx_config.conf index e63915d..dd26642 100644 --- a/etc/nginx_config.conf +++ b/etc/nginx_config.conf @@ -9,10 +9,10 @@ events { http { upstream custom_service { - server 127.0.0.1:8081; + server 127.0.0.1:8000; } server { - listen 8080; + listen %PORT%; server_name docker_proxy; if ($cookie_everware_custom_service_token != "%TOKEN%") { @@ -21,12 +21,24 @@ http { location / { proxy_pass http://custom_service; - proxy_set_header Host localhost:8080; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } - location /service/%USERNAME%/ { + location = /user/%USERNAME% { + return 302 /user/%USERNAME%/; + } + + location /user/%USERNAME%/ { proxy_pass http://custom_service/; - proxy_set_header Host localhost:8080; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } } diff --git a/everware/container_handler.py b/everware/container_handler.py index 270e659..9e3acb0 100644 --- a/everware/container_handler.py +++ b/everware/container_handler.py @@ -1,112 +1,138 @@ from dockerspawner import DockerSpawner from tornado import gen import re +import os.path +import sys +import yaml + +class ShellCommand: + def __init__(self, commands=[]): + self.commands = commands + + def add_commands(self, commands_list): + self.commands.extend(commands_list) + + def extend(self, command): + self.add_commands(command.commands) + + def get_single_command(self): + return 'bash -c "{}"' .format(' && '.join(self.commands)) + +def make_git_command(repourl, commit_sha): + return ShellCommand([ + 'apt-get install -y git', + 'git clone {} /notebooks'.format(repourl), + 'cd /notebooks', + 'git reset --hard {}'.format(commit_sha) + ]) + +def make_nginx_start_command(nginx_config): + return ShellCommand([ + 'apt-get install -y nginx', + "python -c 'print(\\\"{}\\\")' >/etc/nginx/nginx.conf".format(nginx_config), + 'service nginx restart', + 'cat /etc/nginx/nginx.conf' + ]) + +def make_default_start_command(env): + return ShellCommand([ + 'jupyterhub-singleuser --port=8888 --ip=0.0.0.0 --allow-root --user={} --cookie-name={} --base-url={} '.format( + env['JPY_USER'], + env['JPY_COOKIE_NAME'], + env['JPY_BASE_URL'] + ) + '--hub-prefix={} --hub-api-url={} --notebook-dir=/notebooks'.format( + env['JPY_HUB_PREFIX'], + env['JPY_HUB_API_URL'] + ) + ]) + +def make_custom_start_command(command): + return ShellCommand([command]) + class ContainerHandler(DockerSpawner): + def parse_config(self, directory): + self.everware_config = { + 'everware_based': True + } + try: + with open(os.path.join(directory, 'everware.yml')) as fin: + try: + self.everware_config = yaml.load(fin) + except yaml.YAMLError as exc: + self.log.warn('Fail reading everware.yml: {}'.format(exc)) + except IOError: + self.log.info('No everware.yaml in repo') + @gen.coroutine - def prepare_container(self, need_service): + def prepare_container(self): + if self.everware_config.get('everware_based', True): + return container = yield self.get_container() - everware_based = yield self._is_everware_compatible(container) - if not everware_based or need_service: - yield self._init_container(container, everware_based) - if need_service: - nginx_started = yield self._start_nginx( - container, - self.custom_service_token(), - self.user.name + was_cloned = yield self._check_for_git_compatibility(container) + if not was_cloned: + command = make_git_command(self.repo_url_with_token, self.commit_sha) + setup = yield self.docker( + 'exec_create', + container=container, + cmd=command.get_single_command() ) - git_webui_started = False - if nginx_started: - self.log.info('nginx has started in %s' % self.container_id) - git_webui_started = yield self._start_service(container) - if git_webui_started: - self.log.info('git webui has started in %s' % self.container_id) - self._add_to_log('Git Web UI has started') + output = yield self.docker('exec_start', exec_id=setup['Id']) + + + @gen.coroutine + def start(self, image=None): + self.parse_config(self._repo_dir) + start_command = None + extra_create_kwargs = { + 'ports': [self.container_port] + } + if not self.everware_config.get('everware_based', True): + start_command = make_git_command(self.repo_url_with_token, self.commit_sha) + if 'start_command' in self.everware_config: + nginx_config = self._get_nginx_config( + 8888, + self.custom_service_token(), + self.user.name + ) + start_command.extend(make_nginx_start_command(nginx_config)) + start_command.extend(make_custom_start_command(self.everware_config['start_command'])) else: - self.log.info('failed to start nginx in %s' % self.container_id) - if not git_webui_started: - self._add_to_log('Failed to start git web ui') - self.user_options['custom_service'] = False + start_command.extend(make_default_start_command(self.get_env())) + extra_create_kwargs.update({ + 'command': start_command.get_single_command() + }) + + extra_host_config = { + 'port_bindings': { + self.container_port: (self.container_ip,) + } + } + ip, port = yield DockerSpawner.start(self, image, + extra_create_kwargs=extra_create_kwargs, + extra_host_config=extra_host_config) + return ip, port def _encode_conf(self, s): return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) - @gen.coroutine - def _start_nginx(self, container, token, username): + def _get_nginx_config(self, port, token, username): try: result = '' with open('etc/nginx_config.conf') as fin: for line in fin: result += self._encode_conf( - line.replace('%TOKEN%', token).replace('%USERNAME%', username) + line.replace('%TOKEN%', token) + .replace('%USERNAME%', username) + .replace('%PORT%', str(port)) ) - setup = yield self.docker( - 'exec_create', - container=container, - cmd="bash -c \"apt-get install nginx -y && " +\ - "python -c 'print(\\\"%s\\\")' >/etc/nginx/nginx.conf && " % result +\ - "service nginx restart && cat /etc/nginx/nginx.conf\"" - ) - output = yield self.docker('exec_start', exec_id=setup['Id']) - # print(output, file=self.debug_log) - # print(str(output), file=sys.stderr) - return re.search( - r'Restarting nginx.+?\.\.\.done\.', - str(output), - flags=re.DOTALL - ) + return result except OSError: - self.log.info('No nginx config') - return False - - @gen.coroutine - def _start_service(self, container): - setup = yield self.docker( - 'exec_create', - container=container, - cmd="bash -c '"+\ - "curl https://raw.githubusercontent.com/everware/git-webui/master/install/installer.sh >installer.sh" +\ - " && bash installer.sh && cd /notebooks; "+\ - "git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>&1 &'" - ) - output = yield self.docker('exec_start', exec_id=setup['Id']) - return True + self.log.warn('No nginx config') + raise @gen.coroutine - def _init_container(self, container, everware_based): - cmd = "bash -c 'apt-get update && apt-get install git curl net-tools -y" - if everware_based: - cmd += "'" - else: - notebook_dir = '/notebooks' - cmd += " && git clone {} {} && cd {} && git reset --hard {}'".format( - self.repo_url_with_token, - notebook_dir, - notebook_dir, - self.commit_sha - ) - - setup = yield self.docker( - 'exec_create', - container=container, - cmd=cmd - ) - output = yield self.docker('exec_start', exec_id=setup['Id']) - - #@gen.coroutine - #def _run_jupyter(self, container): - # setup = yield self.docker( - # 'exec_create', - # container=container, - # cmd="bash -c 'apt-get install jupyter -y && "+\ - # "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ - # " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" - # ) - # output = yield self.docker('exec_start', exec_id=setup['Id']) - # print(output, file=self.debug_log) - - @gen.coroutine - def _is_everware_compatible(self, container): + def _check_for_git_compatibility(self, container): setup = yield self.docker( 'exec_create', container=container, @@ -114,14 +140,3 @@ def _is_everware_compatible(self, container): ) output = yield self.docker('exec_start', exec_id=setup['Id']) return output != "" - - #@gen.coroutine - #def _is_jupyter_inside(self, container): - # setup = yield self.docker( - # 'exec_create', - # container=container, - # cmd="bash -c 'netstat -peant | grep \":8888 \"'" - # ) - # output = yield self.docker('exec_start', exec_id=setup['Id']) - # print('jupyter check:', output, file=sys.stderr) - # return output != "" diff --git a/everware/home_handler.py b/everware/home_handler.py index d92f98c..14e3203 100755 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -54,15 +54,10 @@ def get(self): repo_url = '' fork_exists = False repository_changed = False - custom_service_url = None - custom_service_name = None if user.running: branch_name = user.spawner.branch_name commit_sha = user.spawner.commit_sha repo_url = user.spawner.repo_url - if user.spawner.need_run_custom_service(): - custom_service_url = user.spawner.custom_service_path - custom_service_name = 'Git Web UI' if user.running and getattr(user, 'login_service', '') == 'github': if do_fork: self.log.info('Will fork %s' % user.spawner.repo_url) @@ -112,8 +107,6 @@ def get(self): version=__version__, g_analitics_id=g_id, ya_metrica_id=ya_id, - custom_service_url=custom_service_url, - custom_service_name=custom_service_name ) self.finish(html) diff --git a/everware/spawner.py b/everware/spawner.py index 026f9ab..2a3661c 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -33,9 +33,6 @@ ssl._create_default_https_context = ssl._create_unverified_context class CustomDockerSpawner(GitMixin, EmailNotificator, ContainerHandler): - container_proxy_port = Int(8080, config=True) - custom_service_port = Int(8081, config=True) - def __init__(self, **kwargs): self._user_log = [] self._is_failed = False @@ -44,7 +41,7 @@ def __init__(self, **kwargs): self._image_handler = ImageHandler() self._cur_waiter = None self._is_empty = False - DockerSpawner.__init__(self, **kwargs) + ContainerHandler.__init__(self, **kwargs) EmailNotificator.__init__(self) @@ -88,10 +85,6 @@ def lister(mm): else: return m(*args, **kwargs) - @property - def proxy(self): - return self.user_options['proxy'] - def clear_state(self): state = super(CustomDockerSpawner, self).clear_state() self.container_id = '' @@ -140,13 +133,6 @@ def _options_form_default(self): checked /> Remove previous container if it exists - """ def options_from_form(self, formdata): @@ -154,9 +140,7 @@ def options_from_form(self, formdata): options['repo_url'] = formdata.get('repository_url', [''])[0].strip() options.update(formdata) need_remove = formdata.get('need_remove', ['on'])[0].strip() - need_custom_service = formdata.get('custom_service', ['off'])[0].strip() options['need_remove'] = need_remove == 'on' - options['custom_service'] = need_custom_service == 'on' if not options['repo_url']: raise Exception('You have to provide the URL to a git repository.') return options @@ -230,11 +214,6 @@ def is_failed(self): def is_building(self): return self._is_building - @gen.coroutine - def service_host(self): - ip, port = yield self.custom_service_ip_and_port() - return 'http://%s:%s' % (ip, port) - def _add_to_log(self, message, level=1): self._user_log.append({ 'text': message, @@ -321,8 +300,6 @@ def build_image(self): raise Exception(full_output) except: raise - finally: - rmtree(tmp_dir, ignore_errors=True) return image_name @@ -368,23 +345,8 @@ def start(self, image=None): self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') - extra_create_kwargs = {} - extra_host_config = {} - if self.need_run_custom_service(): - extra_host_config = { - 'port_bindings': { - self.container_port: (self.container_ip,), - self.container_proxy_port: (self.container_ip,) - } - } - extra_create_kwargs = { - 'ports': [self.container_proxy_port] - } - yield super(CustomDockerSpawner, self).start( image=image_name, - extra_host_config=extra_host_config, - extra_create_kwargs=extra_create_kwargs ) except gen.TimeoutError: self._is_failed = True @@ -407,28 +369,16 @@ def start(self, image=None): yield self.notify_about_fail(message) self._is_building = False raise e + finally: + rmtree(self._repo_dir, ignore_errors=True) try: - yield self.prepare_container( - self.need_run_custom_service() - ) + yield self.prepare_container() except Exception as e: - self.log.warn('exception when preparing container: {}'.format(str(e))) - finally: - self._is_building = False + self.log.warn('Fail to prepare the container: {}'.format(e)) self._add_to_log('Adding to proxy') yield self.wait_up() - if self.need_run_custom_service(): - target_host = yield self.service_host() - self.log.info('Adding service of %s to proxy %s => %s' % ( - self.user.name, - self.custom_service_path, - target_host - )) - yield self.proxy.api_request(self.custom_service_path, method='POST', body={ - 'target': target_host - }) return self.user.server.ip, self.user.server.port # jupyterhub 0.7 prefers returning ip, port @gen.coroutine @@ -455,12 +405,6 @@ def stop(self, now=False): self.container_name, self.container_id[:7]) # remove the container, as well as any associated volumes yield self.docker('remove_container', self.container_id, v=True) - finally: - if self.need_run_custom_service(): - try: - yield self.proxy.api_request(self.custom_service_path, method='DELETE') - except HTTPError: - self.log.info('failed to erase custom service of %s from proxy' % self.user.name) self.clear_state() @@ -517,9 +461,6 @@ def is_running(self): status = yield self.poll() return status is None - def need_run_custom_service(self): - return self.user_options.get('custom_service', False) - def get_env(self): env = super(CustomDockerSpawner, self).get_env() env.update({ @@ -534,15 +475,6 @@ def get_env(self): env.update(self.user_options) return env - @gen.coroutine - def custom_service_ip_and_port(self): - resp = yield self.docker('port', self.container_id, self.container_proxy_port) - if resp is None: - raise RuntimeError("Failed to get custom service port info for %s" % self.container_id) - ip = resp[0]['HostIp'] - port = resp[0]['HostPort'] - return ip, port - class CustomSwarmSpawner(CustomDockerSpawner): container_ip = '0.0.0.0' diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index b3c291d..96d7101 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -29,7 +29,6 @@ def _spawn(self, user, form_options): try: options = user.spawner.options_from_form(form_options) options.update({ - 'proxy': self.proxy, 'service_token': token }) yield self.spawn_single_user(user, options=options) diff --git a/everware/user_wait_handler.py b/everware/user_wait_handler.py index 5fa3655..3d17a40 100755 --- a/everware/user_wait_handler.py +++ b/everware/user_wait_handler.py @@ -47,7 +47,7 @@ def get(self, name, user_path): else: if is_up: self.set_login_cookie(current_user) - target = '/user/%s' % ( + target = '/user/%s/' % ( current_user.name ) self.log.info('redirecting to %s' % target) diff --git a/frontend_tests/normal_scenarios.py b/frontend_tests/normal_scenarios.py index 07b3279..495d46d 100644 --- a/frontend_tests/normal_scenarios.py +++ b/frontend_tests/normal_scenarios.py @@ -96,3 +96,22 @@ def scenario_default_private_repos(user): user.wait_for_pattern_in_page(r"Launch\s+a\s+notebook") driver.find_element_by_id("logout").click() user.log("logout clicked") + +def scenario_r_shiny(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/everware/r-shiny-example") + user.log("spawn clicked") + user.wait_for_pattern_in_page("Iris\s+k-means\s+clustering") + +def scenario_jupyter_only(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/astiunov/qutip-lectures") + user.log("spawn clicked") + user.wait_for_element_present(By.LINK_TEXT, "Control Panel") + driver.find_element_by_link_text("Control Panel").click() + + diff --git a/frontend_tests/test_generator.py b/frontend_tests/test_generator.py index c3857b2..a002f32 100644 --- a/frontend_tests/test_generator.py +++ b/frontend_tests/test_generator.py @@ -17,6 +17,8 @@ normal_scenarios.scenario_no_dockerfile, normal_scenarios.scenario_default_dockerfile, # should go after no_dockerfile normal_scenarios.scenario_default_private_repos, + normal_scenarios.scenario_r_shiny, + normal_scenarios.scenario_jupyter_only, nonstop_scenarios.scenario_simple ] diff --git a/share/static/html/home.html b/share/static/html/home.html index 3d8dcf1..925ec3d 100755 --- a/share/static/html/home.html +++ b/share/static/html/home.html @@ -28,9 +28,6 @@
{% if user.running %} Stop My Server - {% if custom_service_url %} - {{custom_service_name}} - {% endif %} {% endif %} {% if not user.stop_pending %}