From a7972270a43512f4e0c8e336c25f068c1c162db3 Mon Sep 17 00:00:00 2001 From: Alexander Tiunov Date: Fri, 24 Mar 2017 19:56:45 +0300 Subject: [PATCH] working proxy in container --- etc/local_config.py | 3 -- etc/nginx_config.conf | 12 ++------ everware/container_handler.py | 65 ++++++++++++++++++++++++++++--------------- everware/git_processor.py | 18 +++++++++--- everware/spawner.py | 37 +++++++++++++----------- 5 files changed, 79 insertions(+), 56 deletions(-) diff --git a/etc/local_config.py b/etc/local_config.py index f9fb0e9..41ef63a 100644 --- a/etc/local_config.py +++ b/etc/local_config.py @@ -4,6 +4,3 @@ c = get_config() load_subconfig('etc/base_config.py') load_subconfig('etc/github_auth.py') - -# c.Spawner.custom_service_url = 'git' -# c.Spawner.custom_service_name = 'Git UI' diff --git a/etc/nginx_config.conf b/etc/nginx_config.conf index 8af31c8..e63915d 100644 --- a/etc/nginx_config.conf +++ b/etc/nginx_config.conf @@ -21,20 +21,12 @@ http { location / { proxy_pass http://custom_service; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header Host localhost:8080; } location /service/%USERNAME%/ { proxy_pass http://custom_service/; - proxy_redirect off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $server_name; + proxy_set_header Host localhost:8080; } } diff --git a/everware/container_handler.py b/everware/container_handler.py index 6401c2f..270e659 100644 --- a/everware/container_handler.py +++ b/everware/container_handler.py @@ -1,12 +1,12 @@ from dockerspawner import DockerSpawner from tornado import gen -import sys +import re class ContainerHandler(DockerSpawner): @gen.coroutine - def prepare_container(self, everware_based, need_service): - self.debug_log = open('debug_log', 'w') + def prepare_container(self, need_service): container = yield self.get_container() + everware_based = yield self._is_everware_compatible(container) if not everware_based or need_service: yield self._init_container(container, everware_based) if need_service: @@ -22,13 +22,11 @@ def prepare_container(self, everware_based, need_service): if git_webui_started: self.log.info('git webui has started in %s' % self.container_id) self._add_to_log('Git Web UI has started') + else: + self.log.info('failed to start nginx in %s' % self.container_id) if not git_webui_started: self._add_to_log('Failed to start git web ui') - - if everware_based: - return - - yield self._run_all(container) + self.user_options['custom_service'] = False def _encode_conf(self, s): return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) @@ -50,11 +48,16 @@ def _start_nginx(self, container, token, username): "service nginx restart && cat /etc/nginx/nginx.conf\"" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + # print(output, file=self.debug_log) + # print(str(output), file=sys.stderr) + return re.search( + r'Restarting nginx.+?\.\.\.done\.', + str(output), + flags=re.DOTALL + ) except OSError: self.log.info('No nginx config') return False - return True @gen.coroutine def _start_service(self, container): @@ -62,18 +65,16 @@ def _start_service(self, container): 'exec_create', container=container, cmd="bash -c '"+\ - "curl https://raw.githubusercontent.com/alberthier/git-webui/master/install/installer.sh >installer.sh" +\ - " && bash installer.sh && cd {} && git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>/dev/null &'".format( - '/notebooks' - ) + "curl https://raw.githubusercontent.com/everware/git-webui/master/install/installer.sh >installer.sh" +\ + " && bash installer.sh && cd /notebooks; "+\ + "git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>&1 &'" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=sys.stderr) return True @gen.coroutine def _init_container(self, container, everware_based): - cmd = "bash -c 'apt-get update && apt-get install git wget curl -y" + cmd = "bash -c 'apt-get update && apt-get install git curl net-tools -y" if everware_based: cmd += "'" else: @@ -91,16 +92,36 @@ def _init_container(self, container, everware_based): cmd=cmd ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + + #@gen.coroutine + #def _run_jupyter(self, container): + # setup = yield self.docker( + # 'exec_create', + # container=container, + # cmd="bash -c 'apt-get install jupyter -y && "+\ + # "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ + # " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" + # ) + # output = yield self.docker('exec_start', exec_id=setup['Id']) + # print(output, file=self.debug_log) @gen.coroutine - def _run_all(self, container): + def _is_everware_compatible(self, container): setup = yield self.docker( 'exec_create', container=container, - cmd="bash -c 'apt-get install jupyter -y && "+\ - "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\ - " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'" + cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\"" ) output = yield self.docker('exec_start', exec_id=setup['Id']) - print(output, file=self.debug_log) + return output != "" + + #@gen.coroutine + #def _is_jupyter_inside(self, container): + # setup = yield self.docker( + # 'exec_create', + # container=container, + # cmd="bash -c 'netstat -peant | grep \":8888 \"'" + # ) + # output = yield self.docker('exec_start', exec_id=setup['Id']) + # print('jupyter check:', output, file=sys.stderr) + # return output != "" diff --git a/everware/git_processor.py b/everware/git_processor.py index 5731f83..d97c60f 100644 --- a/everware/git_processor.py +++ b/everware/git_processor.py @@ -107,17 +107,27 @@ def prepare_local_repo(self): dockerfile_path = os.path.join(self._repo_dir, 'Dockerfile') if not os.path.isfile(dockerfile_path): - if not os.environ.get('DEFAULT_DOCKER_IMAGE'): + if self.config_files_exist(): + parent_image = 'everware/base:latest' + elif not os.environ.get('DEFAULT_DOCKER_IMAGE'): raise Exception('No dockerfile in repository') + else: + parent_image = os.environ['DEFAULT_DOCKER_IMAGE'] with open(dockerfile_path, 'w') as fout: fout.writelines([ - 'FROM %s\n' % os.environ['DEFAULT_DOCKER_IMAGE'], - 'MAINTAINER Alexander Tiunov ' + 'FROM %s\n' % parent_image ]) return False else: return True + def config_files_exist(self): + # uncomment when this functionality will be implemented + # for conf in ('everware.yml', 'requirements.txt', 'environment.yml'): + # cur_path = os.path.join(self._repo_dir, conf) + # if os.path.isfile(cur_path): + # return True + return False @property def escaped_repo_url(self): @@ -183,4 +193,4 @@ def get_state(self): def load_state(self, state): for key in self.STATE_VARS: if key in state: - setattr(self, key, state[key]) \ No newline at end of file + setattr(self, key, state[key]) diff --git a/everware/spawner.py b/everware/spawner.py index 50d3b95..026f9ab 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -140,14 +140,6 @@ def _options_form_default(self): checked /> Remove previous container if it exists -