diff --git a/Makefile b/Makefile index 7aa9c99..1ad63b7 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,14 @@ install: ## install everware docker-build: ## build docker image docker build --no-cache -t everware/everware:0.10.0 . +clean: ## clean user base + if [ -f ${PIDFILE} ] ; then echo "${PIDFILE} exists, cannot continute" ; exit 1; fi + rm -f jupyterhub.sqlite + +run-linux: clean ## run everware server on linux + source ./env.sh && \ + ${EXECUTOR} -f etc/local_config.py --no-ssl 2>&1 | tee ${LOG} + test: ## run all tests export UPLOADDIR=${UPLOADDIR}; \ py.test everware/ && \ diff --git a/build_tools/frontend_test_normal_config.py b/build_tools/frontend_test_normal_config.py index 7684372..6a1a2cc 100644 --- a/build_tools/frontend_test_normal_config.py +++ b/build_tools/frontend_test_normal_config.py @@ -2,5 +2,5 @@ load_subconfig('etc/base_config.py') c.JupyterHub.authenticator_class = 'everware.DummyTokenAuthenticator' -c.Spawner.start_timeout = 50 +c.Spawner.start_timeout = 120 c.Spawner.http_timeout = 120 # docker sometimes doesn't show up for a while diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index 4b839a0..b719a55 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -18,6 +18,7 @@ function kill_everware { pkill -KILL -f everware-server sleep $WAIT_FOR_STOP fi + pkill -KILL node || true } if [ -z "$UPLOADDIR" ] ; then diff --git a/etc/nginx_config.conf b/etc/nginx_config.conf new file mode 100644 index 0000000..dd26642 --- /dev/null +++ b/etc/nginx_config.conf @@ -0,0 +1,130 @@ +user www-data; +worker_processes 4; +pid /run/nginx.pid; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + upstream custom_service { + server 127.0.0.1:8000; + } + server { + listen %PORT%; + server_name docker_proxy; + + if ($cookie_everware_custom_service_token != "%TOKEN%") { + return 403; + } + + location / { + proxy_pass http://custom_service; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location = /user/%USERNAME% { + return 302 /user/%USERNAME%/; + } + + location /user/%USERNAME%/ { + proxy_pass http://custom_service/; + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # nginx-naxsi config + ## + # Uncomment it if you installed nginx-naxsi + ## + + #include /etc/nginx/naxsi_core.rules; + + ## + # nginx-passenger config + ## + # Uncomment it if you installed nginx-passenger + ## + + #passenger_root /usr; + #passenger_ruby /usr/bin/ruby; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} + + +#mail { +# # See sample authentication script at: +# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript +# +# # auth_http localhost/auth.php; +# # pop3_capabilities "TOP" "USER"; +# # imap_capabilities "IMAP4rev1" "UIDPLUS"; +# +# server { +# listen localhost:110; +# protocol pop3; +# proxy on; +# } +# +# server { +# listen localhost:143; +# protocol imap; +# proxy on; +# } +#} + diff --git a/everware/container_handler.py b/everware/container_handler.py new file mode 100644 index 0000000..9e3acb0 --- /dev/null +++ b/everware/container_handler.py @@ -0,0 +1,142 @@ +from dockerspawner import DockerSpawner +from tornado import gen +import re +import os.path +import sys +import yaml + +class ShellCommand: + def __init__(self, commands=[]): + self.commands = commands + + def add_commands(self, commands_list): + self.commands.extend(commands_list) + + def extend(self, command): + self.add_commands(command.commands) + + def get_single_command(self): + return 'bash -c "{}"' .format(' && '.join(self.commands)) + +def make_git_command(repourl, commit_sha): + return ShellCommand([ + 'apt-get install -y git', + 'git clone {} /notebooks'.format(repourl), + 'cd /notebooks', + 'git reset --hard {}'.format(commit_sha) + ]) + +def make_nginx_start_command(nginx_config): + return ShellCommand([ + 'apt-get install -y nginx', + "python -c 'print(\\\"{}\\\")' >/etc/nginx/nginx.conf".format(nginx_config), + 'service nginx restart', + 'cat /etc/nginx/nginx.conf' + ]) + +def make_default_start_command(env): + return ShellCommand([ + 'jupyterhub-singleuser --port=8888 --ip=0.0.0.0 --allow-root --user={} --cookie-name={} --base-url={} '.format( + env['JPY_USER'], + env['JPY_COOKIE_NAME'], + env['JPY_BASE_URL'] + ) + '--hub-prefix={} --hub-api-url={} --notebook-dir=/notebooks'.format( + env['JPY_HUB_PREFIX'], + env['JPY_HUB_API_URL'] + ) + ]) + +def make_custom_start_command(command): + return ShellCommand([command]) + + +class ContainerHandler(DockerSpawner): + def parse_config(self, directory): + self.everware_config = { + 'everware_based': True + } + try: + with open(os.path.join(directory, 'everware.yml')) as fin: + try: + self.everware_config = yaml.load(fin) + except yaml.YAMLError as exc: + self.log.warn('Fail reading everware.yml: {}'.format(exc)) + except IOError: + self.log.info('No everware.yaml in repo') + + @gen.coroutine + def prepare_container(self): + if self.everware_config.get('everware_based', True): + return + container = yield self.get_container() + was_cloned = yield self._check_for_git_compatibility(container) + if not was_cloned: + command = make_git_command(self.repo_url_with_token, self.commit_sha) + setup = yield self.docker( + 'exec_create', + container=container, + cmd=command.get_single_command() + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + + + @gen.coroutine + def start(self, image=None): + self.parse_config(self._repo_dir) + start_command = None + extra_create_kwargs = { + 'ports': [self.container_port] + } + if not self.everware_config.get('everware_based', True): + start_command = make_git_command(self.repo_url_with_token, self.commit_sha) + if 'start_command' in self.everware_config: + nginx_config = self._get_nginx_config( + 8888, + self.custom_service_token(), + self.user.name + ) + start_command.extend(make_nginx_start_command(nginx_config)) + start_command.extend(make_custom_start_command(self.everware_config['start_command'])) + else: + start_command.extend(make_default_start_command(self.get_env())) + extra_create_kwargs.update({ + 'command': start_command.get_single_command() + }) + + extra_host_config = { + 'port_bindings': { + self.container_port: (self.container_ip,) + } + } + ip, port = yield DockerSpawner.start(self, image, + extra_create_kwargs=extra_create_kwargs, + extra_host_config=extra_host_config) + return ip, port + + def _encode_conf(self, s): + return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) + + def _get_nginx_config(self, port, token, username): + try: + result = '' + with open('etc/nginx_config.conf') as fin: + for line in fin: + result += self._encode_conf( + line.replace('%TOKEN%', token) + .replace('%USERNAME%', username) + .replace('%PORT%', str(port)) + ) + return result + except OSError: + self.log.warn('No nginx config') + raise + + @gen.coroutine + def _check_for_git_compatibility(self, container): + setup = yield self.docker( + 'exec_create', + container=container, + cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\"" + ) + output = yield self.docker('exec_start', exec_id=setup['Id']) + return output != "" diff --git a/everware/git_processor.py b/everware/git_processor.py index 5731f83..d97c60f 100644 --- a/everware/git_processor.py +++ b/everware/git_processor.py @@ -107,17 +107,27 @@ def prepare_local_repo(self): dockerfile_path = os.path.join(self._repo_dir, 'Dockerfile') if not os.path.isfile(dockerfile_path): - if not os.environ.get('DEFAULT_DOCKER_IMAGE'): + if self.config_files_exist(): + parent_image = 'everware/base:latest' + elif not os.environ.get('DEFAULT_DOCKER_IMAGE'): raise Exception('No dockerfile in repository') + else: + parent_image = os.environ['DEFAULT_DOCKER_IMAGE'] with open(dockerfile_path, 'w') as fout: fout.writelines([ - 'FROM %s\n' % os.environ['DEFAULT_DOCKER_IMAGE'], - 'MAINTAINER Alexander Tiunov ' + 'FROM %s\n' % parent_image ]) return False else: return True + def config_files_exist(self): + # uncomment when this functionality will be implemented + # for conf in ('everware.yml', 'requirements.txt', 'environment.yml'): + # cur_path = os.path.join(self._repo_dir, conf) + # if os.path.isfile(cur_path): + # return True + return False @property def escaped_repo_url(self): @@ -183,4 +193,4 @@ def get_state(self): def load_state(self, state): for key in self.STATE_VARS: if key in state: - setattr(self, key, state[key]) \ No newline at end of file + setattr(self, key, state[key]) diff --git a/everware/spawner.py b/everware/spawner.py index 27a7504..f382409 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -14,8 +14,10 @@ from traitlets import ( Integer, Unicode, + Int ) from tornado import gen +from tornado.httpclient import HTTPError import ssl import json @@ -24,13 +26,13 @@ from .image_handler import ImageHandler from .git_processor import GitMixin from .email_notificator import EmailNotificator +from .container_handler import ContainerHandler from . import __version__ ssl._create_default_https_context = ssl._create_unverified_context - -class CustomDockerSpawner(DockerSpawner, GitMixin, EmailNotificator): +class CustomDockerSpawner(GitMixin, EmailNotificator, ContainerHandler): def __init__(self, **kwargs): self._user_log = [] self._is_failed = False @@ -39,7 +41,7 @@ def __init__(self, **kwargs): self._image_handler = ImageHandler() self._cur_waiter = None self._is_empty = False - DockerSpawner.__init__(self, **kwargs) + ContainerHandler.__init__(self, **kwargs) EmailNotificator.__init__(self) @@ -143,6 +145,9 @@ def options_from_form(self, formdata): raise Exception('You have to provide the URL to a git repository.') return options + def custom_service_token(self): + return self.user_options['service_token'] + @property def form_repo_url(self): """Repository URL as submitted by the user.""" @@ -167,8 +172,6 @@ def is_up(self): @gen.coroutine def get_container(self): - - # self.log.debug("Getting container: %s", self.container_name) try: container = yield self.docker( 'inspect_container', self.container_name @@ -225,10 +228,12 @@ def wait_up(self): # copied from jupyterhub, because if user's server didn't appear, it # means that spawn was unsuccessful, need to set is_failed try: - yield self.user.server.wait_up(http=True, timeout=self.http_timeout) ip, port = yield self.get_ip_and_port() self.user.server.ip = ip self.user.server.port = port + yield self.user.server.wait_up(http=True, timeout=self.http_timeout) + self.user.server.ip = ip + self.user.server.port = port self._is_up = True except TimeoutError: self._is_failed = True @@ -298,8 +303,6 @@ def build_image(self): raise Exception(full_output) except: raise - finally: - rmtree(tmp_dir, ignore_errors=True) return image_name @@ -344,10 +347,10 @@ def start(self, image=None): yield self.remove_old_container() self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') - yield super(CustomDockerSpawner, self).start( + + yield ContainerHandler.start(self, image=image_name ) - self._add_to_log('Adding to proxy') except gen.TimeoutError: self._is_failed = True if self._cur_waiter: @@ -358,6 +361,7 @@ def start(self, image=None): level=2 ) yield self.notify_about_fail("Timeout limit %.3f exceeded" % self.start_timeout) + self._is_building = False raise except Exception as e: self._is_failed = True @@ -366,14 +370,20 @@ def start(self, image=None): message = "Container doesn't have jupyter-singleuser inside" self._add_to_log('Something went wrong during building. Error: %s' % message) yield self.notify_about_fail(message) + self._is_building = False raise e finally: - self._is_building = False + rmtree(self._repo_dir, ignore_errors=True) + try: + yield self.prepare_container() + except Exception as e: + self.log.warn('Fail to prepare the container: {}'.format(e)) + + self._add_to_log('Adding to proxy') yield self.wait_up() return self.user.server.ip, self.user.server.port # jupyterhub 0.7 prefers returning ip, port - @gen.coroutine def stop(self, now=False): """Stop the container @@ -436,6 +446,7 @@ def poll(self): if container_state["Running"]: # check if something is listening inside container try: + # self.log.info('poll {}'.format(self.user.server.url)) yield wait_for_http_server(self.user.server.url, timeout=1) except TimeoutError: self.log.warn("Can't reach running container by http") diff --git a/everware/user_spawn_handler.py b/everware/user_spawn_handler.py index 0e84d0a..96d7101 100755 --- a/everware/user_spawn_handler.py +++ b/everware/user_spawn_handler.py @@ -1,6 +1,7 @@ from tornado import web, gen import jupyterhub.handlers.pages as default_handlers import sys +import uuid from . import __version__ from .metrica import MetricaIdsMixin @@ -22,9 +23,14 @@ def _render_form(self, message=''): @gen.coroutine def _spawn(self, user, form_options): + token = uuid.uuid1().hex + self.set_cookie('everware_custom_service_token', token) self.redirect('/user/%s' % user.name) try: options = user.spawner.options_from_form(form_options) + options.update({ + 'service_token': token + }) yield self.spawn_single_user(user, options=options) # if user set another access token (for example he logged with github # and clones from bitbucket) diff --git a/everware/user_wait_handler.py b/everware/user_wait_handler.py index 5fa3655..3d17a40 100755 --- a/everware/user_wait_handler.py +++ b/everware/user_wait_handler.py @@ -47,7 +47,7 @@ def get(self, name, user_path): else: if is_up: self.set_login_cookie(current_user) - target = '/user/%s' % ( + target = '/user/%s/' % ( current_user.name ) self.log.info('redirecting to %s' % target) diff --git a/frontend_tests/normal_scenarios.py b/frontend_tests/normal_scenarios.py index 07b3279..495d46d 100644 --- a/frontend_tests/normal_scenarios.py +++ b/frontend_tests/normal_scenarios.py @@ -96,3 +96,22 @@ def scenario_default_private_repos(user): user.wait_for_pattern_in_page(r"Launch\s+a\s+notebook") driver.find_element_by_id("logout").click() user.log("logout clicked") + +def scenario_r_shiny(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/everware/r-shiny-example") + user.log("spawn clicked") + user.wait_for_pattern_in_page("Iris\s+k-means\s+clustering") + +def scenario_jupyter_only(user): + driver = commons.login(user) + user.wait_for_element_present(By.ID, "start") + driver.find_element_by_id("start").click() + commons.fill_repo_info(driver, user, "https://github.com/astiunov/qutip-lectures") + user.log("spawn clicked") + user.wait_for_element_present(By.LINK_TEXT, "Control Panel") + driver.find_element_by_link_text("Control Panel").click() + + diff --git a/frontend_tests/test_generator.py b/frontend_tests/test_generator.py index c3857b2..a002f32 100644 --- a/frontend_tests/test_generator.py +++ b/frontend_tests/test_generator.py @@ -17,6 +17,8 @@ normal_scenarios.scenario_no_dockerfile, normal_scenarios.scenario_default_dockerfile, # should go after no_dockerfile normal_scenarios.scenario_default_private_repos, + normal_scenarios.scenario_r_shiny, + normal_scenarios.scenario_jupyter_only, nonstop_scenarios.scenario_simple ] diff --git a/requirements.txt b/requirements.txt index 0c80c20..73b3d9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ ipython[notebook] coveralls nose2 +pyyaml pytest>=2.8 GitPython==2.1.1 selenium==2.52.0 jupyterhub==0.7.2 docker-py==1.10.6 --e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner \ No newline at end of file +-e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner