/everware

Permalink
Browse files

support for custom run command and auto git clone inside container

  • Loading branch information...
1 parent a797227 commit e178eb862406710d92397d322a54160c8cd81d13 @astiunov astiunov committed Apr 7, 2017
View
22 etc/nginx_config.conf
@@ -9,10 +9,10 @@ events {
http { http {
upstream custom_service { upstream custom_service {
- server 127.0.0.1:8081; + server 127.0.0.1:8000;
} }
server { server {
- listen 8080; + listen %PORT%;
server_name docker_proxy; server_name docker_proxy;
if ($cookie_everware_custom_service_token != "%TOKEN%") { if ($cookie_everware_custom_service_token != "%TOKEN%") {
@@ -21,12 +21,24 @@ http {
location / { location / {
proxy_pass http://custom_service; proxy_pass http://custom_service;
- proxy_set_header Host localhost:8080; + proxy_set_header Host $host;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
} }
- location /service/%USERNAME%/ { + location = /user/%USERNAME% {
+ return 302 /user/%USERNAME%/;
+ }
+
+ location /user/%USERNAME%/ {
proxy_pass http://custom_service/; proxy_pass http://custom_service/;
- proxy_set_header Host localhost:8080; + proxy_set_header Host $host;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
} }
} }
View
211 everware/container_handler.py
@@ -1,127 +1,142 @@
from dockerspawner import DockerSpawner from dockerspawner import DockerSpawner
from tornado import gen from tornado import gen
import re import re
+import os.path
+import sys
+import yaml
+
+class ShellCommand:
+ def __init__(self, commands=[]):
+ self.commands = commands
+
+ def add_commands(self, commands_list):
+ self.commands.extend(commands_list)
+
+ def extend(self, command):
+ self.add_commands(command.commands)
+
+ def get_single_command(self):
+ return 'bash -c "{}"' .format(' && '.join(self.commands))
+
+def make_git_command(repourl, commit_sha):
+ return ShellCommand([
+ 'apt-get install -y git',
+ 'git clone {} /notebooks'.format(repourl),
+ 'cd /notebooks',
+ 'git reset --hard {}'.format(commit_sha)
+ ])
+
+def make_nginx_start_command(nginx_config):
+ return ShellCommand([
+ 'apt-get install -y nginx',
+ "python -c 'print(\\\"{}\\\")' >/etc/nginx/nginx.conf".format(nginx_config),
+ 'service nginx restart',
+ 'cat /etc/nginx/nginx.conf'
+ ])
+
+def make_default_start_command(env):
+ return ShellCommand([
+ 'jupyterhub-singleuser --port=8888 --ip=0.0.0.0 --allow-root --user={} --cookie-name={} --base-url={} '.format(
+ env['JPY_USER'],
+ env['JPY_COOKIE_NAME'],
+ env['JPY_BASE_URL']
+ ) + '--hub-prefix={} --hub-api-url={} --notebook-dir=/notebooks'.format(
+ env['JPY_HUB_PREFIX'],
+ env['JPY_HUB_API_URL']
+ )
+ ])
+
+def make_custom_start_command(command):
+ return ShellCommand([command])
+
class ContainerHandler(DockerSpawner): class ContainerHandler(DockerSpawner):
+ def parse_config(self, directory):
+ self.everware_config = {
+ 'everware_based': True
+ }
+ try:
+ with open(os.path.join(directory, 'everware.yml')) as fin:
+ try:
+ self.everware_config = yaml.load(fin)
+ except yaml.YAMLError as exc:
+ self.log.warn('Fail reading everware.yml: {}'.format(exc))
+ except IOError:
+ self.log.info('No everware.yaml in repo')
+
@gen.coroutine @gen.coroutine
- def prepare_container(self, need_service): + def prepare_container(self):
+ if self.everware_config.get('everware_based', True):
+ return
container = yield self.get_container() container = yield self.get_container()
- everware_based = yield self._is_everware_compatible(container) + was_cloned = yield self._check_for_git_compatibility(container)
- if not everware_based or need_service: + if not was_cloned:
- yield self._init_container(container, everware_based) + command = make_git_command(self.repo_url_with_token, self.commit_sha)
- if need_service: + setup = yield self.docker(
- nginx_started = yield self._start_nginx( + 'exec_create',
- container, + container=container,
- self.custom_service_token(), + cmd=command.get_single_command()
- self.user.name
) )
- git_webui_started = False + output = yield self.docker('exec_start', exec_id=setup['Id'])
- if nginx_started: +
- self.log.info('nginx has started in %s' % self.container_id) +
- git_webui_started = yield self._start_service(container) + @gen.coroutine
- if git_webui_started: + def start(self, image=None):
- self.log.info('git webui has started in %s' % self.container_id) + self.parse_config(self._repo_dir)
- self._add_to_log('Git Web UI has started') + start_command = None
+ extra_create_kwargs = {
+ 'ports': [self.container_port]
+ }
+ if not self.everware_config.get('everware_based', True):
+ start_command = make_git_command(self.repo_url_with_token, self.commit_sha)
+ if 'start_command' in self.everware_config:
+ nginx_config = self._get_nginx_config(
+ 8888,
+ self.custom_service_token(),
+ self.user.name
+ )
+ start_command.extend(make_nginx_start_command(nginx_config))
+ start_command.extend(make_custom_start_command(self.everware_config['start_command']))
else: else:
- self.log.info('failed to start nginx in %s' % self.container_id) + start_command.extend(make_default_start_command(self.get_env()))
- if not git_webui_started: + extra_create_kwargs.update({
- self._add_to_log('Failed to start git web ui') + 'command': start_command.get_single_command()
- self.user_options['custom_service'] = False + })
+
+ extra_host_config = {
+ 'port_bindings': {
+ self.container_port: (self.container_ip,)
+ }
+ }
+ ip, port = yield DockerSpawner.start(self, image,
+ extra_create_kwargs=extra_create_kwargs,
+ extra_host_config=extra_host_config)
+ return ip, port
def _encode_conf(self, s): def _encode_conf(self, s):
return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s) return ''.join('\\x' + hex(ord(x))[2:].zfill(2) for x in s)
- @gen.coroutine + def _get_nginx_config(self, port, token, username):
- def _start_nginx(self, container, token, username):
try: try:
result = '' result = ''
with open('etc/nginx_config.conf') as fin: with open('etc/nginx_config.conf') as fin:
for line in fin: for line in fin:
result += self._encode_conf( result += self._encode_conf(
- line.replace('%TOKEN%', token).replace('%USERNAME%', username) + line.replace('%TOKEN%', token)
+ .replace('%USERNAME%', username)
+ .replace('%PORT%', str(port))
) )
- setup = yield self.docker( + return result
- 'exec_create',
- container=container,
- cmd="bash -c \"apt-get install nginx -y && " +\
- "python -c 'print(\\\"%s\\\")' >/etc/nginx/nginx.conf && " % result +\
- "service nginx restart && cat /etc/nginx/nginx.conf\""
- )
- output = yield self.docker('exec_start', exec_id=setup['Id'])
- # print(output, file=self.debug_log)
- # print(str(output), file=sys.stderr)
- return re.search(
- r'Restarting nginx.+?\.\.\.done\.',
- str(output),
- flags=re.DOTALL
- )
except OSError: except OSError:
- self.log.info('No nginx config') + self.log.warn('No nginx config')
- return False + raise
-
- @gen.coroutine
- def _start_service(self, container):
- setup = yield self.docker(
- 'exec_create',
- container=container,
- cmd="bash -c '"+\
- "curl https://raw.githubusercontent.com/everware/git-webui/master/install/installer.sh >installer.sh" +\
- " && bash installer.sh && cd /notebooks; "+\
- "git webui --port=8081 --host=0.0.0.0 --no-browser >/dev/null 2>&1 &'"
- )
- output = yield self.docker('exec_start', exec_id=setup['Id'])
- return True
@gen.coroutine @gen.coroutine
- def _init_container(self, container, everware_based): + def _check_for_git_compatibility(self, container):
- cmd = "bash -c 'apt-get update && apt-get install git curl net-tools -y"
- if everware_based:
- cmd += "'"
- else:
- notebook_dir = '/notebooks'
- cmd += " && git clone {} {} && cd {} && git reset --hard {}'".format(
- self.repo_url_with_token,
- notebook_dir,
- notebook_dir,
- self.commit_sha
- )
-
- setup = yield self.docker(
- 'exec_create',
- container=container,
- cmd=cmd
- )
- output = yield self.docker('exec_start', exec_id=setup['Id'])
-
- #@gen.coroutine
- #def _run_jupyter(self, container):
- # setup = yield self.docker(
- # 'exec_create',
- # container=container,
- # cmd="bash -c 'apt-get install jupyter -y && "+\
- # "curl https://raw.githubusercontent.com/jupyterhub/jupyterhub/master/jupyterhub/singleuser.py >singleuser.py" +\
- # " && chmod +x singleuser.py && ./singleuser.py --port=8888 --ip=0.0.0.0 --no-browser &'"
- # )
- # output = yield self.docker('exec_start', exec_id=setup['Id'])
- # print(output, file=self.debug_log)
-
- @gen.coroutine
- def _is_everware_compatible(self, container):
setup = yield self.docker( setup = yield self.docker(
'exec_create', 'exec_create',
container=container, container=container,
cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\"" cmd="bash -c \"ls / | grep -E '\\bnotebooks\\b'\""
) )
output = yield self.docker('exec_start', exec_id=setup['Id']) output = yield self.docker('exec_start', exec_id=setup['Id'])
return output != "" return output != ""
-
- #@gen.coroutine
- #def _is_jupyter_inside(self, container):
- # setup = yield self.docker(
- # 'exec_create',
- # container=container,
- # cmd="bash -c 'netstat -peant | grep \":8888 \"'"
- # )
- # output = yield self.docker('exec_start', exec_id=setup['Id'])
- # print('jupyter check:', output, file=sys.stderr)
- # return output != ""
View
7 everware/home_handler.py
@@ -54,15 +54,10 @@ def get(self):
repo_url = '' repo_url = ''
fork_exists = False fork_exists = False
repository_changed = False repository_changed = False
- custom_service_url = None
- custom_service_name = None
if user.running: if user.running:
branch_name = user.spawner.branch_name branch_name = user.spawner.branch_name
commit_sha = user.spawner.commit_sha commit_sha = user.spawner.commit_sha
repo_url = user.spawner.repo_url repo_url = user.spawner.repo_url
- if user.spawner.need_run_custom_service():
- custom_service_url = user.spawner.custom_service_path
- custom_service_name = 'Git Web UI'
if user.running and getattr(user, 'login_service', '') == 'github': if user.running and getattr(user, 'login_service', '') == 'github':
if do_fork: if do_fork:
self.log.info('Will fork %s' % user.spawner.repo_url) self.log.info('Will fork %s' % user.spawner.repo_url)
@@ -112,8 +107,6 @@ def get(self):
version=__version__, version=__version__,
g_analitics_id=g_id, g_analitics_id=g_id,
ya_metrica_id=ya_id, ya_metrica_id=ya_id,
- custom_service_url=custom_service_url,
- custom_service_name=custom_service_name
) )
self.finish(html) self.finish(html)
Oops, something went wrong.

0 comments on commit e178eb8

Please sign in to comment.