Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Already on GitHub? Sign in to your account

Add basic push to Github functionality #49

Merged
merged 20 commits into from Mar 15, 2016
View
@@ -4,20 +4,24 @@ language: python
sudo: required
services:
- docker
+
addons:
apt:
packages:
- build-essential
+
python:
- 3.4
- 3.5
+
env:
+ global:
+ # GIST_TOKEN=1234 for screen uploading
+ - secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU="
+ matrix:
- JHUB_VERSION=master
- JHUB_VERSION=latest # latest released version
-# GIST_TOKEN=1234 for screen uploading
-secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU="
-
before_install:
# XXX remove IPv6 entry via https://github.com/travis-ci/travis-ci/issues/4978
- sudo [ $(ip addr show | grep "inet6 ::1" | wc -l) -lt "1" ] && sudo sed -i '/^::1/d' /etc/hosts
@@ -35,6 +39,7 @@ before_install:
- pip freeze
- npm list
- if [ -d $HOME/frontend-test-screenshots/ ] ; then ls -alR $HOME/frontend-test-screenshots/ ; fi
+ - python -c "import multiprocessing; print('CPU cores - %d' % multiprocessing.cpu_count())"
install:
- npm install -g configurable-http-proxy
@@ -44,8 +49,7 @@ install:
script:
- nose2 -v --start-dir everware # unit tests live in everware/
- ./build_tools/test_frontend.sh
- - prefix="build $TRAVIS_JOB_NUMBER"
- - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M="travis-CI screens ($JHUB_VERSION v$TRAVIS_PYTHON_VERSION)" ; fi
+ - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M=travis-${TRAVIS_JOB_NUMBER}_${JHUB_VERSION}_v${TRAVIS_PYTHON_VERSION} ; fi
cache:
apt: true
View
@@ -11,8 +11,8 @@ TEST_OPTIONS := -s tests -N 2
TESTS := test_happy_mp
LOG := everware.log
PIDFILE := everware.pid
-IP = $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null)
-OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=${IP}
+IP ?= $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null)
+OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=$${IP}
IS_DOCKER_MACHINE := $(shell which docker-machine > /dev/null ; echo $$?)
UPLOADDIR ?= ~/upload_screens
ifeq (0, $(IS_DOCKER_MACHINE))
@@ -34,7 +34,7 @@ help:
install: ## install everware
npm install
- npm install -g configurable-http-proxy
+ npm install configurable-http-proxy
PYTHON_MAJOR=`python -c 'import sys; print(sys.version_info[0])'` ;\
if [ $${PYTHON_MAJOR} -eq 3 ] ; then \
PYTHON=python ;\
@@ -66,7 +66,7 @@ clean: ## clean user base
run: clean ## run everware server
source ./env.sh && \
- jupyterhub ${OPTIONS}
+ jupyterhub ${OPTIONS} | tee ${LOG}
run-daemon: clean
source ./env.sh && \
@@ -76,9 +76,11 @@ run-daemon: clean
echo "Started. Log saved to ${LOG}"
stop: ${PIDFILE}
- kill -9 `cat ${PIDFILE}`
- pkill -9 -f configurable-http-proxy
rm ${PIDFILE}
+ kill -9 `cat ${PIDFILE}` || pkill -9 -f configurable-http-proxy
+
+stop-zombie:
+ pkill -9 -f jupyterhub || pkill -9 -f configurable-http-proxy
run-test-server: clean ## run everware instance for testing (no auth)
cat jupyterhub_config.py <(echo c.JupyterHub.authenticator_class = 'dummyauthenticator.DummyAuthenticator') \
@@ -94,6 +96,9 @@ run-test-server: clean ## run everware instance for testing (no auth)
logs: ${LOG} ## watch log file
tail -f ${LOG}
+test: ## run tests
+ export UPLOADDIR=${UPLOADDIR} && build_tools/test_frontend.sh
+
test-client: ## run tests
export UPLOADDIR=${UPLOADDIR} ; \
nose2 ${TEST_OPTIONS} ${TESTS}
@@ -117,11 +122,12 @@ upload_screens: ## upload screenshots of failed tests
fi ; \
fi ;\
OPTIONS="--no-open" ; \
- if [ "${M}" != "" ] ; then OPTIONS+=" --description ${M}" ; fi ;\
+ if [ "${M}" != "" ] ; then OPTIONS+=" --description '${M}'" ; fi ;\
gistup $${OPTIONS} ; \
else \
git add * ;\
git commit -am "${M}" ;\
git push ;\
fi ;\
- fi
+ fi
+
@@ -18,14 +18,16 @@ HUB_PID=$!
sleep 3
echo "Start running frontend tests"
+[ -z "$UPLOADDIR" ] && echo "no UPLOADDIR defined" && exit 1
[ -d $UPLOADDIR ] && rm -rf $UPLOADDIR/*
nose2 -v -N $NPROC --start-dir frontend_tests || FAIL=1
if [ -f $LOG ]; then
echo ">>> Frontend test hub log:"
cat $LOG
echo "<<< Frontend test hub log:"
+ docker ps -a
fi
kill ${HUB_PID}
-exit $FAIL
+exit $FAIL
@@ -116,7 +116,7 @@ def get(self):
self.authorize_redirect(
redirect_uri=redirect_uri,
client_id=self.authenticator.client_id,
- scope=[],
+ scope=['repo'],
response_type='code',
extra_params={'state': self.create_signed_value('state', repr(state))})
@@ -142,10 +142,12 @@ def get(self):
self.log.debug('State dict: %s', state)
state.pop('unique')
- username = yield self.authenticator.authenticate(self)
+ username, token = yield self.authenticator.authenticate(self)
if username:
user = self.user_from_username(username)
+ user.token = token
self.set_login_cookie(user)
+ user.login_service = "github"
if 'repourl' in state:
self.log.debug("Redirect with %s", state)
self.redirect(self.hub.server.base_url +'/home?'+urllib.parse.urlencode(state))
@@ -227,7 +229,7 @@ def authenticate(self, handler):
username = resp_json["login"]
if self.whitelist and username not in self.whitelist:
username = None
- raise gen.Return(username)
+ raise gen.Return((username, access_token))
class BitbucketOAuthenticator(Authenticator):
View
@@ -1,17 +1,124 @@
from tornado import web, gen
+from docker.errors import NotFound
from jupyterhub.handlers.base import BaseHandler
from IPython.html.utils import url_path_join
from tornado.httputil import url_concat
+from tornado.httpclient import HTTPRequest, AsyncHTTPClient
+import json
+
+import re
+
+@gen.coroutine
+def _fork_github_repo(url, token):
+ http_client = AsyncHTTPClient()
+
+ headers={"User-Agent": "JupyterHub",
+ "Authorization": "token {}".format(token)
+ }
+
+ result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url)
+ if not result:
+ raise ValueError('URL is not a github URL')
+
+ owner, repo = result[0]
+
+ api_url = "https://api.github.com/repos/%s/%s/forks" % (owner, repo)
+
+ req = HTTPRequest(api_url,
+ method="POST",
+ headers=headers,
+ body='',
+ )
+
+ resp = yield http_client.fetch(req)
+ return json.loads(resp.body.decode('utf8', 'replace'))
+
+@gen.coroutine
+def _github_fork_exists(username, url, token):
+ http_client = AsyncHTTPClient()
+
+ headers={"User-Agent": "JupyterHub",
+ "Authorization": "token {}".format(token)
+ }
+
+ result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url)
+ if not result:
+ raise ValueError('URL is not a github URL')
+
+ owner, repo = result[0]
+ api_url = "https://api.github.com/repos/%s/%s" % (username, repo)
+
+ req = HTTPRequest(api_url,
+ method="GET",
+ headers=headers,
+ )
+
+ try:
+ resp = yield http_client.fetch(req)
+ return True
+ except:
+ return False
+
+@gen.coroutine
+def _repository_changed(user):
+ try:
+ setup = yield user.spawner.docker(
+ 'exec_create',
+ container=user.spawner.container_id,
+ cmd="bash -c 'cd analysis/ && \
+ (git fetch --unshallow > /dev/null 2>&1; true) && \
+ git diff --name-only'",
+ )
+ out = yield user.spawner.docker(
+ 'exec_start',
+ exec_id=setup['Id'],
+ )
+ except NotFound:
+ return False
+ if out:
+ return True
+ else:
+ return False
+
+@gen.coroutine
+def _push_github_repo(user, url, token):
+ result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url)
+ if not result:
+ raise ValueError('URL is not a github URL')
+
+ owner, repo = result[0]
+ fork_url = "https://{}@github.com/{}/{}".format(token, user.name, repo)
+
+ out = yield user.spawner.docker(
+ 'exec_create',
+ container=user.spawner.container_id,
+ cmd="bash -c 'cd analysis/ && \
+ git config --global user.email \"everware@everware.xyz\" && \
+ git config --global user.name \"Everware\" && \
+ (git fetch --unshallow; true) && \
+ git add . && \
+ git commit -m \"Update through everware\" && \
+ (git remote add everware-fork {}; true) && \
+ (git checkout -b everware; true) && \
+ git push everware-fork everware'".format(fork_url),
+ )
+ response = yield user.spawner.docker(
+ 'exec_start',
+ exec_id=out['Id'],
+ )
class HomeHandler(BaseHandler):
"""Render the user's home page."""
@web.authenticated
+ @gen.coroutine
def get(self):
user = self.get_current_user()
repourl = self.get_argument('repourl', '')
+ do_fork = self.get_argument('do_fork', False)
+ do_push = self.get_argument('do_push', False)
if repourl:
self.log.info('Got %s in home' % repourl)
self.redirect(url_concat(
@@ -20,10 +127,52 @@ def get(self):
}
))
return
+
+ stat = yield user.spawner.poll()
+ self.log.debug("The container is {}".format(repr(stat)))
+ if user.running and hasattr(user, "login_service") and user.login_service == "github":
+ if do_fork:
+ self.log.info('Will fork %s' % user.spawner.repo_url)
+ yield _fork_github_repo(
+ user.spawner.repo_url,
+ user.token,
+ )
+ self.redirect('/hub/home')
+ return
+ if do_push:
+ self.log.info('Will push to fork')
+ yield _push_github_repo(
+ user,
+ user.spawner.repo_url,
+ user.token,
+ )
+ self.redirect('/hub/home')
+ return
+ repo_url = user.spawner.repo_url
+ fork_exists = yield _github_fork_exists(
+ user.name,
+ user.spawner.repo_url,
+ user.token,
+ )
+ repository_changed = yield _repository_changed(user)
+ else:
+ repo_url = ''
+ fork_exists = False
+ repository_changed = False
+
+
+ if hasattr(user, 'login_service'):
+ loginservice = user.login_service
+ else:
+ loginservice = 'none'
html = self.render_template('home.html',
- user=user
+ user=user,
+ repourl=repo_url,
+ login_service=loginservice,
+ fork_exists=fork_exists,
+ repository_changed=repository_changed,
)
- self.finish(html)
+ self.finish(html)
@@ -49,7 +49,7 @@ def __enter__(self):
def __exit__(self, exc_type, exc_value, traceback):
self._building_log = []
if isinstance(exc_value, Exception):
- self._exception = exc_type(exc_value)
+ self._exception = exc_value
self._mutex.set()
def add_to_log(self, message, level=1):
View
@@ -25,6 +25,9 @@
from .image_handler import ImageHandler
+import ssl
+
+ssl._create_default_https_context = ssl._create_unverified_context
class CustomDockerSpawner(DockerSpawner):
def __init__(self, **kwargs):
@@ -59,12 +62,15 @@ def _docker(self, method, *args, **kwargs):
if method in generator_methods:
def lister(mm):
ret = []
- for l in mm:
- ret.append(str(l))
- # include only high-level docker's log
- if 'stream' in l and not l['stream'].startswith(' --->'):
- # self._add_to_log(l['stream'], 2)
- self._cur_waiter.add_to_log(l['stream'], 2)
+ try:
+ for l in mm:
+ ret.append(str(l))
+ # include only high-level docker's log
+ if 'stream' in l and not l['stream'].startswith(' --->'):
+ # self._add_to_log(l['stream'], 2)
+ self._cur_waiter.add_to_log(l['stream'], 2)
+ except JSONDecodeError as e:
+ self.warn("Error parsing docker output (%s)" % repr(e))
return ret
return lister(m(*args, **kwargs))
else:
@@ -132,7 +138,7 @@ def options_from_form(self, formdata):
@property
def repo_url(self):
- return self.user_options['repo_url']
+ return self.user_options.get('repo_url', '')
_escaped_repo_url = None
@property
Oops, something went wrong.