[WIP] everware.yml with volumes #190

everware/git_processor.py

@@ -1,9 +1,13 @@
 import re
 import git
 from concurrent.futures import ThreadPoolExecutor
-from tornado import gen
+from tornado import gen,concurrent
+from  tornado.httpclient import AsyncHTTPClient
 import os
 import os.path
+from urllib.parse import urlparse
+import yaml
+import socket
 
 
 class GitMixin:
@@ -118,6 +122,34 @@ def prepare_local_repo(self):
         else:
             return True
 
+    @gen.coroutine
+    def prepare_everware_yml(self, _user_log):
+        self.directory_data = self._repo_dir + "-data"
+        self._user_log = _user_log
+        os.mkdir(self.directory_data)
+        self.everware_yml_param["directory_data"] = self.directory_data
+        everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
+        if os.path.isfile(everware_yml_path):
+            with open(everware_yml_path) as file:
+                text = yaml.load(file)
+            all_data = []
+            for element in text:
+                all_data += text.get(element).get("data")
+            for data in all_data:
+                if type(data) == dict:
+                    url = data.get("url")
+                    ssl = data.get("ssl")
+                else:
+                    url = data
+                url_struct = urlparse(url)
+                self._user_log.append(({'text': "Start download from %s" % url_struct.geturl(), 'level': 2}))
+                if url_struct.scheme == "root":
+                    yield self.download_xrootd(url_struct, ssl)
+                else:
+                    yield self.download_http(url_struct, ssl)
+
+        self.everware_yml_param["user_log"] = self._user_log
+        return self.everware_yml_param
 
     @property
     def escaped_repo_url(self):
@@ -183,4 +215,52 @@ def get_state(self):
     def load_state(self, state):
         for key in self.STATE_VARS:
             if key in state:
-                setattr(self, key, state[key])
+                setattr(self, key, state[key])
+
+    def download_file(self, url):
+        url_struct = urlparse(url)
+        if url_struct.scheme == "root":
+            self.download_xrootd(self, url_struct)
+        else:
+            self.download_http(self, url_struct)
+
+    @concurrent.run_on_executor(executor='_git_executor')
+    def download_xrootd(self, url_struct, ssl):
+        try:
+            from sh import xrdcp
+        except:
+            self.log.info("Sorry, but everware doesn't support xrootd now")
+            return
+        try:
+            n_check_connection = 1
+            self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
+            xrdcp("-DIConnectionRetry", n_check_connection,"-r", url_struct.geturl(), self.directory_data)
+            self.log.info("Downloaded")
+            self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
+        except:
+            self.log.info("Cannot download")
+            self._user_log.append(({'text': "Fail to download from %s" % url_struct.geturl(), 'level': 2}))
+        return
+
+    @gen.coroutine
+    def download_http(self, url_struct, ssl):
+        http_client = AsyncHTTPClient()
+        filename = url_struct.path.split("/")[-1]
+        if len(filename) == 0:
+            filename = url_struct.path.split("/")[-2]
+        #folder with data files is deleted after container removing
+        if ssl:
+            pass
+        try:
+            self.log.info("Downloading from http server %s" % url_struct.hostname)
+            response = yield http_client.fetch(url_struct.geturl())
+            with open(self.directory_data + '/' + filename, "ab") as f:
+                f.write(response.body)
+            self.log.info("Downloaded")
+            self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
+            http_client.close()
+        except:
+            self.log.info("Something went wrong during downloading from %s " % url_struct.hostname)
+            pass
+
+

everware/spawner.py

@@ -39,6 +39,7 @@ def __init__(self, **kwargs):
         self._image_handler = ImageHandler()
         self._cur_waiter = None
         self._is_empty = False
+        self.everware_yml_param = {}
         DockerSpawner.__init__(self, **kwargs)
         EmailNotificator.__init__(self)
 
@@ -268,6 +269,8 @@ def build_image(self):
             ))
             self.log.info('Cloning repo %s' % self.repo_url)
             dockerfile_exists = yield self.prepare_local_repo()
+            self.everware_yml_param = yield self.prepare_everware_yml(self._user_log)
+            self._user_log = self._user_log and self.everware_yml_param["user_log"]
             if not dockerfile_exists:
                 self._add_to_log('No dockerfile. Use the default one %s' % os.environ['DEFAULT_DOCKER_IMAGE'])
 
@@ -371,6 +374,9 @@ def start(self, image=None):
                 yield self.remove_old_container()
             self.log.info("Starting container from image: %s" % image_name)
             self._add_to_log('Creating container')
+            #can't mount into /notebooks folder - it deletes all git files
+            if self.everware_yml_param.get("directory_data"):
+                self.volumes = {self.everware_yml_param.get("directory_data") : "/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )
@@ -425,6 +431,7 @@ def stop(self, now=False):
                     self.container_name, self.container_id[:7])
                 # remove the container, as well as any associated volumes
                 yield self.docker('remove_container', self.container_id, v=True)
+                rmtree(self.everware_yml_param.get("directory_data"))
 
         self.clear_state()
 

requirements.txt

@@ -1,9 +1,11 @@
 ipython[notebook]
 coveralls
 nose2
+sh
+pyyaml
 pytest>=2.8
 GitPython==2.1.1
 selenium==2.52.0
 jupyterhub==0.7.2
 docker-py==1.10.6
--e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner
+-e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner