Merge branch 'slurmoverssh'

containers/tramway-notebook

@@ -59,11 +59,14 @@ The container OS is Ubuntu Xenial and can run on top of old OSes like CentOS6.
     #pip3.6 install --upgrade pip
     pip3.6 uninstall -qy tramway || true
     pip3.6 install . -r requirements.txt
-    pip3.6 install .[roi]
+    pip3.6 install .[animate,roi,webui]
 
-    pip3.6 install tqdm scikit-learn
+    pip3.6 install scikit-learn
     pip3.6 install paramiko
-    pip3.6 install notebook bokeh
+    pip3.6 install notebook
+
+    #apt-get install -y --no-install-recommends \
+    #    firefox firefox-geckodriver
 
     mkdir -p /pasteur
 

scripts/tramway-browse

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import tempfile as tmp
+import sys
+import subprocess
+
+def browse(files=[], browser='Firefox'):
+    if not files:
+        import glob
+        filepattern = '*.rwa'
+        if not glob.glob(filepattern):
+            filepattern = '*/*.rwa'
+            if not glob.glob(filepattern):
+                print('cannot find any matching rwa file')
+                return
+        files = [filepattern]
+
+    script = tmp.NamedTemporaryFile('w', suffix='.py', encoding='utf-8', delete=False)
+    source = """\
+#!/usr/bin/env python3
+
+from tramway.analyzer import *
+from selenium import webdriver
+
+a = RWAnalyzer()
+a.spt_data.from_rwa_files(['{}'])
+a.env.script = '{}'
+a.browser.show_maps(webdriver=webdriver.{})
+""".format("', '".join(files), script.name, browser)
+    script.write(source)
+    script.flush()
+    try:
+        ret = subprocess.run([sys.executable, script.name], check=True)
+    except KeyboardInterrupt:
+        ret = 'interrupted'
+    script.close()
+    return ret
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(prog='tramway-browse',
+        description='Browse TRamWAy-generated .rwa files')
+    parser.add_argument('files', nargs='*', help='for example: *.rwa or */*.rwa')
+    parser.add_argument('--browser', default='Firefox', choices=['Firefox','Chrome','Edge','Ie','Opera','Safari','WebKitGTK','PhantomJS'])
+    print(browse(**parser.parse_args().__dict__))
+
+if __name__ == '__main__':
+    main()
+

setup.py

@@ -6,7 +6,10 @@
 
 # requirements moved to requirements.txt
 install_requires = ['six', 'numpy', 'scipy', 'pandas', 'matplotlib', 'rwa-python>=0.8']
-extras_require = {'animate': ['opencv-python', 'tqdm'], 'roi': ['polytope', 'cvxopt', 'tqdm']}
+extras_require = {
+        'animate':  ['opencv-python', 'scikit-image', 'tqdm'],
+        'roi':  ['polytope', 'cvxopt', 'tqdm'],
+        'webui':    ['bokeh', 'selenium']}
 setup_requires = ['pytest-runner']
 tests_require = ['pytest']
 
@@ -19,7 +22,7 @@
 
 setup(
     name = 'tramway',
-    version = '0.5-beta',
+    version = '0.5-beta2',
     description = 'TRamWAy',
     long_description = long_description,
     url = 'https://github.com/DecBayComp/TRamWAy',
@@ -71,8 +74,9 @@
         'tramway.analyzer.mapper',
         'tramway.analyzer.env',
         'tramway.analyzer.browser',
-        'tramway.analyzer.pipeline'],
-    scripts = ['scripts/tramway'],
+        'tramway.analyzer.pipeline',
+        'tramway.analyzer.images'],
+    scripts = ['scripts/tramway', 'scripts/tramway-browse'],
     install_requires = install_requires,
     extras_require = extras_require,
     setup_requires = setup_requires,

tramway/analyzer/__init__.py

@@ -38,6 +38,7 @@ def proper_parent_name(attr_name):
 from .env       import *
 from .pipeline  import *
 from .browser   import *
+from .images    import *
 
 
 class BasicLogger(object):
@@ -187,7 +188,7 @@ class RWAnalyzer(object):
 
     """
     __slots__ = ( '_logger', '_spt_data', '_roi', '_time', '_tesseller', '_sampler', '_mapper',
-            '_env', '_pipeline', '_browser' )
+            '_env', '_pipeline', '_browser', '_images' )
 
     @property
     def logger(self):
@@ -280,21 +281,34 @@ def _set_env(self, env):
         self._env = env
     env = selfinitializing_property('env', _get_env, _set_env, Environment)
 
+    def _get_images(self):
+        """
+        Single molecule microscopy image stacks.
+
+        See :class:`~images.ImagesInitializer`.
+        """
+        return self._images
+    def _set_images(self, images):
+        self._images = images
+    images = selfinitializing_property('images', _get_images, _set_images, Images)
+
     def __init__(self):
         self._logger = \
                 self._spt_data = \
                 self._roi = \
                 self._tesseller = \
                 self._sampler = \
                 self._mapper = \
-                self._env = None
+                self._env = \
+                self._images = None
         self.spt_data  = SPTDataInitializer
         self.roi       = ROIInitializer
         self.time      = TimeInitializer
         self.tesseller = TessellerInitializer
         self.sampler   = SamplerInitializer
         self.mapper    = MapperInitializer
         self.env       = EnvironmentInitializer
+        self.images    = ImagesInitializer
         self._pipeline = Pipeline(self)
         self._browser  = Browser(self)
 
@@ -313,6 +327,14 @@ def run(self):
         """
         return self.pipeline.run()
 
+    def add_collectible(self, collectible):
+        """
+        designates a file generated at the worker side to be transferred back to the submit side.
+
+        See :meth:`Pipeline.add_collectible`
+        """
+        self.pipeline.add_collectible(collectible)
+
     @property
     def browser(self):
         """

tramway/analyzer/browser/__init__.py

@@ -1,6 +1,5 @@
 
 from ..attribute import *
-from tramway.plot.bokeh.analyzer import *
 
 class Browser(AnalyzerNode):
     """
@@ -55,6 +54,7 @@ class Browser(AnalyzerNode):
     """
     def show_maps(self, **kwargs):
         """ see also :func:`~tramway.plot.bokeh.analyzer.browse_maps`. """
+        from tramway.plot.bokeh.analyzer import browse_maps
         browse_maps(self._eldest_parent, **kwargs)
 
 

tramway/analyzer/env/environments.py

@@ -899,8 +899,12 @@ def collect_results(self, _log_pattern, stage_index=None, _parent_cls='Env'):
                 if dest.startswith(remote):
                     dest = local+dest[len(remote):]
                     break
-            self.ssh.get(end_result_file, dest)
-            any_transfer = True
+            try:
+                self.ssh.get(end_result_file, dest)
+            except FileNotFoundError: # the target file might be empty
+                self.logger.warning('failed')
+            else:
+                any_transfer = True
         return any_transfer
     @classmethod
     def _collectibles_from_log_files(cls, wd, log_pattern, stage_index=None):