Merge branch 'master-20190210-01-progressbar'

.travis.yml

@@ -33,7 +33,10 @@ install:
   - conda create -q -n testenv python=${TRAVIS_PYTHON_VERSION} $([ "$ROOT" != "None" ] && echo "root=${ROOT}") numpy
   - export CONDA_ENV_PATH=$HOME/miniconda/envs/testenv
   - source activate testenv
-  - if [ "$PANDAS" != "None" ]; then pip install pandas=="${PANDAS}"; fi
+  - if [ "$PANDAS" != "None" ]; then
+        pip install pandas=="${PANDAS}";
+        pip install jupyter;
+    fi
   - pip install -r requirements/test.txt
 
 script:

CHANGELOG.md

@@ -3,6 +3,9 @@
 ## [Unreleased]
 
 #### Changes from the previous release: ([diff](https://github.com/alphatwirl/alphatwirl/compare/v0.20.3...master))
+- added progress bars for Jupyter Notebook
+- replaced `multiprocessing` with `threading` for progress bars as
+  Jupyter Notebook doesn't let a fork to display progress bars
 - added function `atpbar`, which initializes `Atpbar`
   - used in `EventLoop` and `ReaderComposite`
 - added iterable `Atpbar`, which wraps another iterable and reports

alphatwirl/parallel/build.py

@@ -83,6 +83,8 @@ def _build_parallel_multiprocessing(quiet, processes):
         progressBar = None
     elif sys.stdout.isatty():
         progressBar = progressbar.ProgressBar()
+    elif is_jupyter_notebook():
+        progressBar = progressbar.ProgressBarJupyter()
     else:
         progressBar = progressbar.ProgressPrint()
 
@@ -97,6 +99,8 @@ def _build_parallel_multiprocessing(quiet, processes):
     return Parallel(progressMonitor, communicationChannel)
 
 ##__________________________________________________________________||
+def is_jupyter_notebook():
+    return True
 
 ##__________________________________________________________________||
 @_deprecated(msg='use alphatwirl.parallel.build.build_parallel() instead.')

alphatwirl/progressbar/ProgressBarJupyter.py

@@ -0,0 +1,104 @@
+# Tai Sakuma <tai.sakuma@gmail.com>
+import time
+
+import ipywidgets as widgets
+from IPython.display import display
+
+##__________________________________________________________________||
+class ProgressBarJupyter(object):
+    def __init__(self):
+        self.interval = 0.1 # [second]
+
+        self.container_widget = None
+        self.active_box_list = [ ]
+        self.complete_box_list = [ ]
+        self.widget_dict = { } # {taskid: [box, bar, label]}
+
+        self._read_time()
+
+    def __repr__(self):
+        return '{}()'.format(
+            self.__class__.__name__
+        )
+
+    def nreports(self):
+        return len(self.active_box_list)
+
+    def present(self, report):
+
+        if not self._need_to_update(report):
+            return
+
+        if self.container_widget is None:
+            self.container_widget = widgets.VBox()
+            display(self.container_widget)
+
+        if report.taskid not in self.widget_dict:
+            self._create_widget(report)
+
+        self._update_widget(report)
+
+        self._reorder_widgets(report)
+
+        self._read_time()
+
+    def _create_widget(self, report):
+        bar = widgets.IntProgress(
+            value=report.done, min=0, max=report.total, step=1,
+            description='',
+            bar_style='', # 'success', 'info', 'warning', 'danger' or ''
+            orientation='horizontal'
+         )
+        label = widgets.HTML(value='')
+        box = widgets.HBox([bar, label])
+        self.active_box_list.append(box)
+        self.container_widget.children = self.complete_box_list + self.active_box_list
+        self.widget_dict[report.taskid] = [box, bar, label]
+
+    def _update_widget(self, report):
+
+        percent = float(report.done)/report.total if report.total > 0 else 1
+        percent = round(percent * 100, 2)
+        percent = '<pre>{:6.2f}%</pre>'.format(percent)
+
+        box = self.widget_dict[report.taskid][0]
+
+        bar = self.widget_dict[report.taskid][1]
+        bar.value = report.done
+        bar.max = report.total
+        bar.description = percent
+        if report.done >= report.total:
+            bar.bar_style = 'success'
+
+        label = self.widget_dict[report.taskid][2]
+        name_field_length = 32
+        percent = float(report.done)/report.total if report.total > 0 else 1
+        bar = (':' * int(percent * 40)).ljust(40, " ")
+        percent = round(percent * 100, 2)
+        name = report.name[0:name_field_length]
+        label.value = '<pre> | {:8d} / {:8d} |:  {:<{}s}</pre>'.format(report.done, report.total, name, name_field_length)
+
+    def _reorder_widgets(self, report):
+        if report.done >= report.total:
+            box, bar, label = self.widget_dict[report.taskid]
+            if box in self.active_box_list:
+                self.active_box_list.remove(box)
+                self.complete_box_list.append(box)
+                self.container_widget.children = self.complete_box_list + self.active_box_list
+
+    def _need_to_update(self, report):
+        if self._time() - self.last_time > self.interval:
+            return True
+        if report.done == report.total:
+            return True
+        if report.done == 0:
+            return True
+        return False
+
+    def _time(self):
+        return time.time()
+
+    def _read_time(self):
+        self.last_time = self._time()
+
+##__________________________________________________________________||

alphatwirl/progressbar/ProgressReportPickup.py

@@ -1,11 +1,11 @@
 # Tai Sakuma <tai.sakuma@gmail.com>
-import multiprocessing
+import threading
 import time
 
 ##__________________________________________________________________||
-class ProgressReportPickup(multiprocessing.Process):
+class ProgressReportPickup(threading.Thread):
     def __init__(self, queue, presentation):
-        multiprocessing.Process.__init__(self)
+        threading.Thread.__init__(self)
         self.queue = queue
         self.presentation = presentation
         self.last_wait_time = 1.0 # [second]

alphatwirl/progressbar/__init__.py

@@ -8,6 +8,12 @@
 from .ProgressReporter import ProgressReporter
 from .main import atpbar
 
+##__________________________________________________________________||
+try:
+    from .ProgressBarJupyter import ProgressBarJupyter
+except ImportError:
+    pass
+
 ##__________________________________________________________________||
 _progress_reporter = None
 

tests/unit/parallel/test_build_parallel.py

@@ -9,6 +9,13 @@
 except ImportError:
     import mock
 
+has_jupyter_notebook = False
+try:
+    from alphatwirl.progressbar import ProgressBarJupyter
+    has_jupyter_notebook = True
+except ImportError:
+    pass
+
 from alphatwirl.parallel import build_parallel
 from alphatwirl.parallel.build import build_parallel_multiprocessing
 
@@ -27,10 +34,25 @@ def isatty(request, monkeypatch):
     monkeypatch.setattr(module, 'sys', f)
     return ret
 
+##__________________________________________________________________||
+if has_jupyter_notebook:
+    is_jupyter_notebook_parames = [True, False]
+else:
+    is_jupyter_notebook_parames = [False]
+
+@pytest.fixture(params=is_jupyter_notebook_parames)
+def is_jupyter_notebook(request, monkeypatch):
+    ret = request.param
+    f = mock.Mock()
+    f.return_value = ret
+    module = sys.modules['alphatwirl.parallel.build']
+    monkeypatch.setattr(module, 'is_jupyter_notebook', f)
+    return ret
+
 ##__________________________________________________________________||
 @pytest.mark.parametrize('processes', [0, 1, 3])
 @pytest.mark.parametrize('quiet', [True, False])
-def test_build_parallel_multiprocessing(quiet, processes, isatty):
+def test_build_parallel_multiprocessing(quiet, processes, isatty, is_jupyter_notebook):
 
     parallel_mode = 'multiprocessing'
     parallel = build_parallel(
@@ -50,6 +72,8 @@ def test_build_parallel_multiprocessing(quiet, processes, isatty):
     if not quiet:
         if isatty:
             assert 'ProgressBar' == parallel.progressMonitor.presentation.__class__.__name__
+        elif is_jupyter_notebook:
+            assert 'ProgressBarJupyter' == parallel.progressMonitor.presentation.__class__.__name__
         else:
             assert 'ProgressPrint' == parallel.progressMonitor.presentation.__class__.__name__