Merge pull request #343 from pymor/appveyor

[WIP] Enable Automated testing on Windows

.ci/appveyor.yml

@@ -0,0 +1,93 @@
+environment:
+  matrix:
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+    # building pymor extensions fails on 2.7 currently
+    #- PYTHON_VERSION: "2.7" # currently 2.7.11
+      #PYTHON_ARCH: "32"
+      #CONDA: "C:\\Miniconda"
+      #MARKER: "not grid"
+
+    #- PYTHON_VERSION: "2.7" # currently 2.7.11
+      #PYTHON_ARCH: "32"
+      #CONDA: "C:\\Miniconda"
+      #MARKER: "grid"
+
+    - PYTHON_VERSION: "3.4" # currently 3.4.3
+      PYTHON_ARCH: "32"
+      CONDA: "C:\\Miniconda3"
+      MARKER: "not grid"
+      QT_API: "pyqt5"
+
+    - PYTHON_VERSION: "3.4" # currently 3.4.3
+      PYTHON_ARCH: "32"
+      CONDA: "C:\\Miniconda3"
+      MARKER: "grid"
+      QT_API: "pyqt5"
+
+    - PYTHON_VERSION: "3.5" # currently 3.5.1
+      PYTHON_ARCH: "32"
+      CONDA: "C:\\Miniconda35"
+      MARKER: "not grid"
+      QT_API: "pyqt5"
+      ALLOW_FAIL: "true"
+
+    - PYTHON_VERSION: "3.5" # currently 3.5.1
+      PYTHON_ARCH: "32"
+      CONDA: "C:\\Miniconda35"
+      MARKER: "grid"
+      QT_API: "pyqt5"
+
+matrix:
+  allow_failures:
+    - ALLOW_FAIL: "true"
+
+install:
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  # credits: JuliaLang developers.
+    - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+          throw "There are newer queued builds for this pull request, failing early." }
+    - set PATH=%CONDA%;%CONDA%\Scripts;%PATH%
+    - conda config --set always_yes yes --set changeps1 no
+    - conda update -q conda
+    # hardcode a few installs that don't succeed otherwise
+    - conda install numpy pip m2-unzip
+    - conda install -c anaconda pyqt=5.6.0
+    - if [%PYTHON_VERSION%] == [3.4] ( conda remove numpy && pip install numpy && pip install https://bitbucket.org/pauloh/pyevtk/get/tip.tar.gz)
+
+    - for /f %%i in (requirements.txt requirements-optional.txt requirements-travis.txt ) do (
+                conda install %%i 1> nul 2>&1 & pip install %%i 1> nul 2>&1 & echo %%i
+            )
+    - curl -fsS -o gmsh.zip http://gmsh.info/bin/Windows/gmsh-2.16.0-Windows32.zip && unzip -j gmsh.zip -d %CONDA%\Scripts
+
+build_script:
+    - set PATH=%CONDA%;%CONDA%\Scripts;%PATH%
+    - pip freeze
+    - set PYTHONPATH=%cd%\src;%PYTHONPATH%;
+    - python setup.py build_ext -i
+
+test_script:
+    - set PYTHONPATH=%cd%\src;%PYTHONPATH%;
+    - py.test -r sxX -k "%MARKER%"
+
+#after_test:
+  # If tests are successful, create binary packages for the project.
+  #- "%CMD_IN_ENV% python setup.py bdist_wheel"
+  #- "%CMD_IN_ENV% python setup.py bdist_wininst"
+  #- "%CMD_IN_ENV% python setup.py bdist_msi"
+  #- ps: "ls dist"
+
+#artifacts:
+  # Archive the generated packages in the ci.appveyor.com build report.
+  #- path: dist\*
+
+#on_success:
+#  - TODO: upload the content of dist/*.whl to a public wheelhouse
+#

requirements-optional.txt

@@ -1,13 +1,14 @@
+-r requirements.txt
 https://bitbucket.org/pauloh/pyevtk/get/tip.tar.gz
 ipyparallel
 ipython>=3.0
 matplotlib
 mpi4py
 pyamg
 pyopengl
-pyside ; python_version == '2.7'
-https://pymor.github.io/wheels/PySide-1.2.2-cp34-cp34m-linux_x86_64.whl ; python_version == '3.4'
-https://pymor.github.io/wheels/PySide-1.2.2-cp35-cp35m-linux_x86_64.whl ; python_version == '3.5'
+qtpy
+PySide ; python_version < '3.5'
+PyQt5 ; python_version >= '3.5'
 pytest>=3.0
 pytest-cov
 pillow

requirements-travis.txt

@@ -1,3 +1,4 @@
+-r requirements.txt
 pytest-cov
 check-manifest
 python-coveralls

requirements.txt

@@ -4,3 +4,4 @@ docopt
 numpy>=1.8.1
 pytest-runner>=2.9
 scipy>=0.13.3
+psutil

setup.cfg

@@ -20,13 +20,13 @@ exclude = __init__.py
 
 [check-manifest]
 ignore =
-    src/pymor/version.py
     .ci/*
     .ci
     .travis.yml
     debian
     debian/*
     .landscape.yaml
+    .appveyor.yml
     .mailmap
     graveyard/*
     graveyard

setup.py

@@ -115,7 +115,7 @@ def __init__(self, *args, **kwargs):
         )
         self.rtool.refactor_dir('src', write=True)
         self.rtool.refactor_dir('docs', write=True)
-        open(checkpoint_fn, 'wta').write('converted')
+        open(checkpoint_fn, 'wt').write('converted')
 
 cmdclass = versioneer.get_cmdclass()
 if sys.version_info[0] < 3:

src/pymor/core/cache.py

@@ -86,6 +86,10 @@ def cleanup_non_persisten_regions():
             region.clear()
 
 
+def _safe_filename(old_name):
+    return ''.join(x for x in old_name if (x.isalnum() or x in '._- '))
+
+
 class CacheRegion(object):
     """Base class for all pyMOR cache regions.
 
@@ -198,7 +202,7 @@ def get(self, key):
             raise RuntimeError('Cache is corrupt!')
 
     def set(self, key, value):
-        fd, file_path = tempfile.mkstemp('.dat', datetime.datetime.now().isoformat()[:-7] + '-', self.path)
+        fd, file_path = tempfile.mkstemp('.dat', _safe_filename(datetime.datetime.now().isoformat()[:-7]) + '-', self.path)
         filename = os.path.basename(file_path)
         with os.fdopen(fd, 'wb') as f:
             dump(value, f)

src/pymor/core/config.py

@@ -15,6 +15,12 @@ def _get_fenics_version():
     return version
 
 
+def is_windows_platform():
+    return sys.platform == 'win32' or sys.platform == 'cygwin'
+
+
+    if is_windows_platform():
+        matplotlib.use('Qt4Agg')
 def _get_ipython_version():
     try:
         import ipyparallel

src/pymor/core/logger.py

@@ -10,7 +10,6 @@
 :func:`set_log_levels` methods.
 """
 
-import curses
 import logging
 import os
 import time
@@ -65,6 +64,7 @@ def __init__(self):
             self.use_color = False
         else:
             try:
+                import curses
                 curses.setupterm()
                 self.use_color = curses.tigetnum("colors") > 1
             except Exception:

src/pymor/gui/qt.py

@@ -14,8 +14,11 @@
 import multiprocessing
 import os
 import signal
+import sys
+import psutil
 
 from pymor.core.config import config
+from pymor.core.config import is_windows_platform
 from pymor.core.defaults import defaults
 from pymor.core.logger import getLogger
 from pymor.core.exceptions import QtMissing
@@ -28,7 +31,8 @@
 if config.HAVE_QT:
     from Qt.QtWidgets import (QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QSlider, QApplication, QLCDNumber,
                               QAction, QStyle, QToolBar, QLabel, QFileDialog, QMessageBox)
-    from Qt.QtCore import Qt, QCoreApplication, QTimer
+    from Qt.QtCore import Qt, QCoreApplication, QTimer, Slot
+
 
     class PlotMainWindow(QWidget):
         """Base class for plot main windows."""
@@ -170,37 +174,44 @@ def step_backward(self):
                 self.slider.setValue(ind)
 
 
-_launch_qt_app_pids = set()
+_launch_qt_processes = set()
 
 
 def _launch_qt_app(main_window_factory, block):
     """Wrapper to display plot in a separate process."""
 
-    def doit():
+    def _doit(factory):
         try:
             app = QApplication([])
         except RuntimeError:
             app = QCoreApplication.instance()
-        main_window = main_window_factory()
+        main_window = factory()
+        if getattr(sys, '_called_from_test', False) and is_windows_platform():
+            QTimer.singleShot(500, app, Slot('quit()'))
         main_window.show()
         app.exec_()
 
     import sys
-    if block and not getattr(sys, '_called_from_test', False):
-        doit()
+    if (block and not getattr(sys, '_called_from_test', False)) or is_windows_platform():
+        _doit(main_window_factory)
     else:
-        p = multiprocessing.Process(target=doit)
+        p = multiprocessing.Process(target=_doit, args=(main_window_factory,))
         p.start()
-        _launch_qt_app_pids.add(p.pid)
+        _launch_qt_processes.add(psutil.Process(p.pid))
 
 
 def stop_gui_processes():
-    for p in multiprocessing.active_children():
-        if p.pid in _launch_qt_app_pids:
-            try:
-                os.kill(p.pid, signal.SIGKILL)
-            except OSError:
-                pass
+    active_pids = {p.pid for p in multiprocessing.active_children()}
+    kill_procs = [pr for pr in _launch_qt_processes if pr.pid in active_pids]
+    for p in kill_procs:
+        try:
+            # active_children apparently contains false positives sometimes
+            p.terminate()
+        except psutil.NoSuchProcess:
+            pass
+    gone, still_alive = psutil.wait_procs(kill_procs, timeout=1)
+    for p in still_alive:
+        p.kill()
 
 
 @defaults('backend', sid_ignore=('backend',))

src/pymor/tools/io.py

@@ -1,10 +1,13 @@
 # This file is part of the pyMOR project (http://www.pymor.org).
 # Copyright 2013-2017 pyMOR developers and contributors. All rights reserved.
 # License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)
-
 from scipy.io import loadmat, mmread
 from scipy.sparse import issparse
 import numpy as np
+import tempfile
+import os
+from contextlib import contextmanager
+import shutil
 
 from pymor.core.logger import getLogger
 
@@ -110,3 +113,19 @@ def load_matrix(path, key=None):
             pass
 
     raise IOError('Could not load file {} (key = {})'.format(path, key))
+
+
+@contextmanager
+def SafeTemporaryFileName(name=None, parent_dir=None):
+    """Cross Platform safe equivalent of re-opening a NamedTemporaryFile
+    Creates an automatically cleaned up temporary directory with a single file therein.
+
+    name: filename component, defaults to 'temp_file'
+    dir: the parent dir of the new tmp dir. defaults to tempfile.gettempdir()
+    """
+    parent_dir = parent_dir or tempfile.gettempdir()
+    name = name or 'temp_file'
+    dirname = tempfile.mkdtemp(dir=parent_dir)
+    path = os.path.join(dirname, name)
+    yield path
+    shutil.rmtree(dirname)

src/pymordemos/thermalblock_gui.py

@@ -39,6 +39,9 @@
 import numpy as np
 import OpenGL
 
+from pymor.core.config import is_windows_platform
+from pymor.gui.matplotlib import MatplotlibPatchWidget
+
 OpenGL.ERROR_ON_COPY = True
 
 from pymor.core.exceptions import QtMissing
@@ -89,10 +92,14 @@ def __init__(self, parent, sim):
         super().__init__(parent)
         self.sim = sim
         box = QtWidgets.QHBoxLayout()
-        self.solution = GLPatchWidget(self, self.sim.grid, vmin=0., vmax=0.8)
-        self.bar = ColorBarWidget(self, vmin=0., vmax=0.8)
-        box.addWidget(self.solution, 2)
-        box.addWidget(self.bar, 2)
+        if is_windows_platform():
+            self.solution = MatplotlibPatchWidget(self, self.sim.grid, vmin=0., vmax=0.8)
+            box.addWidget(self.solution, 2)
+        else:
+            self.solution = GLPatchWidget(self, self.sim.grid, vmin=0., vmax=0.8)
+            self.bar = ColorBarWidget(self, vmin=0., vmax=0.8)
+            box.addWidget(self.solution, 2)
+            box.addWidget(self.bar, 2)
         self.param_panel = ParamRuler(self, sim)
         box.addWidget(self.param_panel)
         self.setLayout(box)

src/pymortests/affine_grid.py

@@ -7,6 +7,7 @@
 import pytest
 
 from pymor.grids.interfaces import ReferenceElementInterface
+from pymortests.base import runmodule
 from pymortests.fixtures.grid import grid, grid_with_orthogonal_centers
 
 # monkey np.testing.assert_allclose to behave the same as np.allclose
@@ -307,3 +308,6 @@ def test_orthogonal_centers(grid_with_orthogonal_centers):
         SEGMENT = C[SUE[s, 0]] - C[SUE[s, 1]]
         SPROD = EMB[s].dot(SEGMENT)
         np.testing.assert_allclose(SPROD, 0)
+
+if __name__ == "__main__":
+    runmodule(filename=__file__)

src/pymortests/analyticalproblem.py

@@ -1,7 +1,7 @@
 # This file is part of the pyMOR project (http://www.pymor.org).
 # Copyright 2013-2017 pyMOR developers and contributors. All rights reserved.
 # License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)
-
+from pymortests.base import runmodule
 from pymortests.fixtures.analyticalproblem import analytical_problem, picklable_analytical_problem
 from pymortests.pickling import assert_picklable, assert_picklable_without_dumps_function
 
@@ -12,3 +12,6 @@ def test_pickle(analytical_problem):
 
 def test_pickle_without_dumps_function(picklable_analytical_problem):
     assert_picklable_without_dumps_function(picklable_analytical_problem)
+
+if __name__ == "__main__":
+    runmodule(filename=__file__)