Block usage of h5py 3+, numpy 1.19.4+, and test py3.9 nightly (#461)

.circleci/config.yml

@@ -133,7 +133,7 @@ jobs:
      - TEST_TOX_ENV: "py36"
      - COVERAGE_TOX_ENV: ""
      - BUILD_TOX_ENV: "build-py36"
-     - TEST_WHEELINSTALL_ENV: "wheelinstall-py36"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *ci-steps
 
   python37:
@@ -143,7 +143,7 @@ jobs:
      - TEST_TOX_ENV: "py37"
      - COVERAGE_TOX_ENV: ""
      - BUILD_TOX_ENV: "build-py37"
-     - TEST_WHEELINSTALL_ENV: "wheelinstall-py37"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *ci-steps
 
   python38:
@@ -153,7 +153,7 @@ jobs:
      - TEST_TOX_ENV: "py38"
      - COVERAGE_TOX_ENV: "coverage"
      - BUILD_TOX_ENV: "build-py38"
-     - TEST_WHEELINSTALL_ENV: "wheelinstall-py38"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
      - UPLOAD_WHEELS: "true"
     <<: *ci-steps
 
@@ -164,9 +164,18 @@ jobs:
      - TEST_TOX_ENV: "py39"
      - COVERAGE_TOX_ENV: ""
      - BUILD_TOX_ENV: "build-py39"
-     - TEST_WHEELINSTALL_ENV: "wheelinstall-py39"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *ci-steps
 
+  python38-upgrade-dev:
+    docker:
+      - image: circleci/python:3.8.6-buster
+    environment:
+     - TEST_TOX_ENV: "py38-upgrade-dev"
+     - COVERAGE_TOX_ENV: ""
+     - BUILD_TOX_ENV: "build-py38-upgrade-dev"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
+    <<: *ci-steps
 
   python36-min-req:
     docker:
@@ -175,7 +184,7 @@ jobs:
      - TEST_TOX_ENV: "py36-min-req"
      - COVERAGE_TOX_ENV: ""
      - BUILD_TOX_ENV: "build-py36-min-req"
-     - TEST_WHEELINSTALL_ENV: "wheelinstall-py36-min-req"
+     - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *ci-steps
 
   miniconda36:
@@ -185,7 +194,7 @@ jobs:
       - CONDA_PYTHON_VER: "3.6"
       - TEST_TOX_ENV: "py36"
       - BUILD_TOX_ENV: "build-py36"
-      - TEST_WHEELINSTALL_ENV: "wheelinstall-py36"
+      - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *conda-steps
 
   miniconda37:
@@ -195,7 +204,7 @@ jobs:
       - CONDA_PYTHON_VER: "3.7"
       - TEST_TOX_ENV: "py37"
       - BUILD_TOX_ENV: "build-py37"
-      - TEST_WHEELINSTALL_ENV: "wheelinstall-py37"
+      - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *conda-steps
 
   miniconda38:
@@ -205,7 +214,7 @@ jobs:
       - CONDA_PYTHON_VER: "3.8"
       - TEST_TOX_ENV: "py38"
       - BUILD_TOX_ENV: "build-py38"
-      - TEST_WHEELINSTALL_ENV: "wheelinstall-py38"
+      - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *conda-steps
 
   miniconda39:
@@ -215,7 +224,7 @@ jobs:
       - CONDA_PYTHON_VER: "3.9"
       - TEST_TOX_ENV: "py39"
       - BUILD_TOX_ENV: "build-py39"
-      - TEST_WHEELINSTALL_ENV: "wheelinstall-py39"
+      - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *conda-steps
 
   gallery36:
@@ -246,6 +255,13 @@ jobs:
      - TEST_TOX_ENV: "gallery-py39"
     <<: *gallery-steps
 
+  gallery38-upgrade-dev:
+    docker:
+      - image: circleci/python:3.8.1-buster
+    environment:
+     - TEST_TOX_ENV: "gallery-py38-upgrade-dev"
+    <<: *gallery-steps
+
   gallery36-min-req:
     docker:
       - image: circleci/python:3.6.10-stretch
@@ -346,7 +362,7 @@ workflows:
           <<: *no_filters
       - python38:
           <<: *no_filters
-      - python39:
+      - python38-upgrade-dev:
           <<: *no_filters
       - python36-min-req:
           <<: *no_filters
@@ -356,15 +372,13 @@ workflows:
           <<: *no_filters
       - miniconda38:
           <<: *no_filters
-      - miniconda39:
-          <<: *no_filters
       - gallery36:
           <<: *no_filters
       - gallery37:
           <<: *no_filters
       - gallery38:
           <<: *no_filters
-      - gallery39:
+      - gallery38-upgrade-dev:
           <<: *no_filters
       - gallery36-min-req:
           <<: *no_filters
@@ -374,16 +388,15 @@ workflows:
             - python36
             - python37
             - python38
-            - python39
+            - python38-upgrade-dev
             - python36-min-req
             - miniconda36
             - miniconda37
             - miniconda38
-            - miniconda39
             - gallery36
             - gallery37
             - gallery38
-            - gallery39
+            - gallery38-upgrade-dev
             - gallery36-min-req
           filters:
             tags:
@@ -399,16 +412,15 @@ workflows:
             - python36
             - python37
             - python38
-            - python39
+            - python38-upgrade-dev
             - python36-min-req
             - miniconda36
             - miniconda37
             - miniconda38
-            - miniconda39
             - gallery36
             - gallery37
             - gallery38
-            - gallery39
+            - gallery38-upgrade-dev
             - gallery36-min-req
           filters:
             tags:
@@ -419,3 +431,18 @@ workflows:
           filters:
             branches:
               ignore: dev
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 5 * * *"
+          filters:
+            branches:
+              only: dev
+    jobs:
+      - python39:
+          <<: *no_filters
+      - miniconda39:
+          <<: *no_filters
+      - gallery39:
+          <<: *no_filters

CHANGELOG.md

@@ -15,6 +15,8 @@
 - Add SimpleMultiContainer, a data_type for storing a Container and Data objects together. @ajtritt (#449)
 - Support `pathlib.Path` paths in `HDMFIO.__init__`, `HDF5IO.__init__`, and `HDF5IO.load_namespaces`. @dsleiter (#439)
 - Use hdmf-common-schema 1.2.1. See https://hdmf-common-schema.readthedocs.io/en/latest/format_release_notes.html for details.
+- Block usage of h5py 3+. h5py>=2.9, <3 is supported.
+- Block usage of numpy>=1.19.4 due to a known issue with numpy on some Windows 10 systems. numpy>1.16, <1.19.4 is supported.
 
 ### Internal improvements
 - Refactor `HDF5IO.write_dataset` to be more readable. @rly (#428)

azure-pipelines-nightly.yml

@@ -0,0 +1,69 @@
+trigger: none
+
+pr: none
+
+schedules:
+- cron: "0 5 * * *"
+  displayName: nightly
+  branches:
+    include: dev
+
+jobs:
+
+- job: 'Test'
+  displayName: "Test HDMF"
+
+  strategy:
+    matrix:
+      macOS-py3.9:
+        imageName: 'macos-10.15'
+        pythonVersion: '3.9'
+        testToxEnv: 'py39'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py39'
+        testWheelInstallEnv: 'wheelinstall'
+
+      Windows-py3.9:
+        imageName: 'vs2017-win2016'
+        pythonVersion: '3.9'
+        testToxEnv: 'py39'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py39'
+        testWheelInstallEnv: 'wheelinstall'
+
+  pool:
+    vmImage: $(imageName)
+
+  steps:
+
+  - checkout: self
+    submodules: true
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(pythonVersion)'
+      architecture: 'x64'
+
+  - script: |
+      python -m pip install --upgrade pip
+      python -m pip install --upgrade setuptools
+      python -m pip install setuptools wheel virtualenv tox
+    displayName: 'Install build dependencies'
+
+  - bash: |
+      tox -e $(testToxEnv)
+    displayName: 'Run tox tests'
+
+  - bash: |
+      if [[ "$(coverageToxEnv)" != "" ]]; then
+        tox -e $(coverageToxEnv)
+      fi
+    displayName: 'Run coverage tests if coverageToxEnv != ""'
+
+  - bash: |
+      tox -e $(buildToxEnv)
+    displayName: 'Run build tests'
+
+  - bash: |
+      tox -e $(testWheelInstallEnv) --recreate --installpkg dist/*-none-any.whl
+    displayName: 'Run wheel install tests'

azure-pipelines.yml

@@ -8,85 +8,85 @@ jobs:
 
   strategy:
     matrix:
-      macOS-py3.9:
-        imageName: 'macos-10.15'
-        pythonVersion: '3.9'
-        testToxEnv: 'py39'
-        coverageToxEnv: ''
-        buildToxEnv: 'build-py39'
-        testWheelInstallEnv: 'wheelinstall-py39'
-
       macOS-py3.8:
         imageName: 'macos-10.15'
         pythonVersion: '3.8'
         testToxEnv: 'py38'
         coverageToxEnv: 'coverage'
         buildToxEnv: 'build-py38'
-        testWheelInstallEnv: 'wheelinstall-py38'
+        testWheelInstallEnv: 'wheelinstall'
 
       macOS-py3.7:
         imageName: 'macos-10.15'
         pythonVersion: '3.7'
         testToxEnv: 'py37'
         coverageToxEnv: ''
         buildToxEnv: 'build-py37'
-        testWheelInstallEnv: 'wheelinstall-py37'
+        testWheelInstallEnv: 'wheelinstall'
 
       macOS-py3.6:
         imageName: 'macos-10.15'
         pythonVersion: '3.6'
         testToxEnv: 'py36'
         coverageToxEnv: ''
         buildToxEnv: 'build-py36'
-        testWheelInstallEnv: 'wheelinstall-py36'
+        testWheelInstallEnv: 'wheelinstall'
+
+      macOS-py3.8-upgrade-dev:
+        imageName: 'macos-10.15'
+        pythonVersion: '3.8'
+        testToxEnv: 'py38-upgrade-dev'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py38-upgrade-dev'
+        testWheelInstallEnv: 'wheelinstall'
 
       macOS-py3.6-min-req:
         imageName: 'macos-10.15'
         pythonVersion: '3.6'
         testToxEnv: 'py36-min-req'
         coverageToxEnv: ''
         buildToxEnv: 'build-py36-min-req'
-        testWheelInstallEnv: 'wheelinstall-py36-min-req'
-
-      Windows-py3.9:
-        imageName: 'vs2017-win2016'
-        pythonVersion: '3.9'
-        testToxEnv: 'py39'
-        coverageToxEnv: ''
-        buildToxEnv: 'build-py39'
-        testWheelInstallEnv: 'wheelinstall-py39'
+        testWheelInstallEnv: 'wheelinstall'
 
       Windows-py3.8:
         imageName: 'vs2017-win2016'
         pythonVersion: '3.8'
         testToxEnv: 'py38'
         coverageToxEnv: 'coverage'
         buildToxEnv: 'build-py38'
-        testWheelInstallEnv: 'wheelinstall-py38'
+        testWheelInstallEnv: 'wheelinstall'
 
       Windows-py3.7:
         imageName: 'vs2017-win2016'
         pythonVersion: '3.7'
         testToxEnv: 'py37'
         coverageToxEnv: ''
         buildToxEnv: 'build-py37'
-        testWheelInstallEnv: 'wheelinstall-py37'
+        testWheelInstallEnv: 'wheelinstall'
 
       Windows-py3.6:
         imageName: 'vs2017-win2016'
         pythonVersion: '3.6'
         testToxEnv: 'py36'
         coverageToxEnv: ''
         buildToxEnv: 'build-py36'
-        testWheelInstallEnv: 'wheelinstall-py36'
+        testWheelInstallEnv: 'wheelinstall'
+
+      Windows-py3.8-upgrade-dev:
+        imageName: 'vs2017-win2016'
+        pythonVersion: '3.8'
+        testToxEnv: 'py38-upgrade-dev'
+        coverageToxEnv: ''
+        buildToxEnv: 'build-py38-upgrade-dev'
+        testWheelInstallEnv: 'wheelinstall'
 
       Windows-py3.6-min-req:
         imageName: 'vs2017-win2016'
         pythonVersion: '3.6'
         testToxEnv: 'py36-min-req'
         coverageToxEnv: ''
         buildToxEnv: 'build-py36-min-req'
-        testWheelInstallEnv: 'wheelinstall-py36-min-req'
+        testWheelInstallEnv: 'wheelinstall'
 
   pool:
     vmImage: $(imageName)

docs/source/update_requirements.rst

@@ -74,3 +74,5 @@ Minimum requirements should be updated manually if a new feature or bug fix is a
 for proper running of HDMF. Minimum requirements should also be updated if a user requests that HDMF be installable
 with an older version of a dependency, all tests pass using the older version, and there is no valid reason for the
 minimum version to be as high as it is.
+
+These minimum requirements are used by ``setup.py`` which replaces "==" with ">=".

requirements-dev.txt

@@ -1,3 +1,5 @@
+# pinned dependencies to reproduce an entire development environment to use HDMF, run HDMF tests, check code style,
+# compute coverage, and create test environments
 codecov==2.1.8
 coverage==5.2
 flake8==3.8.3

requirements-doc.txt

@@ -1,3 +1,4 @@
+# dependencies to generate the documentation for HDMF
 sphinx
 sphinx_rtd_theme
 sphinx-gallery