Mark more io tests

devops/pr-gate-os.yml

@@ -11,15 +11,24 @@ stages:
           parameters:
             pyVersion: '3.6'
 
-  - stage: run_tests
+  - stage: run_full_test_matrix
     dependsOn: static_analysis
-    pool:
-      vmImage: 'ubuntu-18.04'
     jobs:
-      - template: templates/run-tests-on-multiple-py-versions.yml
+      - template: templates/run-tests-on-multiple-os-py.yml
         parameters:
           pyVersions: ['3.6', '3.7', '3.8']
-          testTypes: ['fast']
+          testTypes: ['unit', 'io']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+
+  - stage: run_e2e_test_matrix
+    dependsOn: static_analysis
+    jobs:
+      - template: templates/run-tests-on-multiple-os-py.yml
+        parameters:
+          pyVersions: ['3.6']
+          testTypes: ['slow', 'azure']
+          imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
 
 
 

devops/templates/publish-cov.yml

@@ -1,11 +1,15 @@
+parameters:
+- name: pyVersion
+  type: string
+
 steps:
 - task: PublishTestResults@2
   inputs:
     testResultsFormat: 'JUnit' 
     testResultsFiles: 'junit/*.xml' 
     searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName)_$(pyVersion) Build
-    buildPlatform: $(imageName)
+    testRunTitle: $(Agent.OS) py$(pyVersion) Build
+    buildPlatform: $(Agent.OS)
   condition: always() # Always publish test results
   displayName: 'Publish unit test report'
 

devops/templates/run-tests-on-multiple-os-py.yml

@@ -0,0 +1,40 @@
+parameters:
+- name: pyVersions
+  type: object
+  default: ['3.6', '3.7', '3.8']
+- name: testTypes
+  type: object
+  default: ['fast', 'slow']
+- name: imageOSs
+  type: object
+  default: ['ubuntu-latest']
+
+jobs:
+  - ${{ each imageOS in parameters.imageOSs }}:
+    - ${{ each pyVersion in parameters.pyVersions }}:
+      - job:
+        displayName: ${{imageOS}} py${{pyVersion}}
+        pool:
+          vmImage: ${{imageOS}}
+        steps:
+
+        - task: UsePythonVersion@0
+          inputs:
+            versionSpec: ${{pyVersion}}
+            addToPath: true
+            architecture: 'x64'
+          displayName: 'Use Python ${{pyVersion}}'
+
+        - template: install-dependencies.yml
+
+        - ${{ each testType in parameters.testTypes }}:
+          - template: run-tests.yml
+            parameters:
+              testType: ${{testType}}
+
+          - template: publish-test-results.yml
+            parameters:
+              pyVersion: ${{pyVersion}}
+
+        - template: publish-cov.yml
+

devops/templates/run-tests.yml

@@ -7,17 +7,18 @@ parameters:
   values:
     - slow
     - azure
-    - fast
+    - unit
+    - io
     - all
 
 steps:
 - bash: |
     if [[ '${{parameters.testType}}' == 'all' ]]
     then
       tox -e py 
-    elif [[ '${{parameters.testType}}' == 'fast' ]]
+    elif [[ '${{parameters.testType}}' == 'unit' ]]
     then
-      tox -e py -- -m "not slow and not azure"
+      tox -e py -- tests/unit
     else
       tox -e py -- -m "${{parameters.testType}}"
     fi

tests/e2e/test_image_channel.py

@@ -15,6 +15,7 @@ def doc_generator():
     return DocumentGenerator(template_path=TEMPLATE_PATH)
 
 
+@pytest.mark.io
 def test_red_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -25,6 +26,7 @@ def test_red_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "red.png", img_array)
 
 
+@pytest.mark.io
 def test_green_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator:
@@ -35,6 +37,7 @@ def test_green_channel(doc_generator):
         cv2.imwrite(TEST_OUT_FOLDER + "green.png", img_array)
 
 
+@pytest.mark.io
 def test_blue_channel(doc_generator):
     generator = doc_generator.create_generator(CONTENT, ["solid_bg.html.jinja"])
     for doc in generator: