Update CI pipelines

- Drop support for Python < 3.6, resolves #264 - Update Singularity to 3.7.3, resolves #359 - Update and fix shellcheck issues - Remove nodeps - Create separate Singularity image replication pipeline - Move code sign to CI - Remove Travis CI
Azure · Apr 21, 2021 · 1282779 · 1282779
1 parent 2cfc07d
commit 1282779
Show file tree

Hide file tree

Showing 43 changed files with 306 additions and 446 deletions.
diff --git a/.travis.yml b/.travis.yml
diff --git a/.vsts/pipeline-replicate-singularity.yml b/.vsts/pipeline-replicate-singularity.yml
@@ -0,0 +1,10 @@
+name: $(SourceBranch)$(Rev:.r)
+
+jobs:
+  - job: Replicate Singularity Images
+    pool:
+      vmImage: ubuntu-20.04
+    steps:
+      - template: replicate-singularity.yml
+        parameters:
+          enabled: True
diff --git a/.vsts/pipeline.yml b/.vsts/pipeline.yml
diff --git a/.vsts/replicate-singularity.yml b/.vsts/replicate-singularity.yml
@@ -0,0 +1,38 @@
+parameters:
+  enabled: false
+
+steps:
+  - script: |
+      set -e
+      set -o pipefail
+      docker version
+      docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)"
+      export DOCKER_CLI_EXPERIMENTAL=enabled
+      singularity_version=$(grep -m1 _SINGULARITY_VERSION convoy/misc.py | cut -d "'" -f 2)
+      echo "Replicating Singularity version $singularity_version images to MCR"
+      chkImage=mcr.microsoft.com/azure-batch/shipyard:${singularity_version}-singularity-mnt
+      set +e
+      if docker manifest inspect "$chkImage"; then
+        echo "$chkImage exists, skipping replication"
+      else
+        set -e
+        dhImage="alfpark/singularity:${singularity_version}-mnt"
+        mcrImage="$(docker.servername)/public/azure-batch/shipyard:${singularity_version}-singularity-mnt"
+        docker pull "$dhImage"
+        docker tag "$dhImage" "$mcrImage"
+        docker push "$mcrImage"
+      fi
+      chkImage=mcr.microsoft.com/azure-batch/shipyard:${singularity_version}-singularity-mnt-resource
+      set +e
+      if docker manifest inspect "$chkImage"; then
+        echo "$chkImage exists, skipping replication"
+      else
+        set -e
+        dhImage="alfpark/singularity:${singularity_version}-mnt-resource"
+        mcrImage="$(docker.servername)/public/azure-batch/shipyard:${singularity_version}-singularity-mnt-resource"
+        docker pull "$dhImage"
+        docker tag "$dhImage" "$mcrImage"
+        docker push "$mcrImage"
+      fi
+    displayName: Replicate Singularity Container Images
+    condition: ${{ parameters.enabled }}
diff --git a/.vsts/sign-exec.yml b/.vsts/sign-exec.yml
@@ -0,0 +1,39 @@
+parameters:
+  enabled: false
+  folder: ./bin
+  pattern: '*.dll,*.exe'
+
+steps:
+  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+    displayName: "Code Sign - Executables and Libraries"
+    condition: ${{ parameters.enabled }}
+    inputs:
+      ConnectedServiceName: 'ESRP CodeSign'
+      FolderPath: ${{ parameters.folder }}
+      Pattern: ${{ parameters.pattern }}
+      UseMinimatch: false
+      signConfigType: inlineSignParams
+      inlineOperation: |
+        [
+          {
+            "KeyCode" : "CP-230012",
+            "OperationCode" : "SigntoolSign",
+            "Parameters" : {
+                "OpusName" : "Microsoft",
+                "OpusInfo" : "http://www.microsoft.com",
+                "FileDigest" : "/fd \"SHA256\"",
+                "PageHash" : "/NPH",
+                "TimeStamp" : "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+            },
+            "ToolName" : "sign",
+            "ToolVersion" : "1.0"
+          },
+          {
+            "KeyCode" : "CP-230012",
+            "OperationCode" : "SigntoolVerify",
+            "Parameters" : {},
+            "ToolName" : "sign",
+            "ToolVersion" : "1.0"
+          }
+        ]
+      SessionTimeout: 60
diff --git a/.vsts/sign-nuget.yml b/.vsts/sign-nuget.yml
@@ -0,0 +1,33 @@
+parameters:
+  enabled: false
+  folder: ./bin
+  pattern: '*.nupkg'
+
+steps:
+  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+    displayName: "Code Sign - Nuget Packages"
+    condition: ${{ parameters.enabled }}
+    inputs:
+      ConnectedServiceName: 'ESRP CodeSign'
+      FolderPath: ${{ parameters.folder }}
+      Pattern: ${{ parameters.pattern }}
+      UseMinimatch: false
+      signConfigType: inlineSignParams
+      inlineOperation: |
+        [
+          {
+            "KeyCode" : "CP-401405",
+            "OperationCode" : "NuGetSign",
+            "Parameters" : {},
+            "ToolName" : "sign",
+            "ToolVersion" : "1.0"
+          },
+          {
+            "KeyCode" : "CP-401405",
+            "OperationCode" : "NuGetVerify",
+            "Parameters" : {},
+            "ToolName" : "sign",
+            "ToolVersion" : "1.0"
+          }
+        ]
+      SessionTimeout: 60
diff --git a/README.md b/README.md
@@ -1,5 +1,4 @@
 [![Build Status](https://azurebatch.visualstudio.com/batch-shipyard/_apis/build/status/batch-shipyard-CI)](https://azurebatch.visualstudio.com/batch-shipyard/_build/latest?definitionId=11)
-[![Build Status](https://travis-ci.org/Azure/batch-shipyard.svg?branch=master)](https://travis-ci.org/Azure/batch-shipyard)
 [![Build status](https://ci.appveyor.com/api/projects/status/3a0j0gww57o6nkpw/branch/master?svg=true)](https://ci.appveyor.com/project/alfpark/batch-shipyard)
 
 # Batch Shipyard

diff --git a/appveyor.yml b/appveyor.yml
@@ -7,14 +7,21 @@ cache:
 
 environment:
   matrix:
-  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-    PYTHON: "C:\\Python35-x64"
-    PYTHON_VERSION: "3.5"
-    PYTHON_ARCH: "64"
   - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
     PYTHON: "C:\\Python36-x64"
     PYTHON_VERSION: "3.6"
     PYTHON_ARCH: "64"
+    TOX_ENV: "py36"
+  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+    PYTHON: "C:\\Python37-x64"
+    PYTHON_VERSION: "3.7"
+    PYTHON_ARCH: "64"
+    TOX_ENV: "py37"
+  - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+    PYTHON: "C:\\Python38-x64"
+    PYTHON_VERSION: "3.8"
+    PYTHON_ARCH: "64"
+    TOX_ENV: "py38"
 
 init:
 - echo %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%
@@ -24,14 +31,13 @@ install:
 - where pip
 - pip install --upgrade setuptools wheel
 - pip install -r requirements.txt
-- pip install --no-deps -r req_nodeps.txt
-- pip install flake8==3.6.0
+- pip install flake8
 - pip list --format=columns --outdated
 
 build: off
 
 test_script:
-- flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy\\*.py
-- IF "%PYTHON_VERSION%" GEQ "3.5" (
-  flake8 --select F,E,W --ignore W504 --statistics cascade\\*.py cargo\\*.py federation\\*.py heimdall\\*.py slurm\\*.py
+- flake8 --exit-zero --select F,E,W --ignore W504 --statistics shipyard.py convoy\\*.py
+- IF "%PYTHON_VERSION%" GEQ "3.8" (
+  flake8 --exit-zero --select F,E,W --ignore W504 --statistics cascade\\*.py cargo\\*.py federation\\*.py heimdall\\*.py slurm\\*.py
   )
diff --git a/cargo/requirements.txt b/cargo/requirements.txt
@@ -1,4 +1,4 @@
-azure-batch==8.0.0
-msrest==0.6.10
-requests==2.22.0
-ruamel.yaml==0.16.5
+azure-batch==9.0.0
+msrest==0.6.21
+requests>=2.25.1,<3
+ruamel.yaml>=0.17.4,<1
diff --git a/cascade/requirements.txt b/cascade/requirements.txt
@@ -1,2 +1,2 @@
 azure-cosmosdb-table==1.0.6
-azure-storage-blob==2.1.0
+azure-storage-blob>=2.1.0,<3
diff --git a/convoy/aad.py b/convoy/aad.py
@@ -22,22 +22,11 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import datetime
-import io
 import json
 import logging
-try:
-    import pathlib2 as pathlib
-except ImportError:
-    import pathlib
+import pathlib
 import os
 # non-stdlib imports
 import adal
@@ -149,20 +138,10 @@ def signed_session(self):
             if cache_token and util.is_not_empty(self._token_cache_file):
                 logger.debug('storing token to local cache: {}'.format(
                     self._token_cache_file))
-                if util.on_python2():
-                    with io.open(
-                            self._token_cache_file,
-                            'w', encoding='utf8') as fd:
-                        fd.write(json.dumps(
-                            self._token, indent=4, sort_keys=True,
-                            ensure_ascii=False))
-                else:
-                    with open(
-                            self._token_cache_file,
-                            'w', encoding='utf8') as fd:
-                        json.dump(
-                            self._token, fd, indent=4, sort_keys=True,
-                            ensure_ascii=False)
+                with open(self._token_cache_file, 'w', encoding='utf8') as fd:
+                    json.dump(
+                        self._token, fd, indent=4, sort_keys=True,
+                        ensure_ascii=False)
                 if not util.on_windows():
                     os.chmod(self._token_cache_file, 0o600)
         except adal.AdalError as err:

diff --git a/convoy/autoscale.py b/convoy/autoscale.py
@@ -22,13 +22,6 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import collections
 # non-stdlib imports

diff --git a/convoy/batch.py b/convoy/batch.py
@@ -22,13 +22,6 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import codecs
 import collections
@@ -40,10 +33,7 @@
 import logging
 import multiprocessing
 import os
-try:
-    import pathlib2 as pathlib
-except ImportError:
-    import pathlib
+import pathlib
 import ssl
 import sys
 import time
@@ -3121,10 +3111,7 @@ def get_remote_login_settings(
                     pool_id, node.id)
             for node_id in futures:
                 ret[node_id] = futures[node_id].result()
-        if util.on_python2():
-            ret = collections.OrderedDict(sorted(ret.iteritems()))
-        else:
-            ret = collections.OrderedDict(sorted(ret.items()))
+        ret = collections.OrderedDict(sorted(ret.items()))
         if not suppress_output:
             for node_id in ret:
                 logger.info('node {}: ip {} port {}'.format(

diff --git a/convoy/clients.py b/convoy/clients.py
@@ -22,13 +22,6 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import logging
 # non-stdlib imports

diff --git a/convoy/crypto.py b/convoy/crypto.py
@@ -22,24 +22,14 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import base64
 import collections
 import datetime
 import getpass
 import logging
 import os
-try:
-    import pathlib2 as pathlib
-except ImportError:
-    import pathlib
+import pathlib
 import tempfile
 import stat
 import subprocess

diff --git a/convoy/data.py b/convoy/data.py
@@ -22,27 +22,14 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-# compat imports
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-from builtins import (  # noqa
-    bytes, dict, int, list, object, range, str, ascii, chr, hex, input,
-    next, oct, open, pow, round, super, filter, map, zip)
 # stdlib imports
 import datetime
 import fnmatch
 import logging
 import math
 import os
-try:
-    import pathlib2 as pathlib
-except ImportError:
-    import pathlib
-try:
-    from shlex import quote as shellquote
-except ImportError:
-    from pipes import quote as shellquote
+import pathlib
+from shlex import quote as shellquote
 import threading
 import time
 # non-stdlib imports
@@ -59,7 +46,7 @@
 logger = logging.getLogger(__name__)
 util.setup_logger(logger)
 # global defines
-_BLOBXFER_VERSION = '1.9.4'
+_BLOBXFER_VERSION = '1.10.0'
 _MEGABYTE = 1048576
 _MAX_READ_BLOCKSIZE_BYTES = 4194304
 _FILE_SPLIT_PREFIX = '_shipyard-'