fix s3_local script; add docker init to CI on all platforms

jpmorganchase · Apr 1, 2020 · 865f4e5 · 865f4e5
1 parent 648c1a1
commit 865f4e5
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 20 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -4,7 +4,7 @@ trigger:
     - '*'  # must quote since "*" is a YAML reserved character; we want a string
 
 parameters:
-- name: initializationSteps
+- name: initSteps
   type: stepList
   default:
     - task: UsePythonVersion@0
@@ -37,27 +37,47 @@ parameters:
 
     - bash: env
       displayName: "List all environment variables"
-- name: s3proxySteps
+
+- name: initDockerSteps
+  type: stepList
+  default:
+    - task: DockerInstaller@0
+      displayName: Docker Installer
+      inputs:
+        dockerVersion: 19.03.8
+        releaseType: stable
+
+- name: initDockerStepsMac
+  type: stepList
+  default:
+    - script: |
+        ci/install_docker_desktop_mac.sh
+        ci/start_docker_desktop_mac.sh
+      displayName: 'Setup Docker Desktop Mac'
+
+- name: initS3proxySteps
   type: stepList
   default:
-    - bash: nohup ci/s3_local.sh
+    - bash: |
+        nohup ci/s3_local.sh -d
+        sleep 5
       displayName: "Start up local S3 server"
 
     - script: curl http://127.0.0.1:9000/
       displayName: 'Test local S3 server'
-- name: s3proxyMacSteps
+
+- name: initS3proxyStepsMac
   type: stepList
   default:
     - script: |
-        ci/install_docker_desktop_mac.sh
-        ci/start_docker_desktop_mac.sh
         docker pull andrewgaul/s3proxy
-        nohup sudo docker run -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy &
-        sleep 5
+        sudo docker run --detach -p 9000:80 --env S3PROXY_AUTHORIZATION=none andrewgaul/s3proxy
+        sleep 15
       displayName: 'Start s3proxy in a docker'
 
     - script: curl http://127.0.0.1:9000/
       displayName: 'Test local S3 server'
+
 - name: testSteps
   type: stepList
   default:
@@ -72,6 +92,7 @@ parameters:
 
     - script: make testjs
       displayName: 'Test JS'
+
 - name: publishSteps
   type: stepList
   default:
@@ -97,8 +118,9 @@ jobs:
         python.version: '3.7'
 
   steps:
-    - ${{ parameters.initializationSteps }}
-    - ${{ parameters.s3proxySteps }}
+    - ${{ parameters.initDockerSteps }}
+    - ${{ parameters.initS3proxySteps }}
+    - ${{ parameters.initSteps }}
     - ${{ parameters.testSteps }}
     - ${{ parameters.publishSteps }}
 
@@ -112,22 +134,23 @@ jobs:
         python.version: '3.7'
 
   steps:
-    - ${{ parameters.s3proxyMacSteps }}
-    - ${{ parameters.initializationSteps }}
+    - ${{ parameters.initDockerStepsMac }}
+    - ${{ parameters.initS3proxyStepsMac }}
+    - ${{ parameters.initSteps }}
     - ${{ parameters.testSteps }}
     - ${{ parameters.publishSteps }}
 
 - job: 'Windows'
   pool:
-    vmImage: 'vs2017-win2016'
+    vmImage: 'windows-2019'
 
   strategy:
     matrix:
       Python37:
         python.version: '3.7'
 
   steps:
-    - ${{ parameters.initializationSteps }}
-    - ${{ parameters.s3proxySteps }}
+    - ${{ parameters.initS3proxySteps }}
+    - ${{ parameters.initSteps }}
     - ${{ parameters.testSteps }}
     - ${{ parameters.publishSteps }}
diff --git a/ci/s3_local.sh b/ci/s3_local.sh
@@ -53,7 +53,6 @@ EOT
 # s3proxy.credential=s3_local
 
 # get built s3proxy jar
-command -v java -jar s3proxy >/dev/null 2>&1 || \
 curl -L https://github.com/gaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy -o s3proxy
 
 # run s3proxy as a background job

diff --git a/ci/start_docker_desktop_mac.sh b/ci/start_docker_desktop_mac.sh
@@ -11,16 +11,16 @@ printf "Starting Docker.app, if necessary"
 # Wait for the server to start up, if applicable.
 retries=0
 while ! docker system info &>/dev/null; do
-    if (( retries % 10 == 0 )); then
+    if (( retries % 30 == 0 )); then
         if pgrep -xq -- "Docker"; then
             printf '\nDocker init still running'
         else
             (( retries != 0 )) && printf '\nDocker not running, restart'
-            /Applications/Docker.app/Contents/MacOS/Docker &
+            open -g -a Docker.app || exit
         fi
 
-        if [[ ${retries} -gt 150 ]]; then
-            >&2 printf '\nFailed to run Docker'
+        if [[ ${retries} -gt 600 ]]; then
+            printf '\nFailed to run Docker'
             exit 1
         fi
     fi