[vcpkg] Fix OSX CI by ensuring the downloads directory exists (#11839)

* [vcpkg] Remove do-nothing Set-Content from Windows azure-pipelines.yml. * [vcpkg] Fix OSX CI by ensuring the downloads directory exists in advance, and extract common command line parameters with powershell splatting. * [tensorflow-cc] Prevent hang building tensorflow-cc asking to configure iOS. * Skip ignition-msgs5:x64-osx
microsoft · Jun 9, 2020 · 1d3e985 · 1d3e985
1 parent cb8a9fe
commit 1d3e985
Show file tree

Hide file tree

Showing 9 changed files with 73 additions and 59 deletions.
diff --git a/ports/tensorflow-cc/CONTROL b/ports/tensorflow-cc/CONTROL
@@ -1,5 +1,5 @@
 Source: tensorflow-cc
-Version: 1.14-1
+Version: 1.14-2
 Description: Library for computation using data flow graphs for scalable machine learning
 Build-Depends: c-ares
-Supports: !x86
+Supports: !x86
diff --git a/ports/tensorflow-cc/portfile.cmake b/ports/tensorflow-cc/portfile.cmake
@@ -1,9 +1,7 @@
-include(vcpkg_common_functions)
-
 message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.")
 
 if (VCPKG_TARGET_ARCHITECTURE STREQUAL x86)
-    message(FATAL_ERROR "TensorFlow does not support 32bit system.")
+    message(FATAL_ERROR "TensorFlow does not support 32bit systems.")
 endif()
 
 vcpkg_from_github(
@@ -76,6 +74,7 @@ set(ENV{TF_NCCL_VERSION} 2.3)
 set(ENV{NCCL_INSTALL_PATH} "")
 set(ENV{CC_OPT_FLAGS} "/arch:AVX")
 set(ENV{TF_NEED_CUDA} 0)
+set(ENV{TF_CONFIGURE_IOS} 0)
 
 message(STATUS "Configuring TensorFlow")
 
@@ -118,4 +117,4 @@ file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tenso
 file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright)
 
 file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc)
-file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)
+file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)
diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml
@@ -14,27 +14,19 @@ jobs:
       displayName: 'Initialize Environment'
       inputs:
         filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1'
-    - script: rmdir /s /q archives\fail
-      displayName: 'Delete archives\fail'
+    - script: rmdir /s /q W:\fail
+      displayName: 'Delete W:\fail'
   - job: linux
     displayName: 'Clean Linux Tombstones'
     pool: $(linux-pool)
     steps:
-    - task: Bash@3
-      displayName: 'Initialize Environment'
-      inputs:
-        filePath: scripts/azure-pipelines/linux/initialize-environment.sh
-    - bash: rm -rf archives/fail
-      displayName: 'Delete archives/fail'
+    - bash: rm -rf /archives/fail
+      displayName: 'Delete /archives/fail'
   - job: osx
     displayName: 'Clean MacOS Tombstones'
     pool:
       name: vcpkgAgentPool
       demands: Agent.OS -equals Darwin
     steps:
-    - task: Bash@3
-      displayName: 'Initialize Environment'
-      inputs:
-        filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'
-    - bash: rm -rf archives/fail
-      displayName: 'Delete archives/fail'
+    - bash: rm -rf /Users/vagrant/Data/archives/fail
+      displayName: 'Delete /Users/vagrant/Data/archives/fail'
diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml
@@ -33,11 +33,20 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - powershell: |
-      ./vcpkg x-ci-clean --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages
+      $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,/archives,upload',
+        '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees',
+        '--x-install-root=/mnt/vcpkg-ci/install',
+        '--x-packages-root=/mnt/vcpkg-ci/packages'
+      )
+
+      ./vcpkg x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
       mkdir $(System.ArtifactsDirectory)/xml-results
       $env:HOME = '/home/agent'
-      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/archives`,upload --x-buildtrees-root=/mnt/vcpkg-ci/buildtrees --x-install-root=/mnt/vcpkg-ci/install --x-packages-root=/mnt/vcpkg-ci/packages --downloads-root=/mnt/vcpkg-ci/downloads
+      ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml
@@ -13,22 +13,19 @@ jobs:
   steps:
   - bash: |
       df -h
-    displayName: 'Report on Disk Space Before Environment'
+    displayName: 'Report on Disk Space'
     condition: always()
-  - task: Bash@3
-    displayName: 'Initialize Environment'
-    inputs:
-      filePath: 'scripts/azure-pipelines/osx/initialize-environment.sh'
   - bash: |
       brew list autoconf || brew install autoconf
       brew list automake || brew install automake
       brew list libtool || brew install libtool
       brew list gfortran || brew cask install gfortran
     displayName: 'Install brew dependencies'
   - bash: |
-      df -h
-    displayName: 'Report on Disk Space After Environment'
-    condition: always()
+      sudo mkdir /Users/vagrant/Data/downloads || 0
+      sudo chmod 777 /Users/vagrant/Data/downloads || 0
+      exit 0
+    displayName: 'Create /Users/vagrant/Data/downloads'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
@@ -39,10 +36,19 @@ jobs:
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - powershell: |
-      ./vcpkg x-ci-clean --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads
+      $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload',
+        '--x-buildtrees-root=/Users/vagrant/Data/buildtrees',
+        '--x-install-root=/Users/vagrant/Data/installed',
+        '--x-packages-root=/Users/vagrant/Data/packages'
+      )
+
+      ./vcpkg x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt"
       mkdir $(System.ArtifactsDirectory)/xml-results
-      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,/Users/vagrant/Data/archives`,upload --x-buildtrees-root=/Users/vagrant/Data/buildtrees --x-install-root=/Users/vagrant/Data/installed --x-packages-root=/Users/vagrant/Data/packages --downloads-root=/Users/vagrant/Data/downloads
+      ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/azure-pipelines/osx/initialize-environment.sh b/scripts/azure-pipelines/osx/initialize-environment.sh
diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml
@@ -23,17 +23,25 @@ jobs:
         set VCPKG_DOWNLOADS=D:\downloads
         .\bootstrap-vcpkg.bat
   - powershell: |
-      .\vcpkg.exe x-ci-clean --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
+      $env:VCPKG_DOWNLOADS='D:\downloads'
+      $commonArgs = @(
+        '--binarycaching',
+        '--x-binarysource=clear;files,W:\,upload',
+        '--x-buildtrees-root=E:\buildtrees',
+        '--x-install-root=E:\install',
+        '--x-packages-root=E:\packages'
+      )
+
+      .\vcpkg.exe x-ci-clean @commonArgs
       $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt
       mkdir $(System.ArtifactsDirectory)\xml-results
       # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed.
       # Install them so the CI succeeds:
       if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) {
-          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
+          .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs
       }
 
-      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList --binarycaching --x-binarysource=clear`;files`,W:\`,upload --x-buildtrees-root=E:\buildtrees --x-install-root=E:\install --x-packages-root=E:\packages --downloads-root=D:\downloads
-      Set-Content -Path 'run_ci.cmd' -Value $command -Encoding ASCII
+      .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs
     displayName: '** Test Modified Ports **'
   - task: PowerShell@2
     displayName: 'Analyze results and prepare test logs'

diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
@@ -621,7 +621,7 @@ ignition-msgs5:x64-linux=fail
 ignition-msgs5:arm64-windows=fail
 ignition-msgs5:arm-uwp=fail
 ignition-msgs5:x64-uwp=fail
-ignition-msgs5:x64-osx=fail
+ignition-msgs5:x64-osx=skip
 imgui-sfml:x64-linux=ignore
 intel-ipsec:arm64-windows=fail
 intel-ipsec:arm-uwp=fail

diff --git a/toolsrc/src/vcpkg/commands.ciclean.cpp b/toolsrc/src/vcpkg/commands.ciclean.cpp
@@ -2,31 +2,39 @@
 
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/system.print.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/vcpkgcmdarguments.h>
 
 using namespace vcpkg;
 
-namespace vcpkg::Commands::CIClean
-{
-    void perform_and_exit(const VcpkgCmdArguments&, const VcpkgPaths& paths)
+namespace {
+    void clear_directory(Files::Filesystem& fs, const fs::path& target)
     {
-        auto& fs = paths.get_filesystem();
-        if (fs.is_directory(paths.buildtrees))
+        using vcpkg::System::print2;
+        if (fs.is_directory(target))
         {
-            fs.remove_all_inside(paths.buildtrees, VCPKG_LINE_INFO);
+            print2("Clearing contents of ", target.u8string(), "\n");
+            fs.remove_all_inside(target, VCPKG_LINE_INFO);
         }
-
-        if (fs.is_directory(paths.installed))
-        {
-            fs.remove_all_inside(paths.installed, VCPKG_LINE_INFO);
-        }
-
-        if (fs.is_directory(paths.packages))
+        else
         {
-            fs.remove_all_inside(paths.packages, VCPKG_LINE_INFO);
+            print2("Skipping clearing contents of ", target.u8string(), " because it was not a directory\n");
         }
+    }
+}
 
+namespace vcpkg::Commands::CIClean
+{
+    void perform_and_exit(const VcpkgCmdArguments&, const VcpkgPaths& paths)
+    {
+        using vcpkg::System::print2;
+        auto& fs = paths.get_filesystem();
+        print2("Starting vcpkg CI clean\n");
+        clear_directory(fs, paths.buildtrees);
+        clear_directory(fs, paths.installed);
+        clear_directory(fs, paths.packages);
+        print2("Completed vcpkg CI clean\n");
         Checks::exit_success(VCPKG_LINE_INFO);
     }
 }