[vcpkg] Add vcpkg export to E2E tests. Enable E2E tests on all platforms.

scripts/azure-pipelines/end-to-end-tests.ps1

@@ -115,7 +115,11 @@ Require-FileNotExists "$installRoot/$Triplet/include/rapidjson/rapidjson.h"
 Require-FileNotExists "$buildtreesRoot/rapidjson/src"
 Require-FileExists "$TestingRoot/packages.config"
 
-& $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
+if ($IsLinux -or $IsMacOS) {
+    mono $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
+} else {
+    & $(./vcpkg fetch nuget) restore $TestingRoot/packages.config -OutputDirectory "$NuGetRoot2" -Source "$NuGetRoot"
+}
 
 Remove-Item -Recurse -Force $NuGetRoot -ErrorAction SilentlyContinue
 mkdir $NuGetRoot
@@ -132,3 +136,15 @@ Require-FileExists "$buildtreesRoot/tinyxml/src"
 if ((Get-ChildItem $NuGetRoot -Filter '*.nupkg' | Measure-Object).Count -ne 1) {
     throw "In '$CurrentTest': did not create exactly 1 NuGet package"
 }
+
+# Test export
+$args = $commonArgs + @("export","rapidjson","tinyxml","--nuget","--nuget-id=vcpkg-export","--nuget-version=1.0.0","--output=vcpkg-export-output","--raw","--zip","--output-dir=$TestingRoot")
+$CurrentTest = "./vcpkg $($args -join ' ')"
+Write-Host $CurrentTest
+Require-FileNotExists "$TestingRoot/vcpkg-export-output"
+Require-FileNotExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"
+Require-FileNotExists "$TestingRoot/vcpkg-export-output.zip"
+./vcpkg @args
+Require-FileExists "$TestingRoot/vcpkg-export-output"
+Require-FileExists "$TestingRoot/vcpkg-export.1.0.0.nupkg"
+Require-FileExists "$TestingRoot/vcpkg-export-output.zip"

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -9,6 +9,12 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: /mnt/vcpkg-ci
+  - name: VCPKG_DOWNLOADS
+    value: /mnt/vcpkg-ci/downloads
+
   steps:
   - bash: df -h
     displayName: 'Report on Disk Space'
@@ -19,25 +25,23 @@ jobs:
     displayName: 'Create /home/agent'
     # Note: /mnt is the Azure machines' temporary disk.
   - bash: |
-      sudo mkdir /mnt/vcpkg-ci -m=777
-      sudo mkdir /mnt/vcpkg-ci/downloads -m=777
+      sudo mkdir ${{ variables.WORKING_ROOT }} -m=777
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777
       exit 0
-    displayName: 'Create /mnt/vcpkg-ci/downloads'
+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
       filePath: bootstrap-vcpkg.sh
       arguments: "-buildTests"
-    env:
-      VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-linux.xml'

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -10,6 +10,12 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: /Users/vagrant/Data
+  - name: VCPKG_DOWNLOADS
+    value: /Users/vagrant/Data/downloads
+
   steps:
   - bash: |
       df -h
@@ -21,27 +27,31 @@ jobs:
       brew list libtool || brew install libtool
       brew list bison || brew install bison
       brew list gfortran || brew cask install gfortran
+      brew list mono || brew install mono
     displayName: 'Install brew dependencies'
   - bash: |
-      sudo mkdir /Users/vagrant/Data/downloads || 0
-      sudo chmod 777 /Users/vagrant/Data/downloads || 0
+      sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} || 0
+      sudo chmod 777 ${{ variables.VCPKG_DOWNLOADS }} || 0
       exit 0
-    displayName: 'Create /Users/vagrant/Data/downloads'
+    displayName: 'Create ${{ variables.VCPKG_DOWNLOADS }}'
   - task: Bash@3
     displayName: 'Build vcpkg'
     inputs:
       filePath: bootstrap-vcpkg.sh
       arguments: '-buildTests'
-    env:
-      VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
+  - task: PowerShell@2
+    displayName: 'Run vcpkg end-to-end tests'
+    inputs:
+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'
+      arguments: '-Triplet x64-osx -WorkingRoot ${{ variables.WORKING_ROOT }}'
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot ${{ variables.WORKING_ROOT }}/archives -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-osx.xml'

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -9,6 +9,11 @@ jobs:
   workspace:
     clean: resources
   timeoutInMinutes: 1440 # 1 day
+  variables:
+  - name: WORKING_ROOT
+    value: D:\
+  - name: VCPKG_DOWNLOADS
+    value: D:\downloads
 
   steps:
   - task: PowerShell@2
@@ -21,35 +26,35 @@ jobs:
     inputs:
       filePath: 'scripts/azure-pipelines/windows/disk-space.ps1'
     # Note: D: is the Azure machines' temporary disk.
-  - task: CmdLine@2
+  - script: .\bootstrap-vcpkg.bat
     displayName: 'Build vcpkg'
-    inputs:
-      script: |
-        set VCPKG_DOWNLOADS=D:\downloads
-        .\bootstrap-vcpkg.bat
   - task: CmdLine@2
     displayName: "Build vcpkg with CMake and Run Tests"
     condition: eq('${{ parameters.triplet }}', 'x86-windows')
     inputs:
       script: |
         :: TRANSITION, get these tools on the VMs next time we roll them
-        set VCPKG_DOWNLOADS=D:\downloads
         .\vcpkg.exe fetch cmake
         .\vcpkg.exe fetch ninja
-        set PATH=D:\downloads\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;D:\downloads\tools\ninja-1.10.0-windows;%PATH%
+        set PATH=${{ variables.VCPKG_DOWNLOADS }}\tools\cmake-3.17.2-windows\cmake-3.17.2-win32-x86\bin;${{ variables.VCPKG_DOWNLOADS }}\tools\ninja-1.10.0-windows;%PATH%
         call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x86 -host_arch=x86
         rmdir /s /q build.x86.debug > nul 2> nul
         cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=ON -B build.x86.debug -S toolsrc
         ninja.exe -C build.x86.debug
         build.x86.debug\vcpkg-test.exe
-        powershell.exe -NoProfile -ExecutionPolicy Bypass "scripts\azure-pipelines\end-to-end-tests.ps1 -WorkingRoot \"%cd%\testing\" -triplet x86-windows"
       failOnStderr: true
+  - task: PowerShell@2
+    displayName: 'Run vcpkg end-to-end tests'
+    condition: eq('${{ parameters.triplet }}', 'x86-windows')
+    inputs:
+      filePath: 'scripts/azure-pipelines/end-to-end-tests.ps1'
+      arguments: '-Triplet ${{ parameters.triplet }} -WorkingRoot ${{ variables.WORKING_ROOT }}'
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:
       failOnStderr: true
       filePath: 'scripts/azure-pipelines/test-modified-ports.ps1'
-      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
+      arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot ${{ variables.WORKING_ROOT }} -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)'
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/${{ parameters.triplet }}.xml'

toolsrc/src/vcpkg/base/files.cpp

@@ -904,7 +904,7 @@ namespace vcpkg::Files
                 return res;
             }
 
-            System::printf("Waiting to take filesystem lock on %s...\n", path.u8string());
+            Debug::print("Waiting to take filesystem lock on ", path.u8string(), "...\n");
             auto wait = std::chrono::milliseconds(100);
             // waits, at most, a second and a half.
             while (wait < std::chrono::milliseconds(1000))

toolsrc/src/vcpkg/export.cpp

@@ -160,13 +160,19 @@ namespace vcpkg::Export
         fs.write_contents(nuspec_file_path, nuspec_file_content, VCPKG_LINE_INFO);
 
         // -NoDefaultExcludes is needed for ".vcpkg-root"
-        const auto cmd_line = Strings::format(R"("%s" pack -OutputDirectory "%s" "%s" -NoDefaultExcludes)",
-                                              nuget_exe.u8string(),
-                                              output_dir.u8string(),
-                                              nuspec_file_path.u8string());
+        System::CmdLineBuilder cmd;
+#ifndef _WIN32
+        cmd.path_arg(paths.get_tool_exe(Tools::MONO));
+#endif
+        cmd.path_arg(nuget_exe)
+            .string_arg("pack")
+            .path_arg(nuspec_file_path)
+            .string_arg("-OutputDirectory")
+            .path_arg(output_dir)
+            .string_arg("-NoDefaultExcludes");
 
         const int exit_code =
-            System::cmd_execute_and_capture_output(cmd_line, System::get_clean_environment()).exit_code;
+            System::cmd_execute_and_capture_output(cmd.extract(), System::get_clean_environment()).exit_code;
         Checks::check_exit(VCPKG_LINE_INFO, exit_code == 0, "Error: NuGet package creation failed");
 
         const fs::path output_path = output_dir / (nuget_id + "." + nuget_version + ".nupkg");
@@ -272,6 +278,7 @@ namespace vcpkg::Export
         bool all_installed = false;
 
         Optional<std::string> maybe_output;
+        fs::path output_dir;
 
         Optional<std::string> maybe_nuget_id;
         Optional<std::string> maybe_nuget_version;
@@ -283,6 +290,7 @@ namespace vcpkg::Export
     };
 
     static constexpr StringLiteral OPTION_OUTPUT = "--output";
+    static constexpr StringLiteral OPTION_OUTPUT_DIR = "--output-dir";
     static constexpr StringLiteral OPTION_DRY_RUN = "--dry-run";
     static constexpr StringLiteral OPTION_RAW = "--raw";
     static constexpr StringLiteral OPTION_NUGET = "--nuget";
@@ -324,8 +332,9 @@ namespace vcpkg::Export
         {OPTION_ALL_INSTALLED, "Export all installed packages"},
     }};
 
-    static constexpr std::array<CommandSetting, 15> EXPORT_SETTINGS = {{
+    static constexpr std::array<CommandSetting, 16> EXPORT_SETTINGS = {{
         {OPTION_OUTPUT, "Specify the output name (used to construct filename)"},
+        {OPTION_OUTPUT_DIR, "Specify the output directory for produced artifacts"},
         {OPTION_NUGET_ID, "Specify the id for the exported NuGet package (overrides --output)"},
         {OPTION_NUGET_VERSION, "Specify the version for the exported NuGet package"},
         {OPTION_IFW_REPOSITORY_URL, "Specify the remote repository URL for the online installer"},
@@ -352,7 +361,8 @@ namespace vcpkg::Export
         nullptr,
     };
 
-    static ExportArguments handle_export_command_arguments(const VcpkgCmdArguments& args,
+    static ExportArguments handle_export_command_arguments(const VcpkgPaths& paths,
+                                                           const VcpkgCmdArguments& args,
                                                            Triplet default_triplet,
                                                            const StatusParagraphs& status_db)
     {
@@ -371,6 +381,15 @@ namespace vcpkg::Export
         ret.prefab_options.enable_maven = options.switches.find(OPTION_PREFAB_ENABLE_MAVEN) != options.switches.cend();
         ret.prefab_options.enable_debug = options.switches.find(OPTION_PREFAB_ENABLE_DEBUG) != options.switches.cend();
         ret.maybe_output = maybe_lookup(options.settings, OPTION_OUTPUT);
+        auto maybe_output_dir = maybe_lookup(options.settings, OPTION_OUTPUT_DIR);
+        if (auto output_dir = maybe_output_dir.get())
+        {
+            ret.output_dir = Files::combine(paths.original_cwd, fs::u8path(*output_dir));
+        }
+        else
+        {
+            ret.output_dir = paths.root;
+        }
         ret.all_installed = options.switches.find(OPTION_ALL_INSTALLED) != options.switches.end();
 
         if (ret.all_installed)
@@ -489,8 +508,7 @@ namespace vcpkg::Export
                                         const VcpkgPaths& paths)
     {
         Files::Filesystem& fs = paths.get_filesystem();
-        const fs::path export_to_path = paths.root;
-        const fs::path raw_exported_dir_path = export_to_path / export_id;
+        const fs::path raw_exported_dir_path = opts.output_dir / export_id;
         fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
 
         // TODO: error handling
@@ -548,7 +566,7 @@ namespace vcpkg::Export
             const std::string nuget_id = opts.maybe_nuget_id.value_or(raw_exported_dir_path.filename().string());
             const std::string nuget_version = opts.maybe_nuget_version.value_or("1.0.0");
             const fs::path output_path =
-                do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, export_to_path);
+                do_nuget_export(paths, nuget_id, nuget_version, raw_exported_dir_path, opts.output_dir);
             System::print2(System::Color::success, "NuGet package exported at: ", output_path.u8string(), "\n");
 
             System::printf(R"(
@@ -564,7 +582,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
         {
             System::print2("Creating zip archive...\n");
             const fs::path output_path =
-                do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::ZIP);
+                do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::ZIP);
             System::print2(System::Color::success, "Zip archive exported at: ", output_path.u8string(), "\n");
             print_next_step_info("[...]");
         }
@@ -573,7 +591,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
         {
             System::print2("Creating 7zip archive...\n");
             const fs::path output_path =
-                do_archive_export(paths, raw_exported_dir_path, export_to_path, ArchiveFormatC::SEVEN_ZIP);
+                do_archive_export(paths, raw_exported_dir_path, opts.output_dir, ArchiveFormatC::SEVEN_ZIP);
             System::print2(System::Color::success, "7zip archive exported at: ", output_path.u8string(), "\n");
             print_next_step_info("[...]");
         }
@@ -594,7 +612,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
                 "may use export in classic mode by running vcpkg outside of a manifest-based project.");
         }
         const StatusParagraphs status_db = database_load_check(paths);
-        const auto opts = handle_export_command_arguments(args, default_triplet, status_db);
+        const auto opts = handle_export_command_arguments(paths, args, default_triplet, status_db);
         for (auto&& spec : opts.specs)
             Input::check_triplet(spec.triplet(), paths);
 

toolsrc/src/vcpkg/tools.cpp

@@ -83,8 +83,12 @@ namespace vcpkg
         const bool has_tool_entry = std::regex_search(XML.cbegin(), XML.cend(), match_tool_entry, tool_regex);
         if (!has_tool_entry)
         {
-            return Strings::format(
-                "Could not find entry for tool %s in %s for os=%s", tool, XML_PATH.u8string(), OS_STRING);
+            return Strings::format("Could not automatically acquire %s because there is no entry in %s for os=%s. You "
+                                   "may be able to install %s via your system package manager.",
+                                   tool,
+                                   XML_PATH.u8string(),
+                                   OS_STRING,
+                                   tool);
         }
 
         const std::string tool_data =