[vcpkg] Miscellaneous internal improvements extracted from #15424

scripts/azure-pipelines/linux/azure-pipelines.yml

@@ -36,6 +36,8 @@ jobs:
       arguments: '-buildTests'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
+    env:
+      VCPKG_DEBUG: 1
   - task: PowerShell@2
     displayName: '*** Test Modified Ports and Prepare Test Logs ***'
     inputs:

scripts/azure-pipelines/osx/azure-pipelines.yml

@@ -33,6 +33,8 @@ jobs:
       arguments: '-buildTests'
   - bash: toolsrc/build.rel/vcpkg-test
     displayName: 'Run vcpkg tests'
+    env:
+      VCPKG_DEBUG: 1
   - task: PowerShell@2
     displayName: 'Run vcpkg end-to-end tests'
     inputs:

scripts/azure-pipelines/windows/azure-pipelines.yml

@@ -53,6 +53,7 @@ jobs:
         rmdir /s /q build.x86.debug > nul 2> nul
         cmake.exe -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DVCPKG_DEVELOPMENT_WARNINGS=ON -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=ON -B build.x86.debug -S toolsrc
         ninja.exe -C build.x86.debug
+        set VCPKG_DEBUG=1
         build.x86.debug\vcpkg-test.exe
         cmake -G "Visual Studio 16 2019" -A Win32 -T v140 -DBUILD_TESTING=OFF -DVCPKG_DEVELOPMENT_WARNINGS=OFF -DVCPKG_WARNINGS_AS_ERRORS=ON -DVCPKG_BUILD_FUZZING=OFF -B build.x86.vs2015 -S toolsrc
         cmake --build build.x86.vs2015

toolsrc/include/vcpkg-test/util.h

@@ -32,6 +32,12 @@ namespace Catch
                                           value.package_spec.triplet());
         }
     };
+
+    template<>
+    struct StringMaker<vcpkg::Triplet>
+    {
+        static const std::string& convert(const vcpkg::Triplet& triplet) { return triplet.canonical_name(); }
+    };
 }
 
 namespace vcpkg::Test

toolsrc/include/vcpkg/base/optional.h

@@ -290,32 +290,46 @@ namespace vcpkg
         using map_t = decltype(std::declval<F&>()(std::declval<const T&>()));
 
         template<class F, class U = map_t<F>>
-        U then(F f) const&
+        Optional<U> map(F f) const&
         {
             if (has_value())
             {
                 return f(this->m_base.value());
             }
-            else
+            return nullopt;
+        }
+
+        template<class F, class U = map_t<F>>
+        U then(F f) const&
+        {
+            if (has_value())
             {
-                return nullopt;
+                return f(this->m_base.value());
             }
+            return nullopt;
         }
 
         template<class F>
         using move_map_t = decltype(std::declval<F&>()(std::declval<T&&>()));
 
         template<class F, class U = move_map_t<F>>
-        U then(F f) &&
+        Optional<U> map(F f) &&
         {
             if (has_value())
             {
                 return f(std::move(this->m_base.value()));
             }
-            else
+            return nullopt;
+        }
+
+        template<class F, class U = move_map_t<F>>
+        U then(F f) &&
+        {
+            if (has_value())
             {
-                return nullopt;
+                return f(std::move(this->m_base.value()));
             }
+            return nullopt;
         }
 
         friend bool operator==(const Optional& lhs, const Optional& rhs)

toolsrc/include/vcpkg/dependencies.h

@@ -132,7 +132,7 @@ namespace vcpkg::Dependencies
         RequestType request_type;
 
         Optional<const BinaryParagraph&> core_paragraph() const;
-        std::vector<PackageSpec> dependencies(Triplet triplet) const;
+        std::vector<PackageSpec> dependencies() const;
 
     private:
         Optional<InstalledPackageView> m_installed_package;

toolsrc/include/vcpkg/remove.h

@@ -23,7 +23,6 @@ namespace vcpkg::Remove
 
     extern const CommandStructure COMMAND_STRUCTURE;
 
-    void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet);
     void remove_package(const VcpkgPaths& paths, const PackageSpec& spec, StatusParagraphs* status_db);
 
     struct RemoveCommand : Commands::TripletCommand

toolsrc/src/vcpkg-test/catch.cpp

@@ -2,10 +2,11 @@
 #include <catch2/catch.hpp>
 
 #include <vcpkg/base/system.debug.h>
+#include <vcpkg/base/system.h>
 
 int main(int argc, char** argv)
 {
-    vcpkg::Debug::g_debugging = true;
+    if (vcpkg::System::get_environment_variable("VCPKG_DEBUG").value_or("") == "1") vcpkg::Debug::g_debugging = true;
 
     return Catch::Session().run(argc, argv);
 }

toolsrc/src/vcpkg-test/manifests.cpp

@@ -691,13 +691,21 @@ TEST_CASE ("Serialize all the ports", "[manifests]")
         const auto manifest = dir / fs::u8path("vcpkg.json");
         if (fs.exists(control))
         {
+            INFO(fs::u8string(control));
             auto contents = fs.read_contents(control, VCPKG_LINE_INFO);
             auto pghs = Paragraphs::parse_paragraphs(contents, fs::u8string(control));
             REQUIRE(pghs);
 
-            scfs.push_back(std::move(*SourceControlFile::parse_control_file(
-                                          fs::u8string(control), std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
-                                          .value_or_exit(VCPKG_LINE_INFO)));
+            auto scf = SourceControlFile::parse_control_file(fs::u8string(control),
+                                                             std::move(pghs).value_or_exit(VCPKG_LINE_INFO));
+            if (!scf)
+            {
+                INFO(scf.error()->name);
+                INFO(scf.error()->error);
+                REQUIRE(scf);
+            }
+
+            scfs.push_back(std::move(*scf.value_or_exit(VCPKG_LINE_INFO)));
         }
         else if (fs.exists(manifest))
         {

toolsrc/src/vcpkg-test/optional.cpp

@@ -80,6 +80,33 @@ TEST_CASE ("value conversion", "[optional]")
     REQUIRE(o_v.get()->size() == 3);
 }
 
+TEST_CASE ("optional.map", "[optional]")
+{
+    using vcpkg::NullOpt;
+    using vcpkg::nullopt;
+    using vcpkg::Optional;
+
+    const Optional<std::unique_ptr<int>> move_only;
+
+    Optional<int*> m = move_only.map([](auto&& p) { return p.get(); });
+    Optional<Optional<int*>> n =
+        move_only.map([](auto&& p) -> Optional<int*> { return p ? Optional<int*>{p.get()} : nullopt; });
+    Optional<NullOpt> o = move_only.map([](auto&&) { return nullopt; });
+
+    Optional<int> five = 5;
+
+    struct MoveTest
+    {
+        int operator()(int&&) { return 1; }
+        int operator()(const int&) { return -1; }
+    } move_test;
+
+    Optional<int> dst = std::move(five).map(move_test);
+    REQUIRE(dst == 1);
+    Optional<int> dst2 = five.map(move_test);
+    REQUIRE(dst2 == -1);
+}
+
 TEST_CASE ("common_projection", "[optional]")
 {
     using vcpkg::Util::common_projection;

toolsrc/src/vcpkg/build.cpp

@@ -948,7 +948,8 @@ namespace vcpkg::Build
 
         const auto& abi_info = action.abi_info.value_or_exit(VCPKG_LINE_INFO);
         const auto& triplet_abi = paths.get_triplet_info(abi_info);
-        abi_tag_entries.emplace_back("triplet", triplet_abi);
+        abi_tag_entries.emplace_back("triplet", triplet.canonical_name());
+        abi_tag_entries.emplace_back("triplet_abi", triplet_abi);
         abi_entries_from_abi_info(abi_info, abi_tag_entries);
 
         // If there is an unusually large number of files in the port then

toolsrc/src/vcpkg/commands.ci.cpp

@@ -271,7 +271,7 @@ namespace vcpkg::Commands::CI
         std::vector<FullPackageSpec> unknown;
         std::map<PackageSpec, Build::BuildResult> known;
         std::map<PackageSpec, std::vector<std::string>> features;
-        std::unordered_map<std::string, SourceControlFileLocation> default_feature_provider;
+        Dependencies::ActionPlan plan;
         std::map<PackageSpec, std::string> abi_map;
     };
 
@@ -324,28 +324,20 @@ namespace vcpkg::Commands::CI
 
         auto timer = Chrono::ElapsedTimer::create_started();
 
-        Checks::check_exit(VCPKG_LINE_INFO,
-                           action_plan.already_installed.empty(),
-                           "Cannot use CI command with packages already installed.");
+        Checks::check_exit(VCPKG_LINE_INFO, action_plan.already_installed.empty());
+        Checks::check_exit(VCPKG_LINE_INFO, action_plan.remove_actions.empty());
 
         Build::compute_all_abis(paths, action_plan, var_provider, {});
 
+        auto precheck_results = binary_provider_precheck(paths, action_plan, binaryprovider);
         {
             vcpkg::System::BufferedPrint stdout_print;
-            auto precheck_results = binary_provider_precheck(paths, action_plan, binaryprovider);
 
             for (auto&& action : action_plan.install_actions)
             {
                 auto p = &action;
                 ret->abi_map.emplace(action.spec, action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi);
                 ret->features.emplace(action.spec, action.feature_list);
-                if (auto scfl = p->source_control_file_location.get())
-                {
-                    auto emp = ret->default_feature_provider.emplace(p->spec.name(), scfl->clone());
-                    emp.first->second.source_control_file->core_paragraph->default_features = p->feature_list;
-
-                    p->build_options = vcpkg::Build::backcompat_prohibiting_package_options;
-                }
 
                 auto precheck_result = precheck_results.at(&action);
                 bool b_will_build = false;
@@ -399,6 +391,30 @@ namespace vcpkg::Commands::CI
             }
         } // flush stdout_print
 
+        // This algorithm consumes the previous action plan to build and return a reduced one.
+        std::vector<InstallPlanAction>&& input_install_actions = std::move(action_plan.install_actions);
+        std::vector<InstallPlanAction*> rev_install_actions;
+        rev_install_actions.reserve(input_install_actions.size());
+        std::set<PackageSpec> to_keep;
+        for (auto it = input_install_actions.rbegin(); it != input_install_actions.rend(); ++it)
+        {
+            if (!Util::Sets::contains(ret->known, it->spec))
+            {
+                to_keep.insert(it->spec);
+            }
+
+            if (Util::Sets::contains(to_keep, it->spec))
+            {
+                rev_install_actions.push_back(&*it);
+                to_keep.insert(it->package_dependencies.begin(), it->package_dependencies.end());
+            }
+        }
+
+        for (auto it = rev_install_actions.rbegin(); it != rev_install_actions.rend(); ++it)
+        {
+            ret->plan.install_actions.push_back(std::move(**it));
+        }
+
         System::printf("Time to determine pass/fail: %s\n", timer.elapsed());
         return ret;
     }
@@ -480,10 +496,6 @@ namespace vcpkg::Commands::CI
                 return FullPackageSpec{spec, std::move(default_features)};
             });
 
-            auto split_specs = find_unknown_ports_for_ci(
-                paths, exclusions_set, provider, var_provider, all_default_full_specs, binaryprovider);
-            PortFileProvider::MapPortFileProvider new_default_provider(split_specs->default_feature_provider);
-
             Dependencies::CreateInstallPlanOptions serialize_options;
 
             struct RandomizerInstance : Graphs::Randomizer
@@ -503,8 +515,10 @@ namespace vcpkg::Commands::CI
                 serialize_options.randomizer = &randomizer_instance;
             }
 
-            auto action_plan = Dependencies::create_feature_install_plan(
-                new_default_provider, var_provider, split_specs->unknown, status_db, serialize_options);
+            auto split_specs = find_unknown_ports_for_ci(
+                paths, exclusions_set, provider, var_provider, all_default_full_specs, binaryprovider);
+
+            auto& action_plan = split_specs->plan;
 
             for (auto&& action : action_plan.install_actions)
             {

toolsrc/src/vcpkg/dependencies.cpp

@@ -500,7 +500,7 @@ namespace vcpkg::Dependencies
         return nullopt;
     }
 
-    std::vector<PackageSpec> ExportPlanAction::dependencies(Triplet) const
+    std::vector<PackageSpec> ExportPlanAction::dependencies() const
     {
         if (auto p_ip = m_installed_package.get())
             return p_ip->dependencies();
@@ -587,7 +587,7 @@ namespace vcpkg::Dependencies
 
             std::vector<PackageSpec> adjacency_list(const ExportPlanAction& plan) const override
             {
-                return plan.dependencies(plan.spec.triplet());
+                return plan.dependencies();
             }
 
             ExportPlanAction load_vertex_data(const PackageSpec& spec) const override

toolsrc/src/vcpkg/export.prefab.cpp

@@ -379,7 +379,7 @@ namespace vcpkg::Export::Prefab
         for (const auto& action : export_plan)
         {
             const std::string name = action.spec.name();
-            auto dependencies = action.dependencies(default_triplet);
+            auto dependencies = action.dependencies();
 
             const auto action_build_info = Build::read_build_info(utils, paths.build_info_file_path(action.spec));
             const bool is_empty_package = action_build_info.policies.is_enabled(Build::BuildPolicy::EMPTY_PACKAGE);

toolsrc/src/vcpkg/paragraphs.cpp

@@ -398,6 +398,16 @@ namespace vcpkg::Paragraphs
             bcf.features =
                 Util::fmap(*p, [&](auto&& raw_feature) -> BinaryParagraph { return BinaryParagraph(raw_feature); });
 
+            if (bcf.core_paragraph.spec != spec)
+            {
+                return Strings::concat("Mismatched spec in package at ",
+                                       fs::u8string(paths.package_dir(spec)),
+                                       ": expected ",
+                                       spec,
+                                       ", actual ",
+                                       bcf.core_paragraph.spec);
+            }
+
             return bcf;
         }
 

toolsrc/src/vcpkg/remove.cpp

@@ -214,7 +214,9 @@ namespace vcpkg::Remove
         &valid_arguments,
     };
 
-    void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, Triplet default_triplet)
+    void RemoveCommand::perform_and_exit(const VcpkgCmdArguments& args,
+                                         const VcpkgPaths& paths,
+                                         Triplet default_triplet) const
     {
         if (paths.manifest_mode_enabled())
         {
@@ -332,11 +334,4 @@ namespace vcpkg::Remove
 
         Checks::exit_success(VCPKG_LINE_INFO);
     }
-
-    void RemoveCommand::perform_and_exit(const VcpkgCmdArguments& args,
-                                         const VcpkgPaths& paths,
-                                         Triplet default_triplet) const
-    {
-        Remove::perform_and_exit(args, paths, default_triplet);
-    }
 }