Index V2 consumption

.github/actions/spelling/expect.txt

@@ -361,7 +361,8 @@
 PSHOST
 psobject
 ptstr
-publickey
+publickey
+PVD
 pvk
 pvm
 pwabuilder
@@ -410,8 +411,8 @@
 servercertificate
 setmetadatabymanifestid
 SETTINGCHANGE
 SETTINGMAPPING
 sfs
 sfsclient
 SHCONTF
 SHGDN
@@ -433,7 +434,7 @@
 srs
 startswith
 STARTUPINFOW
 STDMETHODCALLTYPE
 storeapps
 storeorigin
 STRRET

azure-pipelines.yml

@@ -243,6 +243,13 @@ jobs:
       isPackaged: true
       filter: "TestCategory!=InProcess&TestCategory!=OutOfProcess"
 
+  - template: templates/e2e-test.template.yml
+    parameters:
+      title: "E2E Tests Packaged (Index V2)"
+      isPackaged: true
+      filter: "TestCategory!=InProcess&TestCategory!=OutOfProcess"
+      experimentalFeatures: indexV2
+
   - template: templates/e2e-test.template.yml
     parameters:
       title: "Microsoft.Management.Deployment E2E Tests (In-process)"

doc/Settings.md

@@ -345,3 +345,20 @@ You can enable the feature as shown below.
        "configureExport": true
    },
 ```
+
+### indexV2
+
+This feature enables the `winget` source to retrieve the V2 index, which is significantly smaller.
+Regardless of the state of this feature, if the index on the machine contains a V2 index, it will be used.
+If there is a bug with the V2 index stopping the `winget` CLI from working, disable the feature in your settings file and run this command:
+```
+> winget uninstall -s msstore Microsoft.Winget.Source_8wekyb3d8bbwe
+```
+
+You can enable the feature as shown below.
+
+```json
+   "experimentalFeatures": {
+       "indexV2": true
+   },
+```

schemas/JSON/settings/settings.schema.0.2.json

@@ -290,6 +290,11 @@
           "description": "Enable support for the configure export command",
           "type": "boolean",
           "default": false
+        },
+        "indexV2": {
+          "description": "Enable retrieving the V2 index for the winget source",
+          "type": "boolean",
+          "default": false
         }
       }
     }

src/AppInstallerCLIE2ETests/Constants.cs

@@ -29,6 +29,7 @@ public class Constants
         public const string PackageCertificatePathParameter = "PackageCertificatePath";
         public const string PowerShellModulePathParameter = "PowerShellModulePath";
         public const string SkipTestSourceParameter = "SkipTestSource";
+        public const string ForcedExperimentalFeaturesParameter = "ForcedExperimentalFeatures";
 
         // Test Sources
         public const string DefaultWingetSourceName = @"winget";

src/AppInstallerCLIE2ETests/FeaturesCommand.cs

@@ -1,4 +1,4 @@
-// -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // <copyright file="FeaturesCommand.cs" company="Microsoft Corporation">
 //     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 // </copyright>
@@ -41,7 +41,14 @@ public void DisplayFeatures()
             var result = TestCommon.RunAICLICommand("features", string.Empty);
             Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode);
             Assert.True(result.StdOut.Contains("Direct MSI Installation"));
-            Assert.False(result.StdOut.Contains("Enabled"));
+            if (WinGetSettingsHelper.ForcedExperimentalFeatures.Length > 0 && WinGetSettingsHelper.ForcedExperimentalFeatures[0] != "none")
+            {
+                Assert.True(result.StdOut.Contains("Enabled"));
+            }
+            else
+            {
+                Assert.False(result.StdOut.Contains("Enabled"));
+            }
         }
 
         /// <summary>

src/AppInstallerCLIE2ETests/Helpers/TestCommon.cs

@@ -667,6 +667,9 @@ public static void ModifyPortableARPEntryValue(string productCode, string name,
         /// <param name="useGroupPolicyForTestSource">Use group policy.</param>
         public static void SetupTestSource(bool useGroupPolicyForTestSource = false)
         {
+            // Remove the test source so that its package is also removed.
+            RunAICLICommand("source remove", Constants.TestSourceName);
+
             RunAICLICommand("source reset", "--force");
             RunAICLICommand("source remove", Constants.DefaultWingetSourceName);
             RunAICLICommand("source remove", Constants.DefaultMSStoreSourceName);

src/AppInstallerCLIE2ETests/Helpers/TestSetup.cs

@@ -1,4 +1,4 @@
-// -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // <copyright file="TestSetup.cs" company="Microsoft Corporation">
 //     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 // </copyright>
@@ -43,6 +43,8 @@ private TestSetup()
             this.ExeInstallerPath = this.InitializeFileParam(Constants.ExeInstallerPathParameter);
             this.MsiInstallerPath = this.InitializeFileParam(Constants.MsiInstallerPathParameter);
             this.MsixInstallerPath = this.InitializeFileParam(Constants.MsixInstallerPathParameter);
+
+            this.ForcedExperimentalFeatures = this.InitializeStringArrayParam(Constants.ForcedExperimentalFeaturesParameter);
         }
 
         /// <summary>
@@ -137,6 +139,11 @@ public string SettingsJsonFilePath
             }
         }
 
+        /// <summary>
+        /// Gets the experimental features that should be forcibly enabled.
+        /// </summary>
+        public string[] ForcedExperimentalFeatures { get; }
+
         /// <summary>
         /// Gets a value indicating whether is the default parameters.
         /// </summary>
@@ -162,6 +169,16 @@ private string InitializeStringParam(string paramName, string defaultValue = nul
             return TestContext.Parameters.Get(paramName);
         }
 
+        private string[] InitializeStringArrayParam(string paramName, string[] defaultValue = null)
+        {
+            if (this.IsDefault || !TestContext.Parameters.Exists(paramName))
+            {
+                return defaultValue;
+            }
+
+            return TestContext.Parameters.Get(paramName).Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        }
+
         private string InitializeFileParam(string paramName, string defaultValue = null)
         {
             if (!TestContext.Parameters.Exists(paramName))

src/AppInstallerCLIE2ETests/Helpers/WinGetSettingsHelper.cs

@@ -1,4 +1,4 @@
-// -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // <copyright file="WinGetSettingsHelper.cs" company="Microsoft Corporation">
 //     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 // </copyright>
@@ -16,6 +16,11 @@ namespace AppInstallerCLIE2ETests.Helpers
     /// </summary>
     internal static class WinGetSettingsHelper
     {
+        /// <summary>
+        /// Gets or sets the experimental features that should be forcibly enabled.
+        /// </summary>
+        public static string[] ForcedExperimentalFeatures { get; set; }
+
         /// <summary>
         /// Gets the user settings path by calling winget settings export.
         /// </summary>
@@ -33,19 +38,19 @@ public static string GetUserSettingsPath()
         /// </summary>
         public static void InitializeWingetSettings()
         {
+            Hashtable experimentalFeatures = new Hashtable();
+
+            var forcedExperimentalFeatures = ForcedExperimentalFeatures;
+            foreach (var feature in forcedExperimentalFeatures)
+            {
+                experimentalFeatures[feature] = true;
+            }
+
             var settingsJson = new Hashtable()
             {
                 {
                     "experimentalFeatures",
-                    new Hashtable()
-                    {
-                        { "experimentalArg", false },
-                        { "experimentalCmd", false },
-                        { "directMSI", false },
-                        { "windowsFeature", false },
-                        { "resume", false },
-                        { "reboot", false },
-                    }
+                    experimentalFeatures
                 },
                 {
                     "debugging",
@@ -67,27 +72,9 @@ public static void InitializeWingetSettings()
             // when running in unpackaged context
             TestCommon.RunAICLICommand(string.Empty, "-v");
 
-            SetWingetSettings(settingsJson);
-        }
-
-        /// <summary>
-        /// Converts a hashtable to json and writes to the settings file.
-        /// </summary>
-        /// <param name="settingsJson">Settings to set.</param>
-        public static void SetWingetSettings(Hashtable settingsJson)
-        {
             SetWingetSettings(JsonConvert.SerializeObject(settingsJson, Formatting.Indented));
         }
 
-        /// <summary>
-        /// Writes string to settings file.
-        /// </summary>
-        /// <param name="settings">Settings as string.</param>
-        public static void SetWingetSettings(string settings)
-        {
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settings);
-        }
-
         /// <summary>
         /// Configure experimental features.
         /// </summary>
@@ -96,10 +83,8 @@ public static void SetWingetSettings(string settings)
         public static void ConfigureFeature(string featureName, bool status)
         {
             JObject settingsJson = GetJsonSettingsObject("experimentalFeatures");
-            var experimentalFeatures = settingsJson["experimentalFeatures"];
-            experimentalFeatures[featureName] = status;
-
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            ConfigureFeature(settingsJson, featureName, status);
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -113,7 +98,7 @@ public static void ConfigureInstallBehavior(string settingName, string value)
             var installBehavior = settingsJson["installBehavior"];
             installBehavior[settingName] = value;
 
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -134,7 +119,7 @@ public static void ConfigureInstallBehaviorPreferences(string settingName, strin
             var preferences = installBehavior["preferences"];
             preferences[settingName] = value;
 
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -155,7 +140,7 @@ public static void ConfigureInstallBehaviorPreferences(string settingName, strin
             var preferences = installBehavior["preferences"];
             preferences[settingName] = new JArray(value);
 
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -176,7 +161,7 @@ public static void ConfigureInstallBehaviorRequirements(string settingName, stri
             var requirements = installBehavior["requirements"];
             requirements[settingName] = value;
 
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -197,7 +182,7 @@ public static void ConfigureInstallBehaviorRequirements(string settingName, stri
             var requirements = installBehavior["requirements"];
             requirements[settingName] = new JArray(value);
 
-            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString());
+            SetWingetSettings(settingsJson);
         }
 
         /// <summary>
@@ -206,12 +191,28 @@ public static void ConfigureInstallBehaviorRequirements(string settingName, stri
         /// <param name="status">Initialized feature value.</param>
         public static void InitializeAllFeatures(bool status)
         {
-            ConfigureFeature("experimentalArg", status);
-            ConfigureFeature("experimentalCmd", status);
-            ConfigureFeature("directMSI", status);
-            ConfigureFeature("windowsFeature", status);
-            ConfigureFeature("resume", status);
-            ConfigureFeature("reboot", status);
+            JObject settingsJson = GetJsonSettingsObject("experimentalFeatures");
+
+            ConfigureFeature(settingsJson, "experimentalArg", status);
+            ConfigureFeature(settingsJson, "experimentalCmd", status);
+            ConfigureFeature(settingsJson, "directMSI", status);
+            ConfigureFeature(settingsJson, "windowsFeature", status);
+            ConfigureFeature(settingsJson, "resume", status);
+            ConfigureFeature(settingsJson, "reboot", status);
+
+            SetWingetSettings(settingsJson);
+        }
+
+        /// <summary>
+        /// Configure experimental features.
+        /// </summary>
+        /// <param name="settingsJson">The settings JSON object to modify.</param>
+        /// <param name="featureName">Feature name.</param>
+        /// <param name="status">Status.</param>
+        private static void ConfigureFeature(JObject settingsJson, string featureName, bool status)
+        {
+            var experimentalFeatures = settingsJson["experimentalFeatures"];
+            experimentalFeatures[featureName] = status;
         }
 
         private static JObject GetJsonSettingsObject(string objectName)
@@ -225,5 +226,29 @@ private static JObject GetJsonSettingsObject(string objectName)
 
             return settingsJson;
         }
+
+        /// <summary>
+        /// Converts a JObject to a string and writes to the settings file.
+        /// </summary>
+        /// <param name="settingsJson">Settings to set.</param>
+        private static void SetWingetSettings(JObject settingsJson)
+        {
+            var forcedExperimentalFeatures = ForcedExperimentalFeatures;
+            foreach (var feature in forcedExperimentalFeatures)
+            {
+                ConfigureFeature(settingsJson, feature, true);
+            }
+
+            SetWingetSettings(settingsJson.ToString());
+        }
+
+        /// <summary>
+        /// Writes string to settings file.
+        /// </summary>
+        /// <param name="settings">Settings as string.</param>
+        private static void SetWingetSettings(string settings)
+        {
+            File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settings);
+        }
     }
 }

src/AppInstallerCLIE2ETests/README.md

@@ -1,4 +1,4 @@
-# How to Run End-To-End Tests for Windows Package Manager Client
+# How to Run End-To-End Tests for Windows Package Manager Client
 
 Most of the tests require having the local test source added into winget. The local test source must be hosted in a localhost web server.
 
@@ -67,7 +67,8 @@ The localhost web server needs to be running for the duration of the tests. The
 | **CertPassword** | Mandatory | HTTPS Developer Certificate Password |
 | **Port** | Optional | Port number [Default Port Number: 5001] |
 | **OutCertFile** | Optional | The exported certificate used |
-| **LocalSourceJson** | Optional | The local source definition. If set generates the source. |
+| **LocalSourceJson** | Optional | The local source definition. If set generates the source. |
+| **ForcedExperimentalFeatures** | Optional | Experimental features that should be forcibly enabled always. |
 
 ### How to create and trust an ASP.NET Core HTTPS Development Certificate
 Windows Package Manager Client (WinGet.exe) requires new sources added to the WinGet repositories be securely accessed through HTTPS. Therefore, in order to verify the LocalhostWebServer, you will need to create a self-signed development certificate to verify the localhost address. 

src/AppInstallerCLIE2ETests/SetUpFixture.cs

@@ -1,4 +1,4 @@
-// -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 // <copyright file="SetUpFixture.cs" company="Microsoft Corporation">
 //     Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 // </copyright>
@@ -61,6 +61,7 @@ public void Setup()
                 TestIndex.GenerateE2ESource();
             }
 
+            WinGetSettingsHelper.ForcedExperimentalFeatures = testParams.ForcedExperimentalFeatures;
             WinGetSettingsHelper.InitializeWingetSettings();
         }