Bug 1882413 - Part 3: Modify powershell set default to more accurately mirror the Win32 API set default implementation. r=nalexander,mhughes

This modifies the implementation to delete the registry keys via `DeleteSubKey` instead of `DeleteSubKeyTree`. This no longer throws when the DENY permission is set on a registry key as is the case for file association keys. This does throw when we are unable to delete a registry key as has been observed in newer versions of windows. Similar to the Win32 API implementation, this now halts execution and returns an error to the invoking process.

Differential Revision: https://phabricator.services.mozilla.com/D203069

Author: PrototypeNM1

browser/components/shell/WindowsUserChoice.cpp

@@ -284,19 +284,17 @@ UniquePtr<wchar_t[]> GetAssociationKeyPath(const wchar_t* aExt) {
   return keyPath;
 }
 
-nsresult AppendAssociationKeyPath(const wchar_t* aExt, nsString& output) {
+void AppendAssociationKeyPath(const wchar_t* aExt, nsAString& aOutput) {
   if (aExt[0] == L'.') {
-    output.AppendLiteral(
+    aOutput.AppendLiteral(
         u"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\FileExts\\");
   } else {
-    output.AppendLiteral(
+    aOutput.AppendLiteral(
         u"SOFTWARE\\Microsoft\\Windows\\Shell\\Associations\\UrlAssociations"
         u"\\");
   }
 
-  output.Append(aExt);
-
-  return NS_OK;
+  aOutput.Append(aExt);
 }
 
 UniquePtr<wchar_t[]> GenerateUserChoiceHash(const wchar_t* aExt,

browser/components/shell/WindowsUserChoice.h

@@ -65,9 +65,10 @@ mozilla::UniquePtr<wchar_t[]> GetAssociationKeyPath(const wchar_t* aExt);
  * Appends the registry path for the given association, file extension or
  * protocol to the parameter string.
  *
- * @return The path, or nullptr on failure.
+ * @param aExt      File extension or protocol association to return path to.
+ * @param aOutput   String to append registry path to.
  */
-nsresult AppendAssociationKeyPath(const wchar_t* aExt, nsString& output);
+void AppendAssociationKeyPath(const wchar_t* aExt, nsAString& aOutput);
 
 /*
  * Get the current user's SID

toolkit/mozapps/defaultagent/SetDefaultBrowser.cpp

@@ -512,7 +512,7 @@ static UINT GetSystemSleepIntervalInMilliseconds(UINT defaultValue) {
 }
 
 /*
- * MSIX implementation o SetDefaultExtensionHandlersUserChoice.
+ * MSIX implementation for SetDefaultExtensionHandlersUserChoice.
  *
  * Due to the fact that MSIX builds run in a virtual, walled off environment,
  * calling into the Win32 registry APIs doesn't work to set registry keys.
@@ -530,7 +530,7 @@ static UINT GetSystemSleepIntervalInMilliseconds(UINT defaultValue) {
  * Powershell, to make it as quick as possible.
  *
  */
-nsresult SetDefaultExtensionHandlersUserChoiceImplMsix(
+static nsresult SetDefaultExtensionHandlersUserChoiceImplMsix(
     const wchar_t* aAumi, const wchar_t* const aSid,
     const nsTArray<nsString>& aFileExtensions) {
   mozilla::UniquePtr<wchar_t[]> exePath;
@@ -544,24 +544,20 @@ nsresult SetDefaultExtensionHandlersUserChoiceImplMsix(
   // extension, done below.
   nsString startScript(
       uR"(
+# Force exceptions to stop execution
+$ErrorActionPreference = 'Stop'
+
 function Set-DefaultHandlerRegistry($Path, $ProgID, $Hash) {
   $Path = "$Path\UserChoice"
   $CurrentUser = [Microsoft.Win32.Registry]::CurrentUser
-  $ReadWriteSubTreePerm = [Microsoft.Win32.RegistryKeyPermissionCheck]::ReadWriteSubTree
-  try {
-    $CurrentUser.DeleteSubKeyTree($Path, $false)
-    $key = $CurrentUser.CreateSubKey($Path, $ReadWriteSubTreePerm)
-  } catch {
-    $key = $CurrentUser.OpenSubKey($Path, $ReadWriteSubTreePerm, [System.Security.AccessControl.RegistryRights]::ChangePermissions)
-    $acl = $key.GetAccessControl()
-    $CurrentName = [Security.Principal.WindowsIdentity]::GetCurrent().Name
-    $rule = New-Object System.Security.AccessControl.RegistryAccessRule($CurrentName, 'SetValue', 'Deny')
-    $acl.RemoveAccessRule($rule)
-    $key.SetAccessControl($acl)
-    $key.Close()
-
-    $key = $CurrentUser.OpenSubKey($Path, $ReadWriteSubTreePerm)
-  }
+
+  # DeleteSubKey throws if we don't have sufficient permissions to delete key,
+  # signaling failure to launching process.
+  #
+  # Note: DeleteSubKeyTree fails when DENY permissions are set on key, whereas
+  # DeleteSubKey succeeds.
+  $CurrentUser.DeleteSubKey($Path, $false)
+  $key = $CurrentUser.CreateSubKey($Path)
 
   $StringType = [Microsoft.Win32.RegistryValueKind]::String
   $key.SetValue('ProgID', $ProgID, $StringType)
@@ -620,28 +616,20 @@ function Set-DefaultHandlerRegistry($Path, $ProgID, $Hash) {
     for (size_t i = 0; i + 1 < aFileExtensions.Length(); i += 2) {
       const wchar_t* fileExtension = aFileExtensions[i].get();
 
-      // Append a line to the script buffer in the form:
-      // Set-DefaultHandlerRegistry $RegistryKeyPath $ProgID $UserChoiceHash
-
-      // Use Append to minimize string allocation and processing
+      nsAutoString keyPath;
+      AppendAssociationKeyPath(fileExtension, keyPath);
 
-      scriptBuffer.AppendLiteral(u"Set-DefaultHandlerRegistry ");
-      rv = AppendAssociationKeyPath(fileExtension, scriptBuffer);
-      NS_ENSURE_SUCCESS(rv, rv);
-
-      scriptBuffer.AppendLiteral(u" ");
-
-      scriptBuffer.Append(progIDs[i / 2].get());
-      scriptBuffer.AppendLiteral(u" ");
-
-      auto hash = GenerateUserChoiceHash(fileExtension, aSid,
-                                         progIDs[i / 2].get(), hashTimestamp);
-      if (!hash) {
+      auto hashWchar = GenerateUserChoiceHash(
+          fileExtension, aSid, progIDs[i / 2].get(), hashTimestamp);
+      if (!hashWchar) {
         return NS_ERROR_FAILURE;
       }
+      auto hash = nsDependentString(hashWchar.get());
 
-      scriptBuffer.Append(hash.get());
-      scriptBuffer.AppendLiteral(u"\n");
+      // Append a line to the script buffer in the form:
+      // Set-DefaultHandlerRegistry $RegistryKeyPath $ProgID $UserChoiceHash
+      scriptBuffer += u"Set-DefaultHandlerRegistry "_ns + keyPath + u" "_ns +
+                      progIDs[i / 2] + u" "_ns + hash + u"\n"_ns;
     }
 
     // The hash changes at the end of each minute, so check that the hash should