Bug 1991427 - Add support for post-patch-actions to moz.yaml files r=tjr

This patch adds a new `post-patch-actions` section to be added in a
`moz.yaml` file.

This is useful when you need to run scripts that depend on
Mozilla-specific patches being applied first.

Differential Revision: https://phabricator.services.mozilla.com/D266665

Author: michaelvanstraten

python/mozbuild/mozbuild/vendor/docs/index.rst

@@ -64,3 +64,37 @@ In the presence of patches, two steps are needed:
 
 In the absence of patches, a single step is needed, and no extra argument is
 required.
+
+
+Vendoring Actions
+=================
+
+Vendoring actions in the ``moz.yaml`` file can be configured to run either before
+or after patches are applied using separate sections:
+
+* Actions in ``update-actions`` run **before** patches are applied
+* Actions in ``post-patch-actions`` run **after** patches are applied
+
+This separation is useful when you need to run scripts that depend on Mozilla-specific
+patches being applied first, such as:
+
+* Code generation scripts that need patched configuration files
+* Build system updates that depend on patched build definitions
+* Processing steps that require Mozilla-specific modifications to be in place
+
+Example:
+
+.. code-block:: yaml
+
+   # Actions that run before patches are applied
+   update-actions:
+     - action: run-script
+       script: '{yaml_dir}/pre_patch_script.sh'
+       cwd: '{yaml_dir}'
+
+   # Actions that run after patches are applied
+   post-patch-actions:
+     - action: run-script
+       script: '{yaml_dir}/post_patch_script.sh'
+       cwd: '{yaml_dir}'
+       args: ['{revision}']

python/mozbuild/mozbuild/vendor/docs/template.yaml

@@ -158,7 +158,7 @@ vendoring:
   # All three file/path parameters ("keep", "exclude", and "include") support
   # filenames, directory names, and globs/wildcards.
 
-  # Actions to take after updating. Applied in order.
+  # Actions to take after updating but before applying patches. Applied in order.
   # The action subfield is required. It must be one of:
   #   - copy-file
   #   - move-file
@@ -231,6 +231,15 @@ vendoring:
       script: '{cwd}/generate_sources.sh'
       cwd: '{yaml_dir}'
 
+  # Actions to take after patches have been applied. Applied in order.
+  # Uses the same action types as update-actions.
+  # optional
+  post-patch-actions:
+    - action: run-script
+      script: '{yaml_dir}/post_patch_script.py'
+      cwd: '{yaml_dir}'
+      args: ['{revision}']
+
 
 # Configuration for automatic updating system.
 # optional

python/mozbuild/mozbuild/vendor/moz_yaml.py

@@ -119,6 +119,39 @@ def load_moz_yaml(filename, verify=True, require_license_file=True):
 
 def _schema_1():
     """Returns Voluptuous Schema object."""
+
+    actions_schema = All(
+        VendoringActions(),
+        [
+            {
+                Required("action"): In(
+                    [
+                        "copy-file",
+                        "move-file",
+                        "move-dir",
+                        "replace-in-file",
+                        "replace-in-file-regex",
+                        "run-script",
+                        "run-command",
+                        "delete-path",
+                        "vcs-add-remove-files",
+                    ],
+                    msg="Invalid action specified in vendoring-actions",
+                ),
+                "from": All(str, Length(min=1)),
+                "to": All(str, Length(min=1)),
+                "pattern": All(str, Length(min=1)),
+                "with": All(str, Length(min=1)),
+                "file": All(str, Length(min=1)),
+                "script": All(str, Length(min=1)),
+                "command": All(str, Length(min=1)),
+                "args": All([All(str, Length(min=1))]),
+                "cwd": All(str, Length(min=1)),
+                "path": All(str, Length(min=1)),
+            }
+        ],
+    )
+
     return Schema(
         {
             Required("schema"): "1",
@@ -201,37 +234,8 @@ def _schema_1():
                 "individual-files-default-upstream": str,
                 "individual-files-default-destination": All(str, Length(min=1)),
                 "individual-files-list": Unique([str]),
-                "update-actions": All(
-                    UpdateActions(),
-                    [
-                        {
-                            Required("action"): In(
-                                [
-                                    "copy-file",
-                                    "move-file",
-                                    "move-dir",
-                                    "replace-in-file",
-                                    "replace-in-file-regex",
-                                    "run-script",
-                                    "run-command",
-                                    "delete-path",
-                                    "vcs-add-remove-files",
-                                ],
-                                msg="Invalid action specified in update-actions",
-                            ),
-                            "from": All(str, Length(min=1)),
-                            "to": All(str, Length(min=1)),
-                            "pattern": All(str, Length(min=1)),
-                            "with": All(str, Length(min=1)),
-                            "file": All(str, Length(min=1)),
-                            "script": All(str, Length(min=1)),
-                            "command": All(str, Length(min=1)),
-                            "args": All([All(str, Length(min=1))]),
-                            "cwd": All(str, Length(min=1)),
-                            "path": All(str, Length(min=1)),
-                        }
-                    ],
-                ),
+                "update-actions": actions_schema,
+                "post-patch-actions": actions_schema,
             },
         }
     )
@@ -438,8 +442,8 @@ def _schema_1_transform(manifest):
     return manifest
 
 
-class UpdateActions:
-    """Voluptuous validator which verifies the update actions(s) are valid."""
+class VendoringActions:
+    """Voluptuous validator which verifies the vendoring actions(s) are valid."""
 
     def __call__(self, values):
         for v in values:
@@ -497,7 +501,7 @@ def __call__(self, values):
         return values
 
     def __repr__(self):
-        return "UpdateActions"
+        return "VendoringActions"
 
 
 class UpdatebotTasks:

python/mozbuild/mozbuild/vendor/vendor_manifest.py

@@ -137,6 +137,7 @@ def vendor(
                 os.path.dirname(self.yaml_file),
                 self.manifest["vendoring"]["vendor-directory"],
             )
+            self.run_vendoring_actions(revision, "post-patch-actions")
             return
 
         # ==========================================================
@@ -272,7 +273,7 @@ def process_regular_or_individual(
             self.logInfo({}, "Skipping fetching upstream source.")
 
         self.logInfo({}, "Checking for update actions")
-        self.update_files(new_revision)
+        self.run_vendoring_actions(new_revision, "update-actions")
 
         if self.patch_mode == "check":
             self.import_local_patches(
@@ -698,14 +699,14 @@ def spurious_check(self, revision, ignore_modified):
             "Version '{rev}' has changed {num} files.",
         )
 
-    def update_files(self, revision):
-        if "update-actions" not in self.manifest["vendoring"]:
+    def run_vendoring_actions(self, revision, actions_type="update-actions"):
+        if actions_type not in self.manifest["vendoring"]:
             return
 
-        for update in self.manifest["vendoring"]["update-actions"]:
-            if update["action"] == "copy-file":
-                src = self.get_full_path(update["from"])
-                dst = self.get_full_path(update["to"])
+        for action in self.manifest["vendoring"][actions_type]:
+            if action["action"] == "copy-file":
+                src = self.get_full_path(action["from"])
+                dst = self.get_full_path(action["to"])
 
                 self.logInfo(
                     {"s": src, "d": dst}, "action: copy-file src: {s} dst: {d}"
@@ -715,24 +716,24 @@ def update_files(self, revision):
                     contents = f.read()
                 with open(dst, "w") as f:
                     f.write(contents)
-            elif update["action"] == "vcs-add-remove-files":
-                directory = self.get_full_path(update["path"])
+            elif action["action"] == "vcs-add-remove-files":
+                directory = self.get_full_path(action["path"])
 
                 self.logInfo({"d": directory}, "action: vcs-add-remove-files dir: {d}")
 
                 self.repository.add_remove_files(directory)
-            elif update["action"] == "move-file":
-                src = self.get_full_path(update["from"])
-                dst = self.get_full_path(update["to"])
+            elif action["action"] == "move-file":
+                src = self.get_full_path(action["from"])
+                dst = self.get_full_path(action["to"])
 
                 self.logInfo(
                     {"s": src, "d": dst}, "action: move-file src: {s} dst: {d}"
                 )
 
                 shutil.move(src, dst)
-            elif update["action"] == "move-dir":
-                src = self.get_full_path(update["from"])
-                dst = self.get_full_path(update["to"])
+            elif action["action"] == "move-dir":
+                src = self.get_full_path(action["from"])
+                dst = self.get_full_path(action["to"])
 
                 self.logInfo(
                     {"src": src, "dst": dst}, "action: move-dir src: {src} dst: {dst}"
@@ -761,32 +762,32 @@ def copy_tree(src, dst):
                 copy_tree(src, dst)
                 shutil.rmtree(src)
 
-            elif update["action"] in ["replace-in-file", "replace-in-file-regex"]:
-                file = self.get_full_path(update["file"])
+            elif action["action"] in ["replace-in-file", "replace-in-file-regex"]:
+                file = self.get_full_path(action["file"])
 
                 self.logInfo({"file": file}, "action: replace-in-file file: {file}")
 
-                replacement = update["with"].replace("{revision}", revision)
+                replacement = action["with"].replace("{revision}", revision)
                 _replace_in_file(
                     file,
-                    update["pattern"],
+                    action["pattern"],
                     replacement,
-                    regex=update["action"] == "replace-in-file-regex",
+                    regex=action["action"] == "replace-in-file-regex",
                 )
-            elif update["action"] == "delete-path":
-                path = self.get_full_path(update["path"])
+            elif action["action"] == "delete-path":
+                path = self.get_full_path(action["path"])
                 self.logInfo({"path": path}, "action: delete-path path: {path}")
                 mozfile.remove(path)
-            elif update["action"] in ["run-script", "run-command"]:
-                if update["action"] == "run-script":
-                    command = self.get_full_path(update["script"], support_cwd=True)
+            elif action["action"] in ["run-script", "run-command"]:
+                if action["action"] == "run-script":
+                    command = self.get_full_path(action["script"], support_cwd=True)
                 else:
-                    command = update["command"]
+                    command = action["command"]
 
-                run_dir = self.get_full_path(update["cwd"], support_cwd=True)
+                run_dir = self.get_full_path(action["cwd"], support_cwd=True)
 
                 args = []
-                for a in update.get("args", []):
+                for a in action.get("args", []):
                     if a == "{revision}":
                         args.append(revision)
                     elif any(
@@ -808,7 +809,7 @@ def copy_tree(src, dst):
                         "command": command,
                         "run_dir": run_dir,
                         "args": args,
-                        "type": update["action"],
+                        "type": action["action"],
                     },
                     "action: {type} command: {command} working dir: {run_dir} args: {args}",
                 )