feat: Support glob paths in file plugin update extra (meltano#6596)

* Support glob paths in file plugin `update` extra * Add upgrade files glob path test * Update docs * Update tests/meltano/cli/test_upgrade.py Co-authored-by: Edgar R. M. <edgarrm358@gmail.com> * Fix missing closing `p` tag * Make linter happy Co-authored-by: Edgar R. M. <edgarrm358@gmail.com>
BuzzCutNorman · Aug 16, 2022 · 004ce90 · 004ce90
1 parent ecac36b
commit 004ce90
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 8 deletions.
diff --git a/docs/src/_concepts/plugins.md b/docs/src/_concepts/plugins.md
@@ -721,9 +721,9 @@ The file bundle itself will not be added to your [`meltano.yml` project file](pr
 - [Environment variable](/guide/configuration#configuring-settings): `<BUNDLE>__UPDATE`, e.g. `DBT__UPDATE`
 - Default: `{}` (an empty object)
 
-A file bundle's `update` [extra](/guide/configuration#plugin-extras) holds an object mapping file paths (of files inside the bundle, relative to the project root) to booleans.
+A file bundle's `update` [extra](/guide/configuration#plugin-extras) holds an object mapping file paths (of files inside the bundle, relative to the project root) to booleans. [Glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns are supported to allow matching of multiple files with a single path.
 
-When a file path's value is `True`, the file is considered to be managed by the file bundle and updated automatically when [`meltano upgrade`](/reference/command-line-interface#upgrade) is run.
+When a file path's value is `True`, the matching files are considered to be managed by the file bundle and updated automatically when [`meltano upgrade`](/reference/command-line-interface#upgrade) is run.
 
 ##### How to use
 
@@ -734,8 +734,19 @@ files:
 - name: dbt
   update:
     transform/dbt_project.yml: false
+    profiles/*.yml: true
 ```
 
+<div class="notification is-info">
+  <p>If a file path starts with a <code>*</code>, it must be wrapped in quotes to be considered valid YAML. For example, using <code>*.yml</code> to match all <code>.yml</code> files:</p>
+<pre>
+files:
+- name: dbt
+  update:
+    '*.yml': true
+</pre>
+</div>
+
 Alternatively, manage this extra using [`meltano config`](/reference/command-line-interface#config) or an [environment variable](/guide/configuration#configuring-settings):
 
 ```bash
@@ -745,8 +756,9 @@ export <BUNDLE>__UPDATE='{"<path>": <true/false>}'
 
 # For example:
 meltano config --plugin-type=files dbt set _update transform/dbt_project.yml false
+meltano config --plugin-type=files dbt set _update profiles/*.yml true
 
-export DBT__UPDATE='{"transform/dbt_project.yml": false}'
+export DBT__UPDATE='{"transform/dbt_project.yml": false, "profiles/*.yml": true}'
 ```
 
 ### Utilities

diff --git a/src/meltano/core/plugin/file.py b/src/meltano/core/plugin/file.py
@@ -107,7 +107,7 @@ def project_file_contents(
         """
 
         def with_update_header(content: str, relative_path: PathLike):
-            if str(relative_path) in paths_to_update:
+            if any(relative_path.match(path) for path in paths_to_update):
                 content = "\n\n".join([self.update_file_header(relative_path), content])
 
             return content
@@ -209,12 +209,11 @@ def files_to_update(
         Returns:
             A dictionary of file names and their contents.
         """
+        file_contents = self.project_file_contents(project, paths_to_update)
         return {
             relative_path: content
-            for relative_path, content in self.project_file_contents(
-                project, paths_to_update
-            ).items()
-            if str(relative_path) in paths_to_update
+            for relative_path, content in file_contents.items()
+            if any(relative_path.match(path) for path in paths_to_update)
         }
 
     def create_files(

diff --git a/tests/meltano/cli/test_upgrade.py b/tests/meltano/cli/test_upgrade.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import platform
 import shutil
 
@@ -155,6 +156,49 @@ def test_upgrade_files(
         assert_cli_runner(result)
         assert "Updated orchestrate/dags/meltano.py" in output
 
+    def test_upgrade_files_glob_path(
+        self, session, project, cli_runner, config_service, meltano_hub_service
+    ):
+        if platform.system() == "Windows":
+            pytest.xfail(
+                "Doesn't pass on windows, this is currently being tracked here https://github.com/meltano/meltano/issues/3444"
+            )
+
+        with mock.patch(
+            "meltano.core.project_plugins_service.MeltanoHubService",
+            return_value=meltano_hub_service,
+        ):
+            result = cli_runner.invoke(cli, ["add", "files", "airflow"])
+        assert_cli_runner(result)
+
+        file_path = project.root_dir("orchestrate/dags/meltano.py")
+        file_path.write_text("Overwritten!")
+
+        # override airflow--meltano.lock update extra config
+        result = cli_runner.invoke(
+            cli,
+            [
+                "config",
+                "--plugin-type",
+                "files",
+                "airflow",
+                "set",
+                "_update",
+                json.dumps(
+                    {
+                        "orchestrate/dags/meltano.py": False,
+                        "*.py": True,
+                    },
+                ),
+            ],
+        )
+        assert_cli_runner(result)
+
+        result = cli_runner.invoke(cli, ["upgrade", "files"])
+        output = result.stdout + result.stderr
+        assert_cli_runner(result)
+        assert "Updated orchestrate/dags/meltano.py" in output
+
     def test_upgrade_database(self, project, cli_runner):
         result = cli_runner.invoke(cli, ["upgrade", "database"])
         assert_cli_runner(result)