Merge branch 'main' into edgarrmondragon/feat/logging-callsite-parame…

…ters
meltano · Apr 29, 2024 · e0128d1 · e0128d1
2 parents 452406e + 25da80a
commit e0128d1
Show file tree

Hide file tree

Showing 36 changed files with 623 additions and 349 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -18,7 +18,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Install Poetry
       env:
@@ -73,7 +73,7 @@ jobs:
     environment: publishing
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Download artifacts
       uses: actions/download-artifact@v4

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -21,8 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repository
-        uses: actions/checkout@v4.1.2
+        uses: actions/checkout@v4.1.4
 
       - name: GitHub dependency vulnerability check
         if: ${{ github.event_name == 'pull_request_target' }}
-        uses: actions/dependency-review-action@v4.2.5
+        uses: actions/dependency-review-action@V4.3.0
diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml
@@ -51,7 +51,7 @@ jobs:
         echo "dry_run=${{ format('{0}', github.event.inputs.dry_run) || env.DEFAULT_DRY_RUN }}" >> $GITHUB_ENV
         echo "registry=${{ github.event.inputs.registry || env.DEFAULT_REGISTRY }}" >> $GITHUB_ENV
 
-    - uses: actions/checkout@v4.1.2
+    - uses: actions/checkout@v4.1.4
 
     - name: Get Meltano version
       id: get-meltano-version

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -46,7 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Install Poetry
       env:

diff --git a/.github/workflows/lint_snowplow_schemas.yml b/.github/workflows/lint_snowplow_schemas.yml
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the repository
-        uses: actions/checkout@v4.1.2
+        uses: actions/checkout@v4.1.4
 
       - name: Iglu Schema Lint
         uses: ./.github/actions/igluctl-lint

diff --git a/.github/workflows/resources/constraints.txt b/.github/workflows/resources/constraints.txt
@@ -1,4 +1,4 @@
-nox==2024.3.2
+nox==2024.4.15
 nox-poetry==1.0.3
 pip==24.0
 poetry==1.8.2
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -92,7 +92,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Install Poetry
       env:
@@ -247,7 +247,7 @@ jobs:
     needs: tests
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Install Poetry
       env:
@@ -302,7 +302,7 @@ jobs:
 
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v4.1.2
+      uses: actions/checkout@v4.1.4
 
     - name: Install Poetry
       env:

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -34,7 +34,7 @@ jobs:
       pull-requests: write  # to create and update PRs
 
     steps:
-    - uses: actions/checkout@v4.1.2
+    - uses: actions/checkout@v4.1.4
       with:
         fetch-depth: 0
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -74,7 +74,7 @@ repos:
           - "prettier@3.0.0"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.7
+    rev: v0.4.1
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix, --show-fixes]
@@ -88,6 +88,7 @@ repos:
         exclude: ^(src/meltano/migrations/)
         args:
           - "--max-local-variables=8"
+        language_version: python3.11
         additional_dependencies:
           - flake8==4.0.1
           - flake8-bandit==3.0.0

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,32 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/).
 
+## v3.4.0 (2024-04-18)
+
+### ✨ New
+
+- [#8459](https://github.com/meltano/meltano/issues/8459) `meltano run` now has a `--run-id` option to allow for custom run UUIDs
+- [#8465](https://github.com/meltano/meltano/issues/8465) Support `uv` as an optional virtualenv backend
+- [#8355](https://github.com/meltano/meltano/issues/8355) Support installing multiple plugins of any type -- _**Thanks @ReubenFrankel!**_
+
+### 🐛 Fixes
+
+- [#8486](https://github.com/meltano/meltano/issues/8486) "`kind: {kind}` is deprecated..." is no longer logged if the corresponding replacement is in place
+- [#8489](https://github.com/meltano/meltano/issues/8489) Environment variables from `.env` are now passed to the plugin installation subprocesses
+- [#8490](https://github.com/meltano/meltano/issues/8490) An explicit error message is now logged when Meltano fails to retrieve tap state from the state backend
+- [#8447](https://github.com/meltano/meltano/issues/8447) `meltano run` no longer creates empty `venv` plugin directories for inherited plugins
+- [#8446](https://github.com/meltano/meltano/issues/8446) Added `run_id` and `job_name` properties to `meltano run` log messages
+
+### ⚙️ Under the Hood
+
+- [#8499](https://github.com/meltano/meltano/issues/8499) Only lookup uv executable once
+- [#8469](https://github.com/meltano/meltano/issues/8469) Fixed some log serialization issues with `meltano.core.proj…ec_plugins_service.DefinitionSource` and `meltano.core.plugin.project_plugin.ProjectPlugin`
+
+### 📚 Documentation Improvements
+
+- [#8449](https://github.com/meltano/meltano/issues/8449) Fixed a broken link to custom plugin definitions in the project concept page
+- [#8432](https://github.com/meltano/meltano/issues/8432) Added the 2024-Q1 changelog
+
 ## v3.3.2 (2024-03-06)
 
 ### 🐛 Fixes

diff --git a/docs/docs/concepts/plugins.mdx b/docs/docs/concepts/plugins.mdx
@@ -981,8 +981,8 @@ You can install mappers like any other other plugin using [`meltano add`](/refer
 
 ```bash
 $ meltano add mapper transform-field
-Installing mapper 'transform-field'...
-Installed mapper 'transform-field'
+2024-01-01T00:25:40.604941Z [info     ] Installing mapper 'transform-field'
+2024-01-01T00:25:53.152127Z [info     ] Installed mapper 'transform-field'
 
 To learn more about mapper 'transform-field', visit https://github.com/transferwise/pipelinewise-transform-field
 ```

diff --git a/docs/docs/getting-started/part1.mdx b/docs/docs/getting-started/part1.mdx
@@ -131,8 +131,8 @@ Variant:        meltanolabs (default)
 Repository:     https://github.com/meltanolabs/tap-github
 Documentation:  https://hub.meltano.com/extractors/tap-github
 
-Installing extractor 'tap-github'...
-Installed extractor 'tap-github'
+2024-01-01T00:25:40.604941Z [info     ] Installing extractor 'tap-github'
+2024-01-01T00:25:53.152127Z [info     ] Installed extractor 'tap-github'
 
 To learn more about extractor 'tap-github', visit https://hub.meltano.com/extractors/tap-github
 ```
@@ -378,8 +378,8 @@ Variant:        andyh1203 (default)
 Repository:     https://github.com/andyh1203/target-jsonl
 Documentation:  https://hub.meltano.com/loaders/target-jsonl--andyh1203
 
-Installing loader 'target-jsonl'...
-Installed loader 'target-jsonl'
+2024-01-01T00:25:40.604941Z [info     ] Installing loader 'target-jsonl'
+2024-01-01T00:25:53.152127Z [info     ] Installed loader 'target-jsonl'
 
 To learn more about loader 'target-jsonl', visit https://hub.meltano.com/loaders/target-jsonl--andyh1203
 ```

diff --git a/docs/docs/getting-started/part2.mdx b/docs/docs/getting-started/part2.mdx
@@ -59,10 +59,10 @@ Variant:        meltanolabs (default)
 Repository:     https://github.com/MeltanoLabs/target-postgres
 Documentation:  https://hub.meltano.com/loaders/target-postgres--meltanolabs
 
-Installing loader 'target-postgres'...
+2024-01-01T00:25:40.604941Z [info     ] Installing loader 'target-postgres'
 ---> 100%
 
-Installed loader 'target-postgres'
+2024-01-01T00:25:53.152127Z [info     ] Installed loader 'target-postgres'
 
 To learn more about loader 'target-postgres', visit https://hub.meltano.com/loaders/target-postgres--meltanolabs
 ```

diff --git a/docs/docs/getting-started/part3.mdx b/docs/docs/getting-started/part3.mdx
@@ -84,11 +84,12 @@ $ meltano add utility dbt-postgres
 2022-09-22T11:32:35.601357Z [info     ] Environment 'dev' is active
 Added utility 'dbt-postgres' to your Meltano project
 ...
-Installing utility 'dbt-postgres'...
+2024-01-01T00:25:40.604941Z [info     ] Installing utility 'dbt-postgres'
 ---> 100%
 ...
 Installed utility 'dbt-postgres'
-Installed 1/1 plugins
+2024-01-01T00:25:53.152127Z [info     ] Installed utility 'dbt-postgres'
+2024-01-01T00:25:53.152894Z [info     ] Installed 1/1 plugins
 ```
 
 </Termy>

diff --git a/docs/docs/reference/command-line-interface.md b/docs/docs/reference/command-line-interface.md
@@ -982,6 +982,7 @@ meltano run --state-id-suffix=<STATE_ID_SUFFIX> tap-gitlab target-postgres
 - `--force` will force a job run even if a conflicting job with the same generated ID is in progress.
 - `--state-id-suffix` define a custom suffix to generate a state ID with for each EL pair.
 - `--merge-state` will merge state with that of previous runs. See the [example in the Meltano repository](https://github.com/meltano/meltano/blob/main/integration/example-library/meltano-run-merge-states/index.md).
+- `--run-id` will use the provided UUID for the current run. This is useful when your workflow is managed by an external system and you want to track the run in Meltano.
 
 Examples:
 

diff --git a/docs/docs/reference/settings.mdx b/docs/docs/reference/settings.mdx
@@ -884,6 +884,24 @@ export MELTANO_STATE_BACKEND_GCS_APPLICATION_CREDENTIALS="path/to/creds.json"
 - Options: `virtualenv`, `uv`
 - Default: `virtualenv`
 
+<Tabs className="meltano-tabs" queryString="meltano-tabs">
+  <TabItem className="meltano-tab-content" value="meltano config" label="meltano config" default>
+
+```bash
+meltano config meltano set venv.backend uv
+meltano install --clean
+```
+
+  </TabItem>
+  <TabItem className="meltano-tab-content" value="env" label="env" default>
+
+```bash
+MELTANO_VENV_BACKEND=uv meltano install --clean
+```
+
+  </TabItem>
+</Tabs>
+
 ## Snowplow Tracking
 
 ### `snowplow.collector_endpoints`

diff --git a/docs/docs/tutorials/custom-extractor.md b/docs/docs/tutorials/custom-extractor.md
@@ -324,9 +324,9 @@ You can leave the capabilities and settings fields blank for now. The command wi
 ```
 Added extractor 'tap-jsonplaceholder' to your Meltano project
 
-Installing extractor 'tap-jsonplaceholder'...
+2024-01-01T00:25:40.604941Z [info     ] Installing extractor 'tap-jsonplaceholder'...
+2024-01-01T00:25:53.152127Z [info     ] Installed extractor 'tap-jsonplaceholder'
 
-Installed extractor 'tap-jsonplaceholder'
 ```
 
 Alternatively, you can create a [plugin definition](/concepts/project#custom-plugin-definitions) YAML file locally and add to your project using the `--from-ref` option:

diff --git a/docs/package.json b/docs/package.json
@@ -1,7 +1,7 @@
 {
   "name": "docs.meltano.com",
   "license": "MIT",
-  "version": "3.4.0a2",
+  "version": "3.4.0",
   "description": "https://meltano.com",
   "private": true,
   "scripts": {