airbyte-ci: poetry install --no-root in builder (#35010)

airbyte-ci/connectors/pipelines/README.md

@@ -296,7 +296,7 @@ flowchart TD
 | `--code-tests-only`                                     | True     | False         | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. |
 | `--concurrent-cat`                                      | False    | False         | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits.                                                                                              |
 | `--<step-id>.<extra-parameter>=<extra-parameter-value>` | True     |               | You can pass extra parameters for specific test steps. More details in the extra parameters section below                                                                                                |
-| `--ci-requirements`                                     | False    |               |                                                                                                                                                                                                          | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. 
+| `--ci-requirements`                                     | False    |               |                                                                                                                                                                                                          | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use.
 
 Note:
 
@@ -620,11 +620,11 @@ You can find the list of internal packages [here](https://github.com/airbytehq/a
 
 #### Options
 
-| Option                    | Required | Multiple| Description                                                                                 |
-| ------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------- |
-| `--poetry-package-path/-p`| False    | True    | Poetry packages path to run the poe tasks for.                                              |
-| `--modified`              | False    | False   | Run poe tasks of modified internal poetry packages.                                         |
-| `--ci-requirements`       | False    | False   | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
+| Option                     | Required | Multiple | Description                                                                                 |
+| -------------------------- | -------- | -------- | ------------------------------------------------------------------------------------------- |
+| `--poetry-package-path/-p` | False    | True     | Poetry packages path to run the poe tasks for.                                              |
+| `--modified`               | False    | False    | Run poe tasks of modified internal poetry packages.                                         |
+| `--ci-requirements`        | False    | False    | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
 
 #### Examples
 You can pass multiple `--poetry-package-path` options to run poe tasks.
@@ -640,10 +640,12 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
-| 4.1.2  | [#34945](https://github.com/airbytehq/airbyte/pull/34945)  | Only install main dependencies when running poetry install.                                                                |
-| 4.1.1  | [#34430](https://github.com/airbytehq/airbyte/pull/34430)  | Speed up airbyte-ci startup (and airbyte-ci format).                                         |
-| 4.1.0  | [#34923](https://github.com/airbytehq/airbyte/pull/34923)  | Include gradle test reports in HTML connector test report.                                                                 |
-| 4.0.0  | [#34736](https://github.com/airbytehq/airbyte/pull/34736)  | Run poe tasks declared in internal poetry packages.                                         |
+| 4.1.3   | [#TBD](https://github.com/airbytehq/airbyte/pull/TBD)  | Use `poetry install --no-root` in the builder container.                                                                |
+
+| 4.1.2   | [#34945](https://github.com/airbytehq/airbyte/pull/34945)  | Only install main dependencies when running poetry install.                                                                |
+| 4.1.1   | [#34430](https://github.com/airbytehq/airbyte/pull/34430)  | Speed up airbyte-ci startup (and airbyte-ci format).                                                                       |
+| 4.1.0   | [#34923](https://github.com/airbytehq/airbyte/pull/34923)  | Include gradle test reports in HTML connector test report.                                                                 |
+| 4.0.0   | [#34736](https://github.com/airbytehq/airbyte/pull/34736)  | Run poe tasks declared in internal poetry packages.                                                                        |
 | 3.10.4  | [#34867](https://github.com/airbytehq/airbyte/pull/34867)  | Remove connector ops team                                                                                                  |
 | 3.10.3  | [#34836](https://github.com/airbytehq/airbyte/pull/34836)  | Add check for python registry publishing enabled for certified python sources.                                             |
 | 3.10.2  | [#34044](https://github.com/airbytehq/airbyte/pull/34044)  | Add pypi validation testing.                                                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -45,12 +45,14 @@ async def _create_builder_container(self, base_container: Container) -> Containe
         Returns:
             Container: The builder container, with installed dependencies.
         """
-        ONLY_PYTHON_BUILD_FILES = ["setup.py", "requirements.txt", "pyproject.toml", "poetry.lock"]
+        ONLY_BUILD_FILES = ["pyproject.toml", "poetry.lock", "poetry.toml", "setup.py", "requirements.txt"]
+
         builder = await with_python_connector_installed(
             self.context,
             base_container,
             str(self.context.connector.code_directory),
-            include=ONLY_PYTHON_BUILD_FILES,
+            install_root_package=False,
+            include=ONLY_BUILD_FILES
         )
 
         return builder

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py

@@ -156,10 +156,13 @@ def _install_python_dependencies_from_requirements_txt(container: Container) ->
 def _install_python_dependencies_from_poetry(
     container: Container,
     additional_dependency_groups: Optional[Sequence[str]] = None,
+    install_root_package: bool = True,
 ) -> Container:
     pip_install_poetry_cmd = ["pip", "install", "poetry"]
     poetry_disable_virtual_env_cmd = ["poetry", "config", "virtualenvs.create", "false"]
     poetry_install_cmd = ["poetry", "install"]
+    if not install_root_package:
+        poetry_install_cmd += ["--no-root"]
     if additional_dependency_groups:
         for group in additional_dependency_groups:
             poetry_install_cmd += ["--with", group]
@@ -175,6 +178,7 @@ async def with_installed_python_package(
     additional_dependency_groups: Optional[Sequence[str]] = None,
     exclude: Optional[List] = None,
     include: Optional[List] = None,
+    install_root_package: bool = True,
 ) -> Container:
     """Install a python package in a python environment container.
 
@@ -184,6 +188,8 @@ async def with_installed_python_package(
         package_source_code_path (str): The local path to the package source code.
         additional_dependency_groups (Optional[Sequence[str]]): extra_requires dependency of setup.py to install. Defaults to None.
         exclude (Optional[List]): A list of file or directory to exclude from the python package source code.
+        include (Optional[List]): A list of file or directory to include from the python package source code.
+        install_root_package (bool): Whether to install the root package. Defaults to True.
 
     Returns:
         Container: A python environment container with the python package installed.
@@ -200,7 +206,7 @@ async def with_installed_python_package(
 
     if has_pyproject_toml:
         container = with_poetry_cache(container, context.dagger_client)
-        container = _install_python_dependencies_from_poetry(container, additional_dependency_groups)
+        container = _install_python_dependencies_from_poetry(container, additional_dependency_groups, install_root_package)
     elif has_setup_py:
         container = with_pip_cache(container, context.dagger_client)
         container = _install_python_dependencies_from_setup_py(container, additional_dependency_groups)
@@ -251,6 +257,7 @@ async def with_python_connector_installed(
     additional_dependency_groups: Optional[Sequence[str]] = None,
     exclude: Optional[List[str]] = None,
     include: Optional[List[str]] = None,
+    install_root_package: bool = True,
 ) -> Container:
     """Install an airbyte python connectors  dependencies."""
 
@@ -264,6 +271,7 @@ async def with_python_connector_installed(
         additional_dependency_groups=additional_dependency_groups,
         exclude=exclude,
         include=include,
+        install_root_package=install_root_package,
     )
 
     container = await apply_python_development_overrides(context, container)

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.1.2"
+version = "4.1.3"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]