add new connectorBaseImage field to metadata (#30336)

airbytehq · Sep 13, 2023 · c6728bd · c6728bd
1 parent 13a5a40
commit c6728bd
Show file tree

Hide file tree

Showing 12 changed files with 118 additions and 46 deletions.
diff --git a/...onnectors/metadata_service/lib/metadata_service/models/generated/ConnectorBuildOptions.py b/...onnectors/metadata_service/lib/metadata_service/models/generated/ConnectorBuildOptions.py
@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+# generated by datamodel-codegen:
+#   filename:  ConnectorBuildOptions.yaml
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel, Extra
+
+
+class ConnectorBuildOptions(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    baseImage: Optional[str] = None
diff --git a/...s/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/...s/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py
@@ -15,6 +15,13 @@
 from typing_extensions import Literal
 
 
+class ConnectorBuildOptions(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    baseImage: Optional[str] = None
+
+
 class ReleaseStage(BaseModel):
     __root__: Literal["alpha", "beta", "generally_available", "custom"] = Field(
         ...,
@@ -103,9 +110,7 @@ class Config:
         ...,
         description="The deadline by which to upgrade before the breaking change takes effect.",
     )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
+    message: str = Field(..., description="Descriptive message detailing the breaking change.")
     migrationDocumentationUrl: Optional[AnyUrl] = Field(
         None,
         description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
@@ -194,6 +199,7 @@ class Config:
     name: str
     icon: Optional[str] = None
     definitionId: UUID
+    connectorBuildOptions: Optional[ConnectorBuildOptions] = None
     connectorType: Literal["destination", "source"]
     dockerRepository: str
     dockerImageTag: str
@@ -210,12 +216,8 @@ class Config:
         None,
         description="The date when this connector was first released, in yyyy-mm-dd format.",
     )
-    protocolVersion: Optional[str] = Field(
-        None, description="the Airbyte Protocol version supported by the connector"
-    )
-    connectorSubtype: Literal[
-        "api", "database", "file", "custom", "message_queue", "unknown"
-    ]
+    protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
+    connectorSubtype: Literal["api", "database", "file", "custom", "message_queue", "unknown"]
     releaseStage: ReleaseStage
     supportLevel: Optional[SupportLevel] = None
     tags: Optional[List[str]] = Field(

diff --git a/...a_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py b/...a_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py
@@ -93,9 +93,7 @@ class Config:
         ...,
         description="The deadline by which to upgrade before the breaking change takes effect.",
     )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
+    message: str = Field(..., description="Descriptive message detailing the breaking change.")
     migrationDocumentationUrl: Optional[AnyUrl] = Field(
         None,
         description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
@@ -170,9 +168,7 @@ class Config:
         False,
         description="true if this connector definition is available to all workspaces",
     )
-    custom: Optional[bool] = Field(
-        False, description="whether this is a custom connector definition"
-    )
+    custom: Optional[bool] = Field(False, description="whether this is a custom connector definition")
     releaseStage: Optional[ReleaseStage] = None
     supportLevel: Optional[SupportLevel] = None
     releaseDate: Optional[date] = Field(
@@ -184,9 +180,7 @@ class Config:
         description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
     )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
-    protocolVersion: Optional[str] = Field(
-        None, description="the Airbyte Protocol version supported by the connector"
-    )
+    protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None
     supportsDbt: Optional[bool] = Field(
         None,

diff --git a/...tadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py b/...tadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py
@@ -85,9 +85,7 @@ class Config:
         ...,
         description="The deadline by which to upgrade before the breaking change takes effect.",
     )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
+    message: str = Field(..., description="Descriptive message detailing the breaking change.")
     migrationDocumentationUrl: Optional[AnyUrl] = Field(
         None,
         description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
@@ -163,19 +161,15 @@ class Config:
         False,
         description="true if this connector definition is available to all workspaces",
     )
-    custom: Optional[bool] = Field(
-        False, description="whether this is a custom connector definition"
-    )
+    custom: Optional[bool] = Field(False, description="whether this is a custom connector definition")
     releaseStage: Optional[ReleaseStage] = None
     supportLevel: Optional[SupportLevel] = None
     releaseDate: Optional[date] = Field(
         None,
         description="The date when this connector was first released, in yyyy-mm-dd format.",
     )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
-    protocolVersion: Optional[str] = Field(
-        None, description="the Airbyte Protocol version supported by the connector"
-    )
+    protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
     allowedHosts: Optional[AllowedHosts] = None
     suggestedStreams: Optional[SuggestedStreams] = None
     maxSecondsBetweenMessages: Optional[int] = Field(

diff --git a/.../connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py b/.../connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py
@@ -93,9 +93,7 @@ class Config:
         ...,
         description="The deadline by which to upgrade before the breaking change takes effect.",
     )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
+    message: str = Field(..., description="Descriptive message detailing the breaking change.")
     migrationDocumentationUrl: Optional[AnyUrl] = Field(
         None,
         description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
@@ -181,19 +179,15 @@ class Config:
         False,
         description="true if this connector definition is available to all workspaces",
     )
-    custom: Optional[bool] = Field(
-        False, description="whether this is a custom connector definition"
-    )
+    custom: Optional[bool] = Field(False, description="whether this is a custom connector definition")
     releaseStage: Optional[ReleaseStage] = None
     supportLevel: Optional[SupportLevel] = None
     releaseDate: Optional[date] = Field(
         None,
         description="The date when this connector was first released, in yyyy-mm-dd format.",
     )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
-    protocolVersion: Optional[str] = Field(
-        None, description="the Airbyte Protocol version supported by the connector"
-    )
+    protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
     allowedHosts: Optional[AllowedHosts] = None
     suggestedStreams: Optional[SuggestedStreams] = None
     maxSecondsBetweenMessages: Optional[int] = Field(
@@ -224,9 +218,7 @@ class Config:
         False,
         description="true if this connector definition is available to all workspaces",
     )
-    custom: Optional[bool] = Field(
-        False, description="whether this is a custom connector definition"
-    )
+    custom: Optional[bool] = Field(False, description="whether this is a custom connector definition")
     releaseStage: Optional[ReleaseStage] = None
     supportLevel: Optional[SupportLevel] = None
     releaseDate: Optional[date] = Field(
@@ -238,9 +230,7 @@ class Config:
         description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
     )
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
-    protocolVersion: Optional[str] = Field(
-        None, description="the Airbyte Protocol version supported by the connector"
-    )
+    protocolVersion: Optional[str] = Field(None, description="the Airbyte Protocol version supported by the connector")
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None
     supportsDbt: Optional[bool] = Field(
         None,

diff --git a/...ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py b/...ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py
@@ -21,9 +21,7 @@ class Config:
         ...,
         description="The deadline by which to upgrade before the breaking change takes effect.",
     )
-    message: str = Field(
-        ..., description="Descriptive message detailing the breaking change."
-    )
+    message: str = Field(..., description="Descriptive message detailing the breaking change.")
     migrationDocumentationUrl: Optional[AnyUrl] = Field(
         None,
         description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",

diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py
@@ -1,7 +1,12 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by generate-python-classes
 from .ActorDefinitionResourceRequirements import *
 from .AirbyteInternal import *
 from .AllowedHosts import *
+from .ConnectorBuildOptions import *
 from .ConnectorMetadataDefinitionV0 import *
 from .ConnectorRegistryDestinationDefinition import *
 from .ConnectorRegistrySourceDefinition import *

diff --git a/...ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml b/...ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml
@@ -0,0 +1,10 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml
+title: ConnectorBuildOptions
+description: metadata specific to the build process.
+type: object
+additionalProperties: false
+properties:
+  baseImage:
+    type: string
diff --git a/...ctors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml b/...ctors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml
@@ -34,6 +34,8 @@ properties:
       definitionId:
         type: string
         format: uuid
+      connectorBuildOptions:
+        "$ref": ConnectorBuildOptions.yaml
       connectorType:
         type: string
         enum:

diff --git a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metadata-service"
-version = "0.1.5"
+version = "0.1.6"
 description = ""
 authors = ["Ben Church <ben@airbyte.io>"]
 readme = "README.md"

diff --git a/...ce/lib/tests/fixtures/metadata_validate/invalid/metadata_build_base_image_wrong_type.yaml b/...ce/lib/tests/fixtures/metadata_validate/invalid/metadata_build_base_image_wrong_type.yaml
@@ -0,0 +1,29 @@
+data:
+  allowedHosts:
+    hosts:
+      - "*.googleapis.com"
+  connectorBuildOptions:
+    unexpectedField: additionalProperties are not allowed ('unexpectedField' was unexpected)
+  connectorSubtype: file
+  connectorType: source
+  definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
+  dockerImageTag: 0.3.7
+  dockerRepository: airbyte/source-google-sheets
+  githubIssueLabel: source-google-sheets
+  icon: google-sheets.svg
+  license: Elv2
+  name: Google Sheets
+  registries:
+    cloud:
+      enabled: true
+    oss:
+      enabled: true
+  releaseStage: generally_available
+  documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets
+  tags:
+    - language:python
+  ab_internal:
+    sl: 300
+    ql: 400
+  supportLevel: certified
+metadataSpecVersion: "1.0"
diff --git a/...etadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_build_base_image.yaml b/...etadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_build_base_image.yaml
@@ -0,0 +1,29 @@
+data:
+  allowedHosts:
+    hosts:
+      - "*.googleapis.com"
+  connectorBuildOptions:
+    baseImage: airbyte/airbyte-python-connectors-base:0.1.0
+  connectorSubtype: file
+  connectorType: source
+  definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7
+  dockerImageTag: 0.3.7
+  dockerRepository: airbyte/source-google-sheets
+  githubIssueLabel: source-google-sheets
+  icon: google-sheets.svg
+  license: Elv2
+  name: Google Sheets
+  registries:
+    cloud:
+      enabled: true
+    oss:
+      enabled: true
+  releaseStage: generally_available
+  documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets
+  tags:
+    - language:python
+  ab_internal:
+    sl: 300
+    ql: 400
+  supportLevel: certified
+metadataSpecVersion: "1.0"