Source Azure Blob Storage: support OAuth (#36825)

Signed-off-by: Artem Inzhyyants <artem.inzhyyants@gmail.com>

airbyte-integrations/connectors/source-azure-blob-storage/README.md

@@ -11,6 +11,26 @@ For information about how to use this connector within Airbyte, see [the documen
 * Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
 
 
+### Generate new oauth token
+
+Tenant id should be provided by user, reason:
+https://learn.microsoft.com/en-us/answers/questions/1531138/which-tenant-id-do-i-have-to-use-to-get-tokens-and
+
+1. GET https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/authorize
+    ?response_type=code
+    &client_id=<client_id>
+    &scope=offline_access https://storage.azure.com/.default
+    &redirect_uri=http://localhost:8000/auth_flow
+    &response_mode=query
+    &state=1234
+
+2. POST https://login.microsoftonline.com/<tenant_id>/oauth2/v2.0/token
+client_id:<client_id>
+code:<code obtained from previous request>
+redirect_uri:http://localhost:8000/auth_flow
+grant_type:authorization_code
+client_secret:<client_secret>
+
 ### Installing the connector
 From this connector directory, run:
 ```bash

airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml

@@ -9,7 +9,7 @@ acceptance_tests:
         expect_records:
           path: integration_tests/expected_records/csv_custom_encoding.jsonl
           exact_order: true
-      - config_path: secrets/csv_custom_format_config.json
+      - config_path: secrets/csv_custom_format_encoding_config.json
         expect_records:
           path: integration_tests/expected_records/csv_custom_format.jsonl
           exact_order: true
@@ -64,7 +64,7 @@ acceptance_tests:
         status: succeed
       - config_path: secrets/csv_custom_encoding_config.json
         status: succeed
-      - config_path: secrets/csv_custom_format_config.json
+      - config_path: secrets/csv_custom_format_encoding_config.json
         status: succeed
       - config_path: secrets/csv_user_schema_config.json
         status: succeed
@@ -92,7 +92,7 @@ acceptance_tests:
     tests:
       - config_path: secrets/config.json
       - config_path: secrets/csv_custom_encoding_config.json
-      - config_path: secrets/csv_custom_format_config.json
+      - config_path: secrets/csv_custom_format_encoding_config.json
       - config_path: secrets/csv_user_schema_config.json
       - config_path: secrets/csv_no_header_config.json
       - config_path: secrets/csv_skip_rows_config.json

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_avro.json

@@ -1,7 +1,10 @@
 {
   "azure_blob_storage_endpoint": "http://localhost:10000/account1",
   "azure_blob_storage_account_name": "account1",
-  "azure_blob_storage_account_key": "key1",
+  "credentials": {
+    "auth_type": "storage_account_key",
+    "azure_blob_storage_account_key": "key1"
+  },
   "azure_blob_storage_container_name": "testcontainer",
   "streams": [
     {

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_csv.json

@@ -1,7 +1,10 @@
 {
   "azure_blob_storage_endpoint": "http://localhost:10000/account1",
   "azure_blob_storage_account_name": "account1",
-  "azure_blob_storage_account_key": "key1",
+  "credentials": {
+    "auth_type": "storage_account_key",
+    "azure_blob_storage_account_key": "key1"
+  },
   "azure_blob_storage_container_name": "testcontainer",
   "streams": [
     {

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_jsonl.json

@@ -1,7 +1,10 @@
 {
   "azure_blob_storage_endpoint": "http://localhost:10000/account1",
   "azure_blob_storage_account_name": "account1",
-  "azure_blob_storage_account_key": "key1",
+  "credentials": {
+    "auth_type": "storage_account_key",
+    "azure_blob_storage_account_key": "key1"
+  },
   "azure_blob_storage_container_name": "testcontainer",
   "streams": [
     {

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configs/config_integration_parquet.json

@@ -1,7 +1,10 @@
 {
   "azure_blob_storage_endpoint": "http://localhost:10000/account1",
   "azure_blob_storage_account_name": "account1",
-  "azure_blob_storage_account_key": "key1",
+  "credentials": {
+    "auth_type": "storage_account_key",
+    "azure_blob_storage_account_key": "key1"
+  },
   "azure_blob_storage_container_name": "testcontainer",
   "streams": [
     {

airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json

@@ -358,15 +358,81 @@
         "order": 2,
         "type": "string"
       },
-      "azure_blob_storage_account_key": {
-        "title": "Azure Blob Storage account key",
-        "description": "The Azure blob storage account key.",
-        "airbyte_secret": true,
-        "examples": [
-          "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
-        ],
+      "credentials": {
+        "title": "Authentication",
+        "description": "Credentials for connecting to the Azure Blob Storage",
+        "type": "object",
         "order": 3,
-        "type": "string"
+        "oneOf": [
+          {
+            "title": "Authenticate via Oauth2",
+            "type": "object",
+            "properties": {
+              "auth_type": {
+                "title": "Auth Type",
+                "default": "oauth2",
+                "const": "oauth2",
+                "enum": ["oauth2"],
+                "type": "string"
+              },
+              "tenant_id": {
+                "title": "Tenant ID",
+                "description": "Tenant ID of the Microsoft Azure Application user",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "client_id": {
+                "title": "Client ID",
+                "description": "Client ID of your Microsoft developer application",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "client_secret": {
+                "title": "Client Secret",
+                "description": "Client Secret of your Microsoft developer application",
+                "airbyte_secret": true,
+                "type": "string"
+              },
+              "refresh_token": {
+                "title": "Refresh Token",
+                "description": "Refresh Token of your Microsoft developer application",
+                "airbyte_secret": true,
+                "type": "string"
+              }
+            },
+            "required": [
+              "tenant_id",
+              "client_id",
+              "client_secret",
+              "refresh_token",
+              "auth_type"
+            ]
+          },
+          {
+            "title": "Authenticate via Storage Account Key",
+            "type": "object",
+            "properties": {
+              "auth_type": {
+                "title": "Auth Type",
+                "default": "storage_account_key",
+                "const": "storage_account_key",
+                "enum": ["storage_account_key"],
+                "type": "string"
+              },
+              "azure_blob_storage_account_key": {
+                "title": "Azure Blob Storage account key",
+                "description": "The Azure blob storage account key.",
+                "airbyte_secret": true,
+                "examples": [
+                  "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="
+                ],
+                "order": 3,
+                "type": "string"
+              }
+            },
+            "required": ["azure_blob_storage_account_key", "auth_type"]
+          }
+        ]
       },
       "azure_blob_storage_container_name": {
         "title": "Azure blob storage container (Bucket) Name",
@@ -386,8 +452,61 @@
     "required": [
       "streams",
       "azure_blob_storage_account_name",
-      "azure_blob_storage_account_key",
+      "credentials",
       "azure_blob_storage_container_name"
     ]
+  },
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "oauth2",
+    "oauth_config_specification": {
+      "oauth_user_input_from_connector_config_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "tenant_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "tenant_id"]
+          }
+        }
+      },
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "refresh_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "refresh_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
+      }
+    }
   }
 }

airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml

@@ -7,7 +7,7 @@ data:
   connectorSubtype: file
   connectorType: source
   definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093
-  dockerImageTag: 0.3.6
+  dockerImageTag: 0.4.0
   dockerRepository: airbyte/source-azure-blob-storage
   documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage
   githubIssueLabel: source-azure-blob-storage

airbyte-integrations/connectors/source-azure-blob-storage/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "0.3.6"
+version = "0.4.0"
 name = "source-azure-blob-storage"
 description = "Source implementation for Azure Blob Storage."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py

@@ -2,11 +2,51 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Literal, Optional, Union
 
 import dpath.util
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
-from pydantic import AnyUrl, Field
+from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
+from pydantic import AnyUrl, BaseModel, Field
+
+
+class Oauth2(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Authenticate via Oauth2"
+        discriminator = "auth_type"
+
+    auth_type: Literal["oauth2"] = Field("oauth2", const=True)
+    tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft Azure Application user", airbyte_secret=True)
+    client_id: str = Field(
+        title="Client ID",
+        description="Client ID of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+    client_secret: str = Field(
+        title="Client Secret",
+        description="Client Secret of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+    refresh_token: str = Field(
+        title="Refresh Token",
+        description="Refresh Token of your Microsoft developer application",
+        airbyte_secret=True,
+    )
+
+
+class StorageAccountKey(BaseModel):
+    class Config(OneOfOptionConfig):
+        title = "Authenticate via Storage Account Key"
+        discriminator = "auth_type"
+
+    auth_type: Literal["storage_account_key"] = Field("storage_account_key", const=True)
+    azure_blob_storage_account_key: str = Field(
+        title="Azure Blob Storage account key",
+        description="The Azure blob storage account key.",
+        airbyte_secret=True,
+        examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
+        order=3,
+    )
 
 
 class Config(AbstractFileBasedSpec):
@@ -25,11 +65,11 @@ def documentation_url(cls) -> AnyUrl:
         examples=["airbyte5storage"],
         order=2,
     )
-    azure_blob_storage_account_key: str = Field(
-        title="Azure Blob Storage account key",
-        description="The Azure blob storage account key.",
-        airbyte_secret=True,
-        examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="],
+    credentials: Union[Oauth2, StorageAccountKey] = Field(
+        title="Authentication",
+        description="Credentials for connecting to the Azure Blob Storage",
+        discriminator="auth_type",
+        type="object",
         order=3,
     )
     azure_blob_storage_container_name: str = Field(