Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update SDKs for google provider package #30067

Merged
merged 31 commits into from
May 17, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
0829a7a
Update SDK versions for Google provider
Mar 13, 2023
73d2aa7
Adjust google ads operators to v12
Mar 13, 2023
86d2beb
Fix BigQuery system tests
moiseenkov Mar 13, 2023
5af5781
Fix GCS system tests
moiseenkov Mar 27, 2023
aeec47e
Fix CloudBuild unit test
moiseenkov Mar 28, 2023
9c0a6bc
Update BigTable operators to accomodate for new dependencies.
kosteev Mar 10, 2023
61ed6ae
Fix Cloud Tasks System tests
ahidalgob Mar 15, 2023
3947709
Add setup docstring to Tasks system tests.
ahidalgob Mar 16, 2023
ce791d2
Update Vision operators to accommodate new dependencies.
Mar 20, 2023
213db85
Update SpeechToText operators to accommodate new dependencies.
Mar 16, 2023
a951e2f
Update Translate Speech operators to accommodate new dependencies.
Mar 15, 2023
4d957ba
Update VideoIntelligence operators to accommodate new dependencies.
Mar 17, 2023
deb017c
Update Compute Engine operators to accomodate for new dependencies.
VladaZakharova Mar 13, 2023
75ae7cd
Fix Stackdriver system test
ahidalgob Mar 21, 2023
081e340
Update Natural Language operators to accommodate new dependencies.
Mar 20, 2023
d15e09b
Update Composer system tests.
kosteev Mar 22, 2023
8945d37
Update AutoML operators to accommodate new dependencies.
VladaZakharova Mar 21, 2023
89a6cca
Fix Cloud SQL delete operator
ahidalgob Mar 24, 2023
42e8dc0
Update VertexAI operators to accommodate new dependencies.
VladaZakharova Mar 24, 2023
781b3f6
Add SQL to Sheets Test instructions
ahidalgob Mar 27, 2023
734b0b0
Update Dataproc Metastore operators to accommodate new dependencies.
Mar 29, 2023
d34fbf4
Update Dataproc operators to accommodate new dependencies.
Mar 31, 2023
5562618
Update Dataflow sys tests to new sdk
ahidalgob Apr 4, 2023
1b7aa2e
Update Dataproc on gke operators to accommodate new dependencies.
Apr 6, 2023
7420103
Update MLEngine operators to accomodate new dependencies.
VladaZakharova Apr 11, 2023
d783329
Update Dataprep operators to accommodate new dependencies.
Apr 12, 2023
b160d3b
Add Dataflow Go system test
ahidalgob Apr 13, 2023
0bc3cc6
Update providers.yaml for google
Mar 10, 2023
c60683c
fixup! Update providers.yaml for google
potiuk Apr 27, 2023
51a6787
Google SDK Fixes after rebase
ahidalgob May 12, 2023
31d7cd6
Fix Google providers type errors
ahidalgob May 16, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow/providers/apache/beam/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ versions:

dependencies:
- apache-airflow>=2.4.0
- apache-beam>=2.33.0
- apache-beam>=2.47.0

integrations:
- integration-name: Apache Beam
Expand Down
22 changes: 7 additions & 15 deletions airflow/providers/google/ads/hooks/ads.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,18 @@
from tempfile import NamedTemporaryFile
from typing import IO, Any

from google.ads.googleads.client import GoogleAdsClient
from google.ads.googleads.errors import GoogleAdsException
from google.ads.googleads.v12.services.services.customer_service import CustomerServiceClient
from google.ads.googleads.v12.services.services.google_ads_service import GoogleAdsServiceClient
from google.ads.googleads.v12.services.types.google_ads_service import GoogleAdsRow
from google.api_core.page_iterator import GRPCIterator
from google.auth.exceptions import GoogleAuthError

from airflow import AirflowException
from airflow.compat.functools import cached_property
from airflow.hooks.base import BaseHook
from airflow.providers.google.common.hooks.base_google import get_field
from airflow.providers.google_vendor.googleads.client import GoogleAdsClient
from airflow.providers.google_vendor.googleads.errors import GoogleAdsException
from airflow.providers.google_vendor.googleads.v12.services.services.customer_service import (
CustomerServiceClient,
)
from airflow.providers.google_vendor.googleads.v12.services.services.google_ads_service import (
GoogleAdsServiceClient,
)
from airflow.providers.google_vendor.googleads.v12.services.services.google_ads_service.pagers import (
SearchPager,
)
from airflow.providers.google_vendor.googleads.v12.services.types.google_ads_service import (
GoogleAdsRow,
)


class GoogleAdsHook(BaseHook):
Expand Down Expand Up @@ -238,7 +230,7 @@ def _search(

return self._extract_rows(iterators)

def _extract_rows(self, iterators: list[SearchPager]) -> list[GoogleAdsRow]:
def _extract_rows(self, iterators: list[GRPCIterator]) -> list[GoogleAdsRow]:
"""
Convert Google Page Iterator (GRPCIterator) objects to Google Ads Rows

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def get_secret(self, secret_id: str, project_id: str, secret_version: str = "lat
"""
name = self.client.secret_version_path(project_id, secret_id, secret_version)
try:
response = self.client.access_secret_version(name)
response = self.client.access_secret_version(request={"name": name})
value = response.payload.data.decode("UTF-8")
return value
except NotFound:
Expand Down
107 changes: 0 additions & 107 deletions airflow/providers/google/cloud/example_dags/example_compute.py

This file was deleted.

90 changes: 0 additions & 90 deletions airflow/providers/google/cloud/example_dags/example_compute_ssh.py

This file was deleted.

9 changes: 9 additions & 0 deletions airflow/providers/google/cloud/hooks/automl.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
)
from google.protobuf.field_mask_pb2 import FieldMask

from airflow import AirflowException
from airflow.compat.functools import cached_property
from airflow.providers.google.common.consts import CLIENT_INFO
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
Expand Down Expand Up @@ -93,6 +94,14 @@ def get_conn(self) -> AutoMlClient:
self._client = AutoMlClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
return self._client

def wait_for_operation(self, operation: Operation, timeout: float | None = None):
"""Waits for long-lasting operation to complete."""
try:
return operation.result(timeout=timeout)
except Exception:
error = operation.exception(timeout=timeout)
raise AirflowException(error)

@cached_property
def prediction_client(self) -> PredictionServiceClient:
"""
Expand Down