Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new Compute Engine Operators and fix system tests #25608

Merged
merged 31 commits into from
Nov 7, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
82a6d24
Add example_compute.py and test_compute.py
VladaZakharova Jul 4, 2022
0fa2329
Add compute_igm.py and compute_ssh.py + changed comment in compute_ss…
VladaZakharova Jul 5, 2022
58e637b
Add links + end system tests for compute.py and compute_igm.py
VladaZakharova Jul 7, 2022
565abfb
Finish CreateInstanceTemplate operator
VladaZakharova Jul 11, 2022
bf555ed
Add Insert and Get hooks + changes to ComputeEngineCreateInstanceTemp…
VladaZakharova Jul 12, 2022
b49d050
Add DeleteInstanceTemplateOperator + DeleteHook
VladaZakharova Jul 13, 2022
4672b1e
Add DeleteInstanceOperator + hooks + CreateInstanceFromTemplateOperat…
VladaZakharova Jul 14, 2022
b4d63ec
Add InstanceGroupManager operators + hooks
VladaZakharova Jul 27, 2022
a2b1e6a
Add unittests for created Operators
VladaZakharova Aug 2, 2022
920ecd7
Create system tests for operators + changed old operators
VladaZakharova Aug 4, 2022
5f88ab2
fix conflicts
VladaZakharova Aug 4, 2022
734ff7a
revert changes to dependencies files
VladaZakharova Sep 7, 2022
d85d0cf
Update provider.yaml
VladaZakharova Sep 7, 2022
4db0f43
Update provider_dependencies.json
VladaZakharova Sep 7, 2022
79f757e
Update provider_dependencies.json
VladaZakharova Sep 8, 2022
0c2fcc9
Update provider.yaml
VladaZakharova Sep 9, 2022
0ff6ad3
Rebase from upstream
VladaZakharova Sep 20, 2022
86740b5
fix errors with typing
VladaZakharova Sep 20, 2022
e6ff29f
Update compute.py
VladaZakharova Sep 20, 2022
067dfa8
update compute.py docs
VladaZakharova Sep 21, 2022
b8b2a5e
Update compute.py
VladaZakharova Sep 21, 2022
9821206
update link
VladaZakharova Sep 22, 2022
0db0119
change doc string in hook
VladaZakharova Sep 22, 2022
0dda483
fix spelling
VladaZakharova Sep 23, 2022
c5c3064
add mock for project_id default value
VladaZakharova Oct 14, 2022
7e38018
Update test_compute.py
VladaZakharova Oct 14, 2022
65def3e
Update index.rst
VladaZakharova Oct 14, 2022
c8380a6
update string formatting
VladaZakharova Oct 24, 2022
2074fcd
update hook file
VladaZakharova Oct 28, 2022
253b85f
Update compute.py
VladaZakharova Oct 31, 2022
20e20ba
update import string
VladaZakharova Oct 31, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
542 changes: 486 additions & 56 deletions airflow/providers/google/cloud/hooks/compute.py

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion airflow/providers/google/cloud/hooks/compute_ssh.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ def _connect_to_instance(self, user, hostname, pkey, proxy_command) -> paramiko.
raise
self.log.info("Failed to connect. Waiting %ds to retry", time_to_wait)
time.sleep(time_to_wait)
raise AirflowException("Caa not connect to instance")
raise AirflowException("Can not connect to instance")

def _authorize_compute_engine_instance_metadata(self, pubkey):
self.log.info("Appending SSH public key to instance metadata")
Expand Down
112 changes: 112 additions & 0 deletions airflow/providers/google/cloud/links/compute.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""This module contains Google Compute Engine links."""
from __future__ import annotations

from typing import TYPE_CHECKING

from airflow.models import BaseOperator
from airflow.providers.google.cloud.links.base import BaseGoogleLink

if TYPE_CHECKING:
from airflow.utils.context import Context

COMPUTE_BASE_LINK = "https://console.cloud.google.com/compute"
COMPUTE_LINK = (
COMPUTE_BASE_LINK + "/instancesDetail/zones/{location_id}/instances/{resource_id}?project={project_id}"
)
COMPUTE_TEMPLATE_LINK = COMPUTE_BASE_LINK + "/instanceTemplates/details/{resource_id}?project={project_id}"
COMPUTE_GROUP_MANAGER_LINK = (
COMPUTE_BASE_LINK + "/instanceGroups/details/{location_id}/{resource_id}?project={project_id}"
)


class ComputeInstanceDetailsLink(BaseGoogleLink):
"""Helper class for constructing Compute Instance details Link"""

name = "Compute Instance details"
key = "compute_instance_details"
format_str = COMPUTE_LINK

@staticmethod
def persist(
context: Context,
task_instance: BaseOperator,
location_id: str,
resource_id: str,
project_id: str | None,
):
task_instance.xcom_push(
context,
key=ComputeInstanceDetailsLink.key,
value={
"location_id": location_id,
"resource_id": resource_id,
"project_id": project_id,
},
)


class ComputeInstanceTemplateDetailsLink(BaseGoogleLink):
"""Helper class for constructing Compute Instance Template details Link"""

name = "Compute Instance Template details"
key = "compute_instance_template_details"
format_str = COMPUTE_TEMPLATE_LINK

@staticmethod
def persist(
context: Context,
task_instance: BaseOperator,
resource_id: str,
project_id: str | None,
):
task_instance.xcom_push(
context,
key=ComputeInstanceTemplateDetailsLink.key,
value={
"resource_id": resource_id,
"project_id": project_id,
},
)


class ComputeInstanceGroupManagerDetailsLink(BaseGoogleLink):
"""Helper class for constructing Compute Instance Group Manager details Link"""

name = "Compute Instance Group Manager"
key = "compute_instance_group_manager_details"
format_str = COMPUTE_GROUP_MANAGER_LINK

@staticmethod
def persist(
context: Context,
task_instance: BaseOperator,
location_id: str,
resource_id: str,
project_id: str | None,
):
task_instance.xcom_push(
context,
key=ComputeInstanceGroupManagerDetailsLink.key,
value={
"location_id": location_id,
"resource_id": resource_id,
"project_id": project_id,
},
)
Loading