-
Notifications
You must be signed in to change notification settings - Fork 6.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs(samples): Adding samples for template usage (#41)
Preparing another code sample. This time for the [`Create a job from a Compute Engine instance template`](https://cloud.google.com/batch/docs/create-run-job#create-job-instance-template) section.
- Loading branch information
1 parent
89e38c6
commit add4cb0
Showing
3 changed files
with
204 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
pytest==7.1.3 | ||
pytest-parallel==0.1.1 | ||
google-cloud-storage==2.5.0 | ||
google-cloud-compute==1.5.2 | ||
google-cloud-resource-manager==1.6.1 | ||
google-cloud-storage==2.5.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
# Copyright 2022 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# [START batch_create_job_with_template] | ||
from google.cloud import batch_v1 | ||
|
||
|
||
def create_script_job_with_template(project_id: str, region: str, job_name: str, template_link: str) -> batch_v1.Job: | ||
""" | ||
This method shows how to create a sample Batch Job that will run | ||
a simple command on Cloud Compute instances created using a provided Template. | ||
Args: | ||
project_id: project ID or project number of the Cloud project you want to use. | ||
region: name of the region you want to use to run the job. Regions that are | ||
available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations | ||
job_name: the name of the job that will be created. | ||
It needs to be unique for each project and region pair. | ||
template_link: a link to an existing Instance Template. Acceptable formats: | ||
* "projects/{project_id}/global/instanceTemplates/{template_name}" | ||
* "{template_name}" - if the template is defined in the same project as used to create the Job. | ||
Returns: | ||
A job object representing the job created. | ||
""" | ||
client = batch_v1.BatchServiceClient() | ||
|
||
# Define what will be done as part of the job. | ||
task = batch_v1.TaskSpec() | ||
runnable = batch_v1.Runnable() | ||
runnable.script = batch_v1.Runnable.Script() | ||
runnable.script.text = "echo Hello world! This is task ${BATCH_TASK_INDEX}. This job has a total of ${BATCH_TASK_COUNT} tasks." | ||
# You can also run a script from a file. Just remember, that needs to be a script that's | ||
# already on the VM that will be running the job. Using runnable.script.text and runnable.script.path is mutually | ||
# exclusive. | ||
# runnable.script.path = '/tmp/test.sh' | ||
task.runnables = [runnable] | ||
|
||
# We can specify what resources are requested by each task. | ||
resources = batch_v1.ComputeResource() | ||
resources.cpu_milli = 2000 # in milliseconds per cpu-second. This means the task requires 2 whole CPUs. | ||
resources.memory_mib = 16 | ||
task.compute_resource = resources | ||
|
||
task.max_retry_count = 2 | ||
task.max_run_duration = "3600s" | ||
|
||
# Tasks are grouped inside a job using TaskGroups. | ||
group = batch_v1.TaskGroup() | ||
group.task_count = 4 | ||
group.task_spec = task | ||
|
||
# Policies are used to define on what kind of virtual machines the tasks will run on. | ||
# In this case, we tell the system to use an instance template that defines all the | ||
# required parameters. | ||
allocation_policy = batch_v1.AllocationPolicy() | ||
instances = batch_v1.AllocationPolicy.InstancePolicyOrTemplate() | ||
instances.instance_template = template_link | ||
allocation_policy.instances = [instances] | ||
|
||
job = batch_v1.Job() | ||
job.task_groups = [group] | ||
job.allocation_policy = allocation_policy | ||
job.labels = {"env": "testing", "type": "script"} | ||
# We use Cloud Logging as it's an out of the box available option | ||
job.logs_policy = batch_v1.LogsPolicy() | ||
job.logs_policy.destination = batch_v1.LogsPolicy.Destination.CLOUD_LOGGING | ||
|
||
create_request = batch_v1.CreateJobRequest() | ||
create_request.job = job | ||
create_request.job_id = job_name | ||
# The job's parent is the region in which the job will run | ||
create_request.parent = f"projects/{project_id}/locations/{region}" | ||
|
||
return client.create_job(create_request) | ||
# [END batch_create_job_with_template] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
# Copyright 2022 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import uuid | ||
|
||
import google.auth | ||
from google.cloud import batch_v1 | ||
from google.cloud import compute_v1 | ||
from google.cloud import resourcemanager_v3 | ||
import pytest | ||
|
||
|
||
from .test_basics import _test_body | ||
|
||
from ..create.create_with_template import create_script_job_with_template | ||
|
||
PROJECT = google.auth.default()[1] | ||
|
||
PROJECT_NUMBER = resourcemanager_v3.ProjectsClient().get_project(name=f"projects/{PROJECT}").name.split("/")[1] | ||
|
||
REGION = 'europe-north1' | ||
|
||
TIMEOUT = 600 # 10 minutes | ||
|
||
WAIT_STATES = { | ||
batch_v1.JobStatus.State.STATE_UNSPECIFIED, | ||
batch_v1.JobStatus.State.QUEUED, | ||
batch_v1.JobStatus.State.RUNNING, | ||
batch_v1.JobStatus.State.SCHEDULED, | ||
} | ||
|
||
|
||
@pytest.fixture | ||
def job_name(): | ||
return f"test-job-{uuid.uuid4().hex[:10]}" | ||
|
||
|
||
@pytest.fixture | ||
def instance_template(): | ||
disk = compute_v1.AttachedDisk() | ||
initialize_params = compute_v1.AttachedDiskInitializeParams() | ||
initialize_params.source_image = ( | ||
"projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts" | ||
) | ||
initialize_params.disk_size_gb = 25 | ||
initialize_params.disk_type = 'pd-balanced' | ||
disk.initialize_params = initialize_params | ||
disk.auto_delete = True | ||
disk.boot = True | ||
|
||
network_interface = compute_v1.NetworkInterface() | ||
network_interface.name = "global/networks/default" | ||
|
||
access = compute_v1.AccessConfig() | ||
access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name | ||
access.name = "External NAT" | ||
access.network_tier = access.NetworkTier.PREMIUM.name | ||
network_interface.access_configs = [access] | ||
|
||
template = compute_v1.InstanceTemplate() | ||
template.name = "test-template-" + uuid.uuid4().hex[:10] | ||
template.properties = compute_v1.InstanceProperties() | ||
template.properties.disks = [disk] | ||
template.properties.machine_type = "e2-standard-16" | ||
template.properties.network_interfaces = [network_interface] | ||
|
||
template.properties.scheduling = compute_v1.Scheduling() | ||
template.properties.scheduling.on_host_maintenance = compute_v1.Scheduling.OnHostMaintenance.MIGRATE.name | ||
template.properties.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.STANDARD.name | ||
template.properties.scheduling.automatic_restart = True | ||
|
||
template.properties.service_accounts = [ | ||
{ | ||
"email": f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com", | ||
"scopes": [ | ||
"https://www.googleapis.com/auth/devstorage.read_only", | ||
"https://www.googleapis.com/auth/logging.write", | ||
"https://www.googleapis.com/auth/monitoring.write", | ||
"https://www.googleapis.com/auth/servicecontrol", | ||
"https://www.googleapis.com/auth/service.management.readonly", | ||
"https://www.googleapis.com/auth/trace.append" | ||
] | ||
} | ||
] | ||
|
||
template_client = compute_v1.InstanceTemplatesClient() | ||
operation_client = compute_v1.GlobalOperationsClient() | ||
op = template_client.insert_unary( | ||
project=PROJECT, instance_template_resource=template | ||
) | ||
operation_client.wait(project=PROJECT, operation=op.name) | ||
|
||
template = template_client.get(project=PROJECT, instance_template=template.name) | ||
|
||
yield template | ||
|
||
op = template_client.delete_unary(project=PROJECT, instance_template=template.name) | ||
operation_client.wait(project=PROJECT, operation=op.name) | ||
|
||
|
||
def test_template_job(job_name, instance_template): | ||
job = create_script_job_with_template(PROJECT, REGION, job_name, instance_template.self_link) | ||
_test_body(job) |