diff --git a/google/cloud/orchestration/airflow/service/v1/BUILD.bazel b/google/cloud/orchestration/airflow/service/v1/BUILD.bazel index 9e7d670ca046f..8749f2354547a 100644 --- a/google/cloud/orchestration/airflow/service/v1/BUILD.bazel +++ b/google/cloud/orchestration/airflow/service/v1/BUILD.bazel @@ -72,6 +72,7 @@ java_grpc_library( java_gapic_library( name = "service_java_gapic", srcs = [":service_proto_with_info"], + gapic_yaml = None, grpc_service_config = "composer_grpc_service_config.json", rest_numeric_enums = True, service_yaml = "composer_v1.yaml", @@ -175,12 +176,12 @@ py_gapic_library( name = "service_py_gapic", srcs = [":service_proto"], grpc_service_config = "composer_grpc_service_config.json", - opt_args = [ - "warehouse-package-name=google-cloud-orchestration-airflow", - ], + opt_args = ["warehouse-package-name=google-cloud-orchestration-airflow"], rest_numeric_enums = True, service_yaml = "composer_v1.yaml", transport = "grpc+rest", + deps = [ + ], ) py_test( @@ -224,7 +225,9 @@ php_gapic_library( rest_numeric_enums = True, service_yaml = "composer_v1.yaml", transport = "grpc+rest", - deps = [":service_php_proto"], + deps = [ + ":service_php_proto", + ], ) # Open Source Packages @@ -292,10 +295,10 @@ ruby_cloud_gapic_library( name = "service_ruby_gapic", srcs = [":service_proto_with_info"], extra_protoc_parameters = [ - "ruby-cloud-gem-name=google-cloud-orchestration-airflow-service-v1", - "ruby-cloud-product-url=https://cloud.google.com/composer", "ruby-cloud-api-id=composer.googleapis.com", "ruby-cloud-api-shortname=composer", + "ruby-cloud-gem-name=google-cloud-orchestration-airflow-service-v1", + "ruby-cloud-product-url=https://cloud.google.com/composer", ], grpc_service_config = "composer_grpc_service_config.json", rest_numeric_enums = True, @@ -332,6 +335,7 @@ load( csharp_proto_library( name = "service_csharp_proto", + extra_opts = [], deps = [":service_proto"], ) diff --git a/google/cloud/orchestration/airflow/service/v1/composer_v1.yaml b/google/cloud/orchestration/airflow/service/v1/composer_v1.yaml index 251b0bafd34f2..d5d5691e16e7d 100644 --- a/google/cloud/orchestration/airflow/service/v1/composer_v1.yaml +++ b/google/cloud/orchestration/airflow/service/v1/composer_v1.yaml @@ -13,6 +13,7 @@ types: - name: google.cloud.orchestration.airflow.service.v1.DatabaseFailoverResponse - name: google.cloud.orchestration.airflow.service.v1.ExecuteAirflowCommandResponse - name: google.cloud.orchestration.airflow.service.v1.FetchDatabasePropertiesResponse +- name: google.cloud.orchestration.airflow.service.v1.ListWorkloadsResponse - name: google.cloud.orchestration.airflow.service.v1.LoadSnapshotResponse - name: google.cloud.orchestration.airflow.service.v1.OperationMetadata - name: google.cloud.orchestration.airflow.service.v1.PollAirflowCommandResponse diff --git a/google/cloud/orchestration/airflow/service/v1/environments.proto b/google/cloud/orchestration/airflow/service/v1/environments.proto index e83af80a94ac7..72fbe14f37954 100644 --- a/google/cloud/orchestration/airflow/service/v1/environments.proto +++ b/google/cloud/orchestration/airflow/service/v1/environments.proto @@ -121,6 +121,143 @@ service Environments { }; } + // Lists workloads in a Cloud Composer environment. Workload is a unit that + // runs a single Composer component. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc ListWorkloads(ListWorkloadsRequest) returns (ListWorkloadsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/environments/*}/workloads" + }; + option (google.api.method_signature) = "parent"; + } + + // Creates a user workloads Secret. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc CreateUserWorkloadsSecret(CreateUserWorkloadsSecretRequest) + returns (UserWorkloadsSecret) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsSecrets" + body: "user_workloads_secret" + }; + option (google.api.method_signature) = "parent,user_workloads_secret"; + } + + // Gets an existing user workloads Secret. + // Values of the "data" field in the response are cleared. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc GetUserWorkloadsSecret(GetUserWorkloadsSecretRequest) + returns (UserWorkloadsSecret) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists user workloads Secrets. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc ListUserWorkloadsSecrets(ListUserWorkloadsSecretsRequest) + returns (ListUserWorkloadsSecretsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsSecrets" + }; + option (google.api.method_signature) = "parent"; + } + + // Updates a user workloads Secret. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc UpdateUserWorkloadsSecret(UpdateUserWorkloadsSecretRequest) + returns (UserWorkloadsSecret) { + option (google.api.http) = { + put: "/v1/{user_workloads_secret.name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" + body: "user_workloads_secret" + }; + option (google.api.method_signature) = "user_workloads_secret"; + } + + // Deletes a user workloads Secret. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc DeleteUserWorkloadsSecret(DeleteUserWorkloadsSecretRequest) + returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Creates a user workloads ConfigMap. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc CreateUserWorkloadsConfigMap(CreateUserWorkloadsConfigMapRequest) + returns (UserWorkloadsConfigMap) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsConfigMaps" + body: "user_workloads_config_map" + }; + option (google.api.method_signature) = "parent,user_workloads_config_map"; + } + + // Gets an existing user workloads ConfigMap. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc GetUserWorkloadsConfigMap(GetUserWorkloadsConfigMapRequest) + returns (UserWorkloadsConfigMap) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists user workloads ConfigMaps. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc ListUserWorkloadsConfigMaps(ListUserWorkloadsConfigMapsRequest) + returns (ListUserWorkloadsConfigMapsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsConfigMaps" + }; + option (google.api.method_signature) = "parent"; + } + + // Updates a user workloads ConfigMap. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc UpdateUserWorkloadsConfigMap(UpdateUserWorkloadsConfigMapRequest) + returns (UserWorkloadsConfigMap) { + option (google.api.http) = { + put: "/v1/{user_workloads_config_map.name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" + body: "user_workloads_config_map" + }; + option (google.api.method_signature) = "user_workloads_config_map"; + } + + // Deletes a user workloads ConfigMap. + // + // This method is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + rpc DeleteUserWorkloadsConfigMap(DeleteUserWorkloadsConfigMapRequest) + returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" + }; + option (google.api.method_signature) = "name"; + } + // Creates a snapshots of a Cloud Composer environment. // // As a result of this operation, snapshot of environment's state is stored @@ -468,6 +605,313 @@ message PollAirflowCommandResponse { ExitInfo exit_info = 3; } +// Create user workloads Secret request. +message CreateUserWorkloadsSecretRequest { + // Required. The environment name to create a Secret for, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/Environment" + } + ]; + + // Required. User workloads Secret to create. + UserWorkloadsSecret user_workloads_secret = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Get user workloads Secret request. +message GetUserWorkloadsSecretRequest { + // Required. The resource name of the Secret to get, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/UserWorkloadsSecret" + } + ]; +} + +// List user workloads Secrets request. +message ListUserWorkloadsSecretsRequest { + // Required. List Secrets in the given environment, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/Environment" + } + ]; + + // Optional. The maximum number of Secrets to return. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The next_page_token value returned from a previous List request, + // if any. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Update user workloads Secret request. +message UpdateUserWorkloadsSecretRequest { + // Optional. User workloads Secret to override. + UserWorkloadsSecret user_workloads_secret = 1 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Delete user workloads Secret request. +message DeleteUserWorkloadsSecretRequest { + // Required. The Secret to delete, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/UserWorkloadsSecret" + } + ]; +} + +// Create user workloads ConfigMap request. +message CreateUserWorkloadsConfigMapRequest { + // Required. The environment name to create a ConfigMap for, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/Environment" + } + ]; + + // Required. User workloads ConfigMap to create. + UserWorkloadsConfigMap user_workloads_config_map = 2 + [(google.api.field_behavior) = REQUIRED]; +} + +// Get user workloads ConfigMap request. +message GetUserWorkloadsConfigMapRequest { + // Required. The resource name of the ConfigMap to get, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/UserWorkloadsConfigMap" + } + ]; +} + +// List user workloads ConfigMaps request. +message ListUserWorkloadsConfigMapsRequest { + // Required. List ConfigMaps in the given environment, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/Environment" + } + ]; + + // Optional. The maximum number of ConfigMaps to return. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The next_page_token value returned from a previous List request, + // if any. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +// Update user workloads ConfigMap request. +message UpdateUserWorkloadsConfigMapRequest { + // Optional. User workloads ConfigMap to override. + UserWorkloadsConfigMap user_workloads_config_map = 1 + [(google.api.field_behavior) = OPTIONAL]; +} + +// Delete user workloads ConfigMap request. +message DeleteUserWorkloadsConfigMapRequest { + // Required. The ConfigMap to delete, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/UserWorkloadsConfigMap" + } + ]; +} + +// User workloads Secret used by Airflow tasks that run with Kubernetes executor +// or KubernetesPodOperator. +message UserWorkloadsSecret { + option (google.api.resource) = { + type: "composer.googleapis.com/UserWorkloadsSecret" + pattern: "projects/{project}/locations/{location}/environments/{environment}/userWorkloadsSecrets/{user_workloads_secret}" + plural: "userWorkloadsSecrets" + singular: "userWorkloadsSecret" + }; + + // Identifier. The resource name of the Secret, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" + string name = 1 [(google.api.field_behavior) = IDENTIFIER]; + + // Optional. The "data" field of Kubernetes Secret, organized in key-value + // pairs, which can contain sensitive values such as a password, a token, or a + // key. The values for all keys have to be base64-encoded strings. For details + // see: https://kubernetes.io/docs/concepts/configuration/secret/ + map data = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// The user workloads Secrets for a given environment. +message ListUserWorkloadsSecretsResponse { + // The list of Secrets returned by a ListUserWorkloadsSecretsRequest. + repeated UserWorkloadsSecret user_workloads_secrets = 1; + + // The page token used to query for the next page if one exists. + string next_page_token = 2; +} + +// User workloads ConfigMap used by Airflow tasks that run with Kubernetes +// executor or KubernetesPodOperator. +message UserWorkloadsConfigMap { + option (google.api.resource) = { + type: "composer.googleapis.com/UserWorkloadsConfigMap" + pattern: "projects/{project}/locations/{location}/environments/{environment}/userWorkloadsConfigMaps/{user_workloads_config_map}" + plural: "userWorkloadsConfigMaps" + singular: "userWorkloadsConfigMap" + }; + + // Identifier. The resource name of the ConfigMap, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" + string name = 1 [(google.api.field_behavior) = IDENTIFIER]; + + // Optional. The "data" field of Kubernetes ConfigMap, organized in key-value + // pairs. For details see: + // https://kubernetes.io/docs/concepts/configuration/configmap/ + map data = 2 [(google.api.field_behavior) = OPTIONAL]; +} + +// The user workloads ConfigMaps for a given environment. +message ListUserWorkloadsConfigMapsResponse { + // The list of ConfigMaps returned by a ListUserWorkloadsConfigMapsRequest. + repeated UserWorkloadsConfigMap user_workloads_config_maps = 1; + + // The page token used to query for the next page if one exists. + string next_page_token = 2; +} + +// Request for listing workloads in a Cloud Composer environment. +message ListWorkloadsRequest { + // Required. The environment name to get workloads for, in the form: + // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "composer.googleapis.com/Environment" + } + ]; + + // Optional. The maximum number of environments to return. + int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The next_page_token value returned from a previous List request, + // if any. + string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The list filter. + // Currently only supports equality on the type field. The value of a field + // specified in the filter expression must be one ComposerWorkloadType enum + // option. It's possible to get multiple types using "OR" operator, e.g.: + // "type=SCHEDULER OR type=CELERY_WORKER". If not specified, all items are + // returned. + string filter = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response to ListWorkloadsRequest. +message ListWorkloadsResponse { + // Information about a single workload. + message ComposerWorkload { + // Name of a workload. + string name = 1; + + // Type of a workload. + ComposerWorkloadType type = 2; + + // Output only. Status of a workload. + ComposerWorkloadStatus status = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; + } + + // Supported workload types. + enum ComposerWorkloadType { + // Not able to determine the type of the workload. + COMPOSER_WORKLOAD_TYPE_UNSPECIFIED = 0; + + // Celery worker. + CELERY_WORKER = 1; + + // Kubernetes worker. + KUBERNETES_WORKER = 2; + + // Workload created by Kubernetes Pod Operator. + KUBERNETES_OPERATOR_POD = 3; + + // Airflow scheduler. + SCHEDULER = 4; + + // Airflow Dag processor. + DAG_PROCESSOR = 5; + + // Airflow triggerer. + TRIGGERER = 6; + + // Airflow web server UI. + WEB_SERVER = 7; + + // Redis. + REDIS = 8; + } + + // Workload status. + message ComposerWorkloadStatus { + // Output only. Workload state. + ComposerWorkloadState state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Text to provide more descriptive status. + string status_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Detailed message of the status. + string detailed_status_message = 3 + [(google.api.field_behavior) = OUTPUT_ONLY]; + } + + // Workload states. + enum ComposerWorkloadState { + // Not able to determine the status of the workload. + COMPOSER_WORKLOAD_STATE_UNSPECIFIED = 0; + + // Workload is in pending state and has not yet started. + PENDING = 1; + + // Workload is running fine. + OK = 2; + + // Workload is running but there are some non-critical problems. + WARNING = 3; + + // Workload is not running due to an error. + ERROR = 4; + + // Workload has finished execution with success. + SUCCEEDED = 5; + + // Workload has finished execution with failure. + FAILED = 6; + } + + // The list of environment workloads. + repeated ComposerWorkload workloads = 1; + + // The page token used to query for the next page if one exists. + string next_page_token = 2; +} + // Request to create a snapshot of a Cloud Composer environment. message SaveSnapshotRequest { // The resource name of the source environment in the form: @@ -556,6 +1000,13 @@ message FetchDatabasePropertiesResponse { bool is_failover_replica_available = 3; } +// The configuration for data storage in the environment. +message StorageConfig { + // Optional. The name of the Cloud Storage bucket used by the environment. No + // `gs://` prefix. + string bucket = 1 [(google.api.field_behavior) = OPTIONAL]; +} + // Configuration information for an environment. message EnvironmentConfig { // The size of the Cloud Composer environment. @@ -692,6 +1143,11 @@ message EnvironmentConfig { // This field is supported for Cloud Composer environments in versions // composer-2.2.0-airflow-*.*.* and newer. ResilienceMode resilience_mode = 19 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The configuration setting for Airflow database data retention + // mechanism. + DataRetentionConfig data_retention_config = 21 + [(google.api.field_behavior) = OPTIONAL]; } // Network-level access control policy for the Airflow web server. @@ -726,6 +1182,13 @@ message DatabaseConfig { // Supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string machine_type = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The Compute Engine zone where the Airflow database is created. If + // zone is provided, it must be in the region selected for the environment. If + // zone is not provided, a zone is automatically selected. The zone can only + // be set during environment creation. Supported for Cloud Composer + // environments in versions composer-2.*.*-airflow-*.*.*. + string zone = 2 [(google.api.field_behavior) = OPTIONAL]; } // The configuration settings for the Airflow web server App Engine instance. @@ -784,6 +1247,18 @@ message MaintenanceWindow { // Specifies the selection and configuration of software inside the environment. message SoftwareConfig { + // Web server plugins mode of the Cloud Composer environment. + enum WebServerPluginsMode { + // Default mode. + WEB_SERVER_PLUGINS_MODE_UNSPECIFIED = 0; + + // Web server plugins are not supported. + PLUGINS_DISABLED = 1; + + // Web server plugins are supported. + PLUGINS_ENABLED = 2; + } + // The version of the software running in the environment. // This encapsulates both the version of Cloud Composer functionality and the // version of Apache Airflow. It must match the regular expression @@ -877,6 +1352,18 @@ message SoftwareConfig { // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-2.*.*. int32 scheduler_count = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The configuration for Cloud Data Lineage integration. + CloudDataLineageIntegration cloud_data_lineage_integration = 8 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Whether or not the web server uses custom plugins. + // If unspecified, the field defaults to `PLUGINS_ENABLED`. + // + // This field is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + WebServerPluginsMode web_server_plugins_mode = 10 + [(google.api.field_behavior) = OPTIONAL]; } // Configuration for controlling how IPs are allocated in the @@ -1060,6 +1547,31 @@ message NodeConfig { // See: // https://cloud.google.com/kubernetes-engine/docs/how-to/ip-masquerade-agent bool enable_ip_masq_agent = 11 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Network Attachment that Cloud Composer environment is connected + // to, which provides connectivity with a user's VPC network. Takes precedence + // over network and subnetwork settings. If not provided, but network and + // subnetwork are defined during environment, it will be provisioned. If not + // provided and network and subnetwork are also empty, then connectivity to + // user's VPC network is disabled. Network attachment must be provided in + // format + // projects/{project}/regions/{region}/networkAttachments/{networkAttachment}. + // + // This field is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + string composer_network_attachment = 12 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The IP range in CIDR notation to use internally by Cloud + // Composer. IP addresses are not reserved - and the same range can be used by + // multiple Cloud Composer environments. In case of overlap, IPs from this + // range will not be accessible in the user's VPC network. Cannot be updated. + // If not specified, the default value of '100.64.128.0/20' is used. + // + // This field is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + string composer_internal_ipv4_cidr_block = 13 + [(google.api.field_behavior) = OPTIONAL]; } // Configuration options for the private GKE cluster in a Cloud Composer @@ -1117,6 +1629,17 @@ message PrivateEnvironmentConfig { // composer-1.*.*-airflow-*.*.*. bool enable_private_environment = 1 [(google.api.field_behavior) = OPTIONAL]; + // Optional. If `true`, builds performed during operations that install Python + // packages have only private connectivity to Google services (including + // Artifact Registry) and VPC network (if either `NodeConfig.network` and + // `NodeConfig.subnetwork` fields or `NodeConfig.composer_network_attachment` + // field are specified). If `false`, the builds also have access to the + // internet. + // + // This field is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + bool enable_private_builds_only = 11 [(google.api.field_behavior) = OPTIONAL]; + // Optional. Configuration for the private GKE cluster for a Private IP // Cloud Composer environment. PrivateClusterConfig private_cluster_config = 2 @@ -1234,6 +1757,38 @@ message WorkloadsConfig { int32 max_count = 5 [(google.api.field_behavior) = OPTIONAL]; } + // Configuration for resources used by Airflow triggerers. + message TriggererResource { + // Optional. The number of triggerers. + int32 count = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. CPU request and limit for a single Airflow triggerer replica. + float cpu = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Memory (GB) request and limit for a single Airflow triggerer + // replica. + float memory_gb = 3 [(google.api.field_behavior) = OPTIONAL]; + } + + // Configuration for resources used by Airflow DAG processors. + message DagProcessorResource { + // Optional. CPU request and limit for a single Airflow DAG processor + // replica. + float cpu = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Memory (GB) request and limit for a single Airflow DAG + // processor replica. + float memory_gb = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Storage (GB) request and limit for a single Airflow DAG + // processor replica. + float storage_gb = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The number of DAG processors. If not provided or set to 0, a + // single DAG processor instance will be created. + int32 count = 4 [(google.api.field_behavior) = OPTIONAL]; + } + // Optional. Resources used by Airflow schedulers. SchedulerResource scheduler = 1 [(google.api.field_behavior) = OPTIONAL]; @@ -1242,6 +1797,16 @@ message WorkloadsConfig { // Optional. Resources used by Airflow workers. WorkerResource worker = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Resources used by Airflow triggerers. + TriggererResource triggerer = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Resources used by Airflow DAG processors. + // + // This field is supported for Cloud Composer environments in versions + // composer-3.*.*-airflow-*.*.* and newer. + DagProcessorResource dag_processor = 5 + [(google.api.field_behavior) = OPTIONAL]; } // The Recovery settings of an environment. @@ -1293,6 +1858,12 @@ message MasterAuthorizedNetworksConfig { repeated CidrBlock cidr_blocks = 2; } +// Configuration for Cloud Data Lineage integration. +message CloudDataLineageIntegration { + // Optional. Whether or not Cloud Data Lineage integration is enabled. + bool enabled = 1 [(google.api.field_behavior) = OPTIONAL]; +} + // An environment for running orchestration tasks. message Environment { option (google.api.resource) = { @@ -1354,6 +1925,12 @@ message Environment { // * Both keys and values are additionally constrained to be <= 128 bytes in // size. map labels = 7; + + // Output only. Reserved for future use. + bool satisfies_pzs = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Optional. Storage configuration for this environment. + StorageConfig storage_config = 9 [(google.api.field_behavior) = OPTIONAL]; } // Message containing information about the result of an upgrade check @@ -1390,3 +1967,30 @@ message CheckUpgradeResponse { // when the build was triggered. map pypi_dependencies = 6; } + +// The configuration setting for Airflow database data retention mechanism. +message DataRetentionConfig { + // Optional. The configuration settings for task logs retention + TaskLogsRetentionConfig task_logs_retention_config = 2 + [(google.api.field_behavior) = OPTIONAL]; +} + +// The configuration setting for Task Logs. +message TaskLogsRetentionConfig { + // The definition of task_logs_storage_mode. + enum TaskLogsStorageMode { + // This configuration is not specified by the user. + TASK_LOGS_STORAGE_MODE_UNSPECIFIED = 0; + + // Store task logs in Cloud Logging and in the environment's Cloud Storage + // bucket. + CLOUD_LOGGING_AND_CLOUD_STORAGE = 1; + + // Store task logs in Cloud Logging only. + CLOUD_LOGGING_ONLY = 2; + } + + // Optional. The mode of storage for Airflow workers task logs. For details, + // see go/composer-store-task-logs-in-cloud-logging-only-design-doc + TaskLogsStorageMode storage_mode = 2 [(google.api.field_behavior) = OPTIONAL]; +} diff --git a/google/cloud/orchestration/airflow/service/v1beta1/BUILD.bazel b/google/cloud/orchestration/airflow/service/v1beta1/BUILD.bazel index 7b2c05a8295b1..fe4b5fe29cdb6 100644 --- a/google/cloud/orchestration/airflow/service/v1beta1/BUILD.bazel +++ b/google/cloud/orchestration/airflow/service/v1beta1/BUILD.bazel @@ -72,6 +72,7 @@ java_grpc_library( java_gapic_library( name = "service_java_gapic", srcs = [":service_proto_with_info"], + gapic_yaml = None, grpc_service_config = "composer_grpc_service_config.json", rest_numeric_enums = True, service_yaml = "composer_v1beta1.yaml", @@ -178,6 +179,8 @@ py_gapic_library( rest_numeric_enums = True, service_yaml = "composer_v1beta1.yaml", transport = "grpc+rest", + deps = [ + ], ) py_test( @@ -217,10 +220,13 @@ php_gapic_library( name = "service_php_gapic", srcs = [":service_proto_with_info"], grpc_service_config = "composer_grpc_service_config.json", + migration_mode = "PRE_MIGRATION_SURFACE_ONLY", rest_numeric_enums = True, service_yaml = "composer_v1beta1.yaml", transport = "grpc+rest", - deps = [":service_php_proto"], + deps = [ + ":service_php_proto", + ], ) # Open Source Packages @@ -288,10 +294,10 @@ ruby_cloud_gapic_library( name = "service_ruby_gapic", srcs = [":service_proto_with_info"], extra_protoc_parameters = [ - "ruby-cloud-gem-name=google-cloud-orchestration-airflow-service-v1beta1", - "ruby-cloud-product-url=https://cloud.google.com/composer", "ruby-cloud-api-id=composer.googleapis.com", "ruby-cloud-api-shortname=composer", + "ruby-cloud-gem-name=google-cloud-orchestration-airflow-service-v1beta1", + "ruby-cloud-product-url=https://cloud.google.com/composer", ], grpc_service_config = "composer_grpc_service_config.json", rest_numeric_enums = True, @@ -328,6 +334,7 @@ load( csharp_proto_library( name = "service_csharp_proto", + extra_opts = [], deps = [":service_proto"], ) @@ -364,4 +371,20 @@ csharp_gapic_assembly_pkg( ############################################################################## # C++ ############################################################################## -# Put your C++ rules here +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_grpc_library", + "cc_proto_library", +) + +cc_proto_library( + name = "service_cc_proto", + deps = [":service_proto"], +) + +cc_grpc_library( + name = "service_cc_grpc", + srcs = [":service_proto"], + grpc_only = True, + deps = [":service_cc_proto"], +)