Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs: update taxonomy display_name comment
feat: added Dataplex specific fields PiperOrigin-RevId: 440386238
- Loading branch information
Google APIs
authored and
Copybara-Service
committed
Apr 8, 2022
1 parent
d4ca15b
commit 99fd8be
Showing
9 changed files
with
243 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -48,4 +48,7 @@ enum IntegratedSystem { | |
|
|
||
| // Dataproc Metastore. | ||
| DATAPROC_METASTORE = 3; | ||
|
|
||
| // Dataplex. | ||
| DATAPLEX = 4; | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| // Copyright 2022 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| syntax = "proto3"; | ||
|
|
||
| package google.cloud.datacatalog.v1; | ||
|
|
||
| import "google/cloud/datacatalog/v1/common.proto"; | ||
| import "google/cloud/datacatalog/v1/physical_schema.proto"; | ||
|
|
||
| option cc_enable_arenas = true; | ||
| option csharp_namespace = "Google.Cloud.DataCatalog.V1"; | ||
| option go_package = "google.golang.org/genproto/googleapis/cloud/datacatalog/v1;datacatalog"; | ||
| option java_multiple_files = true; | ||
| option java_outer_classname = "DataplexSpecProto"; | ||
| option java_package = "com.google.cloud.datacatalog.v1"; | ||
| option php_namespace = "Google\\Cloud\\DataCatalog\\V1"; | ||
| option ruby_package = "Google::Cloud::DataCatalog::V1"; | ||
|
|
||
| // Common Dataplex fields. | ||
| message DataplexSpec { | ||
| // Fully qualified resource name of an asset in Dataplex, to which the | ||
| // underlying data source (Cloud Storage bucket or BigQuery dataset) of the | ||
| // entity is attached. | ||
| string asset = 1; | ||
|
|
||
| // Format of the data. | ||
| PhysicalSchema data_format = 2; | ||
|
|
||
| // Compression format of the data, e.g., zip, gzip etc. | ||
| string compression_format = 3; | ||
|
|
||
| // Project ID of the underlying Cloud Storage or BigQuery data. Note that | ||
| // this may not be the same project as the correspondingly Dataplex lake / | ||
| // zone / asset. | ||
| string project_id = 4; | ||
| } | ||
|
|
||
| // Entry specyfication for a Dataplex fileset. | ||
| message DataplexFilesetSpec { | ||
| // Common Dataplex fields. | ||
| DataplexSpec dataplex_spec = 1; | ||
| } | ||
|
|
||
| // Entry specification for a Dataplex table. | ||
| message DataplexTableSpec { | ||
| // List of external tables registered by Dataplex in other systems based on | ||
| // the same underlying data. | ||
| // | ||
| // External tables allow to query this data in those systems. | ||
| repeated DataplexExternalTable external_tables = 1; | ||
|
|
||
| // Common Dataplex fields. | ||
| DataplexSpec dataplex_spec = 2; | ||
|
|
||
| // Indicates if the table schema is managed by the user or not. | ||
| bool user_managed = 3; | ||
| } | ||
|
|
||
| // External table registered by Dataplex. | ||
| // Dataplex publishes data discovered from an asset into multiple other systems | ||
| // (BigQuery, DPMS) in form of tables. We call them "external tables". External | ||
| // tables are also synced into the Data Catalog. | ||
| // This message contains pointers to | ||
| // those external tables (fully qualified name, resource name et cetera) within | ||
| // the Data Catalog. | ||
| message DataplexExternalTable { | ||
| // Service in which the external table is registered. | ||
| IntegratedSystem system = 1; | ||
|
|
||
| // Fully qualified name (FQN) of the external table. | ||
| string fully_qualified_name = 28; | ||
|
|
||
| // Google Cloud resource name of the external table. | ||
| string google_cloud_resource = 3; | ||
|
|
||
| // Name of the Data Catalog entry representing the external table. | ||
| string data_catalog_entry = 4; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| // Copyright 2022 Google LLC | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| syntax = "proto3"; | ||
|
|
||
| package google.cloud.datacatalog.v1; | ||
|
|
||
| option cc_enable_arenas = true; | ||
| option csharp_namespace = "Google.Cloud.DataCatalog.V1"; | ||
| option go_package = "google.golang.org/genproto/googleapis/cloud/datacatalog/v1;datacatalog"; | ||
| option java_multiple_files = true; | ||
| option java_outer_classname = "PhysicalSchemaProto"; | ||
| option java_package = "com.google.cloud.datacatalog.v1"; | ||
| option php_namespace = "Google\\Cloud\\DataCatalog\\V1"; | ||
| option ruby_package = "Google::Cloud::DataCatalog::V1"; | ||
|
|
||
| // Native schema used by a resource represented as an entry. Used by query | ||
| // engines for deserializing and parsing source data. | ||
| message PhysicalSchema { | ||
| // Schema in Avro JSON format. | ||
| message AvroSchema { | ||
| // JSON source of the Avro schema. | ||
| string text = 1; | ||
| } | ||
|
|
||
| // Schema in Thrift format. | ||
| message ThriftSchema { | ||
| // Thrift IDL source of the schema. | ||
| string text = 1; | ||
| } | ||
|
|
||
| // Schema in protocol buffer format. | ||
| message ProtobufSchema { | ||
| // Protocol buffer source of the schema. | ||
| string text = 1; | ||
| } | ||
|
|
||
| // Marks a Parquet-encoded data source. | ||
| message ParquetSchema { | ||
|
|
||
| } | ||
|
|
||
| // Marks an ORC-encoded data source. | ||
| message OrcSchema { | ||
|
|
||
| } | ||
|
|
||
| // Marks a CSV-encoded data source. | ||
| message CsvSchema { | ||
|
|
||
| } | ||
|
|
||
| oneof schema { | ||
| // Schema in Avro JSON format. | ||
| AvroSchema avro = 1; | ||
|
|
||
| // Schema in Thrift format. | ||
| ThriftSchema thrift = 2; | ||
|
|
||
| // Schema in protocol buffer format. | ||
| ProtobufSchema protobuf = 3; | ||
|
|
||
| // Marks a Parquet-encoded data source. | ||
| ParquetSchema parquet = 4; | ||
|
|
||
| // Marks an ORC-encoded data source. | ||
| OrcSchema orc = 5; | ||
|
|
||
| // Marks a CSV-encoded data source. | ||
| CsvSchema csv = 6; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters