Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 53a9c3a

Browse files
yirutangyoshi-code-bot
yirutang
and
yoshi-code-bot
authored
feat: v1beta2 manual client library for BigQueryWrite (#714)
* feat: v1beta2 manual client library for BigQueryWrite * . * Update google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java Co-authored-by: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com> * Update google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/OnSchemaUpdateRunnable.java Co-authored-by: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com> * . * fix the messed headers * fix the flakiness of a test * fix integration test failure due to API breaking change * remove a failing test * remove a failing test * . * . * . Co-authored-by: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com>
1 parent 55a2cf5 commit 53a9c3a

19 files changed

+8226
-0
lines changed

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigquery.storage.v1beta2;
17+
18+
import com.google.common.base.Preconditions;
19+
import com.google.common.collect.ImmutableList;
20+
import com.google.common.collect.ImmutableMap;
21+
import com.google.protobuf.DescriptorProtos.DescriptorProto;
22+
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
23+
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
24+
import com.google.protobuf.Descriptors;
25+
import com.google.protobuf.Descriptors.Descriptor;
26+
import com.google.protobuf.Descriptors.FileDescriptor;
27+
import java.util.ArrayList;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
31+
/**
32+
* Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase
33+
* when constructing the protobuf descriptor. The mapping between field types and field modes are
34+
* shown in the ImmutableMaps below.
35+
*/
36+
public class BQTableSchemaToProtoDescriptor {
37+
private static ImmutableMap<TableFieldSchema.Mode, FieldDescriptorProto.Label>
38+
BQTableSchemaModeMap =
39+
ImmutableMap.of(
40+
TableFieldSchema.Mode.NULLABLE, FieldDescriptorProto.Label.LABEL_OPTIONAL,
41+
TableFieldSchema.Mode.REPEATED, FieldDescriptorProto.Label.LABEL_REPEATED,
42+
TableFieldSchema.Mode.REQUIRED, FieldDescriptorProto.Label.LABEL_REQUIRED);
43+
44+
private static ImmutableMap<TableFieldSchema.Type, FieldDescriptorProto.Type>
45+
BQTableSchemaTypeMap =
46+
new ImmutableMap.Builder<TableFieldSchema.Type, FieldDescriptorProto.Type>()
47+
.put(TableFieldSchema.Type.BOOL, FieldDescriptorProto.Type.TYPE_BOOL)
48+
.put(TableFieldSchema.Type.BYTES, FieldDescriptorProto.Type.TYPE_BYTES)
49+
.put(TableFieldSchema.Type.DATE, FieldDescriptorProto.Type.TYPE_INT32)
50+
.put(TableFieldSchema.Type.DATETIME, FieldDescriptorProto.Type.TYPE_INT64)
51+
.put(TableFieldSchema.Type.DOUBLE, FieldDescriptorProto.Type.TYPE_DOUBLE)
52+
.put(TableFieldSchema.Type.GEOGRAPHY, FieldDescriptorProto.Type.TYPE_STRING)
53+
.put(TableFieldSchema.Type.INT64, FieldDescriptorProto.Type.TYPE_INT64)
54+
.put(TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_BYTES)
55+
.put(TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
56+
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
57+
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
58+
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
59+
.build();
60+
61+
/**
62+
* Converts TableFieldSchema to a Descriptors.Descriptor object.
63+
*
64+
* @param BQTableSchema
65+
* @throws Descriptors.DescriptorValidationException
66+
*/
67+
public static Descriptor convertBQTableSchemaToProtoDescriptor(TableSchema BQTableSchema)
68+
throws Descriptors.DescriptorValidationException {
69+
Preconditions.checkNotNull(BQTableSchema, "BQTableSchema is null.");
70+
return convertBQTableSchemaToProtoDescriptorImpl(
71+
BQTableSchema, "root", new HashMap<ImmutableList<TableFieldSchema>, Descriptor>());
72+
}
73+
74+
/**
75+
* Converts a TableFieldSchema to a Descriptors.Descriptor object.
76+
*
77+
* @param BQTableSchema
78+
* @param scope Keeps track of current scope to prevent repeated naming while constructing
79+
* descriptor.
80+
* @param dependencyMap Stores already constructed descriptors to prevent reconstruction
81+
* @throws Descriptors.DescriptorValidationException
82+
*/
83+
private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
84+
TableSchema BQTableSchema,
85+
String scope,
86+
HashMap<ImmutableList<TableFieldSchema>, Descriptor> dependencyMap)
87+
throws Descriptors.DescriptorValidationException {
88+
List<FileDescriptor> dependenciesList = new ArrayList<FileDescriptor>();
89+
List<FieldDescriptorProto> fields = new ArrayList<FieldDescriptorProto>();
90+
int index = 1;
91+
for (TableFieldSchema BQTableField : BQTableSchema.getFieldsList()) {
92+
String currentScope = scope + "__" + BQTableField.getName();
93+
if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
94+
ImmutableList<TableFieldSchema> fieldList =
95+
ImmutableList.copyOf(BQTableField.getFieldsList());
96+
if (dependencyMap.containsKey(fieldList)) {
97+
Descriptor descriptor = dependencyMap.get(fieldList);
98+
dependenciesList.add(descriptor.getFile());
99+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, descriptor.getName()));
100+
} else {
101+
Descriptor descriptor =
102+
convertBQTableSchemaToProtoDescriptorImpl(
103+
TableSchema.newBuilder().addAllFields(fieldList).build(),
104+
currentScope,
105+
dependencyMap);
106+
dependenciesList.add(descriptor.getFile());
107+
dependencyMap.put(fieldList, descriptor);
108+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
109+
}
110+
} else {
111+
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
112+
}
113+
}
114+
FileDescriptor[] dependenciesArray = new FileDescriptor[dependenciesList.size()];
115+
dependenciesArray = dependenciesList.toArray(dependenciesArray);
116+
DescriptorProto descriptorProto =
117+
DescriptorProto.newBuilder().setName(scope).addAllField(fields).build();
118+
FileDescriptorProto fileDescriptorProto =
119+
FileDescriptorProto.newBuilder().addMessageType(descriptorProto).build();
120+
FileDescriptor fileDescriptor =
121+
FileDescriptor.buildFrom(fileDescriptorProto, dependenciesArray);
122+
Descriptor descriptor = fileDescriptor.findMessageTypeByName(scope);
123+
return descriptor;
124+
}
125+
126+
/**
127+
* Converts a BQTableField to ProtoField
128+
*
129+
* @param BQTableField BQ Field used to construct a FieldDescriptorProto
130+
* @param index Index for protobuf fields.
131+
* @param scope used to name descriptors
132+
*/
133+
private static FieldDescriptorProto convertBQTableFieldToProtoField(
134+
TableFieldSchema BQTableField, int index, String scope) {
135+
TableFieldSchema.Mode mode = BQTableField.getMode();
136+
String fieldName = BQTableField.getName().toLowerCase();
137+
if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
138+
return FieldDescriptorProto.newBuilder()
139+
.setName(fieldName)
140+
.setTypeName(scope)
141+
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode))
142+
.setNumber(index)
143+
.build();
144+
}
145+
return FieldDescriptorProto.newBuilder()
146+
.setName(fieldName)
147+
.setType((FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()))
148+
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode))
149+
.setNumber(index)
150+
.build();
151+
}
152+
}

0 commit comments

Comments
 (0)