Skip to content
Permalink
Browse files
feat: proto converter library (#100)
* ProtoConverter library

	modified:   google-cloud-bigquerystorage/pom.xml
	new file:   google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java
	new file:   google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java
	new file:   google-cloud-bigquerystorage/src/test/proto/test.proto
	modified:   pom.xml

* Adjust dependency ordering
  • Loading branch information
yirutang committed Mar 15, 2020
1 parent 555ce99 commit 8fbb80eccdfafe8ffd5ff24fe04132878d09d9ae
@@ -16,6 +16,32 @@
<properties>
<site.installationModule>google-cloud-bigquerystorage</site.installationModule>
</properties>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.6.2</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>com.google.protobuf.tools</groupId>
<artifactId>maven-protoc-plugin</artifactId>
<version>0.4.2</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier}</protocArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>test-compile</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>io.grpc</groupId>
@@ -125,6 +151,10 @@
<artifactId>google-cloud-core</artifactId>
<version>${google.core.version}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>

<dependency>
<groupId>com.google.api.grpc</groupId>
@@ -169,4 +199,4 @@
</dependencies>
</profile>
</profiles>
</project>
</project>
@@ -0,0 +1,78 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigquery.storage.v1alpha2;

import com.google.api.gax.grpc.GrpcStatusCode;
import com.google.api.gax.rpc.InvalidArgumentException;
import com.google.cloud.bigquery.storage.v1alpha2.ProtoBufProto.ProtoSchema;
import com.google.protobuf.DescriptorProtos.DescriptorProto;
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.FieldDescriptor;
import io.grpc.Status;
import java.util.*;

// A Converter class that turns a native protobuf::DescriptorProto to a self contained
// protobuf::DescriptorProto
// that can be reconstructed by the backend.
public class ProtoSchemaConverter {
private static class StructName {
public String getName() {
return "__S" + (count++);
}

private int count = 0;
}

private static ProtoSchema convertInternal(
Descriptor input, List<String> visitedTypes, StructName structName) {
DescriptorProto.Builder resultProto = DescriptorProto.newBuilder();
resultProto.setName(structName.getName());
visitedTypes.add(input.getFullName());
for (int i = 0; i < input.getFields().size(); i++) {
FieldDescriptor inputField = input.getFields().get(i);
FieldDescriptorProto.Builder resultField = inputField.toProto().toBuilder();
if (inputField.getType() == FieldDescriptor.Type.GROUP
|| inputField.getType() == FieldDescriptor.Type.MESSAGE) {
if (visitedTypes.contains(inputField.getMessageType().getFullName())) {
throw new InvalidArgumentException(
"Recursive type is not supported:" + inputField.getMessageType().getFullName(),
null,
GrpcStatusCode.of(Status.Code.INVALID_ARGUMENT),
false);
}
resultProto.addNestedType(
convertInternal(inputField.getMessageType(), visitedTypes, structName)
.getProtoDescriptor());
visitedTypes.remove(inputField.getMessageType().getFullName());
resultField.setTypeName(
resultProto.getNestedType(resultProto.getNestedTypeCount() - 1).getName());
}
if (inputField.getType() == FieldDescriptor.Type.ENUM) {
resultProto.addEnumType(inputField.getEnumType().toProto());
resultField.setTypeName(
resultProto.getEnumType(resultProto.getEnumTypeCount() - 1).getName());
}
resultProto.addField(resultField);
}
return ProtoSchema.newBuilder().setProtoDescriptor(resultProto.build()).build();
}

public static ProtoSchema convert(Descriptor descriptor) {
ArrayList<String> visitedTypes = new ArrayList<String>();
return convertInternal(descriptor, visitedTypes, new StructName());
}
}
@@ -0,0 +1,162 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigquery.storage.v1alpha2;

import com.google.api.gax.rpc.InvalidArgumentException;
import com.google.cloud.bigquery.storage.test.Test.*;
import org.junit.*;

public class ProtoSchemaConverterTest {
@Test
public void convertSimple() {
AllSupportedTypes testProto = AllSupportedTypes.newBuilder().setStringValue("abc").build();
ProtoBufProto.ProtoSchema protoSchema =
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
Assert.assertEquals(
"name: \"__S0\"\n"
+ "field {\n"
+ " name: \"int32_value\"\n"
+ " number: 1\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_INT32\n"
+ "}\n"
+ "field {\n"
+ " name: \"int64_value\"\n"
+ " number: 2\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_INT64\n"
+ "}\n"
+ "field {\n"
+ " name: \"uint32_value\"\n"
+ " number: 3\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_UINT32\n"
+ "}\n"
+ "field {\n"
+ " name: \"uint64_value\"\n"
+ " number: 4\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_UINT64\n"
+ "}\n"
+ "field {\n"
+ " name: \"float_value\"\n"
+ " number: 5\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_FLOAT\n"
+ "}\n"
+ "field {\n"
+ " name: \"double_value\"\n"
+ " number: 6\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_DOUBLE\n"
+ "}\n"
+ "field {\n"
+ " name: \"bool_value\"\n"
+ " number: 7\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_BOOL\n"
+ "}\n"
+ "field {\n"
+ " name: \"enum_value\"\n"
+ " number: 8\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_ENUM\n"
+ " type_name: \"TestEnum\"\n"
+ "}\n"
+ "field {\n"
+ " name: \"string_value\"\n"
+ " number: 9\n"
+ " label: LABEL_REQUIRED\n"
+ " type: TYPE_STRING\n"
+ "}\n"
+ "enum_type {\n"
+ " name: \"TestEnum\"\n"
+ " value {\n"
+ " name: \"TestEnum0\"\n"
+ " number: 0\n"
+ " }\n"
+ " value {\n"
+ " name: \"TestEnum1\"\n"
+ " number: 1\n"
+ " }\n"
+ "}\n",
protoSchema.getProtoDescriptor().toString());
}

@Test
public void convertNested() {
ComplicateType testProto = ComplicateType.newBuilder().build();
ProtoBufProto.ProtoSchema protoSchema =
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
Assert.assertEquals(
"name: \"__S0\"\n"
+ "field {\n"
+ " name: \"nested_repeated_type\"\n"
+ " number: 1\n"
+ " label: LABEL_REPEATED\n"
+ " type: TYPE_MESSAGE\n"
+ " type_name: \"__S1\"\n"
+ "}\n"
+ "field {\n"
+ " name: \"inner_type\"\n"
+ " number: 2\n"
+ " label: LABEL_OPTIONAL\n"
+ " type: TYPE_MESSAGE\n"
+ " type_name: \"__S3\"\n"
+ "}\n"
+ "nested_type {\n"
+ " name: \"__S1\"\n"
+ " field {\n"
+ " name: \"inner_type\"\n"
+ " number: 1\n"
+ " label: LABEL_REPEATED\n"
+ " type: TYPE_MESSAGE\n"
+ " type_name: \"__S2\"\n"
+ " }\n"
+ " nested_type {\n"
+ " name: \"__S2\"\n"
+ " field {\n"
+ " name: \"value\"\n"
+ " number: 1\n"
+ " label: LABEL_REPEATED\n"
+ " type: TYPE_STRING\n"
+ " }\n"
+ " }\n"
+ "}\n"
+ "nested_type {\n"
+ " name: \"__S3\"\n"
+ " field {\n"
+ " name: \"value\"\n"
+ " number: 1\n"
+ " label: LABEL_REPEATED\n"
+ " type: TYPE_STRING\n"
+ " }\n"
+ "}\n",
protoSchema.getProtoDescriptor().toString());
}

@Test
public void convertRecursive() {
try {
RecursiveType testProto = RecursiveType.newBuilder().build();
ProtoBufProto.ProtoSchema protoSchema =
ProtoSchemaConverter.convert(testProto.getDescriptorForType());
Assert.fail("No exception raised");
} catch (InvalidArgumentException e) {
// Expected exception
}
}
}
@@ -0,0 +1,56 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
syntax = "proto2";

package com.google.cloud.bigquery.storage.test;

import "google/protobuf/descriptor.proto";

enum TestEnum {
TestEnum0 = 0;
TestEnum1 = 1;
}

message AllSupportedTypes {
optional int32 int32_value = 1;
optional int64 int64_value = 2;
optional uint32 uint32_value = 3;
optional uint64 uint64_value = 4;
optional float float_value = 5;
optional double double_value = 6;
optional bool bool_value = 7;
optional TestEnum enum_value = 8;
required string string_value = 9;
}

message InnerType {
repeated string value = 1;
}
message NestedType {
repeated InnerType inner_type = 1;
}

message ComplicateType {
repeated NestedType nested_repeated_type = 1;
optional InnerType inner_type = 2;
}

message ContainsRecursive {
optional RecursiveType field = 1;
}
message RecursiveType {
optional ContainsRecursive field = 2;
}
19 pom.xml
@@ -61,6 +61,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<project.protobuf-java.version>3.11.4</project.protobuf-java.version>
<github.global.server>github</github.global.server>
<site.installationModule>google-cloud-bigquerystorage-parent</site.installationModule>
<google.core.version>1.93.2</google.core.version>
@@ -78,6 +79,24 @@
<animal-sniffer.version>1.18</animal-sniffer.version>
</properties>

<pluginRepositories>
<pluginRepository>
<releases>
<updatePolicy>never</updatePolicy>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
<name>Central Repository</name>
<url>https://repo.maven.apache.org/maven2</url>
</pluginRepository>
<pluginRepository>
<id>protoc-plugin</id>
<url>https://dl.bintray.com/sergei-ivanov/maven/</url>
</pluginRepository>
</pluginRepositories>

<dependencyManagement>
<dependencies>
<dependency>

0 comments on commit 8fbb80e

Please sign in to comment.