Skip to content

Commit

Permalink
feat: improve json stream writer json to proto conversion speed by
Browse files Browse the repository at this point in the history
caching the schema. This will introduce approximately 2x improvement to
append speed
  • Loading branch information
GaoleMeng committed Jul 11, 2023
1 parent b5ebe90 commit a6f49fa
Show file tree
Hide file tree
Showing 6 changed files with 365 additions and 131 deletions.
5 changes: 5 additions & 0 deletions google-cloud-bigquerystorage/clirr-ignored-differences.xml
Original file line number Diff line number Diff line change
Expand Up @@ -157,5 +157,10 @@
<className>com/google/cloud/bigquery/storage/v1/JsonStreamWriter</className>
<method>boolean isDone()</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/bigquery/storage/v1/ToProtoConverter</className>
<method>java.util.List convertToProtoMessage(com.google.protobuf.Descriptors$Descriptor, com.google.cloud.bigquery.storage.v1.TableSchema, java.lang.Iterable, boolean)</method>
</difference>
</differences>

Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,29 @@ public AppendSerializationError(
}
}

/** This exception is thrown from proto converter to wrap the row index to error mapping. */
static class RowIndexToErrorException extends IllegalArgumentException {
Map<Integer, String> rowIndexToErrorMessage;

boolean hasDataUnknownError;

public RowIndexToErrorException(
Map<Integer, String> rowIndexToErrorMessage, boolean hasDataUnknownError) {
this.rowIndexToErrorMessage = rowIndexToErrorMessage;
this.hasDataUnknownError = hasDataUnknownError;
}

// This message should not be exposed to the user directly.
// Please examine individual row's error through `rowIndexToErrorMessage`.
public String getMessage() {
return "The map of row index to error message is " + rowIndexToErrorMessage.toString();
}

public boolean hasDataUnknownError() {
return hasDataUnknownError;
}
}

/** This exception is used internally to handle field level parsing errors. */
public static class FieldParseError extends IllegalArgumentException {
private final String fieldName;
Expand Down

0 comments on commit a6f49fa

Please sign in to comment.