From be2b9c63362b3ef4c38211bb4f98b0d1df449c9f Mon Sep 17 00:00:00 2001 From: Mayur Gubrele Date: Tue, 30 Aug 2022 11:20:11 +0530 Subject: [PATCH 01/11] feat: Bigtable Sink (#33) * feat: add bigtable sink with stringified odpf messages as values * feat: add BIGTABLE_CREDENTIAL_PATH to access google cloud * feat: add wrapper class on bigtable client * refactor: fix checkstyle * feat: add bigtable parser tests * feat: add bigtable sink tests * feat: add bigtable client tests * chore: revert version bump * chore: revert version change in build.gradle Co-authored-by: mayur.gubrele <2310-mayur.gubrele@users.noreply.source.golabs.io> Co-authored-by: lavkesh --- build.gradle | 1 + .../io/odpf/depot/bigtable/BigTableSink.java | 51 +++++++++++ .../depot/bigtable/BigTableSinkFactory.java | 41 +++++++++ .../depot/bigtable/client/BigTableClient.java | 75 ++++++++++++++++ .../depot/bigtable/model/BigTableRecord.java | 15 ++++ .../bigtable/parser/BigTableRecordParser.java | 53 ++++++++++++ .../parser/BigTableResponseParser.java | 15 ++++ .../bigtable/parser/BigTableRowKeyParser.java | 14 +++ .../bigtable/response/BigTableResponse.java | 19 +++++ .../odpf/depot/config/BigTableSinkConfig.java | 18 ++++ .../odpf/depot/bigtable/BigTableSinkTest.java | 85 +++++++++++++++++++ .../bigtable/client/BigTableClientTest.java | 83 ++++++++++++++++++ .../parser/BigTableRecordParserTest.java | 67 +++++++++++++++ .../parser/BigTableRowKeyParserTest.java | 21 +++++ .../org.mockito.plugins.MockMaker | 1 + 15 files changed, 559 insertions(+) create mode 100644 src/main/java/io/odpf/depot/bigtable/BigTableSink.java create mode 100644 src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java create mode 100644 src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java create mode 100644 src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java create mode 100644 src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java create mode 100644 src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java create mode 100644 src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java create mode 100644 src/main/java/io/odpf/depot/bigtable/response/BigTableResponse.java create mode 100644 src/main/java/io/odpf/depot/config/BigTableSinkConfig.java create mode 100644 src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java create mode 100644 src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java create mode 100644 src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java create mode 100644 src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java create mode 100644 src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/build.gradle b/build.gradle index ee5985e0..69aedf61 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation group: 'io.odpf', name: 'stencil', version: '0.2.1' exclude group: 'org.slf4j' implementation group: 'org.aeonbits.owner', name: 'owner', version: '1.0.9' implementation 'com.google.cloud:google-cloud-bigquery:1.115.0' + implementation 'com.google.cloud:google-cloud-bigtable:2.10.0' implementation "io.grpc:grpc-all:1.38.0" implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.35' implementation group: 'redis.clients', name: 'jedis', version: '3.0.1' diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSink.java b/src/main/java/io/odpf/depot/bigtable/BigTableSink.java new file mode 100644 index 00000000..b0cf0f95 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSink.java @@ -0,0 +1,51 @@ +package io.odpf.depot.bigtable; + +import io.odpf.depot.OdpfSink; +import io.odpf.depot.OdpfSinkResponse; +import io.odpf.depot.bigtable.client.BigTableClient; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.parser.BigTableRecordParser; +import io.odpf.depot.bigtable.parser.BigTableResponseParser; +import io.odpf.depot.bigtable.response.BigTableResponse; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.message.OdpfMessage; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class BigTableSink implements OdpfSink { + private final BigTableClient bigTableClient; + private final BigTableRecordParser bigTableRecordParser; + + public BigTableSink(BigTableClient bigTableClient, BigTableRecordParser bigTableRecordParser) { + this.bigTableClient = bigTableClient; + this.bigTableRecordParser = bigTableRecordParser; + } + + @Override + public OdpfSinkResponse pushToSink(List messages) { + List records = bigTableRecordParser.convert(messages); + Map> splitterRecords = records.stream().collect(Collectors.partitioningBy(BigTableRecord::isValid)); + List invalidRecords = splitterRecords.get(Boolean.FALSE); + List validRecords = splitterRecords.get(Boolean.TRUE); + + OdpfSinkResponse odpfSinkResponse = new OdpfSinkResponse(); + invalidRecords.forEach(invalidRecord -> odpfSinkResponse.addErrors(invalidRecord.getIndex(), invalidRecord.getErrorInfo())); + + if (validRecords.size() > 0) { + BigTableResponse bigTableResponse = bigTableClient.send(validRecords); + if (bigTableResponse != null && bigTableResponse.hasErrors()) { + Map errorInfoMap = BigTableResponseParser.parseAndFillOdpfSinkResponse(validRecords, bigTableResponse); + errorInfoMap.forEach(odpfSinkResponse::addErrors); + } + } + + return odpfSinkResponse; + } + + @Override + public void close() throws IOException { + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java new file mode 100644 index 00000000..0349a4d9 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -0,0 +1,41 @@ +package io.odpf.depot.bigtable; + +import io.odpf.depot.OdpfSink; +import io.odpf.depot.bigtable.client.BigTableClient; +import io.odpf.depot.bigtable.parser.BigTableRecordParser; +import io.odpf.depot.bigtable.parser.BigTableRowKeyParser; +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.message.OdpfMessageParser; +import io.odpf.depot.message.OdpfMessageParserFactory; +import io.odpf.depot.metrics.StatsDReporter; + +import java.io.IOException; + +public class BigTableSinkFactory { + private BigTableClient bigTableClient; + private final BigTableSinkConfig sinkConfig; + private final StatsDReporter statsDReporter; + private BigTableRecordParser bigTableRecordParser; + + public BigTableSinkFactory(BigTableSinkConfig sinkConfig, StatsDReporter statsDReporter) { + this.sinkConfig = sinkConfig; + this.statsDReporter = statsDReporter; + } + + public void init() { + try { + OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); + BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); + this.bigTableClient = new BigTableClient(sinkConfig); + this.bigTableRecordParser = new BigTableRecordParser(sinkConfig, odpfMessageParser, bigTableRowKeyParser); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public OdpfSink create() { + return new BigTableSink( + bigTableClient, + bigTableRecordParser); + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java new file mode 100644 index 00000000..030e97a2 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -0,0 +1,75 @@ +package io.odpf.depot.bigtable.client; + +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.MutateRowsException; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.response.BigTableResponse; +import io.odpf.depot.config.BigTableSinkConfig; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.List; + +public class BigTableClient { + private final BigtableTableAdminClient bigtableTableAdminClient; + private final BigtableDataClient bigtableDataClient; + private final BigTableSinkConfig sinkConfig; + + public BigTableClient(BigTableSinkConfig sinkConfig) throws IOException { + this.bigtableDataClient = getBigTableDataClient(); + this.bigtableTableAdminClient = getBigTableAdminClient(); + this.sinkConfig = sinkConfig; + } + + public BigTableClient(BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigTableSinkConfig sinkConfig) { + this.bigtableDataClient = bigtableDataClient; + this.bigtableTableAdminClient = bigtableTableAdminClient; + this.sinkConfig = sinkConfig; + } + + private BigtableDataClient getBigTableDataClient() throws IOException { + BigtableDataSettings settings = BigtableDataSettings.newBuilder() + .setProjectId(sinkConfig.getGCloudProjectID()) + .setInstanceId(sinkConfig.getBigtableInstanceId()) + .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getBigTableCredentialPath())))) + .build(); + return BigtableDataClient.create(settings); + } + + private BigtableTableAdminClient getBigTableAdminClient() throws IOException { + BigtableTableAdminSettings settings = BigtableTableAdminSettings.newBuilder() + .setProjectId(sinkConfig.getGCloudProjectID()) + .setInstanceId(sinkConfig.getBigtableInstanceId()) + .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getBigTableCredentialPath())))) + .build(); + + return BigtableTableAdminClient.create(settings); + } + + public BigTableResponse send(List records) throws MutateRowsException { + BulkMutation batch = BulkMutation.create(sinkConfig.getTableId()); + BigTableResponse bigTableResponse = null; + for (BigTableRecord record : records) { + batch.add(record.getRowMutationEntry()); + } + try { + bigtableDataClient.bulkMutateRows(batch); + } catch (MutateRowsException e) { + List failedMutations = e.getFailedMutations(); + bigTableResponse = new BigTableResponse(failedMutations); + } + + return bigTableResponse; + } + + public boolean tableExists(String tableId) { + return bigtableTableAdminClient.exists(tableId); + } + +} diff --git a/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java b/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java new file mode 100644 index 00000000..dcb28c66 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java @@ -0,0 +1,15 @@ +package io.odpf.depot.bigtable.model; + +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import io.odpf.depot.error.ErrorInfo; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +@Getter +public class BigTableRecord { + private final RowMutationEntry rowMutationEntry; + private final long index; + private final ErrorInfo errorInfo; + private final boolean valid; +} diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java new file mode 100644 index 00000000..5a9b9629 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -0,0 +1,53 @@ +package io.odpf.depot.bigtable.parser; + +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageParser; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; + +import java.util.ArrayList; +import java.util.List; + +public class BigTableRecordParser { + private final BigTableSinkConfig sinkConfig; + private final OdpfMessageParser odpfMessageParser; + private final BigTableRowKeyParser bigTableRowKeyParser; + + public BigTableRecordParser(BigTableSinkConfig sinkConfig, OdpfMessageParser odpfMessageParser, BigTableRowKeyParser bigTableRowKeyParser) { + this.sinkConfig = sinkConfig; + this.odpfMessageParser = odpfMessageParser; + this.bigTableRowKeyParser = bigTableRowKeyParser; + } + + public List convert(List messages) { + ArrayList records = new ArrayList<>(); + for (int index = 0; index < messages.size(); index++) { + OdpfMessage message = messages.get(index); + BigTableRecord record = createRecord(message, index); + records.add(record); + } + return records; + } + + private BigTableRecord createRecord(OdpfMessage message, long index) { + SinkConnectorSchemaMessageMode mode = sinkConfig.getSinkConnectorSchemaMessageMode(); + String schemaClass = mode == SinkConnectorSchemaMessageMode.LOG_MESSAGE + ? sinkConfig.getSinkConnectorSchemaProtoMessageClass() : sinkConfig.getSinkConnectorSchemaProtoKeyClass(); + try { + ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, mode, schemaClass); + String rowKey = bigTableRowKeyParser.parse(sinkConfig.getRowKeyTemplate(), message); + RowMutationEntry rowMutationEntry = RowMutationEntry + .create(rowKey) + .setCell("family-test", "odpf-message", parsedOdpfMessage.toString()); + return new BigTableRecord(rowMutationEntry, index, null, true); + } catch (Exception e) { + ErrorInfo errorInfo = new ErrorInfo(e, ErrorType.DESERIALIZATION_ERROR); + return new BigTableRecord(null, index, errorInfo, false); + } + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java new file mode 100644 index 00000000..47f7e4c1 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java @@ -0,0 +1,15 @@ +package io.odpf.depot.bigtable.parser; + +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.response.BigTableResponse; +import io.odpf.depot.error.ErrorInfo; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class BigTableResponseParser { + public static Map parseAndFillOdpfSinkResponse(List validRecords, BigTableResponse bigTableResponse) { + return new HashMap<>(); + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java new file mode 100644 index 00000000..de9d453c --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java @@ -0,0 +1,14 @@ +package io.odpf.depot.bigtable.parser; + +import io.odpf.depot.message.OdpfMessage; + +import java.util.Random; + +public class BigTableRowKeyParser { + public String parse(String rowKeyTemplate, OdpfMessage message) { + Random rand = new Random(); + int n = rand.nextInt(); + System.out.println("Rowkey created: key-test-" + n); + return "key-test-" + n; + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/response/BigTableResponse.java b/src/main/java/io/odpf/depot/bigtable/response/BigTableResponse.java new file mode 100644 index 00000000..bb56875a --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/response/BigTableResponse.java @@ -0,0 +1,19 @@ +package io.odpf.depot.bigtable.response; + +import com.google.cloud.bigtable.data.v2.models.MutateRowsException; +import lombok.Getter; + +import java.util.List; + +@Getter +public class BigTableResponse { + private final List failedMutations; + + public BigTableResponse(List failedMutations) { + this.failedMutations = failedMutations; + } + + public boolean hasErrors() { + return !failedMutations.isEmpty(); + } +} diff --git a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java new file mode 100644 index 00000000..1d2941c6 --- /dev/null +++ b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java @@ -0,0 +1,18 @@ +package io.odpf.depot.config; + +public interface BigTableSinkConfig extends OdpfSinkConfig { + @Key("SINK_BIGTABLE_GOOGLE_CLOUD_PROJECT_ID") + String getGCloudProjectID(); + + @Key("SINK_BIGTABLE_INSTANCE_ID") + String getBigtableInstanceId(); + + @Key("SINK_BIGTABLE_TABLE_ID") + String getTableId(); + + @Key("SINK_BIGTABLE_CREDENTIAL_PATH") + String getBigTableCredentialPath(); + + @Key("ROW_KEY_TEMPLATE") + String getRowKeyTemplate(); +} diff --git a/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java b/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java new file mode 100644 index 00000000..7af46fb0 --- /dev/null +++ b/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java @@ -0,0 +1,85 @@ +package io.odpf.depot.bigtable; + +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import io.odpf.depot.OdpfSinkResponse; +import io.odpf.depot.TestBookingLogKey; +import io.odpf.depot.TestBookingLogMessage; +import io.odpf.depot.TestServiceType; +import io.odpf.depot.bigtable.client.BigTableClient; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.parser.BigTableRecordParser; +import io.odpf.depot.error.ErrorInfo; +import io.odpf.depot.error.ErrorType; +import io.odpf.depot.message.OdpfMessage; +import org.aeonbits.owner.util.Collections; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +import java.util.List; + +public class BigTableSinkTest { + + @Mock + private BigTableRecordParser bigTableRecordParser; + @Mock + private BigTableClient bigTableClient; + + private BigTableSink bigTableSink; + private List messages; + private List validRecords; + private List invalidRecords; + private ErrorInfo errorInfo; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); + TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); + TestBookingLogKey bookingLogKey2 = TestBookingLogKey.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").build(); + TestBookingLogMessage bookingLogMessage2 = TestBookingLogMessage.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").setServiceType(TestServiceType.Enum.GO_SHOP).build(); + + OdpfMessage message1 = new OdpfMessage(bookingLogKey1.toByteArray(), bookingLogMessage1.toByteArray()); + OdpfMessage message2 = new OdpfMessage(bookingLogKey2.toByteArray(), bookingLogMessage2.toByteArray()); + messages = Collections.list(message1, message2); + + RowMutationEntry rowMutationEntry = RowMutationEntry.create("rowKey").setCell("family", "qualifier", "value"); + BigTableRecord bigTableRecord1 = new BigTableRecord(rowMutationEntry, 1, null, true); + BigTableRecord bigTableRecord2 = new BigTableRecord(rowMutationEntry, 2, null, true); + validRecords = Collections.list(bigTableRecord1, bigTableRecord2); + + errorInfo = new ErrorInfo(new Exception("test-exception-message"), ErrorType.DEFAULT_ERROR); + BigTableRecord bigTableRecord3 = new BigTableRecord(null, 3, errorInfo, false); + BigTableRecord bigTableRecord4 = new BigTableRecord(null, 4, errorInfo, false); + invalidRecords = Collections.list(bigTableRecord3, bigTableRecord4); + + bigTableSink = new BigTableSink(bigTableClient, bigTableRecordParser); + } + + @Test + public void shouldSendValidBigTableRecordsToBigTableSink() { + Mockito.when(bigTableRecordParser.convert(messages)).thenReturn(validRecords); + Mockito.when(bigTableClient.send(validRecords)).thenReturn(null); + + OdpfSinkResponse response = bigTableSink.pushToSink(messages); + + Mockito.verify(bigTableClient, Mockito.times(1)).send(validRecords); + Assert.assertEquals(0, response.getErrors().size()); + } + + @Test + public void shouldAddErrorsFromInvalidRecordsToOdpfResponse() { + Mockito.when(bigTableRecordParser.convert(messages)).thenReturn(invalidRecords); + + OdpfSinkResponse response = bigTableSink.pushToSink(messages); + + Mockito.verify(bigTableClient, Mockito.times(0)).send(validRecords); + Assert.assertTrue(response.hasErrors()); + Assert.assertEquals(2, response.getErrors().size()); + Assert.assertEquals(errorInfo, response.getErrorsFor(3)); + Assert.assertEquals(errorInfo, response.getErrorsFor(4)); + } +} diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java new file mode 100644 index 00000000..8f45fb8e --- /dev/null +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -0,0 +1,83 @@ +package io.odpf.depot.bigtable.client; + +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.MutateRowsException; +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.response.BigTableResponse; +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import org.aeonbits.owner.ConfigFactory; +import org.aeonbits.owner.util.Collections; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; + +public class BigTableClientTest { + + @Mock + private BigtableDataClient bigTableDataClient; + @Mock + private BigtableTableAdminClient bigtableTableAdminClient; + @Mock + private ApiException apiException; + + private BigTableClient bigTableClient; + private List validRecords; + + @Before + public void setUp() throws IOException { + MockitoAnnotations.openMocks(this); + System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage"); + System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE)); + System.setProperty("SINK_BIGTABLE_GOOGLE_CLOUD_PROJECT_ID", "test-gcloud-project"); + System.setProperty("SINK_BIGTABLE_INSTANCE_ID", "test-instance"); + System.setProperty("SINK_BIGTABLE_TABLE_ID", "test-table"); + System.setProperty("SINK_BIGTABLE_CREDENTIAL_PATH", "Users/github/bigtable/test-credential"); + + RowMutationEntry rowMutationEntry = RowMutationEntry.create("rowKey").setCell("family", "qualifier", "value"); + BigTableRecord bigTableRecord1 = new BigTableRecord(rowMutationEntry, 1, null, true); + BigTableRecord bigTableRecord2 = new BigTableRecord(rowMutationEntry, 2, null, true); + validRecords = Collections.list(bigTableRecord1, bigTableRecord2); + + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + bigTableClient = new BigTableClient(bigTableDataClient, bigtableTableAdminClient, sinkConfig); + } + + @Test + public void shouldReturnNullBigTableResponseWhenBulkMutateRowsDoesNotThrowAnException() { + doNothing().when(bigTableDataClient).bulkMutateRows(isA(BulkMutation.class)); + + BigTableResponse bigTableResponse = bigTableClient.send(validRecords); + + Assert.assertNull(bigTableResponse); + } + + @Test + public void shouldReturnBigTableResponseWithFailedMutationsWhenBulkMutateRowsThrowsMutateRowsException() { + List failedMutations = new ArrayList<>(); + failedMutations.add(MutateRowsException.FailedMutation.create(0, apiException)); + failedMutations.add(MutateRowsException.FailedMutation.create(1, apiException)); + MutateRowsException mutateRowsException = new MutateRowsException(null, failedMutations, false); + + doThrow(mutateRowsException).when(bigTableDataClient).bulkMutateRows(isA(BulkMutation.class)); + + BigTableResponse bigTableResponse = bigTableClient.send(validRecords); + + Assert.assertTrue(bigTableResponse.hasErrors()); + Assert.assertEquals(2, bigTableResponse.getFailedMutations().size()); + } +} diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java new file mode 100644 index 00000000..f0f75a22 --- /dev/null +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java @@ -0,0 +1,67 @@ +package io.odpf.depot.bigtable.parser; + +import io.odpf.depot.TestBookingLogKey; +import io.odpf.depot.TestBookingLogMessage; +import io.odpf.depot.TestServiceType; +import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.stencil.client.ClassLoadStencilClient; +import org.aeonbits.owner.ConfigFactory; +import org.aeonbits.owner.util.Collections; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +import java.util.List; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.CALLS_REAL_METHODS; + +public class BigTableRecordParserTest { + + @Mock + private ClassLoadStencilClient stencilClient; + private BigTableRecordParser bigTableRecordParser; + private List messages; + + @Before + public void setUp() { + System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage"); + System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE)); + + TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); + TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); + TestBookingLogKey bookingLogKey2 = TestBookingLogKey.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").build(); + TestBookingLogMessage bookingLogMessage2 = TestBookingLogMessage.newBuilder().setOrderNumber("order#2").setOrderUrl("order-url#2").setServiceType(TestServiceType.Enum.GO_SHOP).build(); + + OdpfMessage message1 = new OdpfMessage(bookingLogKey1.toByteArray(), bookingLogMessage1.toByteArray()); + OdpfMessage message2 = new OdpfMessage(bookingLogKey2.toByteArray(), bookingLogMessage2.toByteArray()); + messages = Collections.list(message1, message2); + + stencilClient = Mockito.mock(ClassLoadStencilClient.class, CALLS_REAL_METHODS); + ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + + bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser()); + } + + @Test + public void shouldReturnValidRecordsForListOfValidOdpfMessages() { + List records = bigTableRecordParser.convert(messages); + assertTrue(records.get(0).isValid()); + assertTrue(records.get(1).isValid()); + assertNull(records.get(0).getErrorInfo()); + assertNull(records.get(1).getErrorInfo()); + } + + @Test + public void shouldReturnInvalidRecordForAnyNullOdpfMessage() { + List records = bigTableRecordParser.convert(Collections.list(new OdpfMessage(null, null))); + assertFalse(records.get(0).isValid()); + assertNotNull(records.get(0).getErrorInfo()); + } +} diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java new file mode 100644 index 00000000..532fc04b --- /dev/null +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java @@ -0,0 +1,21 @@ +package io.odpf.depot.bigtable.parser; + +import io.odpf.depot.TestBookingLogKey; +import io.odpf.depot.TestBookingLogMessage; +import io.odpf.depot.TestServiceType; +import io.odpf.depot.message.OdpfMessage; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class BigTableRowKeyParserTest { + + @Test + public void shouldReturnParsedRowKey() { + BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); + TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); + TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); + String parsedRowKey = bigTableRowKeyParser.parse("template", new OdpfMessage(bookingLogKey1, bookingLogMessage1)); + assertTrue(parsedRowKey.contains("key-test-")); + } +} diff --git a/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000..ca6ee9ce --- /dev/null +++ b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file From 3cc97b247d1255b293f91de7e4d652a866d0ec32 Mon Sep 17 00:00:00 2001 From: lavkesh Date: Wed, 14 Sep 2022 17:24:56 +0800 Subject: [PATCH 02/11] chore: qa and review fixes --- build.gradle | 2 +- .../depot/bigtable/BigTableSinkFactory.java | 10 ++++++++-- .../depot/bigtable/client/BigTableClient.java | 20 +++++++++---------- .../odpf/depot/config/BigTableSinkConfig.java | 6 +++--- .../bigtable/client/BigTableClientTest.java | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 69aedf61..5dfa1bc7 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation group: 'io.odpf', name: 'stencil', version: '0.2.1' exclude group: 'org.slf4j' implementation group: 'org.aeonbits.owner', name: 'owner', version: '1.0.9' implementation 'com.google.cloud:google-cloud-bigquery:1.115.0' - implementation 'com.google.cloud:google-cloud-bigtable:2.10.0' + implementation('com.google.cloud:google-cloud-bigtable:2.11.2') implementation "io.grpc:grpc-all:1.38.0" implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.35' implementation group: 'redis.clients', name: 'jedis', version: '3.0.1' diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 0349a4d9..71ad99ce 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -1,5 +1,6 @@ package io.odpf.depot.bigtable; +import com.timgroup.statsd.NoOpStatsDClient; import io.odpf.depot.OdpfSink; import io.odpf.depot.bigtable.client.BigTableClient; import io.odpf.depot.bigtable.parser.BigTableRecordParser; @@ -12,9 +13,9 @@ import java.io.IOException; public class BigTableSinkFactory { - private BigTableClient bigTableClient; private final BigTableSinkConfig sinkConfig; private final StatsDReporter statsDReporter; + private BigTableClient bigTableClient; private BigTableRecordParser bigTableRecordParser; public BigTableSinkFactory(BigTableSinkConfig sinkConfig, StatsDReporter statsDReporter) { @@ -22,6 +23,11 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig, StatsDReporter statsDR this.statsDReporter = statsDReporter; } + public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { + this(sinkConfig, new StatsDReporter(new NoOpStatsDClient())); + } + + public void init() { try { OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); @@ -29,7 +35,7 @@ public void init() { this.bigTableClient = new BigTableClient(sinkConfig); this.bigTableRecordParser = new BigTableRecordParser(sinkConfig, odpfMessageParser, bigTableRowKeyParser); } catch (IOException e) { - e.printStackTrace(); + throw new IllegalArgumentException("Exception occurred while creating sink", e); } } diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java index 030e97a2..d82f4dd5 100644 --- a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -22,31 +22,29 @@ public class BigTableClient { private final BigTableSinkConfig sinkConfig; public BigTableClient(BigTableSinkConfig sinkConfig) throws IOException { - this.bigtableDataClient = getBigTableDataClient(); - this.bigtableTableAdminClient = getBigTableAdminClient(); - this.sinkConfig = sinkConfig; + this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig)); } - public BigTableClient(BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigTableSinkConfig sinkConfig) { + public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient) { + this.sinkConfig = sinkConfig; this.bigtableDataClient = bigtableDataClient; this.bigtableTableAdminClient = bigtableTableAdminClient; - this.sinkConfig = sinkConfig; } - private BigtableDataClient getBigTableDataClient() throws IOException { + private static BigtableDataClient getBigTableDataClient(BigTableSinkConfig sinkConfig) throws IOException { BigtableDataSettings settings = BigtableDataSettings.newBuilder() .setProjectId(sinkConfig.getGCloudProjectID()) - .setInstanceId(sinkConfig.getBigtableInstanceId()) - .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getBigTableCredentialPath())))) + .setInstanceId(sinkConfig.getInstanceId()) + .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getCredentialPath())))) .build(); return BigtableDataClient.create(settings); } - private BigtableTableAdminClient getBigTableAdminClient() throws IOException { + private static BigtableTableAdminClient getBigTableAdminClient(BigTableSinkConfig sinkConfig) throws IOException { BigtableTableAdminSettings settings = BigtableTableAdminSettings.newBuilder() .setProjectId(sinkConfig.getGCloudProjectID()) - .setInstanceId(sinkConfig.getBigtableInstanceId()) - .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getBigTableCredentialPath())))) + .setInstanceId(sinkConfig.getInstanceId()) + .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getCredentialPath())))) .build(); return BigtableTableAdminClient.create(settings); diff --git a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java index 1d2941c6..172bff39 100644 --- a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java @@ -5,14 +5,14 @@ public interface BigTableSinkConfig extends OdpfSinkConfig { String getGCloudProjectID(); @Key("SINK_BIGTABLE_INSTANCE_ID") - String getBigtableInstanceId(); + String getInstanceId(); @Key("SINK_BIGTABLE_TABLE_ID") String getTableId(); @Key("SINK_BIGTABLE_CREDENTIAL_PATH") - String getBigTableCredentialPath(); + String getCredentialPath(); - @Key("ROW_KEY_TEMPLATE") + @Key("SINK_BIGTABLE_ROW_KEY_TEMPLATE") String getRowKeyTemplate(); } diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index 8f45fb8e..4459d8d7 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -54,7 +54,7 @@ public void setUp() throws IOException { validRecords = Collections.list(bigTableRecord1, bigTableRecord2); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigTableClient = new BigTableClient(bigTableDataClient, bigtableTableAdminClient, sinkConfig); + bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient); } @Test From 342a755b90e6c1d0655ff201c8b13b14f185cab7 Mon Sep 17 00:00:00 2001 From: lavkesh Date: Thu, 15 Sep 2022 18:47:52 +0800 Subject: [PATCH 03/11] chore: change default SCHEMA_REGISTRY_STENCIL_CACHE_AUTO_REFRESH to true --- src/main/java/io/odpf/depot/config/OdpfSinkConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java index 277151e1..b2bc1c61 100644 --- a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java @@ -42,7 +42,7 @@ public interface OdpfSinkConfig extends Config { List
getSchemaRegistryStencilFetchHeaders(); @Key("SCHEMA_REGISTRY_STENCIL_CACHE_AUTO_REFRESH") - @DefaultValue("false") + @DefaultValue("true") Boolean getSchemaRegistryStencilCacheAutoRefresh(); @Key("SCHEMA_REGISTRY_STENCIL_CACHE_TTL_MS") From fb787d6ecaa45c552fbc34faecb12428ab411937 Mon Sep 17 00:00:00 2001 From: Mayur Gubrele Date: Mon, 26 Sep 2022 15:19:47 +0530 Subject: [PATCH 04/11] feat: Bigtable record parser (#39) * feat: Bigtable Sink (#33) * feat: add bigtable sink with stringified odpf messages as values * feat: add BIGTABLE_CREDENTIAL_PATH to access google cloud * feat: add wrapper class on bigtable client * refactor: fix checkstyle * feat: add bigtable parser tests * feat: add bigtable sink tests * feat: add bigtable client tests * chore: revert version bump * chore: revert version change in build.gradle Co-authored-by: mayur.gubrele <2310-mayur.gubrele@users.noreply.source.golabs.io> Co-authored-by: lavkesh * feat: create bigtable records using InputOutputFieldMapping provided as configuration * refactor: fix checkstyle and add unit tests * review: minor refactor * refactor: add BigTableSchemaTest and fix BigTableRecordParserTest * refactor: fix checkstyle * tests: add few more tests Co-authored-by: mayur.gubrele <2310-mayur.gubrele@users.noreply.source.golabs.io> Co-authored-by: lavkesh --- build.gradle | 2 +- .../depot/bigtable/BigTableSinkFactory.java | 19 ++- .../depot/bigtable/client/BigTableClient.java | 33 ++++- .../BigTableInvalidSchemaException.java | 11 ++ .../depot/bigtable/model/BigtableSchema.java | 34 +++++ .../bigtable/parser/BigTableRecordParser.java | 32 +++-- .../odpf/depot/config/BigTableSinkConfig.java | 3 + .../io/odpf/depot/config/OdpfSinkConfig.java | 6 +- .../exception/ConfigurationException.java | 4 + .../bigtable/client/BigTableClientTest.java | 36 +++++- .../bigtable/model/BigtableSchemaTest.java | 116 ++++++++++++++++++ .../parser/BigTableRecordParserTest.java | 17 ++- 12 files changed, 288 insertions(+), 25 deletions(-) create mode 100644 src/main/java/io/odpf/depot/bigtable/exception/BigTableInvalidSchemaException.java create mode 100644 src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java create mode 100644 src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java diff --git a/build.gradle b/build.gradle index 5dfa1bc7..05a43bbe 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation group: 'io.odpf', name: 'stencil', version: '0.2.1' exclude group: 'org.slf4j' implementation group: 'org.aeonbits.owner', name: 'owner', version: '1.0.9' implementation 'com.google.cloud:google-cloud-bigquery:1.115.0' - implementation('com.google.cloud:google-cloud-bigtable:2.11.2') + implementation 'com.google.cloud:google-cloud-bigtable:2.11.2' implementation "io.grpc:grpc-all:1.38.0" implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.35' implementation group: 'redis.clients', name: 'jedis', version: '3.0.1' diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 71ad99ce..9b28c219 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -5,10 +5,15 @@ import io.odpf.depot.bigtable.client.BigTableClient; import io.odpf.depot.bigtable.parser.BigTableRecordParser; import io.odpf.depot.bigtable.parser.BigTableRowKeyParser; +import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.exception.ConfigurationException; import io.odpf.depot.message.OdpfMessageParser; import io.odpf.depot.message.OdpfMessageParserFactory; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; import io.odpf.depot.metrics.StatsDReporter; +import io.odpf.depot.utils.MessageConfigUtils; import java.io.IOException; @@ -30,12 +35,20 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { public void init() { try { - OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); this.bigTableClient = new BigTableClient(sinkConfig); - this.bigTableRecordParser = new BigTableRecordParser(sinkConfig, odpfMessageParser, bigTableRowKeyParser); + bigTableClient.validateBigTableSchema(); + Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); + OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); + OdpfMessageSchema schema = odpfMessageParser.getSchema(modeAndSchema.getSecond()); + this.bigTableRecordParser = new BigTableRecordParser( + sinkConfig, + odpfMessageParser, + bigTableRowKeyParser, + modeAndSchema, + schema); } catch (IOException e) { - throw new IllegalArgumentException("Exception occurred while creating sink", e); + throw new ConfigurationException("Exception occurred while creating sink", e); } } diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java index d82f4dd5..3d9694ed 100644 --- a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -4,17 +4,22 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.models.BulkMutation; import com.google.cloud.bigtable.data.v2.models.MutateRowsException; +import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; +import org.json.JSONObject; import java.io.FileInputStream; import java.io.IOException; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; public class BigTableClient { private final BigtableTableAdminClient bigtableTableAdminClient; @@ -46,13 +51,12 @@ private static BigtableTableAdminClient getBigTableAdminClient(BigTableSinkConfi .setInstanceId(sinkConfig.getInstanceId()) .setCredentialsProvider(FixedCredentialsProvider.create(GoogleCredentials.fromStream(new FileInputStream(sinkConfig.getCredentialPath())))) .build(); - return BigtableTableAdminClient.create(settings); } public BigTableResponse send(List records) throws MutateRowsException { - BulkMutation batch = BulkMutation.create(sinkConfig.getTableId()); BigTableResponse bigTableResponse = null; + BulkMutation batch = BulkMutation.create(sinkConfig.getTableId()); for (BigTableRecord record : records) { batch.add(record.getRowMutationEntry()); } @@ -66,8 +70,29 @@ public BigTableResponse send(List records) throws MutateRowsExce return bigTableResponse; } - public boolean tableExists(String tableId) { - return bigtableTableAdminClient.exists(tableId); + public void validateBigTableSchema() throws BigTableInvalidSchemaException { + this.tableExists(sinkConfig.getTableId()); + this.columnFamiliesExist(sinkConfig.getColumnFamilyMapping(), sinkConfig.getTableId()); } + private void tableExists(String tableId) throws BigTableInvalidSchemaException { + if (!bigtableTableAdminClient.exists(tableId)) { + throw new BigTableInvalidSchemaException(String.format("Table: %s does not exist", tableId)); + } + } + + private void columnFamiliesExist(String inputOutputFieldMapping, String tableId) throws BigTableInvalidSchemaException { + List existingColumnFamilies = bigtableTableAdminClient.getTable(tableId) + .getColumnFamilies() + .stream() + .map(ColumnFamily::getId) + .collect(Collectors.toList()); + + Set columnFamilies = new JSONObject(inputOutputFieldMapping).keySet(); + for (String columnFamily : columnFamilies) { + if (!existingColumnFamilies.contains(columnFamily)) { + throw new BigTableInvalidSchemaException(String.format("Column family: %s does not exist!", columnFamily)); + } + } + } } diff --git a/src/main/java/io/odpf/depot/bigtable/exception/BigTableInvalidSchemaException.java b/src/main/java/io/odpf/depot/bigtable/exception/BigTableInvalidSchemaException.java new file mode 100644 index 00000000..fba1a8b7 --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/exception/BigTableInvalidSchemaException.java @@ -0,0 +1,11 @@ +package io.odpf.depot.bigtable.exception; + +public class BigTableInvalidSchemaException extends RuntimeException { + public BigTableInvalidSchemaException(String message, Throwable cause) { + super(message, cause); + } + + public BigTableInvalidSchemaException(String messsage) { + super(messsage); + } +} diff --git a/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java b/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java new file mode 100644 index 00000000..0f857e6b --- /dev/null +++ b/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java @@ -0,0 +1,34 @@ +package io.odpf.depot.bigtable.model; + +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import org.json.JSONObject; + +import java.util.Set; + +public class BigtableSchema { + + private final JSONObject columnFamilyMapping; + + public BigtableSchema(BigTableSinkConfig sinkConfig) { + String columnMapping = sinkConfig.getColumnFamilyMapping(); + if (columnMapping == null || columnMapping.isEmpty()) { + throw new ConfigurationException("Column Mapping should not be empty or null"); + } + this.columnFamilyMapping = new JSONObject(sinkConfig.getColumnFamilyMapping()); + } + + public String getField(String columnFamily, String columnName) { + JSONObject columns = columnFamilyMapping.getJSONObject(columnFamily); + return columns.getString(columnName); + } + + public Set getColumnFamilies() { + return columnFamilyMapping.keySet(); + } + + public Set getColumns(String family) { + return columnFamilyMapping.getJSONObject(family).keySet(); + } + +} diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java index 5a9b9629..6c0ac169 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -2,11 +2,14 @@ import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.model.BigtableSchema; +import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.error.ErrorInfo; import io.odpf.depot.error.ErrorType; import io.odpf.depot.message.OdpfMessage; import io.odpf.depot.message.OdpfMessageParser; +import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; @@ -17,11 +20,21 @@ public class BigTableRecordParser { private final BigTableSinkConfig sinkConfig; private final OdpfMessageParser odpfMessageParser; private final BigTableRowKeyParser bigTableRowKeyParser; + private final BigtableSchema bigTableSchema; + private final OdpfMessageSchema schema; + private final Tuple modeAndSchema; - public BigTableRecordParser(BigTableSinkConfig sinkConfig, OdpfMessageParser odpfMessageParser, BigTableRowKeyParser bigTableRowKeyParser) { + public BigTableRecordParser(BigTableSinkConfig sinkConfig, + OdpfMessageParser odpfMessageParser, + BigTableRowKeyParser bigTableRowKeyParser, + Tuple modeAndSchema, + OdpfMessageSchema schema) { this.sinkConfig = sinkConfig; this.odpfMessageParser = odpfMessageParser; this.bigTableRowKeyParser = bigTableRowKeyParser; + this.modeAndSchema = modeAndSchema; + this.bigTableSchema = new BigtableSchema(sinkConfig); + this.schema = schema; } public List convert(List messages) { @@ -35,15 +48,18 @@ public List convert(List messages) { } private BigTableRecord createRecord(OdpfMessage message, long index) { - SinkConnectorSchemaMessageMode mode = sinkConfig.getSinkConnectorSchemaMessageMode(); - String schemaClass = mode == SinkConnectorSchemaMessageMode.LOG_MESSAGE - ? sinkConfig.getSinkConnectorSchemaProtoMessageClass() : sinkConfig.getSinkConnectorSchemaProtoKeyClass(); try { - ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, mode, schemaClass); String rowKey = bigTableRowKeyParser.parse(sinkConfig.getRowKeyTemplate(), message); - RowMutationEntry rowMutationEntry = RowMutationEntry - .create(rowKey) - .setCell("family-test", "odpf-message", parsedOdpfMessage.toString()); + RowMutationEntry rowMutationEntry = RowMutationEntry.create(rowKey); + ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, modeAndSchema.getFirst(), modeAndSchema.getSecond()); + bigTableSchema.getColumnFamilies().forEach( + columnFamily -> bigTableSchema + .getColumns(columnFamily) + .forEach(column -> { + String fieldName = bigTableSchema.getField(columnFamily, column); + String value = String.valueOf(parsedOdpfMessage.getFieldByName(fieldName, schema)); + rowMutationEntry.setCell(columnFamily, column, value); + })); return new BigTableRecord(rowMutationEntry, index, null, true); } catch (Exception e) { ErrorInfo errorInfo = new ErrorInfo(e, ErrorType.DESERIALIZATION_ERROR); diff --git a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java index 172bff39..6ae75a22 100644 --- a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java @@ -15,4 +15,7 @@ public interface BigTableSinkConfig extends OdpfSinkConfig { @Key("SINK_BIGTABLE_ROW_KEY_TEMPLATE") String getRowKeyTemplate(); + + @Key("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING") + String getColumnFamilyMapping(); } diff --git a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java index b2bc1c61..0d36e1d5 100644 --- a/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/OdpfSinkConfig.java @@ -1,10 +1,10 @@ package io.odpf.depot.config; -import io.odpf.depot.config.converter.SinkConnectorSchemaDataTypeConverter; -import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; -import io.odpf.depot.config.converter.SinkConnectorSchemaMessageModeConverter; import io.odpf.depot.config.converter.SchemaRegistryHeadersConverter; import io.odpf.depot.config.converter.SchemaRegistryRefreshConverter; +import io.odpf.depot.config.converter.SinkConnectorSchemaDataTypeConverter; +import io.odpf.depot.config.converter.SinkConnectorSchemaMessageModeConverter; +import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; import io.odpf.stencil.cache.SchemaRefreshStrategy; import org.aeonbits.owner.Config; diff --git a/src/main/java/io/odpf/depot/exception/ConfigurationException.java b/src/main/java/io/odpf/depot/exception/ConfigurationException.java index 5ec3e107..9fb93bca 100644 --- a/src/main/java/io/odpf/depot/exception/ConfigurationException.java +++ b/src/main/java/io/odpf/depot/exception/ConfigurationException.java @@ -4,4 +4,8 @@ public class ConfigurationException extends RuntimeException { public ConfigurationException(String message) { super(message); } + + public ConfigurationException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index 4459d8d7..8d3665bb 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -1,11 +1,14 @@ package io.odpf.depot.bigtable.client; import com.google.api.gax.rpc.ApiException; +import com.google.bigtable.admin.v2.ColumnFamily; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.Table; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.models.BulkMutation; import com.google.cloud.bigtable.data.v2.models.MutateRowsException; import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; @@ -23,8 +26,7 @@ import java.util.List; import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.*; public class BigTableClientTest { @@ -37,6 +39,7 @@ public class BigTableClientTest { private BigTableClient bigTableClient; private List validRecords; + private BigTableSinkConfig sinkConfig; @Before public void setUp() throws IOException { @@ -47,13 +50,14 @@ public void setUp() throws IOException { System.setProperty("SINK_BIGTABLE_INSTANCE_ID", "test-instance"); System.setProperty("SINK_BIGTABLE_TABLE_ID", "test-table"); System.setProperty("SINK_BIGTABLE_CREDENTIAL_PATH", "Users/github/bigtable/test-credential"); + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"family-test\" : { \"qualifier_name1\" : \"input_field1\", \"qualifier_name2\" : \"input_field2\"} }"); RowMutationEntry rowMutationEntry = RowMutationEntry.create("rowKey").setCell("family", "qualifier", "value"); BigTableRecord bigTableRecord1 = new BigTableRecord(rowMutationEntry, 1, null, true); BigTableRecord bigTableRecord2 = new BigTableRecord(rowMutationEntry, 2, null, true); validRecords = Collections.list(bigTableRecord1, bigTableRecord2); - BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient); } @@ -80,4 +84,30 @@ public void shouldReturnBigTableResponseWithFailedMutationsWhenBulkMutateRowsThr Assert.assertTrue(bigTableResponse.hasErrors()); Assert.assertEquals(2, bigTableResponse.getFailedMutations().size()); } + + @Test + public void shouldThrowInvalidSchemaExceptionIfTableDoesNotExist() { + when(bigtableTableAdminClient.exists(sinkConfig.getTableId())).thenReturn(false); + try { + bigTableClient.validateBigTableSchema(); + } catch (BigTableInvalidSchemaException e) { + Assert.assertEquals("Table: " + sinkConfig.getTableId() + " does not exist", e.getMessage()); + } + } + + @Test + public void shouldThrowInvalidSchemaExceptionIfColumnFamilyDoesNotExist() { + Table testTable = Table.fromProto(com.google.bigtable.admin.v2.Table.newBuilder() + .setName("projects/" + sinkConfig.getGCloudProjectID() + "/instances/" + sinkConfig.getInstanceId() + "/tables/" + sinkConfig.getTableId()) + .putColumnFamilies("existing-family-test", ColumnFamily.newBuilder().build()) + .build()); + + when(bigtableTableAdminClient.exists(sinkConfig.getTableId())).thenReturn(true); + when(bigtableTableAdminClient.getTable(sinkConfig.getTableId())).thenReturn(testTable); + try { + bigTableClient.validateBigTableSchema(); + } catch (BigTableInvalidSchemaException e) { + Assert.assertEquals("Column family: family-test does not exist!", e.getMessage()); + } + } } diff --git a/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java b/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java new file mode 100644 index 00000000..76cb629b --- /dev/null +++ b/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java @@ -0,0 +1,116 @@ +package io.odpf.depot.bigtable.model; + +import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.exception.ConfigurationException; +import org.aeonbits.owner.ConfigFactory; +import org.json.JSONException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +public class BigtableSchemaTest { + private BigtableSchema bigtableSchema; + + @Before + public void setUp() { + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{\n" + + "\"family_name1\" : {\n" + + "\"qualifier_name1\" : \"data.is_complete\",\n" + + "\"qualifier_name2\" : \"data.content\"\n" + + "},\n" + + "\"family_name2\" : {\n" + + "\"qualifier_name3\" : \"base_content3\",\n" + + "\"qualifier_name4\" : \"base_content4\"\n" + + "}\n" + + "}"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + bigtableSchema = new BigtableSchema(sinkConfig); + } + + @Test + public void shouldGeSetOfColumnFamilies() { + Set columnFamilies = bigtableSchema.getColumnFamilies(); + assertEquals(2, columnFamilies.size()); + assertTrue(columnFamilies.contains("family_name1")); + assertTrue(columnFamilies.contains("family_name2")); + } + + @Test + public void shouldGetFieldNameForGivenColumnFamilyAndQualifier() { + assertEquals("data.is_complete", bigtableSchema.getField("family_name1", "qualifier_name1")); + assertEquals("data.content", bigtableSchema.getField("family_name1", "qualifier_name2")); + + assertEquals("base_content3", bigtableSchema.getField("family_name2", "qualifier_name3")); + assertEquals("base_content4", bigtableSchema.getField("family_name2", "qualifier_name4")); + } + + @Test + public void shouldGetColumnsForGivenColumnFamily() { + assertEquals(2, bigtableSchema.getColumns("family_name1").size()); + assertTrue(bigtableSchema.getColumns("family_name1").contains("qualifier_name1")); + assertTrue(bigtableSchema.getColumns("family_name1").contains("qualifier_name2")); + + assertEquals(2, bigtableSchema.getColumns("family_name2").size()); + assertTrue(bigtableSchema.getColumns("family_name2").contains("qualifier_name3")); + assertTrue(bigtableSchema.getColumns("family_name2").contains("qualifier_name4")); + } + + @Test + public void shouldThrowConfigurationExceptionWhenColumnMappingIsEmpty() { + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", ""); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> new BigtableSchema(sinkConfig)); + Assert.assertEquals("Column Mapping should not be empty or null", configurationException.getMessage()); + } + + @Test + public void shouldReturnEmptySetIfNoColumnFamilies() { + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + bigtableSchema = new BigtableSchema(sinkConfig); + Set columnFamilies = bigtableSchema.getColumnFamilies(); + Assert.assertEquals(0, columnFamilies.size()); + } + + @Test + public void shouldReturnEmptySetIfNoColumnsPresent() { + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{\n" + + "\"family_name1\" : {\n" + + "\"qualifier_name1\" : \"data.is_complete\",\n" + + "\"qualifier_name2\" : \"data.content\"\n" + + "},\n" + + "\"family_name2\" : {}\n" + + "}"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + bigtableSchema = new BigtableSchema(sinkConfig); + Set columnFamilies = bigtableSchema.getColumnFamilies(); + Assert.assertEquals(2, columnFamilies.size()); + Set columns = bigtableSchema.getColumns("family_name2"); + Assert.assertEquals(0, columns.size()); + } + + @Test + public void shouldThrowJsonException() { + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{\n" + + "\"family_name1\" : {\n" + + "\"qualifier_name1\" : \"data.is_complete\",\n" + + "\"qualifier_name2\" : \"data.content\"\n" + + "},\n" + + "\"family_name2\" : {}\n" + + "}"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + bigtableSchema = new BigtableSchema(sinkConfig); + Set columnFamilies = bigtableSchema.getColumnFamilies(); + Assert.assertEquals(2, columnFamilies.size()); + JSONException jsonException = assertThrows(JSONException.class, () -> bigtableSchema.getColumns("family_name3")); + Assert.assertEquals("JSONObject[\"family_name3\"] not found.", jsonException.getMessage()); + + jsonException = assertThrows(JSONException.class, () -> bigtableSchema.getField("family_name1", "qualifier_name3")); + Assert.assertEquals("JSONObject[\"qualifier_name3\"] not found.", jsonException.getMessage()); + } + +} diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java index f0f75a22..c242ca25 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java @@ -4,10 +4,13 @@ import io.odpf.depot.TestBookingLogMessage; import io.odpf.depot.TestServiceType; import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.utils.MessageConfigUtils; import io.odpf.stencil.client.ClassLoadStencilClient; import org.aeonbits.owner.ConfigFactory; import org.aeonbits.owner.util.Collections; @@ -16,22 +19,29 @@ import org.mockito.Mock; import org.mockito.Mockito; +import java.io.IOException; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.CALLS_REAL_METHODS; public class BigTableRecordParserTest { @Mock private ClassLoadStencilClient stencilClient; + @Mock + private OdpfMessageSchema schema; private BigTableRecordParser bigTableRecordParser; private List messages; @Before - public void setUp() { + public void setUp() throws IOException { System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage"); System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE)); + System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); @@ -45,8 +55,9 @@ public void setUp() { stencilClient = Mockito.mock(ClassLoadStencilClient.class, CALLS_REAL_METHODS); ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); - bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser()); + bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser(), modeAndSchema, schema); } @Test From fcad819509de1752d28b54d3a8e91329fe4e6bff Mon Sep 17 00:00:00 2001 From: lavkesh Date: Mon, 26 Sep 2022 20:35:26 +0800 Subject: [PATCH 05/11] chore: schema refactoring --- .../depot/bigtable/BigTableSinkFactory.java | 6 ++-- .../depot/bigtable/client/BigTableClient.java | 33 ++++++++++--------- .../depot/bigtable/model/BigtableSchema.java | 18 +++++++--- .../bigtable/parser/BigTableRecordParser.java | 2 +- .../bigtable/client/BigTableClientTest.java | 7 ++-- .../bigtable/model/BigtableSchemaTest.java | 31 ++++++++++++++--- 6 files changed, 66 insertions(+), 31 deletions(-) diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 9b28c219..56f384a6 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -3,6 +3,7 @@ import com.timgroup.statsd.NoOpStatsDClient; import io.odpf.depot.OdpfSink; import io.odpf.depot.bigtable.client.BigTableClient; +import io.odpf.depot.bigtable.model.BigtableSchema; import io.odpf.depot.bigtable.parser.BigTableRecordParser; import io.odpf.depot.bigtable.parser.BigTableRowKeyParser; import io.odpf.depot.common.Tuple; @@ -36,12 +37,13 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { public void init() { try { BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); - this.bigTableClient = new BigTableClient(sinkConfig); + BigtableSchema bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigTableClient = new BigTableClient(sinkConfig, bigtableSchema); bigTableClient.validateBigTableSchema(); Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); OdpfMessageSchema schema = odpfMessageParser.getSchema(modeAndSchema.getSecond()); - this.bigTableRecordParser = new BigTableRecordParser( + bigTableRecordParser = new BigTableRecordParser( sinkConfig, odpfMessageParser, bigTableRowKeyParser, diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java index 3d9694ed..773400b1 100644 --- a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -11,9 +11,9 @@ import com.google.cloud.bigtable.data.v2.models.MutateRowsException; import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.model.BigtableSchema; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; -import org.json.JSONObject; import java.io.FileInputStream; import java.io.IOException; @@ -25,15 +25,17 @@ public class BigTableClient { private final BigtableTableAdminClient bigtableTableAdminClient; private final BigtableDataClient bigtableDataClient; private final BigTableSinkConfig sinkConfig; + private final BigtableSchema bigtableSchema; - public BigTableClient(BigTableSinkConfig sinkConfig) throws IOException { - this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig)); + public BigTableClient(BigTableSinkConfig sinkConfig, BigtableSchema bigtableSchema) throws IOException { + this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig), bigtableSchema); } - public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient) { + public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigtableSchema bigtableSchema) { this.sinkConfig = sinkConfig; this.bigtableDataClient = bigtableDataClient; this.bigtableTableAdminClient = bigtableTableAdminClient; + this.bigtableSchema = bigtableSchema; } private static BigtableDataClient getBigTableDataClient(BigTableSinkConfig sinkConfig) throws IOException { @@ -71,28 +73,27 @@ public BigTableResponse send(List records) throws MutateRowsExce } public void validateBigTableSchema() throws BigTableInvalidSchemaException { - this.tableExists(sinkConfig.getTableId()); - this.columnFamiliesExist(sinkConfig.getColumnFamilyMapping(), sinkConfig.getTableId()); + String tableId = sinkConfig.getTableId(); + checkIfTableExists(tableId); + checkIfColumnFamiliesExist(tableId); } - private void tableExists(String tableId) throws BigTableInvalidSchemaException { + private void checkIfTableExists(String tableId) throws BigTableInvalidSchemaException { if (!bigtableTableAdminClient.exists(tableId)) { throw new BigTableInvalidSchemaException(String.format("Table: %s does not exist", tableId)); } } - private void columnFamiliesExist(String inputOutputFieldMapping, String tableId) throws BigTableInvalidSchemaException { - List existingColumnFamilies = bigtableTableAdminClient.getTable(tableId) + private void checkIfColumnFamiliesExist(String tableId) throws BigTableInvalidSchemaException { + Set existingColumnFamilies = bigtableTableAdminClient.getTable(tableId) .getColumnFamilies() .stream() .map(ColumnFamily::getId) - .collect(Collectors.toList()); - - Set columnFamilies = new JSONObject(inputOutputFieldMapping).keySet(); - for (String columnFamily : columnFamilies) { - if (!existingColumnFamilies.contains(columnFamily)) { - throw new BigTableInvalidSchemaException(String.format("Column family: %s does not exist!", columnFamily)); - } + .collect(Collectors.toSet()); + Set missingColumnFamilies = bigtableSchema.getMissingColumnFamilies(existingColumnFamilies); + if (missingColumnFamilies.size() > 0) { + throw new BigTableInvalidSchemaException( + String.format("Column families %s do not exist in table %s!", missingColumnFamilies, tableId)); } } } diff --git a/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java b/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java index 0f857e6b..94b9dd32 100644 --- a/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java +++ b/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java @@ -1,21 +1,20 @@ package io.odpf.depot.bigtable.model; -import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.exception.ConfigurationException; import org.json.JSONObject; +import java.util.HashSet; import java.util.Set; public class BigtableSchema { private final JSONObject columnFamilyMapping; - public BigtableSchema(BigTableSinkConfig sinkConfig) { - String columnMapping = sinkConfig.getColumnFamilyMapping(); + public BigtableSchema(String columnMapping) { if (columnMapping == null || columnMapping.isEmpty()) { throw new ConfigurationException("Column Mapping should not be empty or null"); } - this.columnFamilyMapping = new JSONObject(sinkConfig.getColumnFamilyMapping()); + this.columnFamilyMapping = new JSONObject(columnMapping); } public String getField(String columnFamily, String columnName) { @@ -31,4 +30,15 @@ public Set getColumns(String family) { return columnFamilyMapping.getJSONObject(family).keySet(); } + /** + * Returns missing column families. + * + * @param existingColumnFamilies existing column families in a table. + * @return set of missing column families + */ + public Set getMissingColumnFamilies(Set existingColumnFamilies) { + Set tempSet = new HashSet<>(getColumnFamilies()); + tempSet.removeAll(existingColumnFamilies); + return tempSet; + } } diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java index 6c0ac169..263b2404 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -33,7 +33,7 @@ public BigTableRecordParser(BigTableSinkConfig sinkConfig, this.odpfMessageParser = odpfMessageParser; this.bigTableRowKeyParser = bigTableRowKeyParser; this.modeAndSchema = modeAndSchema; - this.bigTableSchema = new BigtableSchema(sinkConfig); + this.bigTableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); this.schema = schema; } diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index 8d3665bb..8d1c4c51 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -10,6 +10,7 @@ import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.model.BigtableSchema; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; @@ -56,9 +57,9 @@ public void setUp() throws IOException { BigTableRecord bigTableRecord1 = new BigTableRecord(rowMutationEntry, 1, null, true); BigTableRecord bigTableRecord2 = new BigTableRecord(rowMutationEntry, 2, null, true); validRecords = Collections.list(bigTableRecord1, bigTableRecord2); - sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient); + BigtableSchema schema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient, schema); } @Test @@ -107,7 +108,7 @@ public void shouldThrowInvalidSchemaExceptionIfColumnFamilyDoesNotExist() { try { bigTableClient.validateBigTableSchema(); } catch (BigTableInvalidSchemaException e) { - Assert.assertEquals("Column family: family-test does not exist!", e.getMessage()); + Assert.assertEquals("Column families [family-test] do not exist in table test-table!", e.getMessage()); } } } diff --git a/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java b/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java index 76cb629b..f02c5f34 100644 --- a/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java +++ b/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java @@ -8,6 +8,7 @@ import org.junit.Before; import org.junit.Test; +import java.util.HashSet; import java.util.Set; import static org.junit.jupiter.api.Assertions.*; @@ -28,7 +29,7 @@ public void setUp() { + "}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig); + bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); } @Test @@ -63,7 +64,7 @@ public void shouldGetColumnsForGivenColumnFamily() { public void shouldThrowConfigurationExceptionWhenColumnMappingIsEmpty() { System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", ""); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> new BigtableSchema(sinkConfig)); + ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> new BigtableSchema(sinkConfig.getColumnFamilyMapping())); Assert.assertEquals("Column Mapping should not be empty or null", configurationException.getMessage()); } @@ -71,7 +72,7 @@ public void shouldThrowConfigurationExceptionWhenColumnMappingIsEmpty() { public void shouldReturnEmptySetIfNoColumnFamilies() { System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig); + bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(0, columnFamilies.size()); } @@ -86,7 +87,7 @@ public void shouldReturnEmptySetIfNoColumnsPresent() { + "\"family_name2\" : {}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig); + bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(2, columnFamilies.size()); Set columns = bigtableSchema.getColumns("family_name2"); @@ -103,7 +104,7 @@ public void shouldThrowJsonException() { + "\"family_name2\" : {}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig); + bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(2, columnFamilies.size()); JSONException jsonException = assertThrows(JSONException.class, () -> bigtableSchema.getColumns("family_name3")); @@ -113,4 +114,24 @@ public void shouldThrowJsonException() { Assert.assertEquals("JSONObject[\"qualifier_name3\"] not found.", jsonException.getMessage()); } + @Test + public void shouldReturnEmptySetOfMissingColumnFamilies() { + Set missingColumnFamilies = bigtableSchema.getMissingColumnFamilies(new HashSet() {{ + add("family_name1"); + add("family_name2"); + }}); + Assert.assertEquals(0, missingColumnFamilies.size()); + } + + @Test + public void shouldReturnMissingColumnFamilies() { + Set missingColumnFamilies = bigtableSchema.getMissingColumnFamilies(new HashSet() {{ + add("family_name3"); + add("family_name2"); + add("family_name4"); + }}); + Assert.assertEquals(new HashSet() {{ + add("family_name1"); + }}, missingColumnFamilies); + } } From 1014226996fbe68b8bc907da3c666ddaa311fa3f Mon Sep 17 00:00:00 2001 From: lavkesh Date: Wed, 28 Sep 2022 10:46:03 +0800 Subject: [PATCH 06/11] chore: naming conventions --- .../odpf/depot/bigtable/BigTableSinkFactory.java | 7 ++++--- .../odpf/depot/bigtable/client/BigTableClient.java | 8 ++++---- .../{BigtableSchema.java => BigTableSchema.java} | 4 ++-- .../bigtable/parser/BigTableRecordParser.java | 9 +++++---- .../depot/bigtable/client/BigTableClientTest.java | 4 ++-- ...ableSchemaTest.java => BigTableSchemaTest.java} | 14 +++++++------- .../bigtable/parser/BigTableRecordParserTest.java | 9 +++++---- 7 files changed, 29 insertions(+), 26 deletions(-) rename src/main/java/io/odpf/depot/bigtable/model/{BigtableSchema.java => BigTableSchema.java} (94%) rename src/test/java/io/odpf/depot/bigtable/model/{BigtableSchemaTest.java => BigTableSchemaTest.java} (93%) diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 56f384a6..6e1a26c8 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -3,7 +3,7 @@ import com.timgroup.statsd.NoOpStatsDClient; import io.odpf.depot.OdpfSink; import io.odpf.depot.bigtable.client.BigTableClient; -import io.odpf.depot.bigtable.model.BigtableSchema; +import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.bigtable.parser.BigTableRecordParser; import io.odpf.depot.bigtable.parser.BigTableRowKeyParser; import io.odpf.depot.common.Tuple; @@ -37,7 +37,7 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { public void init() { try { BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); - BigtableSchema bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); bigTableClient = new BigTableClient(sinkConfig, bigtableSchema); bigTableClient.validateBigTableSchema(); Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); @@ -48,7 +48,8 @@ public void init() { odpfMessageParser, bigTableRowKeyParser, modeAndSchema, - schema); + schema, + bigtableSchema); } catch (IOException e) { throw new ConfigurationException("Exception occurred while creating sink", e); } diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java index 773400b1..fd31ef6f 100644 --- a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -11,7 +11,7 @@ import com.google.cloud.bigtable.data.v2.models.MutateRowsException; import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; -import io.odpf.depot.bigtable.model.BigtableSchema; +import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; @@ -25,13 +25,13 @@ public class BigTableClient { private final BigtableTableAdminClient bigtableTableAdminClient; private final BigtableDataClient bigtableDataClient; private final BigTableSinkConfig sinkConfig; - private final BigtableSchema bigtableSchema; + private final BigTableSchema bigtableSchema; - public BigTableClient(BigTableSinkConfig sinkConfig, BigtableSchema bigtableSchema) throws IOException { + public BigTableClient(BigTableSinkConfig sinkConfig, BigTableSchema bigtableSchema) throws IOException { this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig), bigtableSchema); } - public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigtableSchema bigtableSchema) { + public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigTableSchema bigtableSchema) { this.sinkConfig = sinkConfig; this.bigtableDataClient = bigtableDataClient; this.bigtableTableAdminClient = bigtableTableAdminClient; diff --git a/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java b/src/main/java/io/odpf/depot/bigtable/model/BigTableSchema.java similarity index 94% rename from src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java rename to src/main/java/io/odpf/depot/bigtable/model/BigTableSchema.java index 94b9dd32..3311eb2d 100644 --- a/src/main/java/io/odpf/depot/bigtable/model/BigtableSchema.java +++ b/src/main/java/io/odpf/depot/bigtable/model/BigTableSchema.java @@ -6,11 +6,11 @@ import java.util.HashSet; import java.util.Set; -public class BigtableSchema { +public class BigTableSchema { private final JSONObject columnFamilyMapping; - public BigtableSchema(String columnMapping) { + public BigTableSchema(String columnMapping) { if (columnMapping == null || columnMapping.isEmpty()) { throw new ConfigurationException("Column Mapping should not be empty or null"); } diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java index 263b2404..2f550536 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -2,7 +2,7 @@ import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import io.odpf.depot.bigtable.model.BigTableRecord; -import io.odpf.depot.bigtable.model.BigtableSchema; +import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.error.ErrorInfo; @@ -20,7 +20,7 @@ public class BigTableRecordParser { private final BigTableSinkConfig sinkConfig; private final OdpfMessageParser odpfMessageParser; private final BigTableRowKeyParser bigTableRowKeyParser; - private final BigtableSchema bigTableSchema; + private final BigTableSchema bigTableSchema; private final OdpfMessageSchema schema; private final Tuple modeAndSchema; @@ -28,13 +28,14 @@ public BigTableRecordParser(BigTableSinkConfig sinkConfig, OdpfMessageParser odpfMessageParser, BigTableRowKeyParser bigTableRowKeyParser, Tuple modeAndSchema, - OdpfMessageSchema schema) { + OdpfMessageSchema schema, + BigTableSchema bigTableSchema) { this.sinkConfig = sinkConfig; this.odpfMessageParser = odpfMessageParser; this.bigTableRowKeyParser = bigTableRowKeyParser; this.modeAndSchema = modeAndSchema; - this.bigTableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); this.schema = schema; + this.bigTableSchema = bigTableSchema; } public List convert(List messages) { diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index 8d1c4c51..36306fc1 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -10,7 +10,7 @@ import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import io.odpf.depot.bigtable.exception.BigTableInvalidSchemaException; import io.odpf.depot.bigtable.model.BigTableRecord; -import io.odpf.depot.bigtable.model.BigtableSchema; +import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; @@ -58,7 +58,7 @@ public void setUp() throws IOException { BigTableRecord bigTableRecord2 = new BigTableRecord(rowMutationEntry, 2, null, true); validRecords = Collections.list(bigTableRecord1, bigTableRecord2); sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - BigtableSchema schema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + BigTableSchema schema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient, schema); } diff --git a/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java b/src/test/java/io/odpf/depot/bigtable/model/BigTableSchemaTest.java similarity index 93% rename from src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java rename to src/test/java/io/odpf/depot/bigtable/model/BigTableSchemaTest.java index f02c5f34..811dbd34 100644 --- a/src/test/java/io/odpf/depot/bigtable/model/BigtableSchemaTest.java +++ b/src/test/java/io/odpf/depot/bigtable/model/BigTableSchemaTest.java @@ -13,8 +13,8 @@ import static org.junit.jupiter.api.Assertions.*; -public class BigtableSchemaTest { - private BigtableSchema bigtableSchema; +public class BigTableSchemaTest { + private BigTableSchema bigtableSchema; @Before public void setUp() { @@ -29,7 +29,7 @@ public void setUp() { + "}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); } @Test @@ -64,7 +64,7 @@ public void shouldGetColumnsForGivenColumnFamily() { public void shouldThrowConfigurationExceptionWhenColumnMappingIsEmpty() { System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", ""); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> new BigtableSchema(sinkConfig.getColumnFamilyMapping())); + ConfigurationException configurationException = assertThrows(ConfigurationException.class, () -> new BigTableSchema(sinkConfig.getColumnFamilyMapping())); Assert.assertEquals("Column Mapping should not be empty or null", configurationException.getMessage()); } @@ -72,7 +72,7 @@ public void shouldThrowConfigurationExceptionWhenColumnMappingIsEmpty() { public void shouldReturnEmptySetIfNoColumnFamilies() { System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(0, columnFamilies.size()); } @@ -87,7 +87,7 @@ public void shouldReturnEmptySetIfNoColumnsPresent() { + "\"family_name2\" : {}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(2, columnFamilies.size()); Set columns = bigtableSchema.getColumns("family_name2"); @@ -104,7 +104,7 @@ public void shouldThrowJsonException() { + "\"family_name2\" : {}\n" + "}"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); - bigtableSchema = new BigtableSchema(sinkConfig.getColumnFamilyMapping()); + bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); Set columnFamilies = bigtableSchema.getColumnFamilies(); Assert.assertEquals(2, columnFamilies.size()); JSONException jsonException = assertThrows(JSONException.class, () -> bigtableSchema.getColumns("family_name3")); diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java index c242ca25..69617ae9 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java @@ -4,6 +4,7 @@ import io.odpf.depot.TestBookingLogMessage; import io.odpf.depot.TestServiceType; import io.odpf.depot.bigtable.model.BigTableRecord; +import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.OdpfMessage; @@ -22,10 +23,10 @@ import java.io.IOException; import java.util.List; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.CALLS_REAL_METHODS; public class BigTableRecordParserTest { @@ -56,8 +57,8 @@ public void setUp() throws IOException { ProtoOdpfMessageParser protoOdpfMessageParser = new ProtoOdpfMessageParser(stencilClient); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); - - bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser(), modeAndSchema, schema); + BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); + bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser(), modeAndSchema, schema, bigtableSchema); } @Test From 8ab42b8efa4ca418243eea9fa014218d6a36c13f Mon Sep 17 00:00:00 2001 From: Mayur Gubrele Date: Thu, 29 Sep 2022 10:36:15 +0530 Subject: [PATCH 07/11] feat: add functionality to create rowkey from configured template (#44) --- .../depot/bigtable/BigTableSinkFactory.java | 7 +- .../bigtable/parser/BigTableRecordParser.java | 10 +- .../bigtable/parser/BigTableRowKeyParser.java | 18 ++-- .../{redis/parsers => common}/Template.java | 2 +- .../odpf/depot/config/BigTableSinkConfig.java | 3 + .../parsers/RedisEntryParserFactory.java | 1 + .../parsers/RedisHashSetEntryParser.java | 1 + .../parsers/RedisKeyValueEntryParser.java | 1 + .../redis/parsers/RedisListEntryParser.java | 1 + .../bigtable/client/BigTableClientTest.java | 1 + .../parser/BigTableRecordParserTest.java | 6 +- .../parser/BigTableRowKeyParserTest.java | 92 +++++++++++++++++-- .../depot/redis/parsers/TemplateTest.java | 1 + 13 files changed, 115 insertions(+), 29 deletions(-) rename src/main/java/io/odpf/depot/{redis/parsers => common}/Template.java (97%) diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 6e1a26c8..04e28825 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -6,6 +6,7 @@ import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.bigtable.parser.BigTableRecordParser; import io.odpf.depot.bigtable.parser.BigTableRowKeyParser; +import io.odpf.depot.common.Template; import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.exception.ConfigurationException; @@ -36,15 +37,17 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { public void init() { try { - BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); bigTableClient = new BigTableClient(sinkConfig, bigtableSchema); bigTableClient.validateBigTableSchema(); + Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); OdpfMessageParser odpfMessageParser = OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); OdpfMessageSchema schema = odpfMessageParser.getSchema(modeAndSchema.getSecond()); + + Template keyTemplate = new Template(sinkConfig.getRowKeyTemplate()); + BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(keyTemplate, schema); bigTableRecordParser = new BigTableRecordParser( - sinkConfig, odpfMessageParser, bigTableRowKeyParser, modeAndSchema, diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java index 2f550536..567d6c80 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -4,7 +4,6 @@ import io.odpf.depot.bigtable.model.BigTableRecord; import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.common.Tuple; -import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.error.ErrorInfo; import io.odpf.depot.error.ErrorType; import io.odpf.depot.message.OdpfMessage; @@ -17,20 +16,17 @@ import java.util.List; public class BigTableRecordParser { - private final BigTableSinkConfig sinkConfig; private final OdpfMessageParser odpfMessageParser; private final BigTableRowKeyParser bigTableRowKeyParser; private final BigTableSchema bigTableSchema; private final OdpfMessageSchema schema; private final Tuple modeAndSchema; - public BigTableRecordParser(BigTableSinkConfig sinkConfig, - OdpfMessageParser odpfMessageParser, + public BigTableRecordParser(OdpfMessageParser odpfMessageParser, BigTableRowKeyParser bigTableRowKeyParser, Tuple modeAndSchema, OdpfMessageSchema schema, BigTableSchema bigTableSchema) { - this.sinkConfig = sinkConfig; this.odpfMessageParser = odpfMessageParser; this.bigTableRowKeyParser = bigTableRowKeyParser; this.modeAndSchema = modeAndSchema; @@ -50,9 +46,9 @@ public List convert(List messages) { private BigTableRecord createRecord(OdpfMessage message, long index) { try { - String rowKey = bigTableRowKeyParser.parse(sinkConfig.getRowKeyTemplate(), message); - RowMutationEntry rowMutationEntry = RowMutationEntry.create(rowKey); ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, modeAndSchema.getFirst(), modeAndSchema.getSecond()); + String rowKey = bigTableRowKeyParser.parse(parsedOdpfMessage); + RowMutationEntry rowMutationEntry = RowMutationEntry.create(rowKey); bigTableSchema.getColumnFamilies().forEach( columnFamily -> bigTableSchema .getColumns(columnFamily) diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java index de9d453c..1130031c 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParser.java @@ -1,14 +1,16 @@ package io.odpf.depot.bigtable.parser; -import io.odpf.depot.message.OdpfMessage; - -import java.util.Random; +import io.odpf.depot.common.Template; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import lombok.AllArgsConstructor; +@AllArgsConstructor public class BigTableRowKeyParser { - public String parse(String rowKeyTemplate, OdpfMessage message) { - Random rand = new Random(); - int n = rand.nextInt(); - System.out.println("Rowkey created: key-test-" + n); - return "key-test-" + n; + private final Template keyTemplate; + private final OdpfMessageSchema schema; + + public String parse(ParsedOdpfMessage parsedOdpfMessage) { + return keyTemplate.parse(parsedOdpfMessage, schema); } } diff --git a/src/main/java/io/odpf/depot/redis/parsers/Template.java b/src/main/java/io/odpf/depot/common/Template.java similarity index 97% rename from src/main/java/io/odpf/depot/redis/parsers/Template.java rename to src/main/java/io/odpf/depot/common/Template.java index 55e605e2..b9af3e18 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/Template.java +++ b/src/main/java/io/odpf/depot/common/Template.java @@ -1,4 +1,4 @@ -package io.odpf.depot.redis.parsers; +package io.odpf.depot.common; import com.google.common.base.Splitter; import io.odpf.depot.message.OdpfMessageSchema; diff --git a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java index 6ae75a22..675c9207 100644 --- a/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java +++ b/src/main/java/io/odpf/depot/config/BigTableSinkConfig.java @@ -1,5 +1,8 @@ package io.odpf.depot.config; +import org.aeonbits.owner.Config; + +@Config.DisableFeature(Config.DisableableFeature.PARAMETER_FORMATTING) public interface BigTableSinkConfig extends OdpfSinkConfig { @Key("SINK_BIGTABLE_GOOGLE_CLOUD_PROJECT_ID") String getGCloudProjectID(); diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java index 8f519b87..0271e406 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java @@ -1,5 +1,6 @@ package io.odpf.depot.redis.parsers; +import io.odpf.depot.common.Template; import io.odpf.depot.config.RedisSinkConfig; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.metrics.StatsDReporter; diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java index 74e000f5..209a341b 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisHashSetEntryParser.java @@ -1,5 +1,6 @@ package io.odpf.depot.redis.parsers; +import io.odpf.depot.common.Template; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.metrics.Instrumentation; diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java index 51e31b1c..89546cbd 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisKeyValueEntryParser.java @@ -1,5 +1,6 @@ package io.odpf.depot.redis.parsers; +import io.odpf.depot.common.Template; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.metrics.Instrumentation; diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java b/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java index 3a730746..6db0e6b8 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisListEntryParser.java @@ -1,6 +1,7 @@ package io.odpf.depot.redis.parsers; +import io.odpf.depot.common.Template; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.metrics.Instrumentation; diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index 36306fc1..b06df1ba 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -52,6 +52,7 @@ public void setUp() throws IOException { System.setProperty("SINK_BIGTABLE_TABLE_ID", "test-table"); System.setProperty("SINK_BIGTABLE_CREDENTIAL_PATH", "Users/github/bigtable/test-credential"); System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{ \"family-test\" : { \"qualifier_name1\" : \"input_field1\", \"qualifier_name2\" : \"input_field2\"} }"); + System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key-constant-string"); RowMutationEntry rowMutationEntry = RowMutationEntry.create("rowKey").setCell("family", "qualifier", "value"); BigTableRecord bigTableRecord1 = new BigTableRecord(rowMutationEntry, 1, null, true); diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java index 69617ae9..a1ee94fe 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java @@ -5,6 +5,7 @@ import io.odpf.depot.TestServiceType; import io.odpf.depot.bigtable.model.BigTableRecord; import io.odpf.depot.bigtable.model.BigTableSchema; +import io.odpf.depot.common.Template; import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.OdpfMessage; @@ -43,6 +44,8 @@ public void setUp() throws IOException { System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage"); System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE)); System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); + System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key-constant-string"); + TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); @@ -58,7 +61,8 @@ public void setUp() throws IOException { BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); - bigTableRecordParser = new BigTableRecordParser(sinkConfig, protoOdpfMessageParser, new BigTableRowKeyParser(), modeAndSchema, schema, bigtableSchema); + + bigTableRecordParser = new BigTableRecordParser(protoOdpfMessageParser, new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema), modeAndSchema, schema, bigtableSchema); } @Test diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java index 532fc04b..f38b05a6 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java @@ -1,21 +1,93 @@ package io.odpf.depot.bigtable.parser; -import io.odpf.depot.TestBookingLogKey; -import io.odpf.depot.TestBookingLogMessage; -import io.odpf.depot.TestServiceType; +import com.google.protobuf.Descriptors; +import com.timgroup.statsd.NoOpStatsDClient; +import io.odpf.depot.TestKey; +import io.odpf.depot.TestMessage; +import io.odpf.depot.TestNestedMessage; +import io.odpf.depot.TestNestedRepeatedMessage; +import io.odpf.depot.common.Template; +import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.message.OdpfMessageSchema; +import io.odpf.depot.message.ParsedOdpfMessage; +import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.message.proto.ProtoOdpfMessageParser; +import io.odpf.depot.metrics.StatsDReporter; +import org.aeonbits.owner.ConfigFactory; import org.junit.Test; +import org.junit.jupiter.api.Assertions; -import static org.junit.Assert.*; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; public class BigTableRowKeyParserTest { + private final Map descriptorsMap = new HashMap() {{ + put(String.format("%s", TestKey.class.getName()), TestKey.getDescriptor()); + put(String.format("%s", TestMessage.class.getName()), TestMessage.getDescriptor()); + put(String.format("%s", TestNestedMessage.class.getName()), TestNestedMessage.getDescriptor()); + put(String.format("%s", TestNestedRepeatedMessage.class.getName()), TestNestedRepeatedMessage.getDescriptor()); + }}; + @Test - public void shouldReturnParsedRowKey() { - BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(); - TestBookingLogKey bookingLogKey1 = TestBookingLogKey.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").build(); - TestBookingLogMessage bookingLogMessage1 = TestBookingLogMessage.newBuilder().setOrderNumber("order#1").setOrderUrl("order-url#1").setServiceType(TestServiceType.Enum.GO_SEND).build(); - String parsedRowKey = bigTableRowKeyParser.parse("template", new OdpfMessage(bookingLogKey1, bookingLogMessage1)); - assertTrue(parsedRowKey.contains("key-test-")); + public void shouldReturnParsedRowKeyForValidParameterisedTemplate() throws IOException { + System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-%s$key#%s*test,order_number,order_details"); + System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestMessage"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(sinkConfig, new StatsDReporter(new NoOpStatsDClient()), null); + OdpfMessageSchema schema = odpfMessageParser.getSchema(sinkConfig.getSinkConnectorSchemaProtoMessageClass(), descriptorsMap); + + byte[] logMessage = TestMessage.newBuilder() + .setOrderNumber("xyz-order") + .setOrderDetails("eureka") + .build() + .toByteArray(); + OdpfMessage message = new OdpfMessage(null, logMessage); + ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, SinkConnectorSchemaMessageMode.LOG_MESSAGE, sinkConfig.getSinkConnectorSchemaProtoMessageClass()); + + BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema); + String parsedRowKey = bigTableRowKeyParser.parse(parsedOdpfMessage); + assertEquals("row-xyz-order$key#eureka*test", parsedRowKey); } + + @Test + public void shouldReturnTheRowKeySameAsTemplateWhenTemplateIsValidAndContainsOnlyConstantStrings() throws IOException { + System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key#constant$String"); + System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestMessage"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(sinkConfig, new StatsDReporter(new NoOpStatsDClient()), null); + OdpfMessageSchema schema = odpfMessageParser.getSchema(sinkConfig.getSinkConnectorSchemaProtoMessageClass(), descriptorsMap); + + byte[] logMessage = TestMessage.newBuilder() + .setOrderNumber("xyz-order") + .setOrderDetails("eureka") + .build() + .toByteArray(); + OdpfMessage message = new OdpfMessage(null, logMessage); + ParsedOdpfMessage parsedOdpfMessage = odpfMessageParser.parse(message, SinkConnectorSchemaMessageMode.LOG_MESSAGE, sinkConfig.getSinkConnectorSchemaProtoMessageClass()); + + BigTableRowKeyParser bigTableRowKeyParser = new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema); + String parsedRowKey = bigTableRowKeyParser.parse(parsedOdpfMessage); + assertEquals("row-key#constant$String", parsedRowKey); + } + + @Test + public void shouldThrowErrorForInvalidTemplate() throws IOException { + System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key%s"); + System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestMessage"); + BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); + + ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(sinkConfig, new StatsDReporter(new NoOpStatsDClient()), null); + OdpfMessageSchema schema = odpfMessageParser.getSchema(sinkConfig.getSinkConnectorSchemaProtoMessageClass(), descriptorsMap); + + IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema)); + assertEquals("Template is not valid, variables=1, validArgs=1, values=0", illegalArgumentException.getMessage()); + } + } diff --git a/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java b/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java index 53590b16..60bfd6e9 100644 --- a/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java +++ b/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java @@ -5,6 +5,7 @@ import io.odpf.depot.TestKey; import io.odpf.depot.TestLocation; import io.odpf.depot.TestMessage; +import io.odpf.depot.common.Template; import io.odpf.depot.config.RedisSinkConfig; import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; import io.odpf.depot.message.OdpfMessage; From 7fa5ac9e5690ed0a0e31992b279ea54c293542e4 Mon Sep 17 00:00:00 2001 From: lavkesh Date: Thu, 29 Sep 2022 15:17:01 +0800 Subject: [PATCH 08/11] chore: refactor Template validation --- .../depot/bigtable/BigTableSinkFactory.java | 3 +- .../java/io/odpf/depot/common/Template.java | 9 +++--- .../exception/InvalidTemplateException.java | 7 ++++ .../parsers/RedisEntryParserFactory.java | 17 ++++++++-- .../parser/BigTableRecordParserTest.java | 3 +- .../parser/BigTableRowKeyParserTest.java | 7 ++-- .../parsers => common}/TemplateTest.java | 32 +++++++++---------- 7 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 src/main/java/io/odpf/depot/exception/InvalidTemplateException.java rename src/test/java/io/odpf/depot/{redis/parsers => common}/TemplateTest.java (83%) diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 04e28825..533c5e6d 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -10,6 +10,7 @@ import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.exception.ConfigurationException; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessageParser; import io.odpf.depot.message.OdpfMessageParserFactory; import io.odpf.depot.message.OdpfMessageSchema; @@ -53,7 +54,7 @@ public void init() { modeAndSchema, schema, bigtableSchema); - } catch (IOException e) { + } catch (IOException | InvalidTemplateException e) { throw new ConfigurationException("Exception occurred while creating sink", e); } } diff --git a/src/main/java/io/odpf/depot/common/Template.java b/src/main/java/io/odpf/depot/common/Template.java index b9af3e18..501a8991 100644 --- a/src/main/java/io/odpf/depot/common/Template.java +++ b/src/main/java/io/odpf/depot/common/Template.java @@ -1,6 +1,7 @@ package io.odpf.depot.common; import com.google.common.base.Splitter; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.utils.StringUtils; @@ -12,9 +13,9 @@ public class Template { private final String templatePattern; private final List patternVariableFieldNames; - public Template(String template) { + public Template(String template) throws InvalidTemplateException { if (template == null || template.isEmpty()) { - throw new IllegalArgumentException("Template '" + template + "' is invalid"); + throw new InvalidTemplateException("Template '" + template + "' is invalid"); } List templateStrings = new ArrayList<>(); Splitter.on(",").omitEmptyStrings().split(template).forEach(s -> templateStrings.add(s.trim())); @@ -23,12 +24,12 @@ public Template(String template) { validate(); } - private void validate() { + private void validate() throws InvalidTemplateException { int validArgs = StringUtils.countVariables(templatePattern); int values = patternVariableFieldNames.size(); int variables = StringUtils.count(templatePattern, '%'); if (validArgs != values || variables != values) { - throw new IllegalArgumentException(String.format("Template is not valid, variables=%d, validArgs=%d, values=%d", variables, validArgs, values)); + throw new InvalidTemplateException(String.format("Template is not valid, variables=%d, validArgs=%d, values=%d", variables, validArgs, values)); } } diff --git a/src/main/java/io/odpf/depot/exception/InvalidTemplateException.java b/src/main/java/io/odpf/depot/exception/InvalidTemplateException.java new file mode 100644 index 00000000..db618f4a --- /dev/null +++ b/src/main/java/io/odpf/depot/exception/InvalidTemplateException.java @@ -0,0 +1,7 @@ +package io.odpf.depot.exception; + +public class InvalidTemplateException extends Exception { + public InvalidTemplateException(String message) { + super(message); + } +} diff --git a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java index 0271e406..974522c6 100644 --- a/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java +++ b/src/main/java/io/odpf/depot/redis/parsers/RedisEntryParserFactory.java @@ -2,6 +2,7 @@ import io.odpf.depot.common.Template; import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.metrics.StatsDReporter; @@ -18,7 +19,12 @@ public static RedisEntryParser getRedisEntryParser( RedisSinkConfig redisSinkConfig, StatsDReporter statsDReporter, OdpfMessageSchema schema) { - Template keyTemplate = new Template(redisSinkConfig.getSinkRedisKeyTemplate()); + Template keyTemplate; + try { + keyTemplate = new Template(redisSinkConfig.getSinkRedisKeyTemplate()); + } catch (InvalidTemplateException e) { + throw new IllegalArgumentException(e.getMessage()); + } switch (redisSinkConfig.getSinkRedisDataType()) { case KEYVALUE: String fieldName = redisSinkConfig.getSinkRedisKeyValueDataFieldName(); @@ -37,8 +43,15 @@ public static RedisEntryParser getRedisEntryParser( if (properties == null || properties.isEmpty()) { throw new IllegalArgumentException("Empty config SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING found"); } + Map fieldTemplates = properties.entrySet().stream().collect(Collectors.toMap( - kv -> kv.getKey().toString(), kv -> new Template(kv.getValue().toString()) + kv -> kv.getKey().toString(), kv -> { + try { + return new Template(kv.getValue().toString()); + } catch (InvalidTemplateException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } )); return new RedisHashSetEntryParser(statsDReporter, keyTemplate, fieldTemplates, schema); } diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java index a1ee94fe..7f20a775 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRecordParserTest.java @@ -8,6 +8,7 @@ import io.odpf.depot.common.Template; import io.odpf.depot.common.Tuple; import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessage; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; @@ -40,7 +41,7 @@ public class BigTableRecordParserTest { private List messages; @Before - public void setUp() throws IOException { + public void setUp() throws IOException, InvalidTemplateException { System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestBookingLogMessage"); System.setProperty("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE", String.valueOf(SinkConnectorSchemaMessageMode.LOG_MESSAGE)); System.setProperty("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING", "{}"); diff --git a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java index f38b05a6..cfe45f94 100644 --- a/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java +++ b/src/test/java/io/odpf/depot/bigtable/parser/BigTableRowKeyParserTest.java @@ -8,6 +8,7 @@ import io.odpf.depot.TestNestedRepeatedMessage; import io.odpf.depot.common.Template; import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessage; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; @@ -34,7 +35,7 @@ public class BigTableRowKeyParserTest { }}; @Test - public void shouldReturnParsedRowKeyForValidParameterisedTemplate() throws IOException { + public void shouldReturnParsedRowKeyForValidParameterisedTemplate() throws IOException, InvalidTemplateException { System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-%s$key#%s*test,order_number,order_details"); System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestMessage"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); @@ -56,7 +57,7 @@ public void shouldReturnParsedRowKeyForValidParameterisedTemplate() throws IOExc } @Test - public void shouldReturnTheRowKeySameAsTemplateWhenTemplateIsValidAndContainsOnlyConstantStrings() throws IOException { + public void shouldReturnTheRowKeySameAsTemplateWhenTemplateIsValidAndContainsOnlyConstantStrings() throws IOException, InvalidTemplateException { System.setProperty("SINK_BIGTABLE_ROW_KEY_TEMPLATE", "row-key#constant$String"); System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "io.odpf.depot.TestMessage"); BigTableSinkConfig sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); @@ -86,7 +87,7 @@ public void shouldThrowErrorForInvalidTemplate() throws IOException { ProtoOdpfMessageParser odpfMessageParser = new ProtoOdpfMessageParser(sinkConfig, new StatsDReporter(new NoOpStatsDClient()), null); OdpfMessageSchema schema = odpfMessageParser.getSchema(sinkConfig.getSinkConnectorSchemaProtoMessageClass(), descriptorsMap); - IllegalArgumentException illegalArgumentException = Assertions.assertThrows(IllegalArgumentException.class, () -> new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema)); + InvalidTemplateException illegalArgumentException = Assertions.assertThrows(InvalidTemplateException.class, () -> new BigTableRowKeyParser(new Template(sinkConfig.getRowKeyTemplate()), schema)); assertEquals("Template is not valid, variables=1, validArgs=1, values=0", illegalArgumentException.getMessage()); } diff --git a/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java b/src/test/java/io/odpf/depot/common/TemplateTest.java similarity index 83% rename from src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java rename to src/test/java/io/odpf/depot/common/TemplateTest.java index 60bfd6e9..2d422218 100644 --- a/src/test/java/io/odpf/depot/redis/parsers/TemplateTest.java +++ b/src/test/java/io/odpf/depot/common/TemplateTest.java @@ -1,13 +1,13 @@ -package io.odpf.depot.redis.parsers; +package io.odpf.depot.common; import com.google.protobuf.Descriptors; import io.odpf.depot.TestBookingLogMessage; import io.odpf.depot.TestKey; import io.odpf.depot.TestLocation; import io.odpf.depot.TestMessage; -import io.odpf.depot.common.Template; -import io.odpf.depot.config.RedisSinkConfig; +import io.odpf.depot.config.OdpfSinkConfig; import io.odpf.depot.config.enums.SinkConnectorSchemaDataType; +import io.odpf.depot.exception.InvalidTemplateException; import io.odpf.depot.message.OdpfMessage; import io.odpf.depot.message.OdpfMessageParserFactory; import io.odpf.depot.message.OdpfMessageSchema; @@ -34,7 +34,7 @@ @RunWith(MockitoJUnitRunner.class) public class TemplateTest { @Mock - private RedisSinkConfig redisSinkConfig; + private OdpfSinkConfig sinkConfig; @Mock private StatsDReporter statsDReporter; private ParsedOdpfMessage parsedTestMessage; @@ -59,65 +59,65 @@ public void setUp() throws Exception { parsedTestMessage = new ProtoOdpfParsedMessage(protoParserTest.parse((byte[]) message.getLogMessage())); Parser protoParserBooking = StencilClientFactory.getClient().getParser(TestBookingLogMessage.class.getName()); parsedBookingMessage = new ProtoOdpfParsedMessage(protoParserBooking.parse((byte[]) bookingMessage.getLogMessage())); - when(redisSinkConfig.getSinkConnectorSchemaDataType()).thenReturn(SinkConnectorSchemaDataType.PROTOBUF); - ProtoOdpfMessageParser messageParser = (ProtoOdpfMessageParser) OdpfMessageParserFactory.getParser(redisSinkConfig, statsDReporter); + when(sinkConfig.getSinkConnectorSchemaDataType()).thenReturn(SinkConnectorSchemaDataType.PROTOBUF); + ProtoOdpfMessageParser messageParser = (ProtoOdpfMessageParser) OdpfMessageParserFactory.getParser(sinkConfig, statsDReporter); schemaTest = messageParser.getSchema("io.odpf.depot.TestMessage", descriptorsMap); schemaBooking = messageParser.getSchema("io.odpf.depot.TestBookingLogMessage", descriptorsMap); } @Test - public void shouldParseStringMessageForCollectionKeyTemplate() { + public void shouldParseStringMessageForCollectionKeyTemplate() throws InvalidTemplateException { Template template = new Template("Test-%s,order_number"); assertEquals("Test-test-order", template.parse(parsedTestMessage, schemaTest)); } @Test - public void shouldParseStringMessageWithSpacesForCollectionKeyTemplate() { + public void shouldParseStringMessageWithSpacesForCollectionKeyTemplate() throws InvalidTemplateException { Template template = new Template("Test-%s, order_number"); assertEquals("Test-test-order", template.parse(parsedTestMessage, schemaTest)); } @Test - public void shouldParseFloatMessageForCollectionKeyTemplate() { + public void shouldParseFloatMessageForCollectionKeyTemplate() throws InvalidTemplateException { Template template = new Template("Test-%.2f,amount_paid_by_cash"); assertEquals("Test-12.30", template.parse(parsedBookingMessage, schemaBooking)); } @Test - public void shouldParseLongMessageForCollectionKeyTemplate() { + public void shouldParseLongMessageForCollectionKeyTemplate() throws InvalidTemplateException { Template template = new Template("Test-%d,customer_total_fare_without_surge"); assertEquals("Test-2000", template.parse(parsedBookingMessage, schemaBooking)); } @Test public void shouldThrowExceptionForNullCollectionKeyTemplate() { - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template(null)); + InvalidTemplateException e = assertThrows(InvalidTemplateException.class, () -> new Template(null)); assertEquals("Template 'null' is invalid", e.getMessage()); } @Test public void shouldThrowExceptionForEmptyCollectionKeyTemplate() { - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template("")); + InvalidTemplateException e = assertThrows(InvalidTemplateException.class, () -> new Template("")); assertEquals("Template '' is invalid", e.getMessage()); } @Test - public void shouldAcceptStringForCollectionKey() { + public void shouldAcceptStringForCollectionKey() throws InvalidTemplateException { Template template = new Template("Test"); assertEquals("Test", template.parse(parsedBookingMessage, schemaBooking)); } @Test public void shouldNotAcceptStringWithPatternForCollectionKeyWithEmptyVariables() { - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> new Template("Test-%s%d%b,t1,t2")); + InvalidTemplateException e = assertThrows(InvalidTemplateException.class, () -> new Template("Test-%s%d%b,t1,t2")); Assert.assertEquals("Template is not valid, variables=3, validArgs=3, values=2", e.getMessage()); - e = assertThrows(IllegalArgumentException.class, () -> new Template("Test-%s%s%y,order_number,order_details")); + e = assertThrows(InvalidTemplateException.class, () -> new Template("Test-%s%s%y,order_number,order_details")); Assert.assertEquals("Template is not valid, variables=3, validArgs=2, values=2", e.getMessage()); } @Test - public void shouldAcceptStringWithPatternForCollectionKeyWithMultipleVariables() { + public void shouldAcceptStringWithPatternForCollectionKeyWithMultipleVariables() throws InvalidTemplateException { Template template = new Template("Test-%s::%s, order_number, order_details"); assertEquals("Test-test-order::ORDER-DETAILS", template.parse(parsedTestMessage, schemaTest)); } From 0170197b181233cb1f5fdf548d32833a032b19e4 Mon Sep 17 00:00:00 2001 From: lavkesh Date: Thu, 29 Sep 2022 15:22:41 +0800 Subject: [PATCH 09/11] chore: change exception message in Template --- src/main/java/io/odpf/depot/common/Template.java | 2 +- src/test/java/io/odpf/depot/common/TemplateTest.java | 4 ++-- .../odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/odpf/depot/common/Template.java b/src/main/java/io/odpf/depot/common/Template.java index 501a8991..dc62ff3d 100644 --- a/src/main/java/io/odpf/depot/common/Template.java +++ b/src/main/java/io/odpf/depot/common/Template.java @@ -15,7 +15,7 @@ public class Template { public Template(String template) throws InvalidTemplateException { if (template == null || template.isEmpty()) { - throw new InvalidTemplateException("Template '" + template + "' is invalid"); + throw new InvalidTemplateException("Template cannot be empty"); } List templateStrings = new ArrayList<>(); Splitter.on(",").omitEmptyStrings().split(template).forEach(s -> templateStrings.add(s.trim())); diff --git a/src/test/java/io/odpf/depot/common/TemplateTest.java b/src/test/java/io/odpf/depot/common/TemplateTest.java index 2d422218..b2a87b52 100644 --- a/src/test/java/io/odpf/depot/common/TemplateTest.java +++ b/src/test/java/io/odpf/depot/common/TemplateTest.java @@ -92,13 +92,13 @@ public void shouldParseLongMessageForCollectionKeyTemplate() throws InvalidTempl @Test public void shouldThrowExceptionForNullCollectionKeyTemplate() { InvalidTemplateException e = assertThrows(InvalidTemplateException.class, () -> new Template(null)); - assertEquals("Template 'null' is invalid", e.getMessage()); + assertEquals("Template cannot be empty", e.getMessage()); } @Test public void shouldThrowExceptionForEmptyCollectionKeyTemplate() { InvalidTemplateException e = assertThrows(InvalidTemplateException.class, () -> new Template("")); - assertEquals("Template '' is invalid", e.getMessage()); + assertEquals("Template cannot be empty", e.getMessage()); } @Test diff --git a/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java b/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java index 73e7b44a..eaebe1c0 100644 --- a/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java +++ b/src/test/java/io/odpf/depot/redis/parsers/RedisEntryParserFactoryTest.java @@ -78,7 +78,7 @@ public void shouldThrowExceptionForEmptyMappingKeyHashSet() { when(redisSinkConfig.getSinkRedisHashsetFieldToColumnMapping()).thenReturn(new JsonToPropertiesConverter().convert(null, "{\"order_details\":\"\"}")); IllegalArgumentException e = Assert.assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); - assertEquals("Template '' is invalid", e.getMessage()); + assertEquals("Template cannot be empty", e.getMessage()); } @Test @@ -104,6 +104,6 @@ public void shouldThrowExceptionForEmptyRedisTemplate() { when(redisSinkConfig.getSinkRedisKeyTemplate()).thenReturn(""); IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, () -> RedisEntryParserFactory.getRedisEntryParser(redisSinkConfig, statsDReporter, schema)); - assertEquals("Template '' is invalid", illegalArgumentException.getMessage()); + assertEquals("Template cannot be empty", illegalArgumentException.getMessage()); } } From d72d28f506420f5bcc78ea939070fdab5fc20515 Mon Sep 17 00:00:00 2001 From: Mayur Gubrele Date: Mon, 17 Oct 2022 09:58:17 +0530 Subject: [PATCH 10/11] feat: add bigtable sink metrics and logging (#51) --- .../io/odpf/depot/bigtable/BigTableSink.java | 6 +++- .../depot/bigtable/BigTableSinkFactory.java | 20 +++++++++-- .../depot/bigtable/client/BigTableClient.java | 28 +++++++++++++--- .../depot/bigtable/model/BigTableRecord.java | 5 +++ .../bigtable/parser/BigTableRecordParser.java | 8 ++++- .../odpf/depot/metrics/BigTableMetrics.java | 21 ++++++++++++ .../odpf/depot/bigtable/BigTableSinkTest.java | 6 +++- .../bigtable/client/BigTableClientTest.java | 33 +++++++++++++++++-- 8 files changed, 115 insertions(+), 12 deletions(-) create mode 100644 src/main/java/io/odpf/depot/metrics/BigTableMetrics.java diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSink.java b/src/main/java/io/odpf/depot/bigtable/BigTableSink.java index b0cf0f95..3f7642fa 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSink.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSink.java @@ -9,6 +9,7 @@ import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.error.ErrorInfo; import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.metrics.Instrumentation; import java.io.IOException; import java.util.List; @@ -18,10 +19,12 @@ public class BigTableSink implements OdpfSink { private final BigTableClient bigTableClient; private final BigTableRecordParser bigTableRecordParser; + private final Instrumentation instrumentation; - public BigTableSink(BigTableClient bigTableClient, BigTableRecordParser bigTableRecordParser) { + public BigTableSink(BigTableClient bigTableClient, BigTableRecordParser bigTableRecordParser, Instrumentation instrumentation) { this.bigTableClient = bigTableClient; this.bigTableRecordParser = bigTableRecordParser; + this.instrumentation = instrumentation; } @Override @@ -40,6 +43,7 @@ public OdpfSinkResponse pushToSink(List messages) { Map errorInfoMap = BigTableResponseParser.parseAndFillOdpfSinkResponse(validRecords, bigTableResponse); errorInfoMap.forEach(odpfSinkResponse::addErrors); } + instrumentation.logInfo("Pushed a batch of {} records to BigTable.", validRecords.size()); } return odpfSinkResponse; diff --git a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java index 533c5e6d..7fb47f5b 100644 --- a/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java +++ b/src/main/java/io/odpf/depot/bigtable/BigTableSinkFactory.java @@ -15,6 +15,8 @@ import io.odpf.depot.message.OdpfMessageParserFactory; import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.metrics.BigTableMetrics; +import io.odpf.depot.metrics.Instrumentation; import io.odpf.depot.metrics.StatsDReporter; import io.odpf.depot.utils.MessageConfigUtils; @@ -38,8 +40,20 @@ public BigTableSinkFactory(BigTableSinkConfig sinkConfig) { public void init() { try { + Instrumentation instrumentation = new Instrumentation(statsDReporter, BigTableSinkFactory.class); + String bigtableConfig = String.format("\n\tbigtable.gcloud.project = %s\n\tbigtable.instance = %s\n\tbigtable.table = %s" + + "\n\tbigtable.credential.path = %s\n\tbigtable.row.key.template = %s\n\tbigtable.column.family.mapping = %s\n\t", + sinkConfig.getGCloudProjectID(), + sinkConfig.getInstanceId(), + sinkConfig.getTableId(), + sinkConfig.getCredentialPath(), + sinkConfig.getRowKeyTemplate(), + sinkConfig.getColumnFamilyMapping()); + + instrumentation.logInfo(bigtableConfig); BigTableSchema bigtableSchema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); - bigTableClient = new BigTableClient(sinkConfig, bigtableSchema); + BigTableMetrics bigtableMetrics = new BigTableMetrics(sinkConfig); + bigTableClient = new BigTableClient(sinkConfig, bigtableSchema, bigtableMetrics, new Instrumentation(statsDReporter, BigTableClient.class)); bigTableClient.validateBigTableSchema(); Tuple modeAndSchema = MessageConfigUtils.getModeAndSchema(sinkConfig); @@ -54,6 +68,7 @@ public void init() { modeAndSchema, schema, bigtableSchema); + instrumentation.logInfo("Connection to bigtable established successfully"); } catch (IOException | InvalidTemplateException e) { throw new ConfigurationException("Exception occurred while creating sink", e); } @@ -62,6 +77,7 @@ public void init() { public OdpfSink create() { return new BigTableSink( bigTableClient, - bigTableRecordParser); + bigTableRecordParser, + new Instrumentation(statsDReporter, BigTableSink.class)); } } diff --git a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java index fd31ef6f..5ae34703 100644 --- a/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java +++ b/src/main/java/io/odpf/depot/bigtable/client/BigTableClient.java @@ -14,9 +14,12 @@ import io.odpf.depot.bigtable.model.BigTableSchema; import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; +import io.odpf.depot.metrics.BigTableMetrics; +import io.odpf.depot.metrics.Instrumentation; import java.io.FileInputStream; import java.io.IOException; +import java.time.Instant; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -26,16 +29,20 @@ public class BigTableClient { private final BigtableDataClient bigtableDataClient; private final BigTableSinkConfig sinkConfig; private final BigTableSchema bigtableSchema; + private final BigTableMetrics bigtableMetrics; + private final Instrumentation instrumentation; - public BigTableClient(BigTableSinkConfig sinkConfig, BigTableSchema bigtableSchema) throws IOException { - this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig), bigtableSchema); + public BigTableClient(BigTableSinkConfig sinkConfig, BigTableSchema bigtableSchema, BigTableMetrics bigtableMetrics, Instrumentation instrumentation) throws IOException { + this(sinkConfig, getBigTableDataClient(sinkConfig), getBigTableAdminClient(sinkConfig), bigtableSchema, bigtableMetrics, instrumentation); } - public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigTableSchema bigtableSchema) { + public BigTableClient(BigTableSinkConfig sinkConfig, BigtableDataClient bigtableDataClient, BigtableTableAdminClient bigtableTableAdminClient, BigTableSchema bigtableSchema, BigTableMetrics bigtableMetrics, Instrumentation instrumentation) { this.sinkConfig = sinkConfig; this.bigtableDataClient = bigtableDataClient; this.bigtableTableAdminClient = bigtableTableAdminClient; this.bigtableSchema = bigtableSchema; + this.bigtableMetrics = bigtableMetrics; + this.instrumentation = instrumentation; } private static BigtableDataClient getBigTableDataClient(BigTableSinkConfig sinkConfig) throws IOException { @@ -63,7 +70,18 @@ public BigTableResponse send(List records) throws MutateRowsExce batch.add(record.getRowMutationEntry()); } try { + Instant startTime = Instant.now(); bigtableDataClient.bulkMutateRows(batch); + instrumentation.captureDurationSince( + bigtableMetrics.getBigtableOperationLatencyMetric(), + startTime, + String.format(BigTableMetrics.BIGTABLE_INSTANCE_TAG, sinkConfig.getInstanceId()), + String.format(BigTableMetrics.BIGTABLE_TABLE_TAG, sinkConfig.getTableId())); + instrumentation.captureCount( + bigtableMetrics.getBigtableOperationTotalMetric(), + (long) batch.getEntryCount(), + String.format(BigTableMetrics.BIGTABLE_INSTANCE_TAG, sinkConfig.getInstanceId()), + String.format(BigTableMetrics.BIGTABLE_TABLE_TAG, sinkConfig.getTableId())); } catch (MutateRowsException e) { List failedMutations = e.getFailedMutations(); bigTableResponse = new BigTableResponse(failedMutations); @@ -74,13 +92,15 @@ public BigTableResponse send(List records) throws MutateRowsExce public void validateBigTableSchema() throws BigTableInvalidSchemaException { String tableId = sinkConfig.getTableId(); + instrumentation.logDebug(String.format("Validating schema for table: %s...", tableId)); checkIfTableExists(tableId); checkIfColumnFamiliesExist(tableId); + instrumentation.logDebug("Validation complete, Schema is valid."); } private void checkIfTableExists(String tableId) throws BigTableInvalidSchemaException { if (!bigtableTableAdminClient.exists(tableId)) { - throw new BigTableInvalidSchemaException(String.format("Table: %s does not exist", tableId)); + throw new BigTableInvalidSchemaException(String.format("Table: %s does not exist!", tableId)); } } diff --git a/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java b/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java index dcb28c66..86a60d7c 100644 --- a/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java +++ b/src/main/java/io/odpf/depot/bigtable/model/BigTableRecord.java @@ -12,4 +12,9 @@ public class BigTableRecord { private final long index; private final ErrorInfo errorInfo; private final boolean valid; + + @Override + public String toString() { + return String.format("RowMutationEntry: %s ValidRecord=%s", rowMutationEntry != null ? rowMutationEntry.toString() : "NULL", valid); + } } diff --git a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java index 567d6c80..679c53d4 100644 --- a/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java +++ b/src/main/java/io/odpf/depot/bigtable/parser/BigTableRecordParser.java @@ -11,10 +11,12 @@ import io.odpf.depot.message.OdpfMessageSchema; import io.odpf.depot.message.ParsedOdpfMessage; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.List; +@Slf4j public class BigTableRecordParser { private final OdpfMessageParser odpfMessageParser; private final BigTableRowKeyParser bigTableRowKeyParser; @@ -57,7 +59,11 @@ private BigTableRecord createRecord(OdpfMessage message, long index) { String value = String.valueOf(parsedOdpfMessage.getFieldByName(fieldName, schema)); rowMutationEntry.setCell(columnFamily, column, value); })); - return new BigTableRecord(rowMutationEntry, index, null, true); + BigTableRecord bigTableRecord = new BigTableRecord(rowMutationEntry, index, null, true); + if (log.isDebugEnabled()) { + log.debug(bigTableRecord.toString()); + } + return bigTableRecord; } catch (Exception e) { ErrorInfo errorInfo = new ErrorInfo(e, ErrorType.DESERIALIZATION_ERROR); return new BigTableRecord(null, index, errorInfo, false); diff --git a/src/main/java/io/odpf/depot/metrics/BigTableMetrics.java b/src/main/java/io/odpf/depot/metrics/BigTableMetrics.java new file mode 100644 index 00000000..73a08e3c --- /dev/null +++ b/src/main/java/io/odpf/depot/metrics/BigTableMetrics.java @@ -0,0 +1,21 @@ +package io.odpf.depot.metrics; + +import io.odpf.depot.config.OdpfSinkConfig; + +public class BigTableMetrics extends SinkMetrics { + + public static final String BIGTABLE_SINK_PREFIX = "bigtable_"; + public static final String BIGTABLE_INSTANCE_TAG = "instance=%s"; + public static final String BIGTABLE_TABLE_TAG = "table=%s"; + public BigTableMetrics(OdpfSinkConfig config) { + super(config); + } + + public String getBigtableOperationLatencyMetric() { + return getApplicationPrefix() + SINK_PREFIX + BIGTABLE_SINK_PREFIX + "operation_latency_milliseconds"; + } + + public String getBigtableOperationTotalMetric() { + return getApplicationPrefix() + SINK_PREFIX + BIGTABLE_SINK_PREFIX + "operation_total"; + } +} diff --git a/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java b/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java index 7af46fb0..9124b5f1 100644 --- a/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java +++ b/src/test/java/io/odpf/depot/bigtable/BigTableSinkTest.java @@ -11,6 +11,8 @@ import io.odpf.depot.error.ErrorInfo; import io.odpf.depot.error.ErrorType; import io.odpf.depot.message.OdpfMessage; +import io.odpf.depot.metrics.Instrumentation; +import io.odpf.depot.metrics.StatsDReporter; import org.aeonbits.owner.util.Collections; import org.junit.Assert; import org.junit.Before; @@ -27,6 +29,8 @@ public class BigTableSinkTest { private BigTableRecordParser bigTableRecordParser; @Mock private BigTableClient bigTableClient; + @Mock + private StatsDReporter statsDReporter; private BigTableSink bigTableSink; private List messages; @@ -56,7 +60,7 @@ public void setUp() { BigTableRecord bigTableRecord4 = new BigTableRecord(null, 4, errorInfo, false); invalidRecords = Collections.list(bigTableRecord3, bigTableRecord4); - bigTableSink = new BigTableSink(bigTableClient, bigTableRecordParser); + bigTableSink = new BigTableSink(bigTableClient, bigTableRecordParser, new Instrumentation(statsDReporter, BigTableSink.class)); } @Test diff --git a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java index b06df1ba..096d40a9 100644 --- a/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java +++ b/src/test/java/io/odpf/depot/bigtable/client/BigTableClientTest.java @@ -14,12 +14,15 @@ import io.odpf.depot.bigtable.response.BigTableResponse; import io.odpf.depot.config.BigTableSinkConfig; import io.odpf.depot.message.SinkConnectorSchemaMessageMode; +import io.odpf.depot.metrics.BigTableMetrics; +import io.odpf.depot.metrics.Instrumentation; import org.aeonbits.owner.ConfigFactory; import org.aeonbits.owner.util.Collections; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.io.IOException; @@ -27,7 +30,11 @@ import java.util.List; import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doThrow; public class BigTableClientTest { @@ -37,6 +44,10 @@ public class BigTableClientTest { private BigtableTableAdminClient bigtableTableAdminClient; @Mock private ApiException apiException; + @Mock + private BigTableMetrics bigtableMetrics; + @Mock + private Instrumentation instrumentation; private BigTableClient bigTableClient; private List validRecords; @@ -60,7 +71,7 @@ public void setUp() throws IOException { validRecords = Collections.list(bigTableRecord1, bigTableRecord2); sinkConfig = ConfigFactory.create(BigTableSinkConfig.class, System.getProperties()); BigTableSchema schema = new BigTableSchema(sinkConfig.getColumnFamilyMapping()); - bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient, schema); + bigTableClient = new BigTableClient(sinkConfig, bigTableDataClient, bigtableTableAdminClient, schema, bigtableMetrics, instrumentation); } @Test @@ -93,7 +104,7 @@ public void shouldThrowInvalidSchemaExceptionIfTableDoesNotExist() { try { bigTableClient.validateBigTableSchema(); } catch (BigTableInvalidSchemaException e) { - Assert.assertEquals("Table: " + sinkConfig.getTableId() + " does not exist", e.getMessage()); + Assert.assertEquals("Table: " + sinkConfig.getTableId() + " does not exist!", e.getMessage()); } } @@ -112,4 +123,20 @@ public void shouldThrowInvalidSchemaExceptionIfColumnFamilyDoesNotExist() { Assert.assertEquals("Column families [family-test] do not exist in table test-table!", e.getMessage()); } } + + @Test + public void shouldCaptureBigtableMetricsWhenBulkMutateRowsDoesNotThrowAnException() { + doNothing().when(bigTableDataClient).bulkMutateRows(isA(BulkMutation.class)); + + bigTableClient.send(validRecords); + + Mockito.verify(instrumentation, Mockito.times(1)).captureDurationSince(eq(bigtableMetrics.getBigtableOperationLatencyMetric()), + any(), + eq(String.format(BigTableMetrics.BIGTABLE_INSTANCE_TAG, sinkConfig.getInstanceId())), + eq(String.format(BigTableMetrics.BIGTABLE_TABLE_TAG, sinkConfig.getTableId()))); + Mockito.verify(instrumentation, Mockito.times(1)).captureCount(eq(bigtableMetrics.getBigtableOperationTotalMetric()), + any(), + eq(String.format(BigTableMetrics.BIGTABLE_INSTANCE_TAG, sinkConfig.getInstanceId())), + eq(String.format(BigTableMetrics.BIGTABLE_TABLE_TAG, sinkConfig.getTableId()))); + } } From 3ddd8090da4cdce80ccc2a72ae11ecd4e309562c Mon Sep 17 00:00:00 2001 From: MayurGubrele Date: Mon, 7 Nov 2022 11:39:53 +0530 Subject: [PATCH 11/11] docs: add bigtable sink documentation --- docs/README.md | 1 + docs/reference/configuration/README.md | 2 + docs/reference/configuration/bigtable.md | 61 ++++++++++++++++++++++++ docs/reference/metrics.md | 18 ++++++- docs/reference/odpf_sink_response.md | 3 +- docs/sinks/bigtable.md | 40 ++++++++++++++++ 6 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 docs/reference/configuration/bigtable.md create mode 100644 docs/sinks/bigtable.md diff --git a/docs/README.md b/docs/README.md index 31a55f8d..f3285d8c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,6 +11,7 @@ GRPC) * Log Sink * Bigquery Sink * Redis Sink +* Bigtable Sink Depot is a sink connector, which acts as a bridge between data processing systems and real sink. The APIs in this library can be used to push data to various sinks. Common sinks implementations will be added in this repo. diff --git a/docs/reference/configuration/README.md b/docs/reference/configuration/README.md index 642d1f34..859a854b 100644 --- a/docs/reference/configuration/README.md +++ b/docs/reference/configuration/README.md @@ -7,4 +7,6 @@ This page contains reference for all the configurations for sink connectors. * [Generic](generic.md) * [Stencil Client](stencil-client.md) * [Bigquery Sink](bigquery-sink.md) +* [Redis Sink](redis.md) +* [Bigtable Sink](bigtable.md) diff --git a/docs/reference/configuration/bigtable.md b/docs/reference/configuration/bigtable.md new file mode 100644 index 00000000..d99066a4 --- /dev/null +++ b/docs/reference/configuration/bigtable.md @@ -0,0 +1,61 @@ +# Bigtable Sink + +A Bigtable sink requires the following variables to be set along with Generic ones + +## `SINK_BIGTABLE_GOOGLE_CLOUD_PROJECT_ID` + +Contains information of google cloud project id of the bigtable table where the records need to be inserted/updated. Further +documentation on google cloud [project id](https://cloud.google.com/resource-manager/docs/creating-managing-projects). + +* Example value: `gcp-project-id` +* Type: `required` + +## `SINK_BIGTABLE_INSTANCE_ID` + +A Bigtable instance is a container for your data, which contain clusters that your applications can connect to. Each cluster contains nodes, compute units that manage your data and perform maintenance tasks. + +A table belongs to an instance, not to a cluster or node. Here you provide the name of that bigtable instance your table belongs to. Further +documentation on [bigtable Instances, clusters, and nodes](https://cloud.google.com/bigtable/docs/instances-clusters-nodes). + +* Example value: `cloud-bigtable-instance-id` +* Type: `required` + +## `SINK_BIGTABLE_CREDENTIAL_PATH` + +Full path of google cloud credentials file. Further documentation of google cloud authentication +and [credentials](https://cloud.google.com/docs/authentication/getting-started). + +* Example value: `/.secret/google-cloud-credentials.json` +* Type: `required` + +## `SINK_BIGTABLE_TABLE_ID` + +Bigtable stores data in massively scalable tables, each of which is a sorted key/value map. + +Here you provide the name of the table where the records need to be inserted/updated. Further documentation on +[bigtable tables](https://cloud.google.com/bigtable/docs/managing-tables). + +* Example value: `depot-sample-table` +* Type: `required` + +## `SINK_BIGTABLE_ROW_KEY_TEMPLATE` + +Bigtable tables are composed of rows, each of which typically describes a single entity. Each row is indexed by a single row key. + +Here you provide a string template which will be used to create row keys using one or many fields of your input data. Further documentation on [Bigtable storage model](https://cloud.google.com/bigtable/docs/overview#storage-model). + +In the example below, If field_1 and field_2 are `String` and `Integer` data types respectively with values as `alpha` and `10` for a specific record, row key generated for this record will be: `key-alpha-10` + +* Example value: `key-%s-%d, field_1, field_2` +* Type: `required` + +## `SINK_BIGTABLE_COLUMN_FAMILY_MAPPING` + +Bigtable columns that are related to one another are typically grouped into a column family. Each column is identified by a combination of the column family and a column qualifier, which is a unique name within the column family. + +Here you provide the mapping of the table's `column families` and `qualifiers`, and the field names from input data that we intent to insert into the table. Further documentation on [Bigtable storage model](https://cloud.google.com/bigtable/docs/overview#storage-model). + +Please note that `Column families` being provided in this configuration, need to exist in the table beforehand. While `Column Qualifiers` will be created if they don't exist. + +* Example value: `{ "depot-sample-family" : { "depot-sample-qualifier-1" : "field_1", "depot-sample-qualifier-2" : "field_7", "depot-sample-qualifier-3" : "field_5"} }` +* Type: `required` diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 8b926388..2ee2bdb6 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -6,10 +6,11 @@ Sinks can have their own metrics, and they will be emmited while using sink conn ## Table of Contents * [Bigquery Sink](metrics.md#bigquery-sink) +* [Bigtable Sink](metrics.md#bigtable-sink) ## Bigquery Sink -### `Biquery Operation Total` +### `Bigquery Operation Total` Total number of bigquery API operation performed @@ -19,7 +20,20 @@ Time taken for bigquery API operation performed ### `Bigquery Errors Total` -Total numbers of error occurred on bigquery insert operation. +Total numbers of error occurred on bigquery insert operation +## Bigtable Sink + +### `Bigtable Operation Total` + +Total number of bigtable insert/update operation performed + +### `Bigtable Operation Latency` + +Time taken for bigtable insert/update operation performed + +### `Bigtable Errors Total` + +Total numbers of error occurred on bigtable insert/update operation diff --git a/docs/reference/odpf_sink_response.md b/docs/reference/odpf_sink_response.md index d2cf9880..1fe749f3 100644 --- a/docs/reference/odpf_sink_response.md +++ b/docs/reference/odpf_sink_response.md @@ -17,7 +17,8 @@ These errors are returned by sinks in the OdpfSinkResponse object. The error typ * UNKNOWN_FIELDS_ERROR * SINK_4XX_ERROR * SINK_5XX_ERROR +* SINK_RETRYABLE_ERROR * SINK_UNKNOWN_ERROR * DEFAULT_ERROR - * If no error is specified + * If no error is specified (To be deprecated soon) diff --git a/docs/sinks/bigtable.md b/docs/sinks/bigtable.md new file mode 100644 index 00000000..2bcb7f52 --- /dev/null +++ b/docs/sinks/bigtable.md @@ -0,0 +1,40 @@ +# Bigtable Sink + +## Overview +Depot Bigtable Sink translates protobuf messages to bigtable records and insert them to a bigtable table. Its other responsibilities include validating the provided [column-family-schema](../reference/configuration/bigtable.md#sink_bigtable_column_family_mapping), and check whether the configured table exists in [Bigtable instance](../reference/configuration/bigtable.md#sink_bigtable_instance_id) or not. + +Depot uses [Java Client Library for the Cloud Bigtable API](https://cloud.google.com/bigtable/docs/reference/libraries) to perform any operations on Bigtable. + +## Setup Required +To be able to insert/update records in Bigtable, One must have following setup in place: + +* [Bigtable Instance](../reference/configuration/bigtable.md#sink_bigtable_instance_id) belonging to the [GCP project](../reference/configuration/bigtable.md#sink_bigtable_google_cloud_project_id) provided in configuration +* Bigtable [Table](../reference/configuration/bigtable.md#sink_bigtable_table_id) where the records are supposed to be inserted/updated +* Column families that are provided as part of [column-family-mapping](../reference/configuration/bigtable.md#sink_bigtable_column_family_mapping) +* Google cloud [Bigtable IAM permission](https://cloud.google.com/bigtable/docs/access-control) required to access and modify the configured Bigtable Instance and Table + +## Metrics + +Check out the list of [metrics](../reference/metrics.md#bigtable-sink) captured under Bigtable Sink. + +## Error Handling + +[BigtableResponse](../../src/main/java/io/odpf/depot/bigtable/response/BigTableResponse.java) class have the list of failed [mutations](https://cloud.google.com/bigtable/docs/writes#write-types). [BigtableResponseParser](../../src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java) looks for errors from each failed mutation and create [ErrorInfo](../../src/main/java/io/odpf/depot/error/ErrorInfo.java) objects based on the type/HttpStatusCode of the underlying error. This error info is then sent to the application. + +| Error From Bigtable | Error Type Captured | +| --------------- | -------------------- | +| Retryable Error | SINK_RETRYABLE_ERROR | +| Having status code in range 400-499 | SINK_4XX_ERROR | +| Having status code in range 500-599 | SINK_5XX_ERROR | +| Any other Error | SINK_UNKNOWN_ERROR | + +### Error Telemetry + +[BigtableResponseParser](../../src/main/java/io/odpf/depot/bigtable/parser/BigTableResponseParser.java) looks for any specific error types sent from Bigtable and capture those under [BigtableTotalErrorMetrics](../reference/metrics.md#bigtable-sink) with suitable error tags. + +| Error Type | Error Tag Assigned | +| --------------- | -------------------- | +| Bad Request | BAD_REQUEST | +| Quota Failure | QUOTA_FAILURE | +| Precondition Failure | PRECONDITION_FAILURE | +| Any other Error | RPC_FAILURE |