From bf31a0122c602e2207778dd524193beccfa332d1 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Tue, 24 Dec 2024 16:24:41 +0530 Subject: [PATCH 1/5] Data chunk and task result enums and dtos --- .../dataimport/datachunk/ImportDataChunk.java | 14 +++++++ .../datachunk/ImportDataChunkStatus.java | 41 +++++++++++++++++++ .../datachunk/ImportDataChunkStatusState.java | 8 ++++ .../core/dataimport/datachunk/ImportRow.java | 11 +++++ .../dataimport/task/ImportTaskAction.java | 9 ++++ .../task/result/ImportResultStatus.java | 12 ++++++ .../task/result/ImportTargetResult.java | 19 +++++++++ .../task/result/ImportTargetResultStatus.java | 12 ++++++ .../task/result/ImportTaskResult.java | 27 ++++++++++++ 9 files changed, 153 insertions(+) create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java new file mode 100644 index 0000000000..69ed97421a --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunk.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import java.util.List; +import lombok.Builder; +import lombok.Data; + +/** * Import data chunk data */ +@Data +@Builder +public class ImportDataChunk { + + int dataChunkId; + List sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java new file mode 100644 index 0000000000..d6db3e1e7f --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatus.java @@ -0,0 +1,41 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.time.Instant; +import lombok.Builder; +import lombok.Data; + +/** * A DTO to store import data chunk details */ +@Data +@Builder +@JsonDeserialize(builder = ImportDataChunkStatus.ImportDataChunkStatusBuilder.class) +public class ImportDataChunkStatus { + + @JsonProperty("dataChunkId") + private final int dataChunkId; + + @JsonProperty("startTime") + private final Instant startTime; + + @JsonProperty("endTime") + private final Instant endTime; + + @JsonProperty("totalRecords") + private final int totalRecords; + + @JsonProperty("successCount") + private final int successCount; + + @JsonProperty("failureCount") + private final int failureCount; + + @JsonProperty("batchCount") + private final int batchCount; + + @JsonProperty("totalDurationInMilliSeconds") + private final int totalDurationInMilliSeconds; + + @JsonProperty("status") + private final ImportDataChunkStatusState status; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java new file mode 100644 index 0000000000..013c640dcf --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java @@ -0,0 +1,8 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +/** * Status of the import data chunk which during the import process */ +public enum ImportDataChunkStatusState { + START, + IN_PROGRESS, + COMPLETE +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java new file mode 100644 index 0000000000..824ca4ffa4 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportRow.java @@ -0,0 +1,11 @@ +package com.scalar.db.dataloader.core.dataimport.datachunk; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Value; + +/** Stores data related to a single row on import file */ +@Value +public class ImportRow { + int rowNumber; + JsonNode sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java new file mode 100644 index 0000000000..b1850cb84d --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java @@ -0,0 +1,9 @@ +package com.scalar.db.dataloader.core.dataimport.task; + +/*** + * Import task actions + */ +public enum ImportTaskAction { + INSERT, + UPDATE, +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java new file mode 100644 index 0000000000..70c03b9c37 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportResultStatus.java @@ -0,0 +1,12 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +public enum ImportResultStatus { + SUCCESS, + PARTIAL_SUCCESS, + FAILURE, + VALIDATION_FAILED, + RETRIEVAL_FAILED, + MAPPING_FAILED, + TIMEOUT, + CANCELLED +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java new file mode 100644 index 0000000000..0fe4e0379d --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResult.java @@ -0,0 +1,19 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +import com.fasterxml.jackson.databind.JsonNode; +import com.scalar.db.dataloader.core.dataimport.task.ImportTaskAction; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +@Builder +@Value +public class ImportTargetResult { + String namespace; + String tableName; + ImportTaskAction importAction; + List errors; + boolean dataMapped; + JsonNode importedRecord; + ImportTargetResultStatus status; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java new file mode 100644 index 0000000000..d774f8f823 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTargetResultStatus.java @@ -0,0 +1,12 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +public enum ImportTargetResultStatus { + VALIDATION_FAILED, + RETRIEVAL_FAILED, + MISSING_COLUMNS, + DATA_ALREADY_EXISTS, + DATA_NOT_FOUND, + SAVE_FAILED, + SAVED, + ABORTED +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java new file mode 100644 index 0000000000..bebffdbe4d --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java @@ -0,0 +1,27 @@ +package com.scalar.db.dataloader.core.dataimport.task.result; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.List; +import lombok.Builder; +import lombok.Setter; +import lombok.Value; + +@Builder +@Value +@JsonDeserialize(builder = ImportTaskResult.ImportTaskResultBuilder.class) +public class ImportTaskResult { + @JsonProperty("rowNumber") + int rowNumber; + + @JsonProperty("targets") + List targets; + + @JsonProperty("rawRecord") + JsonNode rawRecord; + + @Setter + @JsonProperty("dataChunkId") + int dataChunkId; +} From 57cd3304a09e2434ea1e6e553b7241ecd2705720 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Tue, 24 Dec 2024 16:43:35 +0530 Subject: [PATCH 2/5] Spotless applied --- .../db/dataloader/core/dataimport/task/ImportTaskAction.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java index b1850cb84d..6b8c95d2a8 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/ImportTaskAction.java @@ -1,8 +1,6 @@ package com.scalar.db.dataloader.core.dataimport.task; -/*** - * Import task actions - */ +/** * Import task actions */ public enum ImportTaskAction { INSERT, UPDATE, From 7a39564e310a2072cf9af155da564513e7d6af61 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 26 Dec 2024 11:44:40 +0530 Subject: [PATCH 3/5] Changes --- .../core/dataimport/task/result/ImportTaskResult.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java index bebffdbe4d..3e08cc709b 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/task/result/ImportTaskResult.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.List; import lombok.Builder; -import lombok.Setter; import lombok.Value; @Builder @@ -21,7 +20,6 @@ public class ImportTaskResult { @JsonProperty("rawRecord") JsonNode rawRecord; - @Setter @JsonProperty("dataChunkId") int dataChunkId; } From be4583c8a2c35e0d8bae97546f2d113a84a5b036 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Fri, 3 Jan 2025 17:17:19 +0530 Subject: [PATCH 4/5] Added transaction batch dtos --- .../ImportTransactionBatch.java | 14 ++++++++ .../ImportTransactionBatchResult.java | 32 +++++++++++++++++++ .../ImportTransactionBatchStatus.java | 18 +++++++++++ 3 files changed, 64 insertions(+) create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java create mode 100644 data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java new file mode 100644 index 0000000000..a922fd8afa --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatch.java @@ -0,0 +1,14 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.scalar.db.dataloader.core.dataimport.datachunk.ImportRow; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Transaction batch details */ +@Builder +@Value +public class ImportTransactionBatch { + int transactionBatchId; + List sourceData; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java new file mode 100644 index 0000000000..0e44b66953 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchResult.java @@ -0,0 +1,32 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Transaction batch result */ +@Builder +@Value +@JsonDeserialize(builder = ImportTransactionBatchResult.ImportTransactionBatchResultBuilder.class) +public class ImportTransactionBatchResult { + @JsonProperty("dataChunkId") + int dataChunkId; + + @JsonProperty("transactionBatchId") + int transactionBatchId; + + @JsonProperty("transactionId") + String transactionId; + + @JsonProperty("records") + List records; + + @JsonProperty("errors") + List errors; + + @JsonProperty("success") + boolean success; +} diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java new file mode 100644 index 0000000000..1b7bae34c6 --- /dev/null +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/transactionbatch/ImportTransactionBatchStatus.java @@ -0,0 +1,18 @@ +package com.scalar.db.dataloader.core.dataimport.transactionbatch; + +import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult; +import java.util.List; +import lombok.Builder; +import lombok.Value; + +/** Batch status details */ +@Builder +@Value +public class ImportTransactionBatchStatus { + int dataChunkId; + int transactionBatchId; + String transactionId; + List records; + List errors; + boolean success; +} From e584ace89421f00d1e8dd1287fa0a4bf5dcc18b9 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 29 Jan 2025 13:45:22 +0530 Subject: [PATCH 5/5] Corrected comment --- .../core/dataimport/datachunk/ImportDataChunkStatusState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java index 013c640dcf..9a6398a45e 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/datachunk/ImportDataChunkStatusState.java @@ -1,6 +1,6 @@ package com.scalar.db.dataloader.core.dataimport.datachunk; -/** * Status of the import data chunk which during the import process */ +/** * Status of the import data chunk during the import process */ public enum ImportDataChunkStatusState { START, IN_PROGRESS,