From 6669311e00247695e4fac8c7f19488a974a1e275 Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:57:36 +0100 Subject: [PATCH 1/7] fix empty arrow part --- src/api/transaction/transactionUtils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/transaction/transactionUtils.ts b/src/api/transaction/transactionUtils.ts index 565745b..d7ccce7 100644 --- a/src/api/transaction/transactionUtils.ts +++ b/src/api/transaction/transactionUtils.ts @@ -77,6 +77,10 @@ const MAX_ARROW_SIZE = 2147483647; export async function readArrowFiles(files: TransactionAsyncFile[]) { const results: ArrowResult[] = []; + if (!files.length) { + throw new Error('No files found'); + } + for (const file of files) { if ( typeof file.file !== 'string' && @@ -92,7 +96,7 @@ export async function readArrowFiles(files: TransactionAsyncFile[]) { ); } - const table = await tableFromIPC(file.file.stream()); + const table = tableFromIPC(file.file.stream()); results.push({ relationId: file.name, From 9a2b47f24a371eb75cbbd464c217fef29e0a0f2d Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:00:11 +0100 Subject: [PATCH 2/7] add await --- src/api/transaction/transactionUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/transaction/transactionUtils.ts b/src/api/transaction/transactionUtils.ts index d7ccce7..dffaacc 100644 --- a/src/api/transaction/transactionUtils.ts +++ b/src/api/transaction/transactionUtils.ts @@ -96,7 +96,7 @@ export async function readArrowFiles(files: TransactionAsyncFile[]) { ); } - const table = tableFromIPC(file.file.stream()); + const table = await tableFromIPC(file.file.stream()); results.push({ relationId: file.name, From 967951feae1b12edab63ecb6fc9c88c2d54b1ced Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:13:31 +0100 Subject: [PATCH 3/7] fix add empty relation size error --- src/api/transaction/transactionUtils.ts | 8 +++----- src/errors.ts | 11 +++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/api/transaction/transactionUtils.ts b/src/api/transaction/transactionUtils.ts index dffaacc..09e693a 100644 --- a/src/api/transaction/transactionUtils.ts +++ b/src/api/transaction/transactionUtils.ts @@ -16,7 +16,7 @@ import { tableFromIPC } from 'apache-arrow'; -import { MaxRelationSizeError } from '../../errors'; +import { EmptyRelationSizeError, MaxRelationSizeError } from '../../errors'; import { MetadataInfo } from '../../proto/generated/message'; import { RelationId } from '../../proto/generated/schema'; import { @@ -77,10 +77,6 @@ const MAX_ARROW_SIZE = 2147483647; export async function readArrowFiles(files: TransactionAsyncFile[]) { const results: ArrowResult[] = []; - if (!files.length) { - throw new Error('No files found'); - } - for (const file of files) { if ( typeof file.file !== 'string' && @@ -94,6 +90,8 @@ export async function readArrowFiles(files: TransactionAsyncFile[]) { file.file.size, MAX_ARROW_SIZE, ); + } else if (file.file.size === 0) { + throw new EmptyRelationSizeError(file.name); } const table = await tableFromIPC(file.file.stream()); diff --git a/src/errors.ts b/src/errors.ts index 318e21d..7ae78b5 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -82,6 +82,16 @@ export class MaxRelationSizeError extends Error { } } +export class EmptyRelationSizeError extends Error { + constructor(public relationId: string) { + const message = `Empty relation size of 0 bytes. Relation: ${relationId}`; + + super(message); + + this.name = 'EmptyRelationSizeError'; + } +} + export class AbortError extends Error { constructor(message?: string) { super(message); @@ -99,4 +109,5 @@ export type SdkError = | ApiError | TransactionError | MaxRelationSizeError + | EmptyRelationSizeError | Error; From 39a1bed50fca0aec66f2e4af8bd5fae632733995 Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:13:13 +0100 Subject: [PATCH 4/7] added comment --- src/api/transaction/transactionUtils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/api/transaction/transactionUtils.ts b/src/api/transaction/transactionUtils.ts index 09e693a..3685c88 100644 --- a/src/api/transaction/transactionUtils.ts +++ b/src/api/transaction/transactionUtils.ts @@ -90,7 +90,12 @@ export async function readArrowFiles(files: TransactionAsyncFile[]) { file.file.size, MAX_ARROW_SIZE, ); - } else if (file.file.size === 0) { + } + + // The part that exceeds 2GB is returned as a blob of 0 bytes, + // all the remaining parts are empty as well in Windows’s Chrome, + // therefore, throwing the error here to avoid failures downstream + if (file.file.size === 0) { throw new EmptyRelationSizeError(file.name); } From aee1390fe9419241504a2cd247729e25df926b25 Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:41:07 +0100 Subject: [PATCH 5/7] added changelog and updated version --- CHANGELOG.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b02438d..3b87391 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Change Log +## v0.7.6-alpha +- Added error to handle empty relation size of 0 bytes when reading arrow files + ## v0.7.5-alpha - Updated some dependencies. diff --git a/package.json b/package.json index aba4bb1..6c2cf80 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@relationalai/rai-sdk-javascript", "description": "RelationalAI SDK for JavaScript", - "version": "0.7.5-alpha", + "version": "0.7.6-alpha", "author": { "name": "RelationalAI", "url": "https://relational.ai" From 8e262a33c5d10b135a742bb0c8d60c25a2e4112a Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:42:22 +0100 Subject: [PATCH 6/7] add space --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b87391..84518dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log ## v0.7.6-alpha + - Added error to handle empty relation size of 0 bytes when reading arrow files ## v0.7.5-alpha From fe35c88fa5267a91b6e5c290a3ca9a7e6be80cd8 Mon Sep 17 00:00:00 2001 From: meruyert93 <61665812+meruyert93@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:42:14 +0100 Subject: [PATCH 7/7] fix grammar --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84518dc..a148a7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v0.7.6-alpha -- Added error to handle empty relation size of 0 bytes when reading arrow files +- Added an error to handle empty relation size of 0 bytes when reading arrow files ## v0.7.5-alpha