feat: improve memory usage of record export (#311)

.eslintrc.js

@@ -19,6 +19,15 @@ const config = {
         prefer: "type-imports",
       },
     ],
+    "@typescript-eslint/no-unused-vars": [
+      "warn",
+      {
+        argsIgnorePattern: "^_",
+        varsIgnorePattern: "^_",
+        caughtErrorsIgnorePattern: "^_",
+        destructuredArrayIgnorePattern: "^_",
+      },
+    ],
   },
 };
 module.exports = config;

src/record/delete/parsers/error.ts

@@ -1,29 +1,18 @@
 import { CsvError } from "csv-parse";
+import { CliKintoneError } from "../../../utils/error";
 
-export class ParserError extends Error {
-  private readonly cause: unknown;
-
+export class ParserError extends CliKintoneError {
   constructor(cause: unknown) {
     const message = "Failed to parse input";
-    super(message);
-
+    super(message, cause);
     this.name = "ParserError";
-    this.message = message;
-    this.cause = cause;
-
-    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    // Set the prototype explicitly.
     Object.setPrototypeOf(this, ParserError.prototype);
   }
 
-  toString(): string {
-    let errorMessage = "";
-    errorMessage += this.message + "\n";
+  protected _toStringCause(): string {
     if (this.cause instanceof CsvError) {
-      errorMessage += `${this.cause.code}: ${this.cause.message}\n`;
-    } else {
-      errorMessage += this.cause + "\n";
+      return `${this.cause.code}: ${this.cause.message}\n`;
     }
-    return errorMessage;
+    return super._toStringCause();
   }
 }

src/record/delete/usecases/deleteAll/error.ts

@@ -2,51 +2,40 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client";
 import type { KintoneRecordForDeleteAllParameter } from "../../../../kintone/types";
 import { kintoneAllRecordsErrorToString } from "../../../error";
 import { ErrorParser } from "../../utils/error";
+import { CliKintoneError } from "../../../../utils/error";
+
+export class DeleteAllRecordsError extends CliKintoneError {
+  readonly detail: string;
 
-export class DeleteAllRecordsError extends Error {
-  private readonly cause: unknown;
   private readonly records: KintoneRecordForDeleteAllParameter[];
   private readonly numOfSuccess: number;
   private readonly numOfTotal: number;
 
   constructor(cause: unknown, records: KintoneRecordForDeleteAllParameter[]) {
     const message = "Failed to delete all records.";
-    super(message);
+    super(message, cause);
 
     this.name = "DeleteAllRecordsError";
-    this.message = message;
-    this.cause = cause;
     this.records = records;
     this.numOfTotal = this.records.length;
     this.numOfSuccess = 0;
     if (this.cause instanceof KintoneAllRecordsError) {
       this.numOfSuccess = this.cause.numOfProcessedRecords;
     }
 
-    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    // Set the prototype explicitly.
-    Object.setPrototypeOf(this, DeleteAllRecordsError.prototype);
-  }
-
-  toString(): string {
-    let errorMessage = "";
-    errorMessage += this.message + "\n";
-
     if (this.numOfSuccess === 0) {
-      errorMessage += `No records are deleted.\n`;
+      this.detail = `No records are deleted.`;
     } else {
-      errorMessage += `${this.numOfSuccess}/${this.numOfTotal} records are deleted successfully.\n`;
+      this.detail = `${this.numOfSuccess}/${this.numOfTotal} records are deleted successfully.`;
     }
 
+    Object.setPrototypeOf(this, DeleteAllRecordsError.prototype);
+  }
+
+  protected _toStringCause(): string {
     if (this.cause instanceof KintoneAllRecordsError) {
-      errorMessage += kintoneAllRecordsErrorToString(
-        new ErrorParser(this.cause)
-      );
-    } else if (this.cause instanceof DeleteAllRecordsError) {
-      errorMessage += this.cause.toString();
-    } else {
-      errorMessage += this.cause + "\n";
+      return kintoneAllRecordsErrorToString(new ErrorParser(this.cause));
     }
-    return errorMessage;
+    return super._toStringCause();
   }
 }

src/record/delete/usecases/deleteByRecordNumber/error.ts

@@ -2,51 +2,40 @@ import { KintoneAllRecordsError } from "@kintone/rest-api-client";
 import type { KintoneRecordForDeleteAllParameter } from "../../../../kintone/types";
 import { kintoneAllRecordsErrorToString } from "../../../error";
 import { ErrorParser } from "../../utils/error";
+import { CliKintoneError } from "../../../../utils/error";
+
+export class DeleteSpecifiedRecordsError extends CliKintoneError {
+  readonly detail: string;
 
-export class DeleteSpecifiedRecordsError extends Error {
-  private readonly cause: unknown;
   private readonly records: KintoneRecordForDeleteAllParameter[];
   private readonly numOfSuccess: number;
   private readonly numOfTotal: number;
 
   constructor(cause: unknown, records: KintoneRecordForDeleteAllParameter[]) {
     const message = "Failed to delete records.";
-    super(message);
+    super(message, cause);
 
     this.name = "DeleteSpecifiedRecordsError";
-    this.message = message;
-    this.cause = cause;
     this.records = records;
     this.numOfTotal = this.records.length;
     this.numOfSuccess = 0;
     if (this.cause instanceof KintoneAllRecordsError) {
       this.numOfSuccess = this.cause.numOfProcessedRecords;
     }
 
-    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    // Set the prototype explicitly.
-    Object.setPrototypeOf(this, DeleteSpecifiedRecordsError.prototype);
-  }
-
-  toString(): string {
-    let errorMessage = "";
-    errorMessage += this.message + "\n";
-
     if (this.numOfSuccess === 0) {
-      errorMessage += `No records are deleted.\n`;
+      this.detail = `No records are deleted.`;
     } else {
-      errorMessage += `${this.numOfSuccess}/${this.numOfTotal} records are deleted successfully.\n`;
+      this.detail = `${this.numOfSuccess}/${this.numOfTotal} records are deleted successfully.`;
     }
 
+    Object.setPrototypeOf(this, DeleteSpecifiedRecordsError.prototype);
+  }
+
+  protected _toStringCause(): string {
     if (this.cause instanceof KintoneAllRecordsError) {
-      errorMessage += kintoneAllRecordsErrorToString(
-        new ErrorParser(this.cause)
-      );
-    } else if (this.cause instanceof DeleteSpecifiedRecordsError) {
-      errorMessage += this.cause.toString();
-    } else {
-      errorMessage += this.cause + "\n";
+      return kintoneAllRecordsErrorToString(new ErrorParser(this.cause));
     }
-    return errorMessage;
+    return super._toStringCause();
   }
 }

src/record/delete/validator/error.ts

@@ -1,24 +1,12 @@
-export class ValidatorError extends Error {
-  private readonly cause: unknown;
+import { CliKintoneError } from "../../../utils/error";
 
+export class ValidatorError extends CliKintoneError {
   constructor(cause: unknown) {
     const message = "Failed to delete records";
-    super(message);
+    super(message, cause);
 
     this.name = "ValidatorError";
-    this.message = message;
-    this.cause = cause;
 
-    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
-    // Set the prototype explicitly.
     Object.setPrototypeOf(this, ValidatorError.prototype);
   }
-
-  toString(): string {
-    let errorMessage = "";
-    errorMessage += this.message + "\n";
-    errorMessage += this.cause + "\n";
-
-    return errorMessage;
-  }
 }

src/record/delete/validator/index.ts

@@ -20,7 +20,7 @@ export const validateRecordNumbers: (
   const errorHandler = new ErrorHandler();
   const hasAppCodePrevious = hasAppCode(recordNumbers[0].value, appCode);
   const allRecordIds = await getAllRecordIds(apiClient, app);
-  recordNumbers.forEach((recordNumber, index) => {
+  recordNumbers.forEach((recordNumber) => {
     const value = recordNumber.value;
     if (value.length === 0 || !isValidRecordNumber(value, appCode)) {
       errorHandler.addInvalidValueError(recordNumber);

src/record/export/index.ts

@@ -3,11 +3,12 @@ import iconv from "iconv-lite";
 import type { RestAPIClientOptions } from "../../kintone/client";
 import { buildRestAPIClient } from "../../kintone/client";
 import { getRecords } from "./usecases/get";
-import { stringifierFactory } from "./stringifiers";
 import { createSchema } from "./schema";
 import { formLayout as defaultTransformer } from "./schema/transformers/formLayout";
 import { userSelected } from "./schema/transformers/userSelected";
 import { logger } from "../../utils/log";
+import { LocalRecordRepositoryFromStream } from "./repositories/localRecordRepositoryFromStream";
+import { Transform } from "stream";
 
 export type ExportFileEncoding = "utf8" | "sjis";
 
@@ -42,18 +43,22 @@ export const run: (
         ? userSelected(fields, fieldsJson, layoutJson)
         : defaultTransformer(layoutJson)
     );
-    const records = await getRecords(apiClient, app, schema, {
+
+    const repository = new LocalRecordRepositoryFromStream(
+      () => {
+        const encodeStream = Transform.from(iconv.encodeStream(encoding));
+        encodeStream.pipe(process.stdout);
+        return encodeStream;
+      },
+      schema,
+      !!attachmentsDir
+    );
+
+    await getRecords(apiClient, app, repository, schema, {
       condition,
       orderBy,
       attachmentsDir,
     });
-    const stringifier = stringifierFactory({
-      format: "csv",
-      schema,
-      useLocalFilePath: !!attachmentsDir,
-    });
-    const stringifiedRecords = stringifier(records);
-    process.stdout.write(iconv.encode(stringifiedRecords, encoding));
   } catch (e) {
     logger.error(e);
     // eslint-disable-next-line no-process-exit

src/record/export/repositories/localRecordRepositoryFromStream.ts

@@ -0,0 +1,30 @@
+import type { LocalRecordRepository } from "../usecases/interface";
+import type { RecordSchema } from "../types/schema";
+import { stringifierFactory } from "./stringifiers";
+
+export class LocalRecordRepositoryFromStream implements LocalRecordRepository {
+  readonly format = "csv";
+
+  private readonly openWritableSink: () => NodeJS.WritableStream;
+  private readonly schema: RecordSchema;
+  private readonly useLocalFilePath: boolean;
+
+  constructor(
+    openWritableSink: () => NodeJS.WritableStream,
+    schema: RecordSchema,
+    useLocalFilePath: boolean
+  ) {
+    this.openWritableSink = openWritableSink;
+    this.schema = schema;
+    this.useLocalFilePath = useLocalFilePath;
+  }
+  writer() {
+    const stringifier = stringifierFactory({
+      format: this.format,
+      schema: this.schema,
+      useLocalFilePath: this.useLocalFilePath,
+    });
+    stringifier.pipe(this.openWritableSink());
+    return stringifier;
+  }
+}

src/record/export/repositories/localRecordRepositoryMock.ts

@@ -0,0 +1,26 @@
+import type { LocalRecordRepository, Writer } from "../usecases/interface";
+import type { LocalRecord } from "../types/record";
+
+export class LocalRecordRepositoryMock implements LocalRecordRepository {
+  readonly format = "csv";
+
+  private underlyingSink: WritableMock = new WritableMock();
+
+  writer() {
+    this.underlyingSink = new WritableMock();
+    return this.underlyingSink;
+  }
+  receivedRecords() {
+    return this.underlyingSink.underlyingSink;
+  }
+}
+
+class WritableMock implements Writer {
+  public underlyingSink: LocalRecord[] = [];
+  async write(chunk: LocalRecord[]) {
+    this.underlyingSink.push(...chunk);
+  }
+  async end() {
+    /* noop */
+  }
+}

src/record/export/repositories/stringifiers/__tests__/fixtures/canStringifyRecordsCorrectly/index.ts

@@ -0,0 +1,9 @@
+import { input } from "./input";
+import { expected } from "./expected_csv";
+import { schema } from "./schema";
+
+export const pattern = {
+  input: input,
+  expected: expected,
+  schema: schema,
+};

src/record/export/stringifiers/__tests__/fixtures/input.ts → src/record/export/repositories/stringifiers/__tests__/fixtures/canStringifyRecordsCorrectly/input.ts

@@ -1,6 +1,6 @@
-import type { KintoneRecord } from "../../../types/record";
+import type { LocalRecord } from "../../../../../types/record";
 
-export const input: KintoneRecord[] = [
+export const input: LocalRecord[] = [
   {
     レコード番号: {
       type: "RECORD_NUMBER",

src/record/export/stringifiers/__tests__/fixtures/schema.ts → src/record/export/repositories/stringifiers/__tests__/fixtures/canStringifyRecordsCorrectly/schema.ts

@@ -1,4 +1,4 @@
-import type { RecordSchema } from "../../../types/schema";
+import type { RecordSchema } from "../../../../../types/schema";
 
 export const schema: RecordSchema = {
   hasSubtable: false,

src/record/export/repositories/stringifiers/__tests__/fixtures/canStringifyRecordsCorrectlyIterative/expected_csv.ts

@@ -0,0 +1,5 @@
+export const expected = `"レコード番号","文字列__1行_","作成日時"
+"1","文字列１２３","2022-05-02T07:58:00Z"
+"2","文字列２３４","2022-05-02T07:58:00Z"
+"3","文字列３４５","2022-05-02T07:58:00Z"
+`;

src/record/export/repositories/stringifiers/__tests__/fixtures/canStringifyRecordsCorrectlyIterative/index.ts

@@ -0,0 +1,9 @@
+import { input } from "./input";
+import { expected } from "./expected_csv";
+import { schema } from "./schema";
+
+export const pattern = {
+  input: input,
+  expected: expected,
+  schema: schema,
+};