Fix/normalize export data

api/src/lib/file/index.ts

@@ -1,7 +1,47 @@
+import { join } from "@/lib/path/index.ts";
+
 let tmpDir: string;
+
+/**
+ * Create a new temporary directory if it doesn't exist yet.
+ * Otherwise, return the existing one from the tmpDir variable.
+ */
 export function getTmpDir(): string {
   if (!tmpDir) {
-    tmpDir = Deno.makeTempDirSync();
+    const parts = Deno.makeTempDirSync().split("/");
+    parts.pop();
+    tmpDir = "/" + join(...parts);
   }
+
   return tmpDir;
 }
+
+export type OpenFileOptions = Deno.OpenOptions;
+export type OpenFileDescriptor = Deno.FsFile;
+
+/**
+ * Create and open a file for reading and writing.
+ *
+ * @example
+ * const fd = await open("example.txt", { read: true, write: true });
+ * fd.close();
+ *
+ * @example
+ * const df = await open("example.txt", { write: true, append: true });
+ * await fd.write(new TextEncoder().encode("Hello, World!"));
+ * await fd.write(new TextEncoder().encode("Hello, World!"));
+ * fd.close();
+ *
+ * @param filepath
+ * @param options
+ * @returns
+ */
+export function open(
+  filepath: string,
+  options: OpenFileOptions = { read: true },
+): Promise<OpenFileDescriptor> {
+  return Deno.open(filepath, {
+    ...options,
+    create: true,
+  });
+}

api/src/lib/object/get.unit.spec.ts

@@ -0,0 +1,49 @@
+import { assertEquals, describe, it } from "@/dev_deps.ts";
+import { get } from "@/lib/object/index.ts";
+
+describe("get object helper", () => {
+  it("should get value from object", () => {
+    const obj = { a: { b: { c: 1 } } };
+    assertEquals(get(obj, "a.b.c"), 1);
+  });
+
+  it("should get value from indexed array", () => {
+    const obj = { a: { b: { c: [1, 2, 3] } } };
+    assertEquals(get(obj, "a.b.c.1"), 2);
+  });
+
+  it("should return default value if not found", () => {
+    const obj = { a: { b: { c: 1 } } };
+    assertEquals(get(obj, "a.b.d", 2), 2);
+  });
+
+  it("should return value from an array of objects", () => {
+    const obj = { a: [{ b: 1 }, { b: 2 }] };
+    assertEquals(get(obj, "a.1.b"), 2);
+  });
+
+  it("should return value from an array of objects at root level", () => {
+    const arr = [{ b: 1 }, { b: 2 }];
+    assertEquals(get(arr, "0.b"), 1);
+    assertEquals(get(arr, "1.b"), 2);
+  });
+
+  it("should return default value if the obj is null or undefined", () => {
+    assertEquals(get(null, "a.b.c", 2), 2);
+    assertEquals(get(undefined, "a.b.c", 2), 2);
+  });
+
+  it("should return null or undefined if the path ends and the value is null", () => {
+    const obj = { a: { b: null, c: undefined } };
+    assertEquals(get(obj, "a.b"), null);
+    assertEquals(get(obj, "a.c"), undefined);
+    assertEquals(get(obj, "a.d", 2), 2);
+  });
+
+  it("should return undefined as defaultValue if the defaultValue is not provided", () => {
+    const obj = { a: { b: 1 } };
+    assertEquals(get(obj, "a.b"), 1);
+    assertEquals(get(obj, "a.c", 2), 2);
+    assertEquals(get(obj, "a.d"), undefined);
+  });
+});

api/src/lib/object/index.ts

@@ -1,21 +1,33 @@
 import { collections } from "@/deps.ts";
 
-export function get<T, K extends keyof T>(
-  obj: T,
+/**
+ * Get the value at the given path of object.
+ * If the resolved value is undefined, the defaultValue is returned in its place.
+ *
+ * @param obj
+ * @param path
+ * @param defaultValue
+ * @returns
+ */
+export function get<TObject, TValue>(
+  obj: TObject,
   path: string | string[],
-  defaultValue?: any,
-): any {
+  defaultValue: TValue | undefined = undefined,
+): TValue | null | undefined {
   const keys = Array.isArray(path) ? path : path.split(".");
-  let result: any = obj;
+  let result: unknown = obj;
 
-  for (const key of keys) {
-    result = result[key as K];
-    if (result === undefined) {
-      return defaultValue;
-    }
+  while (keys.length) {
+    const key = keys.shift()!;
+
+    if (result === undefined) return defaultValue;
+    if (result === null && keys.length) return defaultValue;
+    if (key in Object(result) === false) return defaultValue;
+    if (result === null && !keys.length) return null;
+    result = (result as Record<string, unknown>)[key];
   }
 
-  return result;
+  return result as TValue | null | undefined;
 }
 
 export function set<T>(obj: T, path: string | string[], value: any): T {

api/src/lib/process/index.ts

@@ -85,9 +85,9 @@ export function catchErrors(
   }
   globalThis.addEventListener("error", uncaughtExceptionHandler);
 
-  async function unhandledRejectionHandler(e: Event) {
+  async function unhandledRejectionHandler(e: Event & { reason: unknown }) {
+    logger.error("Unhandled Rejection", e.reason);
     e.preventDefault();
-    logger.error("unhandled promise rejection", e);
 
     // shut down anyway after `timeout` seconds
     if (timeout) {

api/src/pdc/providers/middleware/Cast/CastToArrayMiddleware.ts

@@ -32,7 +32,9 @@ export class CastToArrayMiddleware implements MiddlewareInterface<HelperArgs> {
       }
 
       // cast the property to an array
-      set(params, path, Array.isArray(oldValue) ? oldValue : [oldValue]);
+      if (oldValue !== undefined) {
+        set(params, path, Array.isArray(oldValue) ? oldValue : [oldValue]);
+      }
     }
 
     return next(params, context);

api/src/pdc/services/trip/actions/file/BuildFile.ts

@@ -1,6 +1,6 @@
-import { FileHandle, open, Stringifier, stringify } from "@/deps.ts";
+import { Stringifier, stringify } from "@/deps.ts";
 import { provider } from "@/ilos/common/index.ts";
-import { getTmpDir } from "@/lib/file/index.ts";
+import { getTmpDir, open, OpenFileDescriptor } from "@/lib/file/index.ts";
 import { logger } from "@/lib/logger/index.ts";
 import { join } from "@/lib/path/index.ts";
 import { v4 as uuidV4 } from "@/lib/uuid/index.ts";
@@ -19,7 +19,7 @@ import { BuildExportAction } from "../BuildExportAction.ts";
 
 @provider()
 export class BuildFile {
-  constructor() {}
+  private readonly batchSize = 1000;
 
   public async buildCsvFromCursor(
     cursor: PgCursorHandler<ExportTripInterface>,
@@ -28,9 +28,14 @@ export class BuildFile {
     isOpendata: boolean,
   ): Promise<string> {
     // CSV file
-    const { filename, tz } = this.cast(params.type, params, date);
+    const { filename, tz } = this.cast(
+      params.type || "opendata",
+      params,
+      date,
+    );
     const filepath = join(getTmpDir(), filename);
-    const fd = await open(filepath, "a");
+    const fd = await open(filepath, { write: true, append: true });
+    logger.debug(`Exporting file to ${filepath}`);
 
     // Transform data
     const stringifier = this.configure(fd, params.type);
@@ -39,7 +44,7 @@ export class BuildFile {
     try {
       let count = 0;
       do {
-        const results = await cursor.read(10);
+        const results = await cursor.read(this.batchSize);
         count = results.length;
         for (const line of results) {
           stringifier.write(normalizeMethod(line, tz));
@@ -49,15 +54,18 @@ export class BuildFile {
       // Release the db, end the stream and close the file
       await cursor.release();
       stringifier.end();
-      await fd.close();
+      fd.close();
 
       logger.debug(`Finished exporting file: ${filepath}`);
 
       return filepath;
     } catch (e) {
       await cursor.release();
-      await fd.close();
-      logger.error(e.message, e.stack);
+      stringifier.end();
+      fd.close();
+
+      logger.error(e.message);
+      logger.debug(e.stack);
       throw e;
     }
   }
@@ -76,27 +84,28 @@ export class BuildFile {
   }
 
   private configure(
-    fd: FileHandle,
+    fd: OpenFileDescriptor,
     type = "opendata",
   ): Stringifier {
     const stringifier = stringify({
       delimiter: ";",
       header: true,
       columns: BuildExportAction.getColumns(type),
       cast: {
-        boolean: (b: Boolean): string => (b ? "OUI" : "NON"),
+        boolean: (b: boolean): string => (b ? "OUI" : "NON"),
         date: (d: Date): string => d.toISOString(),
-        number: (n: Number): string => n.toString().replace(".", ","),
+        number: (n: number): string => n.toString().replace(".", ","),
       },
       quoted: true,
       quoted_empty: true,
       quoted_string: true,
     });
 
     stringifier.on("readable", async () => {
-      let row: string;
+      let row: string = "";
       while ((row = stringifier.read()) !== null) {
-        await fd.appendFile(row, { encoding: "utf8" });
+        if (row === "") continue;
+        await fd.write(new TextEncoder().encode(row + "\n"));
       }
     });
 

api/src/pdc/services/trip/helpers/normalizeExportDataHelper.ts

@@ -77,8 +77,8 @@ function normalize(
   timeZone: string,
 ): {
   data: FlattenTripInterface;
-  driver_incentive_raw;
-  passenger_incentive_raw;
+  driver_incentive_raw: any;
+  passenger_incentive_raw: any;
 } {
   const jsd = toZonedTime(src.journey_start_datetime, timeZone);
   const jed = toZonedTime(src.journey_end_datetime, timeZone);

api/src/shared/trip/sendExport.contract.ts

@@ -1,4 +1,4 @@
-import { TerritorySelectorsInterface } from '../territory/common/interfaces/TerritoryCodeInterface.ts';
+import { TerritorySelectorsInterface } from "../territory/common/interfaces/TerritoryCodeInterface.ts";
 
 export interface ParamsInterface {
   format: {
@@ -21,13 +21,21 @@ export interface ParamsInterface {
   };
 }
 
-export type ExportType = 'opendata' | 'export' | 'registry' | 'operator' | 'territory';
+export const ExportTypeList = [
+  "opendata",
+  "export",
+  "registry",
+  "operator",
+  "territory",
+] as const;
+export type ExportType = typeof ExportTypeList[number];
 
-export type ResultInterface = void;
+export type ResultInterface = undefined;
 
 export const handlerConfig = {
-  service: 'trip',
-  method: 'sendExport',
+  service: "trip",
+  method: "sendExport",
 } as const;
 
-export const signature = `${handlerConfig.service}:${handlerConfig.method}` as const;
+export const signature =
+  `${handlerConfig.service}:${handlerConfig.method}` as const;