Merge pull request #22 from rowyio/develop

Develop

deploy.sh

@@ -19,7 +19,7 @@ while test $# -gt 0; do
                    return 1;
                    ;;
           esac
-  done  
+  done
 
 if [[ -z "$project_id" ]];
 then

package.json

@@ -1,7 +1,7 @@
 {
   "name": "rowy-backend",
   "description": "cloud run instance for running Rowy's backend functionality",
-  "version": "1.6.4",
+  "version": "1.7.0",
   "private": true,
   "main": "build/index.js",
   "scripts": {

src/functionBuilder/compiler/loader/index.ts

@@ -55,6 +55,7 @@ export const getConfigFromTableSchema = async (
       fieldTypes,
       extensions,
       runtimeOptions,
+      tableSchema: schemaData,
     };
     if (schemaData.searchIndex) {
       config.searchIndex = {
@@ -86,6 +87,7 @@ export const combineConfigs = (configs: any[]) =>
         extensions,
         searchIndex,
         runtimeOptions,
+        tableSchema,
       } = cur;
       return {
         derivativeColumns: [...acc.derivativeColumns, ...derivativeColumns],
@@ -105,6 +107,7 @@ export const combineConfigs = (configs: any[]) =>
           ? [...acc.searchIndices, searchIndex]
           : acc.searchIndices,
         runtimeOptions,
+        tableSchema,
       };
     },
     {
@@ -131,6 +134,7 @@ export const generateFile = async (configData, buildFolderTimestamp) => {
     region,
     searchHost,
     runtimeOptions,
+    tableSchema,
   } = configData;
   const serializedConfigData = {
     fieldTypes: JSON.stringify(fieldTypes),
@@ -144,6 +148,7 @@ export const generateFile = async (configData, buildFolderTimestamp) => {
       serviceAccount: `rowy-functions@${projectId}.iam.gserviceaccount.com`,
       ...runtimeOptions,
     }),
+    tableSchema: JSON.stringify(tableSchema),
     region: JSON.stringify(region),
     searchHost: JSON.stringify(searchHost),
     searchIndices: JSON.stringify(searchIndices),

src/functionBuilder/compiler/loader/serialisers.ts

@@ -1,12 +1,16 @@
 import { IExtension } from "./types";
 import { getRequiredPackages } from "../../../utils";
 
-const removeInlineVersioning = (code) =>
+const removeInlineVersioning = (code: string) =>
   code.replace(
     /(?:require\(.*)@\d+\.\d+\.\d+/g,
     (capture) => capture.split("@")[0]
   );
 
+const removeTrailingColon = (code: string) => {
+  return code.replace(/\s*;\s*$/, "");
+};
+
 /* Convert extension objects into a single readable string */
 export const serialiseExtension = (extensions: IExtension[]): string =>
   "[" +
@@ -31,9 +35,12 @@ export const serialiseExtension = (extensions: IExtension[]): string =>
             requiredPackages:${JSON.stringify(
               getRequiredPackages(extension.extensionBody)
             )},
-          extensionBody: ${removeInlineVersioning(extension.extensionBody)
-            .replace(/^.*:\s*\w*Body\s*=/, "")
-            .replace(/\s*;\s*$/, "")}
+          extensionBody: ${removeTrailingColon(
+            removeInlineVersioning(extension.extensionBody).replace(
+              /^.*:\s*\w*Body\s*=/,
+              ""
+            )
+          )}
         }`
     )
     .join(",") +
@@ -53,7 +60,7 @@ export const serialiseDerivativeColumns = (derivativeColumns: any[]): string =>
     return `${acc}{\nfieldName:'${currColumn.key}'
     ,requiredPackages:${JSON.stringify(getRequiredPackages(functionBody))}
     ,evaluate:async ({row,ref,db,auth,storage,utilFns}) =>
-      ${removeInlineVersioning(functionBody)}
+      ${removeTrailingColon(removeInlineVersioning(functionBody))}
   ,\nlistenerFields:[${listenerFields
     .map((fieldKey: string) => `"${fieldKey}"`)
     .join(",\n")}]},\n`;
@@ -77,7 +84,7 @@ export const serialiseDefaultValueColumns = (
     type:"${type}",
     requiredPackages:${JSON.stringify(getRequiredPackages(functionBody))},
     script:async ({row,ref,db,auth,utilFns}) => {
-      ${removeInlineVersioning(functionBody)}
+      ${removeTrailingColon(removeInlineVersioning(functionBody))}
   },
    },\n`;
     } else {

src/functionBuilder/compiler/loader/types.ts

@@ -50,4 +50,5 @@ export interface TableConfig {
     fields: string[];
   };
   runtimeOptions: IRuntimeOptions;
+  tableSchema: any;
 }

src/functionBuilder/functions/src/extensions/index.ts

@@ -3,7 +3,7 @@ import utilFns, { hasRequiredFields, getTriggerType } from "../utils";
 import { db, auth, storage } from "../firebaseConfig";
 
 const extension =
-  (extensionConfig, fieldTypes) =>
+  (extensionConfig, fieldTypes, tableSchema) =>
   async (
     change: functions.Change<functions.firestore.DocumentSnapshot>,
     context: functions.EventContext
@@ -33,6 +33,7 @@ const extension =
         utilFns,
         fieldTypes,
         storage,
+        tableSchema,
       };
       if (!triggers.includes(triggerType)) return false; //check if trigger type is included in the extension
       if (

src/functionBuilder/functions/src/index.ts

@@ -20,7 +20,11 @@ export const R = {
           extensionConfig.triggers.includes(triggerType)
         )
         .map((extensionConfig) =>
-          extension(extensionConfig, functionConfig.fieldTypes)(change, context)
+          extension(
+            extensionConfig,
+            functionConfig.fieldTypes,
+            functionConfig.tableSchema
+          )(change, context)
         );
       console.log(
         `#${

src/logging/LoggingFactory.ts

@@ -0,0 +1,90 @@
+import { Logging } from "@google-cloud/logging";
+import { getProjectId } from "../metadataService";
+
+type FunctionType = "action" | "connector" | "derivative";
+
+interface RowyLogging {
+  log: (payload: any) => void;
+  warn: (payload: any) => void;
+  error: (payload: any) => void;
+}
+
+class LoggingFactory {
+  public static async createActionLogging(fieldName: string, rowId: string) {
+    const projectId = await getProjectId();
+    return new LoggingFieldAndRow(projectId, fieldName, rowId, "action");
+  }
+
+  public static async createConnectorLogging(fieldName: string, rowId: string) {
+    const projectId = await getProjectId();
+    return new LoggingFieldAndRow(projectId, fieldName, rowId, "connector");
+  }
+
+  public static async createDerivativeLogging(
+    fieldName: string,
+    rowId: string
+  ) {
+    const projectId = await getProjectId();
+    return new LoggingFieldAndRow(projectId, fieldName, rowId, "derivative");
+  }
+}
+
+class LoggingAbstract implements RowyLogging {
+  protected readonly functionType;
+  protected readonly logging: Logging;
+
+  constructor(projectId: string, functionType: FunctionType) {
+    this.functionType = functionType;
+    this.logging = new Logging({ projectId });
+  }
+
+  protected async logWithSeverity(payload: any, severity: string) {
+    throw new Error("logWithSeverity must be implemented");
+  }
+
+  async log(payload: any) {
+    await this.logWithSeverity(payload, "DEFAULT");
+  }
+
+  async warn(payload: any) {
+    await this.logWithSeverity(payload, "WARNING");
+  }
+
+  async error(payload: any) {
+    await this.logWithSeverity(payload, "ERROR");
+  }
+}
+
+class LoggingFieldAndRow extends LoggingAbstract implements RowyLogging {
+  private readonly fieldName: string;
+  private readonly rowId: string;
+
+  constructor(
+    projectId: string,
+    fieldName: string,
+    rowId: string,
+    functionType: FunctionType
+  ) {
+    super(projectId, functionType);
+    this.fieldName = fieldName;
+    this.rowId = rowId;
+  }
+
+  async logWithSeverity(payload: any, severity: string) {
+    const log = this.logging.log(`rowy-logging`);
+    const metadata = {
+      severity,
+    };
+    const payloadSize = JSON.stringify(payload).length;
+    const entry = log.entry(metadata, {
+      loggingSource: "backend-scripts",
+      functionType: this.functionType,
+      fieldName: this.fieldName,
+      rowId: this.rowId,
+      payload: payloadSize > 250000 ? { v: "payload too large" } : payload,
+    });
+    await log.write(entry);
+  }
+}
+
+export { LoggingFactory, RowyLogging };

src/logging/index.ts

@@ -61,3 +61,5 @@ export async function getLogs(req: Request) {
     return entry.toJSON();
   });
 }
+
+export { LoggingFactory, RowyLogging } from "./LoggingFactory";

src/scripts/action.ts

@@ -7,6 +7,7 @@ import { FieldValue } from "firebase-admin/firestore";
 import rowy from "./rowy";
 import { installDependenciesIfMissing } from "../utils";
 import { telemetryRuntimeDependencyPerformance } from "../rowyService";
+import { LoggingFactory } from "../logging";
 
 type Ref = {
   id: string;
@@ -87,8 +88,13 @@ export const actionScript = async (req: Request, res: Response) => {
         `action ${column.key} in ${ref.path}`
       );
 
+    const logging = await LoggingFactory.createActionLogging(
+      column.key,
+      ref.id
+    );
+
     const _actionScript = eval(
-      `async ({ row, db, ref, auth, utilFns, actionParams, user, fetch, rowy }) => ${codeToRun}`
+      `async ({ row, db, ref, auth, utilFns, actionParams, user, fetch, rowy, tableSchema, logging }) => ${codeToRun}`
     );
     const getRows = refs
       ? refs.map(async (r) => db.doc(r.path).get())
@@ -118,6 +124,8 @@ export const actionScript = async (req: Request, res: Response) => {
           user: { ...authUser2rowyUser(user), roles: userRoles },
           fetch,
           rowy,
+          logging,
+          tableSchema: schemaDocData,
         });
         if (result.success || result.status) {
           const cellValue = {

src/scripts/connector.ts

@@ -8,6 +8,7 @@ import { Auth } from "firebase-admin/auth";
 import * as admin from "firebase-admin";
 import { installDependenciesIfMissing } from "../utils";
 import { telemetryRuntimeDependencyPerformance } from "../rowyService";
+import { LoggingFactory, RowyLogging } from "../logging";
 
 type ConnectorRequest = {
   rowDocPath: string;
@@ -25,6 +26,8 @@ type ConnectorArgs = {
   rowy: Rowy;
   fetch: any;
   storage: admin.storage.Storage;
+  logging: RowyLogging;
+  tableSchema: any;
 };
 
 type Connector = (args: ConnectorArgs) => Promise<any[]> | any[];
@@ -69,8 +72,13 @@ export const connector = async (req: Request, res: Response) => {
         `connector ${columnKey} in ${rowDocPath}`
       );
 
+    const logging = await LoggingFactory.createConnectorLogging(
+      columnKey,
+      schemaDoc.ref.id
+    );
+
     const connectorScript = eval(
-      `async ({ row, db, ref, auth, fetch, rowy, storage }) =>` +
+      `async ({ row, db, ref, auth, fetch, rowy, storage, logging, tableSchema }) =>` +
         connectorFnBody
     ) as Connector;
     const pattern = /row(?!y)/;
@@ -88,6 +96,8 @@ export const connector = async (req: Request, res: Response) => {
       user,
       storage,
       rowy,
+      logging,
+      tableSchema: schemaDocData,
     });
 
     const functionEndTime = Date.now();

src/scripts/derivative.ts

@@ -6,6 +6,7 @@ import { DocumentReference } from "firebase-admin/firestore";
 import rowy from "./rowy";
 import { installDependenciesIfMissing } from "../utils";
 import { telemetryRuntimeDependencyPerformance } from "../rowyService";
+import { LoggingFactory } from "../logging";
 
 type RequestData = {
   refs?: DocumentReference[]; // used in bulkAction
@@ -62,8 +63,13 @@ export const evaluateDerivative = async (req: Request, res: Response) => {
         `derivative ${columnKey} in ${collectionPath}`
       );
 
+    const logging = await LoggingFactory.createDerivativeLogging(
+      columnKey,
+      schemaDoc.ref.id
+    );
+
     const derivativeFunction = eval(
-      `async ({ row, db, ref, auth, fetch, rowy }) =>` +
+      `async ({ row, db, ref, auth, fetch, rowy, logging, tableSchema }) =>` +
         code.replace(/^.*=>/, "")
     );
     let rowSnapshots: FirebaseFirestore.DocumentSnapshot<FirebaseFirestore.DocumentData>[] =
@@ -90,6 +96,8 @@ export const evaluateDerivative = async (req: Request, res: Response) => {
             ref: doc.ref,
             fetch,
             rowy,
+            logging,
+            tableSchema: schemaDocData,
           });
           const update = { [columnKey]: result };
           if (schemaDocData?.audit !== false) {