fix(export): upload and notify (#2526)

* refactor dates helper import

* cast start_at and end_at as Dates in ExportParams

* add Export tests

* fix set() import

* document middlewares

* fix inputs casting and validation

* move middlewares to pdc/middlewares

* move permissions config to user service

* check user existence

* add jq to flake

* add Storage and Notify services

* add minio-client to flake

* notify support on export error

* logger fileCreateService error

* fix: decrease log level

---------

Co-authored-by: Nicolas Mérigot <nicolas.merigot@beta.gouv.fr>

api/src/config/contacts.ts

@@ -0,0 +1,12 @@
+import { env_or_default } from "@/lib/env/index.ts";
+
+export const support = {
+  email: env_or_default(
+    "APP_CONTACT_SUPPORT_EMAIL",
+    "technique@covoiturage.beta.gouv.fr",
+  ),
+  fullname: env_or_default(
+    "APP_CONTACT_SUPPORT_FULLNAME",
+    "Equipe technique covoiturage",
+  ),
+};

api/src/ilos/connection-postgres/PostgresConnection.ts

@@ -42,7 +42,7 @@ export class PgPool extends Pool {
 
     this.on("remove", () => {
       const { totalCount, idleCount, waitingCount } = this;
-      logger.info(
+      logger.debug(
         `[pg] client removed ` +
           `(total: ${totalCount}, idle: ${idleCount}, waiting: ${waitingCount})`,
       );

api/src/pdc/middlewares/DefaultTimezoneMiddleware.ts

@@ -0,0 +1,23 @@
+import { defaultTimezone } from "@/config/time.ts";
+import { NextFunction } from "@/deps.ts";
+import { ContextType, middleware } from "@/ilos/common/index.ts";
+import { Timezone } from "@/pdc/providers/validator/types.ts";
+
+/**
+ * Set the params.tz property to the default time zone
+ * if it is not already set.
+ */
+@middleware()
+export class DefaultTimezoneMiddleware<TParams extends { tz: Timezone }> {
+  async process(
+    params: TParams,
+    context: ContextType,
+    next: NextFunction,
+  ): Promise<unknown> {
+    if (!params.tz) {
+      params.tz = defaultTimezone;
+    }
+
+    return next(params, context);
+  }
+}

api/src/pdc/providers/middleware/Cast/CastToArrayMiddleware.ts

@@ -9,13 +9,6 @@ import {
 import { get, set } from "@/lib/object/index.ts";
 import { ConfiguredMiddleware } from "../interfaces.ts";
 
-/*
- * CastToArrayMiddleware middleware and its companion helper function
- * castToArrayMiddleware are used to cast one or many properties
- * to an array.
- *
- * Not found or undefined props are skipped.
- */
 @middleware()
 export class CastToArrayMiddleware implements MiddlewareInterface<HelperArgs> {
   async process(
@@ -53,6 +46,16 @@ const alias = "cast.to_array";
 
 export const castToArrayMiddlewareBinding = [alias, CastToArrayMiddleware];
 
+/**
+ * Cast one or many properties to an array. Not found or undefined props are skipped.
+ *
+ * @param properties - single or array of multiple properties to cast to an array
+ *
+ * @example
+ * middlewares: [
+ *   castToArrayMiddleware(["operator_id", "territory_id", "recipients"]),
+ * ],
+ */
 export function castToArrayMiddleware(
   properties: HelperArgs,
 ): ConfiguredMiddleware<HelperArgs> {

api/src/pdc/providers/middleware/CopyFromContext/CopyFromContextMiddleware.ts

@@ -45,6 +45,27 @@ export const copyFromContextMiddlewareBinding = [
   CopyFromContextMiddleware,
 ];
 
+/**
+ * Copy context data to request params
+
+ *
+ * @param fromPath - the path in the context to copy from
+ * @param toPath - the path in the request params to copy to
+ * @param preserve - whether to preserve the existing value
+ *                             in the request params or override it
+ *
+ * @example
+ * middlewares: [
+ *   // override the operator_id to scope the request to the owner if it is
+ *   // an operator.
+ *   copyFromContextMiddleware("call.user.operator_id", "operator_id", false),
+ *
+ *   // copy the user id to the created_by field only if it is not already set
+ *   // in the request params.
+ *   copyFromContextMiddleware("call.user._id", "created_by", true),
+ * ],
+ *
+ */
 export function copyFromContextMiddleware(
   fromPath: string,
   toPath: string,

api/src/pdc/providers/middleware/HasPermission/HasPermissionMiddleware.ts

@@ -1,3 +1,4 @@
+import { NextFunction } from "@/deps.ts";
 import {
   ContextType,
   ForbiddenException,
@@ -19,7 +20,7 @@ export class HasPermissionMiddleware
   async process(
     params: ParamsType,
     context: ContextType,
-    next: Function,
+    next: NextFunction,
     neededPermissions: HasPermissionMiddlewareParams,
   ): Promise<ResultType> {
     if (!Array.isArray(neededPermissions) || neededPermissions.length === 0) {
@@ -51,6 +52,14 @@ const alias = "has_permission";
 
 export const hasPermissionMiddlewareBinding = [alias, HasPermissionMiddleware];
 
+/**
+ * Define allowed permissions to access the handler.
+ *
+ * User's permissions are extracted from context.user.permissions. The list
+ * must contain permissions from the middleware configuration.
+ *
+ * @param params - list of allowed permissions
+ */
 export function hasPermissionMiddleware(
   ...params: HasPermissionMiddlewareParams
 ): ConfiguredMiddleware<HasPermissionMiddlewareParams> {

api/src/pdc/providers/middleware/ValidateDate/ValidateDateMiddleware.ts

@@ -1,4 +1,4 @@
-import { endOfDay, startOfDay } from "@/deps.ts";
+import { endOfDay, NextFunction, startOfDay } from "@/deps.ts";
 import {
   ContextType,
   InvalidParamsException,
@@ -19,7 +19,7 @@ export class ValidateDateMiddleware
   async process(
     params: ParamsType,
     context: ContextType,
-    next: Function,
+    next: NextFunction,
     options: ValidateDateMiddlewareParams,
   ): Promise<ResultType> {
     if (
@@ -92,6 +92,32 @@ const alias = "validate.date";
 
 export const validateDateMiddlewareBinding = [alias, ValidateDateMiddleware];
 
+/**
+ * Validate start and end date inputs. Make sure start is before end.
+ * Apply limits and defaults.
+ *
+ * @param startPath - Path to the start date in the params object
+ * @param endPath - Path to the end date in the params object
+ * @param minStart - Minimum start date. If the start date is before this date, an error is thrown
+ * @param maxEnd - Maximum end date. If the end date is after this date, an error is thrown
+ * @param applyDefault - If true, set the start date to minStart and end date to maxEnd if they are not provided
+ *
+ * @example
+ * middlewares: [
+ *  validateDateMiddleware({ startPath: "start_at", endPath: "end_at" }),
+ * ],
+ *
+ * @example
+ * middlewares: [
+ *   validateDateMiddleware({
+ *     startPath: "start_at",
+ *     endPath: "end_at",
+ *     minStart: () => new Date(new Date().getTime() - minStartDefault),
+ *     maxEnd: () => new Date(new Date().getTime() - maxEndDefault),
+ *     applyDefault: true,
+ *   }),
+ * ],
+ */
 export function validateDateMiddleware(
   params: ValidateDateMiddlewareParams,
 ): ConfiguredMiddleware<ValidateDateMiddlewareParams> {

api/src/pdc/providers/notification/NotificationMailTransporter.ts

@@ -29,8 +29,8 @@ export class NotificationMailTransporter
       MailTemplateNotificationInterface,
       Partial<MailOptions>
     > {
-  transporter: mailer.Transporter;
-  protected options: NotificationOptions;
+  transporter: mailer.Transporter | null = null;
+  protected options: NotificationOptions = {} as NotificationOptions;
 
   constructor(
     protected config: ConfigInterfaceResolver,
@@ -51,29 +51,28 @@ export class NotificationMailTransporter
   }
 
   protected setOptionsFromConfig(): void {
+    const fromFullname = this.config.get("notification.mail.from.name");
+    const fromEmail = this.config.get("notification.mail.from.email");
+    const toFullname = this.config.get("notification.mail.to.name");
+    const toEmail = this.config.get("notification.mail.to.email");
+    const debug = this.config.get("notification.mail.debug", false);
+
     this.options = {
-      from: `${this.config.get("notification.mail.from.name")} <${
-        this.config.get("notification.mail.from.email")
-      }>`,
-      debug: this.config.get("notification.mail.debug", false),
-      debugToOverride: `${this.config.get("notification.mail.to.name")} <${
-        this.config.get(
-          "notification.mail.to.email",
-        )
-      }>`,
+      from: `${fromFullname} <${fromEmail}>`,
+      debugToOverride: `${toFullname} <${toEmail}>`,
+      debug,
     };
   }
 
   protected async createTransport(verify = false): Promise<void> {
     if (!this.transporter) {
-      this.transporter = mailer.createTransport(
-        this.config.get("notification.mail.smtp"),
-      );
+      const smtp = this.config.get("notification.mail.smtp");
+      this.transporter = mailer.createTransport(smtp);
       if (verify) {
         try {
           await this.transporter.verify();
         } catch (e) {
-          logger.error("Failed to connect to SMTP server");
+          logger.error("Failed to connect to SMTP server", e.message);
           exit(1);
         }
       }
@@ -85,14 +84,38 @@ export class NotificationMailTransporter
     return !mjml ? result : mjml2html(result).html;
   }
 
+  protected moustache(str: string, data: Record<string, unknown>): string {
+    return str.replace(/\{\{([^}]+)\}\}/g, (_, key) => {
+      return data[key.trim()] as string;
+    });
+  }
+
   async send(
     mail: MailTemplateNotificationInterface,
     options = {},
   ): Promise<void> {
     const mailCtor = mail
       .constructor as StaticMailTemplateNotificationInterface;
 
-    await this.transporter.sendMail({
+    if (
+      "message_html" in mail.data && typeof mail.data.message_html === "string"
+    ) {
+      mail.data.message_html = this.moustache(
+        mail.data.message_html,
+        mail.data,
+      );
+    }
+
+    if (
+      "message_text" in mail.data && typeof mail.data.message_text === "string"
+    ) {
+      mail.data.message_text = this.moustache(
+        mail.data.message_text,
+        mail.data,
+      );
+    }
+
+    this.transporter && await this.transporter.sendMail({
       ...options,
       from: this.options.from,
       to: this.options.debug ? this.options.debugToOverride : mail.to,
@@ -102,7 +125,5 @@ export class NotificationMailTransporter
         : undefined,
       text: this.render(new mailCtor.templateText(mail.data)),
     });
-
-    return;
   }
 }

api/src/pdc/services/apdf/commands/ExportCommand.ts

@@ -61,7 +61,7 @@ export class ExportCommand implements CommandInterface {
     },
     {
       signature: "--verbose",
-      description: "Display CLI specific console.info()",
+      description: "Display CLI specific logger.info()",
     },
   ];
 

api/src/pdc/services/application/config/permissions.ts

@@ -1,3 +1,3 @@
-import { operator } from "@/shared/user/permissions.config.ts";
+import { operator } from "@/pdc/services/user/config/permissions.ts";
 
 export const application = [...operator.application.permissions];

api/src/pdc/services/export/ServiceProvider.ts

@@ -5,6 +5,7 @@ import {
   serviceProvider,
 } from "@/ilos/common/index.ts";
 import { ServiceProvider as AbstractServiceProvider } from "@/ilos/core/index.ts";
+import { DefaultTimezoneMiddleware } from "@/pdc/middlewares/DefaultTimezoneMiddleware.ts";
 import { defaultMiddlewareBindings } from "@/pdc/providers/middleware/index.ts";
 import { S3StorageProvider } from "@/pdc/providers/storage/index.ts";
 import {
@@ -21,7 +22,6 @@ import { CreateCommand } from "./commands/CreateCommand.ts";
 import { DebugCommand } from "./commands/DebugCommand.ts";
 import { ProcessCommand } from "./commands/ProcessCommand.ts";
 import { config } from "./config/index.ts";
-import { DefaultTimezoneMiddleware } from "./middlewares/DefaultTimezoneMiddleware.ts";
 import { CampaignRepository } from "./repositories/CampaignRepository.ts";
 import { CarpoolRepository } from "./repositories/CarpoolRepository.ts";
 import { ExportRepository } from "./repositories/ExportRepository.ts";