[CCFPCM-396] update parse runs for aws compatability #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

vyasworks merged 10 commits into main from aws_runs

Feb 23, 2023

.config/.env.example

-Original file line number
+Diff line change
@@ Expand Up / @@ -35,4 +35,5 @@ BASTION_INSTANCE_ID_DEV= @@
     BASTION_INSTANCE_ID_PROD=
     BASTION_INSTANCE_ID_TEST=
-    NODE_ENV=local
+    NODE_ENV=local
+    RUNTIME_ENV=local

.github/workflows/dev.yaml

-Original file line number
+Diff line change
@@ Expand Up / @@ -50,6 +50,5 @@ jobs: @@
           - name: Run Migrations
             run: make aws-run-migrator
-          - name: Deploy API
-            run: make aws-deploy-api
+          - name: Deploy All
+            run: make aws-deploy-all

.github/workflows/prod.yaml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -50,6 +50,6 @@ jobs:
  
          - name: Run Migrations

            run: make aws-run-migrator

          - name: Deploy API

            run: make aws-deploy-api

          - name: Deploy All

            run: aws-deploy-all

.github/workflows/test.yaml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -50,6 +50,6 @@ jobs:
  
          - name: Run Migrations

            run: make aws-run-migrator

          - name: Deploy API

            run: make aws-deploy-api

          - name: Deploy All

            run: aws-deploy-all

Makefile

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -136,7 +136,7 @@ endif
  
    # Builds

    # ======================================================================

    make clean: 

    clean: 

    	@rm -rf apps/backend/dist 

    pre-build:

    @@ -187,31 +187,32 @@ build-backend: pre-build
  
    # Full redirection to /dev/null is required to not leak env variables

    aws-deploy-api:

    	aws lambda update-function-code --function-name paycocoapi --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    aws-deploy-all:

    	@aws lambda update-function-code --function-name paycocoapi --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    	@aws lambda update-function-code --function-name parser --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    	@aws lambda update-function-code --function-name reconciler --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    aws-deploy-migrator:

    	aws lambda update-function-code --function-name migrator --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    aws-deploy-reconciler:

    	aws lambda update-function-code --function-name reconciler --zip-file fileb://./terraform/build/backend.zip --region ca-central-1 > /dev/null

    # ======================================================================

    # AWS Interactions

    # ======================================================================

    aws-sync-data-from-prod: 

    	@aws s3 sync s3://pcc-integration-data-files-prod s3://pcc-integration-data-files-dev

    	@aws s3 sync s3://pcc-integration-data-files-prod s3://pcc-integration-data-files-test

    	@aws s3 sync s3://pcc-integration-data-files-prod s3://pcc-integration-data-files-dev --acl bucket-owner-full-control

    	@aws s3 sync s3://pcc-integration-data-files-prod s3://pcc-integration-data-files-test --acl bucket-owner-full-control

    aws-run-migrator: 

    	@rm migration-results || true

    	@aws lambda invoke --function-name migrator --payload '{}' migration-results --region ca-central-1

    	@cat migration-results | grep "success"

    aws-run-reconciler:

    	@aws lambda invoke --function-name reconciler --payload file://./apps/backend/fixtures/lambda/reconcile.json --region ca-central-1 --cli-binary-format raw-in-base64-out response.txt

    aws-run-parser: 

    	@aws lambda invoke --function-name parser  --payload '{ "eventType": "all" }' --region ca-central-1 --cli-binary-format raw-in-base64-out response.txt

    # ======================================================================

    # CI 

    @@ -228,6 +229,9 @@ close-test:
  
    	@echo "+\n++ Make: Closing test container ...\n+"

    	@docker-compose -f docker-compose.test.yml down

    lint: 

    	@yarn workspace @payment/backend lint

    # ======================================================================

    # Local Development Environment & Testing

    # ======================================================================

    @@ -248,11 +252,11 @@ be-logs:
  
    	@docker logs $(PROJECT)-backend --follow --tail 25

    # ===================================

    # Parse Local

    # Local

    # ===================================

    parse:

    	@docker exec -it $(PROJECT)-backend ./node_modules/.bin/ts-node -e 'require("./apps/backend/src/lambdas/generateData.ts").handler({eventType: "make"})' 

    	@docker exec -it $(PROJECT)-backend ./node_modules/.bin/ts-node -e 'require("./apps/backend/src/lambdas/parser.ts").handler({eventType: "all"})' 

    # TODO update handler to accept args

    reconcile:

apps/backend/fixtures/lambda/reconcile.json

-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+    {
+      "fiscal_start_date": "2023-01-01",
+      "fiscal_end_date": "2023-02-22",
+      "program": "SBC",
+      "location_ids": []
+    }

apps/backend/src/app.module.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -29,13 +29,13 @@ import { TransactionModule } from './transaction/transaction.module';
  
        ReportingModule,

        ConfigModule.forRoot({

          ignoreEnvFile:

            process.env.NODE_ENV === 'local' || process.env.NODE_ENV === 'ci'

            process.env.RUNTIME_ENV === 'local' || process.env.RUNTIME_ENV === 'ci'

              ? false

              : true

        }),

        AwsSdkModule.forRoot({

          defaultServiceOptions: {

            ...(process.env.NODE_ENV === 'local' || process.env.NODE_ENV === 'ci'

            ...(process.env.RUNTIME_ENV === 'local' || process.env.RUNTIME_ENV === 'ci'

              ? {

                  endpoint: process.env.AWS_ENDPOINT || 'http://localhost:9000',

                  region: 'ca-central-1',

apps/backend/src/constants.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,4 +1,4 @@
  
    export const LOCAL = 'local';

    export const ALL = 'all';

    export enum Ministries {

      SBC = 'SBC',

apps/backend/src/database/database.module.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -9,15 +9,13 @@ import { DatabaseLogger } from './database-logger';
  
    const config: PostgresConnectionOptions = {

      type: 'postgres',

      port: +(process.env.DB_PORT || 5432),

      connectTimeoutMS: 5000,

      connectTimeoutMS: 10000,

      host: process.env.DB_HOST || 'db',

      database: process.env.DB_NAME || 'pcc',

      username: process.env.DB_USER || 'postgres',

      password: process.env.DB_PASSWORD,

      entities: ['dist/src/**/entity/*.entity.js'],

      migrations: ['dist/src/database/migrations/*.js'],

      synchronize: false,

      migrationsRun: true,

      migrationsRun: false,

      logging: !!process.env.DEBUG,

      logger: process.env.DEBUG ? new DatabaseLogger() : undefined

    };

    @@ -32,8 +30,7 @@ const getEnvironmentSpecificConfig = (env?: string) => {
  
          };

        default:

          return {

            entities: ['dist/**/*.entity.js'],

            migrations: ['dist/src/database/migrations/*.js'],

            autoLoadEntities: true,

            logging: ['error', 'warn', 'migration'] as LoggerOptions

          };

      }

    @@ -49,8 +46,6 @@ export const appOrmConfig: PostgresConnectionOptions = {
  
      migrationsRun: false

    };

    console.log(appOrmConfig);

    @Module({

      imports: [

        TypeOrmModule.forRootAsync({

apps/backend/src/database/migrate.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,9 +1,9 @@
  
    import { Context } from 'aws-lambda';

    import { MigrationExecutor } from 'typeorm';

    import db from './datasource';

    import { Context } from 'aws-lambda';

    /* eslint-disable @typescript-eslint/no-unused-vars */

    export const handler = async (event?: unknown, context?: Context) => {

    // eslint-disable-next-line @typescript-eslint/no-unused-vars

    export const handler = async (_event?: unknown, _context?: Context) => {

      console.log('Starting migrations...');

      try {

        if (!db.isInitialized) {

apps/backend/src/lambdas/parseFlatFile.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -29,7 +29,7 @@ export const handler = async (event: parseFlatFileEvent, context?: Context) => {
  
        appLogger.log(`...start ${event.type} Parsing`);

        const contents = await s3manager.getObject(

          `pcc-integration-data-files-${process.env.NODE_ENV}`,

          `pcc-integration-data-files-${process.env.RUNTIME_ENV}`,

          `${event.filepath}`

        );

    @@ -59,7 +59,7 @@ export const uploadParsedTDI = async (
  
    ) => {

      try {

        await s3manager.putObject(

          `pcc-integration-data-files-${process.env.NODE_ENV}`,

          `pcc-integration-data-files-${process.env.RUNTIME_ENV}`,

          outputPath ?? `outputs/${type}/${Date.now()}_${type}.json`,

          Buffer.from(JSON.stringify(output))

        );

apps/backend/src/lambdas/generateData.ts → apps/backend/src/lambdas/parser.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -5,7 +5,7 @@ import { getLambdaEventSource } from './utils/eventTypes';
  
    import { parseGarms } from './utils/parseGarms';

    import { parseTDI } from './utils/parseTDI';

    import { AppModule } from '../app.module';

    import { FileNames, FileTypes, LOCAL, Ministries } from '../constants';

    import { FileNames, FileTypes, ALL, Ministries } from '../constants';

    import { CashDepositService } from '../deposits/cash-deposit.service';

    import { CashDepositEntity } from '../deposits/entities/cash-deposit.entity';

    import { POSDepositEntity } from '../deposits/entities/pos-deposit.entity';

    @@ -19,7 +19,7 @@ import { SBCGarmsJson } from '../transaction/interface';
  
    import { PaymentMethodService } from '../transaction/payment-method.service';

    import { TransactionService } from '../transaction/transaction.service';

    export interface LocalEvent {

    export interface ParseEvent {

      eventType: string;

      filename: string;

    }

    @@ -33,13 +33,13 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
      const s3 = app.get(S3ManagerService);

      const posService = app.get(PosDepositService);

      const cashService = app.get(CashDepositService);

      appLogger.log({ event });

      const processLocalFiles = async () => {

      const processAllFiles = async () => {

        appLogger.log('Processing all files...');

        try {

          const fileList =

            (await s3.listBucketContents(

              `pcc-integration-data-files-${process.env.NODE_ENV}`

              `pcc-integration-data-files-${process.env.RUNTIME_ENV}`

            )) || [];

          const allUploadedFiles: string[] = [];

    @@ -71,7 +71,7 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
          // Parse & Save only files that have not been parsed before

          for (const filename of finalParseList) {

            appLogger.log(`Parsing ${filename}..`);

            const event = { eventType: 'local', filename: filename };

            const event = { eventType: 'all', filename: filename };

            await processEvent(event);

          }

        } catch (err) {

    @@ -83,9 +83,9 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
        try {

          const eventType = getLambdaEventSource(event);

          const filename = (() => {

            if (eventType === LOCAL) {

              const localEvent = event as LocalEvent;

              return localEvent.filename;

            if (eventType === ALL) {

              const thisEvent = event as ParseEvent;

              return thisEvent.filename;

            }

            // TODO: use types here

    @@ -98,7 +98,7 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
          })();

          const file = await s3.getObject(

            `pcc-integration-data-files-${process.env.NODE_ENV}`,

            `pcc-integration-data-files-${process.env.RUNTIME_ENV}`,

            filename

          );

    @@ -124,11 +124,8 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
              filename,

              paymentMethods

            );

            garmsSales.map(

              async (data: TransactionEntity) =>

                await transactionService.saveTransaction(data)

            );

            appLogger.log(`txn count: ${garmsSales.length}`);

            await transactionService.saveTransactions(garmsSales);

          }

          if (fileType === FileTypes.TDI17 || fileType === FileTypes.TDI34) {

    @@ -162,11 +159,16 @@ export const handler = async (event?: unknown, _context?: Context) => {
  
        }

      };

      const eventRouting = event as LocalEvent;

      if (eventRouting?.eventType === 'make') {

        await processLocalFiles();

      const eventRouting = event as ParseEvent;

      appLogger.log({ eventRouting });

      if (eventRouting?.eventType === 'all') {

        await processAllFiles();

        return;

      }

      await processEvent(event);

      try {

        await processEvent(event);

      } catch (err) {

        appLogger.error(err);

      }

    };

apps/backend/src/lambdas/reconcile.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -99,10 +99,10 @@ export const handler = async (
  
    };

    const reconcileAll: ReconciliationEventInput = {

      fiscal_start_date: '2023-02-01',

      fiscal_end_date: '2023-02-02',

      fiscal_start_date: '2023-01-01',

      fiscal_end_date: '2023-02-22',

      program: 'SBC',

      location_ids: [31, 61]

      location_ids: []

    };

    handler(reconcileAll);

apps/backend/src/lambdas/utils/eventTypes.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -2,7 +2,6 @@ @@
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     export const getLambdaEventSource = (event?: any) => {
       if (event.Records && event.Records[0].cf) return 'isCloudfront';
       if (event.configRuleId && event.configRuleName && event.configRuleArn)
@@ Expand Down Expand Up / @@ -60,6 +59,5 @@ export const getLambdaEventSource = (event?: any) => { @@
       if (event.Records && event.Records[0].eventSource === 'aws:sqs')
         return 'isSqs';
-      if (event.eventType === 'local')
-        return 'local';
+      if (event.eventType === 'all') return 'all';
     };

apps/backend/src/logger/format.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -19,7 +19,7 @@ export const consoleLogFormat = winston.format.combine( @@
     );
     const consoleTransport = new winston.transports.Console({
-      level: 'INFO',
+      level: 'debug',
       format: consoleLogFormat
     });
@@ Expand Down @@

apps/backend/src/logger/logger.service.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -18,6 +18,7 @@ export class AppLogger implements LoggerService { @@
       log(message: unknown, context?: any) {
         this.logger.log(util.format(message ?? '', context ?? ''));
       }
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       async error(e: any, context?: string) {
         const error = e as Error & { response?: Error };
@@ Expand Down @@

apps/backend/src/reconciliation/pos-reconciliation.service.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -68,8 +68,13 @@ export class POSReconciliationService { @@
       ): Promise<unknown | ReconciliationEventOutput | ReconciliationEventError> {
         const posPayments = await this.transactionService.queryPosPayments(event);
+        if (posPayments.length === 0) {
+          console.log(`****0 POS Payments Found, Skipping****`);
+          return;
+        }
         if (checkPaymentsForFullMatch(posPayments)) {
-          console.log(`****All payments are already matched for this event`);
+          console.log(`****All payments are already matched /for this event****`);
           return;
         }
         const alreadyMatchedPosPayments = posPayments.filter((itm) => itm.match);
@@ Expand Down @@

apps/backend/src/transaction/transaction.service.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -28,6 +28,16 @@ export class TransactionService { @@
         private locationService: LocationService
       ) {}
+      async saveTransactions(data: TransactionEntity[]) {
+        try {
+          const entities = data.map((d) => this.transactionRepo.create(d));
+          return await this.transactionRepo.save(entities);
+        } catch (e) {
+          this.appLogger.error(e);
+          throw e;
+        }
+      }
       async saveTransaction(data: TransactionEntity): Promise<TransactionEntity> {
         try {
           return await this.transactionRepo.save(this.transactionRepo.create(data));
@@ Expand Down @@

apps/backend/test/app.e2e-spec.ts

-Original file line number
+Diff line change
@@ -1,5 +1,5 @@
-    import { Test, TestingModule } from '@nestjs/testing';
     import { INestApplication } from '@nestjs/common';
+    import { Test, TestingModule } from '@nestjs/testing';
     import request from 'supertest';
     import { AppModule } from '../src/app.module';
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[CCFPCM-396] update parse runs for aws compatability #57

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!