feat: script runner data transfer part 1

api/.env.template

@@ -52,3 +52,5 @@ THROTTLE_TTL=3600000
 THROTTLE_LIMIT=100
 # API passkey, requests missing this will not be alllowed to progress
 API_PASS_KEY="some-key-here"
+# this is used to test the script runner's data transfer job
+TEST_CONNECTION_STRING=""

api/src/controllers/auth.controller.ts

@@ -31,12 +31,10 @@ import { mapTo } from '../utilities/mapTo';
 import { User } from '../dtos/users/user.dto';
 import { LoginViaSingleUseCode } from '../dtos/auth/login-single-use-code.dto';
 import { SingleUseCodeAuthGuard } from '../guards/single-use-code.guard';
-import { ApiKeyGuard } from '../guards/api-key.guard';
 
 @Controller('auth')
 @ApiTags('auth')
 @UsePipes(new ValidationPipe(defaultValidationPipeOptions))
-@UseGuards(ApiKeyGuard)
 export class AuthController {
   constructor(private readonly authService: AuthService) {}
 

api/src/controllers/script-runner.controller.ts

@@ -1,4 +1,5 @@
 import {
+  Body,
   Controller,
   Put,
   Request,
@@ -13,6 +14,7 @@ import { defaultValidationPipeOptions } from '../utilities/default-validation-pi
 import { SuccessDTO } from '../dtos/shared/success.dto';
 import { OptionalAuthGuard } from '../guards/optional.guard';
 import { AdminOrJurisdictionalAdminGuard } from '../guards/admin-or-jurisdiction-admin.guard';
+import { DataTransferDTO } from '../dtos/script-runner/data-transfer.dto';
 
 @Controller('scriptRunner')
 @ApiTags('scriptRunner')
@@ -30,4 +32,17 @@ export class ScirptRunnerController {
   async update(@Request() req: ExpressRequest): Promise<SuccessDTO> {
     return await this.scriptRunnerService.example(req);
   }
+
+  @Put('dataTransfer')
+  @ApiOperation({
+    summary: 'A script that pulls data from one source into the current db',
+    operationId: 'dataTransfer',
+  })
+  @ApiOkResponse({ type: SuccessDTO })
+  async dataTransfer(
+    @Body() dataTransferDTO: DataTransferDTO,
+    @Request() req: ExpressRequest,
+  ): Promise<SuccessDTO> {
+    return await this.scriptRunnerService.dataTransfer(req, dataTransferDTO);
+  }
 }

api/src/dtos/script-runner/data-transfer.dto.ts

@@ -0,0 +1,12 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Expose } from 'class-transformer';
+import { IsDefined, IsString } from 'class-validator';
+import { ValidationsGroupsEnum } from '../../enums/shared/validation-groups-enum';
+
+export class DataTransferDTO {
+  @Expose()
+  @IsString({ groups: [ValidationsGroupsEnum.default] })
+  @IsDefined({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
+  connectionString: string;
+}

api/src/services/script-runner.service.ts

@@ -1,9 +1,11 @@
 import { Injectable, BadRequestException } from '@nestjs/common';
+import { PrismaClient } from '@prisma/client';
 import { Request as ExpressRequest } from 'express';
 import { PrismaService } from './prisma.service';
 import { SuccessDTO } from '../dtos/shared/success.dto';
 import { User } from '../dtos/users/user.dto';
 import { mapTo } from '../utilities/mapTo';
+import { DataTransferDTO } from '../dtos/script-runner/data-transfer.dto';
 
 /**
   this is the service for running scripts
@@ -13,6 +15,44 @@ import { mapTo } from '../utilities/mapTo';
 export class ScriptRunnerService {
   constructor(private prisma: PrismaService) {}
 
+  /**
+   *
+   * @param req incoming request object
+   * @param dataTransferDTO data transfer endpoint args. Should contain foreign db connection string
+   * @returns successDTO
+   * @description transfers data from foreign data into the database this api normally connects to
+   */
+  async dataTransfer(
+    req: ExpressRequest,
+    dataTransferDTO: DataTransferDTO,
+  ): Promise<SuccessDTO> {
+    // script runner standard start up
+    const requestingUser = mapTo(User, req['user']);
+    await this.markScriptAsRunStart('data transfer', requestingUser);
+
+    // connect to foreign db based on incoming connection string
+    const client = new PrismaClient({
+      datasources: {
+        db: {
+          url: dataTransferDTO.connectionString,
+        },
+      },
+    });
+    await client.$connect();
+
+    // get data
+    const res =
+      await client.$queryRaw`SELECT id, name FROM jurisdictions WHERE name = 'San Mateo'`;
+    console.log(res);
+
+    // disconnect from foreign db
+    await client.$disconnect();
+
+    // script runner standard spin down
+    await this.markScriptAsComplete('data transfer', requestingUser);
+    return { success: true };
+  }
+
   /**
     this is simply an example
   */

api/test/unit/services/script-runner.service.spec.ts

@@ -1,7 +1,8 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { Logger } from '@nestjs/common';
-import { randomUUID } from 'crypto';
 import { SchedulerRegistry } from '@nestjs/schedule';
+import { randomUUID } from 'crypto';
+import { Request as ExpressRequest } from 'express';
 import { ScriptRunnerService } from '../../../src/services/script-runner.service';
 import { PrismaService } from '../../../src/services/prisma.service';
 import { User } from '../../../src/dtos/users/user.dto';
@@ -23,6 +24,49 @@ describe('Testing script runner service', () => {
     prisma = module.get<PrismaService>(PrismaService);
   });
 
+  it('should transfer data', async () => {
+    prisma.scriptRuns.findUnique = jest.fn().mockResolvedValue(null);
+    prisma.scriptRuns.create = jest.fn().mockResolvedValue(null);
+    prisma.scriptRuns.update = jest.fn().mockResolvedValue(null);
+
+    const id = randomUUID();
+    const scriptName = 'data transfer';
+
+    const res = await service.dataTransfer(
+      {
+        user: {
+          id,
+        } as unknown as User,
+      } as unknown as ExpressRequest,
+      {
+        connectionString: process.env.TEST_CONNECTION_STRING,
+      },
+    );
+
+    expect(res.success).toBe(true);
+
+    expect(prisma.scriptRuns.findUnique).toHaveBeenCalledWith({
+      where: {
+        scriptName,
+      },
+    });
+    expect(prisma.scriptRuns.create).toHaveBeenCalledWith({
+      data: {
+        scriptName,
+        triggeringUser: id,
+      },
+    });
+    expect(prisma.scriptRuns.update).toHaveBeenCalledWith({
+      data: {
+        didScriptRun: true,
+        triggeringUser: id,
+      },
+      where: {
+        scriptName,
+      },
+    });
+  });
+
   // | ---------- HELPER TESTS BELOW ---------- | //
   it('should mark script run as started if no script run present in db', async () => {
     prisma.scriptRuns.findUnique = jest.fn().mockResolvedValue(null);

shared-helpers/src/types/backend-swagger.ts

@@ -2009,6 +2009,28 @@ export class ScriptRunnerService {
 
       configs.data = data
 
+      axios(configs, resolve, reject)
+    })
+  }
+  /**
+   * A script that pulls data from one source into the current db
+   */
+  dataTransfer(
+    params: {
+      /** requestBody */
+      body?: DataTransferDTO
+    } = {} as any,
+    options: IRequestOptions = {}
+  ): Promise<SuccessDTO> {
+    return new Promise((resolve, reject) => {
+      let url = basePath + "/scriptRunner/dataTransfer"
+
+      const configs: IRequestConfig = getConfigs("put", "application/json", url, options)
+
+      let data = params.body
+
+      configs.data = data
+
       axios(configs, resolve, reject)
     })
   }
@@ -5193,6 +5215,11 @@ export interface MapLayer {
   jurisdictionId: string
 }
 
+export interface DataTransferDTO {
+  /**  */
+  connectionString: string
+}
+
 export enum ListingViews {
   "fundamentals" = "fundamentals",
   "base" = "base",