fix(dispatch-service): separated input and output files for simulatio…

…n runs Completed deletion of simulaton run files from S3 Debugged thumbnail deletion Debugged S3 retrying
biosimulations · Jan 6, 2022 · dc98a46 · dc98a46
1 parent 4e3df4f
commit dc98a46
Show file tree

Hide file tree

Showing 10 changed files with 148 additions and 26 deletions.
diff --git a/apps/api/src/files/files.service.ts b/apps/api/src/files/files.service.ts
@@ -4,6 +4,7 @@ import {
   NotFoundException,
   InternalServerErrorException,
   Logger,
+  HttpStatus,
 } from '@nestjs/common';
 import { InjectModel } from '@nestjs/mongoose';
 import { Model } from 'mongoose';
@@ -83,6 +84,16 @@ export class FilesService {
         return this.deleteFile(runId, file.location);
       }),
     );
+
+    await this.storage.deleteSimulationRunFile(runId, 'manifest.xml')
+      .catch((error: any) => {
+        if (!(
+          error.statusCode === HttpStatus.NOT_FOUND &&
+          error.code === 'NoSuchKey'
+        )) {
+          throw error;
+        }
+      });
   }
 
   public async deleteFile(runId: string, fileLocation: string): Promise<void> {

diff --git a/apps/api/src/results/results.service.ts b/apps/api/src/results/results.service.ts
@@ -161,7 +161,15 @@ export class ResultsService {
 
   public async deleteSimulationRunResults(runId: string): Promise<void> {
     await this.results.deleteDatasets(runId);
-    await this.simStorage.deleteSimulationRunResults(runId);
+    await this.simStorage.deleteSimulationRunResults(runId)
+      .catch((error: any) => {
+        if (!(
+          error.statusCode === HttpStatus.NOT_FOUND &&
+          error.code === 'NoSuchKey'
+        )) {
+          throw error;
+        }
+      });
   }
 
   private async parseDataset(

diff --git a/apps/api/src/simulation-run/simulation-run.service.ts b/apps/api/src/simulation-run/simulation-run.service.ts
@@ -238,6 +238,8 @@ export class SimulationRunService {
         throw error;
       }
     }
+
+    await this.simulationStorageService.deleteSimulation(id);
   }
 
   public async update(

diff --git a/apps/dispatch-service/src/app/results/archiver.service.ts b/apps/dispatch-service/src/app/results/archiver.service.ts
@@ -16,7 +16,7 @@ export class ArchiverService {
   ) {}
   // TODO include the output archive in the files endpoint and get size from there
   public async updateResultsSize(id: string): Promise<void> {
-    const path = this.sshService.getSSHResultsDirectory(id);
+    const path = this.sshService.getSSHJobDirectory(id);
     const archive = `${path}/${id}.zip`;
     const command = `du -b ${archive} | cut -f1`;
     this.sshService.execStringCommand(command).then((output) => {

diff --git a/apps/dispatch-service/src/app/results/log.service.ts b/apps/dispatch-service/src/app/results/log.service.ts
@@ -23,7 +23,7 @@ export class LogService {
     extraStdLog?: string,
     update = false,
   ): Promise<CombineArchiveLog> {
-    const path = this.sshService.getSSHResultsDirectory(id);
+    const path = this.sshService.getSSHJobDirectory(id);
     return this.makeLog(id, path, true, extraStdLog).then((value) => {
       return this.uploadLog(id, value, update).catch((error) => {
         this.logger.error(
@@ -56,7 +56,7 @@ export class LogService {
   }
 
   private async readStructuredLog(path: string): Promise<CombineArchiveLog> {
-    const yamlFile = `${path}/log.yml`;
+    const yamlFile = `${path}/outputs/log.yml`;
 
     return this.sshService
       .execStringCommand('cat ' + yamlFile)

diff --git a/apps/dispatch-service/src/app/services/sbatch/sbatch.service.ts b/apps/dispatch-service/src/app/services/sbatch/sbatch.service.ts
@@ -146,7 +146,11 @@ export class SbatchService {
     const simulationRunS3Path = this.filePaths.getSimulationRunPath(runId);
     const simulationRunResultsHsdsPath =
       this.dataPaths.getSimulationRunResultsPath(runId);
-    const outputArchiveS3Subpath = this.filePaths.getSimulationRunOutputPath(
+    const outputArchiveS3Subpath = this.filePaths.getSimulationRunOutputArchivePath(
+      runId,
+      false,
+    );
+    const outputsS3Subpath = this.filePaths.getSimulationRunOutputsPath(
       runId,
       false,
     );
@@ -185,23 +189,53 @@ echo -e '${cyan}============================================ Downloading COMBINE
 
 echo -e ''
 echo -e '${cyan}============================================= Executing COMBINE archive =============================================${nc}'
-srun --job-name="Execute-project" singularity run --tmpdir /local --bind ${workDirname}:/root "${allEnvVarsString}" ${simulatorImage} -i '/root/${combineArchiveFilename}' -o '/root'
+srun --job-name="Execute-project" \
+  singularity run \
+    --tmpdir /local \
+    --bind ${workDirname}:/root \
+    "${allEnvVarsString}" \
+    ${simulatorImage} \
+      -i '/root/${combineArchiveFilename}' \
+      -o '/root/${outputsS3Subpath}'
 
 set +e
 
 echo -e ''
 echo -e '${cyan}=================================================== Saving results ==================================================${nc}'
-srun --job-name="Save-outputs-to-HSDS" hsload --endpoint ${hsdsBasePath} --username ${hsdsUsername} --password ${hsdsPassword} --verbose reports.h5 '${simulationRunResultsHsdsPath}'
+srun --job-name="Save-outputs-to-HSDS" \
+  hsload \
+    --endpoint ${hsdsBasePath} \
+    --username ${hsdsUsername} \
+    --password ${hsdsPassword} \
+    --verbose \
+    ${outputsS3Subpath}/reports.h5 \
+    '${simulationRunResultsHsdsPath}'
 
 set -e
 
 echo -e ''
 echo -e '${cyan}================================================== Zipping outputs ==================================================${nc}'
-srun --job-name="Zip-outputs" zip '${outputArchiveS3Subpath}' reports.h5 log.yml plots.zip job.output
+srun --job-name="Zip-outputs" \
+  zip \
+    -x '${outputsS3Subpath}/plots.zip' \
+    -r \
+    '${outputArchiveS3Subpath}' \
+    ${outputsS3Subpath} \
+    job.output
 
 echo -e ''
 echo -e '${cyan}=================================================== Saving outputs ==================================================${nc}'
-export PYTHONWARNINGS="ignore"; srun --job-name="Save-outputs-to-S3" aws --no-verify-ssl --endpoint-url ${storageEndpoint} s3 sync --acl public-read --exclude "*.sbatch" --exclude "*.omex" . 's3://${storageBucket}/${simulationRunS3Path}'
+export PYTHONWARNINGS="ignore"
+srun --job-name="Save-outputs-to-S3" \
+  aws \
+    --no-verify-ssl \
+    --endpoint-url ${storageEndpoint} \
+    s3 sync \
+      --acl public-read \
+      --exclude "job.sbatch" \
+      --exclude "${combineArchiveFilename}" \
+      . \
+      's3://${storageBucket}/${simulationRunS3Path}'
 `;
 
     return template;

diff --git a/apps/dispatch-service/src/app/services/ssh/ssh.service.ts b/apps/dispatch-service/src/app/services/ssh/ssh.service.ts
@@ -27,9 +27,15 @@ export class SshService {
     '',
   );
   constructor(private configService: ConfigService) {}
-  public getSSHResultsDirectory(id: string): string {
+
+  public getSSHJobDirectory(id: string): string {
     return path.join(this.hpcBase, id);
   }
+
+  public getSSHJobOutputsDirectory(id: string): string {
+    return path.join(this.hpcBase, id, 'outputs');
+  }
+
   public execStringCommand(
     cmd: string,
   ): Promise<{ stdout: string; stderr: string }> {

diff --git a/libs/config/common/src/lib/file-paths.ts b/libs/config/common/src/lib/file-paths.ts
@@ -5,8 +5,9 @@ import { envs } from '@biosimulations/shared/environments';
 export class FilePaths {
   private endpoints: Endpoints;
   private static simulationRunsPath = 'simulations';
-  private static simulationRunContentSubpath = 'contents';
+  private static simulationRunContentsSubpath = 'contents';
   private static simulationRunThumbnailSubpath = 'thumbnails';
+  private static simulationRunOutputsSubpath = 'outputs';
 
   public constructor(env?: envs) {
     this.endpoints = new Endpoints(env);
@@ -80,7 +81,7 @@ export class FilePaths {
   ): string {
     const dirPath = thumbnailType
       ? FilePaths.simulationRunThumbnailSubpath + '/' + thumbnailType
-      : FilePaths.simulationRunContentSubpath;
+      : FilePaths.simulationRunContentsSubpath;
     const filePath = fileLocation !== undefined ? `/${fileLocation}` : '';
     return this.getSimulationRunPath(
       runId,
@@ -93,11 +94,26 @@ export class FilePaths {
    * @param runId Id of the simulation run
    * @param absolute Whether to get the absolute path, or the path relative to the S3 path for the simulation run
    */
-  public getSimulationRunOutputPath(runId: string, absolute = true): string {
+  public getSimulationRunOutputArchivePath(runId: string, absolute = true): string {
+    const relativePath = `${runId}.zip`; 
     if (absolute) {
-      return this.getSimulationRunPath(runId, `${runId}.zip`);
+      return this.getSimulationRunPath(runId, relativePath);
     } else {
-      return `${runId}.zip`;
+      return relativePath;
+    }
+  }
+
+  /**
+   * Create a path for a directory of outputs of a simulation run in an S3 bucket
+   * @param runId Id of the simulation run
+   * @param absolute Whether to get the absolute path, or the path relative to the S3 path for the simulation run
+   */
+  public getSimulationRunOutputsPath(runId: string, absolute = true): string {
+    const relativePath = FilePaths.simulationRunOutputsSubpath;
+    if (absolute) {
+      return this.getSimulationRunPath(runId, relativePath);
+    } else {
+      return relativePath;
     }
   }
 }
diff --git a/libs/shared/storage/src/lib/shared-storage.service.ts b/libs/shared/storage/src/lib/shared-storage.service.ts
@@ -37,6 +37,20 @@ export class SharedStorageService {
     s3.config.update({ region: 'us-east-1' });
   }
 
+  public async listObjects(id: string): Promise<AWS.S3.ListObjectsOutput> {
+    const call = this.retryS3<any>(async (): Promise<any> => {
+      return this.s3.listObjects({ Bucket: this.BUCKET, Prefix: id }).promise();
+    });
+
+    const res = await call;
+
+    if (res.$response.error) {
+      throw res.$response.error.originalError;
+    } else {
+      return res;
+    }
+  }
+
   public async isObject(id: string): Promise<boolean> {
     const call = this.retryS3<any>(async (): Promise<any> => {
       return this.s3.headObject({ Bucket: this.BUCKET, Key: id }).promise();
@@ -182,7 +196,7 @@ export class SharedStorageService {
       async (retry): Promise<T> => {
         return func()
           .catch((error: any) => {
-            if (SharedStorageService.RETRY_ERROR_CODES.includes(error.status)) {
+            if (SharedStorageService.RETRY_ERROR_CODES.includes(error.status || error.statusCode)) {
               retry(error);
             }
             throw error;

diff --git a/libs/shared/storage/src/lib/simulation-storage.service.ts b/libs/shared/storage/src/lib/simulation-storage.service.ts
@@ -29,7 +29,12 @@ export class SimulationStorageService {
   }
 
   public async deleteSimulationRunResults(runId: string): Promise<void> {
-    await this.deleteSimulationArchive(runId);
+    const s3path = this.filePaths.getSimulationRunOutputArchivePath(runId);
+    await this.deleteS3Object(
+      runId,
+      s3path,
+      `COMBINE archive could not be deleted for simulation run '{runId}'.`,
+    );
   }
 
   public async deleteSimulationRunFile(
@@ -56,14 +61,19 @@ export class SimulationStorageService {
           thumbnailType as ThumbnailType,
         );
 
-        const hasThumbnail = await this.storage.isObject(s3path);
-        if (hasThumbnail) {
-          await this.deleteS3Object(
-            runId,
-            s3thumbnailPath,
-            `Thumbnail '${fileLocation}' could not be deleted for simulation run '{runId}'.`,
-          );
-        }
+        await this.deleteS3Object(
+          runId,
+          s3thumbnailPath,
+          `Thumbnail '${fileLocation}' could not be deleted for simulation run '{runId}'.`,
+        )
+        .catch((error: any) => {
+          if (!(
+            error.statusCode === HttpStatus.NOT_FOUND &&
+            error.code === 'NoSuchKey'
+          )) {
+            throw error;
+          }
+        });
       })
     );
   }
@@ -72,7 +82,7 @@ export class SimulationStorageService {
     runId: string,
   ): Promise<S3.GetObjectOutput> {
     const file = await this.storage.getObject(
-      this.filePaths.getSimulationRunOutputPath(runId),
+      this.filePaths.getSimulationRunOutputArchivePath(runId),
     );
     return file;
   }
@@ -128,6 +138,27 @@ export class SimulationStorageService {
     );
   }
 
+  public async deleteSimulation(runId: string): Promise<void> {
+    const s3prefix = this.filePaths.getSimulationRunPath(runId, '');
+    const s3paths: string[] = (await this.storage.listObjects(s3prefix))?.Contents?.flatMap((Content): string[] => {
+      if (Content?.Key) {
+        return [Content.Key];
+      } else {
+        return [];
+      }
+    }) || [];
+
+    await Promise.all(
+      s3paths.map(async (s3path: string): Promise<void> => {
+        return this.deleteS3Object(
+          runId,
+          s3path,
+          `COMBINE archive could not be deleted for simulation run '{runId}'.`,
+        );
+      })
+    );
+  }
+
   private async deleteS3Object(
     runId: string,
     s3path: string,