feat(job-runner): save job artifacts by project id

packages/platform-server/src/modules/job/__tests__/job.spec.e2e.ts

@@ -121,17 +121,22 @@ test('should return 404 if job not found', async (t) => {
 })
 
 test('upload and download artifact', async (t) => {
-  const key = 'test.txt'
+  const filename = 'test.txt'
+
+  const job = await create(Job, { status: JobStatus.Running, runner })
 
   const upload = await request(config.host)
     .post('/api/jobs/artifacts')
-    .query({ key })
+    .query({ key: filename, jobId: job.id })
     .set('x-runner-token', runner.token)
     .set('content-type', 'application/octet-stream')
     .send(Buffer.from('content'))
 
   t.is(upload.statusCode, HttpStatus.CREATED)
 
+  const { key } = upload.body
+  t.truthy(key)
+
   const download = await request(config.host)
     .get('/api/jobs/artifacts')
     .query({ key })
@@ -144,9 +149,11 @@ test('upload and download artifact', async (t) => {
 test('should forbid invalid runner upload & download artifact', async (t) => {
   const key = 'test.txt'
 
+  const job = await create(Job, { status: JobStatus.Running, runner })
+
   const upload = await request(config.host)
     .post('/api/jobs/artifacts')
-    .query({ key })
+    .query({ key, jobId: job.id })
     .set('x-runner-token', 'invalid token')
     .set('content-type', 'application/octet-stream')
     .send(Buffer.from('content'))
@@ -161,6 +168,44 @@ test('should forbid invalid runner upload & download artifact', async (t) => {
   t.is(download.statusCode, HttpStatus.FORBIDDEN)
 })
 
+test('should forbid invalid jobId', async (t) => {
+  const key = 'test.txt'
+
+  const upload = await request(config.host)
+    .post('/api/jobs/artifacts')
+    .query({ key, jobId: 'not-number' })
+    .set('x-runner-token', runner.token)
+    .set('content-type', 'application/octet-stream')
+    .send(Buffer.from('content'))
+
+  t.is(upload.statusCode, HttpStatus.FORBIDDEN)
+
+  const upload2 = await request(config.host)
+    .post('/api/jobs/artifacts')
+    .query({ key, jobId: '123456' })
+    .set('x-runner-token', runner.token)
+    .set('content-type', 'application/octet-stream')
+    .send(Buffer.from('content'))
+
+  t.is(upload2.statusCode, HttpStatus.NOT_FOUND)
+})
+
+test('should return 403 if jobId and runner do not match', async (t) => {
+  const key = 'test.txt'
+
+  const otherRunner = await create(Runner)
+  const job = await create(Job, { status: JobStatus.Running, runner: otherRunner })
+
+  const upload = await request(config.host)
+    .post('/api/jobs/artifacts')
+    .query({ key, jobId: job.id })
+    .set('x-runner-token', runner.token)
+    .set('content-type', 'application/octet-stream')
+    .send(Buffer.from('content'))
+
+  t.is(upload.statusCode, HttpStatus.FORBIDDEN)
+})
+
 test('should return 404 if no artifact found', async (t) => {
   const download = await request(config.host)
     .get('/api/jobs/artifacts')

packages/platform-server/src/modules/job/controller.ts

@@ -14,12 +14,25 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { Controller, Post, Body, Headers, Get, Query, HttpCode, HttpStatus, Res } from '@nestjs/common'
+import {
+  Controller,
+  Post,
+  Body,
+  Headers,
+  Get,
+  Query,
+  HttpCode,
+  HttpStatus,
+  Res,
+  ForbiddenException,
+  NotFoundException,
+} from '@nestjs/common'
 import { Response } from 'express'
 import Redlock from 'redlock'
 
 import { Config } from '@perfsee/platform-server/config'
 import { Cron, CronExpression } from '@perfsee/platform-server/cron'
+import { Job } from '@perfsee/platform-server/db'
 import { EventEmitter } from '@perfsee/platform-server/event'
 import { Logger } from '@perfsee/platform-server/logger'
 import { Metric } from '@perfsee/platform-server/metrics'
@@ -145,9 +158,37 @@ export class JobController {
 
   @Post('/artifacts')
   @HttpCode(HttpStatus.CREATED)
-  async uploadArtifact(@Body() buf: Buffer, @Query('key') key: string, @Headers('x-runner-token') token: string) {
-    await this.runner.authenticateRunner(token)
-    await this.storage.upload(key, buf)
+  async uploadArtifact(
+    @Body() buf: Buffer,
+    @Query('jobId') jobId: string,
+    @Query('key') key: string,
+    @Headers('x-runner-token') token: string,
+  ) {
+    let id
+    try {
+      id = parseInt(jobId)
+      // eslint-disable-next-line no-empty
+    } catch {}
+    if (!id) {
+      throw new ForbiddenException('Invalid jobId')
+    }
+
+    const runner = await this.runner.authenticateRunner(token)
+    const job = await Job.findOneBy({ id })
+
+    if (!job) {
+      throw new NotFoundException('Job not found')
+    }
+
+    if (job.runnerId !== runner.id) {
+      throw new ForbiddenException('JobId not match the runner')
+    }
+
+    const finalKey = 'artifacts/' + job.projectId + '/' + key
+    await this.storage.upload(finalKey, buf)
+    return {
+      key: finalKey,
+    }
   }
 
   @Get('/artifacts')

packages/runner/bundle/src/bundle/index.ts

@@ -73,13 +73,14 @@ export class BundleWorker extends JobWorker<BundleJobPayload> {
     const parser = StatsParser.FromStats(stats, parse(this.statsFilePath).dir, this.logger)
     const { report, moduleTree } = await parser.parse()
 
-    const bundleReportKey = `perfsee/bundle-results/${uuid()}.json`
-    const bundleContentKey = `perfsee/bundle-content/${uuid()}.json`
-    await this.client.uploadArtifact(bundleReportKey, Buffer.from(JSON.stringify(report)))
+    const bundleReportName = `bundle-results/${uuid()}.json`
+    const bundleReportKey = await this.client.uploadArtifact(bundleReportName, Buffer.from(JSON.stringify(report)))
     this.logger.info(`Bundle analysis result uploaded to artifacts. key is: ${bundleReportKey}`)
 
+    const bundleContentName = `bundle-content/${uuid()}.json`
+    let bundleContentKey
     if (moduleTree.length) {
-      await this.client.uploadArtifact(bundleContentKey, Buffer.from(JSON.stringify(moduleTree)))
+      bundleContentKey = await this.client.uploadArtifact(bundleContentName, Buffer.from(JSON.stringify(moduleTree)))
       this.logger.info(`Bundle content result uploaded to artifacts. key is: ${bundleContentKey}`)
     }
 

packages/runner/lab/src/lighthouse/e2e-worker.ts

@@ -160,7 +160,8 @@ export abstract class E2eJobWorker extends JobWorker<E2EJobPayload> {
 
     // run
     let failedReason
-    const screencastStorageKey = `perfsee/screencast/${uuid()}.mp4`
+    const screencastName = `screencast/${uuid()}.mp4`
+    let screencastStorageKey
     let userFlowResult
     const startTime = Date.now()
     try {
@@ -182,7 +183,7 @@ export abstract class E2eJobWorker extends JobWorker<E2EJobPayload> {
 
     if (screenRecorder.isStarted) {
       await screenRecorder.stop()
-      await this.uploadScreencast(screencastStorageKey, videoFile)
+      screencastStorageKey = await this.uploadScreencast(screencastName, videoFile)
     }
 
     try {
@@ -253,9 +254,9 @@ export abstract class E2eJobWorker extends JobWorker<E2EJobPayload> {
     ]
 
     try {
-      const lighthouseStorageKey = `perfsee/snapshots/${uuid()}.json`
-      await this.client.uploadArtifact(
-        lighthouseStorageKey,
+      const lighthouseStorageName = `snapshots/${uuid()}.json`
+      const lighthouseStorageKey = await this.client.uploadArtifact(
+        lighthouseStorageName,
         Buffer.from(
           JSON.stringify({
             metricScores,
@@ -293,9 +294,10 @@ export abstract class E2eJobWorker extends JobWorker<E2EJobPayload> {
 
   private async uploadScreencast(name: string, screencastPath: string) {
     try {
-      await this.client.uploadArtifactFile(name, screencastPath)
+      const fileKey = await this.client.uploadArtifactFile(name, screencastPath)
       this.logger.verbose('Cleanup screencast path')
       await fs.rm(dirname(screencastPath), { recursive: true, force: true })
+      return fileKey
     } catch (e) {
       this.logger.error('Failed to upload video', { error: e })
     }

packages/runner/lab/src/lighthouse/lighthouse-worker.ts

@@ -86,8 +86,10 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
     const jsCoverage = artifacts.JsUsage ?? {}
 
     // artifacts
-    const lighthouseFile = `perfsee/snapshots/${uuid()}.json`
-    const jsCoverageFile = `perfsee/js-coverage/${uuid()}.json`
+    const lighthouseFile = `snapshots/${uuid()}.json`
+    const jsCoverageFile = `js-coverage/${uuid()}.json`
+    let lighthouseStorageKey
+    let jsCoverageStorageKey
 
     // delete useless lighthouse data
     // @ts-expect-error
@@ -101,7 +103,7 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
     lhr.audits['network-requests'] = {}
 
     try {
-      await this.client.uploadArtifact(
+      lighthouseStorageKey = await this.client.uploadArtifact(
         lighthouseFile,
         Buffer.from(
           JSON.stringify({
@@ -127,7 +129,7 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
     }
 
     try {
-      await this.client.uploadArtifact(jsCoverageFile, Buffer.from(JSON.stringify(jsCoverage)))
+      jsCoverageStorageKey = await this.client.uploadArtifact(jsCoverageFile, Buffer.from(JSON.stringify(jsCoverage)))
     } catch (e) {
       this.logger.error('Failed to upload audit result', { error: e })
       return {
@@ -139,9 +141,9 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
     const traceEventsStorageKey = await this.uploadTraceEvents(artifacts)
 
     return {
-      lighthouseStorageKey: lighthouseFile,
+      lighthouseStorageKey,
       screencastStorageKey,
-      jsCoverageStorageKey: jsCoverageFile,
+      jsCoverageStorageKey,
       traceEventsStorageKey,
       metrics,
     }
@@ -303,12 +305,12 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
 
   private async uploadScreencast(screencast: LH.ScreencastGathererResult | null) {
     try {
-      const screencastFile = `perfsee/screencast/${uuid()}.mp4`
+      const screencastFile = `screencast/${uuid()}.mp4`
       if (screencast) {
-        await this.client.uploadArtifactFile(screencastFile, screencast.path)
+        const uploadedFileKey = await this.client.uploadArtifactFile(screencastFile, screencast.path)
         this.logger.verbose('Cleanup screencast path')
         await rm(dirname(screencast.path), { recursive: true, force: true })
-        return screencastFile
+        return uploadedFileKey
       }
     } catch (e) {
       this.logger.error('Failed to upload video', { error: e })
@@ -318,10 +320,9 @@ export abstract class LighthouseJobWorker extends JobWorker<LabJobPayload> {
   private async uploadTraceEvents(artifacts: LH.Artifacts) {
     const { traceEvents } = artifacts.traces['defaultPass']
 
-    const traceEventsFile = `perfsee/trace-events/${uuid()}.json`
+    const traceEventsFile = `trace-events/${uuid()}.json`
     try {
-      await this.client.uploadArtifact(traceEventsFile, Buffer.from(JSON.stringify(traceEvents)))
-      return traceEventsFile
+      return await this.client.uploadArtifact(traceEventsFile, Buffer.from(JSON.stringify(traceEvents)))
     } catch (e) {
       this.logger.error('Failed to upload trace events', { error: e })
     }

packages/runner/shared/src/platform-client.ts

@@ -21,10 +21,10 @@ import { stat } from 'fs/promises'
 import { merge } from 'lodash'
 import fetch, { RequestInit, BodyInit } from 'node-fetch'
 
-import { ServerConfig } from './types'
+import { WorkerData } from './types'
 
 export class PlatformClient {
-  constructor(private readonly config: ServerConfig) {}
+  constructor(private readonly config: WorkerData) {}
 
   async getArtifact(name: string) {
     const res = await this.fetch(`/api/jobs/artifacts?key=${name}`, { method: 'GET' })
@@ -45,7 +45,7 @@ export class PlatformClient {
   }
 
   async uploadArtifact(name: string, body: BodyInit) {
-    const res = await this.fetch(`/api/jobs/artifacts?key=${name}`, {
+    const res = await this.fetch(`/api/jobs/artifacts?jobId=${this.config.job.jobId}&key=${name}`, {
       method: 'POST',
       body,
       headers: {
@@ -56,6 +56,9 @@ export class PlatformClient {
     if (!res.ok) {
       throw new Error('Failed to upload artifact.')
     }
+
+    const { key } = await res.json()
+    return key as string
   }
 
   async uploadArtifactFile(name: string, path: string) {
@@ -97,16 +100,16 @@ export class PlatformClient {
       merge(
         {
           headers: {
-            'x-runner-token': this.config.token,
+            'x-runner-token': this.config.server.token,
           },
-          timeout: this.config.timeout * 1000,
+          timeout: this.config.server.timeout * 1000,
         },
         init,
       ),
     )
   }
 
   async doFetch(path: string, init?: RequestInit) {
-    return fetch(this.config.url + path, init)
+    return fetch(this.config.server.url + path, init)
   }
 }

packages/runner/shared/src/worker.ts

@@ -42,7 +42,7 @@ export abstract class JobWorker<Payload = any> {
   }
 
   constructor(private readonly workerData: WorkerData<Payload>) {
-    this.client = new PlatformClient(this.workerData.server)
+    this.client = new PlatformClient(this.workerData)
   }
 
   /**

packages/runner/source/src/source/index.ts

@@ -77,9 +77,12 @@ export class SourceJobWorker extends JobWorker<SourceAnalyzeJob> {
           )
 
           const { diagnostics, profile } = JSON.parse(analyzeResultStr) as ProfileAnalyzeResult
-          const flameChartStorageKey = `perfsee/flame-charts/${uuid()}.json`
-          this.logger.info(`Uploading analysis flame chart data to artifacts. [key=${flameChartStorageKey}]`)
-          await this.client.uploadArtifact(flameChartStorageKey, Buffer.from(JSON.stringify(profile), 'utf-8'))
+          const flameChartStorageName = `flame-charts/${uuid()}.json`
+          this.logger.info(`Uploading analysis flame chart data to artifacts. [name=${flameChartStorageName}]`)
+          const flameChartStorageKey = await this.client.uploadArtifact(
+            flameChartStorageName,
+            Buffer.from(JSON.stringify(profile), 'utf-8'),
+          )
           this.logger.info(`Finished uploading. [key=${flameChartStorageKey}]`)
 
           const sourceCoverageStorageKey = sourceCoverageResult.find(
@@ -270,9 +273,12 @@ export class SourceJobWorker extends JobWorker<SourceAnalyzeJob> {
       }
 
       const coverageData = await generateSourceCoverageTreemapData({ jsCoverageData, source, pageUrl })
-      const StorageKey = `perfsee/source-coverage/${uuid()}.json`
-      await this.client.uploadArtifact(StorageKey, Buffer.from(JSON.stringify(coverageData), 'utf-8'))
-      result.push({ reportId, sourceCoverageStorageKey: StorageKey })
+      const storageName = `source-coverage/${uuid()}.json`
+      const sourceCoverageStorageKey = await this.client.uploadArtifact(
+        storageName,
+        Buffer.from(JSON.stringify(coverageData), 'utf-8'),
+      )
+      result.push({ reportId, sourceCoverageStorageKey })
     }
 
     return result

packages/server-common/src/types/index.ts

@@ -91,7 +91,7 @@ export type BundleJobPassedUpdate = {
   artifactId: number
   status: BundleJobStatus.Passed
   reportKey: string
-  contentKey: string
+  contentKey: string | undefined
   entryPoints: Record<string, BundleJobEntryPoint>
   duration: number
   score: number