Add env variables to configure different paths for data export

RDB_DATA_EXPORT_PATH is used by Postgres to store CSV files and should be accessible to Node.js via NODE_DATA_EXPORT_PATH.

NODE_DATA_EXPORT_PATH is used to check for whether data files need to be generated and Node stores json files directly into it.

Signed-off-by: Stefan Marr <git@stefan-marr.de>

src/backend/dev-server/server.ts

@@ -2,7 +2,7 @@ import { ParameterizedContext } from 'koa';
 import { readFileSync } from 'node:fs';
 
 import { log } from '../logging.js';
-import { robustPath, robustSrcPath } from '../util.js';
+import { robustPath, robustSrcPath, siteConfig } from '../util.js';
 
 export async function serveStaticResource(
   ctx: ParameterizedContext
@@ -31,11 +31,11 @@ export async function serveStaticResource(
   } else if (filename.endsWith('.json.gz')) {
     ctx.type = 'application/json';
     ctx.set('Content-Encoding', 'gzip');
-    path = robustPath(`../resources/${filename}`);
+    path = `${siteConfig.dataExportPath}/${filename}`;
   } else if (filename.endsWith('.csv.gz')) {
     ctx.type = 'text/csv';
     ctx.set('Content-Encoding', 'gzip');
-    path = robustPath(`../resources/${filename}`);
+    path = `${siteConfig.dataExportPath}/${filename}`;
   } else {
     throw new Error(`Unsupported file type. Filename: ${filename}`);
   }

src/backend/project/data-export.ts

@@ -1,6 +1,6 @@
 import { existsSync } from 'node:fs';
 import { completeRequest, startRequest } from '../perf-tracker.js';
-import { robustPath, siteConfig, storeJsonGzip } from '../util.js';
+import { dbConfig, robustPath, siteConfig, storeJsonGzip } from '../util.js';
 import { log } from '../logging.js';
 import { Database } from '../db/db.js';
 import { ParameterizedContext } from 'koa';
@@ -27,10 +27,9 @@ export async function getExpData(
   }
 
   const expFilePrefix = `${data.project}-${expId}`;
-  const expFileName = `exp-data/${expFilePrefix}.${format}.gz`;
-  const expDataFile = robustPath(`../resources/${expFileName}`);
+  const expFileName = `${expFilePrefix}.${format}.gz`;
 
-  if (existsSync(expDataFile)) {
+  if (existsSync(`${siteConfig.dataExportPath}/${expFileName}`)) {
     data.preparingData = false;
     data.downloadUrl = `${siteConfig.staticUrl}/${expFileName}`;
   } else {
@@ -48,7 +47,10 @@ export async function getExpData(
       const resultP =
         format === 'json'
           ? db.getExperimentMeasurements(expId)
-          : db.storeExperimentMeasurements(expId, expDataFile);
+          : db.storeExperimentMeasurements(
+              expId,
+              `${dbConfig.dataExportPath}/${expFileName}`
+            );
 
       expDataPreparation.set(expRequestId, {
         inProgress: true
@@ -57,7 +59,10 @@ export async function getExpData(
       resultP
         .then(async (data: any[]) => {
           if (format === 'json') {
-            await storeJsonGzip(data, expDataFile);
+            await storeJsonGzip(
+              data,
+              `${siteConfig.dataExportPath}/${expFileName}`
+            );
           }
           expDataPreparation.set(expRequestId, {
             inProgress: false

src/backend/util.ts

@@ -49,11 +49,31 @@ const port: number = process.env.RDB_PORT
   ? parseInt(process.env.RDB_PORT)
   : 5432;
 
+const _rebench_dev = 'https://rebench.dev';
+const reportsUrl = process.env.REPORTS_URL || '/static/reports';
+const staticUrl = process.env.STATIC_URL || '/static';
+const publicUrl = process.env.PUBLIC_URL || _rebench_dev;
+
+// configuration for data export is a little more involved,
+// because the database might run elsewhere, but may produce
+// data files, which we need to be able to serve, at least in the dev mode.
+const dbDataExportPath =
+  process.env.RDB_DATA_EXPORT_PATH || robustPath('../resources/exp-data');
+
+// I assume that Node has access to files produced by itself and PostgreSQL.
+const nodeDataExportPath =
+  process.env.NODE_DATA_EXPORT_PATH || dbDataExportPath;
+
+const dataExportUrlBase = process.env.DATA_URL_BASE || `${staticUrl}/exp-data`;
+
 export const dbConfig = {
   user: process.env.RDB_USER || '',
   password: process.env.RDB_PASS || '',
   host: process.env.RDB_HOST || 'localhost',
   database: process.env.RDB_DB || 'rdb_smde2',
+
+  /** The path where PostgreSQL writes data files to. */
+  dataExportPath: dbDataExportPath,
   port
 };
 
@@ -63,8 +83,6 @@ export const refreshSecret =
 /** How long to still hold on to the cache after it became invalid. In ms. */
 export const cacheInvalidationDelay = 1000 * 60 * 5; /* 5 minutes */
 
-const _rebench_dev = 'https://rebench.dev';
-
 export function isReBenchDotDev(): boolean {
   return siteConfig.publicUrl === _rebench_dev;
 }
@@ -83,9 +101,16 @@ export const statsConfig = {
 
 export const siteConfig = {
   port: process.env.PORT || 33333,
-  reportsUrl: process.env.REPORTS_URL || '/static/reports',
-  staticUrl: process.env.STATIC_URL || '/static',
-  publicUrl: process.env.PUBLIC_URL || _rebench_dev,
+  reportsUrl,
+  staticUrl,
+  publicUrl,
+  dataExportUrlBase,
+
+  /**
+   * The path where Node.js writes data files to,
+   * and Postgres generated files are accessible.
+   */
+  dataExportPath: nodeDataExportPath,
   appId: parseInt(process.env.GITHUB_APP_ID || '') || 76497,
   githubPrivateKey:
     process.env.GITHUB_PK || 'rebenchdb.2020-08-11.private-key.pem',