From 94a65e8b7d33bc45c1249744eb79b30b27b3bbf1 Mon Sep 17 00:00:00 2001
From: il3ven <vaibhavchanana@yahoo.co.in>
Date: Tue, 4 Oct 2022 18:32:36 +0530
Subject: [PATCH] add an outputFilename option for strategies

Strategies can now optionally define outputFilename in crawlPath.

For the following crawl path.

{
  name: "get-tokenuri",
  extractor: {
    args: [resolve(env.DATA_DIR, "soundxyz-call-tokenuri-transformation")],
    outputFilename: "soundxyz-get-tokenuri"
  },
  transformer: {},
}

The results get-tokenuri will be written to soundxyz-get-tokenuri-extraction.

Closes #258
---
 src/lifecycle.mjs | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/lifecycle.mjs b/src/lifecycle.mjs
index 881ab64..18ac136 100644
--- a/src/lifecycle.mjs
+++ b/src/lifecycle.mjs
@@ -128,7 +128,13 @@ export function generatePath(name, type) {
   return path.resolve(dataDir, `${name}-${type}`);
 }
 
-export function extract(strategy, worker, messageRouter, args = []) {
+export function extract(
+  strategy,
+  worker,
+  messageRouter,
+  outputPath,
+  args = []
+) {
   return new Promise(async (resolve, reject) => {
     let numberOfMessages = 0;
     const type = "extraction";
@@ -151,12 +157,11 @@ export function extract(strategy, worker, messageRouter, args = []) {
     }
 
     if (result.write) {
-      const filePath = generatePath(strategy.module.name, type);
       try {
-        appendFileSync(filePath, `${result.write}\n`);
+        appendFileSync(outputPath, `${result.write}\n`);
       } catch (err) {
         const error = new Error(
-          `Couldn't write to file after update. Filepath: "${filePath}", Content: "${result.write}"`
+          `Couldn't write to file after update. Filepath: "${outputPath}", Content: "${result.write}"`
         );
         error.code = EXTRACTOR_CODES.FAILURE;
         clearInterval(interval);
@@ -205,12 +210,11 @@ export function extract(strategy, worker, messageRouter, args = []) {
           }
 
           if (result.write) {
-            const filePath = generatePath(strategy.module.name, type);
             try {
-              appendFileSync(filePath, `${result.write}\n`);
+              appendFileSync(outputPath, `${result.write}\n`);
             } catch (err) {
               const error = new Error(
-                `Couldn't write to file after update. Filepath: "${filePath}", Content: "${result.write}"`
+                `Couldn't write to file after update. Filepath: "${outputPath}", Content: "${result.write}"`
               );
               error.code = EXTRACTOR_CODES.FAILURE;
               messageRouter.off(`${strategy.module.name}-${type}`, callback);
@@ -285,6 +289,10 @@ export async function init(worker, crawlPath) {
     for await (const strategy of segment) {
       if (strategy.extractor) {
         const extractStrategy = finder("extraction", strategy.name);
+        const outputPath = generatePath(
+          strategy.extractor.outputFilename ?? extractStrategy.module.name,
+          "extraction"
+        );
         log(
           `Starting extractor strategy with name "${
             extractStrategy.module.name
@@ -294,6 +302,7 @@ export async function init(worker, crawlPath) {
           extractStrategy,
           worker,
           messageRouter,
+          outputPath,
           strategy.extractor.args
         );
         log(
@@ -310,7 +319,7 @@ export async function init(worker, crawlPath) {
           strategy.transformer.args?.[0] ??
           generatePath(transformStrategy.module.name, "extraction");
         const outputPath = generatePath(
-          transformStrategy.module.name,
+          strategy.transformer.outputFilename ?? transformStrategy.module.name,
           "transformation"
         );
         await transform(