feat: save pipeline executions to current dir if not prod

jupyter-naas · Mar 28, 2024 · 4c8ab54 · 4c8ab54
1 parent 44a37f6
commit 4c8ab54
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 6 deletions.
diff --git a/models/content-engine/__pipeline__.ipynb b/models/content-engine/__pipeline__.ipynb
@@ -194,7 +194,9 @@
    "outputs": [],
    "source": [
     "engine_dir = os.path.join(entity_dir, engine_name, date.today().isoformat())\n",
-    "print(\"✅ Output directory:\", engine_dir)"
+    "print(\"✅ Output directory:\", engine_dir)\n",
+    "pipeline_dir = os.path.join(engine_dir, \"pipeline_executions\") if naas.is_production() else os.path.join(naas_data_product.MODELS_PATH, engine_name, \"custom\", \"pipeline_executions\")\n",
+    "print(\"✅ Pipeline directory:\", pipeline_dir)"
    ]
   },
   {
@@ -280,8 +282,11 @@
     "        name=\"Custom\",\n",
     "        notebook_path=os.path.join(naas_data_product.MODELS_PATH, engine_name, \"custom\", \"__pipeline__.ipynb\"),\n",
     "        parameters={\n",
+    "            \"datalake_dir\": datalake_dir,\n",
+    "            \"api_key\": api_key,\n",
     "            \"entity_name\": entity_name,\n",
     "            \"entity_dir\": entity_dir,\n",
+    "            \"spreadsheet_url\": spreadsheet_url,\n",
     "        }\n",
     "    )\n",
     "else:\n",
@@ -317,7 +322,7 @@
     "\n",
     "pipeline >> extraction >> cleaning >> analytics >> plugin >> email >> custom >> End()\n",
     "\n",
-    "pipeline.run(outputs_path=os.path.join(engine_dir, \"pipeline_executions\"))"
+    "pipeline.run(outputs_path=pipeline_dir)"
    ]
   },
   {

diff --git a/models/growth-engine/__pipeline__.ipynb b/models/growth-engine/__pipeline__.ipynb
@@ -199,7 +199,9 @@
    "outputs": [],
    "source": [
     "engine_dir = os.path.join(entity_dir, engine_name, date.today().isoformat())\n",
-    "print(\"✅ Output directory:\", engine_dir)"
+    "print(\"✅ Output directory:\", engine_dir)\n",
+    "pipeline_dir = os.path.join(engine_dir, \"pipeline_executions\") if naas.is_production() else os.path.join(naas_data_product.MODELS_PATH, engine_name, \"custom\", \"pipeline_executions\")\n",
+    "print(\"✅ Pipeline directory:\", pipeline_dir)"
    ]
   },
   {
@@ -340,8 +342,11 @@
     "        name=\"Custom\",\n",
     "        notebook_path=os.path.join(naas_data_product.MODELS_PATH, engine_name, \"custom\", \"__pipeline__.ipynb\"),\n",
     "        parameters={\n",
+    "            \"datalake_dir\": datalake_dir,\n",
+    "            \"api_key\": api_key,\n",
     "            \"entity_name\": entity_name,\n",
     "            \"entity_dir\": entity_dir,\n",
+    "            \"spreadsheet_url\": spreadsheet_url,\n",
     "        }\n",
     "    )\n",
     "else:\n",
@@ -377,7 +382,7 @@
     "\n",
     "pipeline >> linkedin >> interactions >> growth >> people >> organizations >> contacts >> analytics >> plugin >> email >> custom >> End()\n",
     "\n",
-    "pipeline.run(outputs_path=os.path.join(engine_dir, \"pipeline_executions\"))"
+    "pipeline.run(outputs_path=pipeline_dir)"
    ]
   },
   {

diff --git a/models/sales-engine/__pipeline__.ipynb b/models/sales-engine/__pipeline__.ipynb
@@ -194,7 +194,9 @@
    "outputs": [],
    "source": [
     "engine_dir = os.path.join(entity_dir, engine_name, date.today().isoformat())\n",
-    "print(\"✅ Output directory:\", engine_dir)"
+    "print(\"✅ Output directory:\", engine_dir)\n",
+    "pipeline_dir = os.path.join(engine_dir, \"pipeline_executions\") if naas.is_production() else os.path.join(naas_data_product.MODELS_PATH, engine_name, \"custom\", \"pipeline_executions\")\n",
+    "print(\"✅ Pipeline directory:\", pipeline_dir)"
    ]
   },
   {
@@ -307,7 +309,7 @@
     "\n",
     "pipeline >> deals >> analytics >> plugin >> email >> custom >> End()\n",
     "\n",
-    "pipeline.run(outputs_path=os.path.join(engine_dir, \"pipeline_executions\"))"
+    "pipeline.run(outputs_path=pipeline_dir)"
    ]
   }
  ],