fix: handle lk 302 error + add entity index

jupyter-naas · Mar 28, 2024 · 25927e0 · 25927e0
1 parent 2fdaa91
commit 25927e0
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 10 deletions.
diff --git a/models/growth-engine/core/domain/Google_Sheets_Update_growth_db.ipynb b/models/growth-engine/core/domain/Google_Sheets_Update_growth_db.ipynb
@@ -132,12 +132,13 @@
    "outputs": [],
    "source": [
     "# Inputs\n",
-    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
+    "entity_index = \"0\"\n",
+    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"entity_dir\")\n",
     "input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "file_interactions = \"interactions\"\n",
     "\n",
     "# Outputs\n",
-    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
+    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"abi_spreadsheet\")\n",
     "sheet_growth = \"GROWTH\"\n",
     "output_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "output_file = \"growth\""

diff --git a/models/growth-engine/core/domain/Google_Sheets_Update_interactions_db.ipynb b/models/growth-engine/core/domain/Google_Sheets_Update_interactions_db.ipynb
@@ -131,15 +131,16 @@
    "outputs": [],
    "source": [
     "# Inputs\n",
-    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
+    "entity_index =  \"0\"\n",
+    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"entity_dir\")\n",
     "input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "file_reactions = \"linkedin_post_reactions\"\n",
     "file_comments = \"linkedin_post_comments\"\n",
     "\n",
     "# Outputs\n",
     "output_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "output_file = \"interactions\"\n",
-    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
+    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"abi_spreadsheet\")\n",
     "sheet_interaction = \"INTERACTIONS\""
    ]
   },

diff --git a/models/growth-engine/core/domain/Google_Sheets_Update_organizations_db.ipynb b/models/growth-engine/core/domain/Google_Sheets_Update_organizations_db.ipynb
@@ -140,13 +140,14 @@
    "outputs": [],
    "source": [
     "# Inputs\n",
-    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
+    "entity_index = \"0\"\n",
+    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"entity_dir\")\n",
     "input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "file_interactions = \"interactions\"\n",
     "file_people = \"people\"\n",
     "li_at = os.environ.get(\"LINKEDIN_LI_AT\") or naas.secret.get(\"LINKEDIN_LI_AT\")\n",
     "JSESSIONID = os.environ.get(\"LINKEDIN_JSESSIONID\") or naas.secret.get(\"LINKEDIN_JSESSIONID\")\n",
-    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
+    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"abi_spreadsheet\")\n",
     "sheet_people_organizations = \"ORGANIZATIONS\"\n",
     "sheet_people = \"PEOPLE\"\n",
     "\n",
@@ -500,6 +501,8 @@
     "                    print(\"- ⚠️ LinkedIn call:\", call_linkedin)\n",
     "                except Exception as e:\n",
     "                    print(e)\n",
+    "                    if e.response.status_code == 302:\n",
+    "                        call_linkedin = limit_linkedin\n",
     "                    tmp_df = pd.DataFrame()\n",
     "                \n",
     "            if len(tmp_df) > 0:                \n",
@@ -565,7 +568,7 @@
     "    db_organizations,\n",
     "    output_dir,\n",
     ")\n",
-    "df_organizations#.head(1)"
+    "df_organizations.head(1)"
    ]
   },
   {

diff --git a/models/growth-engine/core/domain/Google_Sheets_Update_people_db.ipynb b/models/growth-engine/core/domain/Google_Sheets_Update_people_db.ipynb
@@ -141,13 +141,14 @@
    "outputs": [],
    "source": [
     "# Inputs\n",
-    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"entity_dir\")\n",
+    "entity_index = \"0\"\n",
+    "entity_dir = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"entity_dir\")\n",
     "input_dir = os.path.join(entity_dir, \"growth-engine\", date.today().isoformat())\n",
     "file_interactions = \"interactions\"\n",
     "api_key = os.environ.get(\"NAAS_API_TOKEN\") or naas.secret.get('NAAS_API_TOKEN')\n",
     "li_at = os.environ.get(\"LINKEDIN_LI_AT\") or naas.secret.get(\"LINKEDIN_LI_AT\")\n",
     "JSESSIONID = os.environ.get(\"LINKEDIN_JSESSIONID\") or naas.secret.get(\"LINKEDIN_JSESSIONID\")\n",
-    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", \"0\"), \"abi_spreadsheet\")\n",
+    "spreadsheet_url = pload(os.path.join(naas_data_product.OUTPUTS_PATH, \"entities\", entity_index), \"abi_spreadsheet\")\n",
     "sheet_people = \"PEOPLE\"\n",
     "prompt_seniority = \"\"\"\n",
     "Find the seniority that matches the most with the occupation extracted from a LinkedIn profile, if there is no exact match, you WON'T return a sentence or ask for more information but stricly return \"Professional/Staff\".\n",
@@ -610,6 +611,9 @@
     "                except Exception as e:\n",
     "                    print(e)\n",
     "                    organization = \"ERROR_LINKEDIN_ENRICHMENT\"\n",
+    "                    if e.response.status_code == 302:\n",
+    "                        organization = \"TBD\"\n",
+    "                        call_linkedin = limit_linkedin\n",
     "                    tmp_df = pd.DataFrame()\n",
     "                call_linkedin += 1\n",
     "                print(\"- ⚠️ LinkedIn call:\", call_linkedin)\n",
@@ -631,7 +635,7 @@
     "    output_dir,\n",
     "    datalake_dir,\n",
     ")\n",
-    "df_people"
+    "df_people.head(1)"
    ]
   },
   {