update parallel scripts

mjhea0 · mjhea0 · commit cfcd2bad03a4 · 2021-12-22T09:23:10.000-06:00
diff --git a/script_parallel_1.py b/script_parallel_1.py
@@ -7,7 +7,7 @@
 from scrapers.scraper import connect_to_base, get_driver, parse_html, write_to_file
 
 
-def run_process(filename, headless):
+def run_process(number, filename, headless):
 
     # init browser
     browser = get_driver(headless)
@@ -39,9 +39,9 @@ def run_process(filename, headless):
     output_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
     output_filename = f"output_{output_timestamp}.csv"
 
-    # scrape and craw
+    # scrape and crawl
     with Pool(cpu_count() - 1) as p:
-        p.starmap(run_process, repeat(output_filename), repeat(headless)))
+        p.starmap(run_process, zip(range(1, 21), repeat(output_filename), repeat(headless)))
     p.close()
     p.join()
 
diff --git a/script_parallel_2.py b/script_parallel_2.py
@@ -6,7 +6,7 @@
 from scrapers.scraper import connect_to_base, get_driver, parse_html, write_to_file
 
 
-def run_process(filename, headless):
+def run_process(number, filename, headless):
 
     # init browser
     browser = get_driver(headless)
@@ -43,7 +43,7 @@ def run_process(filename, headless):
     with ProcessPoolExecutor() as executor:
         for number in range(1, 21):
             futures.append(
-                executor.submit(run_process, output_filename, headless)
+                executor.submit(run_process, number, output_filename, headless)
             )
 
     wait(futures)
