spark connect attempt 4

MrPowers · Feb 10, 2024 · a3ef4a2 · a3ef4a2
1 parent cab663e
commit a3ef4a2
Show file tree

Hide file tree

Showing 6 changed files with 891 additions and 871 deletions.
diff --git a/benchmarks/create_benchmark_df.py b/benchmarks/create_benchmark_df.py
@@ -45,6 +45,7 @@ def save_benchmark_df(
 
     builder = (
         SparkSession.builder.appName("MyApp")
+        .remote("sc://localhost")
         .config("spark.executor.memory", "20G")
         .config("spark.driver.memory", "25G")
         .config("spark.sql.shuffle.partitions", "2")

diff --git a/benchmarks/visualize_benchmarks.py b/benchmarks/visualize_benchmarks.py
@@ -154,6 +154,7 @@ def get_benchmark_date(benchmark_path: str) -> str:
 if __name__ == "__main__":
     spark = (
         SparkSession.builder.appName("MyApp")  # type: ignore # noqa: PGH003
+        .remote("sc://localhost")
         .config("spark.executor.memory", "10G")
         .config("spark.driver.memory", "25G")
         .config("spark.sql.shuffle.partitions", "2")