Remove spark dependency (IntelPython#102)

Remove spark dependency from HPA; use pre-generated sdf_dt.pq
kozlov-alexey · Aug 7, 2019 · 9e77fde · 9e77fde
1 parent 9313df0
commit 9e77fde
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 11 deletions.
diff --git a/buildscripts/hpat-conda-recipe/meta.yaml b/buildscripts/hpat-conda-recipe/meta.yaml
@@ -42,7 +42,6 @@ requirements:
 test:
   requires:
     - h5py
-    - pyspark
     - scipy
   imports:
     - hpat

diff --git a/hpat/tests/gen_test_data.py b/hpat/tests/gen_test_data.py
@@ -39,23 +39,23 @@ def gen_pq_test(cls):
 
 
 def generate_spark_data():
-    from pyspark.sql import SparkSession
-    from pyspark.sql.types import (
-        StructType, StructField, DateType, TimestampType)
-
     # test datetime64, spark dates
     dt1 = pd.DatetimeIndex(['2017-03-03 03:23',
                             '1990-10-23', '1993-07-02 10:33:01'])
     df = pd.DataFrame({'DT64': dt1, 'DATE': dt1.copy()})
     df.to_parquet('pandas_dt.pq')
 
-    spark = SparkSession.builder.appName("GenSparkData").getOrCreate()
-    schema = StructType([StructField('DT64', DateType(), True),
-                         StructField('DATE', TimestampType(), True)])
-    sdf = spark.createDataFrame(df, schema)
-    sdf.write.parquet('sdf_dt.pq', 'overwrite')
+    import os
+    import shutil
+    import tarfile
+
+    if os.path.exists('sdf_dt.pq'):
+      shutil.rmtree('sdf_dt.pq')
 
-    spark.stop()
+    sdf_dt_archive = os.path.join(os.path.dirname(os.path.abspath(__file__)),'sdf_dt.pq.bz2')
+    tar = tarfile.open(sdf_dt_archive, "r:bz2")
+    tar.extractall('.')
+    tar.close()
 
 
 def gen_lr(file_name, N, D):

diff --git a/hpat/tests/sdf_dt.pq.bz2 b/hpat/tests/sdf_dt.pq.bz2