diff --git a/buildscripts/hpat-conda-recipe/meta.yaml b/buildscripts/hpat-conda-recipe/meta.yaml index 2d7a1d6ee..1936a5507 100644 --- a/buildscripts/hpat-conda-recipe/meta.yaml +++ b/buildscripts/hpat-conda-recipe/meta.yaml @@ -42,7 +42,6 @@ requirements: test: requires: - h5py - - pyspark - scipy imports: - hpat diff --git a/hpat/tests/gen_test_data.py b/hpat/tests/gen_test_data.py index dde77aac1..32052714f 100644 --- a/hpat/tests/gen_test_data.py +++ b/hpat/tests/gen_test_data.py @@ -39,23 +39,23 @@ def gen_pq_test(cls): def generate_spark_data(): - from pyspark.sql import SparkSession - from pyspark.sql.types import ( - StructType, StructField, DateType, TimestampType) - # test datetime64, spark dates dt1 = pd.DatetimeIndex(['2017-03-03 03:23', '1990-10-23', '1993-07-02 10:33:01']) df = pd.DataFrame({'DT64': dt1, 'DATE': dt1.copy()}) df.to_parquet('pandas_dt.pq') - spark = SparkSession.builder.appName("GenSparkData").getOrCreate() - schema = StructType([StructField('DT64', DateType(), True), - StructField('DATE', TimestampType(), True)]) - sdf = spark.createDataFrame(df, schema) - sdf.write.parquet('sdf_dt.pq', 'overwrite') + import os + import shutil + import tarfile + + if os.path.exists('sdf_dt.pq'): + shutil.rmtree('sdf_dt.pq') - spark.stop() + sdf_dt_archive = os.path.join(os.path.dirname(os.path.abspath(__file__)),'sdf_dt.pq.bz2') + tar = tarfile.open(sdf_dt_archive, "r:bz2") + tar.extractall('.') + tar.close() def gen_lr(file_name, N, D): diff --git a/hpat/tests/sdf_dt.pq.bz2 b/hpat/tests/sdf_dt.pq.bz2 new file mode 100644 index 000000000..2e926035f Binary files /dev/null and b/hpat/tests/sdf_dt.pq.bz2 differ