From 9e77fde6dcb26b91b76134ccde48b3adf152a56f Mon Sep 17 00:00:00 2001
From: Vyacheslav-Smirnov
 <51660067+Vyacheslav-Smirnov@users.noreply.github.com>
Date: Thu, 1 Aug 2019 16:19:21 +0300
Subject: [PATCH] Remove spark dependency (#102)

Remove spark dependency from HPA; use pre-generated sdf_dt.pq
---
 buildscripts/hpat-conda-recipe/meta.yaml |   1 -
 hpat/tests/gen_test_data.py              |  20 ++++++++++----------
 hpat/tests/sdf_dt.pq.bz2                 | Bin 0 -> 1176 bytes
 3 files changed, 10 insertions(+), 11 deletions(-)
 create mode 100644 hpat/tests/sdf_dt.pq.bz2

diff --git a/buildscripts/hpat-conda-recipe/meta.yaml b/buildscripts/hpat-conda-recipe/meta.yaml
index 2d7a1d6ee..1936a5507 100644
--- a/buildscripts/hpat-conda-recipe/meta.yaml
+++ b/buildscripts/hpat-conda-recipe/meta.yaml
@@ -42,7 +42,6 @@ requirements:
 test:
   requires:
     - h5py
-    - pyspark
     - scipy
   imports:
     - hpat
diff --git a/hpat/tests/gen_test_data.py b/hpat/tests/gen_test_data.py
index dde77aac1..32052714f 100644
--- a/hpat/tests/gen_test_data.py
+++ b/hpat/tests/gen_test_data.py
@@ -39,23 +39,23 @@ def gen_pq_test(cls):
 
 
 def generate_spark_data():
-    from pyspark.sql import SparkSession
-    from pyspark.sql.types import (
-        StructType, StructField, DateType, TimestampType)
-
     # test datetime64, spark dates
     dt1 = pd.DatetimeIndex(['2017-03-03 03:23',
                             '1990-10-23', '1993-07-02 10:33:01'])
     df = pd.DataFrame({'DT64': dt1, 'DATE': dt1.copy()})
     df.to_parquet('pandas_dt.pq')
 
-    spark = SparkSession.builder.appName("GenSparkData").getOrCreate()
-    schema = StructType([StructField('DT64', DateType(), True),
-                         StructField('DATE', TimestampType(), True)])
-    sdf = spark.createDataFrame(df, schema)
-    sdf.write.parquet('sdf_dt.pq', 'overwrite')
+    import os
+    import shutil
+    import tarfile
+
+    if os.path.exists('sdf_dt.pq'):
+      shutil.rmtree('sdf_dt.pq')
 
-    spark.stop()
+    sdf_dt_archive = os.path.join(os.path.dirname(os.path.abspath(__file__)),'sdf_dt.pq.bz2')
+    tar = tarfile.open(sdf_dt_archive, "r:bz2")
+    tar.extractall('.')
+    tar.close()
 
 
 def gen_lr(file_name, N, D):
diff --git a/hpat/tests/sdf_dt.pq.bz2 b/hpat/tests/sdf_dt.pq.bz2
new file mode 100644
index 0000000000000000000000000000000000000000..2e926035f8abd32391f73d2dcb8f653594ba68da
GIT binary patch
literal 1176
zcmV;J1ZVp~T4*^jL0KkKS&J4;Pyh$TfB*m2Pv>`M|MWg}UrYb*|GQAY04czL#2^5e
z07L~635ZYyz38Bn02BZMH~;_u000000000000001kZ=G100000000000000010di4
z00000000000000000uycM5O&Pnt*ywQ_6ZwfYFcx)YBtC14e)YKmarV&;|;WK{Xi}
zG$u@uw5R1gQRvjngVjG&X|&YM9;7`nMl{s*9#Gj#(SnTB{Ydtu=gy}F_1p(EGsFMg
zQW^5G@&w;*>Qg;S2{%F3O$TcEjxA03WRY2yHl>o-x`_$OlpQEg8fm41MA?MGVK)h+
zB;uh1ZnjDW;Rr=$!#GejteROiLK+xxLI*NqfHGvPf%FD8D??KY!oQOlH)T2|Y0L05
zwS#R+nBQ}8l%*t>k&==+GAnF1DWb<gN+q|*q0Po*{vXSNQJ@fDVZ1iz?wTapWXTN{
z24k=S87V>)vJ|??_=Zsf1Pl;<kkd3K{AosWEsUJDfrn_>3WBl*2pAyBrRN2y2Vn}?
zBLe{1U~Vw%ox3I+H#LV5#%wg+Yz0qjB-Yf>%x2QaE({JkL&kNj2-6BmPV}s-*q%sK
z<s}PDd{rx2m90wFrE5~P)|DwNGL*?F4g6BG;4w=kX+qObVm4TPtPGaR_CrMsg)-E*
z6^Voq;R*t<@<XV1PXV}Y77_K78$+;7a(IOWMGBNSrx$y=rE6N%RaGif<{ZOF#WT$F
zc6jPdlVtS|%_%qMHcr9H6bjCvwlE&}%sOM{92^Yl)qzt3_;<@rw}|nMv3Ycp0i5uk
zbd8i``6cL;N)UC&Y4ji4w0FL%oaq!WG$eBBwM8LG74}LSGCL&?@L0O1^)ET2Ir84P
z>`3a&k}bN1Ngs?dNVaasxUoWwi5aG88fMV$i!ZQXH;W5#6KG&reb#A9?M1RL7RXs*
zFDrG!TyPC*|2k*IN@J0vW|5EKrlFS=Zmf-ghC55IB_lCUszyY3OV0_DLs-Ej(ltgG
z+anRGF$#*8jRNs;kd_+7qg>NE0zMhln+8~Rhown3+~hrHjJ&uZ3<L$mCT9j^TtMlx
znJ|{88Q>f)C&14l*&X|sno#06{qz6Bh99@v-I4B_(lN;Lp7A;2@$DTk)-ESpP{veN
zMS%9hq<aUhI1gy{M{MXLFu-6q1|vpA!vn5mjp*|k%>(rrMg;ZESsG?J7#s2}Hb%KE
z83&Zr93w>2I7bQQF&c6|*`#9%6FIPG6p^?!O`@Z*xM&*YLmZ4lljJ!=P|9kJ5mS~e
zIfh}9MW>{Hnc^-N4LKDXvNR6iX{czJhBL}gbLmP_l&8u|g=BN2FE9+!E|H*Ro6~6;
zg~Ke2WkUjkWSeH`+0pbk>0SpVHsohDWNDM<C~rv2Un7`Q)-V+rq)^UE6GlmzMTR27
z+!ZDwbAxc^jd>n<9TTM^Z1l$2KD6zf!>DE+bM_AC#!g6o+QX{CiCBDPGeeFXa?j&Q
q7qL-3>O)hZE{3NkqRAGgB#%Tr5VmqcojNF*LheYX3K9`w$*KU0>-`}B

literal 0
HcmV?d00001