add test verifying fix #64

elastic · Jul 31, 2013 · f140131 · f140131
1 parent 37ff297
commit f140131
Show file tree

Hide file tree

Showing 8 changed files with 3,910 additions and 9 deletions.
diff --git a/cdh4.deps b/cdh4.deps
diff --git a/deps.out b/deps.out
diff --git a/hive-deps.out b/hive-deps.out
diff --git a/pig-deps.out b/pig-deps.out
diff --git a/src/test/java/org/elasticsearch/hadoop/integration/hive/HiveSaveTest.java b/src/test/java/org/elasticsearch/hadoop/integration/hive/HiveSaveTest.java
@@ -122,4 +122,43 @@ public void testCompoundSave() throws Exception {
         System.out.println(server.execute(ddl));
         System.out.println(server.execute(insert));
     }
+
+    @Test
+    public void testTimestampSave() throws Exception {
+        String localTable = "CREATE TABLE timestampsource ("
+                + "id       BIGINT, "
+                + "name     STRING, "
+                + "url      STRING, "
+                + "picture  STRING) "
+                + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'"
+                + "LOCATION '/tmp/hive/warehouse/timestampsource/' ";
+
+        String load = "LOAD DATA INPATH '" + HiveSuite.hdfsResource + "' OVERWRITE INTO TABLE timestampsource";
+
+        // create external table
+        String ddl =
+                "CREATE EXTERNAL TABLE artiststimestampsave ("
+                + "id       BIGINT, "
+                + "date     TIMESTAMP, "
+                + "name     STRING, "
+                + "links    STRUCT<url:STRING, picture:STRING>) "
+                + "STORED BY 'org.elasticsearch.hadoop.hive.ESStorageHandler' "
+                + "WITH SERDEPROPERTIES ('serder.foo' = 'serder.bar') "
+                + "TBLPROPERTIES('es.resource' = 'hive/artiststimestamp') ";
+
+        String currentDate = "SELECT *, from_unixtime(unix_timestamp()) from timestampsource";
+
+        // since the date format is different in Hive vs ISO8601/Joda, save only the date (which is the same) as a string
+        // we do this since unix_timestamp() saves the date as a long (in seconds) and w/o mapping the date is not recognized as data
+        String insert =
+                "INSERT OVERWRITE TABLE artiststimestampsave "
+                + "SELECT NULL, to_date(from_unixtime(unix_timestamp())), s.name, named_struct('url', s.url, 'picture', s.picture) FROM timestampsource s";
+
+        //System.out.println(server.execute(jar));
+        System.out.println(server.execute(ddl));
+        System.out.println(server.execute(localTable));
+        System.out.println(server.execute(load));
+        System.out.println(server.execute(currentDate));
+        System.out.println(server.execute(insert));
+    }
 }
diff --git a/src/test/java/org/elasticsearch/hadoop/integration/hive/HiveSearchTest.java b/src/test/java/org/elasticsearch/hadoop/integration/hive/HiveSearchTest.java
@@ -15,20 +15,13 @@
  */
 package org.elasticsearch.hadoop.integration.hive;
 
-import org.junit.Before;
 import org.junit.Test;
 
 import static org.elasticsearch.hadoop.integration.hive.HiveSuite.*;
 
 public class HiveSearchTest {
 
-    @Before
-    public void cleanConfig() {
-        System.out.println("Refreshing Hive Config...");
-        //server.refreshConfig();
-    }
-
-    @Test
+    //@Test
     public void basicLoad() throws Exception {
 
         String create = "CREATE EXTERNAL TABLE artistsload ("
@@ -73,4 +66,37 @@ public void basicArrayMapping() throws Exception {
         System.out.println(server.execute(create));
         System.out.println(server.execute(select));
     }
-}
+
+    //@Test
+    public void basicTimestampLoad() throws Exception {
+        String create = "CREATE EXTERNAL TABLE timestampload ("
+                + "id       BIGINT, "
+                + "date     TIMESTAMP, "
+                + "name     STRING, "
+                + "links    STRUCT<url:STRING, picture:STRING>) "
+                + "STORED BY 'org.elasticsearch.hadoop.hive.ESStorageHandler' "
+                + "TBLPROPERTIES('es.resource' = 'hive/artiststimestamp/_search?q=*') ";
+
+        String select = "SELECT date FROM timestampload";
+        String select2 = "SELECT unix_timestamp(), date FROM timestampload";
+
+        System.out.println(server.execute(create));
+        System.out.println(server.execute(select));
+        System.out.println(server.execute(select2));
+    }
+
+    @Test
+    public void javaMethodInvocation() throws Exception {
+        String create = "CREATE EXTERNAL TABLE methodInvocation ("
+                + "id       BIGINT, "
+                + "name     STRING, "
+                + "links    STRUCT<url:STRING, picture:STRING>) "
+                + "STORED BY 'org.elasticsearch.hadoop.hive.ESStorageHandler' "
+                + "TBLPROPERTIES('es.resource' = 'hive/artists/_search?q=*') ";
+
+        String select = "SELECT java_method(\"java.lang.System\", \"currentTimeMillis\") FROM methodInvocation LIMIT 1";
+
+        System.out.println(server.execute(create));
+        System.out.println(server.execute(select));
+    }
+}
diff --git a/src/test/resources/test.testing b/src/test/resources/test.testing
@@ -0,0 +1,32 @@
+# default settings for Hadoop test
+
+# ES
+es.host=192.168.1.50
+es.batch.size.bytes=8kb
+es.port=9500
+
+# M&R
+fs.default.name=file:///
+mapred.job.tracker=local
+
+# HDP
+#fs.default.name=hdfs://sandbox:8020
+#mapred.job.tracker=sandbox:50300
+
+# CDH
+#fs.default.name=hdfs://192.168.1.131:8020
+#mapred.job.tracker=192.168.1.131:8021
+
+# MapR
+#fs.maprfs.impl=com.mapr.fs.MapRFileSystem
+#fs.default.name=maprfs://mapr:7222
+#mapred.job.tracker=mapr:9001
+
+# Intel
+#fs.default.name=hdfs://idhvm:8020
+#mapred.job.tracker=idhvm:54311
+
+#Hive - only works if the HDFS is setup as well
+hive=local
+#hive=jdbc:hive2://hdp:10000/
+#hive=jdbc:hive://idhvm:10000/