Adding uploadLocalFileIfChanged to allow HBase & ESearch jobs to run …

…in parallel
infochimps · Sep 7, 2011 · 38a662f · 38a662f
1 parent 224926a
commit 38a662f
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java b/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java
@@ -256,10 +256,10 @@ private void elasticSearchSetup(String location, Job job) {
                     Path hdfsConfigPath = new Path(ES_CONFIG_HDFS_PATH);
                     Path hdfsPluginsPath = new Path(ES_PLUGINS_HDFS_PATH);
 
-                    HadoopUtils.uploadLocalFile(new Path(LOCAL_SCHEME+esConfig), hdfsConfigPath, job.getConfiguration());
+                    HadoopUtils.uploadLocalFileIfChanged(new Path(LOCAL_SCHEME+esConfig), hdfsConfigPath, job.getConfiguration());
                     HadoopUtils.shipFileIfNotShipped(hdfsConfigPath, job.getConfiguration());
 
-                    HadoopUtils.uploadLocalFile(new Path(LOCAL_SCHEME+esPlugins), hdfsPluginsPath, job.getConfiguration());
+                    HadoopUtils.uploadLocalFileIfChanged(new Path(LOCAL_SCHEME+esPlugins), hdfsPluginsPath, job.getConfiguration());
                     HadoopUtils.shipArchiveIfNotShipped(hdfsPluginsPath, job.getConfiguration());
 
                 } catch (Exception e) {