From 38a662f554ac20eda5898cabdc40c05230caae34 Mon Sep 17 00:00:00 2001 From: Nathaniel Eliot Date: Wed, 7 Sep 2011 12:42:13 -0500 Subject: [PATCH] Adding uploadLocalFileIfChanged to allow HBase & ESearch jobs to run in parallel --- .../infochimps/elasticsearch/pig/ElasticSearchStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java b/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java index c87f0f7..a2fac59 100644 --- a/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java +++ b/src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java @@ -256,10 +256,10 @@ private void elasticSearchSetup(String location, Job job) { Path hdfsConfigPath = new Path(ES_CONFIG_HDFS_PATH); Path hdfsPluginsPath = new Path(ES_PLUGINS_HDFS_PATH); - HadoopUtils.uploadLocalFile(new Path(LOCAL_SCHEME+esConfig), hdfsConfigPath, job.getConfiguration()); + HadoopUtils.uploadLocalFileIfChanged(new Path(LOCAL_SCHEME+esConfig), hdfsConfigPath, job.getConfiguration()); HadoopUtils.shipFileIfNotShipped(hdfsConfigPath, job.getConfiguration()); - HadoopUtils.uploadLocalFile(new Path(LOCAL_SCHEME+esPlugins), hdfsPluginsPath, job.getConfiguration()); + HadoopUtils.uploadLocalFileIfChanged(new Path(LOCAL_SCHEME+esPlugins), hdfsPluginsPath, job.getConfiguration()); HadoopUtils.shipArchiveIfNotShipped(hdfsPluginsPath, job.getConfiguration()); } catch (Exception e) {