Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

PIG-2508: PIG can unpredictably ignore deprecated Hadoop config options

git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1344904 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
commit b701d904f1e4a1d71979613859ff92ece171c2dd 1 parent edba800
Dmitriy V. Ryaboy authored
View
2  CHANGES.txt
@@ -158,6 +158,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2508: PIG can unpredictably ignore deprecated Hadoop config options (thw via dvryaboy)
+
PIG-2691: Duplicate TOKENIZE schema (jay23jack via azaroth)
PIG-2173: piggybank datetime conversion javadocs not properly formatted (hluu via daijy)
View
12 contrib/zebra/build.xml
@@ -151,10 +151,14 @@ to call at top-level: ant deploy-contrib compile-core-test
classpath to use build/contrib/*.jar
-->
<sysproperty key="user.dir" value="${test.build.dir}/data"/>
-
- <sysproperty key="fs.default.name" value="${fs.default.name}"/>
- <sysproperty key="pig.test.localoutputfile" value="${pig.test.localoutputfile}"/>
- <sysproperty key="pig.log.dir" value="${pig.log.dir}"/>
+ <syspropertyset>
+ <propertyset>
+ <propertyref name="fs.default.name"/>
+ <propertyref name="pig.test.localoutputfile"/>
+ <propertyref name="pig.log.dir"/>
+ </propertyset>
+ </syspropertyset>
+
<classpath refid="test.classpath"/>
<formatter type="${test.junit.output.format}" />
View
44 src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
@@ -39,7 +39,6 @@
import org.apache.pig.PigException;
import org.apache.pig.backend.datastorage.DataStorage;
import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
@@ -120,13 +119,12 @@ public void init() throws ExecException {
}
@SuppressWarnings("deprecation")
- public void init(Properties properties) throws ExecException {
+ private void init(Properties properties) throws ExecException {
//First set the ssh socket factory
setSSHFactory();
String cluster = null;
String nameNode = null;
- Configuration configuration = null;
// We need to build a configuration object first in the manner described below
// and then get back a properties object to inspect the JOB_TRACKER_LOCATION
@@ -152,7 +150,7 @@ public void init(Properties properties) throws ExecException {
if( hadoop_site == null && core_site == null ) {
throw new ExecException("Cannot find hadoop configurations in classpath (neither hadoop-site.xml nor core-site.xml was found in the classpath)." +
- "If you plan to use local mode, please put -x local option in command line",
+ " If you plan to use local mode, please put -x local option in command line",
4010);
}
@@ -203,18 +201,15 @@ public void init(Properties properties) throws ExecException {
}
log.info("Connecting to hadoop file system at: " + (nameNode==null? LOCAL: nameNode) ) ;
+ // constructor sets DEFAULT_REPLICATION_FACTOR_KEY
ds = new HDataStorage(properties);
- // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties.
- configuration = ConfigurationUtil.toConfiguration(properties);
-
-
if(cluster != null && !cluster.equalsIgnoreCase(LOCAL)){
- log.info("Connecting to map-reduce job tracker at: " + properties.get(JOB_TRACKER_LOCATION));
+ log.info("Connecting to map-reduce job tracker at: " + jc.get(JOB_TRACKER_LOCATION));
}
// Set job-specific configuration knobs
- jobConf = new JobConf(configuration);
+ jobConf = jc;
}
public void updateConfiguration(Properties newConfiguration)
@@ -352,8 +347,8 @@ private void setSSHFactory(){
}
/**
- * Method to recompute pig properties by overriding hadoop properties
- * with pig properties
+ * Method to apply pig properties to JobConf
+ * (replaces properties with resulting jobConf values)
* @param conf JobConf with appropriate hadoop resource files
* @param properties Pig properties that will override hadoop properties; properties might be modified
*/
@@ -362,32 +357,23 @@ private void recomputeProperties(JobConf jobConf, Properties properties) {
// We need to load the properties from the hadoop configuration
// We want to override these with any existing properties we have.
if (jobConf != null && properties != null) {
- Properties hadoopProperties = new Properties();
- Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
- while (iter.hasNext()) {
- Map.Entry<String, String> entry = iter.next();
- hadoopProperties.put(entry.getKey(), entry.getValue());
- }
-
- //override hadoop properties with user defined properties
+ // set user properties on the jobConf to ensure that defaults
+ // and deprecation is applied correctly
Enumeration<Object> propertiesIter = properties.keys();
while (propertiesIter.hasMoreElements()) {
String key = (String) propertiesIter.nextElement();
String val = properties.getProperty(key);
-
// We do not put user.name, See PIG-1419
if (!key.equals("user.name"))
- hadoopProperties.put(key, val);
+ jobConf.set(key, val);
}
-
//clear user defined properties and re-populate
properties.clear();
- Enumeration<Object> hodPropertiesIter = hadoopProperties.keys();
- while (hodPropertiesIter.hasMoreElements()) {
- String key = (String) hodPropertiesIter.nextElement();
- String val = hadoopProperties.getProperty(key);
- properties.put(key, val);
- }
+ Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ properties.put(entry.getKey(), entry.getValue());
+ }
}
}
View
33 src/org/apache/pig/tools/grunt/GruntParser.java
@@ -34,6 +34,7 @@
import java.util.AbstractList;
import java.util.Arrays;
import java.util.Date;
+import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -46,6 +47,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -57,11 +59,10 @@
import org.apache.pig.backend.datastorage.DataStorage;
import org.apache.pig.backend.datastorage.DataStorageException;
import org.apache.pig.backend.datastorage.ElementDescriptor;
-import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine;
-import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.FileLocalizer.FetchFileRet;
@@ -561,7 +562,33 @@ else if (key.equals("default_parallel")) {
}
else
{
- mPigServer.getPigContext().getProperties().setProperty(key, value);
+ //mPigServer.getPigContext().getProperties().setProperty(key, value);
+ // PIG-2508 properties need to be managed through JobConf
+ // since all other code depends on access to properties,
+ // we need to re-populate from updated JobConf
+ //java.util.HashSet<?> keysBefore = new java.util.HashSet<Object>(mPigServer.getPigContext().getProperties().keySet());
+ // set current properties on jobConf
+ Properties properties = mPigServer.getPigContext().getProperties();
+ Configuration jobConf = mPigServer.getPigContext().getExecutionEngine().getJobConf();
+ Enumeration<Object> propertiesIter = properties.keys();
+ while (propertiesIter.hasMoreElements()) {
+ String pkey = (String) propertiesIter.nextElement();
+ String val = properties.getProperty(pkey);
+ // We do not put user.name, See PIG-1419
+ if (!pkey.equals("user.name"))
+ jobConf.set(pkey, val);
+ }
+ // set new value, JobConf will handle deprecation etc.
+ jobConf.set(key, value);
+ // re-initialize to reflect updated JobConf
+ properties.clear();
+ Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ //keysBefore.removeAll(mPigServer.getPigContext().getProperties().keySet());
+ //log.info("PIG-2508: keys dropped from properties: " + keysBefore);
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.