From 60e3f3d63ecb33abcc5fc854cd3768b9dcf82caf Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 10 Jan 2015 04:14:58 +0900 Subject: [PATCH] TAJO-1296: Remove obsolete classes from tajo.master.container package. --- .../master/container/TajoContainerId.java | 3 +- .../{impl/pb => }/TajoContainerIdPBImpl.java | 2 +- .../master/container/TajoConverterUtils.java | 176 ------------------ .../master/container/TajoRecordFactory.java | 31 --- .../container/TajoRecordFactoryPBImpl.java | 104 ----------- .../container/TajoRecordFactoryProvider.java | 70 ------- .../tajo/master/container/TajoRecords.java | 39 ---- .../org/apache/tajo/worker/TaskRunner.java | 2 +- 8 files changed, 3 insertions(+), 424 deletions(-) rename tajo-core/src/main/java/org/apache/tajo/master/container/{impl/pb => }/TajoContainerIdPBImpl.java (98%) delete mode 100644 tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java delete mode 100644 tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java delete mode 100644 tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java delete mode 100644 tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java index 0de5fe0cda..7bc27c662c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.util.Records; /** * This class is borrowed from the following source code : @@ -43,7 +42,7 @@ public abstract class TajoContainerId implements Comparable{ @Unstable public static TajoContainerId newInstance(ApplicationAttemptId appAttemptId, int containerId) { - TajoContainerId id = TajoRecords.newRecord(TajoContainerId.class); + TajoContainerId id = new TajoContainerIdPBImpl(); id.setId(containerId); id.setApplicationAttemptId(appAttemptId); id.build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java rename to tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java index 9d31050a69..cf9e012add 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.master.container.impl.pb; +package org.apache.tajo.master.container; import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java index a6db6548c5..88c4823353 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java @@ -53,86 +53,7 @@ @Private public class TajoConverterUtils { - public static final String APPLICATION_PREFIX = "application"; public static final String CONTAINER_PREFIX = "container"; - public static final String APPLICATION_ATTEMPT_PREFIX = "appattempt"; - - /** - * return a hadoop path from a given url - * - * @param url - * url to convert - * @return path from {@link URL} - * @throws URISyntaxException - */ - public static Path getPathFromYarnURL(URL url) throws URISyntaxException { - String scheme = url.getScheme() == null ? "" : url.getScheme(); - - String authority = ""; - if (url.getHost() != null) { - authority = url.getHost(); - if (url.getUserInfo() != null) { - authority = url.getUserInfo() + "@" + authority; - } - if (url.getPort() > 0) { - authority += ":" + url.getPort(); - } - } - - return new Path( - (new URI(scheme, authority, url.getFile(), null, null)).normalize()); - } - - /** - * change from CharSequence to string for map key and value - * @param env map for converting - * @return string,string map - */ - public static Map convertToString( - Map env) { - - Map stringMap = new HashMap(); - for (Entry entry: env.entrySet()) { - stringMap.put(entry.getKey().toString(), entry.getValue().toString()); - } - return stringMap; - } - - public static URL getYarnUrlFromPath(Path path) { - return getYarnUrlFromURI(path.toUri()); - } - - public static URL getYarnUrlFromURI(URI uri) { - URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class); - if (uri.getHost() != null) { - url.setHost(uri.getHost()); - } - if (uri.getUserInfo() != null) { - url.setUserInfo(uri.getUserInfo()); - } - url.setPort(uri.getPort()); - url.setScheme(uri.getScheme()); - url.setFile(uri.getPath()); - return url; - } - - public static String toString(ApplicationId appId) { - return appId.toString(); - } - - public static ApplicationId toApplicationId(RecordFactory recordFactory, - String appIdStr) { - Iterator it = _split(appIdStr).iterator(); - it.next(); // prefix. TODO: Validate application prefix - return toApplicationId(recordFactory, it); - } - - private static ApplicationId toApplicationId(RecordFactory recordFactory, - Iterator it) { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - return appId; - } private static ApplicationAttemptId toApplicationAttemptId( Iterator it) throws NumberFormatException { @@ -143,32 +64,10 @@ private static ApplicationAttemptId toApplicationAttemptId( return appAttemptId; } - private static ApplicationId toApplicationId( - Iterator it) throws NumberFormatException { - ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()), - Integer.parseInt(it.next())); - return appId; - } - public static String toString(TajoContainerId cId) { return cId == null ? null : cId.toString(); } - public static NodeId toNodeId(String nodeIdStr) { - String[] parts = nodeIdStr.split(":"); - if (parts.length != 2) { - throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr - + "]. Expected host:port"); - } - try { - NodeId nodeId = - NodeId.newInstance(parts[0], Integer.parseInt(parts[1])); - return nodeId; - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid port: " + parts[1], e); - } - } - public static TajoContainerId toTajoContainerId(String containerIdStr) { Iterator it = _split(containerIdStr).iterator(); if (!it.next().equals(CONTAINER_PREFIX)) { @@ -185,79 +84,4 @@ public static TajoContainerId toTajoContainerId(String containerIdStr) { + containerIdStr, n); } } - - public static ApplicationAttemptId toApplicationAttemptId( - String applicationAttmeptIdStr) { - Iterator it = _split(applicationAttmeptIdStr).iterator(); - if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) { - throw new IllegalArgumentException("Invalid AppAttemptId prefix: " - + applicationAttmeptIdStr); - } - try { - return toApplicationAttemptId(it); - } catch (NumberFormatException n) { - throw new IllegalArgumentException("Invalid AppAttemptId: " - + applicationAttmeptIdStr, n); - } - } - - public static ApplicationId toApplicationId( - String appIdStr) { - Iterator it = _split(appIdStr).iterator(); - if (!it.next().equals(APPLICATION_PREFIX)) { - throw new IllegalArgumentException("Invalid ApplicationId prefix: " - + appIdStr + ". The valid ApplicationId should start with prefix " - + APPLICATION_PREFIX); - } - try { - return toApplicationId(it); - } catch (NumberFormatException n) { - throw new IllegalArgumentException("Invalid AppAttemptId: " - + appIdStr, n); - } - } - - /** - * Convert a protobuf token into a rpc token and set its service. Supposed - * to be used for tokens other than RMDelegationToken. For - * RMDelegationToken, use - * {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token, - * org.apache.hadoop.io.Text)} instead. - * - * @param protoToken the yarn token - * @param serviceAddr the connect address for the service - * @return rpc token - */ - public static Token convertFromYarn( - org.apache.hadoop.yarn.api.records.Token protoToken, - InetSocketAddress serviceAddr) { - Token token = new Token(protoToken.getIdentifier().array(), - protoToken.getPassword().array(), - new Text(protoToken.getKind()), - new Text(protoToken.getService())); - if (serviceAddr != null) { - SecurityUtil.setTokenService(token, serviceAddr); - } - return token; - } - - /** - * Convert a protobuf token into a rpc token and set its service. - * - * @param protoToken the yarn token - * @param service the service for the token - */ - public static Token convertFromYarn( - org.apache.hadoop.yarn.api.records.Token protoToken, - Text service) { - Token token = new Token(protoToken.getIdentifier().array(), - protoToken.getPassword().array(), - new Text(protoToken.getKind()), - new Text(protoToken.getService())); - - if (service != null) { - token.setService(service); - } - return token; - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java deleted file mode 100644 index 2fd8697643..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.master.container; - -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * This class is borrowed from the following source code : - * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java - * - */ -@Unstable -public interface TajoRecordFactory { - public T newRecordInstance(Class clazz); -} diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java deleted file mode 100644 index c352a2843c..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.tajo.master.container; - - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.tajo.master.container.TajoRecordFactory; - -/** - * This class is borrowed from the following source code : - * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RecordFactoryPBImpl.java - */ -@Private -public class TajoRecordFactoryPBImpl implements TajoRecordFactory { - - private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb"; - private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl"; - - private static final TajoRecordFactoryPBImpl self = new TajoRecordFactoryPBImpl(); - private Configuration localConf = new Configuration(); - private ConcurrentMap, Constructor> cache = new ConcurrentHashMap, Constructor>(); - - private TajoRecordFactoryPBImpl() { - } - - public static TajoRecordFactory get() { - return self; - } - - @SuppressWarnings("unchecked") - @Override - public T newRecordInstance(Class clazz) { - - Constructor constructor = cache.get(clazz); - if (constructor == null) { - Class pbClazz = null; - try { - pbClazz = localConf.getClassByName(getPBImplClassName(clazz)); - } catch (ClassNotFoundException e) { - throw new YarnRuntimeException("Failed to load class: [" - + getPBImplClassName(clazz) + "]", e); - } - try { - constructor = pbClazz.getConstructor(); - constructor.setAccessible(true); - cache.putIfAbsent(clazz, constructor); - } catch (NoSuchMethodException e) { - throw new YarnRuntimeException("Could not find 0 argument constructor", e); - } - } - try { - Object retObject = constructor.newInstance(); - return (T)retObject; - } catch (InvocationTargetException e) { - throw new YarnRuntimeException(e); - } catch (IllegalAccessException e) { - throw new YarnRuntimeException(e); - } catch (InstantiationException e) { - throw new YarnRuntimeException(e); - } - } - - private String getPBImplClassName(Class clazz) { - String srcPackagePart = getPackageName(clazz); - String srcClassName = getClassName(clazz); - String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX; - String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX; - return destPackagePart + "." + destClassPart; - } - - private String getClassName(Class clazz) { - String fqName = clazz.getName(); - return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length())); - } - - private String getPackageName(Class clazz) { - return clazz.getPackage().getName(); - } -} diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java deleted file mode 100644 index c260e8595d..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.master.container; - - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; - -/** - * This class is borrowed from the following source code : - * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java - */ -@Unstable -public class TajoRecordFactoryProvider { - private static Configuration defaultConf; - - static { - defaultConf = new Configuration(); - } - - private TajoRecordFactoryProvider() { - } - - public static TajoRecordFactory getRecordFactory(Configuration conf) { - if (conf == null) { - //Assuming the default configuration has the correct factories set. - //Users can specify a particular factory by providing a configuration. - conf = defaultConf; - } - return (TajoRecordFactory) getFactoryClassInstance(TajoRecordFactoryPBImpl.class.getCanonicalName()); - } - - private static Object getFactoryClassInstance(String factoryClassName) { - try { - Class clazz = Class.forName(factoryClassName); - Method method = clazz.getMethod("get", null); - method.setAccessible(true); - return method.invoke(null, null); - } catch (ClassNotFoundException e) { - throw new YarnRuntimeException(e); - } catch (NoSuchMethodException e) { - throw new YarnRuntimeException(e); - } catch (InvocationTargetException e) { - throw new YarnRuntimeException(e); - } catch (IllegalAccessException e) { - throw new YarnRuntimeException(e); - } - } -} diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java deleted file mode 100644 index e85edf9895..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.master.container; - - -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -/** - * This class is borrowed from the following source code : - * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/util/Records.java - * - * Convenient API record utils - */ -@Unstable -public class TajoRecords { - // The default record factory - private static final TajoRecordFactory factory = - TajoRecordFactoryProvider.getRecordFactory(null); - - public static T newRecord(Class cls) { - return factory.newRecordInstance(cls); - } -} diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java index f0da0cde01..23efffa847 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java @@ -31,7 +31,7 @@ import org.apache.tajo.engine.query.TaskRequestImpl; import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService; import org.apache.tajo.master.container.TajoContainerId; -import org.apache.tajo.master.container.impl.pb.TajoContainerIdPBImpl; +import org.apache.tajo.master.container.TajoContainerIdPBImpl; import org.apache.tajo.master.container.TajoConverterUtils; import org.apache.tajo.rpc.CallFuture; import org.apache.tajo.rpc.NullCallback;