Skip to content

Commit

Permalink
WHIRR-128. Fix DNS resolution for clients running within EC2. Contrib…
Browse files Browse the repository at this point in the history
…uted by Tibor Kiss.

git-svn-id: https://svn.apache.org/repos/asf/incubator/whirr/trunk@1032844 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
tomwhite committed Nov 9, 2010
1 parent a733c5d commit a98c293
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 7 deletions.
2 changes: 2 additions & 0 deletions CHANGES.txt
Expand Up @@ -8,6 +8,8 @@ Trunk (unreleased changes)

BUG FIXES

WHIRR-128. Fix DNS resolution for clients running within EC2.
(Tibor Kiss via tomwhite)

Release 0.2.0 - 2010-11-04

Expand Down
5 changes: 5 additions & 0 deletions pom.xml
Expand Up @@ -148,6 +148,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
<version>2.0.8</version>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down
4 changes: 4 additions & 0 deletions services/hadoop/pom.xml
Expand Up @@ -95,5 +95,9 @@
<version>0.20.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,60 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.whirr.service.hadoop;

import java.io.IOException;

import org.xbill.DNS.DClass;
import org.xbill.DNS.ExtendedResolver;
import org.xbill.DNS.Message;
import org.xbill.DNS.Name;
import org.xbill.DNS.Record;
import org.xbill.DNS.Resolver;
import org.xbill.DNS.ReverseMap;
import org.xbill.DNS.Section;
import org.xbill.DNS.Type;

public class DnsUtil {

/**
* resolve the reverse dns name for the given IP address
*
* @param hostIp
* @return
* @throws IOException
*/
public static String resolveAddress(String hostIp) throws IOException {
Resolver res = new ExtendedResolver();

Name name = ReverseMap.fromAddress(hostIp);
int type = Type.PTR;
int dclass = DClass.IN;
Record rec = Record.newRecord(name, type, dclass);
Message query = Message.newQuery(rec);
Message response = res.send(query);

Record[] answers = response.getSectionArray(Section.ANSWER);
if (answers.length == 0)
return hostIp;
else {
String revaddr = answers[0].rdataToString();
return revaddr.endsWith(".") ? revaddr.substring(0, revaddr.length() - 1) : revaddr;
}
}
}
Expand Up @@ -53,7 +53,7 @@ public String[] getProxyCommand() throws IOException {
Files.write(ByteStreams.toByteArray(clusterSpec.getPrivateKey().getInput()), identity);
}
String user = Iterables.get(cluster.getInstances(), 0).getLoginCredentials().identity;
String server = cluster.getNamenodePublicAddress().getHostName();
String server = DnsUtil.resolveAddress(cluster.getNamenodePublicAddress().getHostAddress());
return new String[] { "ssh",
"-i", identity.getAbsolutePath(),
"-o", "ConnectTimeout=10",
Expand Down
Expand Up @@ -147,8 +147,8 @@ public HadoopCluster launchCluster(ClusterSpec clusterSpec) throws IOException {
"sun/java/install",
String.format("%s dn,tt -n %s -j %s -c %s",
hadoopInstallRunUrl,
namenodePublicAddress.getHostName(),
jobtrackerPublicAddress.getHostName(),
DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
clusterSpec.getProvider())));

TemplateBuilder slaveTemplateBuilder = computeService.templateBuilder()
Expand Down Expand Up @@ -182,7 +182,7 @@ public HadoopCluster launchCluster(ClusterSpec clusterSpec) throws IOException {

LOG.info("Completed launch of {}", clusterSpec.getClusterName());
LOG.info("Web UI available at http://{}",
namenodePublicAddress.getHostName());
DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()));
Properties config = createClientSideProperties(namenodePublicAddress, jobtrackerPublicAddress);
createClientSideHadoopSiteFile(clusterSpec, config);
HadoopCluster cluster = new HadoopCluster(instances, config);
Expand All @@ -209,8 +209,8 @@ public Instance apply(NodeMetadata node) {
private Properties createClientSideProperties(InetAddress namenode, InetAddress jobtracker) throws IOException {
Properties config = new Properties();
config.setProperty("hadoop.job.ugi", "root,root");
config.setProperty("fs.default.name", String.format("hdfs://%s:8020/", namenode.getHostName()));
config.setProperty("mapred.job.tracker", String.format("%s:8021", jobtracker.getHostName()));
config.setProperty("fs.default.name", String.format("hdfs://%s:8020/", DnsUtil.resolveAddress(namenode.getHostAddress())));
config.setProperty("mapred.job.tracker", String.format("%s:8021", DnsUtil.resolveAddress(jobtracker.getHostAddress())));
config.setProperty("hadoop.socks.server", "localhost:6666");
config.setProperty("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.net.SocksSocketFactory");
return config;
Expand Down Expand Up @@ -258,7 +258,7 @@ private void createProxyScript(ClusterSpec clusterSpec, HadoopCluster cluster) {
HadoopProxy proxy = new HadoopProxy(clusterSpec, cluster);
String script = String.format("echo 'Running proxy to Hadoop cluster at %s. " +
"Use Ctrl-c to quit.'\n",
cluster.getNamenodePublicAddress().getHostName())
DnsUtil.resolveAddress(cluster.getNamenodePublicAddress().getHostAddress()))
+ Joiner.on(" ").join(proxy.getProxyCommand());
Files.write(script, hadoopProxyFile, Charsets.UTF_8);
LOG.info("Wrote Hadoop proxy script {}", hadoopProxyFile);
Expand Down
@@ -0,0 +1,84 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.whirr.service.hadoop;

import static org.junit.Assert.*;
import static java.lang.System.out;

import java.io.IOException;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.UnknownHostException;
import java.util.Collections;
import java.util.Enumeration;

import org.apache.whirr.service.hadoop.DnsUtil;
import org.junit.Test;
import org.xbill.DNS.Address;

public class DnsUtilTest {

@Test
public void testResolveAddress() throws IOException {
// test it with all interfaces
Enumeration<NetworkInterface> en = NetworkInterface.getNetworkInterfaces();
while (en.hasMoreElements()) {
NetworkInterface netint = (NetworkInterface) en.nextElement();
Enumeration<InetAddress> inetAddresses = netint.getInetAddresses();
for (InetAddress inetAddress : Collections.list(inetAddresses)) {
if (inetAddress instanceof Inet4Address) {
long start = System.currentTimeMillis();
String reverse = DnsUtil.resolveAddress(inetAddress.getHostAddress());
long end = System.currentTimeMillis();
// we know that java.net.InetAddress's getHostName takes > 4.5s if
// there is no reverse address assigned to it
// but DnsUtil can resolve any address without this delaying problem.
assertTrue("DnsUtil.resolveAddress takes " + (end - start)
+ " millis, it should be shorter than a second",
end - start < 1000);
if (inetAddress.toString().substring(1).equals(reverse)) {
out.printf(
"InetAddress %s on interface %s does not have reverse dns name, so their reverse remains: %s\n",
inetAddress, netint.getDisplayName(), reverse);
} else {
if (inetAddress.isLoopbackAddress()) {
out.printf(
"InetAddress %s on loopback interface %s obtained reverse name as %s\n",
inetAddress, netint.getDisplayName(), reverse);
} else {
out.printf(
"InetAddress %s on interface %s has reverse dns name: %s\n",
inetAddress, netint.getDisplayName(), reverse);
try {
InetAddress checkedAddr = Address.getByName(reverse);
assertEquals(inetAddress, checkedAddr);
} catch (UnknownHostException uhex) {
fail("InetAddress " + inetAddress + " on interface "
+ netint.getDisplayName() + " got " + reverse
+ " reverse dns name which in return is an unknown host!");
}
}
}
}
}
}
}

}

0 comments on commit a98c293

Please sign in to comment.