Added the AutoConfigured client

client_auto.go

@@ -2,6 +2,7 @@ package hdfs
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -25,7 +26,16 @@ type NameNode struct {
 	Port int
 }
 
-// Get Hadoop Properties
+type nameNodeError struct {
+	Message string
+}
+
+func (e *nameNodeError) Error() string {
+	return fmt.Sprintf("%s", e.Message)
+}
+
+// Get Hadoop Properties - try to open a conf file, marshal the results
+// into a Result object and return the Properties of that object.
 func GetHadoopProperties(path string) ([]Property, error) {
 	result := Result{}
 	f, err := ioutil.ReadFile(path)
@@ -50,8 +60,7 @@ func GetNamenodesFromHDFSConfig(path string) []string {
 	var nns []string
 	for _, prop := range props {
 		if strings.HasPrefix(prop.Name, "dfs.namenode.rpc-address") {
-			nnUrl, _ := url.Parse(prop.Value)
-			nns = append(nns, nnUrl.Host)
+			nns = append(nns, prop.Value)
 		}
 	}
 	return nns
@@ -74,6 +83,7 @@ func GetNamenodesFromSiteConfig(path string) []string {
 }
 
 // AutoConfigClient to create a client by trying to read the hadoop config
+// and returning the first if no namenodes are found look for HADOOP_NAMENODE env var
 func GetAutoConfigClient() (*Client, error) {
 	hadoopHome := os.Getenv("HADOOP_HOME")
 	hadoopConfDir := os.Getenv("HADOOP_CONF_DIR")
@@ -99,7 +109,15 @@ func GetAutoConfigClient() (*Client, error) {
 		nameNodes = append(nameNodes, GetNamenodesFromSiteConfig(confPath)...)
 	}
 
-	address := nameNodes[0]
+	var address string
+	if len(nameNodes) > 0 {
+		address = nameNodes[0]
+	} else if os.Getenv("HADOOP_NAMENODE") != "" {
+		address = os.Getenv("HADOOP_NAMENODE")
+	} else {
+		return nil, &nameNodeError{"Could not determine namenode address."}
+	}
+
 	username, err := Username()
 	if err != nil {
 		return nil, err

client_auto_test.go

@@ -0,0 +1,33 @@
+package hdfs
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAutoClientEnvVar(t *testing.T) {
+	_, err := AutoConfigClient()
+	assert.Nil(t, err)
+}
+
+func TestAutoClientHadoopHome(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_HOME", strings.Join([]string{pwd, "test"}, "/"))
+	_, err := AutoConfigClient()
+	assert.NotNil(t, err)
+	assert.EqualValues(t, "dial tcp: lookup hadoop-namenode-01: no such host", fmt.Sprintf("%s", err))
+	os.Setenv("HADOOP_HOME", "")
+}
+
+func TestAutoClientHadoopConfDir(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_CONF_DIR", strings.Join([]string{pwd, "test"}, "/"))
+	_, err := AutoConfigClient()
+	assert.NotNil(t, err)
+	assert.EqualValues(t, "dial tcp: lookup testnode: no such host", fmt.Sprintf("%s", err))
+	os.Setenv("HADOOP_CONF_DIR", "")
+}

test/conf/hdfs-site.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" ?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration>
+    <property>
+        <name>dfs.nameservices</name>
+        <value>tests</value>
+    </property>
+    <property>
+       <name>dfs.ha.automatic-failover.enabled</name>
+       <value>true</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn1</name>
+        <value>hadoop-namenode-01:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn2</name>
+        <value>hadoop-namenode-02:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn1</name>
+        <value>hadoop-namenode-01:50070</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn2</name>
+        <value>hadoop-namenode-02:50070</value>
+    </property>
+</configuration>

test/core-site.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+      <name>fs.defaultFS</name>
+      <value>hdfs://testnode:9000</value>
+    </property>
+</configuration>