Crozzy auto config

client.go

@@ -31,16 +31,37 @@ func Username() (string, error) {
 }
 
 // New returns a connected Client, or an error if it can't connect. The user
-// will be the user the code is running under.
+// will be the user the code is running under. If address is an empty string
+// it will try and get the namenode address from the hadoop configuration
+// files.
 func New(address string) (*Client, error) {
 	username, err := Username()
 	if err != nil {
 		return nil, err
 	}
 
+	if address == "" {
+		var nnErr error
+		address, nnErr = getNameNodeFromConf()
+		if nnErr != nil {
+			return nil, nnErr
+		}
+	}
+
 	return NewForUser(address, username)
 }
 
+// getNameNodeFromConf return a datanode from the system hadoop configuration
+func getNameNodeFromConf() (string, error) {
+	hadoopConf := LoadHadoopConf("")
+
+	namenodes, nnErr := hadoopConf.Namenodes()
+	if nnErr != nil {
+		return "", nnErr
+	}
+	return namenodes[0], nil
+}
+
 // NewForUser returns a connected Client with the user specified, or an error if
 // it can't connect.
 func NewForUser(address string, user string) (*Client, error) {

conf.go

@@ -0,0 +1,98 @@
+package hdfs
+
+import (
+	"encoding/xml"
+	"errors"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Property is the struct representation of hadoop configuration
+// key value pair.
+type Property struct {
+	Name  string `xml:"name"`
+	Value string `xml:"value"`
+}
+
+type propertyList struct {
+	Property []Property `xml:"property"`
+}
+
+// HadoopConf represents a map of all the key value configutation
+// pairs found in a user's hadoop configuration files.
+type HadoopConf map[string]string
+
+var errUnresolvedNamenode = errors.New("no namenode address in configuration")
+
+// LoadHadoopConf returns a HadoopConf object that is key value map
+// of all the hadoop conf properties, swallows errors reading xml
+// and reading a non-existant file.
+func LoadHadoopConf(inputPath string) HadoopConf {
+	var tryPaths []string
+
+	if inputPath != "" {
+		tryPaths = append(tryPaths, inputPath)
+	} else {
+		hadoopConfDir := os.Getenv("HADOOP_CONF_DIR")
+		hadoopHome := os.Getenv("HADOOP_HOME")
+		if hadoopConfDir != "" {
+			confHdfsPath := filepath.Join(hadoopConfDir, "hdfs-site.xml")
+			confCorePath := filepath.Join(hadoopConfDir, "core-site.xml")
+			tryPaths = append(tryPaths, confHdfsPath, confCorePath)
+		}
+		if hadoopHome != "" {
+			hdfsPath := filepath.Join(hadoopHome, "conf", "hdfs-site.xml")
+			corePath := filepath.Join(hadoopHome, "conf", "core-site.xml")
+			tryPaths = append(tryPaths, hdfsPath, corePath)
+		}
+	}
+	hadoopConf := make(HadoopConf)
+
+	for _, tryPath := range tryPaths {
+		pList := propertyList{}
+		f, err := ioutil.ReadFile(tryPath)
+		if err != nil {
+			continue
+		}
+
+		xmlErr := xml.Unmarshal(f, &pList)
+		if xmlErr != nil {
+			continue
+		}
+
+		for _, prop := range pList.Property {
+			hadoopConf[prop.Name] = prop.Value
+		}
+	}
+	return hadoopConf
+}
+
+// Namenodes returns a slice of deduplicated namenodes named in
+// a user's hadoop configuration files or an error is there are no namenodes.
+func (conf HadoopConf) Namenodes() ([]string, error) {
+	nns := make(map[string]bool)
+	for key, value := range conf {
+		if strings.Contains(key, "fs.defaultFS") {
+			nnUrl, _ := url.Parse(value)
+			nns[nnUrl.Host] = true
+		}
+		if strings.HasPrefix(key, "dfs.namenode.rpc-address") {
+			nns[value] = true
+		}
+	}
+	if len(nns) == 0 {
+		return nil, errUnresolvedNamenode
+	}
+
+	keys := make([]string, len(nns))
+
+	i := 0
+	for k, _ := range nns {
+		keys[i] = k
+		i++
+	}
+	return keys, nil
+}

conf_test.go

@@ -0,0 +1,48 @@
+package hdfs
+
+import (
+	"os"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfNamenodeHadoopHome(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_HOME", strings.Join([]string{pwd, "test"}, "/"))
+	hdConf := LoadHadoopConf("")
+	nns, err := hdConf.Namenodes()
+	assert.Nil(t, err)
+	sort.Strings(nns)
+	expectedNns := []string{"hadoop-namenode-01:8020", "hadoop-namenode-02:8020", "testnode:9000"}
+	assert.EqualValues(t, expectedNns, nns)
+	os.Setenv("HADOOP_HOME", "")
+}
+
+func TestConfNamenodeHadoopConfDir(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_CONF_DIR", strings.Join([]string{pwd, "test"}, "/"))
+	hdConf := LoadHadoopConf("")
+	nns, err := hdConf.Namenodes()
+	expectedNns := []string{"testnode:9000"}
+	assert.Nil(t, err)
+	assert.EqualValues(t, nns, expectedNns)
+	os.Setenv("HADOOP_CONF_DIR", "")
+}
+
+func TestDedupeNamenodes(t *testing.T) {
+	pwd, _ := os.Getwd()
+	os.Setenv("HADOOP_HOME", strings.Join([]string{pwd, "test"}, "/"))
+	os.Setenv("HADOOP_CONF_DIR", strings.Join([]string{pwd, "test"}, "/"))
+	hdConf := LoadHadoopConf("")
+	nns, err := hdConf.Namenodes()
+	assert.Nil(t, err)
+	sort.Strings(nns)
+	expectedNns := []string{"hadoop-namenode-01:8020", "hadoop-namenode-02:8020", "testnode:9000"}
+	assert.EqualValues(t, expectedNns, nns)
+
+	os.Setenv("HADOOP_HOME", "")
+	os.Setenv("HADOOP_CONF_DIR", "")
+}

test/conf/hdfs-site.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" ?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- Put site-specific property overrides in this file. -->
+<configuration>
+    <property>
+        <name>dfs.nameservices</name>
+        <value>tests</value>
+    </property>
+    <property>
+       <name>dfs.ha.automatic-failover.enabled</name>
+       <value>true</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.ha.namenodes.tests</name>
+        <value>nn1,nn2</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn1</name>
+        <value>hadoop-namenode-01:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn2</name>
+        <value>hadoop-namenode-02:8020</value>
+    </property>
+    <property>
+        <name>dfs.namenode.rpc-address.tests.nn3</name>
+        <value>testnode:9000</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn1</name>
+        <value>hadoop-namenode-01:50070</value>
+    </property>
+    <property>
+        <name>dfs.namenode.http-address.tests.nn2</name>
+        <value>hadoop-namenode-02:50070</value>
+    </property>
+</configuration>

test/core-site.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+    <property>
+      <name>fs.defaultFS</name>
+      <value>hdfs://testnode:9000</value>
+    </property>
+</configuration>