support for ls -R (issue #100)

cmd/hdfs/ls.go

@@ -13,7 +13,7 @@ import (
 	"github.com/colinmarc/hdfs/v2"
 )
 
-func ls(paths []string, long, all, humanReadable bool) {
+func ls(paths []string, long, all, humanReadable, recursive bool) {
 	paths, client, err := getClientAndExpandedPaths(paths)
 	if err != nil {
 		fatal(err)
@@ -25,7 +25,8 @@ func ls(paths []string, long, all, humanReadable bool) {
 
 	files := make([]string, 0, len(paths))
 	fileInfos := make([]os.FileInfo, 0, len(paths))
-	dirs := make([]string, 0, len(paths))
+
+	var dirs []string
 	for _, p := range paths {
 		fi, err := client.Stat(p)
 		if err != nil {
@@ -40,30 +41,60 @@ func ls(paths []string, long, all, humanReadable bool) {
 		}
 	}
 
+	// The target is a directory; print its contents instead of the directory
+	// entry itself. Even recursive ls on a single directory still just prints the
+	// toplevel first, without a leading "/foo/bar:".
+	skipTopLevel := false
 	if len(files) == 0 && len(dirs) == 1 {
 		printDir(client, dirs[0], long, all, humanReadable)
+		skipTopLevel = true
+	}
+
+	if long {
+		tw := lsTabWriter()
+		for i, p := range files {
+			printLong(tw, p, fileInfos[i], humanReadable)
+		}
+
+		tw.Flush()
 	} else {
-		if long {
-			tw := lsTabWriter()
-			for i, p := range files {
-				printLong(tw, p, fileInfos[i], humanReadable)
-			}
+		for _, p := range files {
+			fmt.Println(p)
+		}
+	}
 
-			tw.Flush()
-		} else {
-			for _, p := range files {
-				fmt.Println(p)
-			}
+	// Add all nested directories for the recursive case. These are printed as
+	// "siblings".
+	if recursive {
+		var nestedDirs []string
+		for _, dir := range dirs {
+			client.Walk(dir, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+
+				if info.IsDir() {
+					nestedDirs = append(nestedDirs, path)
+				}
+
+				return nil
+			})
 		}
 
-		for i, dir := range dirs {
-			if i > 0 || len(files) > 0 {
-				fmt.Println()
-			}
+		dirs = nestedDirs
+	}
+
+	if skipTopLevel && len(dirs) > 0 {
+		dirs = dirs[1:]
+	}
 
-			fmt.Printf("%s/:\n", dir)
-			printDir(client, dir, long, all, humanReadable)
+	for i, dir := range dirs {
+		if i > 0 || len(files) > 0 || skipTopLevel {
+			fmt.Println()
 		}
+
+		fmt.Printf("%s:\n", dir)
+		printDir(client, dir, long, all, humanReadable)
 	}
 }
 

cmd/hdfs/main.go

@@ -22,7 +22,7 @@ var (
 The flags available are a subset of the POSIX ones, but should behave similarly.
 
 Valid commands:
-  ls [-lah] [FILE]...
+  ls [-lahR] [FILE]...
   rm [-rf] FILE...
   mv [-nT] SOURCE... DEST
   mkdir [-p] FILE...
@@ -46,6 +46,7 @@ Valid commands:
 	lsl    = lsOpts.Bool('l')
 	lsa    = lsOpts.Bool('a')
 	lsh    = lsOpts.Bool('h')
+	lsR    = lsOpts.Bool('R')
 
 	rmOpts = getopt.New()
 	rmr    = rmOpts.Bool('r')
@@ -118,7 +119,7 @@ func main() {
 		fatal("gohdfs version", version)
 	case "ls":
 		lsOpts.Parse(argv)
-		ls(lsOpts.Args(), *lsl, *lsa, *lsh)
+		ls(lsOpts.Args(), *lsl, *lsa, *lsh, *lsR)
 	case "rm":
 		rmOpts.Parse(argv)
 		rm(rmOpts.Args(), *rmr, *rmf)

cmd/hdfs/test/glob.bats

@@ -17,12 +17,12 @@ setup() {
   run $HDFS ls /_test_cmd/glob/dir*/dir
   assert_success
   assert_output <<OUT
-/_test_cmd/glob/dir1/dir/:
+/_test_cmd/glob/dir1/dir:
 a
 b
 c
 
-/_test_cmd/glob/dir2/dir/:
+/_test_cmd/glob/dir2/dir:
 d
 OUT
 }
@@ -33,12 +33,49 @@ OUT
   assert_output <<OUT
 /_test_cmd/glob/dir1/foo
 
-/_test_cmd/glob/dir1/dir/:
+/_test_cmd/glob/dir1/dir:
 a
 b
 c
 
-/_test_cmd/glob/dir2/dir/:
+/_test_cmd/glob/dir2/dir:
+d
+OUT
+}
+
+@test "ls recursive on glob" {
+  run $HDFS ls -R /_test_cmd/glob/dir*
+  assert_success
+  assert_output <<OUT
+/_test_cmd/glob/dir1:
+dir
+foo
+
+/_test_cmd/glob/dir1/dir:
+a
+b
+c
+
+/_test_cmd/glob/dir2:
+dir
+
+/_test_cmd/glob/dir2/dir:
+d
+
+/_test_cmd/glob/dir3:
+OUT
+}
+
+@test "ls recursive on glob of leafs" {
+  run $HDFS ls -R /_test_cmd/glob/dir*/dir
+  assert_success
+  assert_output <<OUT
+/_test_cmd/glob/dir1/dir:
+a
+b
+c
+
+/_test_cmd/glob/dir2/dir:
 d
 OUT
 }
@@ -49,12 +86,12 @@ OUT
   assert_output <<OUT
 /_test_cmd/glob/dir1/foo
 
-/_test_cmd/glob/dir1/dir/:
+/_test_cmd/glob/dir1/dir:
 a
 b
 c
 
-/_test_cmd/glob/dir2/dir/:
+/_test_cmd/glob/dir2/dir:
 d
 OUT
 }
@@ -63,12 +100,12 @@ OUT
   run $HDFS ls /_test_cmd/glob/dir*/dir*
   assert_success
   assert_output <<OUT
-/_test_cmd/glob/dir1/dir/:
+/_test_cmd/glob/dir1/dir:
 a
 b
 c
 
-/_test_cmd/glob/dir2/dir/:
+/_test_cmd/glob/dir2/dir:
 d
 OUT
 }

cmd/hdfs/test/helper.bash

@@ -17,7 +17,7 @@ flunk() {
 
 assert_success() {
   if [ "$status" -ne 0 ]; then
-    flunk "command failed with exit status $status"
+    flunk "command failed with exit status $status. output: $output"
   elif [ "$#" -gt 0 ]; then
     assert_output "$1"
   fi

cmd/hdfs/test/ls.bats

@@ -22,6 +22,44 @@ c
 OUT
 }
 
+@test "ls recursive on nested dir" {
+  run $HDFS ls -R /_test_cmd/ls
+  assert_success
+  assert_output <<OUT
+dir1
+dir2
+dir3
+
+/_test_cmd/ls/dir1:
+a
+b
+c
+
+/_test_cmd/ls/dir2:
+d
+
+/_test_cmd/ls/dir3:
+OUT
+}
+
+@test "ls recursive on leaf dir" {
+  run $HDFS ls -R /_test_cmd/ls/dir1
+  assert_success
+  assert_output <<OUT
+a
+b
+c
+OUT
+}
+
+@test "ls recursive on file" {
+  run $HDFS ls -R /_test_cmd/ls/dir1/c
+  assert_success
+  assert_output <<OUT
+/_test_cmd/ls/dir1/c
+OUT
+}
+
 @test "ls single files" {
   run $HDFS ls /_test_cmd/ls/dir1/a /_test_cmd/ls/dir1/b
   assert_success
@@ -39,6 +77,14 @@ stat /_test_cmd/nonexistent: file does not exist
 OUT
 }
 
+@test "ls recursive nonexistent" {
+  run $HDFS ls -R /_test_cmd/nonexistent
+  assert_failure
+  assert_output <<OUT
+stat /_test_cmd/nonexistent: file does not exist
+OUT
+}
+
 teardown() {
   $HDFS rm -r /_test_cmd/ls
 }