support for ls -R (issue #100)

cmd/hdfs/ls.go

@@ -13,7 +13,7 @@ import (
 	"github.com/colinmarc/hdfs/v2"
 )
 
-func ls(paths []string, long, all, humanReadable bool) {
+func ls(paths []string, long, all, humanReadable, recursive bool) {
 	paths, client, err := getClientAndExpandedPaths(paths)
 	if err != nil {
 		fatal(err)
@@ -23,6 +23,10 @@ func ls(paths []string, long, all, humanReadable bool) {
 		paths = []string{userDir(client)}
 	}
 
+	doLs(client, paths, long, all, humanReadable, recursive)
+}
+
+func doLs(client *hdfs.Client, paths []string, long, all, humanReadable, recursive bool) {
 	files := make([]string, 0, len(paths))
 	fileInfos := make([]os.FileInfo, 0, len(paths))
 	dirs := make([]string, 0, len(paths))
@@ -41,7 +45,10 @@ func ls(paths []string, long, all, humanReadable bool) {
 	}
 
 	if len(files) == 0 && len(dirs) == 1 {
-		printDir(client, dirs[0], long, all, humanReadable)
+		if recursive {
+			fmt.Printf("\n%s:\n", dirs[0])
+		}
+		printDir(client, dirs[0], long, all, humanReadable, recursive)
 	} else {
 		if long {
 			tw := lsTabWriter()
@@ -61,13 +68,13 @@ func ls(paths []string, long, all, humanReadable bool) {
 				fmt.Println()
 			}
 
-			fmt.Printf("%s/:\n", dir)
-			printDir(client, dir, long, all, humanReadable)
+			fmt.Printf("\n%s:\n", dir)
+			printDir(client, dir, long, all, humanReadable, recursive)
 		}
 	}
 }
 
-func printDir(client *hdfs.Client, dir string, long, all, humanReadable bool) {
+func printDir(client *hdfs.Client, dir string, long, all, humanReadable, recursive bool) {
 	dirReader, err := client.Open(dir)
 	if err != nil {
 		fatal(err)
@@ -101,12 +108,30 @@ func printDir(client *hdfs.Client, dir string, long, all, humanReadable bool) {
 	}
 
 	var partial []os.FileInfo
+	var subdirsToVisit []string
+
 	for ; err != io.EOF; partial, err = dirReader.Readdir(100) {
 		if err != nil {
 			fatal(err)
 		}
 
 		printFiles(tw, partial, long, all, humanReadable)
+
+		if recursive {
+			for _, fi := range partial {
+				if fi.IsDir() {
+					subdirsToVisit = append(subdirsToVisit, path.Join(dir, fi.Name()))
+				}
+			}
+		}
+	}
+
+	if long {
+		tw.Flush()
+	}
+
+	for _, subdir := range subdirsToVisit {
+		doLs(client, []string{subdir}, long, all, humanReadable, recursive)
 	}
 
 	if long {

cmd/hdfs/main.go

@@ -22,7 +22,7 @@ var (
 The flags available are a subset of the POSIX ones, but should behave similarly.
 
 Valid commands:
-  ls [-lah] [FILE]...
+  ls [-lahR] [FILE]...
   rm [-rf] FILE...
   mv [-nT] SOURCE... DEST
   mkdir [-p] FILE...
@@ -46,6 +46,7 @@ Valid commands:
 	lsl    = lsOpts.Bool('l')
 	lsa    = lsOpts.Bool('a')
 	lsh    = lsOpts.Bool('h')
+	lsR    = lsOpts.Bool('R')
 
 	rmOpts = getopt.New()
 	rmr    = rmOpts.Bool('r')
@@ -118,7 +119,7 @@ func main() {
 		fatal("gohdfs version", version)
 	case "ls":
 		lsOpts.Parse(argv)
-		ls(lsOpts.Args(), *lsl, *lsa, *lsh)
+		ls(lsOpts.Args(), *lsl, *lsa, *lsh, *lsR)
 	case "rm":
 		rmOpts.Parse(argv)
 		rm(rmOpts.Args(), *rmr, *rmf)

cmd/hdfs/test/ls.bats

@@ -22,6 +22,48 @@ c
 OUT
 }
 
+@test "ls recursive on nested dir" {
+  run $HDFS ls -R /_test_cmd/ls
+  assert_success
+  assert_output <<OUT
+
+/_test_cmd/ls:
+dir1
+dir2
+dir3
+
+/_test_cmd/ls/dir1:
+a
+b
+c
+
+/_test_cmd/ls/dir2:
+d
+
+/_test_cmd/ls/dir3:
+OUT
+}
+
+@test "ls recursive on leaf dir" {
+  run $HDFS ls -R /_test_cmd/ls/dir1
+  assert_success
+  assert_output <<OUT
+
+/_test_cmd/ls/dir1:
+a
+b
+c
+OUT
+}
+
+@test "ls recursive on file" {
+  run $HDFS ls -R /_test_cmd/ls/dir1/c
+  assert_success
+  assert_output <<OUT
+/_test_cmd/ls/dir1/c
+OUT
+}
+
 @test "ls single files" {
   run $HDFS ls /_test_cmd/ls/dir1/a /_test_cmd/ls/dir1/b
   assert_success
@@ -39,6 +81,14 @@ stat /_test_cmd/nonexistent: file does not exist
 OUT
 }
 
+@test "ls recursive nonexistent" {
+  run $HDFS ls -R /_test_cmd/nonexistent
+  assert_failure
+  assert_output <<OUT
+stat /_test_cmd/nonexistent: file does not exist
+OUT
+}
+
 teardown() {
   $HDFS rm -r /_test_cmd/ls
 }