Skip to content
Browse files

simplify/cleanup listing code in cat function

  • Loading branch information...
1 parent 144e411 commit 32efa36a6e7f083eef7e63850d7c885f137a0b61 Klaas Bosteels committed Aug 20, 2012
Showing with 2 additions and 8 deletions.
  1. +2 −8 dumbo/backends/streaming.py
View
10 dumbo/backends/streaming.py
@@ -220,16 +220,10 @@ def cat(self, path, opts):
try:
import typedbytes
ls = os.popen('%s %s -ls %s' % (hadenv, self.hdfs, path))
- if sum(c in path for c in ("*", "?", "{")) > 0:
- # cat each file separately when the path contains special chars
- lineparts = (line.split()[-1] for line in ls)
- subpaths = [part for part in lineparts if part.startswith("/")]
- else:
- # we still do the ls even in this case to make sure we print errors
- subpaths = [path]
+ subpaths = [line.split()[-1] for line in ls if ":" in line]
ls.close()
for subpath in subpaths:
- if subpath.endswith("/_logs"):
+ if subpath.split("/")[-1].startswith("_"):
continue
dumptb = os.popen('%s %s/bin/hadoop jar %s dumptb %s 2> /dev/null'
% (hadenv, self.hadoop, streamingjar, subpath))

0 comments on commit 32efa36

Please sign in to comment.
Something went wrong with that request. Please try again.