Skip to content
Browse files

start adding support for multiple file input

  • Loading branch information...
1 parent d61bca7 commit b486c781a60c01833774711d9c4a05fd2d6da69e @unbracketed committed Aug 28, 2011
Showing with 14 additions and 5 deletions.
  1. +11 −2 bin/redis-import
  2. +3 −3 redline/commands.py
View
13 bin/redis-import
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import argparse
import sys
+import fileinput
from redline import commands
@@ -19,11 +20,19 @@ if __name__ == '__main__':
help='Redis command')
parser.add_argument('key',
help='name of the key for the command')
-
+ parser.add_argument('infiles', nargs='*',
+ #type=argparse.FileType('r'),
+ #default=sys.stdin,
+ metavar='FILE')
options = parser.parse_args()
cmd = options.cmd
if cmd == 'set':
- commands.load_set(options.key, sys.stdin, batch_size=options.batch_size)
+ print sys.argv
+ if len(sys.argv) > 3:
+ input_set = fileinput.input(sys.argv[3:])
+ else:
+ input_set = sys.stdin
+ commands.load_set(options.key, input_set, batch_size=options.batch_size)
elif cmd == 'list':
commands.load_list(options.key, sys.stdin, batch_size=options.batch_size)
View
6 redline/commands.py
@@ -12,12 +12,12 @@ def load_set(key, IN, **kwargs):
r = redis.Redis()
pipeline_redis = r.pipeline()
count = 0
- #batch_size = kwargs['batch_size']
batch_size = kwargs.get('batch_size', 1000)
seen = set([None])
- for member, _ in groupby(reader(IN, delimiter='\t'),
- lambda x: x[0] if len(x) else None):
+ reader_files = reader(IN, delimiter='\t')
+ for member, _ in groupby(reader_files,
+ lambda x: x[0] if len(x) else None):
if member not in seen:
pipeline_redis.sadd(key, member.rstrip())
count += 1

0 comments on commit b486c78

Please sign in to comment.
Something went wrong with that request. Please try again.