keys with scan

README.md

@@ -1,10 +1,26 @@
 # redis-tsv [![Build Status](https://travis-ci.org/maiha/redis-tsv.cr.svg?branch=master)](https://travis-ci.org/maiha/redis-tsv.cr)
 
+# redis-tsv
+
+import and export data from Redis in TSV format
+
+```shell
+% redis-tsv export > backup.tsv
+% redis-tsv import backup.tsv
+```
+
+## TODO
+
+- [x] `keys` should use `SCAN` rather than `KEYS *`
+- [ ] `import` should use `SCAN` rather than `KEYS *`
+- [ ] `export` should use `SCAN` rather than `KEYS *`
+
 ## Installation
 
 - needs [Crystal](http://crystal-lang.org/) to compile
 
 ```shell
+crystal deps  # for the first time only
 make
 cp bin/redis-tsv ~/bin/
 ```
@@ -17,12 +33,20 @@ cp bin/redis-tsv ~/bin/
 
 ```
 redis-tsv import foo.tsv
+redis-tsv -d, import foo.csv
 ```
 
 - export
 
 ```
 redis-tsv export > foo.tsv
+redis-tsv -d, export > foo.csv
+```
+
+- keys
+
+```
+redis-tsv keys > keys.list
 ```
 
 ### information
@@ -31,7 +55,6 @@ util commands for easy access to INFO
 
 - `count` : show a number of keys
 - `version` : show the redis version
-- `keys` : show all keys by using KEYS (be careful!)
 - `ping` : execute PING command
 - `info` (INFO result itself)
 
@@ -41,7 +64,7 @@ util commands for easy access to INFO
 
 ## Contributing
 
-1. Fork it ( https://github.com/maiha/redis-tsv/fork )
+1. Fork it ( https://github.com/maiha/redis-tsv.cr/fork )
 2. Create your feature branch (git checkout -b my-new-feature)
 3. Commit your changes (git commit -am 'Add some feature')
 4. Push to the branch (git push origin my-new-feature)

shard.yml

@@ -4,7 +4,7 @@ version: 0.1.0
 dependencies:
   redis:
     git: https://github.com/stefanwille/crystal-redis.git
-    version: ~> 1.6.1
+    version: ~> 1.6.2
 
 authors:
   - maiha <maiha@wota.jp>

src/bin/redis-tsv.cr

@@ -8,43 +8,52 @@ class Main
   option host : String, "-h HOST", "--host=HOST", "redis host", "localhost"
   option port : Int32 , "-p PORT", "--port=PORT", "redis port", 6379
   option delimiter : String, "-d STRING", "--delimiter=STRING", "default is TAB", "\t"
+  option count : Int32, "-c 1000", "--count=1000", "bulk count passed to Redis SCAN command", 1000
+  option quiet : Bool, "-q", "--quiet", "suppress progress reporting", false
 
   usage <<-EOF
-    Usage: #{$0} (import|export) [file]
+    Usage: #{$0} (import|export|keys) [file]
 
     Options:
 
-    Example:
-      #{$0} import foo.tsv
-      #{$0} export > foo.tsv
+    Example: (bulk commands)
+      #{$0}     import foo.tsv
+      #{$0} -d, import foo.csv
+      #{$0}     export > foo.tsv
+      #{$0} -d, export > foo.csv
+      #{$0}     keys > keys.list
 
-    Undocumented Commands:
+    Other utility commands:
       count, info, ping, version
     EOF
 
   def run
-    op = args.shift { die "missing command: import or export", "", usage }
+    op = args.shift { die "missing command: import or export" }
 
     case op
     when "count"
       puts redis.count
     when "export"
       redis.export(STDOUT, delimiter)
     when "import"
-      file = args.shift { die "missing input tsv file", "", usage }
+      file = args.shift { die "missing input tsv file" }
       File.open(file) {|io| redis.import(io, delimiter) }
     when "info"
       puts redis.info
     when "keys"
-      redis.raw.keys("*").each do |i|
-        puts i.to_s
+      redis.keys(progress: !quiet, count: count) do |key|
+        STDOUT.puts key
       end
     when "ping"
       puts redis.raw.ping
+    when "stest"
+      prefix = args.shift { die "stest expects prefix(String) for 1st arg" }
+      count  = args.shift { die "stest expects count(Int32) for 2nd arg" }
+      redis.stest(prefix, count.to_i)
     when "version"
       puts redis.version
     else
-      die "unknown command: #{op}", "", usage
+      die "unknown command: #{op}"
     end
   rescue err : RedisTsv::ManagedWarn
     STDERR.puts err.to_s.colorize(:yellow)

src/options.cr

@@ -90,10 +90,10 @@ module Options
     end
   end
 
-  protected def die(*args)
-    args.each do |arg|
-      STDERR.puts arg
-    end
+  protected def die(reason : String)
+    STDERR.puts reason.colorize(:red)
+    STDERR.puts ""
+    STDERR.puts usage
     STDERR.flush
     exit -1
   end

src/redis-tsv/bulk.cr

@@ -29,6 +29,33 @@ class RedisTsv
       end
       io.flush
     end
+
+    def keys(progress : Bool, count : Int32)
+      report = build_periodical_report(progress, 3.seconds)
+
+      i = 0
+      raw.each(count: count) do |key|
+        i += 1
+        report.call(i) if (i % 1000) == 0  # reduce method-call overhead
+        yield key
+      end
+    end
+
+    private def build_periodical_report(progress : Bool, interval : Time::Span)
+      return ->(i : Int32){} if progress == false
+      total = count
+      reported = Time.now
+      return ->(i : Int32){
+        now = Time.now
+        if total > 0 && reported + interval < now
+          pcent = [i * 100.0 / total, 100.0].min
+          time = now.to_s("%H:%M:%S")
+          STDERR.puts "%s [%-3.1f%%] (%d/%d)" % [time, pcent, i, total]
+          STDERR.flush
+          reported = now
+        end
+      }
+    end      
   end
 
   include Bulk

src/redis-tsv/each.cr

@@ -15,10 +15,10 @@ class Redis
     # ```
     # redis.each { |key| p key }
     # redis.each(match: "foo:*") { |key| p key }
-    # redis.each(count: 100) { |key| p key }
+    # redis.each(count: 1000) { |key| p key }
     # ```
 
-    def each(match = "*", count = nil)
+    def each(match = "*", count = 1000)
       idx = 0
       while true
         idx, keys = scan(idx, match, count)

src/redis-tsv/stest.cr

@@ -0,0 +1,36 @@
+require "redis"
+
+class RedisTsv
+  module Stest
+    def stest(prefix : String, count : Int32)
+      reporting_interval = 3.seconds
+      started_at = Time.now
+      last_reported_at = started_at
+      last_reported_count = 0
+      one_test = ->(i : Int32) {
+        now = Time.now
+        key = "#{prefix}#{i}"
+        val = "#{i}"
+        raw.set key, val
+        raw.get key
+        if last_reported_at + reporting_interval < now
+          took = now - last_reported_at
+          qps = (i - last_reported_count)*1000.0 / took.total_milliseconds
+          puts "%s %d (%.1f qps)" % [now, i, qps]
+          last_reported_at = now
+          last_reported_count = i
+        end
+      }
+      count.times do |i|
+        one_test.call(i)
+      end
+      # last reporting
+      now = Time.now
+      total_time = now - started_at
+      puts "#{now} #{count} writes/reads done (#{total_time} sec)"
+      puts "%.1f qps (read + write / sec)" % [count / total_time.to_f]
+    end
+  end
+
+  include Stest
+end