Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Stuff in example dir for demo -- ignore it please

  • Loading branch information...
commit 78eac5abd4140d15f1952c2b2bb75e6fc6b5c2c1 1 parent 43f7c60
Philip (flip) Kromer authored
Showing with 127 additions and 0 deletions.
  1. +56 −0 examples/ignore_me/counting.rb
  2. +71 −0 examples/ignore_me/grouper.rb
View
56 examples/ignore_me/counting.rb
@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'wukong'
+
+require 'bloomfilter-rb'
+
+SIZE = 2**24
+
+class BucketCounter
+ def initialize(opts = {})
+ @opts = {
+ :size => 100,
+ :server => {}
+ }.merge opts
+ @db = ::Redis.new(@opts[:server])
+ @size = opts[:size]
+ end
+
+ def key_for val
+ (val.hash % @size)
+ end
+
+ def insert(val)
+ @db.incr(key_for(val))
+ end
+ alias :<< :insert
+
+ def delete(val)
+ if @db.decr(key_for(val)).to_i <= 0
+ @db.del(key_for(val))
+ end
+ end
+
+ def [](val)
+ @db.get(key_for(val)).to_i
+ end
+
+ def clear
+ @db.flushdb
+ end
+end
+
+bf = BucketCounter.new(:size => 1_000, :server => {:host => 'localhost'})
+bf.clear
+counts = Hash.new{|h,k| h[k] = 0 }
+
+doc = File.read(__FILE__)
+doc.split(/\W+/).each do |word|
+ counts[word] += 1
+ bf << word
+end
+
+counts.keys.sort.each do |word|
+ puts [ bf[word] - counts[word], bf[word], counts[word], word.hash % SIZE, word ].join("\t")
+end
View
71 examples/ignore_me/grouper.rb
@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'backports'
+require 'backports/1.8.8'
+require 'extlib'
+
+class Source
+ # include Enumerable
+ attr_reader :streamer
+
+ def recordize line
+ # line.strip.split("\t")
+ [line[0..5]]
+ end
+
+ def each *args
+ $stdin.each(*args) do |raw_record|
+ record = recordize(raw_record)
+ next if record.blank?
+ yield *record
+ break if raw_record =~ /end/
+ end
+ end
+end
+
+# def process_group group
+# end
+#
+
+class Streamer
+
+ def recordize line
+ [line[0..5]]
+ end
+
+ def each_group
+ while not $stdin.eof? do
+ Enumerator.new do |yielder|
+ $stdin.each do |line|
+ yield yielder
+ p yielder
+ break if line =~ /end/
+ end
+ end
+ end
+ end
+end
+
+foo = Streamer.new
+
+foo.each_group do |group|
+ puts "hi"
+ p group.each do |line|
+ p line.reverse
+ end
+ # .map do |record|
+ # 1
+ # end
+end
+
+
+# i = 0
+# # s = source.new(Streamer.new)
+# $stdin.each do
+# process_group do |output|
+# puts output
+# end
+# $stderr.puts [Time.now, i] if (i += 1) % 10 == 0
+# end
+
+
Please sign in to comment.
Something went wrong with that request. Please try again.