Skip to content
Browse files

perfect

  • Loading branch information...
0 parents commit 7e366da8b3b8f45a5c38528ebd2724fc9233086c @technoweenie committed Mar 28, 2011
Showing with 349 additions and 0 deletions.
  1. +22 −0 LICENSE
  2. +135 −0 Rakefile
  3. +71 −0 lib/unique_content_set.rb
  4. +42 −0 test/unique_content_set_test.rb
  5. +79 −0 unique_content_set.gemspec
22 LICENSE
@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) Rick Olson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
135 Rakefile
@@ -0,0 +1,135 @@
+require 'rubygems'
+require 'rake'
+require 'date'
+
+#############################################################################
+#
+# Helper functions
+#
+#############################################################################
+
+def name
+ @name ||= Dir['*.gemspec'].first.split('.').first
+end
+
+def version
+ line = File.read("lib/#{name}.rb")[/^\s*VERSION\s*=\s*.*/]
+ line.match(/.*VERSION\s*=\s*['"](.*)['"]/)[1]
+end
+
+def date
+ Date.today.to_s
+end
+
+def rubyforge_project
+ name
+end
+
+def gemspec_file
+ "#{name}.gemspec"
+end
+
+def gem_file
+ "#{name}-#{version}.gem"
+end
+
+def replace_header(head, header_name)
+ head.sub!(/(\.#{header_name}\s*= ').*'/) { "#{$1}#{send(header_name)}'"}
+end
+
+#############################################################################
+#
+# Standard tasks
+#
+#############################################################################
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+ test.libs << 'lib' << 'test'
+ test.pattern = 'test/**/*_test.rb'
+ test.verbose = true
+end
+
+desc "Open an irb session preloaded with this library"
+task :console do
+ sh "irb -rubygems -r ./lib/#{name}.rb"
+end
+
+#############################################################################
+#
+# Custom tasks (add your own tasks here)
+#
+#############################################################################
+
+
+
+#############################################################################
+#
+# Packaging tasks
+#
+#############################################################################
+
+desc "Create tag v#{version} and build and push #{gem_file} to Rubygems"
+task :release => :build do
+ unless `git branch` =~ /^\* master$/
+ puts "You must be on the master branch to release!"
+ exit!
+ end
+ sh "git commit --allow-empty -a -m 'Release #{version}'"
+ sh "git tag v#{version}"
+ sh "git push origin master"
+ sh "git push origin v#{version}"
+ sh "gem push pkg/#{name}-#{version}.gem"
+end
+
+desc "Build #{gem_file} into the pkg directory"
+task :build => :gemspec do
+ sh "mkdir -p pkg"
+ sh "gem build #{gemspec_file}"
+ sh "mv #{gem_file} pkg"
+end
+
+desc "Generate #{gemspec_file}"
+task :gemspec => :validate do
+ # read spec file and split out manifest section
+ spec = File.read(gemspec_file)
+ head, manifest, tail = spec.split(" # = MANIFEST =\n")
+
+ # replace name version and date
+ replace_header(head, :name)
+ replace_header(head, :version)
+ replace_header(head, :date)
+ #comment this out if your rubyforge_project has a different name
+ replace_header(head, :rubyforge_project)
+
+ # determine file list from git ls-files
+ files = `git ls-files`.
+ split("\n").
+ sort.
+ reject { |file| file =~ /^\./ }.
+ reject { |file| file =~ /^(rdoc|pkg)/ }.
+ map { |file| " #{file}" }.
+ join("\n")
+
+ # piece file back together and write
+ manifest = " s.files = %w[\n#{files}\n ]\n"
+ spec = [head, manifest, tail].join(" # = MANIFEST =\n")
+ File.open(gemspec_file, 'w') { |io| io.write(spec) }
+ puts "Updated #{gemspec_file}"
+end
+
+desc "Validate #{gemspec_file}"
+task :validate do
+ libfiles = Dir['lib/*'] - ["lib/#{name}.rb", "lib/#{name}"]
+ unless libfiles.empty?
+ puts "Directory `lib` should only contain a `#{name}.rb` file and `#{name}` dir."
+ exit!
+ end
+ unless Dir['VERSION*'].empty?
+ puts "A `VERSION` file at root level violates Gem best practices."
+ exit!
+ end
+end
+
71 lib/unique_content_set.rb
@@ -0,0 +1,71 @@
+require 'digest/sha1'
+
+# Stores a set of unique content. This is used to check if new content being
+# added has been seen already.
+class UniqueContentSet
+ VERSION = '0.0.1'
+
+ class << self
+ # Sets are stored in Redis.
+ attr_accessor :redis,
+
+ # Prefix all Redis keys with a certain value.
+ :redis_prefix
+ end
+
+ self.redis_prefix = "unique"
+
+ attr_reader :key
+
+ def initialize(*args)
+ @redis = self.class.redis
+
+ # Turn the given args into a redis key, with pieces separated by ':'.
+ args.unshift self.class.redis_prefix
+ args.compact!
+ args.map! { |a| a.to_s }
+ @key = args * ":"
+ end
+
+ # Public: Adds the given content to the current set, scored by the
+ # given time.
+ #
+ # content - String content to add to the set.
+ # time - The current Time the content was created.
+ #
+ # Returns true if this is the first occurence of the content, or false.
+ def add(content, time = Time.now)
+ @redis.zadd(@key, time.to_i, member_from(content))
+ end
+
+ # Public: Looks for the given content in the current set.
+ #
+ # content - String content that is being checked.
+ #
+ # Returns true if the content is a member of the set, or false.
+ def exist?(content)
+ !!@redis.zscore(@key, member_from(content))
+ end
+
+ # Public: Removes content posted before the given time.
+ #
+ # time - The latest Time that should be purged from the set.
+ #
+ # Returns a Fixnum of the number of removed entries.
+ def delete_before(time)
+ @redis.zremrangebyscore(@key, 0, time.to_i+1)
+ end
+
+private
+ # Encodes the content into a value that can be used to quickly check
+ # uniqueness in the set.
+ #
+ # content - The String content.
+ #
+ # Returns a String of the the size of the content, plus a SHA of the content,
+ # separated by a colon.
+ def member_from(content)
+ content = content.to_s
+ '%d:%s' % [content.size, Digest::SHA1.hexdigest(content)]
+ end
+end
42 test/unique_content_set_test.rb
@@ -0,0 +1,42 @@
+require File.expand_path("../../lib/unique_content_set", __FILE__)
+require 'test/unit'
+require 'rubygems'
+require 'redis'
+
+UniqueContentSet.redis = Redis.new
+UniqueContentSet.redis.select ENV['REDIS_DB'] || 7
+
+class UniqueContentSetTest < Test::Unit::TestCase
+ def setup
+ UniqueContentSet.redis.flushdb
+ @set = UniqueContentSet.new :abc
+ @set.add 'abc', 1
+ end
+
+ def test_adding_unique_content
+ assert_equal true, @set.add('def')
+ end
+
+ def test_adding_repeated_content
+ assert_equal false, @set.add('abc')
+ end
+
+ def test_checking_content_existence
+ assert @set.exist?('abc')
+ assert !@set.exist?('def')
+ end
+
+ def test_removing_old_content
+ @set.add 'def', 2
+ @set.add 'ghi', 3
+ assert @set.exist?('abc')
+ assert @set.exist?('def')
+ assert @set.exist?('ghi')
+
+ @set.delete_before(1)
+
+ assert !@set.exist?('abc')
+ assert !@set.exist?('def')
+ assert @set.exist?('ghi')
+ end
+end
79 unique_content_set.gemspec
@@ -0,0 +1,79 @@
+## This is the rakegem gemspec template. Make sure you read and understand
+## all of the comments. Some sections require modification, and others can
+## be deleted if you don't need them. Once you understand the contents of
+## this file, feel free to delete any comments that begin with two hash marks.
+## You can find comprehensive Gem::Specification documentation, at
+## http://docs.rubygems.org/read/chapter/20
+Gem::Specification.new do |s|
+ s.specification_version = 2 if s.respond_to? :specification_version=
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+ s.rubygems_version = '1.3.5'
+
+ ## Leave these as is they will be modified for you by the rake gemspec task.
+ ## If your rubyforge_project name is different, then edit it and comment out
+ ## the sub! line in the Rakefile
+ s.name = 'email_reply_parser'
+ s.version = '0.2.0'
+ s.date = '2011-03-17'
+ s.rubyforge_project = 'email_reply_parser'
+
+ ## Make sure your summary is short. The description may be as long
+ ## as you like.
+ s.summary = "Short description used in Gem listings."
+ s.description = "Long description. Maybe copied from the README."
+
+ ## List the primary authors. If there are a bunch of authors, it's probably
+ ## better to set the email to an email list or something. If you don't have
+ ## a custom homepage, consider using your GitHub URL or the like.
+ s.authors = ["Rick Olson"]
+ s.email = 'technoweenie@gmail.com'
+ s.homepage = 'http://github.com/technoweenie/unique_content_set'
+
+ ## This gets added to the $LOAD_PATH so that 'lib/NAME.rb' can be required as
+ ## require 'NAME.rb' or'/lib/NAME/file.rb' can be as require 'NAME/file.rb'
+ s.require_paths = %w[lib]
+
+ ## This sections is only necessary if you have C extensions.
+ #s.require_paths << 'ext'
+ #s.extensions = %w[ext/extconf.rb]
+
+ ## If your gem includes any executables, list them here.
+ #s.executables = ["name"]
+ #s.default_executable = 'name'
+
+ ## Specify any RDoc options here. You'll want to add your README and
+ ## LICENSE files to the extra_rdoc_files list.
+ s.rdoc_options = ["--charset=UTF-8"]
+ s.extra_rdoc_files = %w[README.md LICENSE]
+
+ ## List your runtime dependencies here. Runtime dependencies are those
+ ## that are needed for an end user to actually USE your code.
+ s.add_dependency('redis', ["~> 2.1.0", "~> 2.0.0"])
+
+ ## List your development dependencies here. Development dependencies are
+ ## those that are only needed during development
+ #s.add_development_dependency('DEVDEPNAME', [">= 1.1.0", "< 2.0.0"])
+
+ ## Leave this section as-is. It will be automatically generated from the
+ ## contents of your Git repository via the gemspec task. DO NOT REMOVE
+ ## THE MANIFEST COMMENTS, they are used as delimiters by the task.
+ # = MANIFEST =
+ s.files = %w[
+ LICENSE
+ README.md
+ Rakefile
+ email_reply_parser.gemspec
+ lib/email_reply_parser.rb
+ test/email_reply_parser_test.rb
+ test/emails/email_1_1.txt
+ test/emails/email_1_2.txt
+ test/emails/email_1_3.txt
+ ]
+ # = MANIFEST =
+
+ ## Test files will be grabbed from the file list. Make sure the path glob
+ ## matches what you actually use.
+ s.test_files = s.files.select { |path| path =~ /^test\/.*_test\.rb/ }
+end
+
+

0 comments on commit 7e366da

Please sign in to comment.
Something went wrong with that request. Please try again.