Added legacy object support, versioned records, and unit tests.

* Also began to factor out functionality into separate files
avdi · Nov 22, 2009 · 19be731 · 19be731
1 parent c74c534
commit 19be731
Show file tree

Hide file tree

Showing 22 changed files with 1,105 additions and 149 deletions.
diff --git a/README b/README
@@ -50,6 +50,15 @@ http://github.com/devver/dm-adapter-simpledb/
    - Store floats using exponential notation
  * Option to store Date/Time/DateTime as ISO8601
  * Full aggregate support (min/max/etc)
+ * Offset support
+   Note, from the SimpleDB documentation:
+
+   "The next token returned by count(*) and select are interchangeable as long
+   as the where and order by clauses match. For example, if you want to return
+   the 200 items after the first 10,000 (similar to an offset), you can perform
+   a count with a limit clause of 10,000 and use the next token to return the
+   next 200 items with select."
+
  * Option to use libxml if available
  * Parallelized queries for increased throughput
  * Support of normalized 1:1 table:domain schemes that works with associations

diff --git a/Rakefile b/Rakefile
@@ -4,33 +4,44 @@ require 'pathname'
 load 'tasks/devver.rake'
 
 ROOT = Pathname(__FILE__).dirname.expand_path
-require ROOT + 'lib/simpledb_adapter'
 
-task :default => [ :spec ]
+task :default => [ 'spec:unit' ]
 
-desc 'Run specifications'
-Spec::Rake::SpecTask.new(:spec) do |t|
-  if File.exists?('spec/spec.opts')
-    t.spec_opts << '--options' << 'spec/spec.opts'
+namespace :spec do
+  desc 'Run unit-level specifications'
+  Spec::Rake::SpecTask.new(:unit) do |t|
+    if File.exists?('spec/spec.opts')
+      t.spec_opts << '--options' << 'spec/spec.opts'
+    end
+    t.spec_files = Pathname.glob((ROOT + 'spec/unit/**/*_spec.rb').to_s)
+
+    begin
+      t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true
+      t.rcov_opts << '--exclude' << 'spec'
+      t.rcov_opts << '--text-summary'
+      t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
+    rescue Exception
+      # rcov not installed
+    end
   end
-  t.spec_files = Pathname.glob((ROOT + 'spec/**/*_spec.rb').to_s)
-
-  begin
-    t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true
-    t.rcov_opts << '--exclude' << 'spec'
-    t.rcov_opts << '--text-summary'
-    t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
-  rescue Exception
-    # rcov not installed
-  end
-end
 
-desc 'Run specifications without Rcov'
-Spec::Rake::SpecTask.new(:spec_no_rcov) do |t|
-  if File.exists?('spec/spec.opts')
-    t.spec_opts << '--options' << 'spec/spec.opts'
+  desc 'Run integration-level specifications'
+  Spec::Rake::SpecTask.new(:integration) do |t|
+    if File.exists?('spec/spec.opts')
+      t.spec_opts << '--options' << 'spec/spec.opts'
+    end
+    t.spec_files = Pathname.glob((ROOT + 'spec/integration/**/*_spec.rb').to_s)
+
+    begin
+      t.rcov = ENV.has_key?('NO_RCOV') ? ENV['NO_RCOV'] != 'true' : true
+      t.rcov_opts << '--exclude' << 'spec'
+      t.rcov_opts << '--text-summary'
+      t.rcov_opts << '--sort' << 'coverage' << '--sort-reverse'
+    rescue Exception
+      # rcov not installed
+    end
   end
-  t.spec_files = Pathname.glob((ROOT + 'spec/**/*_spec.rb').to_s)
+
 end
 
 begin
@@ -68,6 +79,8 @@ END
     ]
     gem.add_dependency('dm-core',               '~> 0.10.0')
     gem.add_dependency('dm-aggregates',         '~> 0.10.0')
+    gem.add_dependency('dm-migrations',         '~> 0.10.0')
+    gem.add_dependency('dm-types',              '~> 0.10.0')
     gem.add_dependency('uuidtools',             '~> 2.0')
     gem.add_dependency('right_aws',             '~> 1.10')
   end

diff --git a/lib/simpledb/chunked_string.rb b/lib/simpledb/chunked_string.rb
@@ -0,0 +1,54 @@
+module SimpleDB
+  class ChunkedString < String
+    MAX_CHUNK_SIZE = 1019
+
+    def self.valid?(values)
+      values.all?{|v| v =~ /^\d{4}:/}
+    end
+
+    def initialize(string_or_array)
+      case string_or_array
+      when Array then super(chunks_to_string(string_or_array))
+      else super(string_or_array)
+      end
+    end
+
+    def to_ary
+      string_to_chunks(self)
+    end
+
+    alias_method :to_a, :to_ary
+
+    private
+
+    def string_to_chunks(value)
+      return [value] if value.size <= 1019
+      chunks = value.to_s.scan(%r/.{1,1019}/m) # 1024 - '1024:'.size
+      i = -1
+      fmt = '%04d:'
+      chunks.map!{|chunk| [(fmt % (i += 1)), chunk].join}
+      raise ArgumentError, 'that is just too big yo!' if chunks.size >= 256
+      chunks
+    end
+
+    def chunks_to_string(value)
+      begin
+        chunks =
+          Array(value).flatten.map do |chunk|
+          index, text = chunk.split(%r/:/, 2)
+          [Float(index).to_i, text]
+        end
+        chunks.replace chunks.sort_by{|index, text| index}
+        string_result = chunks.map!{|index, text| text}.join
+        string_result
+      rescue ArgumentError, TypeError
+        #return original value, they could have put strings in the system not
+        #using the adapter or previous versions that are larger than chunk size,
+        #but less than 1024
+        value
+      end
+    end
+
+
+  end
+end
diff --git a/lib/simpledb/rake.rb b/lib/simpledb/rake.rb
@@ -0,0 +1,43 @@
+namespace :simpledb do
+  desc "Migrate records to be compatable with current DM/SimpleDB adapter"
+  task :migrate, :domain do |t, args|
+    raise "THIS IS A WORK IN PROGRESS AND WILL DESTROY YOUR DATA"
+    require 'progressbar'
+    require 'right_aws'
+    require 'simpledb/record'
+
+    puts "Initializing connection..."
+    domain = args.domain
+    sdb = RightAws::SdbInterface.new
+    puts "Counting records..."
+    num_legacy_records = 0
+    query = "select count(*) from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
+    next_token = nil
+    while(results = sdb.select(query, next_token)) do
+      next_token = results[:next_token]
+      count = results[:items].first["Domain"]["Count"].first.to_i
+      num_legacy_records += count
+      break if next_token.nil?
+    end
+    puts "Found #{num_legacy_records} to migrate"
+
+    pbar = ProgressBar.new("migrate", num_legacy_records)
+    query = "select * from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
+    while(results = sdb.select(query, next_token)) do
+      next_token = results[:next_token]
+      items = results[:items]
+      items.each do |item|
+        legacy_record = SimpleDB::Record.from_simpledb_hash(item)
+        new_record    = legacy_record.migrate
+        updates       = new_record.writable_attributes
+        deletes       = new_record.deletable_attributes
+        sdb.put_attributes(domain, new_record.item_name, updates)
+        sdb.delete_attributes(domain, new_record.item_name, deletes)
+        pbar.inc
+      end
+      break if next_token.nil?
+    end
+    pbar.finish
+
+  end
+end