added pagination support

README.md

@@ -46,6 +46,9 @@ This gem is released to gemcutter.  Rubyforge is not utilized.
     # Specific rows and columns
     File.open("nodes.txt", "w"){|file| DB[:nodes].filter(:id < 5).select(:id, :name).export(file)}
 
+	# Using pagination extension (for very large datasets)
+    File.open("nodes.txt", "w"){|file| DB[:nodes].export(file, :paginate => true, :page_size => 1000)}
+
 ## Use Rake Tasks
 
 Several rake tasks are made available simply by requiring "tasks/sequel" in your Rakefile 

lib/extensions/export.rb

@@ -13,9 +13,14 @@
 module Sequel
   class Dataset
     def export(fd = $stdout, options = {})
+
       opts[:delimiter]  = options[:delimiter] || "\t"
       opts[:quote_char] = options[:quote_char] || ''
       opts[:headers]    = options[:headers] != false
+      opts[:paginate]   = options[:paginate] || false
+      opts[:page_size]  = options[:page_size] || 5000
+
+      Sequel.extension :pagination if opts[:paginate]
 
       Sequel::Export::Writer.new(fd, self, opts).output
     end
@@ -29,25 +34,18 @@ def initialize(fd, dataset, options)
         @dataset = dataset
         @options = options
       end
-
-      def output 
-        first_row = @dataset.first
-        return unless first_row
-
-        quot = @options[:quote_char]
-        @columns ||= first_row.keys.sort_by{|x|x.to_s}
 
-        if @options[:headers] == true
-          @file.puts @columns.map{|col| "#{quot}#{col}#{quot}"}.join(@options[:delimiter])
-        end
-
-        @dataset.each do |row| 
+      def export_data(ds)
+        quot = @options[:quote_char]
+        ds.each do |row| 
           data = @columns.map do |col|
             case row[col]
             when Date then 
               "#{quot}#{row[col].strftime('%Y-%m-%d')}#{quot}"
-            when DateTime, Time then 
+            when DateTime then
               "#{quot}#{row[col].localtime.strftime('%Y-%m-%dT%H:%M%Z')}#{quot}"
+            when Time then 
+              "#{quot}#{row[col].localtime.strftime('%H:%M%Z')}#{quot}"
             when Float, BigDecimal then 
               row[col].to_f
             when BigDecimal, Bignum, Fixnum then 
@@ -59,6 +57,24 @@ def output
           @file.puts data.join(@options[:delimiter])
         end
       end
+
+      def output 
+        first_row = @dataset.first
+        return unless first_row
+
+        quot = @options[:quote_char]
+        @columns ||= first_row.keys.sort_by{|x|x.to_s}
+
+        if @options[:headers] == true
+          @file.puts @columns.map{|col| "#{quot}#{col}#{quot}"}.join(@options[:delimiter])
+        end
+
+        if @options[:paginate]
+          @dataset.each_page(@options[:page_size]){|paged_ds| export_data paged_ds}
+        else
+          export_data @dataset
+        end
+      end
 
       def build_row(row) 
         quot = @options[:quote_char] 

lib/sequel_tree.rb

@@ -60,8 +60,8 @@ def ancestors
         #
         #   subchild1.ancestors # => [child1, root]
         def descendants
-          nodes = self.children
-          nodes.each{|child| nodes + child.descendants}
+          nodes = self.children.dup
+          nodes.each{|child| nodes.concat(child.descendants)}
           nodes 
         end
 

lib/tasks/sequel.rake

@@ -40,16 +40,22 @@ namespace :sq do
 
   desc "Displays schema of table"
   task :desc, [:table] => :load_config do |t, args|
-    def o(value, size = 25)
-      "%#{-1*size}s" % value.to_s
-    end
     unless args[:table]
       Rake::Task["sq:tables"].invoke
     else
       puts '==[' << args.table << ']' << '=' * (80 - args.table.size - 4)
       DB.schema(args.table.to_sym).each_with_index do |col, i|
-       name, info = col
-       puts "#{o i+1, -3}: #{o name}:#{o info[:type], 15}:#{o info[:db_type], 15}:#{' not null ' unless info[:allow_null]} #{' pk ' if info[:primary_key]} #{' default: ' << info[:default].to_s if info[:default]}"
+        name, info = col
+        values = [
+          "%3s:" % (i + 1),
+          (" %-12s:" % "#{info[:db_type]}#{('(' + info[:max_chars].to_s + ')') if info[:max_chars]}"),
+          ("%15s:" % info[:type]),
+          "%-25s: " % name,
+          (' not null ' unless info[:allow_null]),
+          (' pk ' if info[:primary_key]),
+          (" default: %s" % info[:default] if info[:default]),
+          ]
+        puts values.join
       end
       puts '-' * 80
       indexes = DB.indexes(args.table.to_sym)
@@ -58,7 +64,7 @@ namespace :sq do
       else
         indexes.each_with_index do |idx, i|
           name, attrs = idx
-          puts '  ' << o(name, 28) << ": unique? " << o(attrs[:unique] ? 'yes' : 'no', 6) << ': ' << attrs[:columns].join(', ')
+          puts '  ' << "%-28s" % name << ": unique? " << "%-6s" % (attrs[:unique] ? 'yes' : 'no') << ': ' << attrs[:columns].join(', ')
         end
       end
       puts '=' * 80

test/rake/Rakefile

@@ -31,7 +31,7 @@ task :environment do
     primary_key :id
     String :name
     Integer :ipsum_id
-    Integer :neque
+    Integer :neque, :index => true
   end
 
   LOREMS = [

test/test_export.rb

@@ -116,6 +116,27 @@ module ExportTest
 9,"five","",5
 10,"four","",4
 11,"five.one",9,1
+12,"two.three",2,3
+      TEXT
+    end
+
+    it "should export paginated data with comma delimiter" do 
+      mem_stream = StringIO.new("", "w+")
+      DB[:nodes].export(mem_stream, :quote_char => '"', :delimiter => ',', :paginate => true, :page_size => 2)
+      mem_stream.pos = 0
+      mem_stream.read.should == <<-TEXT
+"id","name","parent_id","position"
+1,"one","",1
+2,"two","",2
+3,"three","",3
+4,"two.one",2,1
+5,"two.two",2,2
+6,"two.two.one",5,1
+7,"one.two",1,2
+8,"one.one",1,1
+9,"five","",5
+10,"four","",4
+11,"five.one",9,1
 12,"two.three",2,3
       TEXT
     end

test/test_sequel_tree.rb

@@ -71,7 +71,7 @@ class Lorem < Sequel::Model
     it "should find all descendants of a node" do 
       two = Node.find(:id => 2)
       two.name.should == "two"
-      two.descendants.map{|m| m[:id]}.should == [4, 5, 12]
+      two.descendants.map{|m| m[:id]}.should == [4, 5, 12, 6]
     end
 
     it "should find all ancestors of a node" do