Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added pagination support

  • Loading branch information...
commit e5723100761dd72ae4f3102853415ba168a41a9b 1 parent 65adc4c
@mwlang authored
View
3  README.md
@@ -46,6 +46,9 @@ This gem is released to gemcutter. Rubyforge is not utilized.
# Specific rows and columns
File.open("nodes.txt", "w"){|file| DB[:nodes].filter(:id < 5).select(:id, :name).export(file)}
+ # Using pagination extension (for very large datasets)
+ File.open("nodes.txt", "w"){|file| DB[:nodes].export(file, :paginate => true, :page_size => 1000)}
+
## Use Rake Tasks
Several rake tasks are made available simply by requiring "tasks/sequel" in your Rakefile
View
42 lib/extensions/export.rb
@@ -13,9 +13,14 @@
module Sequel
class Dataset
def export(fd = $stdout, options = {})
+
opts[:delimiter] = options[:delimiter] || "\t"
opts[:quote_char] = options[:quote_char] || ''
opts[:headers] = options[:headers] != false
+ opts[:paginate] = options[:paginate] || false
+ opts[:page_size] = options[:page_size] || 5000
+
+ Sequel.extension :pagination if opts[:paginate]
Sequel::Export::Writer.new(fd, self, opts).output
end
@@ -29,25 +34,18 @@ def initialize(fd, dataset, options)
@dataset = dataset
@options = options
end
-
- def output
- first_row = @dataset.first
- return unless first_row
-
- quot = @options[:quote_char]
- @columns ||= first_row.keys.sort_by{|x|x.to_s}
- if @options[:headers] == true
- @file.puts @columns.map{|col| "#{quot}#{col}#{quot}"}.join(@options[:delimiter])
- end
-
- @dataset.each do |row|
+ def export_data(ds)
+ quot = @options[:quote_char]
+ ds.each do |row|
data = @columns.map do |col|
case row[col]
when Date then
"#{quot}#{row[col].strftime('%Y-%m-%d')}#{quot}"
- when DateTime, Time then
+ when DateTime then
"#{quot}#{row[col].localtime.strftime('%Y-%m-%dT%H:%M%Z')}#{quot}"
+ when Time then
+ "#{quot}#{row[col].localtime.strftime('%H:%M%Z')}#{quot}"
when Float, BigDecimal then
row[col].to_f
when BigDecimal, Bignum, Fixnum then
@@ -59,6 +57,24 @@ def output
@file.puts data.join(@options[:delimiter])
end
end
+
+ def output
+ first_row = @dataset.first
+ return unless first_row
+
+ quot = @options[:quote_char]
+ @columns ||= first_row.keys.sort_by{|x|x.to_s}
+
+ if @options[:headers] == true
+ @file.puts @columns.map{|col| "#{quot}#{col}#{quot}"}.join(@options[:delimiter])
+ end
+
+ if @options[:paginate]
+ @dataset.each_page(@options[:page_size]){|paged_ds| export_data paged_ds}
+ else
+ export_data @dataset
+ end
+ end
def build_row(row)
quot = @options[:quote_char]
View
4 lib/sequel_tree.rb
@@ -60,8 +60,8 @@ def ancestors
#
# subchild1.ancestors # => [child1, root]
def descendants
- nodes = self.children
- nodes.each{|child| nodes + child.descendants}
+ nodes = self.children.dup
+ nodes.each{|child| nodes.concat(child.descendants)}
nodes
end
View
18 lib/tasks/sequel.rake
@@ -40,16 +40,22 @@ namespace :sq do
desc "Displays schema of table"
task :desc, [:table] => :load_config do |t, args|
- def o(value, size = 25)
- "%#{-1*size}s" % value.to_s
- end
unless args[:table]
Rake::Task["sq:tables"].invoke
else
puts '==[' << args.table << ']' << '=' * (80 - args.table.size - 4)
DB.schema(args.table.to_sym).each_with_index do |col, i|
- name, info = col
- puts "#{o i+1, -3}: #{o name}:#{o info[:type], 15}:#{o info[:db_type], 15}:#{' not null ' unless info[:allow_null]} #{' pk ' if info[:primary_key]} #{' default: ' << info[:default].to_s if info[:default]}"
+ name, info = col
+ values = [
+ "%3s:" % (i + 1),
+ (" %-12s:" % "#{info[:db_type]}#{('(' + info[:max_chars].to_s + ')') if info[:max_chars]}"),
+ ("%15s:" % info[:type]),
+ "%-25s: " % name,
+ (' not null ' unless info[:allow_null]),
+ (' pk ' if info[:primary_key]),
+ (" default: %s" % info[:default] if info[:default]),
+ ]
+ puts values.join
end
puts '-' * 80
indexes = DB.indexes(args.table.to_sym)
@@ -58,7 +64,7 @@ namespace :sq do
else
indexes.each_with_index do |idx, i|
name, attrs = idx
- puts ' ' << o(name, 28) << ": unique? " << o(attrs[:unique] ? 'yes' : 'no', 6) << ': ' << attrs[:columns].join(', ')
+ puts ' ' << "%-28s" % name << ": unique? " << "%-6s" % (attrs[:unique] ? 'yes' : 'no') << ': ' << attrs[:columns].join(', ')
end
end
puts '=' * 80
View
2  test/rake/Rakefile
@@ -31,7 +31,7 @@ task :environment do
primary_key :id
String :name
Integer :ipsum_id
- Integer :neque
+ Integer :neque, :index => true
end
LOREMS = [
View
21 test/test_export.rb
@@ -119,6 +119,27 @@ module ExportTest
12,"two.three",2,3
TEXT
end
+
+ it "should export paginated data with comma delimiter" do
+ mem_stream = StringIO.new("", "w+")
+ DB[:nodes].export(mem_stream, :quote_char => '"', :delimiter => ',', :paginate => true, :page_size => 2)
+ mem_stream.pos = 0
+ mem_stream.read.should == <<-TEXT
+"id","name","parent_id","position"
+1,"one","",1
+2,"two","",2
+3,"three","",3
+4,"two.one",2,1
+5,"two.two",2,2
+6,"two.two.one",5,1
+7,"one.two",1,2
+8,"one.one",1,1
+9,"five","",5
+10,"four","",4
+11,"five.one",9,1
+12,"two.three",2,3
+ TEXT
+ end
it "should export everything with comma delimiter and no quote characters" do
mem_stream = StringIO.new("", "w+")
View
2  test/test_sequel_tree.rb
@@ -71,7 +71,7 @@ class Lorem < Sequel::Model
it "should find all descendants of a node" do
two = Node.find(:id => 2)
two.name.should == "two"
- two.descendants.map{|m| m[:id]}.should == [4, 5, 12]
+ two.descendants.map{|m| m[:id]}.should == [4, 5, 12, 6]
end
it "should find all ancestors of a node" do
Please sign in to comment.
Something went wrong with that request. Please try again.