Permalink
Browse files

Add optimize_model_load setting to speedup loading of model objects, …

…off by default

This setting is off by default as it isn't completely compatible. But
it can speed up model loading by 30-50% for simple tables.
Basically, this is just like the standard way of returning a hash,
except that instead of yielding the hash, a model object is
allocated, the hash is assigned as the values, and the model object
is yielded.
  • Loading branch information...
1 parent b317ac7 commit 112ce399b5cbe2ff8e1f788c53d51a1144abfba9 @jeremyevans committed Oct 25, 2011
Showing with 84 additions and 0 deletions.
  1. +2 −0 CHANGELOG
  2. +25 −0 README.rdoc
  3. +24 −0 ext/sequel_pg/sequel_pg.c
  4. +33 −0 lib/sequel_pg/sequel_pg.rb
View
@@ -1,5 +1,7 @@
=== HEAD
+* Add optimize_model_load setting to speedup loading of model objects, off by default (jeremyevans)
+
* Add major speedup to Dataset#map, #to_hash, #select_map, #select_order_map, and #select_hash (jeremyevans)
* Work with the new Database#timezone setting in Sequel 3.29.0 (jeremyevans)
View
@@ -38,6 +38,31 @@ Here's an example that uses a modified version of swift's benchmarks
sequel #select 0.090000 2.020000 2.110000 2.246688 46.54m
sequel_pg #select 0.000000 0.250000 0.250000 0.361999 7.33m
+sequel_pg also has code to speed up the map, to_hash, select_hash,
+select_map, and select_order_map Dataset methods, which is on by
+default. It also has code to speed up the loading of model objects,
+which is off by default as it isn't fully compatible. It doesn't
+handle overriding Model.call, Model#set_values, or
+Model#after_initialize, which may cause problems with the
+following plugins that ship with Sequel:
+
+* class_table_inheritance
+* force_encoding
+* serializiation
+* single_table_interitance
+* typecast_on_load
+* update_primary_key
+
+If you want to extract that last ounce of performance when loading
+model objects and you can live with the limitations, you can
+enable the model optimization via:
+
+ # All datasets
+ DB.optimize_model_load = true
+
+ # Specific dataset
+ Artist.dataset.optimize_model_load = true
+
== Installing the gem
gem install sequel_pg
View
@@ -34,6 +34,7 @@
#define SPG_YIELD_MKV_HASH 6
#define SPG_YIELD_KMV_HASH 7
#define SPG_YIELD_MKMV_HASH 8
+#define SPG_YIELD_MODEL 9
static VALUE spg_Sequel;
static VALUE spg_Blob;
@@ -47,6 +48,7 @@ static VALUE spg_sym_map;
static VALUE spg_sym_first;
static VALUE spg_sym_array;
static VALUE spg_sym_hash;
+static VALUE spg_sym_model;
static VALUE spg_sym__sequel_pg_type;
static VALUE spg_sym__sequel_pg_value;
@@ -74,6 +76,7 @@ static ID spg_id_conversion_procs;
static ID spg_id_columns;
static ID spg_id_encoding;
+static ID spg_id_values;
#if SPG_ENCODING
static int enc_get_index(VALUE val)
@@ -458,6 +461,8 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
type = SPG_YIELD_MKMV_HASH;
}
}
+ } else if (pg_type == spg_sym_model && rb_type(pg_value) == T_CLASS) {
+ type = SPG_YIELD_MODEL;
}
}
}
@@ -510,6 +515,7 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
}
break;
case SPG_YIELD_KV_HASH:
+ /* Hash with single key and single value */
{
VALUE k, v;
h = rb_hash_new();
@@ -522,6 +528,7 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
}
break;
case SPG_YIELD_MKV_HASH:
+ /* Hash with array of keys and single value */
{
VALUE k, v;
h = rb_hash_new();
@@ -534,6 +541,7 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
}
break;
case SPG_YIELD_KMV_HASH:
+ /* Hash with single keys and array of values */
{
VALUE k, v;
h = rb_hash_new();
@@ -546,6 +554,7 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
}
break;
case SPG_YIELD_MKMV_HASH:
+ /* Hash with array of keys and array of values */
{
VALUE k, v;
h = rb_hash_new();
@@ -557,6 +566,19 @@ static VALUE spg_yield_hash_rows(VALUE self, VALUE rres, VALUE ignore) {
rb_yield(h);
}
break;
+ case SPG_YIELD_MODEL:
+ /* Model object for entire row */
+ for(i=0; i<ntuples; i++) {
+ h = rb_hash_new();
+ for(j=0; j<nfields; j++) {
+ rb_hash_aset(h, colsyms[j], spg__col_value(self, res, i, j, colconvert ENC_INDEX));
+ }
+ /* Abuse local variable */
+ pg_type = rb_obj_alloc(pg_value);
+ rb_ivar_set(pg_type, spg_id_values, h);
+ rb_yield(pg_type);
+ }
+ break;
}
return self;
}
@@ -590,13 +612,15 @@ void Init_sequel_pg(void) {
spg_id_columns = rb_intern("@columns");
spg_id_encoding = rb_intern("@encoding");
+ spg_id_values = rb_intern("@values");
spg_sym_utc = ID2SYM(rb_intern("utc"));
spg_sym_local = ID2SYM(rb_intern("local"));
spg_sym_map = ID2SYM(rb_intern("map"));
spg_sym_first = ID2SYM(rb_intern("first"));
spg_sym_array = ID2SYM(rb_intern("array"));
spg_sym_hash = ID2SYM(rb_intern("hash"));
+ spg_sym_model = ID2SYM(rb_intern("model"));
spg_sym__sequel_pg_type = ID2SYM(rb_intern("_sequel_pg_type"));
spg_sym__sequel_pg_value = ID2SYM(rb_intern("_sequel_pg_value"));
View
@@ -1,5 +1,15 @@
+# Add speedup for model class creation from dataset
+class Sequel::Postgres::Database
+ # Whether to optimize loads for all model datasets created from this dataset.
+ # Has certain limitations, see the README for details.
+ attr_accessor :optimize_model_load
+end
+
# Add faster versions of Dataset#map, #to_hash, #select_map, #select_order_map, and #select_hash
class Sequel::Postgres::Dataset
+ # Set whether to enable optimized model loading for this dataset.
+ attr_writer :optimize_model_load
+
# In the case where an argument is given, use an optimized version.
def map(sym=nil)
if sym
@@ -15,6 +25,12 @@ def map(sym=nil)
end
end
+ # If this dataset has turned model loading on or off, use the default value from
+ # the Database object.
+ def optimize_model_load
+ defined?(@optimize_model_load) ? @optimize_model_load : db.optimize_model_load
+ end
+
# In the case where both arguments given, use an optimized version.
def to_hash(key_column, value_column = nil)
if value_column
@@ -24,6 +40,15 @@ def to_hash(key_column, value_column = nil)
end
end
+ # If model loads are being optimized and this is a model load, use the optimized
+ # version.
+ def each
+ if (rp = row_proc) && optimize_model_load?
+ clone(:_sequel_pg_type=>:model, :_sequel_pg_value=>rp).fetch_rows(sql, &Proc.new)
+ else
+ super
+ end
+ end
protected
@@ -40,4 +65,12 @@ def _select_map_single
clone(:_sequel_pg_type=>:first).fetch_rows(sql){|s| rows << s}
rows
end
+
+ private
+
+ # The model load can only be optimized if it's for a model and it's not a graphed dataset
+ # or using a cursor.
+ def optimize_model_load?
+ (rp = row_proc).is_a?(Class) && (rp < Sequel::Model) && optimize_model_load && !opts[:use_cursor] && !opts[:graph]
+ end
end

0 comments on commit 112ce39

Please sign in to comment.