Skip to content

Commit

Permalink
Item9808: truncate long data to fit into finite fields. The tradeoff …
Browse files Browse the repository at this point in the history 
…is between being able index fields for searching, or use them to store full data at the expense of the indexes. Searching TEXT is fast in MySql and Postgresql, but a disaster in SQL server.

git-svn-id: http://svn.foswiki.org/trunk/DBIStoreContrib@17304 0b4bb1d4-4e5a-0410-9cc4-b2b747904278
  • Loading branch information
CrawfordCurrie authored and CrawfordCurrie committed Mar 4, 2014
1 parent 9be8b5c commit 65ec156
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 54 deletions.
5 changes: 3 additions & 2 deletions data/System/DBIStoreTest.txt
View file
Expand Up @@ -61,5 +61,6 @@ To reload the companion form topic, click on %SCRIPTURL{"view"}%/%WEB%/DBIStoreT
%META:FIELD{name="number" attributes="" title="number" value="99"}%
%META:FIELD{name="string" attributes="" title="string" value="String"}%
%META:FIELD{name="boolean" attributes="" title="boolean" value="Boolean"}%
%META:FIELD{name="Other" attributes="" title="Other" value="DBIStoreTestForm"}%
%META:PREFERENCE{name="Red" value="0"}%
%META:FIELD{name="Other" spunsugar="" title="Other" value="DBIStoreTestForm"}%
%META:FIELD{name="Huge" title="Huge" value="Mrcda6 qsc a prtt0rz0s rf zt0ws t2at zzcd rf qr@ uw0p@ ucd0r @20 ruq, cr@ aff rur fauf@ 0z@20r. I@ rw0 @20w zcup0c@zrc r0prr@u frr 26813, 26814, 26597, 23694, acd 22416, d0@azfu quc 40frw.%0d%0a%0d%0aS@ar@zcq wz@2 S0p@. 3, w20c w0 2ad a w00@zcq wz@2 @20 frffrwzcq ac@zrc z@0wu, acd @20zr crccfuuzrcu wrz@@0c 40c0a@2 0ac2 rc0:%0d%0a%0d%0aCrrfzcq 4ar:%0d%0a%0d%0a- C20cz wz@2 Brzac @r u00 zf @20 uuppfz0r 2au c2acq0d (GF).%0d%0aDrc0: Nr c2acq0%0d%0a%0d%0a- Rzcu0 a cf0ac0d crrfzcq 4ar wz@2 ac0@rc0 rc @r a cf0ac Sz waf0r (wz@2 r0f0r0cc0) @r u00 zf wa@0rzaf u@zff crw0u rff (GF, HA, AA). %0d%0aDrc0, acd a c0w prrc0uu 2au 400c d0v0frp0d au @20r0 wau wa@0rzaf.%0d%0a%0d%0a- Ov0rvz0w rf cf0aczcq prrc0dur0 (HA wz@2 AA, LP)%0d%0aSrw0 drc0, cr r0prr@ au @20 farq0u@ 40c0fz@ ur far wau u00c au @20 cf0aczcq 4ar c2acq0 zc prrc0uu.%0d%0a%0d%0aS6fqard:%0d%0a%0d%0a- R0pfac0w0c@ rf u6rzcq0u wz@2 qfauu par@u (HA)%0d%0aS@zff waz@zcq. N00d da@0.%0d%0a%0d%0a- R0vz0w rf 0cvzrrcw0c@ acd @rrfzcq frr TH3 S6fqard (HA, BL)%0d%0aS@zff waz@zcq. N00d da@0.%0d%0a%0d%0aIcup0c@zrc:%0d%0a%0d%0a- Iccr0au0d 2zq2 prw0r zcup0c@zrc rf zccrwzcq cr6u@afu aqazc (fa4 zcup0c@zrcu 2av0 c2acq0d @r u0v0raf c0w rp0ra@rru, urw0 zuuu0u cr@0d zc zc@0rpr0@a@zrc) (HA, AA, LP)%0d%0a%0d%0aS@zff waz@zcq, cr da@0 u0@, rur vrfuw0 zu wzczwaf, 4u@ @2zu u2rufd 40 rc @20 pfac.%0d%0a%0d%0aOv0c:%0d%0a%0d%0a- Lrrz a@ rv0c r0pfac0w0c@ (BL)%0d%0aI@ frrzu fzz0 a waxzwuw rf $4z @r r0pfac0, acd aff 4u@ frur cr6u@afu uu4;0c@0d @r @2zu rv0c 0cvzrrcw0c@ 40frr0 auu0w4f6 2av0 fazf0d ru@ rf 15.%0d%0a%0d%0a- If cr@ r0pfac0d, dr a @0ar drwc acd cf0ac (aff)%0d%0aNr@ drc0.%0d%0a%0d%0aE-@0u@ 0auzpw0c@:%0d%0a%0d%0a- R0auaf (BL) cr@0: @2zu zu a 2rp0 @r q0@ ac acafrqu0 DC uzqcaf @2a@ crufd 2zc@ a@ pr@0c@zaf fazfur0 40frr0 AC - rzq2@ crw w0 pauu rr fazf, 4u@ pauu zu q0c0raff6 406rcd rur a4zfz@6 @r r0ad a uzqcaf. Krdaz dupfzca@0u rur @0u@ ffzcz wz@2 cr zuuu0u, 4u@ z@ u2rufd ffzcz 40 cr@0d @2a@ f0azaq0 a@ DC zu ac aqqr0qa@0 uzqcaf. Crufd a DC zcdzvzduaf c2acc0f @0u@ 0v0r u2rw zuuu0u rc a cr6u@af przrr @r S6fqard?%0d%0a%0d%0aT2zu dr0uc'@ frrz f0auz4f0, w0 2av0 u00c f0azaq0 curr0c@ acd @20c 4az0d, wz@2 a r0aurca4f0 pauu ra@0 (u@racq0, rcf6 fazfzcq DC @0u@ par@u pauu AC @0u@ fa@0r). Tr frrz a@ uzcqf0 c2acc0fu, w0 quc frrzzcq a@ @wr rrd0ru rf waqcz@ud0 frw0r uzqcaf u@zff, acd a 2acd @0u@ wz@2 @20 Krdaz 4rard. I@ cac dzucuuu fur@20r.%0d%0a%0d%0aIccrwzcq RMA cr6u@af:%0d%0a%0d%0a- Icup0c@zrc 46 @0aw, bu@ 46 GF%0d%0a%0d%0aI 2av0c'@ qr@@0c a cr6u@af 60@. 26813 acd 26814 w0r0 uuppru0d @r 40 frrz0d a@ frr r0uzdu0u u00c rc @20 wzcdrw, au wau 26597. T20 zd0a wau @2a@ w0 wrufd zcup0c@ @20 r0uzdu0 rc @20 wzcdrw @2a@ @206 cr@0d. I@ quc u@ar@zcq @r @racz @20 @zwzcq rf w20c @20 wa@0r crrfzcq fzc0u quc 0prxz0d wz@2 r0fa@zrc @r @20 cr6u@af auu0w4f6 prrc0uu, z@ cac var6 frrw da6u @r w00zu, acd @20 r4vzruu ru@qauuzcq rf @2zu 0prx6 crufd 40 zc@0rf0rzcq wz@2 S6fqard curzcq. T20u0 zcrwc fazfur0u wz@2 vzuz4f0 ru@qau r0uzdu0 w0r0 uuppru0d @r 40 zcup0c@0d acd duf6 cr@0d frr u0v0rz@6, acd @20c @20 S6fqard p00f0d rff @r u00 zf z@ crw0u wrr0 0auzf6 @2ac uuuaf; z0: dzd z@ 0v0c cur0 rc @20 uurfac0? I cac rcf6 fzcd @20 w20r0a4ru@u rf 26597, @2ruq2 w0 2av0 p2r@ru rf 26813 frrw Krdaz w20c @206 w0r0 u0cdzcq z@.%0d%0a%0d%0a%0d%0aS0para@zrc rf Aqzf0c@ prrc0uu:%0d%0a%0d%0a- T2zu wzff 40 rcqrzcq qrrup 0ffrr@, 4u@ zccr0au0d Aqzf0c@ ac@zvz@6 crufd 40 r0fa@0d @r @20 zuuu0. Faczfz@z0u 2au cr pfacu frr c2acqzcq @2zu uz@ua@zrc uc@zf @206 wrv0 ru@ aff @20 ffrw 2rrdu. N00d @r d0@0rwzc0 @2zu uc20duf0 (BL, HA)%0d%0aNr uc20duf0 zu u0@ 60@, faczfz@z0u 2au qzv0c ucrffzczaf uc20duf0u rf zww0dza@0 c2acq0u @r @20 rrrw, cr frrwaf uc20duf0 2au 400c frrz0d a@ frrw @2zu p0rup0c@zv0.%0d%0a%0d%0a-Wz@2zc @20 fau@ 60ar, a TH3 wau u0c@ wz@2ru@ a crrfzcq 4ar @r @0u@ @20 1/4 pzx0f (40frr0 I ;rzc0d) - fzcd r0uuf@ rf @2zu @0u@.%0d%0aT2zu wau cr6u@af 23694, z@ u@zff fazf0d acd w0 w0r0 uuppru0d @r zcup0c@ z@ aq0u aqr.%0d%0a%0d%0aAu w0ff, w0 2av0 ac ac@zrc z@0w @r zcup0c@ RMA0d auu0w4f6 22416. I@ wau ac arc2zv0d cr6u@af, w2zc2 dzd fazf 46 1/4 pzx0f, 4u@ @206 uazd @2a@ ac auu0w4f6 ;zq zwprzc@ wau f0f@ zc @20 AR cra@0d wzcdrw, ur w0 dr 2av0 @r 2rcrur @2a@ RMA. I@ quc ffzcz uuppru0d @r SEM z@ af@0r z@ 2au 400c zcup0c@0d acd d0@0rwzc0d 2rw @2zu warz qr@ @20r0, ;uu@ @r u00 @20 fazfur0 wrd0.%0d%0a%0d%0aGf0c"}%
%META:PREFERENCE{nqme="Red" value="0"}%
10 changes: 7 additions & 3 deletions lib/Foswiki/Contrib/DBIStoreContrib/Config.spec
View file
Expand Up @@ -35,6 +35,10 @@ $Foswiki::cfg{Extensions}{DBIStoreContrib}{AutoloadUnknownMETA} = 0;
# the column.
# The pseudo-type _DEFAULT must exist and must be a text type.
# If {index} is true, then an index will be created for that column.
# If {truncate_to} is set to a length, then the value of that field
# stored in the DB will be truncated to that length. This only affects
# searching. If truncate_to is not set, then trying to store a value
# longer than the field accepts will be an error.
$Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema} = {
_DEFAULT => { type => 'TEXT' },
_USERNAME => { type => 'VARCHAR(64)', index => 1 },
Expand Down Expand Up @@ -74,20 +78,20 @@ $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema} = {
size => { type => 'VARCHAR(32)' },
date => '_DATE',
user => '_USERNAME',
comment => { type => 'VARCHAR(512)' },
comment => { type => 'VARCHAR(512)', truncate_to => 512 },
attr => { type => 'VARCHAR(32)' },
},
FORM => {
name => { type => 'VARCHAR(256)', index => 1 },
},
FIELD => {
name => { type => 'VARCHAR(128)', index => 1 },
value => { type => 'VARCHAR(512)', index => 1 },
value => { type => 'VARCHAR(512)', index => 1, truncate_to => 512 },
title => { type => 'VARCHAR(256)' },
},
PREFERENCE => {
name => { type => 'VARCHAR(64)', index => 1 },
value => { type => 'VARCHAR(512)' },
value => _DEFAULT,
type => { type => 'VARCHAR(32)' },
}
};
Expand Down
69 changes: 53 additions & 16 deletions lib/Foswiki/Contrib/DBIStoreContrib/DBIStore.pm
View file
Expand Up @@ -175,14 +175,10 @@ sub _createTableForMETA {
my $schema = $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{$t};

# Create table
my $cols = join(
',',
my $cols = join( ',',
'tid INT',
map {
personality()->safe_id($_) . ' '
. personality()->column_type( $t, $_ )
} grep( !/^_/, keys %$schema )
);
map { personality()->safe_id($_) . ' ' . _column( $t, $_ )->{type} }
grep( !/^_/, keys %$schema ) );
my $sn = personality()->safe_id($t);
my $sql = "CREATE TABLE $sn ( $cols )";
_say $sql if MONITOR;
Expand Down Expand Up @@ -216,15 +212,14 @@ sub _createTables {
my @cols = ('tid INT PRIMARY KEY');
foreach my $c (qw/web name text raw/) {
push( @cols,
personality()->safe_id($c) . ' '
. personality()->column_type( 'topic', $c ) );
personality()->safe_id($c) . ' ' . _column( 'topic', $c )->{type} );
}
my $colst = join( ',', @cols );
$this->{handle}->do("CREATE TABLE topic ($colst,UNIQUE (tid))");

# Now create the meta-table of known META: tables
$this->{handle}->do( 'CREATE TABLE metatypes(name '
. personality()->column_type( 'metatypes', 'name' )
. _column( 'metatypes', 'name' )->{type}
. ')' );

# Create the tables for each registered META: type
Expand Down Expand Up @@ -280,6 +275,35 @@ sub _convertToUTF8 {
return $text;
}

sub _truncate {
my ( $data, $size ) = @_;
return $data unless defined($size) && length($data) > $size;
print STDERR 'Truncating ' . length($data) . " to $size\n"
if MONITOR;
return substr( $data, 0, $size - 3 ) . '...';
}

# Get the column schema for the given column.
sub _column {
my ( $table, $column ) = @_;

my $l = $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{$table};
$l = $l->{$column} if $l;
if ( defined $l ) {

# If the type name starts with an underscore, map to a default
# type name
if ( $l =~ /^_/ ) {
$l = $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{$l};
ASSERT($l) if DEBUG;
}
return $l;
}
print STDERR "WARNING: Could not determine a type for $table.$column\n"
if DEBUG;
return $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{_DEFAULT};
}

sub _inner_insert {
my ( $this, $mo ) = @_;
return unless defined $mo->topic();
Expand Down Expand Up @@ -358,12 +382,15 @@ sub _inner_insert {

# The column might be in the DB but not in
# the schema. This is unlikely, but possible.
$schema->{$kn} =
personality()->column_type( $type, $kn );

# _column will give us the default if the column
# name isn't matched
$schema->{$kn} = _column( $type, $kn );

$this->{handle}->do( 'ALTER TABLE '
. personality()->safe_id($type) . ' ADD '
. personality()->safe_id($kn) . ' '
. $schema->{$kn} );
. $schema->{$kn}->{type} );
}
}

Expand All @@ -382,11 +409,21 @@ sub _inner_insert {
. join( ',', map { '?' } @kns ) . ")";
shift(@kns);

_say "$sql [tid," . join( ',', map { $item->{$_} } @kns ) . ']'
_say "$sql [tid,"
. join( ',', map { _truncate( $item->{$_}, 80 ) } @kns ) . ']'
if MONITOR;

$this->{handle}->do( $sql, {},
$tid, map { _convertToUTF8( $item->{$_} ) } @kns );
$this->{handle}->do(
$sql,
{},
$tid,
map {
_truncate(
_convertToUTF8( $item->{$_} ),
_column( $type, $_ )->{truncate_to}
)
} @kns
);
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions lib/Foswiki/Contrib/DBIStoreContrib/HoistSQL.pm
View file
Expand Up @@ -674,9 +674,9 @@ sub _hoist {
$l_sel => $lhs{type},
$r_sel => $rhs{type}
);
my $where;
my $where = "($lhs_alias.tid=$rhs_alias.tid)";
if ( $optype == BOOLEAN ) {
$where = $expr;
$where .= " AND ($expr)";
$expr = $TRUE;
$optype = $TRUE_TYPE;
}
Expand Down
31 changes: 0 additions & 31 deletions lib/Foswiki/Contrib/DBIStoreContrib/Personality.pm
View file
Expand Up @@ -313,37 +313,6 @@ sub strcat {
return join( '||', @_ );
}

=begin TML
---++ column_type($table, $column) -> $typename
Determine the best text type to use to represent the given column.
The default implementation uses the
{Extensions}{DBIStoreContrib}{Schema} table to map from a
table name, column name to a type.
=cut

sub column_type {
my ( $this, $table, $column ) = @_;

my $l = $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{$table};
$l = $l->{$column} if $l;
if ( defined $l ) {

# If the type name starts with an underscore, map to a default
# type name
if ( $l =~ /^_/ ) {
$l = $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{$l};
ASSERT($l) if DEBUG;
}
return $l->{type};
}
print STDERR "WARNING: Could not determine a type for $table.$column\n"
if DEBUG;
return $Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema}{_DEFAULT}{type};
}

1;
__DATA__
Expand Down

0 comments on commit 65ec156

Please sign in to comment.