Item9808: limit data sizes and generate indexes for tables using a pe…

…r-installation schema.

git-svn-id: http://svn.foswiki.org/trunk/DBIStoreContrib@17302 0b4bb1d4-4e5a-0410-9cc4-b2b747904278

data/System/DBIStoreContrib.txt

@@ -23,51 +23,8 @@ Contains all wiki topics.
 Contains the names of all meta-tables.
    * =name= - table name, e.g. =TOPICINFO=, =FORM= etc
 
-At a minimum, =metatypes= will include the names of all the
-tables listed below. The columns of these tables have identical semantics
-(and syntax) to their corresponding =%META:= statements
-
-=TOPICINFO= %BR%
-   * =TEXT author=
-   * =TEXT version=
-   * =TEXT date=
-   * =TEXT format=
-   * =TEXT reprev=
-   * =TEXT rev=
-   * =TEXT comment=
-   * =TEXT encoding=
-
-=TOPICMOVED= %BR%
-   * =TEXT from=
-   * =TEXT to=
-   * =TEXT by=
-   * =TEXT date=
-
-=TOPICPARENT= %BR%
-   * =TEXT name=
-
-=FILEATTACHMENT= %BR%
-   * =TEXT name=
-   * =TEXT version=
-   * =TEXT path=
-   * =TEXT size=
-   * =TEXT date=
-   * =TEXT user=
-   * =TEXT comment=
-   * =TEXT attr=
-
-=FORM= %BR%
-   * =TEXT name=
-
-=FIELD= %BR%
-   * =TEXT name=
-   * =TEXT value=
-   * =TEXT title=
-
-=PREFERENCE= %BR%
-   * =TEXT name=
-   * =TEXT value=
-   * =TEXT type=
+The full schema, including the types used to represent different data can
+be found in =configure= under the settings for the Extension.
 
 Note that file attachments are *not* stored in the database at this time.
 
@@ -97,6 +54,14 @@ Date conversion using the =d2n= operator is not supported.
 ---+++ Row indexes
 Integer indexes are not supported. Use queries instead.
 
+---+++ Representational types
+The type defined in the schema must be long enough to store any possible
+value for the given field, but be as short as possible to maximise the
+DB's chance of building a decent index for it.
+
+---+++ length of an array
+The =length()= operator only works on string data, not on tables.
+
 ---++ Installation Instructions
 
 %$INSTALL_INSTRUCTIONS%

data/System/DBIStoreTest.txt

@@ -20,15 +20,18 @@ To reload this topic, click on %SCRIPTURL{"view"}%/%WEB%/%TOPIC%?dbistore_update
 To reload the companion form topic, click on %SCRIPTURL{"view"}%/%WEB%/DBIStoreTestForm?dbistore_update=1
 
 <!-- TA number of hist TB search query TC -->
-   * Non-query: %XA 2 %XB "1" %SOP%%XD%
    * False: %XA% 0 %XB% "0" %XC%
    * True: %XA% 2 %XB% "1" %XC%
-   * Table.selector: %XA% 2 %XB% "form.name='DBIStoreTestForm' AND length('x')=1" %XC%
-   * Array: %XA% 1 %XB% "fields[name='string'].value='String'" %XC%
-   * Field: %XA% 2 %XB% "number" %XC%
+   * > literal: %XA% 0 %XB% "3>20" %XC%
+   * < literal: %XA% 0 %XB% "'3'<'20'" %XC%
+   * Simple regex: %XA% 2 %XB% "'AA'=~'A'" %XC%
+   * Numeric Field: %XA% 2 %XB% "number" %XC%
    * Boolean field: %XA% 1 %XB% "boolean" %XC%
    * Field cmp: %XA% 1 %XB% "number=99" %XC%
-   * Simple regex: %XA% 2 %XB% "'AA'=~'A'" %XC%
+   * Field cmp: %XA% 1 %XB% "99=number" %XC%
+   * Field uop: %XA% 1 %XB% "length(number)=2" %XC%
+   * Array: %XA% 1 %XB% "fields[name='string'].value='String'" %XC%
+   * Table.selector: %XA% 2 %XB% "form.name='DBIStoreTestForm' AND length('x')=1" %XC%
    * Simple LIKE: %XA% 1 %XB% "name~'DBIStoreT*orm'" %XC%
    * Simple AND: %XA% 1 %XB% "number=99 AND string='String'" %XC%
    * Simple OR: %XA% 1 %XB% "number=99 OR string='String'" %XC%
@@ -52,6 +55,7 @@ To reload the companion form topic, click on %SCRIPTURL{"view"}%/%WEB%/DBIStoreT
    * Escapes  %XA% 0 %XB% "name =~ '\\\' OR name ~ '\\\' OR name = '\\\'" %XC%
    * Escapes  %XA% 0 %XB% "name =~ '\\.x.y\\\'" %XC%
    * Table=Table     %#TA% 0 %#TB% "fields=attachments" %#TC%
+   * Non-query: %XA 2 %XB "1" %SOP%%XD%
 
 %META:FORM{name="DBIStoreTestForm"}%
 %META:FIELD{name="number" attributes="" title="number" value="99"}%

lib/Foswiki/Contrib/DBIStoreContrib.pm

@@ -4,8 +4,8 @@ package Foswiki::Contrib::DBIStoreContrib;
 use strict;
 use Foswiki ();
 
-our $VERSION = '1.1';           # plugin version is also locked to this
-our $RELEASE = '25 Feb 2014';
+our $VERSION = '1.1';          # plugin version is also locked to this
+our $RELEASE = '4 Mar 2014';
 
 # Very verbose debugging. Used by all modules in the suite.
 use constant MONITOR => 0;

lib/Foswiki/Contrib/DBIStoreContrib/Config.spec

@@ -25,4 +25,70 @@ $Foswiki::cfg{Extensions}{DBIStoreContrib}{SQLite}{PCRE} = '/usr/lib/sqlite3/pcr
 # encountered in topic text. This should not normally be required, as plugins
 # should register all META that they create. Note that only META:NAME where
 # NAME matches /^[A-Z][A_Z0-9_]+$/ will be loaded.
-$Foswiki::cfg{Extensions}{DBIStoreContrib}{AutoloadUnknownMETA} = 0;
+$Foswiki::cfg{Extensions}{DBIStoreContrib}{AutoloadUnknownMETA} = 0;
+# **PERL**
+# If a column isn't found in the schema, it will use the _DEFAULT type.
+# You should extend this table as required by extra meta-data found in
+# your wiki.
+# If an entry for a column is a string starting with an underscore,
+# that string will be used as an index to get the 'real' schema for
+# the column.
+# The pseudo-type _DEFAULT must exist and must be a text type.
+# If {index} is true, then an index will be created for that column.
+$Foswiki::cfg{Extensions}{DBIStoreContrib}{Schema} = {
+    _DEFAULT => { type => 'TEXT' },
+    _USERNAME => { type => 'VARCHAR(64)', index => 1 },
+    _DATE => { type => 'VARCHAR(32)' },
+    topic => {
+        web  => { type => 'VARCHAR(256)', index => 1 },
+        name => { type => 'VARCHAR(128)', index => 1 },
+        text => '_DEFAULT',
+        raw  => '_DEFAULT'
+        },
+    metatypes => {
+        name => { type => 'VARCHAR(63)', index => 1 },
+        },
+    TOPICINFO => {
+        author => '_USERNAME',
+        version => { type => 'VARCHAR(256)' },
+        date => '_DATE',
+        format => { type => 'VARCHAR(32)' },
+        reprev => { type => 'VARCHAR(32)' },
+        rev => { type => 'VARCHAR(32)' },
+        comment => { type => 'VARCHAR(512)' },
+        encoding => { type => 'VARCHAR(32)' },
+    },
+    TOPICMOVED => {
+        from => { type => 'VARCHAR(256)' },
+        to => { type => 'VARCHAR(256)' },
+        by => { type => 'VARCHAR(256)' },
+        date => '_DATE',
+    },
+    TOPICPARENT => {
+        name => { type => 'VARCHAR(256)', index => 1 },
+    },
+    FILEATTACHMENT => {
+        name => { type => 'VARCHAR(256)', index => 1 },
+        version => { type => 'VARCHAR(32)' },
+        path => { type => 'VARCHAR(256)' },
+        size => { type => 'VARCHAR(32)' },
+        date => '_DATE',
+        user => '_USERNAME',
+        comment => { type => 'VARCHAR(512)' },
+        attr => { type => 'VARCHAR(32)' },
+    },
+    FORM => {
+        name => { type => 'VARCHAR(256)', index => 1 },
+    },
+    FIELD => {
+        name => { type => 'VARCHAR(128)', index => 1 },
+        value => { type => 'VARCHAR(512)', index => 1 },
+        title => { type => 'VARCHAR(256)' },
+    },
+    PREFERENCE => {
+        name => { type => 'VARCHAR(64)', index => 1 },
+        value => { type => 'VARCHAR(512)' },
+        type => { type => 'VARCHAR(32)' },
+    }
+};
+