@@ -175,22 +175,27 @@ class DBSiteStore(common.SiteStore): result[r.key] = common.LazyThing(self, r.key, r.data) return result - def _add_version(self, thing_id, revision=None, timestamp=None, comment=None, machine_comment=None, ip=None, author=None): + def _add_transaction(self, action, author, ip, comment, created): + return self.db.insert('transaction', + action=action, + author_id=author and self.get_metadata(author).id, + ip=ip, + created=created, + comment=comment + ) + + def _add_version(self, thing_id, revision, transaction_id, created): if revision is None: d = self.db.query( - 'UPDATE thing set latest_revision=latest_revision+1, last_modified=$timestamp WHERE id=$thing_id;' + \ + 'UPDATE thing set latest_revision=latest_revision+1, last_modified=$created WHERE id=$thing_id;' + \ 'SELECT latest_revision FROM thing WHERE id=$thing_id', vars=locals()) revision = d[0].latest_revision self.db.insert('version', False, thing_id=thing_id, revision=revision, - created=timestamp, - comment=comment, - machine_comment=machine_comment, - ip=ip, - author_id=author and self.get_metadata(author).id - ) + transaction_id=transaction_id + ) return revision def _update_tables(self, thing_id, key, olddata, newdata): @@ -256,13 +261,15 @@ class DBSiteStore(common.SiteStore): for k in added: do_action(action_insert, new_type, thing_id, k, newdata[k]) - def save_many(self, items, timestamp, comment, machine_comment, ip, author): + def save_many(self, items, timestamp, comment, machine_comment, ip, author, action=None): + action = action or "bulk_update" t = self.db.transaction() - result = [self.save(d['key'], d, timestamp, comment, machine_comment, ip, author) for d in items] + transaction_id = self._add_transaction(action=action, author=author, ip=ip, comment=comment, created=timestamp) + result = [self.save(d['key'], d, transaction_id=transaction_id, timestamp=timestamp) for d in items] t.commit() return result - def save(self, key, data, timestamp=None, comment=None, machine_comment=None, ip=None, author=None): + def save(self, key, data, timestamp=None, comment=None, machine_comment=None, ip=None, author=None, transaction_id=None): timestamp = timestamp or datetime.datetime.utcnow() t = self.db.transaction() @@ -273,16 +280,17 @@ class DBSiteStore(common.SiteStore): revision = None thing_id = thing.id olddata = thing._get_data() + action = "update" else: revision = 1 type_id = self.get(typekey).id thing_id = self.new_thing(key=key, type=type_id, latest_revision=1, last_modified=timestamp, created=timestamp) - olddata = {} - - revision = self._add_version(thing_id, timestamp=timestamp, - comment=comment, machine_comment=machine_comment, - ip=ip, author=author, revision=revision) + action = "create" + + if transaction_id is None: + transaction_id = self._add_transaction(action=action, author=author, ip=ip, comment=comment, created=timestamp) + revision = self._add_version(thing_id=thing_id, revision=revision, transaction_id=transaction_id, created=timestamp) created = olddata and olddata['created'] @@ -514,11 +522,12 @@ class DBSiteStore(common.SiteStore): return web.SQLQuery.join(queries, delim) def versions(self, query): - what = 'thing.key, version.*' - where = 'version.thing_id = thing.id' + what = 'thing.key, version.revision, transaction.*' + where = 'version.thing_id = thing.id AND version.transaction_id = transaction.id' for c in query.conditions: key, value = c.key, c.value + assert key in ['key', 'type', 'author', 'ip', 'comment', 'created'] if key == 'key': key = 'thing_id' @@ -527,8 +536,10 @@ class DBSiteStore(common.SiteStore): key = 'thing.type' value = self.get_metadata(value).id elif key == 'author': - key = 'author_id' + key = 'transaction.author_id' value = self.get_metadata(value).id + else: + key = 'transaction.' + key where += web.reparam(' AND %s=$value' % key, locals()) @@ -540,7 +551,7 @@ class DBSiteStore(common.SiteStore): if config.query_timeout: self.db.query("SELECT set_config('statement_timeout', $query_timeout, false)", dict(query_timeout=config.query_timeout)) - result = self.db.select(['thing','version'], what=what, where=where, offset=query.offset, limit=query.limit, order=sort) + result = self.db.select(['thing','version', 'transaction'], what=what, where=where, offset=query.offset, limit=query.limit, order=sort) result = result.list() author_ids = list(set(r.author_id for r in result if r.author_id)) authors = self.get_metadata_list_from_ids(author_ids) @@ -549,7 +560,6 @@ class DBSiteStore(common.SiteStore): for r in result: r.author = r.author_id and authors[r.author_id].key - del r.thing_id return result def get_user_details(self, key): infogami/infobase/dbstore.py @@ -96,7 +96,7 @@ class Site: def new_key(self, type, kw=None): return self.store.new_key(type, kw or {}) - def write(self, query, timestamp=None, comment=None, machine_comment=None, ip=None, author=None, _internal=False): + def write(self, query, timestamp=None, comment=None, machine_comment=None, ip=None, author=None, action=None, _internal=False): timestamp = timestamp or datetime.datetime.utcnow() author = author or self.get_account_manager().get_user() @@ -104,7 +104,7 @@ class Site: items = p.process(query) items = (item for item in items if item) - result = self.store.save_many(items, timestamp, comment, machine_comment, ip, author and author.key) + result = self.store.save_many(items, timestamp, comment, machine_comment, ip, author and author.key, action=action) created = [r['key'] for r in result if r and r['revision'] == 1] @@ -142,7 +142,7 @@ class Site: self._fire_triggers([result]) return result - def save_many(self, query, timestamp=None, comment=None, machine_comment=None, ip=None, author=None): + def save_many(self, query, timestamp=None, comment=None, machine_comment=None, ip=None, author=None, action=None): timestamp = timestamp or datetime.datetime.utcnow() author = author or self.get_account_manager().get_user() ip = ip or web.ctx.get('ip', '127.0.0.1') @@ -152,7 +152,7 @@ class Site: items = query items = (p.process(item['key'], item) for item in items) items = (item for item in items if item) - result = self.store.save_many(items, timestamp, comment, machine_comment, ip, author and author.key) + result = self.store.save_many(items, timestamp, comment, machine_comment, ip, author and author.key, action=action) event_data = dict(comment=comment, machine_comment=machine_comment, query=query, result=result) self._fire_event("save_many", timestamp, ip, author and author.key, event_data) infogami/infobase/infobase.py @@ -25,15 +25,24 @@ $for name in ['key', 'type', 'latest_revision', 'last_modified', 'created']: $if multisite: create index thing_site_id_idx ON thing(site_id); -create table version ( +create table transaction ( id serial primary key, - thing_id int references thing, - revision int, + action varchar(256), author_id int references thing, ip inet, comment text, machine_comment text, - created timestamp default (current_timestamp at time zone 'utc'), + created timestamp default (current_timestamp at time zone 'utc') +); + +$for name in ['author_id', 'ip', 'created']: + create index transaction_${name}_idx ON transaction($name); + +create table version ( + id serial primary key, + thing_id int references thing, + revision int, + txn_id int references transaction, UNIQUE (thing_id, revision) ); @@ -49,9 +58,6 @@ RETURNS text AS 'select property.name FROM property, thing WHERE thing.type = property.type AND thing.id=$$1 AND property.id=$$2;' LANGUAGE SQL; -$for name in ['author_id', 'ip', 'created']: - create index version_${name}_idx ON version($name); - create table account ( $if multisite: site_id int references site, infogami/infobase/schema.sql @@ -120,9 +120,9 @@ class write: @jsonify def POST(self, sitename): site = get_site(sitename) - i = input('query', comment=None, machine_comment=None) + i = input('query', comment=None, machine_comment=None, action=None) query = from_json(i.query) - result = site.write(query, comment=i.comment, machine_comment=i.machine_comment) + result = site.write(query, comment=i.comment, machine_comment=i.machine_comment, action=i.action) return result class withkey: @@ -161,10 +161,10 @@ class save: class save_many: @jsonify def POST(self, sitename): - i = input('query', comment=None, machine_comment=None) + i = input('query', comment=None, machine_comment=None, action=None) data = from_json(i.query) site = get_site(sitename) - return site.save_many(data, comment=i.comment, machine_comment=i.machine_comment) + return site.save_many(data, comment=i.comment, machine_comment=i.machine_comment, action=i.action) class new_key: @jsonify infogami/infobase/server.py @@ -32,12 +32,37 @@ create table property ( ); """ +TRANSACTION_TABLE = """ +create table transaction ( + id serial primary key, + action varchar(256), + author_id int references thing, + ip inet, + comment text, + created timestamp default (current_timestamp at time zone 'utc') +); +""" + +TRANSACTION_INDEXES = """ +create index transaction_author_id_idx ON transaction(author_id); +create index transaction_ip_idx ON transaction(ip); +create index transaction_created_idx ON transaction(created); +""" + #@@ type to table prefix mappings. #@@ If there are any special tables in your schema, this should be updated. type2table = { } +# exclusive properties. They are some property names, which can't occur with any other type. +exclusive_properties = { + '/type/type': ['properties.name', 'properties.expected_type', 'properties.unique', 'properties.description', 'kind'], + '/type/user': ['displayname'], + '/type/usergroup': ['members'], + '/type/permission': ['readers', 'writers', 'admins'] +} + def get_table_prefix(type): """Returns table prefix for that type and a boolean flag to specify whether the table has more than one type. When the table as values only from a single type, then some of the queries can be optimized. @@ -60,9 +85,24 @@ def fix_property_keys(): keys_table = prefix + "_keys" keys = dict((r.key, r.id) for r in db.query('SELECT * FROM ' + keys_table)) newkeys = {} + + #@@ There is a chance that we may overwrite one update with another when new id is in the same range of old ids. + #@@ Example: + #@@ UPDATE datum_str SET key_id=4 FROM thing WHERE thing.id = property.type AND key_id=1; + #@@ UPDATE datum_str SET key_id=6 FROM thing WHERE thing.id = property.type AND key_id=4; + #@@ In the above example, the second query overwrites the result of first query. + #@@ Making id of property table more than max_id of datum_keys makes sure that this case never happen. + id1 = db.query('SELECT max(id) as x FROM ' + keys_table)[0].x + id2 = db.query('SELECT max(id) as x FROM property')[0].x + print >> web.debug, 'max ids', id1, id2 + if id1 > id2: + db.query("SELECT setval('property_id_seq', $id1)", vars=locals()) + for key in keys: newkeys[key] = db.insert('property', type=type_id, name=key) + total_updated = {} + for d in ['str', 'int', 'float', 'boolean', 'ref']: table = prefix + '_' + d print >> web.debug, 'fixing', type, table @@ -70,9 +110,16 @@ def fix_property_keys(): old_key_id = keys[key] new_key_id = newkeys[key] if multiple_types: - db.query('UPDATE %s SET key_id=$new_key_id FROM thing WHERE thing.id = %s.thing_id AND thing.type=$type_id AND key_id=$old_key_id' % (table, table), vars=locals()) + updated = db.query('UPDATE %s SET key_id=$new_key_id FROM thing WHERE thing.id = %s.thing_id AND thing.type=$type_id AND key_id=$old_key_id' % (table, table), vars=locals()) else: - db.update(table, key_id=new_key_id, where='key_id=$old_key_id', vars=locals()) + updated = db.update(table, key_id=new_key_id, where='key_id=$old_key_id', vars=locals()) + + total_updated[key] = total_updated.get(key, 0) + updated + print >> web.debug, 'updated', updated + + unused = [k for k in total_updated if total_updated[k] == 0] + print >> web.debug, 'unused', unused + db.delete('property', where=web.sqlors('id =', [newkeys[k] for k in unused])) primitive = ['/type/key', '/type/int', '/type/float', '/type/boolean', '/type/string', '/type/datetime'] # add embeddable types too @@ -112,6 +159,21 @@ def process_keys(table): for r in result: print r + +def fix_version_table(): + """Add transaction table and move columns from version table to transaction table.""" + filename = '/tmp/version.%d.txt' % os.getpid() + new_filename = filename + ".new" + db.query('copy version to $filename', vars=locals()); + cmd = """awk -F'\t' 'BEGIN {OFS="\t"}{if ($3 == 1) action = "create"; else action="update"; print $1,action, $4,$5,$6,$8; }' < %s > %s""" % (filename, new_filename) + os.system(cmd) + db.query(TRANSACTION_TABLE) + db.query("copy transaction from $new_filename", vars=locals()) + db.query("SELECT setval('transaction_id_seq', (SELECT max(id) FROM transaction))") + db.query('ALTER TABLE version add column transaction_id int references transaction') + db.query('UPDATE version set transaction_id=id') + db.query(TRANSACTION_INDEXES) + os.system('rm %s %s' % (filename, new_filename)) def main(): parse_args() @@ -121,11 +183,11 @@ def main(): t = db.transaction() db.query(PROPERTY_TABLE) + fix_version_table() drop_key_id_foreign_key() fix_property_keys() add_key_id_foreign_key() - t.commit() if __name__ == "__main__": migration/migrate-0.4-0.5.py 79138c8b4f9361d12bfa7cc51c779917a841f473 Anand Chitipothu anandology@gmail.com http://github.com/infogami/infogami/commit/b2cab7a231cbeffa873b46032d687cf1eae7f9a0 b2cab7a231cbeffa873b46032d687cf1eae7f9a0 2009-02-24T21:30:43-08:00 2009-02-24T21:30:43-08:00 support tracking transactions in Infobase. 95553c12eae14aa5171a3a05b258bfdf105ba429 Anand Chitipothu anandology@gmail.com