Support turning Firstbits support on and off.

Abe/DataStore.py

@@ -1147,6 +1147,46 @@ def initialize(store):
         store.save_config()
         store.commit()
 
+    def get_lock(store):
+        if store.version_below('Abe26'):
+            return None
+        conn = store.connect()
+        cur = conn.cursor()
+        cur.execute("UPDATE abe_lock SET pid = %d WHERE lock_id = 1"
+                    % (os.getpid(),))
+        if cur.rowcount != 1:
+            raise Exception("unexpected rowcount")
+        cur.close()
+
+        # Check whether database supports concurrent updates.  Where it
+        # doesn't (SQLite) we get exclusive access automatically.
+        try:
+            import random
+            letters = "".join([chr(random.randint(65, 90)) for x in xrange(10)])
+            store.sql("""
+                INSERT INTO configvar (configvar_name, configvar_value)
+                VALUES (?, ?)""",
+                      ("upgrade-lock-" + letters, 'x'))
+        except:
+            store.release_lock(conn)
+            conn = None
+
+        store.rollback()
+
+        # XXX Should reread config.
+
+        return conn
+
+    def release_lock(store, conn):
+        if conn:
+            conn.rollback()
+            conn.close()
+
+    def version_below(store, vers):
+        sv = store.config['schema_version'].replace('Abe', '')
+        vers = vers.replace('Abe', '')
+        return float(sv) < float(vers)
+
     def configure(store):
         store.config = {}
 

Abe/abe.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright(C) 2011 by John Tobey <John.Tobey@gmail.com>
+# Copyright(C) 2011,2012 by John Tobey <John.Tobey@gmail.com>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as

Abe/firstbits.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# Copyright(C) 2011,2012 by John Tobey <John.Tobey@gmail.com>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <http://www.gnu.org/licenses/agpl.html>.
+
+"""Reconfigure an Abe instance to use or not use Firstbits."""
+
+def populate_firstbits(store):
+    blocks, fbs = 0, 0
+    log_incr = 1000
+
+    for addr_vers, block_id in store.selectall("""
+        SELECT c.chain_address_version,
+               cc.block_id
+          FROM chain c
+          JOIN chain_candidate cc ON (c.chain_id = cc.chain_id)
+         WHERE cc.block_height IS NOT NULL
+         ORDER BY cc.chain_id, cc.block_height"""):
+        fbs += store.do_vers_firstbits(addr_vers, int(block_id))
+        blocks += 1
+        if blocks % log_incr == 0:
+            store.commit()
+            store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
+
+    if blocks % log_incr > 0:
+        store.commit()
+        store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
+
+def create_firstbits(store):
+    store.log.info("Creating firstbits table.")
+    store.ddl(
+        """CREATE TABLE abe_firstbits (
+            pubkey_id       NUMERIC(26) NOT NULL,
+            block_id        NUMERIC(14) NOT NULL,
+            address_version BIT VARYING(80) NOT NULL,
+            firstbits       VARCHAR(50) NOT NULL,
+            PRIMARY KEY (address_version, pubkey_id, block_id),
+            FOREIGN KEY (pubkey_id) REFERENCES pubkey (pubkey_id),
+            FOREIGN KEY (block_id) REFERENCES block (block_id)
+        )""")
+    store.ddl(
+        """CREATE INDEX x_abe_firstbits
+            ON abe_firstbits (address_version, firstbits)""")
+
+def drop_firstbits(store):
+    store.log.info("Dropping firstbits table.")
+    store.ddl("DROP TABLE abe_firstbits")
+
+def reconfigure(store, args):
+    have = store.config['use_firstbits'] == "true"
+    want = args.use_firstbits
+    if have == want:
+        return
+    lock = store.get_lock()
+    try:
+        # XXX Should temporarily store a new schema_version.
+        if want:
+            create_firstbits(store)
+            populate_firstbits(store)
+            store.config['use_firstbits'] = "true"
+        else:
+            drop_firstbits(store)
+            store.config['use_firstbits'] = "false"
+
+        store.save_configvar("use_firstbits")
+        store.commit()
+
+    finally:
+        store.release_lock(lock)

Abe/reconfigure.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright(C) 2012 by John Tobey <John.Tobey@gmail.com>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <http://www.gnu.org/licenses/agpl.html>.
+
+"""Reconfigure an Abe instance to use or not use Firstbits."""
+
+import sys
+import logging
+
+import DataStore
+import readconf
+import firstbits
+
+def main(argv):
+    conf = {
+        "debug":                    None,
+        "logging":                  None,
+        }
+    conf.update(DataStore.CONFIG_DEFAULTS)
+
+    args, argv = readconf.parse_argv(argv, conf,
+                                     strict=False)
+    if argv and argv[0] in ('-h', '--help'):
+        print ("""Usage: python -m Abe.reconfigure [-h] [--config=FILE] [--CONFIGVAR=VALUE]...
+
+Apply configuration changes to an existing Abe database, if possible.
+
+  --help                    Show this help message and exit.
+  --config FILE             Read options from FILE.
+  --use-firstbits {true|false}
+                            Turn Firstbits support on or off.
+
+All configuration variables may be given as command arguments.""")
+        return 0
+
+    logging.basicConfig(
+        stream=sys.stdout,
+        level=logging.DEBUG,
+        format="%(message)s")
+    if args.logging is not None:
+        import logging.config as logging_config
+        logging_config.dictConfig(args.logging)
+
+    store = DataStore.new(args)
+    firstbits.reconfigure(store, args)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

Abe/upgrade.py

@@ -44,48 +44,12 @@ def run_upgrades_locked(store, upgrades):
             store.config['schema_version'] = sv
 
 def run_upgrades(store, upgrades):
-    lock = get_lock(store)
+    """Guard against concurrent upgrades."""
+    lock = store.get_lock()
     try:
         run_upgrades_locked(store, upgrades)
     finally:
-        release_lock(lock)
-
-def get_lock(store):
-    if version_below(store, 'Abe26'):
-        return None
-    conn = store.connect()
-    cur = conn.cursor()
-    cur.execute("UPDATE abe_lock SET pid = %d WHERE lock_id = 1"
-                % (os.getpid(),))
-    if cur.rowcount != 1:
-        raise Exception("unexpected rowcount")
-    cur.close()
-
-    # Check whether database supports concurrent updates.  Where it
-    # doesn't (SQLite) we get exclusive access automatically.
-    try:
-        import random
-        letters = "".join([chr(random.randint(65, 90)) for x in xrange(10)])
-        store.sql("""
-            INSERT INTO configvar (configvar_name, configvar_value)
-            VALUES (?, ?)""",
-                  ("upgrade-lock-" + letters, 'x'))
-    except:
-        release_lock(conn)
-        conn = None
-
-    store.rollback()
-    return conn
-
-def release_lock(conn):
-    if conn:
-        conn.rollback()
-        conn.close()
-
-def version_below(store, vers):
-    sv = store.config['schema_version'].replace('Abe', '')
-    vers = vers.replace('Abe', '')
-    return float(sv) < float(vers)
+        store.release_lock(lock)
 
 def add_block_value_in(store):
     store.sql("ALTER TABLE block ADD block_value_in NUMERIC(30)")
@@ -848,32 +812,9 @@ def add_fk_search_block_id(store):
     add_constraint(store, "block", "fk1_search_block_id",
                    "FOREIGN KEY (search_block_id) REFERENCES block (block_id)")
 
-def populate_firstbits(store):
-    if store.config['use_firstbits'] != "true":
-        return
-
-    blocks, fbs = 0, 0
-    log_incr = 1000
-
-    for addr_vers, block_id in store.selectall("""
-        SELECT c.chain_address_version,
-               cc.block_id
-          FROM chain c
-          JOIN chain_candidate cc ON (c.chain_id = cc.chain_id)
-         WHERE cc.block_height IS NOT NULL
-         ORDER BY cc.chain_id, cc.block_height"""):
-        fbs += store.do_vers_firstbits(addr_vers, int(block_id))
-        blocks += 1
-        if blocks % log_incr == 0:
-            store.commit()
-            store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
-
-    if blocks % log_incr > 0:
-        store.commit()
-        store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
-
 def create_firstbits(store):
     flag = store.config.get('use_firstbits')
+
     if flag is None:
         if store.args.use_firstbits is None:
             store.log.info("use_firstbits not found, defaulting to false.")
@@ -883,23 +824,15 @@ def create_firstbits(store):
         flag = "true" if store.args.use_firstbits else "false"
         store.config['use_firstbits'] = flag
         store.save_configvar("use_firstbits")
-    if flag == "false":
-        return
 
-    store.log.info("Creating firstbits table.")
-    store.ddl(
-        """CREATE TABLE abe_firstbits (
-            pubkey_id       NUMERIC(26) NOT NULL,
-            block_id        NUMERIC(14) NOT NULL,
-            address_version BIT VARYING(80) NOT NULL,
-            firstbits       VARCHAR(50) NOT NULL,
-            PRIMARY KEY (address_version, pubkey_id, block_id),
-            FOREIGN KEY (pubkey_id) REFERENCES pubkey (pubkey_id),
-            FOREIGN KEY (block_id) REFERENCES block (block_id)
-        )""")
-    store.ddl(
-        """CREATE INDEX x_abe_firstbits
-            ON abe_firstbits (address_version, firstbits)""")
+    if flag == "true":
+        import firstbits
+        firstbits.create_firstbits(store)
+
+def populate_firstbits(store):
+    if store.config['use_firstbits'] == "true":
+        import firstbits
+        firstbits.populate_firstbits(store)
 
 upgrades = [
     ('6',    add_block_value_in),

README-FIRSTBITS.txt

@@ -16,13 +16,12 @@ enable it, add "use-firstbits" to the configuration *before* first
 running a version that supports it.
 
 If you run without use-firstbits, Abe will default it to false and
-will never create the table.  I'd like to have a script that turns
-firstbits on and off, but for now the best you can do is to stop Abe
-and run these (UNTESTED) SQL commands, once you have configured
-use-firstbits=true:
+will never create the table.  The Abe.reconfigure module turns
+firstbits on and off once you have upgraded Abe's schema.  Stop all
+processes using the database, change the use-firstbits setting in
+abe.conf, and run:
 
-DELETE FROM configvar WHERE configvar_name = 'use_firstbits' AND configvar_value <> 'true';
-UPDATE configvar SET configvar_value = 'Abe29.3' WHERE configvar_name = 'schema_version' AND configvar_value = 'Abe30';
+    python -m Abe.reconfigure --config abe.conf
 
 I have tried a few dozen addresses, and they match firstbits.com.
 Please report issues in the forum thread