Permalink
Browse files

Support turning Firstbits support on and off.

  • Loading branch information...
John Tobey John Tobey
John Tobey authored and John Tobey committed Aug 11, 2012
1 parent b0b843c commit 7cf9c9ec4798f5907c9f37f4b6d977496af372fa
Showing with 201 additions and 86 deletions.
  1. +40 −0 Abe/DataStore.py
  2. +1 −1 Abe/abe.py
  3. +81 −0 Abe/firstbits.py
  4. +62 −0 Abe/reconfigure.py
  5. +12 −79 Abe/upgrade.py
  6. +5 −6 README-FIRSTBITS.txt
View
@@ -1147,6 +1147,46 @@ def initialize(store):
store.save_config()
store.commit()
+ def get_lock(store):
+ if store.version_below('Abe26'):
+ return None
+ conn = store.connect()
+ cur = conn.cursor()
+ cur.execute("UPDATE abe_lock SET pid = %d WHERE lock_id = 1"
+ % (os.getpid(),))
+ if cur.rowcount != 1:
+ raise Exception("unexpected rowcount")
+ cur.close()
+
+ # Check whether database supports concurrent updates. Where it
+ # doesn't (SQLite) we get exclusive access automatically.
+ try:
+ import random
+ letters = "".join([chr(random.randint(65, 90)) for x in xrange(10)])
+ store.sql("""
+ INSERT INTO configvar (configvar_name, configvar_value)
+ VALUES (?, ?)""",
+ ("upgrade-lock-" + letters, 'x'))
+ except:
+ store.release_lock(conn)
+ conn = None
+
+ store.rollback()
+
+ # XXX Should reread config.
+
+ return conn
+
+ def release_lock(store, conn):
+ if conn:
+ conn.rollback()
+ conn.close()
+
+ def version_below(store, vers):
+ sv = store.config['schema_version'].replace('Abe', '')
+ vers = vers.replace('Abe', '')
+ return float(sv) < float(vers)
+
def configure(store):
store.config = {}
View
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-# Copyright(C) 2011 by John Tobey <John.Tobey@gmail.com>
+# Copyright(C) 2011,2012 by John Tobey <John.Tobey@gmail.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
View
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# Copyright(C) 2011,2012 by John Tobey <John.Tobey@gmail.com>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program. If not, see
+# <http://www.gnu.org/licenses/agpl.html>.
+
+"""Reconfigure an Abe instance to use or not use Firstbits."""
+
+def populate_firstbits(store):
+ blocks, fbs = 0, 0
+ log_incr = 1000
+
+ for addr_vers, block_id in store.selectall("""
+ SELECT c.chain_address_version,
+ cc.block_id
+ FROM chain c
+ JOIN chain_candidate cc ON (c.chain_id = cc.chain_id)
+ WHERE cc.block_height IS NOT NULL
+ ORDER BY cc.chain_id, cc.block_height"""):
+ fbs += store.do_vers_firstbits(addr_vers, int(block_id))
+ blocks += 1
+ if blocks % log_incr == 0:
+ store.commit()
+ store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
+
+ if blocks % log_incr > 0:
+ store.commit()
+ store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
+
+def create_firstbits(store):
+ store.log.info("Creating firstbits table.")
+ store.ddl(
+ """CREATE TABLE abe_firstbits (
+ pubkey_id NUMERIC(26) NOT NULL,
+ block_id NUMERIC(14) NOT NULL,
+ address_version BIT VARYING(80) NOT NULL,
+ firstbits VARCHAR(50) NOT NULL,
+ PRIMARY KEY (address_version, pubkey_id, block_id),
+ FOREIGN KEY (pubkey_id) REFERENCES pubkey (pubkey_id),
+ FOREIGN KEY (block_id) REFERENCES block (block_id)
+ )""")
+ store.ddl(
+ """CREATE INDEX x_abe_firstbits
+ ON abe_firstbits (address_version, firstbits)""")
+
+def drop_firstbits(store):
+ store.log.info("Dropping firstbits table.")
+ store.ddl("DROP TABLE abe_firstbits")
+
+def reconfigure(store, args):
+ have = store.config['use_firstbits'] == "true"
+ want = args.use_firstbits
+ if have == want:
+ return
+ lock = store.get_lock()
+ try:
+ # XXX Should temporarily store a new schema_version.
+ if want:
+ create_firstbits(store)
+ populate_firstbits(store)
+ store.config['use_firstbits'] = "true"
+ else:
+ drop_firstbits(store)
+ store.config['use_firstbits'] = "false"
+
+ store.save_configvar("use_firstbits")
+ store.commit()
+
+ finally:
+ store.release_lock(lock)
View
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright(C) 2012 by John Tobey <John.Tobey@gmail.com>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program. If not, see
+# <http://www.gnu.org/licenses/agpl.html>.
+
+"""Reconfigure an Abe instance to use or not use Firstbits."""
+
+import sys
+import logging
+
+import DataStore
+import readconf
+import firstbits
+
+def main(argv):
+ conf = {
+ "debug": None,
+ "logging": None,
+ }
+ conf.update(DataStore.CONFIG_DEFAULTS)
+
+ args, argv = readconf.parse_argv(argv, conf,
+ strict=False)
+ if argv and argv[0] in ('-h', '--help'):
+ print ("""Usage: python -m Abe.reconfigure [-h] [--config=FILE] [--CONFIGVAR=VALUE]...
+
+Apply configuration changes to an existing Abe database, if possible.
+
+ --help Show this help message and exit.
+ --config FILE Read options from FILE.
+ --use-firstbits {true|false}
+ Turn Firstbits support on or off.
+
+All configuration variables may be given as command arguments.""")
+ return 0
+
+ logging.basicConfig(
+ stream=sys.stdout,
+ level=logging.DEBUG,
+ format="%(message)s")
+ if args.logging is not None:
+ import logging.config as logging_config
+ logging_config.dictConfig(args.logging)
+
+ store = DataStore.new(args)
+ firstbits.reconfigure(store, args)
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
View
@@ -44,48 +44,12 @@ def run_upgrades_locked(store, upgrades):
store.config['schema_version'] = sv
def run_upgrades(store, upgrades):
- lock = get_lock(store)
+ """Guard against concurrent upgrades."""
+ lock = store.get_lock()
try:
run_upgrades_locked(store, upgrades)
finally:
- release_lock(lock)
-
-def get_lock(store):
- if version_below(store, 'Abe26'):
- return None
- conn = store.connect()
- cur = conn.cursor()
- cur.execute("UPDATE abe_lock SET pid = %d WHERE lock_id = 1"
- % (os.getpid(),))
- if cur.rowcount != 1:
- raise Exception("unexpected rowcount")
- cur.close()
-
- # Check whether database supports concurrent updates. Where it
- # doesn't (SQLite) we get exclusive access automatically.
- try:
- import random
- letters = "".join([chr(random.randint(65, 90)) for x in xrange(10)])
- store.sql("""
- INSERT INTO configvar (configvar_name, configvar_value)
- VALUES (?, ?)""",
- ("upgrade-lock-" + letters, 'x'))
- except:
- release_lock(conn)
- conn = None
-
- store.rollback()
- return conn
-
-def release_lock(conn):
- if conn:
- conn.rollback()
- conn.close()
-
-def version_below(store, vers):
- sv = store.config['schema_version'].replace('Abe', '')
- vers = vers.replace('Abe', '')
- return float(sv) < float(vers)
+ store.release_lock(lock)
def add_block_value_in(store):
store.sql("ALTER TABLE block ADD block_value_in NUMERIC(30)")
@@ -848,32 +812,9 @@ def add_fk_search_block_id(store):
add_constraint(store, "block", "fk1_search_block_id",
"FOREIGN KEY (search_block_id) REFERENCES block (block_id)")
-def populate_firstbits(store):
- if store.config['use_firstbits'] != "true":
- return
-
- blocks, fbs = 0, 0
- log_incr = 1000
-
- for addr_vers, block_id in store.selectall("""
- SELECT c.chain_address_version,
- cc.block_id
- FROM chain c
- JOIN chain_candidate cc ON (c.chain_id = cc.chain_id)
- WHERE cc.block_height IS NOT NULL
- ORDER BY cc.chain_id, cc.block_height"""):
- fbs += store.do_vers_firstbits(addr_vers, int(block_id))
- blocks += 1
- if blocks % log_incr == 0:
- store.commit()
- store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
-
- if blocks % log_incr > 0:
- store.commit()
- store.log.info("%d firstbits in %d blocks" % (fbs, blocks))
-
def create_firstbits(store):
flag = store.config.get('use_firstbits')
+
if flag is None:
if store.args.use_firstbits is None:
store.log.info("use_firstbits not found, defaulting to false.")
@@ -883,23 +824,15 @@ def create_firstbits(store):
flag = "true" if store.args.use_firstbits else "false"
store.config['use_firstbits'] = flag
store.save_configvar("use_firstbits")
- if flag == "false":
- return
- store.log.info("Creating firstbits table.")
- store.ddl(
- """CREATE TABLE abe_firstbits (
- pubkey_id NUMERIC(26) NOT NULL,
- block_id NUMERIC(14) NOT NULL,
- address_version BIT VARYING(80) NOT NULL,
- firstbits VARCHAR(50) NOT NULL,
- PRIMARY KEY (address_version, pubkey_id, block_id),
- FOREIGN KEY (pubkey_id) REFERENCES pubkey (pubkey_id),
- FOREIGN KEY (block_id) REFERENCES block (block_id)
- )""")
- store.ddl(
- """CREATE INDEX x_abe_firstbits
- ON abe_firstbits (address_version, firstbits)""")
+ if flag == "true":
+ import firstbits
+ firstbits.create_firstbits(store)
+
+def populate_firstbits(store):
+ if store.config['use_firstbits'] == "true":
+ import firstbits
+ firstbits.populate_firstbits(store)
upgrades = [
('6', add_block_value_in),
View
@@ -16,13 +16,12 @@ enable it, add "use-firstbits" to the configuration *before* first
running a version that supports it.
If you run without use-firstbits, Abe will default it to false and
-will never create the table. I'd like to have a script that turns
-firstbits on and off, but for now the best you can do is to stop Abe
-and run these (UNTESTED) SQL commands, once you have configured
-use-firstbits=true:
+will never create the table. The Abe.reconfigure module turns
+firstbits on and off once you have upgraded Abe's schema. Stop all
+processes using the database, change the use-firstbits setting in
+abe.conf, and run:
-DELETE FROM configvar WHERE configvar_name = 'use_firstbits' AND configvar_value <> 'true';
-UPDATE configvar SET configvar_value = 'Abe29.3' WHERE configvar_name = 'schema_version' AND configvar_value = 'Abe30';
+ python -m Abe.reconfigure --config abe.conf
I have tried a few dozen addresses, and they match firstbits.com.
Please report issues in the forum thread

0 comments on commit 7cf9c9e

Please sign in to comment.