Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Rename schema names in mbslave-import.py

  • Loading branch information...
commit 736a075e5021c88efbafd30dfe79af2cab16cf97 1 parent f2fba1c
@lalinsky authored
View
37 mbslave-import.py
@@ -1,49 +1,40 @@
#!/usr/bin/env python
-import ConfigParser
-import psycopg2
import tarfile
import sys
import os
+from mbslave import Config, connect_db, parse_name, check_table_exists, fqn
-def load_tar(filename, db, schema, ignored_tables):
+def load_tar(filename, db, config, ignored_tables):
print "Importing data from", filename
tar = tarfile.open(filename, 'r:bz2')
cursor = db.cursor()
for member in tar:
if not member.name.startswith('mbdump/'):
continue
- table = member.name.split('/')[1].replace('_sanitised', '')
- fulltable = schema + "." + table
+ name = member.name.split('/')[1].replace('_sanitised', '')
+ schema, table = parse_name(config, name)
+ fulltable = fqn(schema, table)
if table in ignored_tables:
- print " - Ignoring", fulltable
+ print " - Ignoring", name
+ continue
+ if not check_table_exists(db, schema, table):
+ print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
- print " - Skipping", fulltable, "(already contains data)"
+ print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
- print " - Loading", fulltable
+ print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
-config = ConfigParser.RawConfigParser()
-config.read(os.path.dirname(__file__) + '/mbslave.conf')
-
-opts = {}
-opts['database'] = config.get('DATABASE', 'name')
-opts['user'] = config.get('DATABASE', 'user')
-if config.has_option('DATABASE', 'password'):
- opts['password'] = config.get('DATABASE', 'password')
-if config.has_option('DATABASE', 'host'):
- opts['host'] = config.get('DATABASE', 'host')
-if config.has_option('DATABASE', 'port'):
- opts['port'] = config.get('DATABASE', 'port')
-db = psycopg2.connect(**opts)
+config = Config(os.path.dirname(__file__) + '/mbslave.conf')
+db = connect_db(config)
-schema = config.get('DATABASE', 'schema')
ignored_tables = set(config.get('TABLES', 'ignore').split(','))
for filename in sys.argv[1:]:
- load_tar(filename, db, schema, ignored_tables)
+ load_tar(filename, db, config, ignored_tables)
View
8 mbslave-psql.py
@@ -1,15 +1,14 @@
#!/usr/bin/env python
-import ConfigParser
import os
from optparse import OptionParser
+from mbslave import Config, connect_db
parser = OptionParser()
parser.add_option("-S", "--no-schema", action="store_true", dest="public", default=False, help="don't configure the default schema")
options, args = parser.parse_args()
-config = ConfigParser.RawConfigParser()
-config.read(os.path.dirname(__file__) + '/mbslave.conf')
+config = Config(os.path.dirname(__file__) + '/mbslave.conf')
args = ['psql']
args.append('-U')
@@ -23,7 +22,8 @@
args.append(config.get('DATABASE', 'name'))
if not options.public:
- os.environ['PGOPTIONS'] = '-c search_path=%s' % config.get('DATABASE', 'schema')
+ schema = config.schema.name('musicbrainz')
+ os.environ['PGOPTIONS'] = '-c search_path=%s' % schema
if config.has_option('DATABASE', 'password'):
os.environ['PGPASSWORD'] = config.get('DATABASE', 'password')
os.execvp("psql", args)
View
1  mbslave-sync.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python
-import ConfigParser
import psycopg2
import tarfile
import sys
View
7 mbslave.conf.default
@@ -4,7 +4,6 @@ port=5432
name=musicbrainz
user=musicbrainz
#password=
-schema=musicbrainz
[MUSICBRAINZ]
base_url=http://ftp.musicbrainz.org/pub/musicbrainz/data/replication/
@@ -13,6 +12,12 @@ base_url=http://ftp.musicbrainz.org/pub/musicbrainz/data/replication/
ignore=
#ignore=tracklist_index
+[schemas]
+musicbrainz=musicbrainz
+statistics=musicbrainz
+reports=musicbrainz
+cover_art_archive=musicbrainz
+
[solr]
enabled=no
url=http://localhost:8983/solr/musicbrainz/
View
27 mbslave/__init__.py
@@ -2,8 +2,29 @@
from mbslave.config import Config
from mbslave.replication import ReplicationHook
+
def connect_db(cfg):
- db = psycopg2.connect(**cfg.make_psql_args())
- db.cursor().execute("SET search_path TO %s", (cfg.database.schema,))
- return db
+ return psycopg2.connect(**cfg.make_psql_args())
+
+
+def parse_name(config, table):
+ if '.' in table:
+ schema, table = table.split('.', 1)
+ else:
+ schema = 'musicbrainz'
+ schema = config.schema.name(schema.strip('"'))
+ table = table.strip('"')
+ return schema, table
+
+
+def fqn(schema, table):
+ return '%s.%s' % (schema, table)
+
+
+def check_table_exists(db, schema, table):
+ cursor = db.cursor()
+ cursor.execute("SELECT 1 FROM pg_tables WHERE schemaname=%s AND tablename=%s", (schema, table))
+ if not cursor.fetchone():
+ return False
+ return True
View
17 mbslave/config.py
@@ -41,6 +41,19 @@ def parse(self, parser, section):
self.status_file = parser.get(section, 'status_file')
+class SchemasConfig(object):
+
+ def __init__(self):
+ self.mapping = {}
+
+ def name(self, name):
+ return self.mapping.get(name, name)
+
+ def parse(self, parser, section):
+ for name, value in parser.items(section):
+ self.mapping[name] = value
+
+
class Config(object):
def __init__(self, path):
@@ -50,13 +63,15 @@ def __init__(self, path):
self.get = self.cfg.get
self.has_option = self.cfg.has_option
self.database = ConfigSection()
- self.database.schema = self.cfg.get('DATABASE', 'schema')
self.solr = SolrConfig()
if self.cfg.has_section('solr'):
self.solr.parse(self.cfg, 'solr')
self.monitoring = MonitoringConfig()
if self.cfg.has_section('monitoring'):
self.monitoring.parse(self.cfg, 'monitoring')
+ self.schema = SchemasConfig()
+ if self.cfg.has_section('schemas'):
+ self.schema.parse(self.cfg, 'schemas')
def make_psql_args(self):
opts = {}
Please sign in to comment.
Something went wrong with that request. Please try again.