Merge branch 'bb_docker'

consul_config.py.ctmpl

@@ -25,6 +25,7 @@ MUSICBRAINZ_CLIENT_SECRET = '''{{template "KEY" "musicbrainz/client_secret"}}'''
 {{if service "pgbouncer-master"}}
 {{with index (service "pgbouncer-master") 0}}
 SQLALCHEMY_DATABASE_URI = "postgresql://critiquebrainz:critiquebrainz@{{.Address}}:{{.Port}}/critiquebrainz_db"
+BB_DATABASE_URI = "postgresql://{{template "KEY" "bookbrainz/dbuser"}}:{{template "KEY" "bookbrainz/dbpass"}}@{{.Address}}:{{.Port}}/{{template "KEY" "bookbrainz/dbname"}}"
 {{end}}
 {{end}}
 

critiquebrainz/frontend/__init__.py

@@ -75,6 +75,10 @@ def create_app(debug=None, config_path=None):
 
     add_robots(app)
 
+    # BookBrainz Database
+    import critiquebrainz.frontend.external.bookbrainz_db as bookbrainz_db 
+    bookbrainz_db.init_db_engine(app.config.get("BB_DATABASE_URI"))
+
     # MusicBrainz Database
     from brainzutils import musicbrainz_db
     musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI"))

critiquebrainz/frontend/external/bookbrainz_db/__init__.py

@@ -0,0 +1,17 @@
+from sqlalchemy import create_engine
+from sqlalchemy.pool import NullPool
+
+DEFAULT_CACHE_EXPIRATION = 12 * 60 * 60  # seconds (12 hours)
+bb_engine = None
+
+
+def init_db_engine(connect_str):
+    global bb_engine
+    bb_engine = create_engine(connect_str, poolclass=NullPool)
+
+
+def run_sql_script(sql_file_path):
+    with open(sql_file_path) as sql:
+        connection = bb_engine.connect()
+        connection.execute(sql.read())
+        connection.close()

critiquebrainz/ws/__init__.py

@@ -62,6 +62,10 @@ def create_app(debug=None, config_path=None):
     from critiquebrainz import db as critiquebrainz_db
     critiquebrainz_db.init_db_engine(app.config.get("SQLALCHEMY_DATABASE_URI"))
 
+    # BookBrainz Database
+    import critiquebrainz.frontend.external.bookbrainz_db as bookbrainz_db 
+    bookbrainz_db.init_db_engine(app.config.get("BB_DATABASE_URI"))
+
     # MusicBrainz Database
     from brainzutils import musicbrainz_db
     musicbrainz_db.init_db_engine(app.config.get("MB_DATABASE_URI"))

default_config.py

@@ -7,6 +7,9 @@
 SQLALCHEMY_DATABASE_URI = "postgresql://critiquebrainz:critiquebrainz@db:5432/critiquebrainz"
 SQLALCHEMY_TRACK_MODIFICATIONS = False
 
+# BookBrainz Database
+BB_DATABASE_URI = "postgresql://bookbrainz:bookbrainz@db:5432/bookbrainz"
+
 # MusicBrainz Database
 MB_DATABASE_URI = "postgresql://musicbrainz:musicbrainz@musicbrainz_db:5432/musicbrainz_db"
 

docker/docker-compose.dev.yml

@@ -3,18 +3,18 @@ version: "3.4"
 
 volumes:
   cb_home:
+  cb_postgres:
 
 services:
 
   db:
     image: postgres:12.3
+    user: postgres
     environment:
-      POSTGRES_USER: critiquebrainz
-      POSTGRES_PASSWORD: critiquebrainz
-      POSTGRES_DB: critiquebrainz
-      PGDATA: /var/lib/postgresql/data/pgdata
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
     volumes:
-      - ../data/pgdata:/var/lib/postgresql/data/pgdata:z
+      - cb_postgres:/var/lib/postgresql/data:z
       - ./pg_custom:/docker-entrypoint-initdb.d/
     ports:
       - "127.0.0.1:15432:5432"

docker/docker-compose.test.yml

@@ -4,10 +4,10 @@ services:
 
   db:
     image: postgres:12.3
+    user: postgres
     environment:
-      POSTGRES_USER: critiquebrainz
-      POSTGRES_PASSWORD: critiquebrainz
-      POSTGRES_DB: critiquebrainz
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
     command: postgres -F
     volumes:
       - ./pg_custom:/docker-entrypoint-initdb.d/

docker/pg_custom/create-bb-db.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE USER bookbrainz WITH SUPERUSER PASSWORD 'bookbrainz';
+    CREATE DATABASE bookbrainz;
+    GRANT ALL PRIVILEGES ON DATABASE bookbrainz TO bookbrainz;
+EOSQL

docker/pg_custom/create-cb-db.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE USER critiquebrainz WITH SUPERUSER PASSWORD 'critiquebrainz';
+    CREATE DATABASE critiquebrainz;
+    GRANT ALL PRIVILEGES ON DATABASE critiquebrainz TO critiquebrainz;
+EOSQL

docs/intro.rst

@@ -120,6 +120,12 @@ a single command::
    the ``data/mbdata`` directory by accident. Also make sure that you have about 25GB of free
    space to keep the MusicBrainz data.
 
+A BookBrainz database is also required for running CritiqueBrainz.
+You can import the database dump by downloading and importing the data in
+a single command::
+
+    $ ./develop.sh run --rm critiquebrainz bash scripts/download-import-bookbrainz-dump.sh
+
 Next, initialize the CritiqueBrainz database::
 
    $ ./develop.sh run --rm critiquebrainz python3 manage.py init_db

scripts/download-import-bookbrainz-dump.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+DB_HOSTNAME=db
+DB_PORT=5432
+DB_USER=bookbrainz
+DB_NAME=bookbrainz
+DB_PASSWORD=bookbrainz
+
+DUMP_DIR=/tmp/bookbrainz-dumps
+DUMP_FILE=$DUMP_DIR/latest.sql.bz2
+
+if [ -f $DUMP_FILE ]; then
+    echo "A bookbrainz dump file, already exists. Using that to import."
+    echo "To force a re-download of the data, please remove $DUMP_FILE"
+else
+    mkdir -p $DUMP_DIR
+    curl -o $DUMP_FILE ftp://ftp.musicbrainz.org/pub/musicbrainz/bookbrainz/latest.sql.bz2
+    if [ $? -ne 0 ]
+    then
+        echo "Downloading the bookbrainz data dump failed."
+        exit $?
+    fi
+fi
+
+bzcat $DUMP_FILE | PGPASSWORD=$DB_PASSWORD psql -h $DB_HOSTNAME -p $DB_PORT -U $DB_USER -d $DB_NAME
+if [ $? -ne 0 ]
+then
+    echo "Importing the bookbrainz database failed."
+    exit $?
+fi
+
+# Clean up the dump file if it imported correctly.
+rm -f $DUMP_FILE

test.sh

@@ -37,6 +37,10 @@ function setup {
                 run --rm critiquebrainz dockerize \
                   -wait tcp://db:5432 -timeout 60s \
                 bash -c "python3 manage.py init_db"
+
+    docker-compose -f ${COMPOSE_FILE_LOC} \
+                   -p ${COMPOSE_PROJECT_NAME} \
+                   run --rm critiquebrainz bash scripts/download-import-bookbrainz-dump.sh
 }
 
 function is_db_running {

test_config.py

@@ -7,3 +7,6 @@
 
 # MusicBrainz Database
 MB_DATABASE_URI = "postgresql://musicbrainz@musicbrainz_db:5432/musicbrainz_db"
+
+# BookBrainz Database
+BB_DATABASE_URI = "postgresql://bookbrainz:bookbrainz@db:5432/bookbrainz"