Make DB connection string configurable, add MariaDB support

We may not want to use SQLite all the time, and since we use
SQLAlchemy and Alembic, we can just make it a setting.

Also, to support MariaDB we need to make some adjustments to the
schema, such as making VARCHAR strings have a maximum length. The
Alembic migration achieves that, although for new setups with
MariaDB we will have to manually stamp this revision as the initial
one.

Change-Id: I47052bb20a6a474b6d8e53ede2f6a0f6c98d8e53

dlrn/config.py

@@ -33,6 +33,7 @@ def __init__(self, cp):
         self.pkginfo_driver = cp.get('DEFAULT', 'pkginfo_driver')
         self.workers = cp.getint('DEFAULT', 'workers')
         self.gerrit_topic = cp.get('DEFAULT', 'gerrit_topic')
+        self.database_connection = cp.get('DEFAULT', 'database_connection')
 
         # Handling for optional sections, driver-based
         if cp.has_section('gitrepo_driver'):

dlrn/db.py

@@ -21,6 +21,7 @@
 from sqlalchemy import Integer
 from sqlalchemy import or_
 from sqlalchemy import String
+from sqlalchemy import Text
 
 from sqlalchemy.orm import scoped_session
 
@@ -35,15 +36,15 @@ class Commit(Base):
     dt_commit = Column(Integer)
     dt_distro = Column(Integer)
     dt_build = Column(Integer)
-    project_name = Column(String)
-    repo_dir = Column(String)
-    distgit_dir = Column(String)
-    commit_hash = Column(String)
-    distro_hash = Column(String)
-    commit_branch = Column(String)
-    status = Column(String)
-    rpms = Column(String)
-    notes = Column(String)
+    project_name = Column(String(256))
+    repo_dir = Column(String(1024))
+    distgit_dir = Column(String(1024))
+    commit_hash = Column(String(64))
+    distro_hash = Column(String(64))
+    commit_branch = Column(String(256))
+    status = Column(String(64))
+    rpms = Column(Text)
+    notes = Column(Text)
     flags = Column(Integer, default=0)
 
     def __cmp__(self, b):
@@ -61,7 +62,7 @@ class Project(Base):
     __tablename__ = "projects"
 
     id = Column(Integer, primary_key=True)
-    project_name = Column(String)
+    project_name = Column(String(256))
     last_email = Column(Integer)
 
     # Returns True if the last email sent for this project

dlrn/migrations/versions/f38ba3389b85_set_string_length.py

@@ -0,0 +1,85 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Set string length
+
+Revision ID: f38ba3389b85
+Revises: 1268c799620f
+Create Date: 2017-01-12 18:42:25.593667
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f38ba3389b85'
+down_revision = '1268c799620f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("commits") as batch_op:
+        batch_op.alter_column('project_name', existing_type=sa.String(),
+                              type_=sa.String(256))
+        batch_op.alter_column('repo_dir', existing_type=sa.String(),
+                              type_=sa.String(1024))
+        batch_op.alter_column('distgit_dir', existing_type=sa.String(),
+                              type_=sa.String(1024))
+        batch_op.alter_column('commit_hash', existing_type=sa.String(),
+                              type_=sa.String(64))
+        batch_op.alter_column('distro_hash', existing_type=sa.String(),
+                              type_=sa.String(64))
+        batch_op.alter_column('distgit_dir', existing_type=sa.String(),
+                              type_=sa.String(1024))
+        batch_op.alter_column('commit_branch', existing_type=sa.String(),
+                              type_=sa.String(256))
+        batch_op.alter_column('status', existing_type=sa.String(),
+                              type_=sa.String(64))
+        batch_op.alter_column('rpms', existing_type=sa.String(),
+                              type_=sa.Text())
+        batch_op.alter_column('notes', existing_type=sa.String(),
+                              type_=sa.Text())
+
+    with op.batch_alter_table("projects") as batch_op:
+        batch_op.alter_column('project_name', existing_type=sa.String(),
+                              type_=sa.String(256))
+
+
+def downgrade():
+    with op.batch_alter_table("commits") as batch_op:
+        batch_op.alter_column('project_name', existing_type=sa.String(256),
+                              type_=sa.String())
+        batch_op.alter_column('repo_dir', existing_type=sa.String(1024),
+                              type_=sa.String())
+        batch_op.alter_column('distgit_dir', existing_type=sa.String(1024),
+                              type_=sa.String())
+        batch_op.alter_column('commit_hash', existing_type=sa.String(64),
+                              type_=sa.String())
+        batch_op.alter_column('distro_hash', existing_type=sa.String(64),
+                              type_=sa.String())
+        batch_op.alter_column('distgit_dir', existing_type=sa.String(1024),
+                              type_=sa.String())
+        batch_op.alter_column('commit_branch', existing_type=sa.String(256),
+                              type_=sa.String())
+        batch_op.alter_column('status', existing_type=sa.String(64),
+                              type_=sa.String())
+        batch_op.alter_column('rpms', existing_type=sa.Text(),
+                              type_=sa.String())
+        batch_op.alter_column('notes', existing_type=sa.Text(),
+                              type_=sa.String())
+
+    with op.batch_alter_table("projects") as batch_op:
+        batch_op.alter_column('project_name', existing_type=sa.String(256),
+                              type_=sa.String())

dlrn/purge.py

@@ -83,7 +83,7 @@ def purge():
         if ans.lower() != "y":
             return
 
-    session = getSession('sqlite:///commits.sqlite')
+    session = getSession(cp.get('DEFAULT', 'database_connection'))
 
     # To remove builds we have to start at a point in time and move backwards
     # builds with no build date are also purged as these are legacy

dlrn/reporting.py

@@ -25,7 +25,6 @@
 from dlrn.config import getConfigOptions
 from dlrn.db import Commit
 from dlrn.db import getCommits
-from dlrn.db import getSession
 
 
 def get_commit_url(commit, pkg):
@@ -60,9 +59,7 @@ def _jinja2_filter_get_commit_url(commit, packages):
     return "???"
 
 
-def genreports(packages, head_only):
-    global session
-    session = getSession('sqlite:///commits.sqlite')
+def genreports(packages, head_only, session):
     config_options = getConfigOptions()
 
     # Generate report of the last 300 package builds

dlrn/shell.py

@@ -61,6 +61,7 @@
                    'pkginfo_driver': 'dlrn.drivers.rdoinfo.RdoInfoDriver',
                    'workers': '1',
                    'gerrit_topic': 'rdo-FTBFS',
+                   'database_connection': 'sqlite:///commits.sqlite'
                    }
 
 
@@ -146,8 +147,8 @@ def main():
     if options.order is True:
         options.sequential = True
 
-    session = getSession('sqlite:///commits.sqlite')
     config_options = ConfigOptions(cp)
+    session = getSession(config_options.database_connection)
     pkginfo_driver = config_options.pkginfo_driver
     global pkginfo
     pkginfo = import_object(pkginfo_driver, cfg_options=config_options)
@@ -380,7 +381,7 @@ def my_cmp(a, b):
             commit.status = 'RETRY'
             session.add(commit)
             session.commit()
-    genreports(packages, options.head_only)
+    genreports(packages, options.head_only, session)
     return exit_code
 
 
@@ -496,7 +497,7 @@ def process_build_result(status, packages, session, dev_mode=False,
         session.add(commit)
     if dev_mode is False:
         session.commit()
-    genreports(packages, head_only)
+    genreports(packages, head_only, session)
     # TODO(jpena): could we launch this asynchronously?
     sync_repo(commit)
     return exit_code

doc/source/installation.rst

@@ -66,6 +66,7 @@ The configuration file looks like this:
     rsyncport=22
     workers=1
     gerrit_topic=rdo-FTBFS
+    database_connection=sqlite:///commits.sqlite
 
 
 * ``datadir`` is the directory where the packages and repositories will be
@@ -126,6 +127,10 @@ The configuration file looks like this:
   multiple workers, the mock build part will be handled by a pool of processes,
   while the repo creation and synchronization will still be sequential.
 
+* The ``database_connection`` string defines a database connection string. By
+  default, a local SQLite3 database is used, but it is also possible to set up
+  an external database.
+
 * ``pkginfo_driver`` defines the driver to be used to manage the distgit
   repositories. There are currently two drivers:
 
@@ -186,10 +191,42 @@ data directories, or just make a symbolic link:
     $ sudo ln -s <datadir>/repos .
 
 
+Database support
+----------------
+
+DLRN supports different database engines through SQLAlchemy. SQLite3 and MariaDB have
+been tested so far. You can set the ``database_connection`` parameter in projects.ini
+with the required string, using `the SQLAlchemy syntax
+ <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_. 
+
+For MariaDB, use a mysql+pymysql driver, with the following string:
+
+.. code-block:: ini
+
+    database_connection=mysql+pymysql://user:password@serverIP/dlrn
+
+That requires you to pre-create the ``dlrn``database. You can use the following commands
+on a mysql session to create the database and grant the required permissions:
+
+.. code-block:: mysql
+
+    use mysql;
+    create database dlrn;
+    grant all on dlrn.* to 'user'@'%' identified by 'password';
+    flush privileges
+
+If your MariaDB database is placed on a publicly accessible server, you will also
+want to secure it before:
+
+.. code-block:: bash
+
+    $ sudo mysql_secure_installation
+
+
 Database migration
 ++++++++++++++++++
 
-during DLRN upgrades, you may need to upgrade the database schemas,
+During DLRN upgrades, you may need to upgrade the database schemas,
 in order to keep your old history.
 To migrate database to the latest revision, you need the alembic command-line
 and to run the ``alembic upgrade head`` command.
@@ -200,3 +237,14 @@ and to run the ``alembic upgrade head`` command.
     $ alembic upgrade head
 
 If the database doesn't exist, ``alembic upgrade head`` will create it from scratch.
+
+If you are using a MariaDB database, the initial schema will not be valid. You should
+start by running DLRN a first time, so it creates the basic schema, then run the
+following command to stamp the database to the first version of the schema that
+supported MariaDB:
+
+.. code-block:: bash
+    
+    $ alembic stamp head
+
+After that initial command, you will be able to run future migrations.

projects.ini

@@ -16,6 +16,7 @@ rsyncdest=
 rsyncport=22
 workers=1
 gerrit_topic=rdo-FTBFS
+database_connection=sqlite:///commits.sqlite
 
 [gitrepo_driver]
 # Options to be specified if pkginfo_driver is set to

requirements.txt

@@ -5,10 +5,11 @@ oslo.config
 sh==1.11.0
 six
 SQLAlchemy
+PyMySQL
 PyYAML
 pycparser==2.14
 rdopkg>=0.34
-alembic
+alembic>=0.7.0
 renderspec
 pymod2pkg>=0.5.5
 requests