[0.13] core: Track upgrade step within each schema version

src/core/abstractsqlstorage.cpp

@@ -190,13 +190,13 @@ QString AbstractSqlStorage::queryString(const QString &queryName, int version)
 }
 
 
-QStringList AbstractSqlStorage::setupQueries()
+QList<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::setupQueries()
 {
-    QStringList queries;
+    QList<SqlQueryResource> queries;
     // The current schema is stored in the root folder, including setup scripts.
     QDir dir = QDir(QString(":/SQL/%1/").arg(displayName()));
     foreach(QFileInfo fileInfo, dir.entryInfoList(QStringList() << "setup*", QDir::NoFilter, QDir::Name)) {
-        queries << queryString(fileInfo.baseName());
+        queries << SqlQueryResource(queryString(fileInfo.baseName()), fileInfo.baseName());
     }
     return queries;
 }
@@ -213,10 +213,11 @@ bool AbstractSqlStorage::setup(const QVariantMap &settings, const QProcessEnviro
     }
 
     db.transaction();
-    foreach(QString queryString, setupQueries()) {
-        QSqlQuery query = db.exec(queryString);
+    foreach (auto queryResource, setupQueries()) {
+        QSqlQuery query = db.exec(queryResource.queryString);
         if (!watchQuery(query)) {
-            qCritical() << "Unable to setup Logging Backend!";
+            qCritical() << qPrintable(QString("Unable to setup Logging Backend!  Setup query failed (step: %1).")
+                                      .arg(queryResource.queryFilename));
             db.rollback();
             return false;
         }
@@ -230,13 +231,13 @@ bool AbstractSqlStorage::setup(const QVariantMap &settings, const QProcessEnviro
 }
 
 
-QStringList AbstractSqlStorage::upgradeQueries(int version)
+QList<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::upgradeQueries(int version)
 {
-    QStringList queries;
+    QList<SqlQueryResource> queries;
     // Upgrade queries are stored in the 'version/##' subfolders.
     QDir dir = QDir(QString(":/SQL/%1/version/%2/").arg(displayName()).arg(version));
     foreach(QFileInfo fileInfo, dir.entryInfoList(QStringList() << "upgrade*", QDir::NoFilter, QDir::Name)) {
-        queries << queryString(fileInfo.baseName(), version);
+        queries << SqlQueryResource(queryString(fileInfo.baseName(), version), fileInfo.baseName());
     }
     return queries;
 }
@@ -253,41 +254,76 @@ bool AbstractSqlStorage::upgradeDb()
     // transaction.  This will need careful testing of potential additional space requirements and
     // any database modifications that might not be allowed in a transaction.
 
+    // Check if we're resuming an interrupted multi-step upgrade: is an upgrade step stored?
+    const QString previousLaunchUpgradeStep = schemaVersionUpgradeStep();
+    bool resumingUpgrade = !previousLaunchUpgradeStep.isEmpty();
+
     for (int ver = installedSchemaVersion() + 1; ver <= schemaVersion(); ver++) {
-        foreach(QString queryString, upgradeQueries(ver)) {
-            QSqlQuery query = db.exec(queryString);
+        foreach (auto queryResource, upgradeQueries(ver)) {
+            if (resumingUpgrade) {
+                // An upgrade was interrupted.  Check if this matches the the last successful query.
+                if (previousLaunchUpgradeStep == queryResource.queryFilename) {
+                    // Found the matching query!
+                    quInfo() << qPrintable(QString("Resuming interrupted upgrade for schema version %1 (last step: %2)")
+                                           .arg(QString::number(ver), previousLaunchUpgradeStep));
+
+                    // Stop searching for queries
+                    resumingUpgrade = false;
+                    // Continue past the previous query with the next not-yet-tried query
+                    continue;
+                }
+                else {
+                    // Not yet matched, keep looking
+                    continue;
+                }
+            }
+
+            // Run the upgrade query
+            QSqlQuery query = db.exec(queryResource.queryString);
             if (!watchQuery(query)) {
                 // Individual upgrade query failed, bail out
-                qCritical() << "Unable to upgrade Logging Backend!  Upgrade query in schema version"
-                            << ver << "failed.";
+                qCritical() << qPrintable(QString("Unable to upgrade Logging Backend!  Upgrade query in schema version %1 failed (step: %2).")
+                                          .arg(QString::number(ver), queryResource.queryFilename));
                 return false;
             }
+            else {
+                // Mark as successful
+                setSchemaVersionUpgradeStep(queryResource.queryFilename);
+            }
         }
 
-        // Update the schema version for each intermediate step.  This ensures that any interrupted
-        // upgrades have a greater chance of resuming correctly after core restart.
+        if (resumingUpgrade) {
+            // Something went wrong and the last successful SQL query to resume from couldn't be
+            // found.
+            // 1.  The storage of successful query glitched, or the database was manually changed
+            // 2.  Quassel changed the filenames of upgrade queries, and the local Quassel core
+            //     version was replaced during an interrupted schema upgrade
+            //
+            // Both are unlikely, but it's a good idea to handle it anyways.
+
+            qCritical() << qPrintable(QString("Unable to resume interrupted upgrade in Logging "
+                                              "Backend!  Missing upgrade step in schema version %1 "
+                                              "(expected step: %2)")
+                                      .arg(QString::number(ver), previousLaunchUpgradeStep));
+            return false;
+        }
+
+        // Update the schema version for each intermediate step and mark the step as done.  This
+        // ensures that any interrupted upgrades have a greater chance of resuming correctly after
+        // core restart.
         //
         // Almost all databases make single queries atomic (fully works or fully fails, no partial),
         // and with many of the longest migrations being a single query, this makes upgrade
         // interruptions much more likely to leave the database in a valid intermediate schema
         // version.
-        if (!updateSchemaVersion(ver)) {
+        if (!updateSchemaVersion(ver, true)) {
             // Updating the schema version failed, bail out
             qCritical() << "Unable to upgrade Logging Backend!  Setting schema version"
                         << ver << "failed.";
             return false;
         }
     }
 
-    // Update the schema version for the final step.  Split this out to offer more informative
-    // logging (though setting schema version really should not fail).
-    if (!updateSchemaVersion(schemaVersion())) {
-        // Updating the final schema version failed, bail out
-        qCritical() << "Unable to upgrade Logging Backend!  Setting final schema version"
-                    << schemaVersion() << "failed.";
-        return false;
-    }
-
     // If we made it here, everything seems to have worked!
     return true;
 }
@@ -319,6 +355,13 @@ int AbstractSqlStorage::schemaVersion()
 }
 
 
+QString AbstractSqlStorage::schemaVersionUpgradeStep()
+{
+    // By default, assume there's no pending upgrade
+    return {};
+}
+
+
 bool AbstractSqlStorage::watchQuery(QSqlQuery &query)
 {
     bool queryError = query.lastError().isValid();

src/core/abstractsqlstorage.h

@@ -24,6 +24,7 @@
 
 #include <memory>
 
+#include <QList>
 #include <QSqlDatabase>
 #include <QSqlQuery>
 #include <QSqlError>
@@ -42,6 +43,18 @@ class AbstractSqlStorage : public Storage
     virtual std::unique_ptr<AbstractSqlMigrationReader> createMigrationReader() { return {}; }
     virtual std::unique_ptr<AbstractSqlMigrationWriter> createMigrationWriter() { return {}; }
 
+    /**
+     * An SQL query with associated resource filename
+     */
+    struct SqlQueryResource {
+        QString queryString;   ///< SQL query string
+        QString queryFilename; ///< Path to the resource file providing this query
+
+        SqlQueryResource(const QString& queryString, const QString& queryFilename)
+            : queryString(std::move(queryString)),
+              queryFilename(std::move(queryFilename)) {}
+    };
+
 public slots:
     virtual State init(const QVariantMap &settings = QVariantMap(),
                        const QProcessEnvironment &environment = {},
@@ -74,18 +87,52 @@ public slots:
      */
     QString queryString(const QString &queryName, int version = 0);
 
-    QStringList setupQueries();
+    /**
+     * Gets the collection of SQL setup queries and filenames to create a new database
+     *
+     * @return List of SQL query strings and filenames
+     */
+    QList<SqlQueryResource> setupQueries();
 
-    QStringList upgradeQueries(int ver);
+    /**
+     * Gets the collection of SQL upgrade queries and filenames for a given schema version
+     *
+     * @param ver  SQL schema version
+     * @return List of SQL query strings and filenames
+     */
+    QList<SqlQueryResource> upgradeQueries(int ver);
     bool upgradeDb();
 
     bool watchQuery(QSqlQuery &query);
 
     int schemaVersion();
     virtual int installedSchemaVersion() { return -1; };
-    virtual bool updateSchemaVersion(int newVersion) = 0;
+
+    /**
+     * Update the stored schema version number, optionally clearing the record of mid-schema steps
+     *
+     * @param newVersion        New schema version number
+     * @param clearUpgradeStep  If true, clear the record of any in-progress schema upgrades
+     * @return
+     */
+    virtual bool updateSchemaVersion(int newVersion, bool clearUpgradeStep = true) = 0;
+
     virtual bool setupSchemaVersion(int version) = 0;
 
+    /**
+     * Gets the last successful schema upgrade step, or an empty string if no upgrade is in progress
+     *
+     * @return Filename of last successful schema upgrade query, or empty string if not upgrading
+     */
+    virtual QString schemaVersionUpgradeStep();
+
+    /**
+     * Sets the last successful schema upgrade step
+     *
+     * @param upgradeQuery  The filename of the last successful schema upgrade query
+     * @return True if successfully set, otherwise false
+     */
+    virtual bool setSchemaVersionUpgradeStep(QString upgradeQuery) = 0;
     virtual void setConnectionProperties(const QVariantMap &properties,
                                          const QProcessEnvironment &environment,
                                          bool loadFromEnvironment) = 0;

src/core/postgresqlstorage.cpp

@@ -187,19 +187,39 @@ int PostgreSqlStorage::installedSchemaVersion()
 }
 
 
-bool PostgreSqlStorage::updateSchemaVersion(int newVersion)
+bool PostgreSqlStorage::updateSchemaVersion(int newVersion, bool clearUpgradeStep)
 {
-    QSqlQuery query(logDb());
+    // Atomically update the schema version and clear the upgrade step, if specified
+    // Note: This will need reworked if "updateSchemaVersion" is ever called within a transaction.
+    QSqlDatabase db = logDb();
+    if (!beginTransaction(db)) {
+        qWarning() << "PostgreSqlStorage::updateSchemaVersion(int, bool): cannot start transaction!";
+        qWarning() << " -" << qPrintable(db.lastError().text());
+        return false;
+    }
+
+    QSqlQuery query(db);
     query.prepare("UPDATE coreinfo SET value = :version WHERE key = 'schemaversion'");
     query.bindValue(":version", newVersion);
     safeExec(query);
 
-    bool success = true;
     if (!watchQuery(query)) {
-        qCritical() << "PostgreSqlStorage::updateSchemaVersion(int): Updating schema version failed!";
-        success = false;
+        qCritical() << "PostgreSqlStorage::updateSchemaVersion(int, bool): Updating schema version failed!";
+        db.rollback();
+        return false;
     }
-    return success;
+
+    if (clearUpgradeStep) {
+        // Try clearing the upgrade step if requested
+        if (!setSchemaVersionUpgradeStep("")) {
+            db.rollback();
+            return false;
+        }
+    }
+
+    // Successful, commit and return true
+    db.commit();
+    return true;
 }
 
 
@@ -219,6 +239,52 @@ bool PostgreSqlStorage::setupSchemaVersion(int version)
 }
 
 
+QString PostgreSqlStorage::schemaVersionUpgradeStep()
+{
+    QSqlQuery query(logDb());
+    query.prepare("SELECT value FROM coreinfo WHERE key = 'schemaupgradestep'");
+    safeExec(query);
+    watchQuery(query);
+    if (query.first())
+        return query.value(0).toString();
+
+    // Fall back to the default value
+    return AbstractSqlStorage::schemaVersionUpgradeStep();
+}
+
+
+bool PostgreSqlStorage::setSchemaVersionUpgradeStep(QString upgradeQuery)
+{
+    // Intentionally do not wrap in a transaction so other functions can include multiple operations
+
+    QSqlQuery query(logDb());
+    query.prepare("UPDATE coreinfo SET value = :upgradestep WHERE key = 'schemaupgradestep'");
+    query.bindValue(":upgradestep", upgradeQuery);
+    safeExec(query);
+
+    // Make sure that the query didn't fail (shouldn't ever happen), and that some non-zero number
+    // of rows were affected
+    bool success = watchQuery(query) && query.numRowsAffected() != 0;
+
+    if (!success) {
+        // The key might not exist (Quassel 0.13.0 and older).  Try inserting it...
+        query = QSqlQuery(logDb());
+        query.prepare("INSERT INTO coreinfo (key, value) VALUES ('schemaupgradestep', :upgradestep)");
+        query.bindValue(":upgradestep", upgradeQuery);
+        safeExec(query);
+
+        if (!watchQuery(query)) {
+            qCritical() << Q_FUNC_INFO << "Setting schema upgrade step failed!";
+            success = false;
+        }
+        else {
+            success = true;
+        }
+    }
+    return success;
+}
+
+
 UserId PostgreSqlStorage::addUser(const QString &user, const QString &password, const QString &authenticator)
 {
     QSqlQuery query(logDb());

src/core/postgresqlstorage.h

@@ -133,8 +133,24 @@ public slots:
     QString userName()  override { return _userName; }
     QString password()  override { return _password; }
     int installedSchemaVersion() override;
-    bool updateSchemaVersion(int newVersion) override;
+    bool updateSchemaVersion(int newVersion, bool clearUpgradeStep) override;
     bool setupSchemaVersion(int version) override;
+
+    /**
+     * Gets the last successful schema upgrade step, or an empty string if no upgrade is in progress
+     *
+     * @return Filename of last successful schema upgrade query, or empty string if not upgrading
+     */
+    QString schemaVersionUpgradeStep() override;
+
+    /**
+     * Sets the last successful schema upgrade step
+     *
+     * @param upgradeQuery  The filename of the last successful schema upgrade query
+     * @return True if successfully set, otherwise false
+     */
+    virtual bool setSchemaVersionUpgradeStep(QString upgradeQuery) override;
+
     void safeExec(QSqlQuery &query);
 
     bool beginTransaction(QSqlDatabase &db);