cats: Update DB schema from 2171 to 2192

- Add an INDEX on Job.JobTDate to improve performance (MySQL only)
- Fix integer out of range error by changing BaseFiles.BaseId to
  BIGINT/BIGSERIAL
- Add new DDL update scripts for each supported DB backend
- Set BDB_VERSION in cats.h
- Remove unnecessary INDEX pathvisibility_jobid
- Collation handling PostgreSQL >= 10
- Update basefiles sequence PostgreSQL >= 10

Fixes #1088: Integer out of range when using large amounts of files with Base Jobs
Fixes #1061: Tremendous MySQL load

core/src/cats/cats.h

@@ -563,7 +563,7 @@ class pathid_cache;
 /**
  * Current database version number for all drivers
  */
-#define BDB_VERSION 2171
+#define BDB_VERSION 2192
 
 #ifdef _BDB_PRIV_INTERFACE_
 /*

core/src/cats/create_bareos_database.in

@@ -109,7 +109,7 @@ case ${db_type} in
       # This must be updated for future versions of PostgreSQL
       #
       case ${PSQLVERSION} in
-         9.*|10.*)
+         9.*|10.*|11.*|12.*)
 	    ENCODING="ENCODING 'SQL_ASCII' LC_COLLATE 'C' LC_CTYPE 'C'"
             ;;
          8.[456789])

core/src/cats/ddl/creates/mysql.sql

@@ -130,7 +130,8 @@ CREATE TABLE Job (
    Reviewed TINYINT DEFAULT 0,
    Comment BLOB,
    PRIMARY KEY(JobId),
-   INDEX (Name(128))
+   INDEX (Name(128)),
+   INDEX (JobTDate)
 );
 
 -- Create a table like Job for long term statistics
@@ -316,7 +317,7 @@ CREATE TABLE Log (
 );
 
 CREATE TABLE BaseFiles (
-   BaseId INTEGER UNSIGNED AUTO_INCREMENT,
+   BaseId BIGINT UNSIGNED AUTO_INCREMENT,
    BaseJobId INTEGER UNSIGNED NOT NULL REFERENCES Job,
    JobId INTEGER UNSIGNED NOT NULL REFERENCES Job,
    FileId BIGINT UNSIGNED NOT NULL REFERENCES File,
@@ -369,8 +370,6 @@ CREATE TABLE PathVisibility
    Files int4 DEFAULT 0,
    CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId)
 );
-CREATE INDEX pathvisibility_jobid
-	     ON PathVisibility (JobId);
 
 CREATE TABLE Version (
    VersionId INTEGER UNSIGNED NOT NULL
@@ -460,5 +459,5 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
 -- Initialize Version
 --   DELETE should not be required,
 --   but prevents errors if create script is called multiple times
-DELETE FROM Version WHERE VersionId<=2171;
-INSERT INTO Version (VersionId) VALUES (2171);
+DELETE FROM Version WHERE VersionId<=2192;
+INSERT INTO Version (VersionId) VALUES (2192);

core/src/cats/ddl/creates/postgresql.sql

@@ -322,7 +322,7 @@ CREATE TABLE counters
 
 CREATE TABLE basefiles
 (
-    BaseId            SERIAL      NOT NULL,
+    BaseId            BIGSERIAL   NOT NULL,
     JobId             INTEGER     NOT NULL,
     FileId            BIGINT      NOT NULL,
     FileIndex         INTEGER,
@@ -360,8 +360,6 @@ CREATE TABLE PathVisibility
       Files           INTEGER     DEFAULT 0,
       CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId)
 );
-CREATE INDEX pathvisibility_jobid
-             ON PathVisibility (JobId);
 
 CREATE TABLE version
 (
@@ -484,7 +482,7 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
 -- Initialize Version
 --   DELETE should not be required,
 --   but prevents errors if create script is called multiple times
-DELETE FROM Version WHERE VersionId<=2171;
-INSERT INTO Version (VersionId) VALUES (2171);
+DELETE FROM Version WHERE VersionId<=2192;
+INSERT INTO Version (VersionId) VALUES (2192);
 
 -- Make sure we have appropriate permissions

core/src/cats/ddl/creates/sqlite3.sql

@@ -292,7 +292,7 @@ CREATE TABLE Client (
 );
 
 CREATE TABLE BaseFiles (
-   BaseId INTEGER,
+   BaseId BIGINT,
    BaseJobId INTEGER UNSIGNED REFERENCES Job NOT NULL,
    JobId INTEGER UNSIGNED REFERENCES Job NOT NULL,
    FileId INTEGER UNSIGNED REFERENCES File NOT NULL,
@@ -352,9 +352,6 @@ CREATE TABLE PathVisibility
    CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId)
 );
 
-CREATE INDEX pathvisibility_jobid
-	  ON PathVisibility (JobId);
-
 CREATE TABLE Status (
    JobStatus CHAR(1) NOT NULL,
    JobStatusLong BLOB,
@@ -471,8 +468,8 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES
 -- Initialize Version
 --   DELETE should not be required,
 --   but prevents errors if create script is called multiple times
-DELETE FROM Version WHERE VersionId<=2171;
-INSERT INTO Version (VersionId) VALUES (2171);
+DELETE FROM Version WHERE VersionId<=2192;
+INSERT INTO Version (VersionId) VALUES (2192);
 
 PRAGMA default_cache_size = 100000;
 PRAGMA synchronous = NORMAL;

core/src/cats/ddl/updates/mysql.2171_2192.sql

@@ -0,0 +1,18 @@
+-- update db schema from 2171 to 2192
+
+BEGIN;
+
+-- adapt index
+CREATE INDEX IF NOT EXISTS jobtdate_idx ON Job (JobTDate);
+
+-- change BaseFiles.BaseId data type
+ALTER TABLE BaseFiles MODIFY BaseId BIGINT;
+
+-- remove INDEX pathvisibility_jobid from pathvisibility
+DROP INDEX pathvisibility_jobid ON PathVisibility;
+
+UPDATE Version SET VersionId = 2192;
+
+COMMIT;
+
+ANALYZE TABLE Job;

core/src/cats/ddl/updates/postgresql.2171_2192.sql

@@ -0,0 +1,17 @@
+-- update db schema from 2171 to 2192
+
+-- start transaction
+BEGIN;
+
+ALTER TABLE BaseFiles
+ALTER COLUMN BaseId TYPE BIGINT;
+
+DROP INDEX IF EXISTS pathvisibility_jobid;
+
+UPDATE Version SET VersionId = 2192;
+
+COMMIT;
+
+set client_min_messages = fatal;
+
+ANALYSE;

core/src/cats/ddl/updates/sqlite3.2171_2192.sql

@@ -0,0 +1,16 @@
+-- update db schema from 2171 to 2192
+
+-- Stop on error. Prevents, that tables get droped, when merging data into new table has failed.
+.bail on
+
+BEGIN;
+
+-- adapt index
+CREATE INDEX JobTDateIndex ON Job (JobTDate);
+
+-- remove INDEX pathvisibility_jobid from pathvisibility
+DROP INDEX IF EXISTS pathvisibility_jobid;
+
+UPDATE Version SET VersionId = 2192;
+
+COMMIT;

core/src/cats/ddl/versions.map.in

@@ -10,5 +10,6 @@
 15.2.0=2004
 17.2.2=2170
 17.2.3=2171
+19.2.0=2192
 
 default=@BDB_VERSION@

core/src/cats/update_bareos_tables.in

@@ -170,6 +170,20 @@ do
       postgresql)
          psql -f ${temp_sql_schema} -d ${db_name} $*
          retval=$?
+         if [ $retval -eq 0 ] && [ ${end_version} -eq 2192 ]; then
+             PGSQLVERSION=`psql -d template1 -c 'SELECT version()' $* 2>/dev/null | \
+                    awk '/PostgreSQL/ { print $2 }' | \
+                    cut -d '.' -f 1,2`
+             case ${PGSQLVERSION} in
+                 10.*|11.*|12.*)
+                     psql -d ${db_name} -c 'ALTER SEQUENCE basefiles_baseid_seq AS BIGINT'
+                     result=$?
+                     if [ $result -ne 0 ]; then
+                         echo "WARNING: Failed to update sequence basefiles_baseid_seq"
+                     fi
+                     ;;
+             esac
+         fi
          ;;
    esac