From a8b0f67eeb4b185a221dcbbf025b6538cb7856e6 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Wed, 4 Dec 2019 17:19:55 +0100 Subject: [PATCH] cats: Update DB schema from 2171 to 2192 - Add an INDEX on Job.JobTDate to improve performance (MySQL only) - Fix integer out of range error by changing BaseFiles.BaseId to BIGINT/BIGSERIAL - Add new DDL update scripts for each supported DB backend - Set BDB_VERSION in cats.h - Remove unnecessary INDEX pathvisibility_jobid - Collation handling PostgreSQL >= 10 - Update basefiles sequence PostgreSQL >= 10 Fixes #1088: Integer out of range when using large amounts of files with Base Jobs Fixes #1061: Tremendous MySQL load --- core/src/cats/cats.h | 2 +- core/src/cats/create_bareos_database.in | 2 +- core/src/cats/ddl/creates/mysql.sql | 11 +++++------ core/src/cats/ddl/creates/postgresql.sql | 8 +++----- core/src/cats/ddl/creates/sqlite3.sql | 9 +++------ core/src/cats/ddl/updates/mysql.2171_2192.sql | 18 ++++++++++++++++++ .../cats/ddl/updates/postgresql.2171_2192.sql | 17 +++++++++++++++++ .../src/cats/ddl/updates/sqlite3.2171_2192.sql | 16 ++++++++++++++++ core/src/cats/ddl/versions.map.in | 1 + core/src/cats/update_bareos_tables.in | 14 ++++++++++++++ 10 files changed, 79 insertions(+), 19 deletions(-) create mode 100644 core/src/cats/ddl/updates/mysql.2171_2192.sql create mode 100644 core/src/cats/ddl/updates/postgresql.2171_2192.sql create mode 100644 core/src/cats/ddl/updates/sqlite3.2171_2192.sql diff --git a/core/src/cats/cats.h b/core/src/cats/cats.h index 2b6a1a5e745..de8f35823d7 100644 --- a/core/src/cats/cats.h +++ b/core/src/cats/cats.h @@ -563,7 +563,7 @@ class pathid_cache; /** * Current database version number for all drivers */ -#define BDB_VERSION 2171 +#define BDB_VERSION 2192 #ifdef _BDB_PRIV_INTERFACE_ /* diff --git a/core/src/cats/create_bareos_database.in b/core/src/cats/create_bareos_database.in index 98612c00ff7..391bfaf1248 100755 --- a/core/src/cats/create_bareos_database.in +++ b/core/src/cats/create_bareos_database.in @@ -109,7 +109,7 @@ case ${db_type} in # This must be updated for future versions of PostgreSQL # case ${PSQLVERSION} in - 9.*|10.*) + 9.*|10.*|11.*|12.*) ENCODING="ENCODING 'SQL_ASCII' LC_COLLATE 'C' LC_CTYPE 'C'" ;; 8.[456789]) diff --git a/core/src/cats/ddl/creates/mysql.sql b/core/src/cats/ddl/creates/mysql.sql index 2f6cf22f735..07cd4b6628d 100644 --- a/core/src/cats/ddl/creates/mysql.sql +++ b/core/src/cats/ddl/creates/mysql.sql @@ -130,7 +130,8 @@ CREATE TABLE Job ( Reviewed TINYINT DEFAULT 0, Comment BLOB, PRIMARY KEY(JobId), - INDEX (Name(128)) + INDEX (Name(128)), + INDEX (JobTDate) ); -- Create a table like Job for long term statistics @@ -316,7 +317,7 @@ CREATE TABLE Log ( ); CREATE TABLE BaseFiles ( - BaseId INTEGER UNSIGNED AUTO_INCREMENT, + BaseId BIGINT UNSIGNED AUTO_INCREMENT, BaseJobId INTEGER UNSIGNED NOT NULL REFERENCES Job, JobId INTEGER UNSIGNED NOT NULL REFERENCES Job, FileId BIGINT UNSIGNED NOT NULL REFERENCES File, @@ -369,8 +370,6 @@ CREATE TABLE PathVisibility Files int4 DEFAULT 0, CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId) ); -CREATE INDEX pathvisibility_jobid - ON PathVisibility (JobId); CREATE TABLE Version ( VersionId INTEGER UNSIGNED NOT NULL @@ -460,5 +459,5 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES -- Initialize Version -- DELETE should not be required, -- but prevents errors if create script is called multiple times -DELETE FROM Version WHERE VersionId<=2171; -INSERT INTO Version (VersionId) VALUES (2171); +DELETE FROM Version WHERE VersionId<=2192; +INSERT INTO Version (VersionId) VALUES (2192); diff --git a/core/src/cats/ddl/creates/postgresql.sql b/core/src/cats/ddl/creates/postgresql.sql index 15c08245e1e..e34406a1b5f 100644 --- a/core/src/cats/ddl/creates/postgresql.sql +++ b/core/src/cats/ddl/creates/postgresql.sql @@ -322,7 +322,7 @@ CREATE TABLE counters CREATE TABLE basefiles ( - BaseId SERIAL NOT NULL, + BaseId BIGSERIAL NOT NULL, JobId INTEGER NOT NULL, FileId BIGINT NOT NULL, FileIndex INTEGER, @@ -360,8 +360,6 @@ CREATE TABLE PathVisibility Files INTEGER DEFAULT 0, CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId) ); -CREATE INDEX pathvisibility_jobid - ON PathVisibility (JobId); CREATE TABLE version ( @@ -484,7 +482,7 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES -- Initialize Version -- DELETE should not be required, -- but prevents errors if create script is called multiple times -DELETE FROM Version WHERE VersionId<=2171; -INSERT INTO Version (VersionId) VALUES (2171); +DELETE FROM Version WHERE VersionId<=2192; +INSERT INTO Version (VersionId) VALUES (2192); -- Make sure we have appropriate permissions diff --git a/core/src/cats/ddl/creates/sqlite3.sql b/core/src/cats/ddl/creates/sqlite3.sql index 18c21532295..1c1131c81f3 100644 --- a/core/src/cats/ddl/creates/sqlite3.sql +++ b/core/src/cats/ddl/creates/sqlite3.sql @@ -292,7 +292,7 @@ CREATE TABLE Client ( ); CREATE TABLE BaseFiles ( - BaseId INTEGER, + BaseId BIGINT, BaseJobId INTEGER UNSIGNED REFERENCES Job NOT NULL, JobId INTEGER UNSIGNED REFERENCES Job NOT NULL, FileId INTEGER UNSIGNED REFERENCES File NOT NULL, @@ -352,9 +352,6 @@ CREATE TABLE PathVisibility CONSTRAINT pathvisibility_pkey PRIMARY KEY (JobId, PathId) ); -CREATE INDEX pathvisibility_jobid - ON PathVisibility (JobId); - CREATE TABLE Status ( JobStatus CHAR(1) NOT NULL, JobStatusLong BLOB, @@ -471,8 +468,8 @@ INSERT INTO Status (JobStatus,JobStatusLong,Severity) VALUES -- Initialize Version -- DELETE should not be required, -- but prevents errors if create script is called multiple times -DELETE FROM Version WHERE VersionId<=2171; -INSERT INTO Version (VersionId) VALUES (2171); +DELETE FROM Version WHERE VersionId<=2192; +INSERT INTO Version (VersionId) VALUES (2192); PRAGMA default_cache_size = 100000; PRAGMA synchronous = NORMAL; diff --git a/core/src/cats/ddl/updates/mysql.2171_2192.sql b/core/src/cats/ddl/updates/mysql.2171_2192.sql new file mode 100644 index 00000000000..271de3940be --- /dev/null +++ b/core/src/cats/ddl/updates/mysql.2171_2192.sql @@ -0,0 +1,18 @@ +-- update db schema from 2171 to 2192 + +BEGIN; + +-- adapt index +CREATE INDEX IF NOT EXISTS jobtdate_idx ON Job (JobTDate); + +-- change BaseFiles.BaseId data type +ALTER TABLE BaseFiles MODIFY BaseId BIGINT; + +-- remove INDEX pathvisibility_jobid from pathvisibility +DROP INDEX pathvisibility_jobid ON PathVisibility; + +UPDATE Version SET VersionId = 2192; + +COMMIT; + +ANALYZE TABLE Job; diff --git a/core/src/cats/ddl/updates/postgresql.2171_2192.sql b/core/src/cats/ddl/updates/postgresql.2171_2192.sql new file mode 100644 index 00000000000..fd9a4fdde3e --- /dev/null +++ b/core/src/cats/ddl/updates/postgresql.2171_2192.sql @@ -0,0 +1,17 @@ +-- update db schema from 2171 to 2192 + +-- start transaction +BEGIN; + +ALTER TABLE BaseFiles +ALTER COLUMN BaseId TYPE BIGINT; + +DROP INDEX IF EXISTS pathvisibility_jobid; + +UPDATE Version SET VersionId = 2192; + +COMMIT; + +set client_min_messages = fatal; + +ANALYSE; diff --git a/core/src/cats/ddl/updates/sqlite3.2171_2192.sql b/core/src/cats/ddl/updates/sqlite3.2171_2192.sql new file mode 100644 index 00000000000..4f2e0d17a62 --- /dev/null +++ b/core/src/cats/ddl/updates/sqlite3.2171_2192.sql @@ -0,0 +1,16 @@ +-- update db schema from 2171 to 2192 + +-- Stop on error. Prevents, that tables get droped, when merging data into new table has failed. +.bail on + +BEGIN; + +-- adapt index +CREATE INDEX JobTDateIndex ON Job (JobTDate); + +-- remove INDEX pathvisibility_jobid from pathvisibility +DROP INDEX IF EXISTS pathvisibility_jobid; + +UPDATE Version SET VersionId = 2192; + +COMMIT; diff --git a/core/src/cats/ddl/versions.map.in b/core/src/cats/ddl/versions.map.in index 6ce16aaac1f..d1019b0d92f 100644 --- a/core/src/cats/ddl/versions.map.in +++ b/core/src/cats/ddl/versions.map.in @@ -10,5 +10,6 @@ 15.2.0=2004 17.2.2=2170 17.2.3=2171 +19.2.0=2192 default=@BDB_VERSION@ diff --git a/core/src/cats/update_bareos_tables.in b/core/src/cats/update_bareos_tables.in index e3f3de83512..870a4b0784b 100755 --- a/core/src/cats/update_bareos_tables.in +++ b/core/src/cats/update_bareos_tables.in @@ -170,6 +170,20 @@ do postgresql) psql -f ${temp_sql_schema} -d ${db_name} $* retval=$? + if [ $retval -eq 0 ] && [ ${end_version} -eq 2192 ]; then + PGSQLVERSION=`psql -d template1 -c 'SELECT version()' $* 2>/dev/null | \ + awk '/PostgreSQL/ { print $2 }' | \ + cut -d '.' -f 1,2` + case ${PGSQLVERSION} in + 10.*|11.*|12.*) + psql -d ${db_name} -c 'ALTER SEQUENCE basefiles_baseid_seq AS BIGINT' + result=$? + if [ $result -ne 0 ]; then + echo "WARNING: Failed to update sequence basefiles_baseid_seq" + fi + ;; + esac + fi ;; esac