From d8723f513ea5696cae3fec68a1f8d6556c6c1ec1 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Tue, 28 Jul 2020 14:25:24 +0200 Subject: [PATCH 1/5] cats: set MySQL/SQLite create database deprecated --- core/src/cats/create_bareos_database.in | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/cats/create_bareos_database.in b/core/src/cats/create_bareos_database.in index c2f5ef9776b..0c9ad1a8cb8 100755 --- a/core/src/cats/create_bareos_database.in +++ b/core/src/cats/create_bareos_database.in @@ -81,15 +81,12 @@ fi case ${db_type} in sqlite3) - sqlite3 $* ${working_dir}/${db_name}.db ".tables" - # Allow non root access - chown ${dir_user}:${dir_group} ${working_dir}/${db_name}.db - chmod 0640 ${working_dir}/${db_name}.db - retval=0 + echo "The SQLite database backend is deprecated. Please use PostgreSQL instead." + retval=1 ;; mysql) - mysql $* -e "CREATE DATABASE ${db_name};" - retval=$? + echo "The MySQL database backend is deprecated. Please use PostgreSQL instead." + retval=1 ;; postgresql) # From 42482975401c2833d9deed862b7abd89bdc8bc68 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Tue, 28 Jul 2020 14:40:58 +0200 Subject: [PATCH 2/5] docs: set SQLite database backend deprecated --- .../source/IntroductionAndTutorial/InstallingBareos.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst b/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst index e2094544bd0..a6d6807d796 100644 --- a/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst +++ b/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst @@ -60,7 +60,9 @@ MariaDB/MySQL It is contained in package **bareos-database-mysql**. **This is not the right choice for a new deployment.** -Sqlite +SQLite + .. deprecated:: 20.0.0 + This backend is for testing purposes only. It is contained in package **bareos-database-sqlite3**. **Do not use this in production.** From c392fda554597b86e4008a20eb2ce98eb96309dc Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Tue, 28 Jul 2020 14:45:48 +0200 Subject: [PATCH 3/5] docs: remove MySQL/MariaDB create database instructions --- .../InstallingBareos.rst | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst b/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst index a6d6807d796..377fbce80dc 100644 --- a/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst +++ b/docs/manuals/source/IntroductionAndTutorial/InstallingBareos.rst @@ -360,9 +360,8 @@ If you are not interested in this additional functionality, the commands describ Prepare Bareos database ----------------------- -We assume that you have already your database installed and basically running. Using the PostgreSQL database backend is recommended. - -The easiest way to set up a database is using an system account that have passwordless local access to the database. Often this is the user **root** for MySQL and the user **postgres** for PostgreSQL. +We assume that you have already your database installed and basically running. +Using the PostgreSQL database backend is recommended. For details, see chapter :ref:`CatMaintenanceChapter`. @@ -405,30 +404,6 @@ If your are using PostgreSQL and your PostgreSQL administration user is **postgr su postgres -c /usr/lib/bareos/scripts/make_bareos_tables su postgres -c /usr/lib/bareos/scripts/grant_bareos_privileges -MySQL/MariaDB -^^^^^^^^^^^^^ - -Make sure, that **root** has direct access to the local MySQL server. Check if the command :command:`mysql` connects to the database without defining the password. This is the default on RedHat and SUSE distributions. On other systems (Debian, Ubuntu), create the file :file:`~/.my.cnf` with your authentication informations: - -.. code-block:: cfg - :caption: MySQL credentials file .my.cnf - - [client] - host=localhost - user=root - password=YourPasswordForAccessingMysqlAsRoot - -It is recommended to secure the Bareos database connection with a password. See :ref:`Catalog Maintenance -- MySQL ` on how to achieve this. For testing, using a password-less MySQL connection is probably okay. Setup the Bareos database tables using the following commands: - -.. code-block:: shell-session - :caption: Setup Bareos catalog with MySQL - - /usr/lib/bareos/scripts/create_bareos_database - /usr/lib/bareos/scripts/make_bareos_tables - /usr/lib/bareos/scripts/grant_bareos_privileges - -As some Bareos updates require a database schema update, the file :file:`/root/.my.cnf` might also be useful in the future. - .. _section-StartDaemons: Start the daemons From 7cd0b365ac6fd127ad59074d3c000b3013f79b95 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Tue, 28 Jul 2020 15:40:23 +0200 Subject: [PATCH 4/5] cats: set MySQL/SQLite make bareos tables deprecated --- core/src/cats/make_bareos_tables.in | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/core/src/cats/make_bareos_tables.in b/core/src/cats/make_bareos_tables.in index 318884f0a44..6644087bfa5 100755 --- a/core/src/cats/make_bareos_tables.in +++ b/core/src/cats/make_bareos_tables.in @@ -110,18 +110,14 @@ fi case ${db_type} in sqlite3) - sqlite3 $* ${working_dir}/${db_name}.db < ${temp_sql_schema} - chmod 640 ${working_dir}/${db_name}.db - retval=0 + echo "The SQLite database backend is deprecated. Please use PostgreSQL instead." + echo "Creation of Bareos SQLite tables aborted." + retval=1 ;; mysql) - mysql $* --database=${db_name} -f < ${temp_sql_schema} - retval=$? - if test $retval = 0; then - echo "Creation of Bareos MySQL tables succeeded." - else - echo "Creation of Bareos MySQL tables failed." - fi + echo "The MySQL database backend is deprecated. Please use PostgreSQL instead." + echo "Creation of Bareos MySQL tables aborted." + retval=1 ;; postgresql) PGOPTIONS='--client-min-messages=warning' psql -f ${temp_sql_schema} -d ${db_name} $* From ecc8c3e3e0be0c5f606ef973d9339df981a6ad05 Mon Sep 17 00:00:00 2001 From: Frank Bergkemper Date: Tue, 28 Jul 2020 16:56:13 +0200 Subject: [PATCH 5/5] docs: remove MySQL/SQLite sections from catalog maintenance chapter --- .../TasksAndConcepts/CatalogMaintenance.rst | 375 +----------------- 1 file changed, 2 insertions(+), 373 deletions(-) diff --git a/docs/manuals/source/TasksAndConcepts/CatalogMaintenance.rst b/docs/manuals/source/TasksAndConcepts/CatalogMaintenance.rst index aed5e1e625a..bcb76c92d2b 100644 --- a/docs/manuals/source/TasksAndConcepts/CatalogMaintenance.rst +++ b/docs/manuals/source/TasksAndConcepts/CatalogMaintenance.rst @@ -12,9 +12,9 @@ Bareos stores its catalog in a database. Different database backends are offered - PostgreSQL (recommended) -- MariaDB/MySQL (see :ref:`section-MysqlSupport`) +- MariaDB/MySQL (deprecated) -- Sqlite (only for testing) +- Sqlite (deprecated) What database will be used, can be configured in the |dir| configuration, see the :ref:`DirectorResourceCatalog`. @@ -321,226 +321,6 @@ If **dbconfig-common** did not succeed or you choosed not to use it, run the Bar /usr/lib/bareos/scripts/make_bareos_tables --host=bareos-database.example.com /usr/lib/bareos/scripts/grant_bareos_privileges --host=bareos-database.example.com -.. _catalog-maintenance-mysql: - -MySQL -^^^^^ - -MySQL user authentication is username, password and host-based. The database administrator is the user **root**. - -On some distributions access to the MySQL database is allowed password-less as database user **root**, on other distributions, a password is required. On productive systems you normally want to have password secured access. - -The bareos database preparation scripts require password-less access to the database. To guarantee this, create a MySQL credentials file `:file:`~/.my.cnf` `_ with the credentials of the database administrator: - -.. code-block:: cfg - :caption: MySQL credentials file .my.cnf - - [client] - host=localhost - user=root - password=YourPasswordForAccessingMysqlAsRoot - -Alternatively you can specifiy your database password by adding it to the file :file:`/etc/my.cnf`. - -Verify that you have specified the correct settings by calling the :command:`mysql` command. If this connects you to the database, your credentials are good. Exit the MySQL client. - -For the Bareos database connection, you should specify a database password. Otherwise the Bareos database user gets the permission to connect without password. This is not recommended. Choose a database password and add it into the Bareos Director configuration file :file:`/etc/bareos/bareos-dir.conf`: - -.. code-block:: bareosconfig - :caption: Bareos catalog configuration - - ... - # - # Generic catalog service - # - Catalog { - Name = MyCatalog - dbdriver = "mysql" - dbname = "bareos" - dbuser = "bareos" - dbpassword = "YourSecretPassword" - } - ... - -After this, run the Bareos database preparation scripts. For Bareos <= 13.2.2, the database password must be specified as environment variable ``db_password``\ . From :sinceVersion:`13.2.3: MySQL password from configuration file` the database password is read from the configuration, if no environment variable is given. - -.. code-block:: shell-session - :caption: Setup Bareos catalog database - - export db_password=YourSecretPassword - /usr/lib/bareos/scripts/create_bareos_database - /usr/lib/bareos/scripts/make_bareos_tables - /usr/lib/bareos/scripts/grant_bareos_privileges - -After this, you can use the :command:`mysql` command to verify that your database setup is okay and works with your the Bareos database user. The result should look similar as this (here Bareos 13.2 is used on SLES11): - -.. code-block:: shell-session - :caption: Verify Bareos database on MySQL - - root@linux:~# mysql --user=bareos --password=YourSecretPassword bareos - Welcome to the MySQL monitor. Commands end with ; or \g. - Your MySQL connection id is 162 - Server version: 5.5.32 SUSE MySQL package - - Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. - - Oracle is a registered trademark of Oracle Corporation and/or its - affiliates. Other names may be trademarks of their respective - owners. - - Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. - - mysql> show tables; - +--------------------+ - | Tables_in_bareos | - +--------------------+ - | BaseFiles | - | CDImages | - | Client | - | Counters | - | Device | - | DeviceStats | - | File | - | FileSet | - | Filename | - | Job | - | JobHisto | - | JobMedia | - | JobStats | - | Location | - | LocationLog | - | Log | - | Media | - | MediaType | - | NDMPJobEnvironment | - | NDMPLevelMap | - | Path | - | PathHierarchy | - | PathVisibility | - | Pool | - | Quota | - | RestoreObject | - | Status | - | Storage | - | UnsavedFiles | - | Version | - +--------------------+ - 30 rows in set (0.00 sec) - - mysql> describe Job; - +-----------------+---------------------+------+-----+---------+----------------+ - | Field | Type | Null | Key | Default | Extra | - +-----------------+---------------------+------+-----+---------+----------------+ - | JobId | int(10) unsigned | NO | PRI | NULL | auto_increment | - | Job | tinyblob | NO | | NULL | | - | Name | tinyblob | NO | MUL | NULL | | - | Type | binary(1) | NO | | NULL | | - | Level | binary(1) | NO | | NULL | | - | ClientId | int(11) | YES | | 0 | | - | JobStatus | binary(1) | NO | | NULL | | - | SchedTime | datetime | YES | | NULL | | - | StartTime | datetime | YES | | NULL | | - | EndTime | datetime | YES | | NULL | | - | RealEndTime | datetime | YES | | NULL | | - | JobTDate | bigint(20) unsigned | YES | | 0 | | - | VolSessionId | int(10) unsigned | YES | | 0 | | - | VolSessionTime | int(10) unsigned | YES | | 0 | | - | JobFiles | int(10) unsigned | YES | | 0 | | - | JobBytes | bigint(20) unsigned | YES | | 0 | | - | ReadBytes | bigint(20) unsigned | YES | | 0 | | - | JobErrors | int(10) unsigned | YES | | 0 | | - | JobMissingFiles | int(10) unsigned | YES | | 0 | | - | PoolId | int(10) unsigned | YES | | 0 | | - | FileSetId | int(10) unsigned | YES | | 0 | | - | PriorJobId | int(10) unsigned | YES | | 0 | | - | PurgedFiles | tinyint(4) | YES | | 0 | | - | HasBase | tinyint(4) | YES | | 0 | | - | HasCache | tinyint(4) | YES | | 0 | | - | Reviewed | tinyint(4) | YES | | 0 | | - | Comment | blob | YES | | NULL | | - +-----------------+---------------------+------+-----+---------+----------------+ - 27 rows in set (0,00 sec) - - mysql> select * from Version; - +-----------+ - | VersionId | - +-----------+ - | 2002 | - +-----------+ - 1 row in set (0.00 sec) - - mysql> exit - Bye - -Modify database credentials -''''''''''''''''''''''''''' - -If you want to change the Bareos database credentials, do the following: - -- stop the Bareos director - -- modify the configuration - -- rerun the grant script :command:`grant_bareos_privileges` (or modify database user directly) - -- start the Bareos director - -Modify the configuration, set a new password: - -.. code-block:: bareosconfig - :caption: bareos-dir.d/catalog/MyCatalog.conf - - Catalog { - Name = MyCatalog - dbdriver = "mysql" - dbname = "bareos" - dbuser = "bareos" - dbpassword = "MyNewSecretPassword" - } - -Rerun the Bareos grant script :command:`grant_bareos_privileges` ... - -.. code-block:: shell-session - :caption: Modify database privileges - - export db_password=MyNewSecretPassword - /usr/lib/bareos/scripts/grant_bareos_privileges - - - -Sqlite -^^^^^^ - -There are different versions of Sqlite available. When we use the term Sqlite, we will always refer to Sqlite3. - -Sqlite is a file based database. Access via network connection is not supported. Because its setup is easy, it is a good database for testing. However, please don’t use it in a production environment. - -Sqlite stores a database in a single file. Bareos creates this file at :file:`/var/lib/bareos/bareos.db`. - -Sqlite does not offer access permissions. The only permissions that do apply are the Unix file permissions. - -The database is accessable by the following command: - -.. code-block:: shell-session - :caption: Verify Bareos database on Sqlite3 (bareos-13.2.3) - - sqlite3 /var/lib/bareos/bareos.db - SQLite version 3.7.6.3 - Enter ".help" for instructions - Enter SQL statements terminated with a ";" - sqlite> .tables - BaseFiles Filename Media Pool - CDImages Job MediaType Quota - Client JobHisto NDMPJobEnvironment RestoreObject - Counters JobMedia NDMPLevelMap Status - Device JobStats NextId Storage - DeviceStats Location Path UnsavedFiles - File LocationLog PathHierarchy Version - FileSet Log PathVisibility - sqlite> select * from Version; - 2002 - sqlite> - Retention Periods ----------------- @@ -801,7 +581,6 @@ To setup a scheduled admin job for vacuuming the file table, the following must #includedir /etc/sudoers.d - #. Create the following admin job in the director configuration @@ -838,156 +617,6 @@ The same considerations apply as for :ref:`RepairingMySQL`. Consult the PostgreS For Bareos specific problems, consider using :ref:`bareos-dbcheck` program. -MySQL/MariaDB -------------- - -:index:`\ ` -:index:`\ ` - - -.. _section-MysqlSupport: - -MySQL/MariaDB Support -~~~~~~~~~~~~~~~~~~~~~ - -:index:`\ `\ - -As MariaDB is a fork of MySQL, we use MySQL as synonym for MariaDB and fully support it. We test our packages against the preferred MySQL fork that a distribution provides. - - -.. _CompactingMySQL: - -Compacting Your MySQL Database -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:index:`\ `\ - -Over time, as noted above, your database will tend to grow. Even though Bareos regularly prunes files, MySQL does not automatically reuse the space, and instead continues growing. - -Let's assume that you are using the InnoDB database engine (which is the default since MySQL Version 5.5). - -It is recommended that you use the OPTIMIZE TABLE and ANALYZE TABLE statements regularly. This is to make sure that all indices are up to date and to recycle space inside the database files. - -You can do this via the mysqlcheck command: - -:: - - mysqlcheck -a -o -A - - - -Please note that the database files are never shrunk by MySQL. If you really need to shrink the database files, you need to recreate the database. This only works if you use per-table tablespaces by setting the innodb_file_per_table configuration option. See `http://dev.mysql.com/doc/refman/5.5/en/innodb-multiple-tablespaces.html `_ for details. - - -.. _RepairingMySQL: - -Repairing Your MySQL Database -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:index:`\ `\ :index:`\ `\ - - - - - -If you find that you are getting errors writing to your MySQL database, or Bareos hangs each time it tries to access the database, you should consider running MySQL’s database check and repair routines. - -This can be done by running the :command:`mysqlcheck` command: - -:: - - mysqlcheck --all-databases - - - -If the errors you are getting are simply SQL warnings, then you might try running :command:`bareos-dbcheck` before (or possibly after) using the MySQL database repair program. It can clean up many of the orphaned record problems, and certain other inconsistencies in the Bareos database. - -A typical cause of MySQL database problems is if your partition fills. In such a case, you will need to create additional space on the partition. - -MySQL Table is Full -~~~~~~~~~~~~~~~~~~~ - -:index:`\ `\ :index:`\ `\ - -If you are running into the error The table ’File’ is full ..., it is probably because on version 4.x MySQL, the table is limited by default to a maximum size of 4 GB and you have probably run into the limit. The solution can be found at: `http://dev.mysql.com/doc/refman/5.0/en/full-table.html `_ - -You can display the maximum length of your table with: - - - -:: - - mysql bareos - SHOW TABLE STATUS FROM bareos like "File"; - - - -If the column labeled "Max_data_length" is around 4Gb, this is likely to be the source of your problem, and you can modify it with: - - - -:: - - mysql bareos - ALTER TABLE File MAX_ROWS=281474976710656; - - - -MySQL Server Has Gone Away -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:index:`\ `\ :index:`\ `\ If you are having problems with the MySQL server disconnecting or with messages saying that your MySQL server has gone away, then please read the MySQL documentation, which can be found at: - -`http://dev.mysql.com/doc/refman/5.0/en/gone-away.html `_ - -MySQL Temporary Tables -~~~~~~~~~~~~~~~~~~~~~~ - -When doing backups with large numbers of files, MySQL creates some temporary tables. When these tables are small they can be held in system memory, but as they approach some size, they spool off to disk. The default location for these temp tables is /tmp. Once that space fills up, Bareos daemons such as the Storage daemon doing spooling can get strange errors. E.g. - - - -:: - - Fatal error: spool.c:402 Spool data read error. - Fatal error: backup.c:892 Network send error to SD. ERR=Connection reset by - peer - - - -What you need to do is setup MySQL to use a different (larger) temp directory, which can be set in the /etc/my.cnf with these variables set: - - - -:: - - tmpdir=/path/to/larger/tmpdir - bdb_tmpdir=/path/to/larger/tmpdir - - - -MySQL: Lock Wait Timeout -~~~~~~~~~~~~~~~~~~~~~~~~ - -In large environments, the Bareos |mysql| backend may run in a lock wait timeout. This can be seen as Bareos message, e.g.: - -.. code-block:: bareosmessage - :caption: Bareos error message because of |mysql| lock time timeout - - Fatal error: sql_create.c:899 Fill File table Query failed: INSERT INTO File (FileIndex, JobId, PathId, FilenameId, LStat, MD5, DeltaSeq) SELECT batch.FileIndex, batch.JobId, Path.PathId, Filename.FilenameId,batch.LStat, batch.MD5, batch.DeltaSeq FROM batch JOIN Path ON (batch.Path = Path.Path) JOIN Filename ON (batch.Name = Filename.Name): ERR=Lock wait timeout exceeded; try restarting transaction - -In this case the |mysql| ``innodb_lock_wait_timeout`` must be increased. A value of 300 should be sufficient. - -.. code-block:: cfg - :caption: /etc/my.cnf.d/server.cnf - - ... - [mysqld] - innodb_lock_wait_timeout = 300 - ... - - - .. _BackingUpBareos: Backing Up Your Bareos Database