From 6d13fda3fdd11d69d6ae7eeabb8ebc10046e7ba2 Mon Sep 17 00:00:00 2001 From: Philipp Storz Date: Wed, 12 Dec 2018 10:54:47 +0100 Subject: [PATCH] docs: add autogenerated .rst files We add the autogenerated .rst files to be able to use sphinxcontrib-versioning. --- .../source/appendix-a/requirements.rst | 31 + .../source/appendix-b/supportedoses.rst | 285 +++ .../source/appendix-c/programs.rst | 1477 +++++++++++ .../source/appendix-d/bootstrap.rst | 429 ++++ .../source/appendix-e/verify.rst | 391 +++ .../appendix-f/backward-compatibility.rst | 204 ++ .../source/appendix-g/tables.rst | 90 + .../source/appendix-h/howto.rst | 1201 +++++++++ .../source/appendix-i/rescue.rst | 364 +++ .../source/appendix-j/troubleshooting.rst | 860 +++++++ .../source/appendix-k/debug.rst | 129 + .../source/appendix-l/releasenotes.rst | 776 ++++++ .../source/appendix-m/license.rst | 80 + .../source/chapter01/general.rst | 389 +++ .../source/chapter01/state.rst | 178 ++ .../source/chapter02/install.rst | 418 ++++ .../source/chapter03/webui.rst | 532 ++++ .../source/chapter04/update.rst | 121 + .../source/chapter05/quickstart.rst | 82 + .../source/chapter06/tutorial.rst | 1075 ++++++++ .../source/chapter07/critical.rst | 74 + .../source/chapter08/configure.rst | 1112 +++++++++ .../source/chapter09/dirdconf.rst | 539 ++++ .../source/chapter10/storedconf.rst | 246 ++ .../source/chapter11/filedconf.rst | 166 ++ .../source/chapter12/messagesres.rst | 135 ++ .../source/chapter13/consoleconf.rst | 254 ++ .../source/chapter14/monitorconf.rst | 209 ++ .../source/chapter15/bconsole.rst | 1936 +++++++++++++++ .../source/chapter16/restore.rst | 785 ++++++ .../source/chapter17/disk.rst | 510 ++++ .../source/chapter17/recycling.rst | 557 +++++ .../source/chapter18/pools.rst | 382 +++ .../source/chapter19/autochangers.rst | 899 +++++++ .../chapter20/tape-without-autochanger.rst | 416 ++++ .../source/chapter21/spooling.rst | 80 + .../source/chapter22/migration.rst | 323 +++ .../source/chapter23/always-incremental.rst | 609 +++++ .../source/chapter24/basejob.rst | 56 + .../source/chapter26/plugins.rst | 624 +++++ .../source/chapter27/win32.rst | 717 ++++++ .../chapter28/client-initiated-connection.rst | 88 + .../source/chapter28/lanaddress.rst | 111 + .../source/chapter28/passiveclient.rst | 67 + .../source/chapter29/tls.rst | 302 +++ .../source/chapter30/dataencryption.rst | 156 ++ .../source/chapter31/ndmp.rst | 2158 +++++++++++++++++ .../source/chapter32/catmaintenance.rst | 1513 ++++++++++++ .../source/chapter33/security.rst | 350 +++ 49 files changed, 24486 insertions(+) create mode 100644 docs/manuals/en/new_main_reference/source/appendix-a/requirements.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-b/supportedoses.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-c/programs.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-d/bootstrap.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-e/verify.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-f/backward-compatibility.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-g/tables.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-h/howto.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-i/rescue.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-j/troubleshooting.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-k/debug.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-l/releasenotes.rst create mode 100644 docs/manuals/en/new_main_reference/source/appendix-m/license.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter01/general.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter01/state.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter02/install.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter03/webui.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter04/update.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter05/quickstart.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter06/tutorial.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter07/critical.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter08/configure.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter09/dirdconf.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter10/storedconf.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter11/filedconf.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter12/messagesres.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter13/consoleconf.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter14/monitorconf.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter15/bconsole.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter16/restore.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter17/disk.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter17/recycling.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter18/pools.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter19/autochangers.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter20/tape-without-autochanger.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter21/spooling.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter22/migration.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter23/always-incremental.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter24/basejob.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter26/plugins.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter27/win32.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter28/client-initiated-connection.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter28/lanaddress.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter28/passiveclient.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter29/tls.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter30/dataencryption.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter31/ndmp.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter32/catmaintenance.rst create mode 100644 docs/manuals/en/new_main_reference/source/chapter33/security.rst diff --git a/docs/manuals/en/new_main_reference/source/appendix-a/requirements.rst b/docs/manuals/en/new_main_reference/source/appendix-a/requirements.rst new file mode 100644 index 00000000000..eb9e74acbd1 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-a/requirements.rst @@ -0,0 +1,31 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _SysReqs: + +System Requirements +=================== + +.. index:: + single: System Requirements +.. index:: + pair: Requirements; System + + +- The minimum versions for each of the databases supported by Bareos are: + + - PostgreSQL 8.4 (since Bareos 13.2.3) + + - MySQL 4.1 - 5.6 or compatible fork (e.g. MariaDB), see :ref:`section-MysqlSupport` + + - SQLite 3 + +- Windows release is cross-compiled on Linux + +- Bareos requires a good implementation of pthreads to work. This is not the case on some of the BSD systems. + +- The source code has been written with portability in mind and is mostly POSIX compatible. Thus porting to any POSIX compatible operating system should be relatively easy. + +- Jansson library: + +.. _`jansson}` :raw-latex:`\index[general]{JSON}` :raw-latex:`\index[general]{Jansson!\see{JSON}}` Bareos 15.2.0`: jansson}` :raw-latex:`\index[general]{JSON}` :raw-latex:`\index[general]{Jansson!\see{JSON} Bareos :raw-latex:`\sinceVersion{dir}{requires!jansson}{15.2.0 offers a JSON API mode, see :raw-latex:`\bareosDeveloperGuideApiModeJson`. On some platform, the Jansson library is directory available. On others it can easly be added. For some older platforms, we compile Bareos without JSON API mode. diff --git a/docs/manuals/en/new_main_reference/source/appendix-b/supportedoses.rst b/docs/manuals/en/new_main_reference/source/appendix-b/supportedoses.rst new file mode 100644 index 00000000000..b8cc11bcf77 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-b/supportedoses.rst @@ -0,0 +1,285 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +The Bareos project provides and supports packages that have been released at `http://download.bareos.org/bareos/release/ `_ + +However, the following tabular gives an overview, what components are expected on which platforms to run: + ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| **Operating Systems** | **Version** | **Client Daemon** | **Director Daemon** | **Storage Daemon** | ++==============================================================================================+==================+===============================================================================+===============================================================================+===============================================================================+ +| :raw-latex:`\multicolumn{5}{c}{\textbf{Linux}}` :raw-latex:`\index[general]{Platform!Linux}` | | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Arch Linux :raw-latex:`\index[general]{Platform!Arch Linux}` |   | `X `_ | `X `_ | `X `_ | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| CentOS | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Debian | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Fedora | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Gentoo :raw-latex:`\index[general]{Platform!Gentoo}` |   | `X `_ | `X `_ | `X `_ | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| openSUSE | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| RHEL | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| SLES | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Ubuntu | current | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :ref:`Univention Corporate Linux ` | App Center | v12.4 | v12.4 | v12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :raw-latex:`\multicolumn{5}{c}{\textbf{MS Windows}}` | | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :ref:`MS Windows ` 32bit | 10/8/7 | v12.4 | v15.2 | v15.2 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +|   | 2008/Vista | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +|   | 2003/XP | v12.4–v14.2 | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :ref:`MS Windows ` 64bit | 10/8/2012/7 | v12.4 | v15.2 | v15.2 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +|   | 2008/Vista | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :raw-latex:`\multicolumn{5}{c}{\textbf{Mac OS}}` | | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :ref:`Mac OS X/Darwin ` |   | v14.2 | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :raw-latex:`\multicolumn{5}{c}{\textbf{BSD}}` | | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| FreeBSD :raw-latex:`\index[general]{Platform!FreeBSD}` | :math:`\geq` 5.0 | `X `_ | `X `_ | `X `_ | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| OpenBSD |   | X | |   | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| NetBSD |   | X | |   | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| :raw-latex:`\multicolumn{5}{c}{\textbf{Unix}}` | | | | | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| AIX :raw-latex:`\index[general]{Platform!AIX}` | :math:`\geq` 4.3 | com-13.2 | :math:`\star` | :math:`\star` | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| HP-UX :raw-latex:`\index[general]{Platform!HP-UX}` |   | com-13.2 |   |   | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Irix |   | :math:`\star` |   |   | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| Solaris :raw-latex:`\index[general]{Platform!Solaris}` | :math:`\geq` 8 | com-12.4 | com-12.4 | com-12.4 | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ +| True64 |   | :math:`\star` |   |   | ++----------------------------------------------------------------------------------------------+------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ + ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| **vVV.V** | starting with Bareos version VV.V, this platform is official supported by the Bareos.org project | ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| **com-VV.V** | starting with Bareos version VV.V, this platform is supported. However, pre-build packages are only available from Bareos.com | ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| **nightly** | provided by Bareos nightly build. Bug reports are welcome, however it is not official supported | ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| **X** | known to work | ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ +| **:math:`\star`** | has been reported to work by the community | ++-------------------+-------------------------------------------------------------------------------------------------------------------------------+ + +.. _section-packages: + +Packages for the different Linux platforms +------------------------------------------ + +The following tables summarize what packages are available for the different Linux platforms. + +This information is generated based on `http://download.bareos.com/bareos/release/ `_. In most cases this is identical to the packages provided by `http://download.bareos.org/bareos/release/ `_. Only if a package have been added later in a maintenance release, these information may differ. + +Distributions that are no longer relevant are left out. However, you might still find the packages on our download servers. + +Bareos tries to provide all packages for all current platforms. For extra packages, it depends if the distribution contains the required dependencies. + +For general information about the packages, see :ref:`section-BareosPackages`. + +Packages names not containing the word **bareos** are required packages where we decided to include them ourselves. + + :raw-latex:`\small` :raw-latex:`\input{autogenerated/bareos-packages-table-redhat.tex}` :raw-latex:`\input{autogenerated/bareos-packages-table-fedora.tex}` :raw-latex:`\input{autogenerated/bareos-packages-table-suse.tex}` :raw-latex:`\input{autogenerated/bareos-packages-table-opensuse.tex}` :raw-latex:`\input{autogenerated/bareos-packages-table-debian.tex}` :raw-latex:`\input{autogenerated/bareos-packages-table-ubuntu.tex}` + +Debian.org / Ubuntu Universe +---------------------------- + + +.. index:: + triple: Platform; Debian; Debian.org; + :raw-latex:`\index[general]{Platform!Debian!8}` :raw-latex:`\index[general]{Platform!Ubuntu!Universe}` :raw-latex:`\index[general]{Platform!Ubuntu!Universe!15.04}` + +.. _`section-DebianOrg`: section-DebianOrg + +The distributions of Debian :math:`>=` 8 include a version of Bareos. Ubuntu Universe :math:`>=` 15.04 does also include these packages. + +In the further text, these version will be named **Bareos (Debian.org)** (also for the Ubuntu Universe version, as this is based on the Debian version). + +.. _section-DebianOrgLimitations: + +Limitations of the Debian.org/Ubuntu Universe version of Bareos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Debian.org does not include the libfastlz compression library and therefore the Bareos (Debian.org) packages do not offer the fileset options :option:`compression=LZFAST`, :option:`compression=LZ4` and :option:`compression=LZ4HC`. + +- Debian.org prefers that Bareos (Debian.org) is linked against GnuTLS instead of OpenSSL. Therefore, the Bareos (Debian.org) package only support :ref:`section-TransportEncryption` but no :ref:`DataEncryption`. + +- Debian.org does not include the **bareos-webui** package. + +Mac OS X +-------- + + +.. index:: + triple: Platform; Mac; OS X; + + +.. _`section-macosx`: section-macosx + +Bareos for MacOS X is available either + +- via the `Homebrew project `_ (`http://formulae.brew.sh/formula/bareos-client `_) or + +- as pkg file from `http://download.bareos.org/bareos/release/latest/MacOS/ `_. + +However, you have to choose upfront, which client you want to use. Otherwise conflicts do occur. + +Both packages contain the |bareosFd| and :program:`bconsole`. + +Installing the Bareos Client as PKG +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Installation; MacOS + + +The Bareos installer package for Mac OS X contains the |bareosFd| for Mac OS X 10.5 or later. + +On your local Mac, you must be an admin user. The main user is an admin user. + +Download the :file:`bareos-client*.pkg` installer package from `http://download.bareos.org/bareos/release/latest/MacOS/ `_. + +Find the .pkg you just downloaded. Install the .pkg by holding the CTRL key, left-clicking the installer and choosing :emphasis:`open`. + +Follow the directions given to you and finish the installation. + +Configuration +~~~~~~~~~~~~~ + +To make use of your |bareosFd| on your system, it is required to configure the |bareosDir| and the local |bareosFd|. + +Configure the server-side by follow the instructions at :ref:`section-AddAClient`. + +After configuring the server-side you can either transfer the necessary configuration file using following command or configure the client locally. + +.. raw:: latex + + \subsubsubsection{Option 1: Copy the director resource from the Bareos Director to the Client} + +Assuming your client has the DNS entry :strong:`client2.example.com` and has been added to |bareosDir| as **client2-fd**:sup:`bareos-dir`:sub:`client` : + + + + +.. code-block:: sh + :caption: + + scp /etc/bareos/bareos-dir-export/client/client2-fd/bareos-fd.d/director/bareos-dir.conf root@client2.example.com:/usr/local/etc/bareos/bareos-fd.d/director/ + +This differs in so far, as on Linux the configuration files are located under :file:`/etc/bareos/``, while on MacOS they are located at ``path:/usr/local/etc/bareos/`. + +.. raw:: latex + + \subsubsubsection{Option 2: Edit the director resource on the Client} + +Alternatively, you can edit the file :file:`/usr/local/etc/bareos/bareos-fd.d/director/bareos-dir.conf`. + +This can be done by right-clicking the finder icon in your task bar, select :emphasis:`Go to folder ...` and paste :file:`/usr/local/etc/bareos/bareos-fd.d/director/`. + +Select the :file:`bareos-dir.conf` file and open it. + +Alternatively you can also call following command on the command console: + + + + +.. code-block:: sh + :caption: + + open -t /usr/local/etc/bareos/bareos-fd.d/director/bareos-dir.conf + +The file should look similar to this: + + + + +.. code-block:: sh + :caption: bareos-fd director bareos-dir + + Director { + Name = bareos-dir + Password = "SOME_RANDOM_PASSWORD" + Description = "Allow the configured Director to access this file daemon." + } + +Set this client-side password to the same value as given on the server-side. + +.. raw:: latex + + +.. warning:: + The configuration file contains passwords and therefore must not be accessible for any users except admin users. + +Restart bareos-fd after changing the configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The bareos-fd must be restarted to reread its configuration: + + + + +.. code-block:: sh + :caption: Restart the \bareosFd + + sudo launchctl stop org.bareos.bareos-fd + sudo launchctl start org.bareos.bareos-fd + +Verify that the Bareos File Daemon is working +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Open the :program:`bconsole` on your |bareosDir| and check the status of the client with + + + + +.. code-block:: sh + :caption: + + *status client=client2-fd + +In case, the client does not react, following command are useful the check the status: + + + + +.. code-block:: sh + :caption: Verify the status of \bareosFd + + # check if bareos-fd is started by system: + sudo launchctl list org.bareos.bareos-fd + + # get process id (PID) of bareos-fd + pgrep bareos-fd + + # show files opened by bareos-fd + sudo lsof -p `pgrep bareos-fd` + + # check what process is listening on the \bareosFd port + sudo lsof -n -iTCP:9102 | grep LISTEN + +You can also manually start bareos-fd in debug mode by: + + + + +.. code-block:: sh + :caption: Start \bareosFd in debug mode + + sudo /usr/local/sbin/bareos-fd -f -d 100 diff --git a/docs/manuals/en/new_main_reference/source/appendix-c/programs.rst b/docs/manuals/en/new_main_reference/source/appendix-c/programs.rst new file mode 100644 index 00000000000..4eafba56d5c --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-c/programs.rst @@ -0,0 +1,1477 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _section-Utilities: + +Bareos Programs +=============== + +Bareos Daemons +-------------- + +Daemon Command Line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Daemon; Command Line Options +.. index:: + pair: Command Line Options; Daemon + + +Each of the three daemons (Director, File, Storage) accepts a small set of options on the command line. In general, each of the daemons as well as the Console program accepts the following options: + +-c + Define the file or directory to use for the configuration. See :ref:`section-ConfigurationPathLayout`. + +-d nnn + Set the debug level to **nnn**. Generally anything between 50 and 200 is reasonable. The higher the number, the more output is produced. The output is written to standard output. The debug level can also be set during runtime, see section :ref:`bconsole: setdebug `. + +-f + Run the daemon in the foreground. This option is needed to run the daemon under the debugger. + +-g + Run the daemon under this group. This must be a group name, not a GID. + +-s + Do not trap signals. This option is needed to run the daemon under the debugger. + +-t + Read the configuration file and print any error messages, then immediately exit. Useful for syntax testing of new configuration files. + +-u + Run the daemon as this user. This must be a user name, not a UID. + +-v + Be more verbose or more complete in printing error and informational messages. + +-xc + Print the current configuration and exit. + +-xs + Print configuration schema in JSON format and exit. + +-? + Print the version and list of options. + +.. _command-bareos-dir: + +bareos-dir +~~~~~~~~~~ + +.. index:: + single: Command Line Options + + +|bareosDir|. + +.. _command-bareos-sd: + +bareos-sd +~~~~~~~~~ + +.. index:: + single: Command Line Options + + +|bareosSd|. + +.. _command-bareos-fd: + +bareos-fd +~~~~~~~~~ + +.. index:: + single: Command Line Options + + +|bareosFd|. + +Interactive Programs +-------------------- + +bconsole +~~~~~~~~ + +There is an own chapter on :program:`bconsole`. Please refer to chapter :ref:`section-bconsole`. + +bareos-webui +~~~~~~~~~~~~ + +For further information regarding the Bareos Webui, please refer to :ref:`section-webui`. + +bat +~~~ + +.. index:: + pair: Command; bat + + +.. _`bat`: bat + +The Bacula/Bareos Administration Tool (:program:`bat`) has been a native GUI for Bareos. It has been marked deprecated since 17.2.0 it is no longer part of Bareos. We encourage the use of |bareosWebui| instead. + +Volume Utility Commands +----------------------- + +.. index:: + single: Volume Utility Tools +.. index:: + pair: Tools; Volume Utility + + +.. _`section-VolumeUtilityCommands`: section-VolumeUtilityCommands + +This document describes the utility programs written to aid Bareos users and developers in dealing with Volumes external to Bareos and to perform other useful tasks. + +Parameter +~~~~~~~~~ + +Specifying the Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each of the utilities that deal with Volumes require a valid |bareosSd| configuration (actually, the only part of the configuration file that these programs need is the :sup:`Sd` :strong:`Device` resource definitions). This permits the programs to find the configuration parameters for your **Archive Device**:sup:`Sd`:sub:`Device` . Using the :option:`-c` option a custom |bareosSd| configuration file or directory can be +selected. + +Specifying a Device +^^^^^^^^^^^^^^^^^^^ + +Each of these programs require a :option:`device-name` where the Volume can be found. The device-name is either the name of the |bareosSd| device (**Name**:sup:`Sd`:sub:`Device` ) or its **Archive Device**:sup:`Sd`:sub:`Device` . + +Specifying a Device Name For a Tape +''''''''''''''''''''''''''''''''''' + +In the case of a tape, this is the physical device name such as **/dev/nst0** or **/dev/rmt/0ubn** depending on your system. + +.. raw:: latex + + +.. warning:: + If you have Bareos running and you want to use + one of these programs, you will either need to stop the \bareosSd + or :strong:`unmount` any tape drive you want to use, + otherwise the drive may get busy because Bareos is using it. + After this, you can use the command \command{mtx or \ilink{mtx-changer script}{section-MtxChangerManualUsage} + to load the required volume into the tape drive. + } + +Specifying a Device Name For a File +''''''''''''''''''''''''''''''''''' + +If you are attempting to read or write an archive file rather than a tape, the :option:`device-name` can be the full path to the archive location specified at **Archive Device**:sup:`Sd`:sub:`Device` or this including the filename of the volume. The filename (last part of the specification) will be stripped and used as the Volume name So, the path is equivalent to the **Archive Device**:sup:`Sd`:sub:`Device` and the filename is +equivalent to the volume name. + +Specifying Volumes +^^^^^^^^^^^^^^^^^^ + +.. index:: + single: Bootstrap + + +Often you must specify the Volume name to the programs below. The best method to do so is to specify a bootstrap file on the command line with the :option:`-b` option. As part of the bootstrap file, you will then specify the Volume name or Volume names if more than one volume is needed. For example, suppose you want to read tapes :raw-latex:`\volume{tapevolume1}` and :raw-latex:`\volume{tapevolume2}`. First construct a **bootstrap** file named say, :file:`list.bsr` +which contains: + +.. raw:: latex + + + + + + Volume=tapevolume1|tapevolume2 + +.. raw:: latex + + + +where each Volume is separated by a vertical bar. Then simply use: + + + + +.. code-block:: sh + :caption: + + bls -b list.bsr /dev/nst0 + +In the case of Bareos Volumes that are on files, you may simply append volumes as follows: + + + + +.. code-block:: sh + :caption: + + bls /var/lib/bareos/storage/volume1\|volume2 + +where the backslash (\) was necessary as a shell escape to permit entering the vertical bar (|). + +And finally, if you feel that specifying a Volume name is a bit complicated with a bootstrap file, you can use the :option:`-V` option (on all programs except :program:`bcopy`) to specify one or more Volume names separated by the vertical bar (|). For example: + + + + +.. code-block:: sh + :caption: + + bls /dev/nst0 -V tapevolume1 + +You may also specify an asterisk (*) to indicate that the program should accept any volume. For example: + + + + +.. code-block:: sh + :caption: + + bls /dev/nst0 -V* + +If your |bareosSd| has following resource, + + + + +.. code-block:: sh + :caption: bareos-sd device FileStorage + + Device { + Name = FileStorage + Archive Device = /var/lib/bareos/storage + ... + } + +following calls of :program:`bls` should behave identical: + + + + +.. code-block:: sh + :caption: bls using Storage Device Name + + bls FileStorage -V Full1 + +or + + + + +.. code-block:: sh + :caption: bls using the Archive Device of a Storage Device + + bls /var/lib/bareos/storage -V Full1 + +or + + + + +.. code-block:: sh + :caption: bls using the Archive Device of a Storage Device and volume name + + bls /var/lib/bareos/storage/Full1 + +Specifying Maximum Block Size +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you use Bareos with non-default block sizes defined in the pools (**Maximum Block Size**:sup:`Dir`:sub:`Pool` ), it might be necessary to specify the **Maximum Block Size**:sup:`Sd`:sub:`Device` also in the storage device resource, see :ref:`Direct access to Volumes with non-default blocksizes `. + +bls +~~~ + +.. index:: + single: bls +.. index:: + pair: Command; bls + + +:program:`bls` can be used to do an :program:`ls` type listing of a Bareos tape or file. It is called: + + + + +.. code-block:: sh + :caption: + + Usage: bls [options] + -b specify a bootstrap file + -c specify a Storage configuration file + -D specify a director name specified in the Storage + configuration file for the Key Encryption Key selection + -d set debug level to + -dt print timestamp in debug output + -e exclude list + -i include list + -j list jobs + -k list blocks + (no j or k option) list saved files + -L dump label + -p proceed inspite of errors + -v be verbose + -V specify Volume names (separated by |) + -? print this message + +Normally if no options are specified, :program:`bls` will produce the equivalent output to the :program:`ls -l` command for each volume. + +For example, to list the contents of a tape: + + + + +.. code-block:: sh + :caption: + + bls -V Volume-name /dev/nst0 + +Or to list the contents of a volume file: + + + + +.. code-block:: sh + :caption: + + bls FileStorage -V Full1 + +or + + + + +.. code-block:: sh + :caption: + + bls /var/lib/bareos/storage -V Full1 + +or + + + + +.. code-block:: sh + :caption: + + bls /var/lib/bareos/storage/Full1 + +For example: + + + + +.. code-block:: sh + :caption: + + bls FileStorage -V Full1 + bls: butil.c:282-0 Using device: "/var/lib/bareos/storage" for reading. + 12-Sep 18:30 bls JobId 0: Ready to read from volume "Full1" on device "FileStorage" (/var/lib/bareos/storage). + bls JobId 1: -rwxr-xr-x 1 root root 4614 2013-01-22 22:24:11 /usr/sbin/service + bls JobId 1: -rwxr-xr-x 1 root root 13992 2013-01-22 22:24:12 /usr/sbin/rtcwake + bls JobId 1: -rwxr-xr-x 1 root root 6243 2013-02-06 11:01:29 /usr/sbin/update-fonts-scale + bls JobId 1: -rwxr-xr-x 1 root root 43240 2013-01-22 22:24:10 /usr/sbin/grpck + bls JobId 1: -rwxr-xr-x 1 root root 16894 2013-01-22 22:24:11 /usr/sbin/update-rc.d + bls JobId 1: -rwxr-xr-x 1 root root 9480 2013-01-22 22:47:43 /usr/sbin/gss_clnt_send_err + ... + bls JobId 456: -rw-r----- 1 root bareos 1008 2013-05-23 13:17:45 /etc/bareos/bareos-fd.conf + bls JobId 456: drwxr-xr-x 2 root root 4096 2013-07-04 17:40:21 /etc/bareos/ + 12-Sep 18:30 bls JobId 0: End of Volume at file 0 on device "FileStorage" (/var/lib/bareos/storage), Volume "Full1" + 12-Sep 18:30 bls JobId 0: End of all volumes. + 2972 files found. + +Show Detailed File Information +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To retrieve information, about how a file is stored on the volume, you can use :program:`bls` in verbose mode: + + + + +.. code-block:: sh + :caption: + + bls FileStorage -V TestVolume001 -v + bls: butil.c:273-0 Using device: "FileStorage" for reading. + 22-Jun 19:34 bls JobId 0: Ready to read from volume "TestVolume001" on device "Storage1" (/var/lib/bareos/storage). + Volume Label Record: VolSessionId=1 VolSessionTime=1498152622 JobId=0 DataLen=168 + Begin Job Session Record: VolSessionId=1 VolSessionTime=1498152622 JobId=1 DataLen=169 + FileIndex=1 Stream=1 UATTR DataLen=129 | -rw-rw-r-- 1 root root 5 2017-06-22 19:30:21 + | /srv/data/test1.dat + FileIndex=1 Stream=29 COMPRESSED DataLen=25 | GZIP, level=9, version=1, length=13 + FileIndex=1 Stream=3 MD5 DataLen=16 | 2Oj8otwPiW/Xy0ywAxuiSQ (base64) + FileIndex=2 Stream=1 UATTR DataLen=123 | drwxrwxr-x 2 root root 4096 2017-06-22 19:30:21 + | /srv/data/ + ... + End Job Session Record: VolSessionId=1 VolSessionTime=1498152622 JobId=1 + DataLen=205 + 22-Jun 19:34 bls JobId 0: End of Volume at file 0 on device "FileStorage" (/var/lib/bareos/storage), Volume "TestVolume001" + 22-Jun 19:34 bls JobId 0: End of all volumes. + End of Physical Medium Record: VolSessionId=0 VolSessionTime=0 JobId=0 DataLen=0 + 9 files and directories found. + +For details about the Volume format, see :raw-latex:`\bareosDeveloperGuideStorageMediaOutputFormat`. + +Show Label Information +^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: bls; Label + + +Using the :option:`-L` the label information of a Volume is shown: + + + + +.. code-block:: sh + :caption: bls: show volume label + + bls -L /var/lib/bareos/storage/testvol + bls: butil.c:282-0 Using device: "/var/lib/bareos/storage" for reading. + 12-Sep 18:41 bls JobId 0: Ready to read from volume "testvol" on device "FileStorage" (/var/lib/bareos/storage). + + Volume Label: + Id : Bareos 0.9 mortal + VerNo : 10 + VolName : File002 + PrevVolName : + VolFile : 0 + LabelType : VOL_LABEL + LabelSize : 147 + PoolName : Default + MediaType : File + PoolType : Backup + HostName : debian6 + Date label written: 06-Mar-2013 17:21 + +Listing Jobs +^^^^^^^^^^^^ + +.. index:: + single: Listing Jobs with bls +.. index:: + pair: bls; Listing Jobs + + +If you are listing a Volume to determine what Jobs to restore, normally the :option:`-j` option provides you with most of what you will need as long as you don’t have multiple clients. For example: + + + + +.. code-block:: sh + :caption: bls: list jobs + + bls /var/lib/bareos/storage/testvol -j + bls: butil.c:282-0 Using device: "/var/lib/bareos/storage" for reading. + 12-Sep 18:33 bls JobId 0: Ready to read from volume "testvol" on device "FileStorage" (/var/lib/bareos/storage). + Volume Record: File:blk=0:193 SessId=1 SessTime=1362582744 JobId=0 DataLen=158 + Begin Job Session Record: File:blk=0:64705 SessId=1 SessTime=1362582744 JobId=1 + Job=BackupClient1.2013-03-06_17.22.48_05 Date=06-Mar-2013 17:22:51 Level=F Type=B + End Job Session Record: File:blk=0:6499290 SessId=1 SessTime=1362582744 JobId=1 + Date=06-Mar-2013 17:22:52 Level=F Type=B Files=162 Bytes=6,489,071 Errors=0 Status=T + Begin Job Session Record: File:blk=0:6563802 SessId=2 SessTime=1362582744 JobId=2 + Job=BackupClient1.2013-03-06_23.05.00_02 Date=06-Mar-2013 23:05:02 Level=I Type=B + End Job Session Record: File:blk=0:18832687 SessId=2 SessTime=1362582744 JobId=2 + Date=06-Mar-2013 23:05:02 Level=I Type=B Files=3 Bytes=12,323,791 Errors=0 Status=T + ... + Begin Job Session Record: File:blk=0:319219736 SessId=299 SessTime=1369307832 JobId=454 + Job=BackupClient1.2013-09-11_23.05.00_25 Date=11-Sep-2013 23:05:03 Level=I Type=B + End Job Session Record: File:blk=0:319219736 SessId=299 SessTime=1369307832 JobId=454 + Date=11-Sep-2013 23:05:03 Level=I Type=B Files=0 Bytes=0 Errors=0 Status=T + Begin Job Session Record: File:blk=0:319284248 SessId=301 SessTime=1369307832 JobId=456 + Job=BackupCatalog.2013-09-11_23.10.00_28 Date=11-Sep-2013 23:10:03 Level=F Type=B + End Job Session Record: File:blk=0:320694269 SessId=301 SessTime=1369307832 JobId=456 + Date=11-Sep-2013 23:10:03 Level=F Type=B Files=12 Bytes=1,472,681 Errors=0 Status=T + 12-Sep 18:32 bls JobId 0: End of Volume at file 0 on device "FileStorage" (/var/lib/bareos/storage), Volume "testvol" + 12-Sep 18:32 bls JobId 0: End of all volumes. + +Adding the :option:`-v` option will display virtually all information that is available for each record. + +Listing Blocks +^^^^^^^^^^^^^^ + +.. index:: + single: Listing Blocks with bls +.. index:: + pair: bls; Listing Blocks + + +Normally, except for debugging purposes, you will not need to list Bareos blocks (the "primitive" unit of Bareos data on the Volume). However, you can do so with: + + + + +.. code-block:: sh + :caption: + + bls -k /tmp/File002 + bls: butil.c:148 Using device: /tmp + Block: 1 size=64512 + Block: 2 size=64512 + ... + Block: 65 size=64512 + Block: 66 size=19195 + bls: Got EOF on device /tmp + End of File on device + +By adding the :option:`-v` option, you can get more information, which can be useful in knowing what sessions were written to the volume: + + + + +.. code-block:: sh + :caption: + + bls -k -v /tmp/File002 + Date label written: 2002-10-19 at 21:16 + Block: 1 blen=64512 First rec FI=VOL_LABEL SessId=1 SessTim=1035062102 Strm=0 rlen=147 + Block: 2 blen=64512 First rec FI=6 SessId=1 SessTim=1035062102 Strm=DATA rlen=4087 + Block: 3 blen=64512 First rec FI=12 SessId=1 SessTim=1035062102 Strm=DATA rlen=5902 + Block: 4 blen=64512 First rec FI=19 SessId=1 SessTim=1035062102 Strm=DATA rlen=28382 + ... + Block: 65 blen=64512 First rec FI=83 SessId=1 SessTim=1035062102 Strm=DATA rlen=1873 + Block: 66 blen=19195 First rec FI=83 SessId=1 SessTim=1035062102 Strm=DATA rlen=2973 + bls: Got EOF on device /tmp + End of File on device + +Armed with the SessionId and the SessionTime, you can extract just about anything. + +If you want to know even more, add a second :option:`-v` to the command line to get a dump of every record in every block. + + + + +.. code-block:: sh + :caption: + + bls -k -vv /tmp/File002 + bls: block.c:79 Dump block 80f8ad0: size=64512 BlkNum=1 + Hdrcksum=b1bdfd6d cksum=b1bdfd6d + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=VOL_LABEL Strm=0 len=147 p=80f8b40 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=SOS_LABEL Strm=-7 len=122 p=80f8be7 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=1 Strm=UATTR len=86 p=80f8c75 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=2 Strm=UATTR len=90 p=80f8cdf + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=3 Strm=UATTR len=92 p=80f8d4d + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=3 Strm=DATA len=54 p=80f8dbd + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=3 Strm=MD5 len=16 p=80f8e07 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=4 Strm=UATTR len=98 p=80f8e2b + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=4 Strm=DATA len=16 p=80f8ea1 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=4 Strm=MD5 len=16 p=80f8ec5 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=5 Strm=UATTR len=96 p=80f8ee9 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=5 Strm=DATA len=1783 p=80f8f5d + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=5 Strm=MD5 len=16 p=80f9668 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=UATTR len=95 p=80f968c + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=DATA len=32768 p=80f96ff + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=DATA len=32768 p=8101713 + bls: block.c:79 Dump block 80f8ad0: size=64512 BlkNum=2 + Hdrcksum=9acc1e7f cksum=9acc1e7f + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=contDATA len=4087 p=80f8b40 + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=DATA len=31970 p=80f9b4b + bls: block.c:92 Rec: VId=1 VT=1035062102 FI=6 Strm=MD5 len=16 p=8101841 + ... + +bextract +~~~~~~~~ + +.. index:: + single: bextract +.. index:: + pair: Command; bextract + +.. index:: + triple: Disaster; Recovery; bextract; + + +If you find yourself using :program:`bextract`, you probably have done something wrong. For example, if you are trying to recover a file but are having problems, please see the :ref:`section-RestoreCatalog` chapter. + +Normally, you will restore files by running a **Restore** Job from the **Console** program. However, :program:`bextract` can be used to extract a single file or a list of files from a Bareos tape or file. In fact, :program:`bextract` can be a useful tool to restore files to an empty system assuming you are able to boot, you have statically linked :program:`bextract` and you have an appropriate **bootstrap** file. + +Please note that some of the current limitations of :program:`bextract` are: + +#. It cannot restore access control lists (ACL) that have been backed up along with the file data. + +#. It cannot restore encrypted files. + +#. The command line length is relatively limited, which means that you cannot enter a huge number of volumes. If you need to enter more volumes than the command line supports, please use a bootstrap file (see below). + +#. Extracting files from a Windows backup on a Linux system will only extract the plain files, not the additional Windows file information. If you have to extract files from a Windows backup, you should use the Windows version of :program:`bextract`. + +It is called: + + + + +.. code-block:: sh + :caption: + + Usage: bextract + -b specify a bootstrap file + -c specify a Storage configuration file + -D specify a director name specified in the Storage + configuration file for the Key Encryption Key selection + -d set debug level to + -dt print timestamp in debug output + -e exclude list + -i include list + -p proceed inspite of I/O errors + -v verbose + -V specify Volume names (separated by |) + -? print this message + +where **device-name** is the Archive Device (raw device name or full filename) of the device to be read, and **directory-to-store-files** is a path prefix to prepend to all the files restored. + +.. raw:: latex + + \warning{On Windows systems, if you specify a prefix of say d:/tmp, any file that + would have been restored to \verb|path:C:/My Documents| will be restored to \verb|path:D:/tmp/My Documents|. + That is, the original drive specification will be + stripped. If no prefix is specified, the file will be restored to the original + drive.} + +Extracting with Include or Exclude Lists +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using the **-e** option, you can specify a file containing a list of files to be excluded. Wildcards can be used in the exclusion list. This option will normally be used in conjunction with the **-i** option (see below). Both the **-e** and the **-i** options may be specified at the same time as the **-b** option. The bootstrap filters will be applied first, then the include list, then the exclude list. + +Likewise, and probably more importantly, with the **-i** option, you can specify a file that contains a list (one file per line) of files and directories to include to be restored. The list must contain the full filename with the path. If you specify a path name only, all files and subdirectories of that path will be restored. If you specify a line containing only the filename (e.g. **my-file.txt**) it probably will not be extracted because you have not specified the full path. + +For example, if the file **include-list** contains: + +.. raw:: latex + + + + + + /etc/bareos + /usr/sbin + +.. raw:: latex + + + +Then the command: + + + + +.. code-block:: sh + :caption: + + bextract -i include-list -V Volume /dev/nst0 /tmp + +will restore from the Bareos archive **/dev/nst0** all files and directories in the backup from **/etc/bareos** and from **/usr/sbin**. The restored files will be placed in a file of the original name under the directory **/tmp** (i.e. /tmp/etc/bareos/... and /tmp/usr/sbin/...). + +Extracting With a Bootstrap File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The **-b** option is used to specify a **bootstrap** file containing the information needed to restore precisely the files you want. Specifying a **bootstrap** file is optional but recommended because it gives you the most control over which files will be restored. For more details on the **bootstrap** file, please see :ref:`Restoring Files with the Bootstrap File ` chapter of this document. Note, you may also use a bootstrap file produced by the **restore** +command. For example: + + + + +.. code-block:: sh + :caption: + + bextract -b bootstrap-file /dev/nst0 /tmp + +The bootstrap file allows detailed specification of what files you want restored (extracted). You may specify a bootstrap file and include and/or exclude files at the same time. The bootstrap conditions will first be applied, and then each file record seen will be compared to the include and exclude lists. + +Extracting From Multiple Volumes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you wish to extract files that span several Volumes, you can specify the Volume names in the bootstrap file or you may specify the Volume names on the command line by separating them with a vertical bar. See the section above under the **bls** program entitled **Listing Multiple Volumes** for more information. The same techniques apply equally well to the **bextract** program or read the :ref:`Bootstrap ` chapter of this document. + +Extracting Under Windows +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: Windows; bextract + + +.. raw:: latex + + \warning{If you use \command{bextract} under Windows, the ordering of the parameters is essential.} + +To use :program:`bextract`, the Bareos Storage Daemon must be installed. As bextract works on tapes or disk volumes, these must be configured in the Storage Daemon configuration file, normally found at :file:`C:\ProgrammData\Bareos\bareos-sd.conf`. However, it is not required to start the Bareos Storage Daemon. Normally, if the Storage Daemon would be able to run, :program:`bextract` would not be required. + +After installing, :program:`bextract` can be called via command line: + + + + +.. code-block:: sh + :caption: Call of bextract + + C:\Program Files\Bareos .\bextract.exe -c "C:\ProgrammData\Bareos\bareos-sd.conf" -V + +If you want to use exclude or include files you need to write them like you do on Linux. That means each path begins with a "/" and not with "yourdrive:/". You need to specify the parameter **-e exclude.list** as first parameter. For example: + + + + +.. code-block:: sh + :caption: Example exclude.list + + /Program Files/Bareos/bareos-dir.exe + /ProgramData/ + + + + +.. code-block:: sh + :caption: Call bextract with exclude list + + C:\Program Files\Bareos .\bextract.exe -e exclude.list -c "C:\ProgrammData\Bareos\bareos-sd.conf" -V + +bscan +~~~~~ + +.. index:: + single: bscan +.. index:: + pair: Command; bscan + + +If you find yourself using this program, you have probably done something wrong. For example, the best way to recover a lost or damaged Bareos database is to reload the database by using the bootstrap file that was written when you saved it (default Bareos-dir.conf file). + +The **bscan** program can be used to re-create a database (catalog) records from the backup information written to one or more Volumes. This is normally needed only if one or more Volumes have been pruned or purged from your catalog so that the records on the Volume are no longer in the catalog, or for Volumes that you have archived. Note, if you scan in Volumes that were previously purged, you will be able to do restores from those Volumes. However, unless you modify the Job and File retention +times for the Jobs that were added by scanning, the next time you run any backup Job with the same name, the records will be pruned again. Since it takes a long time to scan Volumes this can be very frustrating. + +With some care, :program:`bscan` can also be used to synchronize your existing catalog with a Volume. Although we have never seen a case of bscan damaging a catalog, since bscan modifies your catalog, we recommend that you do a simple ASCII backup of your database before running :program:`bscan` just to be sure. See :ref:`Compacting Your Database ` for the details of making a copy of your database. + +:program:`bscan` can also be useful in a disaster recovery situation, after the loss of a hard disk, if you do not have a valid **bootstrap** file for reloading your system, or if a Volume has been recycled but not overwritten, you can use :program:`bscan` to re-create your database, which can then be used to **restore** your system or a file to its previous state. + +It is called: + + + + +.. code-block:: sh + :caption: + + Usage: bscan [options] + -B specify the database driver name (default NULL) + -b bootstrap specify a bootstrap file + -c specify configuration file + -d set debug level to nn + -dt print timestamp in debug output + -m update media info in database + -D specify a director name specified in the Storage + configuration file for the Key Encryption Key selection + -n specify the database name (default Bareos) + -u specify database user name (default Bareos) + -P specify database password (default none) + -h specify database host (default NULL) + -t specify database port (default 0) + -p proceed inspite of I/O errors + -r list records + -s synchronize or store in database + -S show scan progress periodically + -v verbose + -V specify Volume names (separated by |) + -w specify working directory (default from conf file) + -? print this message + +As Bareos supports loading its database backend dynamically you need to specify the right database driver to use using the **-B** option. + +If you are using MySQL or PostgreSQL, there is no need to supply a working directory since in that case, bscan knows where the databases are. However, if you have provided security on your database, you may need to supply either the database name (**-b** option), the user name (**-u** option), and/or the password (**-p**) options. + +NOTE: before :program:`bscan` can work, it needs at least a bare bones valid database. If your database exists but some records are missing because they were pruned, then you are all set. If your database was lost or destroyed, then you must first ensure that you have the SQL program running (MySQL or PostgreSQL), then you must create the Bareos database (normally named bareos), and you must create the Bareos tables. This is explained in :ref:`section-CreateDatabase` +chapter of the manual. Finally, before scanning into an empty database, you must start and stop the Director with the appropriate Bareos-dir.conf file so that it can create the Client and Storage records which are not stored on the Volumes. Without these records, scanning is unable to connect the Job records to the proper client. + +Forgetting for the moment the extra complications of a full rebuild of your catalog, let’s suppose that you did a backup to Volumes "Vol001" and "Vol002", then sometime later all records of one or both those Volumes were pruned or purged from the database. By using **bscan** you can recreate the catalog entries for those Volumes and then use the **restore** command in the Console to restore whatever you want. A command something like: + + + + +.. code-block:: sh + :caption: + + bscan -v -V Vol001|Vol002 /dev/nst0 + +will give you an idea of what is going to happen without changing your catalog. Of course, you may need to change the path to the Storage daemon’s conf file, the Volume name, and your tape (or disk) device name. This command must read the entire tape, so if it has a lot of data, it may take a long time, and thus you might want to immediately use the command listed below. Note, if you are writing to a disk file, replace the device name with the path to the directory that contains the Volumes. +This must correspond to the Archive Device in the conf file. + +Then to actually write or store the records in the catalog, add the **-s** option as follows: + + + + +.. code-block:: sh + :caption: + + bscan -s -m -v -V Vol001|Vol002 /dev/nst0 + +When writing to the database, if :program:`bscan` finds existing records, it will generally either update them if something is wrong or leave them alone. Thus if the Volumes you are scanning are all or partially in the catalog already, no harm will be done to that existing data. Any missing data will simply be added. + +If you have multiple tapes, you should scan them with: + + + + +.. code-block:: sh + :caption: + + bscan -s -m -v -V Vol001|Vol002|Vol003 /dev/nst0 + +Since there is a limit on the command line length (511 bytes) accepted by :program:`bscan`, if you have too many Volumes, you will need to manually create a bootstrap file. See the :ref:`Bootstrap ` chapter of this manual for more details, in particular the section entitled :ref:`Bootstrap for bscan `. Basically, the .bsr file for the above example might look like: + +.. raw:: latex + + + + + + Volume=Vol001 + Volume=Vol002 + Volume=Vol003 + +.. raw:: latex + + + +Note: :program:`bscan` does not support supplying Volume names on the command line and at the same time in a bootstrap file. Please use only one or the other. + +You should, always try to specify the tapes in the order they are written. If you do not, any Jobs that span a volume may not be fully or properly restored. However, bscan can handle scanning tapes that are not sequential. Any incomplete records at the end of the tape will simply be ignored in that case. If you are simply repairing an existing catalog, this may be OK, but if you are creating a new catalog from scratch, it will leave your database in an incorrect state. If you do not specify all +necessary Volumes on a single bscan command, bscan will not be able to correctly restore the records that span two volumes. In other words, it is much better to specify two or three volumes on a single bscan command (or in a .bsr file) rather than run bscan two or three times, each with a single volume. + +Note, the restoration process using bscan is not identical to the original creation of the catalog data. This is because certain data such as Client records and other non-essential data such as volume reads, volume mounts, etc is not stored on the Volume, and thus is not restored by bscan. The results of bscanning are, however, perfectly valid, and will permit restoration of any or all the files in the catalog using the normal Bareos console commands. If you are starting with an empty catalog +and expecting bscan to reconstruct it, you may be a bit disappointed, but at a minimum, you must ensure that your Bareos-dir.conf file is the same as what it previously was – that is, it must contain all the appropriate Client resources so that they will be recreated in your new database **before** running bscan. Normally when the Director starts, it will recreate any missing Client records in the catalog. Another problem you will have is that even if the Volumes (Media records) are recreated in +the database, they will not have their autochanger status and slots properly set. As a result, you will need to repair that by using the :strong:`update slots` command. There may be other considerations as well. Rather than bscanning, you should always attempt to recover you previous catalog backup. + +Using bscan to Compare a Volume to an existing Catalog +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: Catalog; Using bscan to Compare a Volume to an existing + + +If you wish to compare the contents of a Volume to an existing catalog without changing the catalog, you can safely do so if and only if you do **not** specify either the **-m** or the **-s** options. However, the comparison routines are not as good or as thorough as they should be, so we don’t particularly recommend this mode other than for testing. + +Using bscan to Recreate a Catalog from a Volume +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: Catalog; Recreate Using bscan +.. index:: + pair: bscan; Recreate Catalog + + +This is the mode for which **bscan** is most useful. You can either **bscan** into a freshly created catalog, or directly into your existing catalog (after having made an ASCII copy as described above). Normally, you should start with a freshly created catalog that contains no data. + +Starting with a single Volume named **TestVolume1**, you run a command such as: + + + + +.. code-block:: sh + :caption: + + bscan -V TestVolume1 -v -s -m /dev/nst0 + +If there is more than one volume, simply append it to the first one separating it with a vertical bar. You may need to precede the vertical bar with a forward slash escape the shell – e.g. **TestVolume1:raw-latex:`\textbar{}`TestVolume2**. The **-v** option was added for verbose output (this can be omitted if desired). The **-s** option that tells :program:`bscan` to store information in the database. The physical device name **/dev/nst0** is specified after all the options. + +For example, after having done a full backup of a directory, then two incrementals, I reinitialized the SQLite database as described above, and using the bootstrap.bsr file noted above, I entered the following command: + + + + +.. code-block:: sh + :caption: + + bscan -b bootstrap.bsr -v -s /dev/nst0 + +which produced the following output: + + + + +.. code-block:: sh + :caption: + + bscan: bscan.c:182 Using Database: Bareos, User: bacula + bscan: bscan.c:673 Created Pool record for Pool: Default + bscan: bscan.c:271 Pool type "Backup" is OK. + bscan: bscan.c:632 Created Media record for Volume: TestVolume1 + bscan: bscan.c:298 Media type "DDS-4" is OK. + bscan: bscan.c:307 VOL_LABEL: OK for Volume: TestVolume1 + bscan: bscan.c:693 Created Client record for Client: Rufus + bscan: bscan.c:769 Created new JobId=1 record for original JobId=2 + bscan: bscan.c:717 Created FileSet record "Users Files" + bscan: bscan.c:819 Updated Job termination record for new JobId=1 + bscan: bscan.c:905 Created JobMedia record JobId 1, MediaId 1 + bscan: Got EOF on device /dev/nst0 + bscan: bscan.c:693 Created Client record for Client: Rufus + bscan: bscan.c:769 Created new JobId=2 record for original JobId=3 + bscan: bscan.c:708 Fileset "Users Files" already exists. + bscan: bscan.c:819 Updated Job termination record for new JobId=2 + bscan: bscan.c:905 Created JobMedia record JobId 2, MediaId 1 + bscan: Got EOF on device /dev/nst0 + bscan: bscan.c:693 Created Client record for Client: Rufus + bscan: bscan.c:769 Created new JobId=3 record for original JobId=4 + bscan: bscan.c:708 Fileset "Users Files" already exists. + bscan: bscan.c:819 Updated Job termination record for new JobId=3 + bscan: bscan.c:905 Created JobMedia record JobId 3, MediaId 1 + bscan: Got EOF on device /dev/nst0 + bscan: bscan.c:652 Updated Media record at end of Volume: TestVolume1 + bscan: bscan.c:428 End of Volume. VolFiles=3 VolBlocks=57 VolBytes=10,027,437 + +The key points to note are that **bscan** prints a line when each major record is created. Due to the volume of output, it does not print a line for each file record unless you supply the **-v** option twice or more on the command line. + +In the case of a Job record, the new JobId will not normally be the same as the original Jobid. For example, for the first JobId above, the new JobId is 1, but the original JobId is 2. This is nothing to be concerned about as it is the normal nature of databases. **bscan** will keep everything straight. + +Although :program:`bscan` claims that it created a Client record for Client: Rufus three times, it was actually only created the first time. This is normal. + +You will also notice that it read an end of file after each Job (Got EOF on device ...). Finally the last line gives the total statistics for the bscan. + +If you had added a second **-v** option to the command line, Bareos would have been even more verbose, dumping virtually all the details of each Job record it encountered. + +Now if you start Bareos and enter a :strong:`list jobs` command to the console program, you will get: + + + + +.. code-block:: sh + :caption: list jobs + + +-------+----------+------------------+------+-----+----------+----------+---------+ + | JobId | Name | StartTime | Type | Lvl | JobFiles | JobBytes | JobStat | + +-------+----------+------------------+------+-----+----------+----------+---------+ + | 1 | usersave | 2002-10-07 14:59 | B | F | 84 | 4180207 | T | + | 2 | usersave | 2002-10-07 15:00 | B | I | 15 | 2170314 | T | + | 3 | usersave | 2002-10-07 15:01 | B | I | 33 | 3662184 | T | + +-------+----------+------------------+------+-----+----------+----------+---------+ + +which corresponds virtually identically with what the database contained before it was re-initialized and restored with bscan. All the Jobs and Files found on the tape are restored including most of the Media record. The Volume (Media) records restored will be marked as **Full** so that they cannot be rewritten without operator intervention. + +It should be noted that :program:`bscan` cannot restore a database to the exact condition it was in previously because a lot of the less important information contained in the database is not saved to the tape. Nevertheless, the reconstruction is sufficiently complete, that you can run **restore** against it and get valid results. + +An interesting aspect of restoring a catalog backup using :program:`bscan` is that the backup was made while Bareos was running and writing to a tape. At the point the backup of the catalog is made, the tape Bareos is writing to will have say 10 files on it, but after the catalog backup is made, there will be 11 files on the tape Bareos is writing. This there is a difference between what is contained in the backed up catalog and what is actually on the tape. If after restoring a +catalog, you attempt to write on the same tape that was used to backup the catalog, Bareos will detect the difference in the number of files registered in the catalog compared to what is on the tape, and will mark the tape in error. + +There are two solutions to this problem. The first is possibly the simplest and is to mark the volume as Used before doing any backups. The second is to manually correct the number of files listed in the Media record of the catalog. This procedure is documented elsewhere in the manual and involves using the :strong:`update volume` command in :program:`bconsole`. + +Using bscan to Correct the Volume File Count +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: bscan; Correct Volume File Count +.. index:: + pair: Volume; File Count + + +If the Storage daemon crashes during a backup Job, the catalog will not be properly updated for the Volume being used at the time of the crash. This means that the Storage daemon will have written say 20 files on the tape, but the catalog record for the Volume indicates only 19 files. + +Bareos refuses to write on a tape that contains a different number of files from what is in the catalog. To correct this situation, you may run a **bscan** with the **-m** option (but without the **-s** option) to update only the final Media record for the Volumes read. + +After bscan +^^^^^^^^^^^ + +.. index:: + pair: bscan; after + + +If you use **bscan** to enter the contents of the Volume into an existing catalog, you should be aware that the records you entered may be immediately pruned during the next job, particularly if the Volume is very old or had been previously purged. To avoid this, after running **bscan**, you can manually set the volume status (VolStatus) to **Read-Only** by using the **update** command in the catalog. This will allow you to restore from the volume without having it immediately purged. When you +have restored and backed up the data, you can reset the VolStatus to **Used** and the Volume will be purged from the catalog. + +bcopy +~~~~~ + +.. index:: + single: bcopy +.. index:: + pair: Command; bcopy + + +The :program:`bcopy` program can be used to copy one Bareos archive file to another. For example, you may copy a tape to a file, a file to a tape, a file to a file, or a tape to a tape. For tape to tape, you will need two tape drives. In the process of making the copy, no record of the information written to the new Volume is stored in the catalog. This means that the new Volume, though it contains valid backup data, cannot be accessed directly from existing catalog entries. If you +wish to be able to use the Volume with the Console restore command, for example, you must first bscan the new Volume into the catalog. + + + + +.. code-block:: sh + :caption: + + Usage: bcopy [-d debug_level] + -b bootstrap specify a bootstrap file + -c specify configuration file + -D specify a director name specified in the Storage + configuration file for the Key Encryption Key selection + -dnn set debug level to nn + -dt print timestamp in debug output + -i specify input Volume names (separated by |) + -o specify output Volume names (separated by |) + -p proceed inspite of I/O errors + -v verbose + -w dir specify working directory (default /tmp) + -? print this message + +By using a **bootstrap** file, you can copy parts of a Bareos archive file to another archive. + +One of the objectives of this program is to be able to recover as much data as possible from a damaged tape. However, the current version does not yet have this feature. + +As this is a new program, any feedback on its use would be appreciated. In addition, I only have a single tape drive, so I have never been able to test this program with two tape drives. + +btape +~~~~~ + +.. index:: + single: btape +.. index:: + pair: Command; btape + + +This program permits a number of elementary tape operations via a tty command interface. It works only with tapes and not with other kinds of Bareos storage media (DVD, File, ...). The **test** command, described below, can be very useful for testing older tape drive compatibility problems. Aside from initial testing of tape drive compatibility with **Bareos**, **btape** will be mostly used by developers writing new tape drivers. + +**btape** can be dangerous to use with existing **Bareos** tapes because it will relabel a tape or write on the tape if so requested regardless that the tape may contain valuable data, so please be careful and use it only on blank tapes. + +To work properly, :program:`btape` needs to read the Storage daemon’s configuration file. + +The physical device name must be specified on the command line, and this same device name must be present in the Storage daemon’s configuration file read by :program:`btape`. + + + + +.. code-block:: sh + :caption: + + Usage: btape + -b specify bootstrap file + -c set configuration file to file + -D specify a director name specified in the Storage + configuration file for the Key Encryption Key selection + -d set debug level to nn + -dt print timestamp in debug output + -p proceed inspite of I/O errors + -s turn off signals + -v be verbose + -? print this message. + +Using btape to Verify your Tape Drive +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + pair: Drive; Verify using btape + + +An important reason for this program is to ensure that a Storage daemon configuration file is defined so that Bareos will correctly read and write tapes. + +It is highly recommended that you run the **test** command before running your first Bareos job to ensure that the parameters you have defined for your storage device (tape drive) will permit **Bareos** to function properly. You only need to mount a blank tape, enter the command, and the output should be reasonably self explanatory. Please see the :ref:`Tape Testing ` Chapter of this manual for the details. + +btape Commands +^^^^^^^^^^^^^^ + +The full list of commands are: + + + + +.. code-block:: sh + :caption: btape commands + + Command Description + ======= =========== + autochanger test autochanger + bsf backspace file + bsr backspace record + cap list device capabilities + clear clear tape errors + eod go to end of Bareos data for append + eom go to the physical end of medium + fill fill tape, write onto second volume + unfill read filled tape + fsf forward space a file + fsr forward space a record + help print this command + label write a Bareos label to the tape + load load a tape + quit quit btape + rawfill use write() to fill tape + readlabel read and print the Bareos tape label + rectest test record handling functions + rewind rewind the tape + scan read() tape block by block to EOT and report + scanblocks Bareos read block by block to EOT and report + speed report drive speed + status print tape status + test General test Bareos tape functions + weof write an EOF on the tape + wr write a single Bareos block + rr read a single record + qfill quick fill command + +The most useful commands are: + +- test – test writing records and EOF marks and reading them back. + +- fill – completely fill a volume with records, then write a few records on a second volume, and finally, both volumes will be read back. This command writes blocks containing random data, so your drive will not be able to compress the data, and thus it is a good test of the real physical capacity of your tapes. + +- readlabel – read and dump the label on a Bareos tape. + +- cap – list the device capabilities and status. + +The **readlabel** command can be used to display the details of a Bareos tape label. This can be useful if the physical tape label was lost or damaged. + +In the event that you want to relabel a Bareos volume, you can simply use the **label** command which will write over any existing label. However, please note for labeling tapes, we recommend that you use the **label** command in the **Console** program since it will never overwrite a valid Bareos tape. + +.. _section-btapespeed: + +Testing your Tape Drive +''''''''''''''''''''''' + +To determine the best configuration of your tape drive, you can run the new ``speed`` command available in the ``btape`` program. + +This command can have the following arguments: + +- Specify the **Maximum File Size**:sup:`Sd`:sub:`Device` for this test. This counter is in GB. + +- Specify the number of file to be written. The amount of data should be greater than your memory (:math:`file\_size*nb\_file`). + +- This flag permits to skip tests with constant data. + +- This flag permits to skip tests with random data. + +- This flag permits to skip tests with raw access. + +- This flag permits to skip tests with Bareos block access. + + + + +.. code-block:: sh + :caption: btape speed + + *speed file_size=3 skip_raw + btape.c:1078 Test with zero data and Bareos block structure. + btape.c:956 Begin writing 3 files of 3.221 GB with blocks of 129024 bytes. + ++++++++++++++++++++++++++++++++++++++++++ + btape.c:604 Wrote 1 EOF to "Drive-0" (/dev/nst0) + btape.c:406 Volume bytes=3.221 GB. Write rate = 44.128 MB/s + ... + btape.c:383 Total Volume bytes=9.664 GB. Total Write rate = 43.531 MB/s + + btape.c:1090 Test with random data, should give the minimum throughput. + btape.c:956 Begin writing 3 files of 3.221 GB with blocks of 129024 bytes. + +++++++++++++++++++++++++++++++++++++++++++ + btape.c:604 Wrote 1 EOF to "Drive-0" (/dev/nst0) + btape.c:406 Volume bytes=3.221 GB. Write rate = 7.271 MB/s + +++++++++++++++++++++++++++++++++++++++++++ + ... + btape.c:383 Total Volume bytes=9.664 GB. Total Write rate = 7.365 MB/s + +When using compression, the random test will give your the minimum throughput of your drive . The test using constant string will give you the maximum speed of your hardware chain. (cpu, memory, scsi card, cable, drive, tape). + +You can change the block size in the Storage Daemon configuration file. + +bscrypto +~~~~~~~~ + +.. index:: + single: bscrypto +.. index:: + pair: Command; bscrypto + + +:program:`bscrypto` is used in the process of encrypting tapes (see also :ref:`LTOHardwareEncryptionGeneral`). The |bareosSd| and the btools (:program:`bls`, :program:`bextract`, :program:`bscan`, :program:`btape`, :program:`bextract`) will use a so called |bareosSd| plugin to perform the setting and clearing of the encryption keys. To bootstrap the encryption support and for +populating things like the crypto cache with encryption keys of volumes that you want to scan, you need to use the bscrypto tool. The bscrypto tool has the following capabilities: + +- Generate a new passphrase + + - | to be used as a so called Key Encryption Key (KEK) for wrapping a passphrase using RFC3394 key wrapping with aes-wrap + | - or - + + - for usage as a clear text encryption key loaded into the tape drive. + +- Base64-encode a key if requested + +- Generate a wrapped passphrase which performs the following steps: + + - generate a semi random clear text passphrase + + - wrap the passphrase using the Key Encryption Key using RFC3394 + + - base64-encode the wrapped key (as the wrapped key is binary, we always need to base64-encode it in order to be able to pass the data as part of the director to storage daemon protocol + +- | show the content of a wrapped or unwrapped keyfile. + | This can be used to reveal the content of the passphrase when a passphrase is stored in the database and you have the urge to change the Key Encryption Key. Normally it is unwise to change the Key Encryption Key, as this means that you have to redo all your stored encryption keys, as they are stored in the database wrapped using the Key Encryption Key available in the config during the label phase of the volume. + +- Clear the crypto cache on the machine running the bareos-sd, which keeps a cache of used encryption keys, which can be used when the bareos-sd is restarted without the need to connect to the bareos-dir to retrieve the encryption keys. + +- Set the encryption key of the drive + +- Clear the encryption key of the drive + +- Show the encryption status of the drive + +- Show the encryption status of the next block (e.g. volume) + +- Populate the crypto cache with data + +Other Programs +-------------- + +The following programs are general utility programs and in general do not need a configuration file nor a device name. + +bsmtp +~~~~~ + +.. index:: + single: bsmtp +.. index:: + pair: Command; bsmtp + + +:program:`bsmtp` is a simple mail transport program that permits more flexibility than the standard mail programs typically found on Unix systems. It can even be used on Windows machines. + +It is called: + + + + +.. code-block:: sh + :caption: bsmtp + + Usage: bsmtp [-f from] [-h mailhost] [-s subject] [-c copy] [recipient ...] + -4 forces bsmtp to use IPv4 addresses only. + -6 forces bsmtp to use IPv6 addresses only. + -8 set charset to UTF-8 + -a use any ip protocol for address resolution + -c set the Cc: field + -d set debug level to + -dt print a timestamp in debug output + -f set the From: field + -h use mailhost:port as the SMTP server + -s set the Subject: field + -r set the Reply-To: field + -l set the maximum number of lines to send (default: unlimited) + -? print this message. + +If the **-f** option is not specified, :program:`bsmtp` will use your userid. If the option **-h** is not specified :program:`bsmtp` will use the value in the environment variable **bsmtpSERVER** or if there is none **localhost**. By default port 25 is used. + +If a line count limit is set with the **-l** option, :program:`bsmtp` will not send an email with a body text exceeding that number of lines. This is especially useful for large restore job reports where the list of files restored might produce very long mails your mail-server would refuse or crash. However, be aware that you will probably suppress the job report and any error messages unless you check the log file written by the Director (see the messages resource in this manual for +details). + +**recipients** is a space separated list of email recipients. + +The body of the email message is read from standard input. + +An example of the use of :program:`bsmtp` would be to put the following statement in the :ref:`Messages resource ` of your |bareosDir| configuration. + + + + +.. code-block:: sh + :caption: bsmtp in Message resource + + Mail Command = "bsmtp -h mail.example.com -f \"\(Bareos\) %r\" -s \"Bareos: %t %e of %c %l\" %r" + Operator Command = "bsmtp -h mail.example.com -f \"\(Bareos\) %r\" -s \"Bareos: Intervention needed for %j\" %r" + +You have to replace **mail.example.com** with the fully qualified name of your SMTP (email) server, which normally listens on port 25. For more details on the substitution characters (e.g. %r) used in the above line, please see the documentation of the :ref:`MailCommand in the Messages Resource ` chapter of this manual. + +It is HIGHLY recommended that you test one or two cases by hand to make sure that the **mailhost** that you specified is correct and that it will accept your email requests. Since **bsmtp** always uses a TCP connection rather than writing in the spool file, you may find that your **from** address is being rejected because it does not contain a valid domain, or because your message is caught in your spam filtering rules. Generally, you should specify a fully qualified domain name in the **from** +field, and depending on whether your bsmtp gateway is Exim or Sendmail, you may need to modify the syntax of the from part of the message. Please test. + +When running :program:`bsmtp` by hand, you will need to terminate the message by entering a ctrl-d in column 1 of the last line. + +If you are getting incorrect dates (e.g. 1970) and you are running with a non-English language setting, you might try adding a :program:`LANG=C` immediately before the :program:`bsmtp` call. + +In general, :program:`bsmtp` attempts to cleanup email addresses that you specify in the from, copy, mailhost, and recipient fields, by adding the necessary < and > characters around the address part. However, if you include a **display-name** (see RFC 5332), some SMTP servers such as Exchange may not accept the message if the **display-name** is also included in < and >. As mentioned above, you must test, and if you run into this situation, you may manually add the < and > to the +Bareos **Mail Command**:sup:`Dir`:sub:`Messages` or **Operator Command**:sup:`Dir`:sub:`Messages` and when :program:`bsmtp` is formatting an address if it already contains a < or > character, it will leave the address unchanged. + +bareos-dbcheck +~~~~~~~~~~~~~~ + + + +.. _`dbcheck}` :raw-latex:`\index[general]{bareos-dbcheck}` :raw-latex:`\index[general]{Command!bareos-dbcheck}` :raw-latex:`\index[general]{Catalog!database check`: dbcheck}` :raw-latex:`\index[general]{bareos-dbcheck}` :raw-latex:`\index[general]{Command!bareos-dbcheck}` :raw-latex:`\index[general]{Catalog!database check + +:program:`bareos-dbcheck` is a simple program that will search for logical inconsistencies in the Bareos tables in your database, and optionally fix them. It is a database maintenance routine, in the sense that it can detect and remove unused rows, but it is not a database repair routine. To repair a database, see the tools furnished by the database vendor. Normally :program:`bareos-dbcheck` should never need to be run, but if Bareos has crashed or you have a lot of +Clients, Pools, or Jobs that you have removed, it could be useful. + +:program:`bareos-dbcheck` is best started as the same user, as the |bareosDir| is running, normally **bareos}`. If you are :raw-latex:`**root**` on Linux, use the following command to switch to user :raw-latex:`\user{bareos**: + + + + +.. code-block:: sh + :caption: Substitute user to bareos + + su -s /bin/bash - bareos + +If not, problems of reading the Bareos configuration or accessing the database can arise. + +:program:`bareos-dbcheck` supports following command line options: + + + + +.. code-block:: sh + :caption: + + Usage: bareos-dbcheck [-c config ] [-B] [-C catalog name] [-d debug level] [-D driver name] [] [] + -b batch mode + -C catalog name in the director conf file + -c Director configuration filename or configuration directory (e.g. /etc/bareos) + -B print catalog configuration and exit + -d set debug level to + -dt print a timestamp in debug output + -D specify the database driver name (default NULL) + -f fix inconsistencies + -v verbose + -? print this message + +When using the default configuration paths, it is not necessary to specify any options. Optionally, as Bareos supports loading its database backend dynamically you may specify the right database driver to use using the :option:`-D` option. + +If the :option:`-B` option is specified, :program:`bareos-dbcheck` will print out catalog information in a simple text based format: + + + + +.. code-block:: sh + :caption: + + # bareos-dbcheck -B + catalog=MyCatalog + db_type=SQLite + db_name=bareos + db_driver= + db_user=bareos + db_password= + db_address= + db_port=0 + db_socket= + +If the :option:`-c` option is given with the |bareosDir| configuration, there is no need to enter any of the command line arguments, in particular the working directory as :program:`bareos-dbcheck` will read them from the file. + +If the :option:`-f` option is specified, :program:`bareos-dbcheck` will repair (**fix**) the inconsistencies it finds. Otherwise, it will report only. + +If the :option:`-b` option is specified, :program:`bareos-dbcheck` will run in batch mode, and it will proceed to examine and fix (if :option:`-f` is set) all programmed inconsistency checks. If the :option:`-b` option is not specified, :program:`bareos-dbcheck` will enter interactive mode and prompt with the following: + + + + +.. code-block:: sh + :caption: + + Hello, this is the database check/correct program. + Modify database is off. Verbose is off. + Please select the function you want to perform. + 1) Toggle modify database flag + 2) Toggle verbose flag + 3) Repair bad Filename records + 4) Repair bad Path records + 5) Eliminate duplicate Filename records + 6) Eliminate duplicate Path records + 7) Eliminate orphaned Jobmedia records + 8) Eliminate orphaned File records + 9) Eliminate orphaned Path records + 10) Eliminate orphaned Filename records + 11) Eliminate orphaned FileSet records + 12) Eliminate orphaned Client records + 13) Eliminate orphaned Job records + 14) Eliminate all Admin records + 15) Eliminate all Restore records + 16) All (3-15) + 17) Quit + Select function number: + +By entering 1 or 2, you can toggle the modify database flag (:raw-latex:`\parameter{-f}` option) and the verbose flag (:raw-latex:`\parameter{-v}`). It can be helpful and reassuring to turn off the modify database flag, then select one or more of the consistency checks (items 3 through 13) to see what will be done, then toggle the modify flag on and re-run the check. + +Since Bareos 16.2.5, when running :program:`bareos-dbcheck` with :option:`-b` and :option:`-v`, it will not interactively ask if results should be printed or not. Instead, it does not print any detail results. + +The inconsistencies examined are the following: + +- Duplicate Filename records. This can happen if you accidentally run two copies of Bareos at the same time, and they are both adding filenames simultaneously. It is a rare occurrence, but will create an inconsistent database. If this is the case, you will receive error messages during Jobs warning of duplicate database records. If you are not getting these error messages, there is no reason to run this check. + +- Repair bad Filename records. This checks and corrects filenames that have a trailing slash. They should not. + +- Repair bad Path records. This checks and corrects path names that do not have a trailing slash. They should. + +- Duplicate Path records. This can happen if you accidentally run two copies of Bareos at the same time, and they are both adding filenames simultaneously. It is a rare occurrence, but will create an inconsistent database. See the item above for why this occurs and how you know it is happening. + +- Orphaned JobMedia records. This happens when a Job record is deleted (perhaps by a user issued SQL statement), but the corresponding JobMedia record (one for each Volume used in the Job) was not deleted. Normally, this should not happen, and even if it does, these records generally do not take much space in your database. However, by running this check, you can eliminate any such orphans. + +- Orphaned File records. This happens when a Job record is deleted (perhaps by a user issued SQL statement), but the corresponding File record (one for each Volume used in the Job) was not deleted. Note, searching for these records can be **very** time consuming (i.e. it may take hours) for a large database. Normally this should not happen as Bareos takes care to prevent it. Just the same, this check can remove any orphaned File records. It is recommended that you run this once a year since + orphaned File records can take a large amount of space in your database. You might want to ensure that you have indexes on JobId, FilenameId, and PathId for the File table in your catalog before running this command. + +- Orphaned Path records. This condition happens any time a directory is deleted from your system and all associated Job records have been purged. During standard purging (or pruning) of Job records, Bareos does not check for orphaned Path records. As a consequence, over a period of time, old unused Path records will tend to accumulate and use space in your database. This check will eliminate them. It is recommended that you run this check at least once a year. + +- Orphaned Filename records. This condition happens any time a file is deleted from your system and all associated Job records have been purged. This can happen quite frequently as there are quite a large number of files that are created and then deleted. In addition, if you do a system update or delete an entire directory, there can be a very large number of Filename records that remain in the catalog but are no longer used. + + During standard purging (or pruning) of Job records, Bareos does not check for orphaned Filename records. As a consequence, over a period of time, old unused Filename records will accumulate and use space in your database. This check will eliminate them. It is strongly recommended that you run this check at least once a year, and for large database (more than 200 Megabytes), it is probably better to run this once every 6 months. + +- Orphaned Client records. These records can remain in the database long after you have removed a client. + +- Orphaned Job records. If no client is defined for a job or you do not run a job for a long time, you can accumulate old job records. This option allow you to remove jobs that are not attached to any client (and thus useless). + +- All Admin records. This command will remove all Admin records, regardless of their age. + +- All Restore records. This command will remove all Restore records, regardless of their age. + +If you are using MySQL, :program:`bareos-dbcheck` in interactive mode will ask you if you want to create temporary indexes to speed up orphaned Path and Filename elimination. In batch mode (:raw-latex:`\parameter{-b}`) the temporary indexes will be created without asking. + +If you are using bvfs (e.g. used by :ref:`bareos-webui `), don’t eliminate orphaned path, else you will have to rebuild :raw-latex:`\variable{brestore_pathvisibility}` and :raw-latex:`\variable{brestore_pathhierarchy}` indexes. + +Normally you should never need to run :program:`bareos-dbcheck` in spite of the recommendations given above, which are given so that users don’t waste their time running :program:`bareos-dbcheck` too often. + +bregex +~~~~~~ + +.. index:: + single: bregex +.. index:: + pair: Command; bregex + + +:program:`bregex` is a simple program that will allow you to test regular expressions against a file of data. This can be useful because the regex libraries on most systems differ, and in addition, regex expressions can be complicated. + +To run it, use: + + + + Usage: bregex [-d debug_level] -f + -f specify file of data to be matched + -l suppress line numbers + -n print lines that do not match + -? print this message. + +The is a filename that contains lines of data to be matched (or not) against one or more patterns. When the program is run, it will prompt you for a regular expression pattern, then apply it one line at a time against the data in the file. Each line that matches will be printed preceded by its line number. You will then be prompted again for another pattern. + +Enter an empty line for a pattern to terminate the program. You can print only lines that do not match by using the -n option, and you can suppress printing of line numbers with the -l option. + +This program can be useful for testing regex expressions to be applied against a list of filenames. + +bwild +~~~~~ + +.. index:: + single: bwild +.. index:: + pair: Command; bwild + + +:program:`bwild` is a simple program that will allow you to test wild-card expressions against a file of data. + +To run it, use: + + + + Usage: bwild [-d debug_level] -f + -f specify file of data to be matched + -l suppress line numbers + -n print lines that do not match + -? print this message. + +The is a filename that contains lines of data to be matched (or not) against one or more patterns. When the program is run, it will prompt you for a wild-card pattern, then apply it one line at a time against the data in the file. Each line that matches will be printed preceded by its line number. You will then be prompted again for another pattern. + +Enter an empty line for a pattern to terminate the program. You can print only lines that do not match by using the -n option, and you can suppress printing of line numbers with the -l option. + +This program can be useful for testing wild expressions to be applied against a list of filenames. + +bpluginfo +~~~~~~~~~ + +.. index:: + single: bpluginfo +.. index:: + pair: Command; bpluginfo + + +The main purpose of bpluginfo is to display different information about Bareos plugin. You can use it to check a plugin name, author, license and short description. You can use -f option to display API implemented by the plugin. Some plugins may require additional ’-a’ option for val- idating a Bareos Daemons API. In most cases it is not required. diff --git a/docs/manuals/en/new_main_reference/source/appendix-d/bootstrap.rst b/docs/manuals/en/new_main_reference/source/appendix-d/bootstrap.rst new file mode 100644 index 00000000000..7227d001a75 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-d/bootstrap.rst @@ -0,0 +1,429 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _BootstrapChapter: + +The Bootstrap File +================== + +.. index:: + pair: File; Bootstrap +.. index:: + pair: Bootstrap; File + + +.. raw:: latex + + \TODO{This chapter is going to be rewritten (by Philipp).} + +The information in this chapter is provided so that you may either create your own bootstrap files, or so that you can edit a bootstrap file produced by Bareos. However, normally the bootstrap file will be automatically created for you during the :strong:`restore` in the Console program, or by using a **Write Bootstrap**:sup:`Dir`:sub:`Job` record in your Backup Jobs, and thus you will never need to know the details of this file. + +The **bootstrap** file contains ASCII information that permits precise specification of what files should be restored, what volume they are on, and where they are on the volume. It is a relatively compact form of specifying the information, is human readable, and can be edited with any text editor. + +Bootstrap File Format +--------------------- + +.. index:: + pair: Bootstrap; File Format + + +The general format of a **bootstrap** file is: + +**= ** + +Where each **keyword** and the **value** specify which files to restore. More precisely the **keyword** and their **values** serve to limit which files will be restored and thus act as a filter. The absence of a keyword means that all records will be accepted. + +Blank lines and lines beginning with a pound sign (#) in the bootstrap file are ignored. + +There are keywords which permit filtering by Volume, Client, Job, FileIndex, Session Id, Session Time, ... + +The more keywords that are specified, the more selective the specification of which files to restore will be. In fact, each keyword is **AND**\ ed with other keywords that may be present. + +For example, + +.. raw:: latex + + + + + + Volume = Test-001 + VolSessionId = 1 + VolSessionTime = 108927638 + +.. raw:: latex + + + +directs the Storage daemon (or the **bextract** program) to restore only those files on Volume Test-001 **AND** having VolumeSessionId equal to one **AND** having VolumeSession time equal to 108927638. + +The full set of permitted keywords presented in the order in which they are matched against the Volume records are: + +Volume +.. index:: + pair: Bootstrap; Volume + The value field specifies what Volume the following commands apply to. Each Volume specification becomes the current Volume, to which all the following commands apply until a new current Volume (if any) is specified. If the Volume name contains spaces, it should be enclosed in quotes. At lease one Volume specification is required. + +Count +.. index:: + pair: Bootstrap; Count + The value is the total number of files that will be restored for this Volume. This allows the Storage daemon to know when to stop reading the Volume. This value is optional. + +VolFile +.. index:: + pair: Bootstrap; VolFile + The value is a file number, a list of file numbers, or a range of file numbers to match on the current Volume. The file number represents the physical file on the Volume where the data is stored. For a tape volume, this record is used to position to the correct starting file, and once the tape is past the last specified file, reading will stop. + +VolBlock +.. index:: + pair: Bootstrap; VolBlock + The value is a block number, a list of block numbers, or a range of block numbers to match on the current Volume. The block number represents the physical block within the file on the Volume where the data is stored. + +VolSessionTime +.. index:: + pair: Bootstrap; VolSessionTime + The value specifies a Volume Session Time to be matched from the current volume. + +VolSessionId +.. index:: + pair: Bootstrap; VolSessionId + The value specifies a VolSessionId, a list of volume session ids, or a range of volume session ids to be matched from the current Volume. Each VolSessionId and VolSessionTime pair corresponds to a unique Job that is backed up on the Volume. + +JobId +.. index:: + pair: Bootstrap; JobId + The value specifies a JobId, list of JobIds, or range of JobIds to be selected from the current Volume. Note, the JobId may not be unique if you have multiple Directors, or if you have reinitialized your database. The JobId filter works only if you do not run multiple simultaneous jobs. This value is optional and not used by Bareos to restore files. + +Job +.. index:: + pair: Bootstrap; Job + The value specifies a Job name or list of Job names to be matched on the current Volume. The Job corresponds to a unique VolSessionId and VolSessionTime pair. However, the Job is perhaps a bit more readable by humans. Standard regular expressions (wildcards) may be used to match Job names. The Job filter works only if you do not run multiple simultaneous jobs. This value is optional and not used by Bareos to restore files. + +Client +.. index:: + pair: Bootstrap; Client + The value specifies a Client name or list of Clients to will be matched on the current Volume. Standard regular expressions (wildcards) may be used to match Client names. The Client filter works only if you do not run multiple simultaneous jobs. This value is optional and not used by Bareos to restore files. + +FileIndex +.. index:: + pair: Bootstrap; FileIndex + The value specifies a FileIndex, list of FileIndexes, or range of FileIndexes to be selected from the current Volume. Each file (data) stored on a Volume within a Session has a unique FileIndex. For each Session, the first file written is assigned FileIndex equal to one and incremented for each file backed up. + + This for a given Volume, the triple VolSessionId, VolSessionTime, and FileIndex uniquely identifies a file stored on the Volume. Multiple copies of the same file may be stored on the same Volume, but for each file, the triple VolSessionId, VolSessionTime, and FileIndex will be unique. This triple is stored in the Catalog database for each file. + + To restore a particular file, this value (or a range of FileIndexes) is required. + +FileRegex +.. index:: + pair: Bootstrap; FileRegex + The value is a regular expression. When specified, only matching filenames will be restored. + + + + FileRegex=^/etc/passwd(.old)? + +Slot +.. index:: + pair: Bootstrap; Slot + The value specifies the autochanger slot. There may be only a single **Slot** specification for each Volume. + +Stream +.. index:: + pair: Bootstrap; Stream + The value specifies a Stream, a list of Streams, or a range of Streams to be selected from the current Volume. Unless you really know what you are doing (the internals of Bareos), you should avoid this specification. This value is optional and not used by Bareos to restore files. + +The **Volume** record is a bit special in that it must be the first record. The other keyword records may appear in any order and any number following a Volume record. + +Multiple Volume records may be specified in the same bootstrap file, but each one starts a new set of filter criteria for the Volume. + +In processing the bootstrap file within the current Volume, each filter specified by a keyword is **AND**\ ed with the next. Thus, + +.. raw:: latex + + + + + + Volume = Test-01 + Client = "My machine" + FileIndex = 1 + +.. raw:: latex + + + +will match records on Volume **Test-01** **AND** Client records for **My machine** **AND** FileIndex equal to **one**. + +Multiple occurrences of the same record are **OR**\ ed together. Thus, + +.. raw:: latex + + + + + + Volume = Test-01 + Client = "My machine" + Client = "Backup machine" + FileIndex = 1 + +.. raw:: latex + + + +will match records on Volume **Test-01** **AND** (Client records for **My machine** **OR** **Backup machine**) **AND** FileIndex equal to **one**. + +For integer values, you may supply a range or a list, and for all other values except Volumes, you may specify a list. A list is equivalent to multiple records of the same keyword. For example, + +.. raw:: latex + + + + + + Volume = Test-01 + Client = "My machine", "Backup machine" + FileIndex = 1-20, 35 + +.. raw:: latex + + + +will match records on Volume **Test-01** **AND** **(**\ Client records for **My machine** **OR** **Backup machine**\ **)** **AND** **(**FileIndex 1 **OR** 2 **OR** 3 ... **OR** 20 **OR** 35\ **)**. + +As previously mentioned above, there may be multiple Volume records in the same bootstrap file. Each new Volume definition begins a new set of filter conditions that apply to that Volume and will be **OR**\ ed with any other Volume definitions. + +As an example, suppose we query for the current set of tapes to restore all files on Client **Rufus** using the **query** command in the console program: + +.. raw:: latex + + + + + + Using default Catalog name=MySQL DB=bareos + *query + Available queries: + 1: List Job totals: + 2: List where a file is saved: + 3: List where the most recent copies of a file are saved: + 4: List total files/bytes by Job: + 5: List total files/bytes by Volume: + 6: List last 10 Full Backups for a Client: + 7: List Volumes used by selected JobId: + 8: List Volumes to Restore All Files: + Choose a query (1-8): 8 + Enter Client Name: Rufus + +-------+------------------+------------+-----------+----------+------------+ + | JobId | StartTime | VolumeName | StartFile | VolSesId | VolSesTime | + +-------+------------------+------------+-----------+----------+------------+ + | 154 | 2002-05-30 12:08 | test-02 | 0 | 1 | 1022753312 | + | 202 | 2002-06-15 10:16 | test-02 | 0 | 2 | 1024128917 | + | 203 | 2002-06-15 11:12 | test-02 | 3 | 1 | 1024132350 | + | 204 | 2002-06-18 08:11 | test-02 | 4 | 1 | 1024380678 | + +-------+------------------+------------+-----------+----------+------------+ + +.. raw:: latex + + + +The output shows us that there are four Jobs that must be restored. The first one is a Full backup, and the following three are all Incremental backups. + +The following bootstrap file will restore those files: + +.. raw:: latex + + + + + + Volume=test-02 + VolSessionId=1 + VolSessionTime=1022753312 + Volume=test-02 + VolSessionId=2 + VolSessionTime=1024128917 + Volume=test-02 + VolSessionId=1 + VolSessionTime=1024132350 + Volume=test-02 + VolSessionId=1 + VolSessionTime=1024380678 + +.. raw:: latex + + + +As a final example, assume that the initial Full save spanned two Volumes. The output from **query** might look like: + +.. raw:: latex + + + + + + +-------+------------------+------------+-----------+----------+------------+ + | JobId | StartTime | VolumeName | StartFile | VolSesId | VolSesTime | + +-------+------------------+------------+-----------+----------+------------+ + | 242 | 2002-06-25 16:50 | File0003 | 0 | 1 | 1025016612 | + | 242 | 2002-06-25 16:50 | File0004 | 0 | 1 | 1025016612 | + | 243 | 2002-06-25 16:52 | File0005 | 0 | 2 | 1025016612 | + | 246 | 2002-06-25 19:19 | File0006 | 0 | 2 | 1025025494 | + +-------+------------------+------------+-----------+----------+------------+ + +.. raw:: latex + + + +and the following bootstrap file would restore those files: + +.. raw:: latex + + + + + + Volume=File0003 + VolSessionId=1 + VolSessionTime=1025016612 + Volume=File0004 + VolSessionId=1 + VolSessionTime=1025016612 + Volume=File0005 + VolSessionId=2 + VolSessionTime=1025016612 + Volume=File0006 + VolSessionId=2 + VolSessionTime=1025025494 + +.. raw:: latex + + + +Automatic Generation of Bootstrap Files +--------------------------------------- + +.. index:: + pair: Files; Automatic Generation of Bootstrap +.. index:: + pair: Bootstrap; Automatic Generation + + +One thing that is probably worth knowing: the bootstrap files that are generated automatically at the end of the job are not as optimized as those generated by the restore command. This is because during Incremental and Differential jobs, the records pertaining to the files written for the Job are appended to the end of the bootstrap file. As consequence, all the files saved to an Incremental or Differential job will be restored first by the Full save, then by any Incremental or Differential +saves. + +When the bootstrap file is generated for the restore command, only one copy (the most recent) of each file is restored. + +So if you have spare cycles on your machine, you could optimize the bootstrap files by doing the following: + +.. raw:: latex + + + + + + bconsole + restore client=xxx select all + done + no + quit + Backup bootstrap file. + +.. raw:: latex + + + +The above will not work if you have multiple FileSets because that will be an extra prompt. However, the **restore client=xxx select all** builds the in-memory tree, selecting everything and creates the bootstrap file. + +The **no** answers the **Do you want to run this (yes/mod/no)** question. + +Bootstrap for bscan +------------------- + +.. index:: + single: bscan +.. index:: + pair: bscan; bootstrap +.. index:: + pair: Bootstrap; bscan +.. index:: + pair: Command; bscan + + +.. _`bscanBootstrap`: bscanBootstrap + +If you have a very large number of Volumes to scan with **bscan**, you may exceed the command line limit (511 characters). In that case, you can create a simple bootstrap file that consists of only the volume names. An example might be: + +.. raw:: latex + + + + + + Volume="Vol001" + Volume="Vol002" + Volume="Vol003" + Volume="Vol004" + Volume="Vol005" + +.. raw:: latex + + + +Bootstrap Example +----------------- + +.. index:: + pair: Example; Bootstrap +.. index:: + pair: Bootstrap; Example + + +If you want to extract or copy a single Job, you can do it by selecting by JobId (code not tested) or better yet, if you know the VolSessionTime and the VolSessionId (printed on Job report and in Catalog), specifying this is by far the best. Using the VolSessionTime and VolSessionId is the way Bareos does restores. A bsr file might look like the following: + +.. raw:: latex + + + + + + Volume="Vol001" + VolSessionId=10 + VolSessionTime=1080847820 + +.. raw:: latex + + + +If you know how many files are backed up (on the job report), you can enormously speed up the selection by adding (let’s assume there are 157 files): + +.. raw:: latex + + + + + + FileIndex=1-157 + Count=157 + +.. raw:: latex + + + +Finally, if you know the File number where the Job starts, you can also cause bcopy to forward space to the right file without reading every record: + +.. raw:: latex + + + + + + VolFile=20 + +.. raw:: latex + + + +There is nothing magic or complicated about a BSR file. Parsing it and properly applying it within Bareos \*is\* magic, but you don’t need to worry about that. + +If you want to see a \*real\* bsr file, simply fire up the **restore** command in the console program, select something, then answer no when it prompts to run the job. Then look at the file **restore.bsr** in your working directory. diff --git a/docs/manuals/en/new_main_reference/source/appendix-e/verify.rst b/docs/manuals/en/new_main_reference/source/appendix-e/verify.rst new file mode 100644 index 00000000000..525be0acc5f --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-e/verify.rst @@ -0,0 +1,391 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _VerifyChapter: + +Verify File Integrity with Bareos +================================= + +.. index:: + pair: Security; Using Bareos to Improve Computer +.. index:: + pair: Verify; File Integrity + + +Since Bareos maintains a catalog of files, their attributes, and either SHA1 or MD5 signatures, it can be an ideal tool for improving computer security. This is done by making a snapshot of your system files with a **Verify** Job and then checking the current state of your system against the snapshot, on a regular basis (e.g. nightly). + +The first step is to set up a **Verify** Job and to run it with: + +.. raw:: latex + + + + + + Level = InitCatalog + +.. raw:: latex + + + +The **InitCatalog** level tells **Bareos** simply to get the information on the specified files and to put it into the catalog. That is your database is initialized and no comparison is done. The **InitCatalog** is normally run one time manually. + +Thereafter, you will run a Verify Job on a daily (or whatever) basis with: + +.. raw:: latex + + + + + + Level = Catalog + +.. raw:: latex + + + +The **Level = Catalog** level tells Bareos to compare the current state of the files on the Client to the last **InitCatalog** that is stored in the catalog and to report any differences. See the example below for the format of the output. + +You decide what files you want to form your "snapshot" by specifying them in a **FileSet** resource, and normally, they will be system files that do not change, or that only certain features change. + +Then you decide what attributes of each file you want compared by specifying comparison options on the **Include** statements that you use in the **FileSet** resource of your **Catalog** Jobs. + +The Details +----------- + +.. index:: + pair: Verify; Details + + +In the discussion that follows, we will make reference to the Verify Configuration Example that is included below in the **A Verify Configuration Example** section. You might want to look it over now to get an idea of what it does. + +The main elements consist of adding a schedule, which will normally be run daily, or perhaps more often. This is provided by the **VerifyCycle** Schedule, which runs at 5:05 in the morning every day. + +Then you must define a Job, much as is done below. We recommend that the Job name contain the name of your machine as well as the word **Verify** or **Check**. In our example, we named it **MatouVerify**. This will permit you to easily identify your job when running it from the Console. + +You will notice that most records of the Job are quite standard, but that the **FileSet** resource contains **verify=pins1** option in addition to the standard **signature=SHA1** option. If you don’t want SHA1 signature comparison, and we cannot imagine why not, you can drop the **signature=SHA1** and none will be computed nor stored in the catalog. Or alternatively, you can use **verify=pins5** and **signature=MD5**, which will use the MD5 hash algorithm. The MD5 hash computes faster than SHA1, +but is cryptographically less secure. + +The **verify=pins1** is ignored during the **InitCatalog** Job, but is used during the subsequent **Catalog** Jobs to specify what attributes of the files should be compared to those found in the catalog. **pins1** is a reasonable set to begin with, but you may want to look at the details of these and other options. They can be found in the :ref:`FileSet Resource ` section of this manual. Briefly, however, the **p** of the **pins1** tells Verify to compare the +permissions bits, the **i** is to compare inodes, the **n** causes comparison of the number of links, the **s** compares the file size, and the **1** compares the SHA1 checksums (this requires the **signature=SHA1** option to have been set also). + +You must also specify the **Client** and the **Catalog** resources for your Verify job, but you probably already have them created for your client and do not need to recreate them, they are included in the example below for completeness. + +As mentioned above, you will need to have a **FileSet** resource for the Verify job, which will have the additional **verify=pins1** option. You will want to take some care in defining the list of files to be included in your **FileSet**. Basically, you will want to include all system (or other) files that should not change on your system. If you select files, such as log files or mail files, which are constantly changing, your automatic Verify job will be constantly finding differences. The +objective in forming the FileSet is to choose all unchanging important system files. Then if any of those files has changed, you will be notified, and you can determine if it changed because you loaded a new package, or because someone has broken into your computer and modified your files. The example below shows a list of files that I use on my Red Hat 7.3 system. Since I didn’t spend a lot of time working on it, it probably is missing a few important files (if you find one, please send it to +me). On the other hand, as long as I don’t load any new packages, none of these files change during normal operation of the system. + +Running the Verify +------------------ + +.. index:: + pair: Verify; Running + + +The first thing you will want to do is to run an **InitCatalog** level Verify Job. This will initialize the catalog to contain the file information that will later be used as a basis for comparisons with the actual file system, thus allowing you to detect any changes (and possible intrusions into your system). + +The easiest way to run the **InitCatalog** is manually with the console program by simply entering **run**. You will be presented with a list of Jobs that can be run, and you will choose the one that corresponds to your Verify Job, **MatouVerify** in this example. + +.. raw:: latex + + + + + + The defined Job resources are: + 1: MatouVerify + 2: usersrestore + 3: Filetest + 4: usersave + Select Job resource (1-4): 1 + +.. raw:: latex + + + +Next, the console program will show you the basic parameters of the Job and ask you: + +.. raw:: latex + + + + + + Run Verify job + JobName: MatouVerify + FileSet: Verify Set + Level: Catalog + Client: MatouVerify + Storage: DLTDrive + Verify Job: + Verify List: /tmp/regress/working/MatouVerify.bsr + OK to run? (yes/mod/no): mod + +.. raw:: latex + + + +Here, you want to respond **mod** to modify the parameters because the Level is by default set to **Catalog** and we want to run an **InitCatalog** Job. After responding **mod**, the console will ask: + +.. raw:: latex + + + + + + Parameters to modify: + 1: Level + 2: Storage + 3: Job + 4: FileSet + 5: Client + 6: When + 7: Priority + 8: Pool + 9: Verify Job + Select parameter to modify (1-5): 1 + +.. raw:: latex + + + +you should select number 2 to modify the **Level**, and it will display: + +.. raw:: latex + + + + + + Levels: + 1: Initialize Catalog + 2: Verify Catalog + 3: Verify Volume to Catalog + 4: Verify Disk to Catalog + 5: Verify Volume Data (not yet implemented) + Select level (1-4): 1 + +.. raw:: latex + + + +Choose item 1, and you will see the final display: + +.. raw:: latex + + + + + + Run Verify job + JobName: MatouVerify + FileSet: Verify Set + Level: Initcatalog + Client: MatouVerify + Storage: DLTDrive + Verify Job: + Verify List: /tmp/regress/working/MatouVerify.bsr + OK to run? (yes/mod/no): yes + +.. raw:: latex + + + +at which point you respond **yes**, and the Job will begin. + +Thereafter the Job will automatically start according to the schedule you have defined. If you wish to immediately verify it, you can simply run a Verify **Catalog** which will be the default. No differences should be found. + +To use a previous job, you can add ``jobid=xxx`` option in run command line. It will run the Verify job against the specified job. + + + + *run jobid=1 job=MatouVerify + Run Verify job + JobName: MatouVerify + Level: Catalog + Client: 127.0.0.1-fd + FileSet: Full Set + Pool: Default (From Job resource) + Storage: File (From Job resource) + Verify Job: MatouVerify.2010-09-08_15.33.33_03 + Verify List: /tmp/regress/working/MatouVerify.bsr + When: 2010-09-08 15:35:32 + Priority: 10 + OK to run? (yes/mod/no): + +What To Do When Differences Are Found +------------------------------------- + +.. index:: + pair: Verify; Differences + + +If you have setup your messages correctly, you should be notified if there are any differences and exactly what they are. For example, below is the email received after doing an update of OpenSSH: + +.. raw:: latex + + + + + + HeadMan: Start Verify JobId 83 Job=RufusVerify.2002-06-25.21:41:05 + HeadMan: Verifying against Init JobId 70 run 2002-06-21 18:58:51 + HeadMan: File: /etc/pam.d/sshd + HeadMan: st_ino differ. Cat: 4674b File: 46765 + HeadMan: File: /etc/rc.d/init.d/sshd + HeadMan: st_ino differ. Cat: 56230 File: 56231 + HeadMan: File: /etc/ssh/ssh_config + HeadMan: st_ino differ. Cat: 81317 File: 8131b + HeadMan: st_size differ. Cat: 1202 File: 1297 + HeadMan: SHA1 differs. + HeadMan: File: /etc/ssh/sshd_config + HeadMan: st_ino differ. Cat: 81398 File: 81325 + HeadMan: st_size differ. Cat: 1182 File: 1579 + HeadMan: SHA1 differs. + HeadMan: File: /etc/ssh/ssh_config.rpmnew + HeadMan: st_ino differ. Cat: 812dd File: 812b3 + HeadMan: st_size differ. Cat: 1167 File: 1114 + HeadMan: SHA1 differs. + HeadMan: File: /etc/ssh/sshd_config.rpmnew + HeadMan: st_ino differ. Cat: 81397 File: 812dd + HeadMan: st_size differ. Cat: 2528 File: 2407 + HeadMan: SHA1 differs. + HeadMan: File: /etc/ssh/moduli + HeadMan: st_ino differ. Cat: 812b3 File: 812ab + HeadMan: File: /usr/bin/scp + HeadMan: st_ino differ. Cat: 5e07e File: 5e343 + HeadMan: st_size differ. Cat: 26728 File: 26952 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/ssh-keygen + HeadMan: st_ino differ. Cat: 5df1d File: 5e07e + HeadMan: st_size differ. Cat: 80488 File: 84648 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/sftp + HeadMan: st_ino differ. Cat: 5e2e8 File: 5df1d + HeadMan: st_size differ. Cat: 46952 File: 46984 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/slogin + HeadMan: st_ino differ. Cat: 5e359 File: 5e2e8 + HeadMan: File: /usr/bin/ssh + HeadMan: st_mode differ. Cat: 89ed File: 81ed + HeadMan: st_ino differ. Cat: 5e35a File: 5e359 + HeadMan: st_size differ. Cat: 219932 File: 234440 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/ssh-add + HeadMan: st_ino differ. Cat: 5e35b File: 5e35a + HeadMan: st_size differ. Cat: 76328 File: 81448 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/ssh-agent + HeadMan: st_ino differ. Cat: 5e35c File: 5e35b + HeadMan: st_size differ. Cat: 43208 File: 47368 + HeadMan: SHA1 differs. + HeadMan: File: /usr/bin/ssh-keyscan + HeadMan: st_ino differ. Cat: 5e35d File: 5e96a + HeadMan: st_size differ. Cat: 139272 File: 151560 + HeadMan: SHA1 differs. + HeadMan: 25-Jun-2002 21:41 + JobId: 83 + Job: RufusVerify.2002-06-25.21:41:05 + FileSet: Verify Set + Verify Level: Catalog + Client: RufusVerify + Start time: 25-Jun-2002 21:41 + End time: 25-Jun-2002 21:41 + Files Examined: 4,258 + Termination: Verify Differences + +.. raw:: latex + + + +At this point, it was obvious that these files were modified during installation of the RPMs. If you want to be super safe, you should run a **Verify Level=Catalog** immediately before installing new software to verify that there are no differences, then run a **Verify Level=InitCatalog** immediately after the installation. + +To keep the above email from being sent every night when the Verify Job runs, we simply re-run the Verify Job setting the level to **InitCatalog** (as we did above in the very beginning). This will re-establish the current state of the system as your new basis for future comparisons. Take care that you don’t do an **InitCatalog** after someone has placed a Trojan horse on your system! + +If you have included in your **FileSet** a file that is changed by the normal operation of your system, you will get false matches, and you will need to modify the **FileSet** to exclude that file (or not to Include it), and then re-run the **InitCatalog**. + +The FileSet that is shown below is what I use on my Red Hat 7.3 system. With a bit more thought, you can probably add quite a number of additional files that should be monitored. + +A Verify Configuration Example +------------------------------ + +.. index:: + pair: Verify; Example + + +.. raw:: latex + + + + + + Schedule { + Name = "VerifyCycle" + Run = Level=Catalog sun-sat at 5:05 + } + Job { + Name = "MatouVerify" + Type = Verify + Level = Catalog # default level + Client = MatouVerify + FileSet = "Verify Set" + Messages = Standard + Storage = DLTDrive + Pool = Default + Schedule = "VerifyCycle" + } + # + # The list of files in this FileSet should be carefully + # chosen. This is a good starting point. + # + FileSet { + Name = "Verify Set" + Include { + Options { + verify=pins1 + signature=SHA1 + } + File = /boot + File = /bin + File = /sbin + File = /usr/bin + File = /lib + File = /root/.ssh + File = /home/user/.ssh + File = /var/named + File = /etc/sysconfig + File = /etc/ssh + File = /etc/security + File = /etc/exports + File = /etc/rc.d/init.d + File = /etc/sendmail.cf + File = /etc/sysctl.conf + File = /etc/services + File = /etc/xinetd.d + File = /etc/hosts.allow + File = /etc/hosts.deny + File = /etc/hosts + File = /etc/modules.conf + File = /etc/named.conf + File = /etc/pam.d + File = /etc/resolv.conf + } + Exclude = { } + } + Client { + Name = MatouVerify + Address = lmatou + Catalog = Bareos + Password = "" + File Retention = 80d # 80 days + Job Retention = 1y # one year + AutoPrune = yes # Prune expired Jobs/Files + } + Catalog { + Name = Bareos + dbname = verify; user = bareos; password = "" + } + +.. raw:: latex + + diff --git a/docs/manuals/en/new_main_reference/source/appendix-f/backward-compatibility.rst b/docs/manuals/en/new_main_reference/source/appendix-f/backward-compatibility.rst new file mode 100644 index 00000000000..b1113755fd9 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-f/backward-compatibility.rst @@ -0,0 +1,204 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +Backward Compatibility +====================== + +.. index:: + pair: Compatibility; Backward + + +.. _`backward-compatibility`: backward-compatibility + +Tape Formats +------------ + +.. index:: + pair: Tape; Format + + +.. _`backward-compatibility-tape-format`: backward-compatibility-tape-format + +One of the major goals of Backup software is to ensure that you can restore tapes (the word tape should also include disk volumes) that you wrote years ago. This means that each new version of the software should be able to read old format tapes. The first problem you will have is to ensure that the hardware is still working some years down the road, and the second problem will be to ensure that the media will still be good, then your OS must be able to interface to the device, and finally +Bareos must be able to recognize old formats. All the problems except the last are ones that we cannot solve, but by careful planning you can. + +Since the very beginning of Bacula (January 2000) until today (2015), there have been three major Bacula/Bareos tape formats. The second format was introduced in Bacula version 1.27 in November of 2002. Bareos has been required to adapt the tape format to avoid potential trademark issues, but is able to read also the old Bacula tape formats. + +Though the tape format is basically fixed, the kinds of data that we can put on the tapes are extensible, and that is how we added new features such as ACLs, Win32 data, encrypted data, ... Obviously, an older version of Bacula/Bareos would not know how to read these newer data streams, but each newer version of Bareos should know how to read all the older streams. + +Compatibility between Bareos and Bacula +--------------------------------------- + +.. index:: + single: Bacula + + +.. _`compat-bacula`: compat-bacula + +A Director and a Storage Daemon should (must) always run at the same version. This is true for Bareos as well as for Bacula. It is not possible to mix these components. This is because the protocol between Director and Storage Daemon itself is not versioned (also true for Bareos and Bacula). If you want to be able to switch back from Bareos to Bacula after using a Bareos director and storage daemon you have to enable the compatible mode in the Bareos storage daemon to have it write the data in +the same format as the Bacula storage daemon. + +The |bareosFd| is compatible with all version of the Bacula director (tested with version 5.2.13 and lower) when you enable the compatible mode in the configuration of the |bareosFd|. The compatible option was set by default in Bareos :math:`<` 15.2.0, and is disabled by default since 15.2.0. + +To be sure this is enabled you can explicitly set the compatible option: + +.. raw:: latex + + \resourceDirectiveValue{Fd}{Client}{Compatible}{True} + +A |bareosDir| can only talk to Bacula file daemons of version 2.0 or higher. Through a change in the Bacula network protocols, it is currently not possible to use a Bacula file daemon :math:`\ge` 6.0 with a |bareosDir|. + +These combinations of Bareos and Bacula are know to work together: + ++--------------+--------------------+-----------------------------------------+-------------+ +| **Director** | **Storage Daemon** | **File Daemon** | **Remarks** | ++==============+====================+=========================================+=============+ +| Bareos  | Bareos  | Bareos  | | ++--------------+--------------------+-----------------------------------------+-------------+ +| Bareos  | Bareos  | 2.0 :math:`\leq` Bacula \ :math:`<` 6.0 | | ++--------------+--------------------+-----------------------------------------+-------------+ +| Bacula  | Bacula  | Bacula  | | ++--------------+--------------------+-----------------------------------------+-------------+ +| Bacula  | Bacula  | Bareos (compatibility mode) | | ++--------------+--------------------+-----------------------------------------+-------------+ + +Other combinations like Bacula Director with |bareosSd| will not work. However this wasn’t even possible with different versions of bacula-dir and bacula-sd. + +Upgrade from Bacula 5.2 to Bareos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Upgrade from Bacula to Bareos +.. index:: + pair: Bareos; Upgrading + + +.. _`upgrade-from-bacula-to-bareos`: upgrade-from-bacula-to-bareos + +Upgrade is supported from Bacula version 5.2.x. If you are running any older version of Bacula, please update to 5.2 first (see Bacula documentation). + +.. raw:: latex + + +.. warning:: + Updating from Bacula ≥ 7.0 to Bareos has not been tested. + +.. raw:: latex + + \warning{As Bareos and Bacula packages bring binaries with identical paths and names, + it is on most platforms not possible to install components from both in parallel. + Your package management tool will warn you about this.} + +Rename user and group before upgrading +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To have bareos running without any permission hassle, it is recommended to rename the user and group **bacula}` to the user and group :raw-latex:`\user{bareos** before upgrading. That way, we minimize the effort for the user to recheck all config files and the rights on every script/directory etc. involved in the existing setup. + +The required commands should look something like this: + + + + +.. code-block:: sh + :caption: + + usermod -l bareos bacula + groupmod -n bareos bacula + +MySQL +^^^^^ + +Proceed with the following steps: + +- Stop bacula services + +- Backup your catalog database: + + + + +.. code-block:: sh + :caption: + + mysqldump bacula > /tmp/bacula_5.2.sql + +- Make the user bareos have the same userid and the group bareos the same groupid as the user/group bacula had before. This will solve a lot of rights problems. + +- Install Bareos packages + +- Run the update script on the old bacula database: + + + + +.. code-block:: sh + :caption: + + export db_name=bacula + /usr/lib/bareos/update_bareos_tables + unset db_name + +- Backup upgraded DB: + + + + +.. code-block:: sh + :caption: + + mysqldump bacula > /tmp/bacula.sql + +- Create bareos database: + + + + +.. code-block:: sh + :caption: + + /usr/lib/bareos/create_bareos_database + +- Insert backuped db into new database: + + + + +.. code-block:: sh + :caption: + + cat /tmp/bacula.sql | mysql bareos + +- Grant permissions: + + + + +.. code-block:: sh + :caption: + + /usr/lib/bareos/grant_mysql_privileges + +- Adapt file permissions to bareos, if you have any file storage + +- Adapt configs (not complete) + + - With bacula the default setting for pid files was :file:`/var/run``, which may not work if the bareos-director runs as user bareos. Best way is to comment out the entry :program:`Pid Directory = "/var/run"` in your director config. Bareos will set a working default value (supposed to be ``path:/var/lib/bareos/`) + +PostgreSQL +^^^^^^^^^^ + +Renaming a postgresql database: + +- Become postgresql user + +- psql template1 + + + + +.. code-block:: sh + :caption: + + ALTER DATABASE bacula RENAME TO bareos; + ALTER USER bacula RENAME TO bareos; + ALTER USER bareos UNENCRYPTED PASSWORD 'password'; diff --git a/docs/manuals/en/new_main_reference/source/appendix-g/tables.rst b/docs/manuals/en/new_main_reference/source/appendix-g/tables.rst new file mode 100644 index 00000000000..17a797793c7 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-g/tables.rst @@ -0,0 +1,90 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +Catalog Tables +============== + +.. index:: + single: Catalog + + +Bareos stores its information in a database, named Catalog. It is configured by :ref:`DirectorResourceCatalog`. + +Job +--- + +.. index:: + pair: Catalog; Job +.. index:: + pair: Job; Catalog + + +JobStatus +~~~~~~~~~ + +.. index:: + pair: Job; JobStatus + +.. index:: + triple: Catalog; Job; JobStatus; + + +The status of a Bareos job is stored as abbreviation in the Catalog database table Job. It is also displayed by some bconsole commands, eg. :strong:`list jobs`. + +This table lists the abbreviations together with its description and weight. The weight is used, when multiple states are applicable for a job. In this case, only the status with the highest weight/priority is applied. + ++-----------+--------------------------------------------------------+------------+ +| **Abbr.** | :raw-latex:`\multicolumn{1}{c|}{\textbf{Description}}` | **Weight** | ++===========+========================================================+============+ +| C | Created, not yet running | 15 | ++-----------+--------------------------------------------------------+------------+ +| R | Running | 15 | ++-----------+--------------------------------------------------------+------------+ +| B | Blocked | 15 | ++-----------+--------------------------------------------------------+------------+ +| T | Completed successfully | 10 | ++-----------+--------------------------------------------------------+------------+ +| E | Terminated with errors | 25 | ++-----------+--------------------------------------------------------+------------+ +| e | Non-fatal error | 20 | ++-----------+--------------------------------------------------------+------------+ +| f | Fatal error | 100 | ++-----------+--------------------------------------------------------+------------+ +| D | Verify found differences | 15 | ++-----------+--------------------------------------------------------+------------+ +| A | Canceled by user | 90 | ++-----------+--------------------------------------------------------+------------+ +| I | Incomplete job | 15 | ++-----------+--------------------------------------------------------+------------+ +| L | Committing data | 15 | ++-----------+--------------------------------------------------------+------------+ +| W | Terminated with warnings | 20 | ++-----------+--------------------------------------------------------+------------+ +| l | Doing data despooling | 15 | ++-----------+--------------------------------------------------------+------------+ +| q | Queued waiting for device | 15 | ++-----------+--------------------------------------------------------+------------+ +| F | Waiting for Client | 15 | ++-----------+--------------------------------------------------------+------------+ +| S | Waiting for Storage daemon | 15 | ++-----------+--------------------------------------------------------+------------+ +| m | Waiting for new media | 15 | ++-----------+--------------------------------------------------------+------------+ +| M | Waiting for media mount | 15 | ++-----------+--------------------------------------------------------+------------+ +| s | Waiting for storage resource | 15 | ++-----------+--------------------------------------------------------+------------+ +| j | Waiting for job resource | 15 | ++-----------+--------------------------------------------------------+------------+ +| c | Waiting for client resource | 15 | ++-----------+--------------------------------------------------------+------------+ +| d | Waiting on maximum jobs | 15 | ++-----------+--------------------------------------------------------+------------+ +| t | Waiting on start time | 15 | ++-----------+--------------------------------------------------------+------------+ +| p | Waiting on higher priority jobs | 15 | ++-----------+--------------------------------------------------------+------------+ +| i | Doing batch insert file records | 15 | ++-----------+--------------------------------------------------------+------------+ +| a | SD despooling attributes | 15 | ++-----------+--------------------------------------------------------+------------+ diff --git a/docs/manuals/en/new_main_reference/source/appendix-h/howto.rst b/docs/manuals/en/new_main_reference/source/appendix-h/howto.rst new file mode 100644 index 00000000000..40b58268b55 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-h/howto.rst @@ -0,0 +1,1201 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _dummydevice: + +Use a dummy device to test the backup +===================================== + + + +.. _`TestUsingFifoDevice`: TestUsingFifoDevice If your are testing your configuration, but don’t want to store the backup data, it is possible to use a dummy FIFO device to test your configuration, see :ref:`Stored configuration `. + +Obviously, it can not be used to do a restore. + + + + +.. code-block:: sh + :caption: FIFO Storage Device Configuration + + Device { + Name = NULL + Media Type = NULL + Device Type = Fifo + Archive Device = /dev/null + LabelMedia = yes + Random Access = no + AutomaticMount = no + RemovableMedia = no + MaximumOpenWait = 60 + AlwaysOpen = no + } + +Backup Of Third Party Databases +=============================== + +.. index:: + pair: Backup; of Third Party Databases +.. index:: + pair: Database; Backup Of Third Party + + +.. _`BackupOtherDBs`: BackupOtherDBs + +If you are running a database in production mode on your machine, Bareos will happily backup the files, but if the database is in use while Bareos is reading it, you may back it up in an unstable state. + +The best solution is to shutdown your database before backing it up, or use some tool specific to your database to make a valid live copy perhaps by dumping the database in ASCII format. + +Backup of MSSQL Databases with Bareos Plugin +-------------------------------------------- + +.. index:: + single: MSSQL Backup +.. index:: + pair: Database; MSSQL +.. index:: + pair: Plugin; MSSQL backup + + +.. _`MSSQL`: MSSQL + +.. raw:: latex + + \sinceVersion{general}{MSSQL}{13.2.0} + +Preparation +~~~~~~~~~~~ + +If you like to use the MSSQL-Plugin to backing up your Databases you need to consider some things: + +- | **Database Mode** + | The database need to run in **Full Recovery Mode**. Otherwise you are not able to use differential and incremental backups or to use point in time recovery. + | +.. warning:: + If you set the databases into the mentionend mode you have to consider some maintance facts. The database doesn't shrink or delete the logs unanttended, so you have to shrink them manual once a week and you have to truncate the logs once in a month. + +- | **Security and Access** + | For connections you can use a SQL-User or a integrated systemaccount (Windows NT user). Both connection types are supported. + + - | Standard Security + | You have to provide user credentials within your options which do belong to user with the sufficent right performing restores and backups from the database. This way stands for an extra backup/restore user. + + - | Trusted Security + | You use a systemaccount which have the sufficent rights to performing backups and restores on a database. This systemaccount have to be same account like the **bareos-filedeamon** runs on. + +- **Permissions and Roles** + + - | Server-Role + | The user should be in the groups **sysadmin** and **dbcreator**. + + - | Database permissions + | The user have to be a **backupoperator** and **dbowner** of the database which you like to backup. + +There is no difference for the rights and roles between using a systemaccount (trusted security method) or a extra backup user (standard security method). Please keep in mind if you use the trusted security method you have to use the same system account like the bareos-filedeamon runs on. + +.. _MssqlPluginInstallation: + +MSSQL Plugin Installation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +For Bareos :math:`<` 14.2, install the Bareos MSSQL plugin onto the MSSQL server you want to backup. Bareos :math:`>=` 14.2 also allows to backup remote MSSQL servers (option :option:`serveraddress`). + +Bareos Windows-Installer +^^^^^^^^^^^^^^^^^^^^^^^^ + +Install the Bareos filedaemon including the component "Bareos FileDameon Plugins". Make sure, that you install the file daemon **without the "compatible" option**. + +Manual install +^^^^^^^^^^^^^^ + +After downloading the plugin you need to copy it into :file:`C:\Program Files\Bareos\Plugins``. Then you need to define the plugin directory and which plugin the **bareos-filedaemon** should use. You have to edit the **bareos-filedaemon** resource in ``path:C:\Program Data\bareos-fd.conf` as follows: + + + + +.. code-block:: sh + :caption: MSSQL plugin configuration + + FileDaemon { + Name = mssqlserver-fd + Maximum Concurrent Jobs = 20 + + # remove comment in next line to load plugins from specified directory + Plugin Directory = "C:/Program Files/Bareos/Plugins" + + Plugin Names = "mssqlvdi" + compatible = no # this is the default since bareos 15 + } + +Plugin Test +~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: status client=mssqlserver-fd + + *status client=mssqlserver-fd + Connecting to Client mssqlserver-fd at 192.168.10.101:9102 + + mssqlserver-fd Version: 13.2.2 (12 November 2013) VSS Linux Cross-compile Win64 + Daemon started 18-Nov-13 11:51. Jobs: run=0 running=0. + Microsoft Windows Server 2012 Standard Edition (build 9200), 64-bit + Heap: heap=0 smbytes=20,320 max_bytes=20,522 bufs=71 max_bufs=73 + Sizeof: boffset_t=8 size_t=8 debug=0 trace=1 bwlimit=0kB/s + Plugin Info: + Plugin : mssqlvdi-fd.dll + Description: Bareos MSSQL VDI Windows File Daemon Plugin + Version : 1, Date: July 2013 + Author : Zilvinas Krapavickas + License : Bareos AGPLv3 + Usage : + mssqlvdi: + serveraddress=: + instance=: + database=: + username=: + password=: + norecovery=: + replace=: + recoverafterrestore=: + stopbeforemark=: + stopatmark=: + stopat= + + examples: + timestamp: 'Apr 15, 2020 12:00 AM' + log sequence number: 'lsn:15000000040000037' + +Configure the FileSet +~~~~~~~~~~~~~~~~~~~~~ + +To use the plugin you need to configure it in the fileset as a plugin resource. For each database instance you need to define a exclusive backup job and fileset. + + + + +.. code-block:: sh + :caption: MSSQL FileSet + + Fileset { + Name = "Mssql" + Enable VSS = no + Include { + Options { + Signature = MD5 + } + Plugin = "mssqlvdi:instance=default:database=myDatabase:username=bareos:password=bareos" + } + } + +In this example we use the standard security method for the connection. + +Used options in the plugin string are: + +mssqlvdi + This is the reference to the MSSQL plugin. + +serveraddress + (14.2.2) Defines the server address to connect to (if empty defaults to localhost). + +instance + Defines the instance within the database server. + +database + Defines the database that should get backuped. + +username and password + Username and Password are required, when the connection is done using a MSSQL user. If the systemaccount the bareos-fd runs with has succifient permissions, this is not required. + +It is recommend to define an additional restore job. + +For every database separate job and FileSet are required. + +Run Backups +~~~~~~~~~~~ + +Here you can see an example for a backup: + + + + +.. code-block:: sh + :caption: run MSSQL backup job + + *run job=MSSQLBak + Using Catalog "MyCatalog" + Run Backup job + JobName: MSSQLBak + Level: Full + Client: mssqlserver-fd + Format: Native + FileSet: Mssql + Pool: File (From Job resource) + Storage: File (From Job resource) + When: 2013-11-21 09:48:27 + Priority: 10 + OK to run? (yes/mod/no): yes + Job queued. JobId=7 + You have no messages. + *mess + 21-Nov 09:48 bareos-dir JobId 7: Start Backup JobId 7, Job=MSSQLBak.2013-11-21_09.48.30_04 + 21-Nov 09:48 bareos-dir JobId 7: Using Device "FileStorage" to write. + 21-Nov 09:49 bareos-sd JobId 7: Volume "test1" previously written, moving to end of data. + 21-Nov 09:49 bareos-sd JobId 7: Ready to append to end of Volume "test1" size=2300114868 + 21-Nov 09:49 bareos-sd JobId 7: Elapsed time=00:00:27, Transfer rate=7.364 M Bytes/second + + 21-Nov 09:49 bareos-dir JobId 7: Bareos bareos-dir 13.4.0 (01Oct13): + Build OS: x86_64-pc-linux-gnu debian Debian GNU/Linux 7.0 (wheezy) + JobId: 7 + Job: MSSQLBak.2013-11-21_09.48.30_04 + Backup Level: Full + Client: "mssqlserver-fd" 13.2.2 (12Nov13) Microsoft Windows Server 2012 Standard Edition (build 9200), 64-bit,Cross-compile,Win64 + FileSet: "Mssql" 2013-11-04 23:00:01 + Pool: "File" (From Job resource) + Catalog: "MyCatalog" (From Client resource) + Storage: "File" (From Job resource) + Scheduled time: 21-Nov-2013 09:48:27 + Start time: 21-Nov-2013 09:49:13 + End time: 21-Nov-2013 09:49:41 + Elapsed time: 28 secs + Priority: 10 + FD Files Written: 1 + SD Files Written: 1 + FD Bytes Written: 198,836,224 (198.8 MB) + SD Bytes Written: 198,836,435 (198.8 MB) + Rate: 7101.3 KB/s + Software Compression: None + VSS: no + Encryption: no + Accurate: no + Volume name(s): test1 + Volume Session Id: 1 + Volume Session Time: 1384961357 + Last Volume Bytes: 2,499,099,145 (2.499 GB) + Non-fatal FD errors: 0 + SD Errors: 0 + FD termination status: OK + SD termination status: OK + Termination: Backup OK + +At least you gain a full backup which contains the follow: + +.. raw:: latex + + + + + + @MSSQL/ + @MSSQL/default/ + @MSSQL/default/myDatabase/ + @MSSQL/default/myDatabase/db-full + +.. raw:: latex + + + +| So if you perform your first full backup your are capable to perfom differntial and incremental backups. +| Differntial FileSet example: + +.. raw:: latex + + + + + + /@MSSQL/ + /@MSSQL/default/ + /@MSSQL/default/myDatabase/ + /@MSSQL/default/myDatabase/db-full + /@MSSQL/default/myDatabase/db-diff + +.. raw:: latex + + + +Incremental FileSet example: + +.. raw:: latex + + + + + + *@MSSQL/ + *default/ + *myDatabase/ + *db-diff + *db-full + *log-2013-11-21 17:32:20 + +.. raw:: latex + + + +Restores +~~~~~~~~ + +If you want to perfom a restore of a full backup without differentials or incrementals you have some options which helps you to restore even the corrupted database still exist. But you have to specifiy the options like plugin, instance and database during every backup. + +replace= + With this option you can replace the database if it still exist. + +instance + Defines the server instance whithin the database is running. + +database + Defines the database you want to backup. + +If you want to restore the actual backup to a set of backup files which you can use to restore a database under an new name or perform any kind of special operations using for example the sql management studio, you can use a where setting for the restore other then ’/’. When the where is set to ’/’ it will restore to the Virtual Device Interface (VDI). + +When you specify for restore a where path which is lets say ’c:/temp’ the plugin will restore the selected backup files under a relocated path under c:/temp/@MSSQL@/... + +Example for a full restore: + + + + +.. code-block:: sh + :caption: restore MSSQL database + + *restore client=mssqlserver-fd + Using Catalog "MyCatalog" + + First you select one or more JobIds that contain files + to be restored. You will be presented several methods + of specifying the JobIds. Then you will be allowed to + select which files from those JobIds are to be restored. + + To select the JobIds, you have the following choices: + 1: List last 20 Jobs run + 2: List Jobs where a given File is saved + 3: Enter list of comma separated JobIds to select + 4: Enter SQL list command + 5: Select the most recent backup for a client + 6: Select backup for a client before a specified time + 7: Enter a list of files to restore + 8: Enter a list of files to restore before a specified time + 9: Find the JobIds of the most recent backup for a client + 10: Find the JobIds for a backup for a client before a specified time + 11: Enter a list of directories to restore for found JobIds + 12: Select full restore to a specified Job date + 13: Cancel + Select item- (1-13): 5 + Automatically selected FileSet: Mssql + +-------+-------+----------+-------------+---------------------+------------+ + | JobId | Level | JobFiles | JobBytes | StartTime | VolumeName | + +-------+-------+----------+-------------+---------------------+------------+ + | 8 | F | 1 | 198,836,224 | 2013-11-21 09:52:28 | test1 | + +-------+-------+----------+-------------+---------------------+------------+ + You have selected the following JobId: 8 + + Building directory tree for JobId(s) 8 ... + 1 files inserted into the tree. + + You are now entering file selection mode where you add (mark) and + remove (unmark) files to be restored. No files are initially added, unless + you used the "all" keyword on the command line. + Enter "done" to leave this mode. + + cwd is: / + $ mark * + 1 file marked. + $ done + Bootstrap records written to /var/lib/bareos/bareos-dir.restore.4.bsr + + The job will require the following + Volume(s) Storage(s) SD Device(s) + =========================================================================== + + test1 File FileStorage + + Volumes marked with "*" are online. + + + 1 file selected to be restored. + + The defined Restore Job resources are: + 1: RestoreMSSQL + 2: RestoreFiles + Select Restore Job (1-2): 1 + Using Catalog "MyCatalog" + Run Restore job + JobName: RestoreMSSQL + Bootstrap: /var/lib/bareos/bareos-dir.restore.4.bsr + Where: / + Replace: Always + FileSet: Mssql + Backup Client: mssqlserver-fd + Restore Client: mssqlserver-fd + Format: Native + Storage: File + When: 2013-11-21 17:12:05 + Catalog: MyCatalog + Priority: 10 + Plugin Options: *None* + OK to run? (yes/mod/no): mod + Parameters to modify: + 1: Level + 2: Storage + 3: Job + 4: FileSet + 5: Restore Client + 6: Backup Format + 7: When + 8: Priority + 9: Bootstrap + 10: Where + 11: File Relocation + 12: Replace + 13: JobId + 14: Plugin Options + Select parameter to modify (1-14): 14 + Please enter Plugin Options string: mssqlvdi:instance=default:database=myDatabase:replace=yes + Run Restore job + JobName: RestoreMSSQL + Bootstrap: /var/lib/bareos/bareos-dir.restore.4.bsr + Where: / + Replace: Always + FileSet: Mssql + Backup Client: mssqlserver-fd + Restore Client: mssqlserver-fd + Format: Native + Storage: File + When: 2013-11-21 17:12:05 + Catalog: MyCatalog + Priority: 10 + Plugin Options: mssqlvdi:instance=default:database=myDatabase:replace=yes + OK to run? (yes/mod/no): yes + Job queued. JobId=10 + You have messages. + *mess + 21-Nov 17:12 bareos-dir JobId 10: Start Restore Job RestoreMSSQL.2013-11-21_17.12.26_11 + 21-Nov 17:12 bareos-dir JobId 10: Using Device "FileStorage" to read. + 21-Nov 17:13 damorgan-sd JobId 10: Ready to read from volume "test1" on device "FileStorage" (/storage). + 21-Nov 17:13 damorgan-sd JobId 10: Forward spacing Volume "test1" to file:block 0:2499099145. + 21-Nov 17:13 damorgan-sd JobId 10: End of Volume at file 0 on device "FileStorage" (/storage), Volume "test1" + 21-Nov 17:13 damorgan-sd JobId 10: End of all volumes. + 21-Nov 17:13 bareos-dir JobId 10: Bareos bareos-dir 13.4.0 (01Oct13): + Build OS: x86_64-pc-linux-gnu debian Debian GNU/Linux 7.0 (wheezy) + JobId: 10 + Job: RestoreMSSQL.2013-11-21_17.12.26_11 + Restore Client: mssqlserver-fd + Start time: 21-Nov-2013 17:12:28 + End time: 21-Nov-2013 17:13:21 + Files Expected: 1 + Files Restored: 1 + Bytes Restored: 198,836,224 + Rate: 3751.6 KB/s + FD Errors: 0 + FD termination status: OK + SD termination status: OK + Termination: Restore OK + +Restore a Backup Chain +^^^^^^^^^^^^^^^^^^^^^^ + +If you like to restore a specific state or a whole chain consists of full, incremental and differential backups you need to use the "norecovery=yes" option. After this the database is in "recovery mode". You can also use a option which put the database right after the restore back into the right mode. If you like to restore certains point with protocols or "LSN" it it not recommend to work with this option. + +norecovery= + This option must be set to yes, if the database server should not do a automatic recovery after the backup. Instead, additional manual maintenace operations are possible. + +recoverafterrestore= + With this command the database is right after backup in the correct mode. If you not use this you have to use the followed tsql statement: :raw-latex:`` + + + + Restore DATABASE yourDatabase WITH RECOVERY + GO + + + + + + +stopbeforemark= + used for point in time recovery. + +stopatmark= + used for point in time recovery. + +stopat= + used for point in time recovery. + +Followed is a example for a restore of full, differential and incremental backup with a replace of the original database: + + + + +.. code-block:: sh + :caption: restore MSSQL database chain + + *restore client=mssqlserver-fd + + First you select one or more JobIds that contain files + to be restored. You will be presented several methods + of specifying the JobIds. Then you will be allowed to + select which files from those JobIds are to be restored. + + To select the JobIds, you have the following choices: + 1: List last 20 Jobs run + 2: List Jobs where a given File is saved + 3: Enter list of comma separated JobIds to select + 4: Enter SQL list command + 5: Select the most recent backup for a client + 6: Select backup for a client before a specified time + 7: Enter a list of files to restore + 8: Enter a list of files to restore before a specified time + 9: Find the JobIds of the most recent backup for a client + 10: Find the JobIds for a backup for a client before a specified time + 11: Enter a list of directories to restore for found JobIds + 12: Select full restore to a specified Job date + 13: Cancel + Select item- (1-13): 5 + Automatically selected FileSet: Mssql + +-------+-------+----------+-------------+---------------------+------------+ + | JobId | Level | JobFiles | JobBytes | StartTime | VolumeName | + +-------+-------+----------+-------------+---------------------+------------+ + | 8 | F | 1 | 198,836,224 | 2013-11-21 09:52:28 | test1 | + | 11 | D | 1 | 2,555,904 | 2013-11-21 17:19:45 | test1 | + | 12 | I | 1 | 720,896 | 2013-11-21 17:29:39 | test1 | + +-------+-------+----------+-------------+---------------------+------------+ + You have selected the following JobIds: 8,11,12 + + Building directory tree for JobId(s) 8,11,12 ... + 3 files inserted into the tree. + + You are now entering file selection mode where you add (mark) and + remove (unmark) files to be restored. No files are initially added, unless + you used the "all" keyword on the command line. + Enter "done" to leave this mode. + + cwd is: / + $ mark * + 3 files marked. + $ lsmark + *@MSSQL/ + *default/ + *myDatabase/ + *db-diff + *db-full + *log-2013-11-21 17:32:20 + $ done + Bootstrap records written to /var/lib/bareos/bareos-dir.restore.6.bsr + + The job will require the following + Volume(s) Storage(s) SD Device(s) + =========================================================================== + + test1 File FileStorage + + Volumes marked with "*" are online. + + + 1 file selected to be restored. + + The defined Restore Job resources are: + 1: RestoreMSSQL + 2: RestoreFiles + Select Restore Job (1-2): 1 + Run Restore job + JobName: RestoreMSSQL + Bootstrap: /var/lib/bareos/bareos-dir.restore.6.bsr + Where: / + Replace: Always + FileSet: Mssql + Backup Client: mssqlserver-fd + Restore Client: mssqlserver-fd + Format: Native + Storage: File + When: 2013-11-21 17:34:23 + Catalog: MyCatalog + Priority: 10 + Plugin Options: *None* + OK to run? (yes/mod/no): mod + Parameters to modify: + 1: Level + 2: Storage + 3: Job + 4: FileSet + 5: Restore Client + 6: Backup Format + 7: When + 8: Priority + 9: Bootstrap + 10: Where + 11: File Relocation + 12: Replace + 13: JobId + 14: Plugin Options + Select parameter to modify (1-14): 14 + Please enter Plugin Options string: mssqlvdi:instance=default:database=myDatabase:replace=yes:norecovery=yes + Run Restore job + JobName: RestoreMSSQL + Bootstrap: /var/lib/bareos/bareos-dir.restore.6.bsr + Where: / + Replace: Always + FileSet: Mssql + Backup Client: mssqlserver-fd + Restore Client: mssqlserver-fd + Format: Native + Storage: File + When: 2013-11-21 17:34:23 + Catalog: MyCatalog + Priority: 10 + Plugin Options: mssqlvdi:instance=default:database=myDatabase:replace=yes:norecovery=yes + OK to run? (yes/mod/no): yes + Job queued. JobId=14 + 21-Nov 17:34 bareos-dir JobId 14: Start Restore Job RestoreMSSQL.2013-11-21_17.34.40_16 + 21-Nov 17:34 bareos-dir JobId 14: Using Device "FileStorage" to read. + 21-Nov 17:35 damorgan-sd JobId 14: Ready to read from volume "test1" on device "FileStorage" (/storage). + 21-Nov 17:35 damorgan-sd JobId 14: Forward spacing Volume "test1" to file:block 0:2499099145. + 21-Nov 17:35 damorgan-sd JobId 14: End of Volume at file 0 on device "FileStorage" (/storage), Volume "test1" + 21-Nov 17:35 damorgan-sd JobId 14: End of all volumes. + 21-Nov 17:35 bareos-dir JobId 14: Bareos bareos-dir 13.4.0 (01Oct13): + Build OS: x86_64-pc-linux-gnu debian Debian GNU/Linux 7.0 (wheezy) + JobId: 14 + Job: RestoreMSSQL.2013-11-21_17.34.40_16 + Restore Client: mssqlserver-fd + Start time: 21-Nov-2013 17:34:42 + End time: 21-Nov-2013 17:35:36 + Files Expected: 1 + Files Restored: 3 + Bytes Restored: 202,113,024 + Rate: 3742.8 KB/s + FD Errors: 0 + FD termination status: OK + SD termination status: OK + Termination: Restore OK + +Backup of a PostgreSQL Database +------------------------------- + +.. index:: + pair: PostgreSQL; Backup + +.. index:: + triple: Database; PostgreSQL; Backup; + + +.. _`backup-postgresql`: backup-postgresql + +In this section, we describe different methods how to do backups of the PostgreSQL databases. + +Backup of a PostgreSQL Database by using the RunScript directive +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: RunScript; Example + + +One method to backup a PostgreSQL database is to use the :program:`pg_dumpall` tool to dump the database into a file and then backup it as a normal file. After the backup, the file can be removed. It may also be an option not to remove it, so that the latest version is always available immediately. On the next job run it will be overwritten anyway. + +This can be done by using **Run Script**:sup:`Dir`:sub:`Job` directives inside a Job Resource, for example: + + + + +.. code-block:: sh + :caption: RunScript job resource for a PostgreSQL backup + + Job { + Name = "BackupDatabase" + JobDefs = "DefaultJob" + Client = dbserver-fd + Level = Full + FileSet="Database" + + # This creates a dump of our database in the local filesystem on the client + RunScript { + FailJobOnError = Yes + RunsOnClient = Yes + RunsWhen = Before + Command = "sh -c 'pg_dumpall -U postgres > /var/lib/bareos/postgresql_dump.sql'" + } + + # This deletes the dump in our local filesystem on the client + RunScript { + RunsOnSuccess = Yes + RunsOnClient = Yes + RunsWhen = After + Command = "rm /var/lib/bareos/postgresql_dump.sql" + } + } + + FileSet { + Name = "Database" + Include { + Options { + signature = MD5 + compression = gzip + } + # database dump file + File = "/var/lib/bareos/postgresql_dump.sql" + } + } + +Note that redirecting the :program:`pg_dumpall` output to a file requires to run the whole command line through a shell, otherwise the :program:`pg_dumpall` would not know what do with the :program:`>` character and the job would fail. As no shell features like redirection or piping are used for the :program:`rm`, the :program:`sh -c` is not needed there. See **Run Script**:sup:`Dir`:sub:`Job` for more details. + +Backup of a PostgreSQL Databases by using the bpipe plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: bpipe; PostgreSQL backup + + +Instead of creating a temporary database dump file, the bpipe plugin can be used. For general information about bpipe, see the :ref:`bpipe` section. The bpipe plugin is configured inside the **Include**:sup:`Dir`:sub:`FileSet` section of a File Set, e.g.: + + + + +.. code-block:: sh + :caption: bpipe directive for PostgreSQL backup + + FileSet { + Name = "postgresql-all" + Include { + Plugin = "bpipe:file=/POSTGRESQL/dump.sql:reader=pg_dumpall -U postgres:writer=psql -U postgres" + Options { + signature = MD5 + compression = gzip + } + } + } + +This causes the File Daemon to call bpipe plugin, which will write its data into the "pseudo" file :file:`/POSTGRESQL/dump.sql` by calling the program :program:`pg_dumpall -U postgres` to read the data during backup. The :program:`pg_dumpall` command outputs all the data for the database, which will be read by the plugin and stored in the backup. During restore, the data that was backed up will be sent to the program specified in the last field, which in this +case is psql. When psql is called, it will read the data sent to it by the plugin then write it back to the same database from which it came from. + +This can also be used, to backup a database that is running on a remote host: + + + + +.. code-block:: sh + :caption: bpipe directive to backup a PostgreSQL database that is running on a remote host + + FileSet { + Name = "postgresql-remote" + Include { + Plugin = "bpipe:file=/POSTGRESQL/dump.sql:reader=pg_dumpall -h -U -W :writer=psql -h -U -W " + Options { + signature = MD5 + compression = gzip + } + } + } + +Backup of a PostgreSQL Databases by using the PGSQL-Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; PostgreSQL Backup + + +.. _`backup-postgresql-plugin`: backup-postgresql-plugin + +The PGSQL-Plugin supports an online (Hot) backup of database files and database transaction logs (WAL) archiving (with pgsql-archlog) and backup. With online database and transaction logs the backup plugin can perform Poin-In-Time-Restore up to a single selected transaction or date/time. + +Database recovery is performed fully automatic with dedicated pgsql-restore utility. + +For a full description, see `https://github.com/bareos/contrib-pgsql-plugin/wiki `_. + +Backup of a MySQL Database +-------------------------- + +.. index:: + pair: MySQL; Backup + +.. index:: + triple: Database; MySQL; Backup; + + +.. _`backup-mysql`: backup-mysql + +In this section, we describe different methods to do a full backup of a MySQL database. + +Backup of MySQL Databases using the Bareos MySQL Percona xtrabackup Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Percona xtrabackup}` :raw-latex:`\index[general]{xtrabackup + + +.. _`backup-mysql-xtrabackup`: backup-mysql-xtrabackup + +This plugin is available since 16.2.4, it uses the :program:`xtrabackup` tool from Percona to perform full and incremental hot-backups of MySQL / MariaDB tables of type InnoDB. It can also backup MyISAM tables but only as full backups. On restore it requires a preparation using the xtrabackup tools, before the tables can be restored. If you simply want to backup full dumps, then using +:ref:`backup-mysql-python` is the easier way. + +Prerequisites +^^^^^^^^^^^^^ + +Install the xtrabackup tool from Percona. Documentation and packages are available here: `https://www.percona.com/software/mysql-database/percona-xtrabackup `_. The plugin was successfully tested with xtrabackup versions 2.3.5 and 2.4.4. + +As it is a Python plugin, it will also require to have the package **bareos-filedaemon-python-plugin** installed on the |bareosFd|, where you run it. + +For authentication the :file:`.mycnf` file of the user running the |bareosFd|. Before proceeding, make sure that xtrabackup can connect to the database and create backups. + +Installation +^^^^^^^^^^^^ + +Make sure you have met the prerequisites. Install the files :file:`BareosFdPercona.py` and :file:`bareos-fd-percona.py` in your Bareos plugin directory (usually :file:`/usr/lib64/bareos/plugins`). These files are available in the Git repository `https://github.com/bareos/bareos-contrib/tree/master/fd-plugins/bareos_percona `_. + +Configuration +^^^^^^^^^^^^^ + +Activate your plugin directory in the |bareosFd| configuration. See :ref:`fdPlugins` for more about plugins in general. + + + + +.. code-block:: sh + :caption: bareos-fd client myself + + Client { + ... + Plugin Directory = /usr/lib64/bareos/plugins + Plugin Names = "python" + } + +Now include the plugin as command-plugin in the Fileset resource: + + + + +.. code-block:: sh + :caption: bareos-dir fileset mysql + + FileSet { + Name = "mysql" + Include { + Options { + compression=GZIP + signature = MD5 + } + File = /etc + #... + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-percona:mycnf=/root/.my.cnf" + } + } + +If used this way, the plugin will call xtrabackup to create a backup of all databases in the xbstream format. This stream will be processed by Bareos. If job level is incremental, xtrabackup will perform an incremental backup since the last backup – for InnoDB tables. If you have MyISAM tables, you will get a full backup of those. + +You can append options to the plugin call as key=value pairs, separated by ’:’. The following options are available: + +- With :option:`mycnf` you can make xtrabackup use a special mycnf-file with login credentials. + +- :option:`dumpbinary` lets you modify the default command xtrabackup. + +- :option:`dumpoptions` to modify the options for xtrabackup. Default setting is: :program:`--backup --datadir=/var/lib/mysql/ --stream=xbstream --extra-lsndir=/tmp/individual_tempdir` + +- :option:`restorecommand` to modify the command for restore. Default setting is: :program:`xbstream -x -C` + +- :option:`strictIncremental`: By default (false), an incremental backup will create data, even if the Log Sequence Number (LSN) wasn’t increased since last backup. This is to ensure, that eventual changes to MYISAM tables get into the backup. MYISAM does not support incremental backups, you will always get a full bakcup of these tables. If set to true, no data will be written into backup, if the LSN wasn’t changed. + +Restore +^^^^^^^ + +With the usual Bareos restore mechanism a file-hierarchy will be created on the restore client under the default restore location: + +.. raw:: latex + + \directory{/tmp/bareos-restores/_percona/} + +Each restore job gets an own subdirectory, because Percona expects an empty directory. In that subdirectory, a new directory is created for every backup job that was part of the Full-Incremental sequence. + +The naming scheme is: :file:`fromLSN_toLSN_jobid` + +Example: + + + + /tmp/bareos-restores/_percona/351/ + |-- 00000000000000000000_00000000000010129154_0000000334 + |-- 00000000000010129154_00000000000010142295_0000000335 + |-- 00000000000010142295_00000000000010201260_0000000338 + +This example shows the restore tree for restore job with ID 351. First subdirectory has all files from the first full backup job with ID 334. It starts at LSN 0 and goes until LSN 10129154. + +Next line is the first incremental job with ID 335, starting at LSN 10129154 until 10142295. The third line is the 2nd incremental job with ID 338. + +To further prepare the restored files, use the :program:`xtrabackup --prepare` command. Read `https://www.percona.com/doc/percona-xtrabackup/2.4/xtrabackup_bin/incremental_backups.html `_ for more information. + +Backup of MySQL Databases using the Python MySQL plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; MySQL Backup + + +.. _`backup-mysql-python`: backup-mysql-python + +The Python plugin from `https://github.com/bareos/bareos-contrib/tree/master/fd-plugins/mysql-python `_ makes a backup of all or selected MySQL databases from the |bareosFd| or any other MySQL server. It makes use of the mysqldump command and basically grabs data from mysqldump via pipe. This plugin is suitable to backup database dumps. If you prefer to use mechanisms like incremental hot-backups of InnoDB tables, please use the Bareos MySQL / MariaDB Percona xtrabackup Plugin +(see :ref:`backup-mysql-xtrabackup`). + +Following settings must be done on the Bareos client (|bareosFd|): + +- install and enable the |bareosFd| Python plugin + +- install the Python MySQL plugin (for some platforms it is available prepackaged from :raw-latex:`\contribDownloadBareosOrg`, on the other platforms: copy the plugin files to the Bareos Plugin Directory) + +- disable bacula compatibility (default for Bareos :math:`>=` 15.2) + + + + +.. code-block:: sh + :caption: bareos-fd client myself + + Client { + ... + Plugin Directory = /usr/lib64/bareos/plugins + Plugin Names = "python" + compatible = no + } + +Configure the plugin in the |bareosDir|: + + + + +.. code-block:: sh + :caption: bareos-dir fileset mysql + + FileSet { + Name = "mysql" + Include { + Options { + signature = MD5 + compression = lz4 + } + Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-mysql:db=test,wikidb" + #Plugin = "python:module_path=/usr/lib64/bareos/plugins:module_name=bareos-fd-mysql:mysqlhost=dbhost:mysqluser=bareos:mysqlpassword=bareos" + } + } + +In the above example the plugin creates and saves a dump from the databases called :strong:`test` and :strong:`wikidb`, running on the file-daemon. The commented example below specifies an explicit MySQL server called :option:`dbhost`, and connects with user :strong:`bareos`, password :strong:`bareos`, to create and save a backup of all databases. + +The plugin creates a pipe internally, thus no extra space on disk is needed. You will find one file per database in the backups in the virtual directory :file:`/_mysqlbackups_`. + +List of supported options: + +db + comma separated list of databases to save, where each database will be stored in a separate file. If ommited, all databases will be saved. + +dumpbinary + command (with or without full path) to create the dumps. Default: :strong:`mysqldump` + +dumpoptions + options for dumpbinary, default: :emphasis:`\argument{--events --single-transaction}` + +drop_and_recreate + if not set to :strong:`false`, adds :strong:`--add-drop-database --databases` to dumpoptions + +mysqlhost + MySQL host to connect to, default: :strong:`localhost` + +mysqluser + MySQL user. Default: unset, the user running the file-daemon will be used (usually root) + +mysqlpassword + MySQL password. Default: unset (better use :file:`my.cnf` to store passwords) + +On restore, the database dumps are restored to the subdirectory :file:`_mysqlbackups_` in the restore path. The database restore must be triggered manually (:program:`mysql < _mysqlbackups_/DATABASENAME.sql`). + +Backup of a MySQL Database by using the RunScript directive +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: RunScript; Example + + +One method to backup a MySQL database is to use the :program:`mysqldump` tool to dump the database into a file and then backup it as a normal file. After the backup, the file can be removed. It may also be an option not to remove it, so that the latest version is always available immediately. On the next job run it will be overwritten anyway. + +This can be done by using **Run Script**:sup:`Dir`:sub:`Job` directives, for example: + + + + +.. code-block:: sh + :caption: RunScript job resource for a MySQL backup + + Job { + Name = "BackupDatabase" + JobDefs = "DefaultJob" + Client = dbserver-fd + Level = Full + FileSet="Database" + + # This creates a dump of our database in the local filesystem on the Client + RunScript { + FailJobOnError = Yes + RunsOnClient = Yes + RunsWhen = Before + Command = "sh -c 'mysqldump --user= --password= --opt --all-databases > /var/lib/bareos/mysql_dump.sql'" + } + + # This deletes the dump in the local filesystem on the Client + RunScript { + RunsOnSuccess = Yes + RunsOnClient = Yes + RunsWhen = After + Command = "rm /var/lib/bareos/mysql_dump.sql" + } + } + + FileSet { + Name = "Database" + Include { + Options { + signature = MD5 + compression = gzip + } + # database dump file + File = "/var/lib/bareos/mysql_dump.sql" + } + } + +Note that redirecting the :program:`mysqldump` output to a file requires to run the whole command line through a shell, otherwise the :program:`mysqldump` would not know what do with the :program:`>` character and the job would fail. As no shell features like redirection or piping are used for the :program:`rm`, the :program:`sh -c` is not needed there. See **Run Script**:sup:`Dir`:sub:`Job` for more details. + +Backup of a MySQL Database by using the bpipe plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: bpipe; MySQL backup + + +Instead of creating a temporary database dump file, the bpipe plugin can be used. For general information about bpipe, see the :ref:`bpipe` section. The bpipe plugin is configured inside the Include section of a File Set, e.g.: + + + + +.. code-block:: sh + :caption: bpipe fileset for MySQL backup + + FileSet { + Name = "mysql-all" + Include { + Plugin = "bpipe:file=/MYSQL/dump.sql:reader=mysqldump --user= --password= --opt --all-databases:writer=mysql --user= --password=" + Options { + signature = MD5 + compression = gzip + } + } + } + +This can also be used, to backup a database that is running on a remote host: + + + + +.. code-block:: sh + :caption: bpipe directive to backup a MySQL database that is running on a remote host + + FileSet{ + Name = "mysql-all" + Include { + Plugin = "bpipe:file=/MYSQL/dump.sql:reader=mysqldump --host= --user= --password= --opt --all-databases:writer=mysql --host= --user= --password=" + Options { + signature = MD5 + compression = gzip + } + } + } + +If you do not want a direct restore of your data in your plugin directive, as shown in the examples above, there is the possibility to restore the dump to the filesystem first, which offers you more control over the restore process, e.g.: + + + + +.. code-block:: sh + :caption: bpipe directive to backup a MySQL database and restore the dump to the filesystem first + + FileSet{ + Name = "mysql-all" + Include { + Plugin = "bpipe:file=/MYSQL/dump.sql:reader=mysqldump --host= --user= --password= --opt --all-databases:writer=/usr/lib/bareos/scripts/bpipe-restore.sh" + Options { + signature = MD5 + compression = gzip + } + } + } + +A very simple corresponding shell script (:program:`bpipe-restore.sh`) to the method above might look like the following one: + + + + +.. code-block:: sh + :caption: bpipe shell script for a restore to filesystem + + #!/bin/bash + cat - > /tmp/dump.sql + exit 0 + +.. _section-StatisticCollection: + +Statistics Collection +===================== + +Statistics Collection can be controlled by a number of configuration directives. If Statistics Collection is enabled, statistics are collected by the |bareosDir| and stored into the Catalog database. So enabling this feature will increase your database size. + +The Statistics are used by the |bareosWebui| to show the status of a running job. :raw-latex:`\index[general]{Webui!Configure Statistics Collection}` + +Director Configuration - Director Resource Directives +----------------------------------------------------- + +- + + + + **Statistics Collect Interval**:sup:`Dir`:sub:`Director` + +- + + + + **Statistics Retention**:sup:`Dir`:sub:`Director` + +Director Configuration - Storage Resource Directives +---------------------------------------------------- + +- + + + + **Collect Statistics**:sup:`Dir`:sub:`Storage` + +Storage Configuration - Storage Resource Directives +--------------------------------------------------- + +- + + + + **Collect Device Statistics**:sup:`Sd`:sub:`Storage` + +- + + + + **Collect Job Statistics**:sup:`Sd`:sub:`Storage` + +- + + + + **Statistics Collect Interval**:sup:`Sd`:sub:`Storage` + +Storage Configuration - Device Resource Directives +-------------------------------------------------- + +- + + + + **Collect Statistics**:sup:`Sd`:sub:`Device` + +See chapter :ref:`section-JobStatistics` for additional information. diff --git a/docs/manuals/en/new_main_reference/source/appendix-i/rescue.rst b/docs/manuals/en/new_main_reference/source/appendix-i/rescue.rst new file mode 100644 index 00000000000..eaf5e3b68e3 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-i/rescue.rst @@ -0,0 +1,364 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _RescueChapter: + +Disaster Recovery Using Bareos +============================== + +.. index:: + pair: Disaster; Recovery +.. index:: + pair: Recovery; Disaster Recovery + + +General +------- + +When disaster strikes, you must have a plan, and you must have prepared in advance, otherwise the work of recovering your system and your files will be considerably greater. For example, if you have not previously saved the partitioning information for your hard disk, how can you properly rebuild it if the disk must be replaced? + +Unfortunately, many of the steps one must take before and immediately after a disaster are very much dependent on the operating system in use. As a consequence, this chapter will discuss in detail disaster recovery only for selected operating systems. + +Important Considerations +~~~~~~~~~~~~~~~~~~~~~~~~ + +Here are a few important considerations concerning disaster recovery that you should take into account before a disaster strikes. + +- If the building which houses your computers burns down or is otherwise destroyed, do you have off-site backup data? + +- Disaster recovery is much easier if you have several machines. If you have a single machine, how will you handle unforeseen events if your only machine is down? + +- Do you want to protect your whole system and use Bareos to recover everything? Or do you want to try to restore your system from the original installation disks, apply any other updates and only restore the user files? + +.. _section-before-disaster: + +Steps to Take Before Disaster Strikes +------------------------------------- + +.. index:: + pair: Disaster; Before + + +- Create a rescue or CDROM for your systems. Generally, they are offered by each distribution, and there are many good rescue disks on the Web + +- Ensure that you always have a valid bootstrap file for your backup and that it is saved to an alternate machine. This will permit you to easily do a full restore of your system. + +- If possible copy your catalog nightly to an alternate machine. If you have a valid bootstrap file, this is not necessary, but can be very useful if you do not want to reload everything. + +- Ensure that you always have a valid :ref:`bootstrap ` file for your catalog backup that is saved to an alternate machine. This will permit you to restore your catalog more easily if needed. + +- Try out how to use the Rescue CDROM before you are forced to use it in an emergency situation. + +- Make a copy of your Bareos .conf files, particularly your bareos-dir.conf, and your bareos-sd.conf files, because if your server goes down, these files will be needed to get it back up and running, and they can be difficult to rebuild from memory. + +.. _section-BareMetalRestoreClient: + +Bare Metal Recovery of Bareos Clients +------------------------------------- + +A so called "Bare Metal" recovery is one where you start with an empty hard disk and you restore your machine. + +Generally, following components are required for a Bare Metal Recovery: + +- A rescue CDROM containing a copy of your OS and including the Bareos File daemon, including all libraries + +- The Bareos client configuration files + +- Useful: a copy of your hard disk information + +- A full Bareos backup of your system + +.. _section-rear: + +Linux +~~~~~ + + +.. index:: + triple: Disaster; Recovery; Linux; + + +From the Relax-and-Recover web site (`http://relax-and-recover.org `_): + + Relax-and-Recover is a setup-and-forget Linux bare metal disaster recovery solution. It is easy to set up and requires no maintenance so there is no excuse for not using it. + +Relax-and-Recover (ReaR) is quite easy to use with Bareos. + +Installation +^^^^^^^^^^^^ + +Bareos is a supported backend for ReaR :math:`>=` 1.15. To use the :option:`BAREOS_CLIENT` option, ReaR :math:`>=` 1.17 is required. If ReaR :math:`>=` 1.17 is not part of your distribution, check the :raw-latex:`\elink{download section on the +ReaR website}{http://relax-and-recover.org/download/}`. + +Configuration +^^^^^^^^^^^^^ + +Assuming you have a working Bareos configuration on the system you want to protect with ReaR and Bareos references this system by the name :option:`bareosclient-fd`, the only configuration for ReaR is: + + + + +.. code-block:: sh + :caption: + + BACKUP=BAREOS + BAREOS_CLIENT=bareosclient-fd + +You also need to specify in your ReaR configuration file (:file:`/etc/rear/local.conf`) where you want to store your recovery images. Please refer to the `ReaR documentation `_ for details. + +For example, if you want to create an ISO image and store it to an NFS server with the IP Address 192.168.10.1, you can use the following configuration: + + + + +.. code-block:: sh + :caption: Full Rear configuration in /etc/rear/local.conf + + # This is default: + #OUTPUT=ISO + # Where to write the iso image + # You can use NFS, if you want to write your iso image to a nfs server + # If you leave this blank, it will + # be written to: /var/lib/rear/output/ + OUTPUT_URL=nfs://192.168.10.1/rear + BACKUP=BAREOS + BAREOS_CLIENT=bareosclient-fd + +Backup +^^^^^^ + +If you have installed and configured ReaR on your system, type + + + + +.. code-block:: sh + :caption: Create Rescue Image + + rear -v mkrescue + +to create the rescue image. If you used the configuration example above, you will get a bootable ISO image which can be burned onto a CD. + +.. raw:: latex + + +.. warning:: + This will not create a Bareos backup on your system! You will have to do that by + other means, e.g. by a regular Bareos backup schedule. + Also \command{rear mkbackup will not create a backup. + In this configuration it will only create the rescue ISO + (same as the \command{rear mkrescue} command).} + +Recovery +^^^^^^^^ + +In case, you want to recover your system, boot it using the generated ReaR recovery ISO. After booting log in as user **root** and type + + + + +.. code-block:: sh + :caption: Restore your system using Rear and Bareos + + rear recover + +ReaR will now use the most recent backup from Bareos to restore your system. When the restore job has finished, ReaR will start a new shell which you can use to verify if the system has been restored correctly. The restored system can be found under the :file:`/mnt/local` directory. When you are done< with the verification, type ’exit’ to leave the shell, getting back to the recovery process. Finally, you will be asked to confirm that everything is correct. Type ’yes’ to continue. After that, +ReaR will restore your bootloader. Recovery is complete. + +.. raw:: latex + + \hide{ + \subsection{FreeBSD} + \label{FreeBSD1} + \index[general]{Disaster!Recovery!FreeBSD} + \index[general]{FreeBSD!Disaster Recovery} + + The same basic techniques described above also apply to FreeBSD. Although we + don't yet have a fully automated procedure, Alex Torres Molina has provided us + with the following instructions with a few additions from Jesse Guardiani and + Dan Langille: + + \begin{enumerate} + \item Boot with the FreeBSD installation disk + \item Go to Custom, Partition and create your slices and go to Label and + create the partitions that you want. Apply changes. + \item Go to Fixit to start an emergency console. + \item Create devs ad0 .. .. if they don't exist under /mnt2/dev (in my situation) + with MAKEDEV. The device or devices you create depend on what hard drives you + have. ad0 is your first ATA drive. da0 would by your first SCSI drive. Under + OS version 5 and greater, your device files are most likely automatically + created for you. + \item mkdir /mnt/disk + this is the root of the new disk + \item mount /mnt2/dev/ad0s1a /mnt/disk + mount /mnt2/dev/ad0s1c /mnt/disk/var + mount /mnt2/dev/ad0s1d /mnt/disk/usr + ..... + The same hard drive issues as above apply here too. Note, under OS version 5 + or higher, your disk devices may be in /dev not /mnt2/dev. + \item Network configuration (ifconfig xl0 ip/mask + route add default + ip-gateway) + \item mkdir /mnt/disk/tmp + \item cd /mnt/disk/tmp + \item Copy bareos-fd and bareos-fd.conf to this path + \item If you need to, use sftp to copy files, after which you must do this: + ln -s /mnt2/usr/bin /usr/bin + \item chmod u+x bareos-fd + \item Modify bareos-fd.conf to fit this machine + \item Copy /bin/sh to /mnt/disk, necessary for chroot + \item Don't forget to put your bareos-dir's IP address and domain name in + /mnt/disk/etc/hosts if it's not on a public net. Otherwise the FD on the + machine you are restoring to won't be able to contact the SD and DIR on the + remote machine. + \item mkdir -p /mnt/disk/var/db/bareos + \item chroot /mnt/disk /tmp/bareos-fd -c /tmp/bareos-fd.conf + to start bareos-fd + \item Now you can go to bareos-dir and restore the job with the entire + contents of the broken server. + \item You must create /proc + \end{enumerate} + } + +.. raw:: latex + + \hide{ + \subsection{Solaris} + \label{solaris} + \index[general]{Solaris!Disaster Recovery} + \index[general]{Disaster!Recovery!Solaris} + + The same basic techniques described above apply to Solaris: + + \begin{itemize} + \item the same restrictions as those given for Linux apply + \item you will need to create a Rescue disk + \end{itemize} + + However, during the recovery phase, the boot and disk preparation procedures + are different: + + \begin{itemize} + \item there is no need to create an emergency boot disk since it is an + integrated part of the Solaris boot. + \item you must partition and format your hard disk by hand following manual + procedures as described in W. Curtis Preston's book "Unix Backup \& + Recovery" + \end{itemize} + + Once the disk is partitioned, formatted and mounted, you can continue with + bringing up the network and reloading Bareos. + + \subsubsection{Preparing Solaris Before a Disaster} + + As mentioned above, before a disaster strikes, you should prepare the + information needed in the case of problems. To do so, in the {\bf + rescue/solaris} subdirectory enter: + + + \ + su + ./getdiskinfo + ./make_rescue_disk + \ + + + The {\bf getdiskinfo} script will, as in the case of Linux described above, + create a subdirectory {\bf diskinfo} containing the output from several system + utilities. In addition, it will contain the output from the {\bf SysAudit} + program as described in Curtis Preston's book. This file {\bf + diskinfo/sysaudit.bsi} will contain the disk partitioning information that + will allow you to manually follow the procedures in the "Unix Backup \& + Recovery" book to repartition and format your hard disk. In addition, the + {\bf getdiskinfo} script will create a {\bf start\_network} script. + + Once you have your disks repartitioned and formatted, do the following: + + \begin{itemize} + \item Start Your Network with the {\bf start\_network} script + \item Restore the Bareos File daemon as documented above + \item Perform a Bareos restore of all your files using the same commands as + described above for Linux + \item Re-install your boot loader using the instructions outlined in the + "Unix Backup \& Recovery" book using installboot + \end{itemize} + } + +.. raw:: latex + + \hide{ + \subsection{Windows} + \label{Win3233} + \index[general]{Disaster!Recovery!Windows} + \index[general]{Windows!Disaster Recovery} + + Due to open system files, and registry problems, Bareos cannot save and + restore a complete Win2K/XP/NT environment. + + A suggestion by Damian Coutts using Microsoft's NTBackup utility in + conjunction with Bareos should permit a Full bare metal restore of Win2K/XP + (and possibly NT systems). His suggestion is to do an NTBackup of the critical + system state prior to running a Bareos backup with the following command: + + + \ + ntbackup backup systemstate /F c:\systemstate.bkf + \ + + + The {\bf backup} is the command, the {\bf systemstate} says to backup only the + system state and not all the user files, and the {\bf /F + c:\textbackslash{}systemstate.bkf} specifies where to write the state file. + this file must then be saved and restored by Bareos. This command + can be put in a Client Run Before Job directive so that it is automatically + run during each backup, and thus saved to a Bareos Volume. + + To restore the system state, you first reload a base operating system, then + you would use Bareos to restore all the users files and to recover the {\bf + c:\textbackslash{}systemstate.bkf} file, and finally, run {\bf NTBackup} and + {\bf catalogue} the system statefile, and then select it for restore. The + documentation says you can't run a command line restore of the systemstate. + + This procedure has been confirmed to work by Ludovic Strappazon -- many + thanks! + } + +Restoring a Bareos Server +------------------------- + +.. index:: + pair: Restore; Bareos Server + + +.. _`section-RestoreServer`: section-RestoreServer + +Above, we considered how to recover a client machine where a valid Bareos server was running on another machine. However, what happens if your server goes down and you no longer have a running Director, Catalog, or Storage daemon? There are several solutions: + +#. Bring up static versions of your Director, Catalog, and Storage daemon on the damaged machine. + +#. Move your server to another machine. + +#. Use a Hot Spare Server on another Machine. + +The first option, is very difficult because it requires you to have created a static version of the Director and the Storage daemon as well as the Catalog. If the Catalog uses MySQL or PostgreSQL, this may or may not be possible. In addition, to loading all these programs on a bare system (quite possible), you will need to make sure you have a valid driver for your tape drive. + +The second suggestion is probably a much simpler solution, and one I have done myself. To do so, you might want to consider the following steps: + +- Install the same database server as on the original system. + +- Install Bareos and initialize the Bareos database. + +- Ideally, you will have a copy of all the Bareos conf files that were being used on your server. If not, you will at a minimum need create a bareos-dir.conf that has the same Client resource that was used to backup your system. + +- If you have a valid saved Bootstrap file as created for your damaged machine with WriteBootstrap, use it to restore the files to the damaged machine, where you have loaded a static Bareos File daemon using the Rescue disk). This is done by using the restore command and at the yes/mod/no prompt, selecting **mod** then specifying the path to the bootstrap file. + +- If you have the Bootstrap file, you should now be back up and running, if you do not have a Bootstrap file, continue with the suggestions below. + +- Using **bscan** scan the last set of backup tapes into your MySQL, PostgreSQL or SQLite database. + +- Start Bareos, and using the Console **restore** command, restore the last valid copy of the Bareos database and the Bareos configuration files. + +- Move the database to the correct location. + +- Start the database, and restart Bareos. Then use the Console **restore** command, restore all the files on the damaged machine, where you have loaded a Bareos File daemon using the Rescue disk. + +For additional details of restoring your database, please see the :ref:`section-RestoreCatalog` chapter. diff --git a/docs/manuals/en/new_main_reference/source/appendix-j/troubleshooting.rst b/docs/manuals/en/new_main_reference/source/appendix-j/troubleshooting.rst new file mode 100644 index 00000000000..545857ff4ee --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-j/troubleshooting.rst @@ -0,0 +1,860 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _section-debug-messages: + +Debug Messages +============== + +The Bareos programs contain a lot of debug messages. Normally, these are not printed. See the :ref:`setdebug ` chapter about how to enable them. + +.. _AccessProblems: + +Client Access Problems +====================== + +.. index:: + pair: Problem; Cannot Access a Client + There are several reasons why a |bareosDir| could not contact a client on a different machine. They are: + +- Check if the client file daemon is really running. + +- The Client address or port is incorrect or not resolved by DNS. See if you can ping the client machine using the same address as in the Client record. + +- You have a firewall, and it is blocking traffic on port 9102 between the Director’s machine and the Client’s machine (or on port 9103 between the Client and the Storage daemon machines). + +- If your system is using Tcpwrapper (:file:`hosts.allow` or :file:`hosts.deny` file), verify that is permitting access. + +- Your password or names are not correct in both the Director and the Client machine. Try configuring everything identical to how you run the client on the same machine as the Director, but just change the address. If that works, make the other changes one step at a time until it works. + +Some of the DNS and Firewall problems can be circumvented by configuring clients using :raw-latex:`\nameref {section-ClientInitiatedConnection}` or as :ref:`PassiveClient`. + +Difficulties Connecting from the FD to the SD +--------------------------------------------- + +.. index:: + pair: Problem; Connecting from the FD to the SD + + +If you are having difficulties getting one or more of your File daemons to connect to the Storage daemon, it is most likely because you have not used a fully qualified domain name on the **Address**:sup:`Dir`:sub:`Storage` directive. That is the resolver on the File daemon’s machine (not on the Director’s) must be able to resolve the name you supply into an IP address. An example of an address that is guaranteed not to work: :strong:`localhost`. An example that +may work: :strong:`bareos-sd1`. An example that is more likely to work: :strong:`bareos-sd1.example.com`. + +You can verify how a |bareosFd| resolves a DNS name by the following command: + + + + +.. code-block:: sh + :caption: Test DNS resolution of the \bareosFd \name{bareos-fd} + + *resolve client=bareos-fd NONEXISTINGHOSTNAME + Connecting to Client bareos-fd at bareos:9102 + bareos-fd: Failed to resolve NONEXISTINGHOSTNAME + *resolve client=bareos-fd bareos-sd1.example.com + Connecting to Client bareos-fd at bareos:9102 + bareos-fd resolves bareos-sd1.example.com to host[ipv4;192.168.0.1] + +If your address is correct, then make sure that no other program is using the port 9103 on the Storage daemon’s machine. The Bacula project has reserved these port numbers by IANA, therefore they should only be used by Bacula and its replacements like Bareos. However, apparently some HP printers do use these port numbers. A :program:`netstat -lntp` on the |bareosSd|’s machine can determine who is listening on the 9103 port (used for FD to SD communications in Bareos). + +Authorization Errors +-------------------- + +.. index:: + single: Concurrent Jobs + + +.. _`AuthorizationErrors`: AuthorizationErrors + +For security reasons, Bareos requires that both the File daemon and the Storage daemon know the name of the Director as well as its password. As a consequence, if you change the Director’s name or password, you must make the corresponding change in the Storage daemon’s and in the File daemon’s configuration files. + +During the authorization process, the Storage daemon and File daemon also require that the Director authenticates itself, so both ends require the other to have the correct name and password. + +If you have edited the configuration files and modified any name or any password, and you are getting authentication errors, then your best bet is to go back to the original configuration files generated by the Bareos installation process. Make only the absolutely necessary modifications to these files – e.g. add the correct email address. Then follow the instructions in the :ref:`Running Bareos ` chapter of this manual. You will run a backup to disk and a restore. +Only when that works, should you begin customization of the configuration files. + +Some users report that authentication fails if there is not a proper reverse DNS lookup entry for the machine. This seems to be a requirement of gethostbyname(), which is what Bareos uses to translate names into IP addresses. If you cannot add a reverse DNS entry, or you don’t know how to do so, you can avoid the problem by specifying an IP address rather than a machine name in the appropriate Bareos configuration file. + +Here is a picture that indicates what names/passwords in which files/Resources must match up: + +|image| + +In the left column, you will find the Director, Storage, and Client resources, with their names and passwords – these are all in the |bareosDir| configuration. The right column is where the corresponding values should be found in the Console, Storage daemon (SD), and File daemon (FD) configuration files. + +Another thing to check is to ensure that the Bareos component you are trying to access has :strong:`Maximum Concurrent Jobs` set large enough to handle each of the Jobs and the Console that want to connect simultaneously. Once the maximum connections has been reached, each Bareos component will reject all new connections. + +.. _ConcurrentJobs: + +Concurrent Jobs +=============== + +.. index:: + single: Running Concurrent Jobs}` :raw-latex:`\index[general]{Concurrent Jobs + + +Bareos can run multiple concurrent jobs. Using the :strong:`Maximum Concurrent Jobs` directives, you can configure how many and which jobs can be run simultaneously: + +|bareosDir| + | + + - + + + + **Maximum Concurrent Jobs**:sup:`Dir`:sub:`Director` + + - + + + + **Maximum Concurrent Jobs**:sup:`Dir`:sub:`Client` + + - + + + + **Maximum Concurrent Jobs**:sup:`Dir`:sub:`Job` + + - + + + + **Maximum Concurrent Jobs**:sup:`Dir`:sub:`Storage` + +|bareosSd| + | + + - + + + + **Maximum Concurrent Jobs**:sup:`Sd`:sub:`Storage` + + - + + + + **Maximum Concurrent Jobs**:sup:`Sd`:sub:`Device` + +|bareosFd| + | + + - + + + + **Maximum Concurrent Jobs**:sup:`Fd`:sub:`Client` + +For example, if you want two different jobs to run simultaneously backing up the same Client to the same Storage device, they will run concurrently only if you have set :strong:`Maximum Concurrent Jobs` greater than one in the :strong:`Director` resource, the :strong:`Client` resource, and the :strong:`Storage` resource in |bareosDir| configuration. + + + +.. _`section-Interleaving`: section-Interleaving When running concurrent jobs without :ref:`section-DataSpooling`, the volume format becomes more complicated, consequently, restores may take longer if Bareos must sort through interleaved volume blocks from multiple simultaneous jobs. This can be avoided by having each simultaneous job write to a different volume or by using data spooling We recommend that you read the :ref:`section-DataSpooling` of this manual first, +then test your multiple concurrent backup including restore testing before you put it into production. + +When using random access media as backup space (e.g. disk), you should also read the chapter about :ref:`ConcurrentDiskJobs`. + +Below is a super stripped down :file:`bareos-dir.conf` file showing you the four places where the the file must be modified to allow the same job **NightlySave**:sup:`Dir`:sub:`Job` to run up to four times concurrently. The change to the Job resource is not necessary if you want different Jobs to run at the same time, which is the normal case. + + + + +.. code-block:: sh + :caption: Concurrent Jobs Example + + # + # Bareos Director Configuration file -- bareos-dir.conf + # + Director { + Name = rufus-dir + Maximum Concurrent Jobs = 4 + ... + } + Job { + Name = "NightlySave" + Maximum Concurrent Jobs = 4 + Client = rufus-fd + Storage = File + ... + } + Client { + Name = rufus-fd + Maximum Concurrent Jobs = 4 + ... + } + Storage { + Name = File + Maximum Concurrent Jobs = 4 + ... + } + +Media VolWrites: integer out of range +===================================== + +.. index:: + pair: Errors; integer out of range + +.. index:: + triple: Catalog; Media; VolWrites; + + +In some situation, you receive an error message similar to this: + + + + +.. code-block:: sh + :caption: + + 12-Apr 15:10 bareos-dir JobId 15860: Fatal error: Catalog error updating Media record. sql_update.c:385 update UPDATE Media SET VolJobs=12,VolFiles=10,VolBlocks=155013,VolBytes=10000263168,VolMounts=233,VolErrors=0,VolWrites=2147626019,MaxVolBytes=0,VolStatus='Append',Slot=1,InChanger=1,VolReadTime=0,VolWriteTime=842658562655,LabelType=0,StorageId=3,PoolId=2,VolRetention=144000,VolUseDuration=82800,MaxVolJobs=0,MaxVolFiles=0,Enabled=1,LocationId=0,ScratchPoolId=0,RecyclePoolId=0,RecycleCount=201,Recycle=1,ActionOnPurge=0,MinBlocksize=0,MaxBlocksize=0 WHERE VolumeName='000194L5' failed: + ERROR: integer out of range + +The database column :raw-latex:`\dbcolumn{VolWrites}` in the :raw-latex:`**Media**` table stores the number of write accesses to a volume. It is only used for statistics. + +However, it has happened that the number of write accesses exceeds the maximum value supported by the database column (on |postgresql| it is currently 2147483647, 32 bit, signed integer). The result is a database error, similar to the one mentioned above. + +As a temporary fix, just reset this counter: + + + + +.. code-block:: sh + :caption: Reset the VolWrites counter + + 1000 OK: bareos-dir Version: 17.2.5 (14 Feb 2018) + Enter a period to cancel a command. + *sqlquery + Automatically selected Catalog: MyCatalog + Using Catalog "MyCatalog" + Entering SQL query mode. + Terminate each query with a semicolon. + Terminate query mode with a blank line. + Enter SQL query: UPDATE Media SET VolWrites = 0 WHERE VolWrites > '2000000000'; + No results to list. + SELECT volwrites FROM media; volwrites > '0'; + +-----------+ + | volwrites | + +-----------+ + | 0 | + | 0 | + | 0 | + | 0 | + +-----------+ + Enter SQL query: + +In the long run, it is planed to modify the database schema to enable storing much larger numbers. + +.. _TapeTestingChapter: + +Tape Drive +========== + +.. index:: + pair: Problem; Tape + + +Autochanger +=========== + +.. _AutochangerTesting: + +Testing Autochanger and Adapting mtx-changer script +--------------------------------------------------- + + + +.. _`section-MtxChangerManualUsage}` :raw-latex:`\index[general]{Autochanger!Testing}` :raw-latex:`\index[general]{Autochanger!mtx-changer}` :raw-latex:`\index[general]{Command!mtx-changer}` :raw-latex:`\index[general]{Problem!Autochanger}` :raw-latex:`\index[general]{Problem!mtx-changer`: section-MtxChangerManualUsage}` :raw-latex:`\index[general]{Autochanger!Testing}` :raw-latex:`\index[general]{Autochanger!mtx-changer}` :raw-latex:`\index[general]{Command!mtx-changer}` :raw-latex:`\index[general]{Problem!Autochanger}` :raw-latex:`\index[general]{Problem!mtx-changer + +In case, Bareos does not work well with the Autochanger, it is preferable to "hand-test" that the changer works. To do so, we suggest you do the following commands: + +Make sure Bareos is not running. + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 list 0 /dev/nst0 0} + +.. index:: + single: mtx-changer list + + +This command should print: + +.. raw:: latex + + + + + + 1: + 2: + 3: + ... + +.. raw:: latex + + + +or one number per line for each slot that is occupied in your changer, and the number should be terminated by a colon (**:**). If your changer has barcodes, the barcode will follow the colon. If an error message is printed, you must resolve the problem (e.g. try a different SCSI control device name if **/dev/sg0** is incorrect). For example, on FreeBSD systems, the autochanger SCSI control device is generally **/dev/pass2**. + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 listall 0 /dev/nst0 0} + +.. index:: + single: mtx-changer listall + + +This command should print: + +.. raw:: latex + + + + + + Drive content: D:Drive num:F:Slot loaded:Volume Name + D:0:F:2:vol2 or D:Drive num:E + D:1:F:42:vol42 + D:3:E + + Slot content: + S:1:F:vol1 S:Slot num:F:Volume Name + S:2:E or S:Slot num:E + S:3:F:vol4 + + Import/Export tray slots: + I:10:F:vol10 I:Slot num:F:Volume Name + I:11:E or I:Slot num:E + I:12:F:vol40 + +.. raw:: latex + + + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 transfer 1 2} + +.. index:: + single: mtx-changer listall + + +This command should transfer a volume from source (1) to destination (2) + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 slots} + +.. index:: + single: mtx-changer slots + + +This command should return the number of slots in your autochanger. + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 unload 1 /dev/nst0 0} + +.. index:: + single: mtx-changer unload + + +If a tape is loaded from slot 1, this should cause it to be unloaded. + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 load 3 /dev/nst0 0} + +.. index:: + single: mtx-changer load + + +Assuming you have a tape in slot 3, it will be loaded into drive (0). + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 loaded 0 /dev/nst0 0} + +.. index:: + single: mtx-changer loaded + + +It should print "3" Note, we have used an "illegal" slot number 0. In this case, it is simply ignored because the slot number is not used. However, it must be specified because the drive parameter at the end of the command is needed to select the correct drive. + +.. raw:: latex + + \command{/usr/lib/bareos/scripts/mtx-changer /dev/sg0 unload 3 /dev/nst0 0} + +.. index:: + single: mtx-changer unload + + +will unload the tape into slot 3. + +Once all the above commands work correctly, assuming that you have the right **Changer Command** in your configuration, Bareos should be able to operate the changer. The only remaining area of problems will be if your autoloader needs some time to get the tape loaded after issuing the command. After the **mtx-changer** script returns, Bareos will immediately rewind and read the tape. If Bareos gets rewind I/O errors after a tape change, you will probably need to configure the +:raw-latex:`\parameter{load_sleep}` paramenter in the config file :file:`/etc/bareos/mtx-changer.conf`. You can test whether or not you need a **sleep** by putting the following commands into a file and running it as a script: + +.. raw:: latex + + + + + + #!/bin/sh + /usr/lib/bareos/scripts/mtx-changer /dev/sg0 unload 1 /dev/nst0 0 + /usr/lib/bareos/scripts/mtx-changer /dev/sg0 load 3 /dev/nst0 0 + mt -f /dev/st0 rewind + mt -f /dev/st0 weof + +.. raw:: latex + + + +If the above script runs, you probably have no timing problems. If it does not run, start by putting a **sleep 30** or possibly a **sleep 60** in the script just after the mtx-changer load command. If that works, then you should configure the :option:`load_sleep` paramenter in the config file :file:`/etc/bareos/mtx-changer.conf` to the specified value so that it will be effective when Bareos runs. + +A second problem that comes up with a small number of autochangers is that they need to have the cartridge ejected before it can be removed. If this is the case, the **load 3** will never succeed regardless of how long you wait. If this seems to be your problem, you can insert an eject just after the unload so that the script looks like: + +.. raw:: latex + + + + + + #!/bin/sh + /usr/lib/bareos/scripts/mtx-changer /dev/sg0 unload 1 /dev/nst0 0 + mt -f /dev/st0 offline + /usr/lib/bareos/scripts/mtx-changer /dev/sg0 load 3 /dev/nst0 0 + mt -f /dev/st0 rewind + mt -f /dev/st0 weof + +.. raw:: latex + + + +If this solves your problems, set the parameter :option:`offline` in the config file :file:`/etc/bareos/mtx-changer.conf` to "1". + +Restore +======= + +Restore a pruned job using a pattern +------------------------------------ + +.. index:: + single: Regex + + +It is possible to configure Bareos in a way, that job information are still stored in the Bareos catalog, while the individual file information are already pruned. + +If all File records are pruned from the catalog for a Job, normally Bareos can restore only all files saved. That is there is no way using the catalog to select individual files. With this new feature, Bareos will ask if you want to specify a Regex expression for extracting only a part of the full backup. + + + + Building directory tree for JobId(s) 1,3 ... + There were no files inserted into the tree, so file selection + is not possible.Most likely your retention policy pruned the files + + Do you want to restore all the files? (yes|no): no + + Regexp matching files to restore? (empty to abort): /etc/.* + Bootstrap records written to /tmp/regress/working/zog4-dir.restore.1.bsr + +See also :ref:`FileRegex bsr option ` for more information. + +Problems Restoring Files +------------------------ + +.. index:: + pair: Problem; Restoring Files + +.. index:: + triple: Problem; Tape; fixed mode; + :raw-latex:`\index[general]{Problem!Tape!variable mode}` + +The most frequent problems users have restoring files are error messages such as: + +.. raw:: latex + + + + + + 04-Jan 00:33 z217-sd: RestoreFiles.2005-01-04_00.31.04 Error: + block.c:868 Volume data error at 20:0! Short block of 512 bytes on + device /dev/tape discarded. + +.. raw:: latex + + + +or + +.. raw:: latex + + + + + + 04-Jan 00:33 z217-sd: RestoreFiles.2005-01-04_00.31.04 Error: + block.c:264 Volume data error at 20:0! Wanted ID: "BB02", got ".". + Buffer discarded. + +.. raw:: latex + + + +Both these kinds of messages indicate that you were probably running your tape drive in fixed block mode rather than variable block mode. Fixed block mode will work with any program that reads tapes sequentially such as tar, but Bareos repositions the tape on a block basis when restoring files because this will speed up the restore by orders of magnitude when only a few files are being restored. There are several ways that you can attempt to recover from this unfortunate situation. + +Try the following things, each separately, and reset your Device resource to what it is now after each individual test: + +#. Set "Block Positioning = no" in your Device resource and try the restore. This is a new directive and untested. + +#. Set "Minimum Block Size = 512" and "Maximum Block Size = 512" and try the restore. If you are able to determine the block size your drive was previously using, you should try that size if 512 does not work. This is a really horrible solution, and it is not at all recommended to continue backing up your data without correcting this condition. Please see the :ref:`TapeTestingChapter` section for more on this. + +#. Try editing the restore.bsr file at the Run xxx yes/mod/no prompt before starting the restore job and remove all the VolBlock statements. These are what causes Bareos to reposition the tape, and where problems occur if you have a fixed block size set for your drive. The VolFile commands also cause repositioning, but this will work regardless of the block size. + +#. Use bextract to extract the files you want – it reads the Volume sequentially if you use the include list feature, or if you use a .bsr file, but remove all the VolBlock statements after the .bsr file is created (at the Run yes/mod/no) prompt but before you start the restore. + +Restoring Files Can Be Slow +--------------------------- + +.. index:: + pair: Restore; slow + +.. index:: + triple: Problem; Restore; slow; + + +Restoring files is generally **much** slower than backing them up for several reasons. The first is that during a backup the tape is normally already positioned and Bareos only needs to write. On the other hand, because restoring files is done so rarely, Bareos keeps only the start file and block on the tape for the whole job rather than on a file by file basis which would use quite a lot of space in the catalog. + +Bareos will forward space to the correct file mark on the tape for the Job, then forward space to the correct block, and finally sequentially read each record until it gets to the correct one(s) for the file or files you want to restore. Once the desired files are restored, Bareos will stop reading the tape. + +Finally, instead of just reading a file for backup, during the restore, Bareos must create the file, and the operating system must allocate disk space for the file as Bareos is restoring it. + +For all the above reasons the restore process is generally much slower than backing up (sometimes it takes three times as long). + +.. _section-RestoreCatalog: + +Restoring When Things Go Wrong +------------------------------ + +.. index:: + pair: Catalog; Restore +.. index:: + pair: Restore; Catalog +.. index:: + pair: Problem; Repair Catalog + + +This and the following sections will try to present a few of the kinds of problems that can come up making restoring more difficult. We will try to provide a few ideas how to get out of these problem situations. In addition to what is presented here, there is more specific information on restoring a :ref:`Client ` and your :ref:`Server ` in the :ref:`RescueChapter` chapter of this manual. + +Problem + My database is broken. + +Solution + For SQLite, use the vacuum command to try to fix the database. For either MySQL or PostgreSQL, see the vendor’s documentation. They have specific tools that check and repair databases, see the :ref:`CatMaintenanceChapter` sections of this manual for links to vendor information. + + Assuming the above does not resolve the problem, you will need to restore or rebuild your catalog. Note, if it is a matter of some inconsistencies in the Bareos tables rather than a broken database, then running :ref:`bareos-dbcheck ` might help, but you will need to ensure that your database indexes are properly setup. + +Problem + How do I restore my catalog? + +Solution with a Catalog backup + If you have backed up your database nightly (as you should) and you have made a bootstrap file, you can immediately load back your database (or the ASCII SQL output). Make a copy of your current database, then re-initialize it, by running the following scripts: + + + + ./drop_bareos_tables + ./make_bareos_tables + + After re-initializing the database, you should be able to run Bareos. If you now try to use the restore command, it will not work because the database will be empty. However, you can manually run a restore job and specify your bootstrap file. You do so by entering the **run** command in the console and selecting the restore job. If you are using the default bareos-dir.conf, this Job will be named **RestoreFiles**. Most likely it will prompt you with something such as: + + + + + + + + Run Restore job + JobName: RestoreFiles + Bootstrap: /home/user/bareos/working/restore.bsr + Where: /tmp/bareos-restores + Replace: always + FileSet: Full Set + Client: rufus-fd + Storage: File + When: 2005-07-10 17:33:40 + Catalog: MyCatalog + Priority: 10 + OK to run? (yes/mod/no): + + + + + + A number of the items will be different in your case. What you want to do is: to use the mod option to change the Bootstrap to point to your saved bootstrap file; and to make sure all the other items such as Client, Storage, Catalog, and Where are correct. The FileSet is not used when you specify a bootstrap file. Once you have set all the correct values, run the Job and it will restore the backup of your database, which is most likely an ASCII dump. + + You will then need to follow the instructions for your database type to recreate the database from the ASCII backup file. See the :raw-latex:`\ilink {Catalog Maintenance}{CatMaintenanceChapter}` chapter of this manual for examples of the command needed to restore a database from an ASCII dump (they are shown in the Compacting Your XXX Database sections). + + Also, please note that after you restore your database from an ASCII backup, you do NOT want to do a **make_bareos_tables** command, or you will probably erase your newly restored database tables. + +Solution with a Job listing + If you did save your database but did not make a bootstrap file, then recovering the database is more difficult. You will probably need to use :program:`bextract` to extract the backup copy. First you should locate the listing of the job report from the last catalog backup. It has important information that will allow you to quickly find your database file. For example, in the job report for the CatalogBackup shown below, the critical items are the Volume name(s), the Volume + Session Id and the Volume Session Time. If you know those, you can easily restore your Catalog. + + + + + + + + 22-Apr 10:22 HeadMan: Start Backup JobId 7510, + Job=CatalogBackup.2005-04-22_01.10.0 + 22-Apr 10:23 HeadMan: Bareos 1.37.14 (21Apr05): 22-Apr-2005 10:23:06 + JobId: 7510 + Job: CatalogBackup.2005-04-22_01.10.00 + Backup Level: Full + Client: Polymatou + FileSet: "CatalogFile" 2003-04-10 01:24:01 + Pool: "Default" + Storage: "DLTDrive" + Start time: 22-Apr-2005 10:21:00 + End time: 22-Apr-2005 10:23:06 + FD Files Written: 1 + SD Files Written: 1 + FD Bytes Written: 210,739,395 + SD Bytes Written: 210,739,521 + Rate: 1672.5 KB/s + Software Compression: None + Volume name(s): DLT-22Apr05 + Volume Session Id: 11 + Volume Session Time: 1114075126 + Last Volume Bytes: 1,428,240,465 + Non-fatal FD errors: 0 + SD Errors: 0 + FD termination status: OK + SD termination status: OK + Termination: Backup OK + + + + + + From the above information, you can manually create a bootstrap file, and then follow the instructions given above for restoring your database. A reconstructed bootstrap file for the above backup Job would look like the following: + + + + + + + + Volume="DLT-22Apr05" + VolSessionId=11 + VolSessionTime=1114075126 + FileIndex=1-1 + + + + + + Where we have inserted the Volume name, Volume Session Id, and Volume Session Time that correspond to the values in the job report. We’ve also used a FileIndex of one, which will always be the case providing that there was only one file backed up in the job. + + The disadvantage of this bootstrap file compared to what is created when you ask for one to be written, is that there is no File and Block specified, so the restore code must search all data in the Volume to find the requested file. A fully specified bootstrap file would have the File and Blocks specified as follows: + + + + + + + + Volume="DLT-22Apr05" + VolSessionId=11 + VolSessionTime=1114075126 + VolFile=118-118 + VolBlock=0-4053 + FileIndex=1-1 + + + + + + Once you have restored the ASCII dump of the database, you will then to follow the instructions for your database type to recreate the database from the ASCII backup file. See the :raw-latex:`\ilink {Catalog Maintenance}{CatMaintenanceChapter}` chapter of this manual for examples of the command needed to restore a database from an ASCII dump (they are shown in the Compacting Your XXX Database sections). + + Also, please note that after you restore your database from an ASCII backup, you do NOT want to do a **make_bareos_tables** command, or you will probably erase your newly restored database tables. + +Solution without a Job Listing + If you do not have a job listing, then it is a bit more difficult. Either you use the :ref:`bscan ` program to scan the contents of your tape into a database, which can be very time consuming depending on the size of the tape, or you can use the :ref:`bls ` program to list everything on the tape, and reconstruct a bootstrap file from the bls listing for the file or files you want following the instructions given above. + + There is a specific example of how to use **bls** below. + +Problem + Trying to restore the last known good full backup by specifying item 3 on the restore menu then the JobId to restore, but Bareos then reports: + + + + + + + + 1 Job 0 Files + + + + + + and restores nothing. + +Solution + Most likely the File records were pruned from the database either due to the File Retention period expiring or by explicitly purging the Job. By using the "llist jobid=nn" command, you can obtain all the important information about the job: + + + + + + + + llist jobid=120 + JobId: 120 + Job: save.2005-12-05_18.27.33 + Job.Name: save + PurgedFiles: 0 + Type: B + Level: F + Job.ClientId: 1 + Client.Name: Rufus + JobStatus: T + SchedTime: 2005-12-05 18:27:32 + StartTime: 2005-12-05 18:27:35 + EndTime: 2005-12-05 18:27:37 + JobTDate: 1133803657 + VolSessionId: 1 + VolSessionTime: 1133803624 + JobFiles: 236 + JobErrors: 0 + JobMissingFiles: 0 + Job.PoolId: 4 + Pool.Name: Full + Job.FileSetId: 1 + FileSet.FileSet: BackupSet + + + + + + Then you can find the Volume(s) used by doing: + + + + + + + + sql + select VolumeName from JobMedia,Media where JobId=1 and JobMedia.MediaId=Media.MediaId; + + + + + + Finally, you can create a bootstrap file as described in the previous problem above using this information. + + Bareos will ask you if you would like to restore all the files in the job, and it will collect the above information and write the bootstrap file for you. + +Problem + You don’t have a bootstrap file, and you don’t have the Job report for the backup of your database, but you did backup the database, and you know the Volume to which it was backed up. + +Solution + Either :program:`bscan` the tape (see below for bscanning), or better use :program:`bls` to find where it is on the tape, then use :program:`bextract` to restore the database. For example, + + + + + + + + ./bls -j -V DLT-22Apr05 /dev/nst0 + + + + + + Might produce the following output: :raw-latex:`` + + + + bls: butil.c:258 Using device: "/dev/nst0" for reading. + 21-Jul 18:34 bls: Ready to read from volume "DLT-22Apr05" on device "DLTDrive" + (/dev/nst0). + Volume Record: File:blk=0:0 SessId=11 SessTime=1114075126 JobId=0 DataLen=164 + ... + Begin Job Session Record: File:blk=118:0 SessId=11 SessTime=1114075126 + JobId=7510 + Job=CatalogBackup.2005-04-22_01.10.0 Date=22-Apr-2005 10:21:00 Level=F Type=B + End Job Session Record: File:blk=118:4053 SessId=11 SessTime=1114075126 + JobId=7510 + Date=22-Apr-2005 10:23:06 Level=F Type=B Files=1 Bytes=210,739,395 Errors=0 + Status=T + ... + 21-Jul 18:34 bls: End of Volume at file 201 on device "DLTDrive" (/dev/nst0), + Volume "DLT-22Apr05" + 21-Jul 18:34 bls: End of all volumes. + + + + + + Of course, there will be many more records printed, but we have indicated the essential lines of output. From the information on the Begin Job and End Job Session Records, you can reconstruct a bootstrap file such as the one shown above. + +Problem + How can I find where a file is stored? + +Solution + Normally, it is not necessary, you just use the **restore** command to restore the most recently saved version (menu option 5), or a version saved before a given date (menu option 8). If you know the JobId of the job in which it was saved, you can use menu option 3 to enter that JobId. + + If you would like to know the JobId where a file was saved, select restore menu option 2. + + You can also use the **query** command to find information such as: :raw-latex:`` + + + + *query + Available queries: + 1: List up to 20 places where a File is saved regardless of the + directory + 2: List where the most recent copies of a file are saved + 3: List last 20 Full Backups for a Client + 4: List all backups for a Client after a specified time + 5: List all backups for a Client + 6: List Volume Attributes for a selected Volume + 7: List Volumes used by selected JobId + 8: List Volumes to Restore All Files + 9: List Pool Attributes for a selected Pool + 10: List total files/bytes by Job + 11: List total files/bytes by Volume + 12: List Files for a selected JobId + 13: List Jobs stored on a selected MediaId + 14: List Jobs stored for a given Volume name + 15: List Volumes Bareos thinks are in changer + 16: List Volumes likely to need replacement from age or errors + Choose a query (1-16): + + + + + +Problem + I didn’t backup my database. What do I do now? + +Solution + This is probably the worst of all cases, and you will probably have to re-create your database from scratch and then bscan in all your volumes, which is a very long, painful, and inexact process. + + There are basically three steps to take: + + #. Ensure that your SQL server is running (MySQL or PostgreSQL) and that the Bareos database (normally bareos) exists. See the :ref:`section-CreateDatabase` chapter of the manual. + + #. Ensure that the Bareos databases are created. This is also described at the above link. + + #. Start and stop the Bareos Director using the propriate bareos-dir.conf file so that it can create the Client and Storage records which are not stored on the Volumes. Without these records, scanning is unable to connect the Job records to the proper client. + + When the above is complete, you can begin bscanning your Volumes. Please see the :ref:`bscan` chapter for more details. + +.. |image| image:: \idir Conf-Diagram + :width: 80.0% diff --git a/docs/manuals/en/new_main_reference/source/appendix-k/debug.rst b/docs/manuals/en/new_main_reference/source/appendix-k/debug.rst new file mode 100644 index 00000000000..d90aef72785 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-k/debug.rst @@ -0,0 +1,129 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. index:: + single: Crash +.. index:: + pair: Debug; crash + + +This chapter describes, how to debug Bareos, when the program crashes. If you are just interested about how to get more information about a running Bareos daemon, please read :ref:`section-debug-messages`. + +If you are running on a Linux system, and you have a set of working configuration files, it is very unlikely that **Bareos** will crash. As with all software, however, it is inevitable that someday, it may crash. + +This chapter explains what you should do if one of the three **Bareos** daemons (Director, File, Storage) crashes. When we speak of crashing, we mean that the daemon terminates abnormally because of an error. There are many cases where Bareos detects errors (such as PIPE errors) and will fail a job. These are not considered crashes. In addition, under certain conditions, Bareos will detect a fatal in the configuration, such as lack of permission to read/write the working directory. In that case, +Bareos will force itself to crash with a SEGFAULT. However, before crashing, Bareos will normally display a message indicating why. For more details, please read on. + +Traceback +========= + +.. index:: + single: Traceback + + +Each of the three Bareos daemons has a built-in exception handler which, in case of an error, will attempt to produce a traceback. If successful the traceback will be emailed to you. + +For this to work, you need to ensure that a few things are setup correctly on your system: + +#. You must have a version of Bareos with debug information and not stripped of debugging symbols. When using a packaged version of Bareos, this requires to install the Bareos debug packages (**bareos-debug** on RPM based systems, **bareos-dbg** on Debian based systems). + +#. On Linux, :program:`gdb` (the GNU debugger) must be installed. On some systems such as Solaris, :program:`gdb` may be replaced by :program:`dbx`. + +#. By default, btraceback uses :program:`bsmtp` to send the traceback via email. Therefore it expects a local mail transfer daemon running. It send the traceback to :raw-latex:`\email{root@localhost}` via :strong:`localhost`. + +#. Some Linux distributions, e.g. Ubuntu:raw-latex:`\index[general]{Platform!Ubuntu!Debug}`, disable the possibility to examine the memory of other processes. While this is a good idea for hardening a system, our debug mechanismen will fail. To disable this feature, run (as root): + + + + +.. code-block:: sh + :caption: disable ptrace protection to enable debugging (required on Ubuntu Linux) + + test -e /proc/sys/kernel/yama/ptrace_scope && echo 0 > /proc/sys/kernel/yama/ptrace_scope + +If all the above conditions are met, the daemon that crashes will produce a traceback report and email it. If the above conditions are not true, you can run the debugger by hand as described below. + +Testing The Traceback +===================== + +.. index:: + pair: Traceback; Test + + +To "manually" test the traceback feature, you simply start **Bareos** then obtain the **PID** of the main daemon thread (there are multiple threads). The output produced here will look different depending on what OS and what version of the kernel you are running. + + + + +.. code-block:: sh + :caption: get the process ID of a running Bareos daemon + + ps fax | grep bareos-dir + 2103 ? S 0:00 /usr/sbin/bareos-dir + +which in this case is 2103. Then while Bareos is running, you call the program giving it the path to the Bareos executable and the **PID**. In this case, it is: + + + + +.. code-block:: sh + :caption: get traceback of running Bareos director daemon + + btraceback /usr/sbin/bareos-dir 2103 + +It should produce an email showing you the current state of the daemon (in this case the Director), and then exit leaving **Bareos** running as if nothing happened. If this is not the case, you will need to correct the problem by modifying the :program:`btraceback` script. + +Getting A Traceback On Other Systems +------------------------------------ + +It should be possible to produce a similar traceback on systems other than Linux, either using :program:`gdb` or some other debugger. Solaris:raw-latex:`\index[general]{Platform!Solaris!Debug}` with :program:`dbx` loaded works quite fine. On other systems, you will need to modify the :program:`btraceback` program to invoke the correct debugger, and possibly correct the :file:`btraceback.gdb` script to have appropriate commands for your debugger. +Please keep in mind that for any debugger to work, it will most likely need to run as root. + +Manually Running Bareos Under The Debugger +========================================== + +If for some reason you cannot get the automatic traceback, or if you want to interactively examine the variable contents after a crash, you can run Bareos under the debugger. Assuming you want to run the Storage daemon under the debugger (the technique is the same for the other daemons, only the name changes), you would do the following: + +#. The Director and the File daemon should be running but the Storage daemon should not. + +#. Start the Storage daemon under the debugger: + + + + +.. code-block:: sh + :caption: run the Bareos Storage daemon in the debugger + + gdb --args /usr/sbin/bareos-sd -f -s -d 200 + (gdb) run + + Parameter: + + -f + foreground + + -s + no signals + + -d nnn + debug level + + See section :ref:`daemon command line options ` for a detailed list of options. + +#. At this point, Bareos will be fully operational. + +#. In another shell command window, start the Console program and do what is necessary to cause Bareos to die. + +#. When Bareos crashes, the **gdb** shell window will become active and **gdb** will show you the error that occurred. + +#. To get a general traceback of all threads, issue the following command: + + + + +.. code-block:: sh + :caption: run the Bareos Storage daemon in the debugger + + (gdb) thread apply all bt + + After that you can issue any debugging command. diff --git a/docs/manuals/en/new_main_reference/source/appendix-l/releasenotes.rst b/docs/manuals/en/new_main_reference/source/appendix-l/releasenotes.rst new file mode 100644 index 00000000000..fbc4273667d --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-l/releasenotes.rst @@ -0,0 +1,776 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + + + +.. _`releasenotes}` :raw-latex:`\index[general]{Releases`: releasenotes}` :raw-latex:`\index[general]{Releases + +The technical changelog is automatically generated from the Bareos bug tracking system, see `http://bugs.bareos.org/changelog_page.php `_. + +Please note, that some of the subreleases are only internal development releases. + +Open issues for a specific version are shown at `http://bugs.bareos.org/roadmap_page.php `_. + +The overview about new feature of a release are shown at `https://github.com/bareos/bareos} and in the :ref:`index `_ of this document. + +This chapter concentrates on things to do when updating an existing Bareos installation. + + +.. warning:: + While all the source code is published on `GitHub `_, the releases of packages on \verb|URL:http://download.bareos.org is limited to the initial versions of a major release. Later maintenance releases are only published on https://download.bareos.com.\| + +.. raw:: latex + + \releasenoteSection{Bareos-17.2} + +.. raw:: latex + + \releasenote{17.2.7}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2018-07-13\\ + Database Version & 2171 (unchanged)\\ + Release Ticket & \ticket{966}\\ + Url & \releaseUrlDownloadBareosCom{17.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item \ticket{892} \package{bareos-storage-droplet}: improve error handling on unavailable backend. + \item \ticket{902} \package{bareos-storage-droplet}: improve job status handling (terminate job after all data is written). + \item \ticket{967} :strong:`Windows`: overwrite symbolic links on restore. + \item \ticket{983} \bareosSd: prevent sporadic crash when \linkResourceDirectiveValue{Sd}{Storage}{Collect Job Statistics}{yes}. + \item :strong:`SLES 12sp2` and :strong:`SLES 12sp3`: provide \package{bareos-storage-ceph} and \package{bareos-filedaemon-ceph-plugin} packages. + + \end{itemize} + + } + +.. raw:: latex + + \releasenote{17.2.6}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2018-06-21\\ + Database Version & 2171 (unchanged)\\ + Release Ticket & \ticket{916}\\ + Url & \releaseUrlDownloadBareosCom{17.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item added platforms: :strong:`Fedora 27`, :strong:`Fedora 28`, :strong:`openSUSE 15.0`, :strong:`Ubuntu 18.04` and :strong:`Univention 4.3`. + \item :strong:`Univention 4.3`: fixes integration. + \item \ticket{872} adapted to new Ceph API. + \item \ticket{943} use \package{tirpc} if Sun-RPC is not provided. + \item \ticket{964} fixes the predefined queries. + \item \ticket{969} fixes a problem of restoring more files then selected in \bareosWebui/BVFS. + \item \bareosDir: fixes for a crash after reload in the statistics thread (\ticket{695}, \ticket{903}). + \item \command{bareos-dbcheck}: cleanup and speedup for some some of the checks. + \item adapted for \postgresql 10. + \item gfapi: stale file handles are treated as warnings + \end{itemize} + + } + +.. raw:: latex + + \releasenote{17.2.5}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2018-02-16\\ + Database Version & 2171 (unchanged)\\ + Release Ticket & \ticket{910}\\ + Url & \releaseUrlDownloadBareosCom{17.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item \bareosFd is ready for :strong:`AIX 7.1.0.0`. + \item :ref:`VMwarePlugin` is also provided for :strong:`Debian 9`. + \item NDMP fixes + \item Virtual Backup fixes + \item \package{bareos-storage-droplet}: improvements + \item \command{bareos-dbcheck} improvements and fixes: with older versions it could happen, that it destroys structures required by :strong:`.bvfs_*`. + \item \ticket{850} fixes a bug on :strong:`Univention`: fixes a problem of regenerating passwords when resyncing settings. + \item \ticket{890} :strong:`.bvfs_update` fix. Before there have been cases where it did not update the cache. + \item :strong:`.bvfs_lsdirs` make limit- and offset-option work correctly. + \item :strong:`.bvfs_lsdirs` show special directory (like \verb|path:@bpipe@/|) on the same level as \verb|path:/|. + \item \ticket{895} added description to the output of \bcommand{show}{filesets}. + \item \bareosWebui: Restore Browser fixes + \begin{itemize} + \item There was the possibility of an endless loop if the BVFS API delivers unexpected results. This has been fixed. See bugreports \ticket{887} and \ticket{893} for details. + \item \ticket{905} fixes a problem with file names containing quotes. + \end{itemize} + \item **NDMP Block Size**:sup:`Dir`:sub:`Client` changed type from \dt{Pint32} to \dt{Size32}. This should not affect any configuration, but is more consistent with other block size configuration directives. + \end{itemize} + + } + +.. raw:: latex + + \releasenote{17.2.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2017-12-14\\ + Database Version & 2171\\ + Release Ticket & \ticket{861}\\ + Url & \releaseUrlDownloadBareosOrg{17.2} \\ + & \releaseUrlDownloadBareosCom{17.2} \\ + \end{tabular} + + This release contains several enhancements. Excerpt: + \begin{itemize} + \item Bareos Distribution (packages) + \begin{itemize} + \item \package{python-bareos} is included in the core distribution. + \item \package{bareos-storage-droplet} is a storage backend for the droplet library. + Most notably it allows backup and restores to a S3 environment. + \betaSince{sd}{bareos-storage-droplet}{17.2.4} + \item \package{bat} has been removed, see section :ref:`bat`. + \item platforms: + \begin{itemize} + \item Windows Clients are still supported since Windows Vista. + \item MacOS: added to build chain. + \item \bareosFd is ready for HP-UX 11.31 (ia64). + \item Linux Distribution: Bareos tries to provide packages for all current platforms. For details, refer to :ref:`section-packages`. + \end{itemize} + \item Linux RPM packages: allow read access to /etc/bareos/ for all users (however, relevant files are still only readable for the user **bareos**). + This allows other programs associated with Bareos to also use this directory. + \end{itemize} + + \item Denormalization of the **File** database table + \begin{itemize} + \item The denormalization of the **File** database table leads to enormous performance improvements in installation, which covering a lot of file (millions and more). + \item For the denormalization the database schema must be modified. + +.. warning:: + Updating the database to schema version ≥ 2170 will increase the required disk space. + Especially it will require around twice the amount of the current database disk space during the migration. + \item The **Filename} database table does no longer exists. Therefore the :strong:`.bvfs_*` commands do no longer output the \dbcolumn{FilenameId** column. + \end{itemize} + + \item NDMP\_NATIVE support has been added. This include the NDMP features DAR and DDAR. For details see :ref:`section-NdmpNative`. + \item Updated the package \package{bareos-vmware-plugin} to utilize the Virtual Disk Development Kit (VDDK) 6.5.x. This includes support for \vSphere 6.5 and the next major release (except new features) and backward compatible with \vSphere 5.5 and 6.0. For details see :ref:`VMwarePlugin`. + \item Soft Quota: automatic quota grace period reset if a job does not exceed the quota. + \item \command{bareos-dbcheck}: disable all interactive questions in batch mode. + \item \bcommand{list}{files}: also show deleted files (accurate mode). + \item \bcommand{list}{jobstatastics}: added. + \item :strong:`purge`: added confirmation. + \item \bcommand{list}{volumes}: fix limit and offset handling. + \item \ticket{629} Windows: restore directory attributes. + \item \ticket{639} tape: fix block size handling, AWS VTL iSCSI devices + \item \ticket{705} support for MySQL 5.7 + \item \ticket{719} allow long JSON messages (has been increased from 100KB to 2GB). + \item \ticket{793} Virtual Backups: skip jobs with no files. + \end{itemize} + + } + +.. raw:: latex + + \releasenoteSection{Bareos-16.2} + +.. raw:: latex + + \releasenote{16.2.8}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2018-07-06\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{863}\\ + Url & \releaseUrlDownloadBareosCom{16.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item gfapi-fd Plugin + \begin{itemize} + \item Allow to use non-accurate backups with glusterfind + \item Fix backups with empty glusterfind filelist. + \item Explicitly close glfs fd on IO-open + \item Don't reinitialize the connection to gluster + \item Fix parsing of missing basedir argument + \item Handle non-fatal Gluster problems properly + \end{itemize} + \item Reset JobStatus to previous JobStatus in status SD and FD loops to fix status all output + \item Backport ceph: ported cephfs-fd and \command{cephfs_device} to new api + \item \ticket{967} Windows: Symbolic links are now replaceable during restore + \end{itemize} + } + +.. raw:: latex + + \releasenote{16.2.7}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2017-10-09\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{836}\\ + Url & \releaseUrlDownloadBareosCom{16.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item Fixes a Director crash, when enabling debugging output + \item :strong:`.bvfs_lsdirs`: improve performance, especially when having a large number of directories + \begin{itemize} + \item To optimize the performance of the SQL query used by :strong:`.bvfs_lsdirs`, it is important to + have the following indexes: + \item PostgreSQL + \begin{itemize} + \item \sqlcommand{CREATE INDEX file_jpfnidpart_idx ON File(PathId,JobId,FilenameId) WHERE FileIndex = 0;} + \item If the index \sqlcommand{file_jfnidpart_idx} mentioned in 16.2.6 release notes exist, drop it:\\ + \sqlcommand{DROP INDEX file_jfnidpart_idx;} + \end{itemize} + \item MySQL/MariaDB + \begin{itemize} + \item \sqlcommand{CREATE INDEX PathId_JobId_FileNameId_FileIndex ON File(PathId,JobId,FilenameId,FileIndex);} + \item If the index \sqlcommand{PathId_JobId_FileIndex_FileNameId} mentioned in 16.2.6 release notes exist, drop it:\\ + \sqlcommand{DROP INDEX PathId_JobId_FileIndex_FileNameId ON File;} + \end{itemize} + \end{itemize} + \item Utilize OpenSSL ≥ 1.1 if available + \item Windows: fixes silent upgrade (\command{winbareos-*.exe /S}) + \item Windows: restore attributes also on directories (not only on files) + \item Fixes problem with SHA1 signature when compiled without OpenSSL (not relevant for bareos.org/bareos.com packages) + \item Packages for openSUSE Leap 42.3 and Fedora 26 have been added. + \item Packages for AIX and current HP-UX 11.31 + \end{itemize} + + } + +.. raw:: latex + + \releasenote{16.2.6}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2017-06-22\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{794}\\ + Url & \releaseUrlDownloadBareosCom{16.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item Prevent from director crash when using incorrect paramaters of :strong:`.bvfs_*` commands. + \item Director now closes all configuration files when reloading failed. + \item Storage daemon now closes the network connection when MaximumConcurrentJobs reached. + \item New directive \configdirective{LanAddress} was added to the Client and Storage Resources of the director to facilitate a network topology where client and storage are situated inside of a LAN, but the Director is outside of that LAN. See :ref:`LanAddress` for details. + \item A Problem in the storage abstraction layer was fixed where the director picked the wrong storage daemon when multiple storages/storage daemons were used. + \item The device spool size calculation when using secure erase was fixed. + \item :strong:`.bvfs_lsdirs` no longer shows empty directories from accurate jobs. + \begin{itemize} + \item \warning{This decreases performance if your environment has a large numbers of directories. Creating an index improves the performance.} + %In Bareos 16.2.6 the SQL Query used by :strong:`.bvfs_lsdirs` was changed to not show + %empty directories from accurate jobs. It turned out that that the changed + %query causes performance issues when larger amounts of directories were backed up.\\ + \item \postgresql + \begin{itemize} + \item When using PostgreSQL, creating the following partial improves the performance sufficiently:\\ + \sqlcommand{CREATE INDEX file_jfnidpart_idx ON File(JobId, FilenameId) WHERE FileIndex = 0;} + + \item Run following command to create the partial index:\\ + \path!su - postgres -c 'echo "CREATE INDEX file_jfnidpart_idx ON File(JobId, FilenameId) WHERE FileIndex = 0; ANALYZE File;" | psql bareos'! + + \end{itemize} + \item \mysql + \begin{itemize} + \item When using MySQL or MariaDB, creating the following index improves the performance:\\ + \sqlcommand{CREATE INDEX PathId_JobId_FileIndex_FileNameId ON File(PathId,JobId,FileIndex,FilenameId);} + + \item Run following command to create the index:\\ + \path!echo "CREATE INDEX PathId_JobId_FileIndex_FileNameId ON File(PathId,JobId,FileIndex,FilenameId);" | mysql -u root bareos! + + \item However, with larger amounts of directories and/or involved jobs, even with this index + the performance of :strong:`.bvfs_lsdirs` may still be insufficient. We are working on optimizing + the SQL query for MySQL/MariaDB to solve this problem. + \end{itemize} + \end{itemize} + + \item Packages for Univention UCS 4.2 have been added. + \item Packages for Debian 9 (Stretch) have been added. + \item WebUI: The post install script of the bareos-webui RPM package for RHEL/CentOS was fixed, it no longer tries to run a2enmod which does not exist on RHEL/CentOS. + \item WebUI: The login form no longer allows redirects to arbitrary URLs + \item WebUI: The used ZendFramework components were updated from version 2.4.10 to 2.4.11. + \item WebUI: jQuery was updated from version 1.12.4 to version 3.2.0., some outdated browsers like Internet Explorer 6-8, Opera 12.1x or Safari 5.1+ will no longer be supported, see `jQuery Browser Support `_ for details. + \end{itemize} + + } + +.. raw:: latex + + \releasenote{16.2.5}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2017-03-03\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{734}\\ + Url & \releaseUrlDownloadBareosCom{16.2} \\ + \end{tabular} + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item NDMP: critical bugfix when restoring large files. + \item truncate command allows to free space on disk storages (replaces an purged volume by an empty volume). + \item Some fixes were added regarding director crashes, Windows backups (VSS), soft-quota reset and API (bvfs) problems. + \item WebUI: handle file names containing special characters, hostnames starting with numbers and long logfiles. + \item WebUI: adds translations for Chinese, Italian and Spanish. + \end{itemize} + + } + +.. raw:: latex + + \releasenote{16.2.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2016-10-28\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{698}\\ + Url & \releaseUrlDownloadBareosOrg{16.2} \\ + & \releaseUrlDownloadBareosCom{16.2} \\ + \end{tabular} + + First stable release of the Bareos 16.2 branch. + + \begin{itemize} + \item Configuration + \begin{itemize} + \item Bareos packages contain the default configuration in :ref:`section-ConfigurationSubdirectories`. Please read :ref:`section-UpdateToConfigurationSubdirectories` before updating (make a copy of your configuration directories for your \bareosDir and \bareosSd before updating). Note: as the old configuration files are still supported, in most cases no changes are required. + \item The default configuration does no longer name the \resourcetype{Dir}{Director} and \resourcetype{Sd}{Storage} resources after the systems hostname (\verb|path:$HOSTNAME-dir| resp. \verb|path:$HOSTNAME-sd|) but use \resourcename{Dir}{Director}{bareos-dir} resp. \resourcename{Sd}{Storage}{bareos-sd} as defaults. The prior solution had the disadvantage, that \verb|path:$HOSTNAME-dir| has also been set on \bareosFd not running on the \bareosDir, which almost ever did require changing this setting. Also the new approach aligns better with :ref:`section-ConfigurationSubdirectories`. + \item Due to limitation of the build system, the default resource \resourcename{Dir}{FileSet}{Linux All} have been renamed to \resourcename{Dir}{FileSet}{LinuxAll} (no space between Linux and All). + \item The configuration of the \package{bareos-traymonitor} has also been split into resource files. + Additional, these resource files are now packaged in other packages: + \begin{itemize} + \item \verb|path:CONFIGDIR/tray-monitor.d/monitor/bareos-mon.conf|: \package{bareos-traymonitor} + \item \verb|path:CONFIGDIR/tray-monitor.d/client/FileDaemon-local.conf|: \package{bareos-filedaemon} + \item \verb|path:CONFIGDIR/tray-monitor.d/storage/StorageDaemon-local.conf|: \package{bareos-storage} + \item \verb|path:CONFIGDIR/tray-monitor.d/director/Director-local.conf|: :file:`bareos-director` + \end{itemize} + This way, the \package{bareos-traymonitor} will be configured automatically for the installed components. + \end{itemize} + \item Strict ACL handling + \begin{itemize} + \item Bareos Console \dt{Acl}s do no longer automatically matches substrings + (to avoid that e.g. \linkResourceDirectiveValue{Dir}{Console}{Pool ACL}{Full} also matches \pool{VirtualFull}). + To configure the ACL to work as before, \linkResourceDirectiveValue{Dir}{Console}{Pool ACL}{.*Full.*} must be set. + Unfortunately the \bareosWebui 15.2 \resourcename{Dir}{Profile}{webui} did use \linkResourceDirectiveValue{Dir}{Console}{Command ACL}{.bvfs*}, which is also no longer works as intended. Moreover, to use all of \bareosWebui 16.2 features, some additional commands must be permitted, so best use the new \resourcename{Dir}{Profile}{webui-admin}. + \end{itemize} + \item \bareosWebui + \begin{itemize} + \item Updating from Bareos 15.2: Adapt \resourcename{Dir}{Profile}{webui} (from bareos 15.2) to allow all commands of \resourcename{Dir}{Profile}{webui-admin} (**Command ACL**:sup:`Dir`:sub:`Console` ). + Alternately modify all \resourcetype{Dir}{Console}s currently using \resourcename{Dir}{Profile}{webui} to use \resourcename{Dir}{Profile}{webui-admin} instead. + \item While RHEL 6 and CentOS 6 are still platforms supported by Bareos, the package \package{bareos-webui} is not available for these platforms, as the required ZendFramework 2.4 do require PHP $>=$ 5.3.17 (5.3.23). However, it is possible to use \package{bareos-webui} 15.2 against \package{bareos-director} 16.2. Also here, the profile must be adapted. + \end{itemize} + \end{itemize} + } + +.. raw:: latex + + \releasenoteSection{Bareos-15.2} + +.. raw:: latex + + \releasenote{15.2.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2016-06-10\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{641} \\ + Url & \releaseUrlDownloadBareosCom{15.2} \\ + \end{tabular} + + For upgrading from 14.2, please see release notes for 15.2.1. + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item Automatic mount of disks by SD + \item NDMP performance enhancements + \item Windows: sparse file restore + \item Director memory leak caused by frequent bconsole calls + \end{itemize} + } + +.. raw:: latex + + \releasenote{15.2.3}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2016-03-11\\ + Database Version & 2004 (unchanged)\\ + Release Ticket & \ticket{625} \\ + Url & \releaseUrlDownloadBareosCom{15.2} \\ + \end{tabular} + + For upgrading from 14.2, please see releasenotes for 15.2.1. + + This release contains several bugfixes and enhancements. Excerpt: + \begin{itemize} + \item VMWare plugin can now restore to VMDK file + \item Ceph support for SLES12 included + \item Multiple gfapi and ceph enhancements + \item NDMP enhancements and bugfixes + \item Windows: multiple VSS Jobs can now run concurrently in one FD, installer fixes + \item bpipe: fix stderr/stdout problems + \item reload command enhancements (limitations eliminated) + \item label barcodes now can run without interaction + \end{itemize} + } + +.. raw:: latex + + \releasenote{15.2.2}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-11-19\\ + Database Version & 2004\\ + & Database update required (if coming from bareos-14.2). See the :ref:`bareos-update` section.\\ + Release Ticket & \ticket{554} \\ + Url & \releaseUrlDownloadBareosOrg{15.2} \\ + & \releaseUrlDownloadBareosCom{15.2} \\ + \end{tabular} + + First stable release of the Bareos 15.2 branch. + + When coming from bareos-14.2.x, the following things have changed (same as in bareos-15.2.1): + \begin{itemize} + \item The default setting for the Bacula Compatbile mode in **Compatible**:sup:`Fd`:sub:`Client` and **Compatible**:sup:`Sd`:sub:`Storage` have been changed from \argument{yes} to \argument{no}. + \item The configuration syntax for Storage Daemon Cloud Backends Ceph and GlusterFS have changed. + Before bareos-15.2, options have been configured as part of the **Archive Device**:sup:`Sd`:sub:`Device` directive, while now the Archive Device contains only information text and options are defined via the **Device Options**:sup:`Sd`:sub:`Device` directive. See examples in **Device Options**:sup:`Sd`:sub:`Device` . + \end{itemize} + + } + +.. raw:: latex + + \releasenoteUnstable{15.2.1}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-09-16\\ + Database Version & 2004\\ + & Database update required, see the :ref:`bareos-update` section.\\ + Release Ticket & \ticket{501} \\ + Url & \releaseUrlDownloadBareosOrg{15.2} \\ + \end{tabular} + + Beta release. + + \begin{itemize} + \item The default setting for the Bacula Compatbile mode in **Compatible**:sup:`Fd`:sub:`Client` and **Compatible**:sup:`Sd`:sub:`Storage` have been changed from \argument{yes} to \argument{no}. + \item The configuration syntax for Storage Daemon Cloud Backends Ceph and GlusterFS have changed. + Before bareos-15.2, options have been configured as part of the **Archive Device**:sup:`Sd`:sub:`Device` directive, while now the Archive Device contains only information text and options are defined via the **Device Options**:sup:`Sd`:sub:`Device` directive. See examples in **Device Options**:sup:`Sd`:sub:`Device` . + % # Old syntax: + % # Archive Device = /etc/ceph/ceph.conf:poolname + % # + % # New syntax: + % # Archive Device = + % # Device Options = "conffile=/etc/ceph/ceph.conf,poolname=poolname" + \end{itemize} + + } + +.. raw:: latex + + \releasenoteSection{Bareos-14.2} + +It is known, that :program:`drop_database` scripts will not longer work on PostgreSQL :math:`<` 8.4. However, as :program:`drop_database` scripts are very seldom needed, package dependencies do not yet enforce PostgreSQL :math:`>=` 8.4. We plan to ensure this in future version of Bareos. + +.. raw:: latex + + \releasenote{14.2.7}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2016-07-11\\ + Database Version & 2003 (unchanged)\\ + Release Ticket & \ticket{584} \\ + Url & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + This release contains several bugfixes. Excerpt: + \begin{itemize} + \item bareos-dir + \begin{itemize} + \item Fixes pretty printing of Fileset options block \\ + \ticket{591}: config pretty-printer does not print filesets correctly + \item run command: fixes changing the pool when changing the backup level in interactive mode \\ + \ticket{633}: Interactive run doesn't update pool on level change + \item Ignore the Fileset option DriveType on non Windows systems \\ + \ticket{644}: Setting DriveType causes empty backups on Linux + \item Suppress already queued jobs for disabled schedules \\ + \ticket{659}: Suppress already queued jobs for disabled schedules + \end{itemize} + \item NDMP + \begin{itemize} + \item Fixes cancel of NDMP jobs\\ + \ticket{604}: Cancel a NDMP Job causes the sd to crash + \end{itemize} + \item bpipe-fd plugin + \begin{itemize} + \item Only take stdout into account, ignore stderr (like earlier versions) \\ + \ticket{632}: fd-bpipe plugin merges stderr with stdout, which can result in corrupted backups + \end{itemize} + \item win32 + \begin{itemize} + \item Fix symlink and junction support\\ + \ticket{575}: charset problem in symlinks/junctions windows restore \\ + \ticket{615}: symlinks/junctions wrong target path on restore (wide chars) + \item Fixes quoting for bmail.exe in bareos-dir.conf \\ + \ticket{581}: Installer is setting up a wrong path to bmail.exe without quotes / bmail not called + \item Fix crash on restore of sparse files \\ + \ticket{640}: File daemon crashed after restoring sparse file on windows + \end{itemize} + \item win32 mssql plugin + \begin{itemize} + \item Allow connecting to non default instance \\ + \ticket{383}: mssqldvi problem with connection to mssql not default instance + \item Fix backup/restore of incremental backups \\ + \ticket{588}: Incremental MSSQL backup fails when database name contains spaces + \end{itemize} + \end{itemize} + } + +.. raw:: latex + + \releasenote{14.2.6}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-12-03\\ + Database Version & 2003 (unchanged)\\ + Release Ticket & \ticket{474} \\ + Url & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + This release contains several bugfixes. + } + +.. raw:: latex + + \releasenote{14.2.5}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-06-01\\ + Database Version & 2003 (unchanged)\\ + Release Ticket & \ticket{447} \\ + Url & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + This release contains several bugfixes and added the platforms :strong:`Debian 8` and :strong:`Fedora 21`. + } + +.. raw:: latex + + \releasenote{14.2.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-03-23 \\ + Database Version & 2003 (unchanged)\\ + Release Ticket & \ticket{420} \\ + Url & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + This release contains several bugfixes, including one major bugfix (\ticket{437}), relevant for those of you using backup to disk with autolabeling enabled. + + It can lead to loss of a 64k block of data when all of this conditions apply: + \begin{itemize} + \item backups are written to disk (tape backups are not affected) + \item autolabelling is enabled + \item a backup spans over multiple volumes + \item the additional volumes are newly created and labeled during the backup + \end{itemize} + If existing volumes are used for backups spanning over multiple volumes, the problem does not occur. + + We recommend to update to the latest packages as soon as possible. + + If an update is not possible immediately, + autolabeling should be disabled and volumes should be labelled manually + until the update can be installed. + + If you are affected by the 64k bug, we recommend that you schedule a full backup after fixing the problem in order to get a + proper full backup of all files. + } + +.. raw:: latex + + \releasenote{14.2.3}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-02-02 \\ + Database Version & 2003 (unchanged)\\ + Release Ticket & \ticket{393}\\ + Url & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + } + +.. raw:: latex + + \releasenote{14.2.2}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2014-12-12 \\ + Database Version & 2003 (unchanged)\\ + & Database update required if updating from version $<$ 14.2.\\ + & See the :ref:`bareos-update` section for details.\\ + Url & \releaseUrlDownloadBareosOrg{14.2} \\ + & \releaseUrlDownloadBareosCom{14.2} \\ + \end{tabular} + + First stable release of the Bareos 14.2 branch. + } + +.. raw:: latex + + \releasenoteUnstable{14.2.1}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2014-09-22 \\ + Database Version & 2003\\ + & Database update required, see the :ref:`bareos-update` section.\\ + Url & \releaseUrlDownloadBareosOrg{14.2} \\ + \end{tabular} + + Beta release. + } + +.. raw:: latex + + \releasenoteSection{Bareos-13.2} + +.. raw:: latex + + \releasenote{13.2.5}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-12-03 \\ + Database Version & 2002 (unchanged)\\ + Url & \releaseUrlDownloadBareosCom{13.2} \\ + \end{tabular} + + This release contains several bugfixes. + } + +.. raw:: latex + + \releasenote{13.2.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2014-11-05 \\ + Database Version & 2002 (unchanged)\\ + Url & \releaseUrlDownloadBareosCom{13.2} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{13.2.3}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2014-03-11 \\ + Database Version & 2002\\ + & Database update required, see the :ref:`bareos-update` section.\\ + Url & \releaseUrlDownloadBareosCom{13.2} \\ + \end{tabular} + + It is known, that \command{drop_database} scripts will not longer work on PostgreSQL $<$ 8.4. However, as \command{drop_database} scripts are very seldom needed, package dependencies do not yet enforce PostgreSQL $>=$ 8.4. We plan to ensure this in future version of Bareos. + } + +.. raw:: latex + + \releasenote{13.2.2}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-11-19 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosOrg{13.2} \\ + & \releaseUrlDownloadBareosCom{13.2} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenoteSection{Bareos-12.4} + +.. raw:: latex + + \releasenote{12.4.8}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2015-11-18 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosCom{12.4} \\ + \end{tabular} + + This release contains several bugfixes. + } + +.. raw:: latex + + \releasenote{12.4.6}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-11-19 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosOrg{12.4} \\ + & \releaseUrlDownloadBareosCom{12.4} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{12.4.5}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-09-10 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosCom{12.4} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{12.4.4}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-06-17 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosOrg{12.4} \\ + & \releaseUrlDownloadBareosCom{12.4} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{12.4.3}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-04-15 \\ + Database Version & 2001 (unchanged)\\ + Url & \releaseUrlDownloadBareosOrg{12.4} \\ + & \releaseUrlDownloadBareosCom{12.4} \\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{12.4.2}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-03-03 \\ + Database Version & 2001 (unchanged)\\ + \end{tabular} + } + +.. raw:: latex + + \releasenote{12.4.1}{ + + \begin{tabular}{p{0.2\textwidth} p{0.8\textwidth}} + Code Release & 2013-02-06 \\ + Database Version & 2001 (initial)\\ + \end{tabular} + + This have been the initial release of Bareos. + + Information about migrating from Bacula to Bareos are available at `Howto upgrade from Bacula to Bareos `_ and in section :ref:`compat-bacula`. + } diff --git a/docs/manuals/en/new_main_reference/source/appendix-m/license.rst b/docs/manuals/en/new_main_reference/source/appendix-m/license.rst new file mode 100644 index 00000000000..3ab5c8e7365 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/appendix-m/license.rst @@ -0,0 +1,80 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +Licenses Overview +================= + +There are a number of different licenses that are used in Bareos. + +FDL +--- + +.. index:: + pair: License; FDL + + +The :ref:`GNU Free Documentation License (FDL) ` is used for this manual, which is a free and open license. This means that you may freely reproduce it and even make changes to it. + +AGPL +---- + +.. index:: + pair: License; AGPL + + +The vast bulk of the source code is released under the :ref:`GNU Affero General Public License (AGPL) version 3 `. + +Most of this code is copyrighted: Copyright ©2000-2012 Free Software Foundation Europe e.V. + +All new code is copyrighted: Copyright ©2013-2013 Bareos GmbH & Co. KG + +Portions may be copyrighted by other people. These files are released under different licenses which are compatible with the AGPL license. + +LGPL +---- + +.. index:: + pair: License; LGPL + + +Some of the Bareos library source code is released under the :ref:`GNU Lesser General Public License (LGPL) `. This permits third parties to use these parts of our code in their proprietary programs to interface to Bareos. + +Public Domain +------------- + +.. index:: + pair: License; Public Domain + + +Some of the Bareos code, or code that Bareos references, has been released to the public domain. E.g. md5.c, SQLite. + +Trademark +--------- + +.. index:: + single: Trademark + + +Bareos:raw-latex:`\raisebox{.6ex}{\textsuperscript{\textregistered}}` is a registered trademark of Bareos GmbH & Co. KG. + +Bacula:raw-latex:`\raisebox{.6ex}{\textsuperscript{\textregistered}}` is a registered trademark of Kern Sibbald. + +Disclaimer +---------- + +.. index:: + single: Disclaimer + + +NO WARRANTY + +BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH +ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +Other Copyrights and Trademarks +------------------------------- + +Certain words and/or products are Copyrighted or Trademarked such as Windows (by Microsoft). Since they are numerous, and we are not necessarily aware of the details of each, we don’t try to list them here. However, we acknowledge all such Copyrights and Trademarks, and if any copyright or trademark holder wishes a specific acknowledgment, notify us, and we will be happy to add it where appropriate. diff --git a/docs/manuals/en/new_main_reference/source/chapter01/general.rst b/docs/manuals/en/new_main_reference/source/chapter01/general.rst new file mode 100644 index 00000000000..77df65a9536 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter01/general.rst @@ -0,0 +1,389 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _GeneralChapter: + +What is Bareos? +=============== + +Bareos is a set of computer programs that permits the system administrator to manage backup, recovery, and verification of computer data across a network of computers of different kinds. Bareos can also run entirely upon a single computer and can backup to various types of media, including tape and disk. + +In technical terms, it is a network Client/Server based backup program. Bareos is relatively easy to use and efficient, while offering many advanced storage management features that make it easy to find and recover lost or damaged files. Due to its modular design, Bareos is scalable from small single computer systems to systems consisting of hundreds of computers located over a large network. + +.. _History: + +History +------- + +Bareos is a `fork `_ of the open source project `Bacula `_ version 5.2. In 2010 the Bacula community developer Marco van Wieringen started to collect rejected or neglected community contributions in his own branch. This branch was later on the base of Bareos and since then was enriched by a lot of new features. + +This documentation also bases on the `original Bacula documentation `_, it is technically also a fork of the documenation created following the rules of the GNU Free Documentation License. + +Original author of Bacula and its documentation is Kern Sibbald. We thank Kern and all contributors to Bacula and it’s documentation. We maintain a list of contributors to Bacula (until the time we’ve started the fork) and Bareos in our `AUTHORS `_ file. + +Who Needs Bareos? +----------------- + +If you are currently using a program such as tar, dump, or bru to backup your computer data, and you would like a network solution, more flexibility, or catalog services, Bareos will most likely provide the additional features you want. However, if you are new to Unix systems or do not have offsetting experience with a sophisticated backup package, the Bareos project does not recommend using Bareos as it is much more difficult to setup and use than tar or dump. + +If you want Bareos to behave like the above mentioned simple programs and write over any tape that you put in the drive, then you will find working with Bareos difficult. Bareos is designed to protect your data following the rules you specify, and this means reusing a tape only as the last resort. It is possible to "force" Bareos to write over any tape in the drive, but it is easier and more efficient to use a simpler program for that kind of operation. + +If you would like a backup program that can write to multiple volumes (i.e. is not limited by your tape drive capacity), Bareos can most likely fill your needs. + +If you are currently using a sophisticated commercial package such as Legato Networker, ARCserveIT, Arkeia, IBM Tivoli Storage Manager or PerfectBackup+, you may be interested in Bareos, which provides many of the same features and is free software available under the GNU AGPLv3 software license. + +Bareos Components or Services +----------------------------- + +Bareos is made up of the following major components or services: Director, Console, File, Storage, and Monitor services. + +.. _DirDef: + +Bareos Director +~~~~~~~~~~~~~~~ + +The Director is the central control program for all the other daemons. It schedules and supervises all the backup, restore, verify and archive operations. The system administrator uses the Bareos Director to schedule backups and to recover files. The Director runs as a daemon (or service) in the background. + +.. _`UADef`: UADef + +Bareos Console +~~~~~~~~~~~~~~ + +The Bareos Console (:program:`bconsole`) is the program that allows the administrator or user to communicate with the |bareosDir|. It runs in a shell window (i.e. TTY interface). Most system administrators will find this completely adequate. For more details see the :ref:`section-bconsole`. + +.. _FDDef: + +Bareos File Daemon +~~~~~~~~~~~~~~~~~~ + +The |bareosFd| is a program that must be installed on each (Client) machine that should be backed up. At the request of the |bareosDir|, it finds the files to be backed up and sends them (their data) to the |bareosSd|. + +It is specific to the operating system on which it runs and is responsible for providing the file attributes and data when requested by the |bareosDir|. + +The |bareosFd| is also responsible for the file system dependent part of restoring the file attributes and data during a recovery operation. This program runs as a daemon on the machine to be backed up. + +.. _SDDef: + +Bareos Storage Daemon +~~~~~~~~~~~~~~~~~~~~~ + +The |bareosSd| is responsible, at the |bareosDir| request, for accepting data from a |bareosFd| and storing the file attributes and data to the physical backup media or volumes. In the case of a restore request, it is responsible to find the data and send it to the |bareosFd|. + +There can be multiple |bareosSd| in your environment, all controlled by the same |bareosDir|. + +The Storage services runs as a daemon on the machine that has the backup device (such as a tape drive). + +.. _DBDefinition: + +Catalog +~~~~~~~ + +The Catalog services are comprised of the software programs responsible for maintaining the file indexes and volume databases for all files backed up. The Catalog services permit the system administrator or user to quickly locate and restore any desired file. The Catalog services sets Bareos apart from simple backup programs like tar and bru, because the catalog maintains a record of all Volumes used, all Jobs run, and all Files saved, permitting efficient restoration and Volume management. +Bareos currently supports three different databases, MySQL, PostgreSQL, and SQLite, one of which must be chosen when building Bareos. + +The three SQL databases currently supported (MySQL, PostgreSQL or SQLite) provide quite a number of features, including rapid indexing, arbitrary queries, and security. Although the Bareos project plans to support other major SQL databases, the current Bareos implementation interfaces only to MySQL, PostgreSQL and SQLite. + +To perform a successful save or restore, the following four daemons must be configured and running: the Director daemon, the File daemon, the Storage daemon, and the Catalog service (MySQL, PostgreSQL or SQLite). + +Bareos Version Numbers and Releases +----------------------------------- + +.. index:: + single: Version numbers}` :raw-latex:`\index[general]{Releases + + +Bareos version numbers consists of three parts: YY.Q.C + ++------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| YY | year (last two digits) | ++------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Q | quarter of the year | ++------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| YY.Q | year and quarter of the code freeze. After this, as a general rule, no new feature should get introduced to this Bareos branch. Subsequent release are for bugfixing. | ++------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| C | Release counter. For every subsequent release, this counter is incremented. Beginning with 16.2, numbers from 1 to 3 represents the month of the quarter during development. After the code freeze, the number is set to 4. So, stable releases get number from 4 onwards. Maintenance releases get numbers starting from 5 onwards. | ++------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Following information can be determined from the Bareos release bareos-16.2.4: + +- 16.2: Code freeze have been in the second quarter of 2016 + +- 4: this is the first stable release of the bareos-16.2 branch + +For details about the different releases see :ref:`releasenotes`. + +.. _section-BareosPackages: + +Bareos Packages +--------------- + +Following Bareos Linux packages are available (release 17.2.4): + ++--------------------------------------+---------------------------------------------------------------------------+ +| **Package Name** | **Description** | ++======================================+===========================================================================+ +| bareos | Backup Archiving REcovery Open Sourced - metapackage | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-bconsole | Bareos administration console (CLI) | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-client | Bareos client Meta-All-In-One package | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-common | Common files, required by multiple Bareos packages | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-database-common | Generic abstraction libs and files to connect to a database | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-database-mysql | Libs and tools for mysql catalog | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-database-postgresql | Libs and tools for postgresql catalog | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-database-sqlite3 | Libs and tools for sqlite3 catalog | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-database-tools | Bareos CLI tools with database dependencies (bareos-dbcheck, bscan) | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-devel | Devel headers | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-director | Bareos Director daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-director-python-plugin | Python plugin for Bareos Director daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-filedaemon | Bareos File daemon (backup and restore client) | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-filedaemon-ceph-plugin | CEPH plugin for Bareos File daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-filedaemon-glusterfs-plugin | GlusterFS plugin for Bareos File daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-filedaemon-ldap-python-plugin | LDAP Python plugin for Bareos File daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-filedaemon-python-plugin | Python plugin for Bareos File daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-regress-config | Required files for bareos-regress | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage | Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-ceph | CEPH support for the Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-droplet | Object Storage support (through libdroplet) for the Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-fifo | FIFO support for the Bareos Storage backend | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-glusterfs | GlusterFS support for the Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-python-plugin | Python plugin for Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-storage-tape | Tape support for the Bareos Storage daemon | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-tools | Bareos CLI tools (bcopy, bextract, bls, bregex, bwild) | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-traymonitor | Bareos Tray Monitor (QT) | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-vadp-dumper | VADP Dumper - vStorage APIs for Data Protection Dumper program | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-vmware-plugin | Bareos VMware plugin | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-vmware-plugin-compat | Bareos VMware plugin compatibility | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-vmware-vix-disklib | VMware vix disklib distributable libraries | ++--------------------------------------+---------------------------------------------------------------------------+ +| bareos-webui | Bareos Web User Interface | ++--------------------------------------+---------------------------------------------------------------------------+ +| python-bareos | Backup Archiving REcovery Open Sourced - Python module | ++--------------------------------------+---------------------------------------------------------------------------+ + +Not all packages (especially optional backends and plugins) are available on all platforms. For details, see :ref:`section-packages`. + +Additionally, packages containing debug information are available. These are named differently depending on the distribution (**bareos-debuginfo** or **bareos-dbg** or :math:`\ldots`). + +Not all packages are required to run Bareos. + +- For the Bareos Director, the package **bareos-director** and one of **bareos-database-postgresql**, **bareos-database-mysql** or **bareos-database-sqlite3** are required. It is recommended to use **bareos-database-postgresql**. + +- For the |bareosSd|, the package **bareos-storage** is required. If you plan to connect tape drives to the storage director, also install the package **bareos-storage-tape**. This is kept separately, because it has additional dependencies for tape tools. + +- On a client, only the package **bareos-filedaemon** is required. If you run it on a workstation, the packages **bareos-traymonitor** gives the user information about running backups. + +- On a Backup Administration system you need to install at least **bareos-bconsole** to have an interactive console to the |bareosDir|. + +Quick Start +----------- + +To get Bareos up and running quickly, the author recommends that you first scan the Terminology section below, then quickly review the next chapter entitled :ref:`The Current State of Bareos `, then the :ref:`Installing Bareos `, the :ref:`Getting Started with Bareos `, which will give you a quick overview of getting Bareos running. After which, you should proceed to the chapter +:ref:`How to Configure Bareos `, and finally the chapter on :ref:`Running Bareos `. + +Terminology +----------- + +.. index:: + single: Terminology + + +Administrator +.. index:: + single: Administrator + The person or persons responsible for administrating the Bareos system. + +Backup +.. index:: + single: Backup + The term Backup refers to a Bareos Job that saves files. + +Bootstrap File +.. index:: + single: Bootstrap File + The bootstrap file is an ASCII file containing a compact form of commands that allow Bareos or the stand-alone file extraction utility (bextract) to restore the contents of one or more Volumes, for example, the current state of a system just backed up. With a bootstrap file, Bareos can restore your system without a Catalog. You can create a bootstrap file from a Catalog to extract any file or files you wish. + +Catalog +.. index:: + single: Catalog + The Catalog is used to store summary information about the Jobs, Clients, and Files that were backed up and on what Volume or Volumes. The information saved in the Catalog permits the administrator or user to determine what jobs were run, their status as well as the important characteristics of each file that was backed up, and most importantly, it permits you to choose what files to restore. The Catalog is an online resource, but does not contain the + data for the files backed up. Most of the information stored in the catalog is also stored on the backup volumes (i.e. tapes). Of course, the tapes will also have a copy of the file data in addition to the File Attributes (see below). + + The catalog feature is one part of Bareos that distinguishes it from simple backup and archive programs such as dump and tar. + +Client +.. index:: + single: Client}` :raw-latex:`\index[general]{File Daemon|see{Client} + In Bareos’s terminology, the word Client refers to the machine being backed up, and it is synonymous with the File services or File daemon, and quite often, it is referred to it as the FD. A Client is defined in a configuration file resource. + +Console +.. index:: + single: Console + The program that interfaces to the Director allowing the user or system administrator to control Bareos. + +Daemon +.. index:: + single: Daemon + Unix terminology for a program that is always present in the background to carry out a designated task. On Windows systems, as well as some Unix systems, daemons are called Services. + +Directive +.. index:: + single: Directive + The term directive is used to refer to a statement or a record within a Resource in a configuration file that defines one specific setting. For example, the **Name** directive defines the name of the Resource. + +Director +.. index:: + single: Director + The main Bareos server daemon that schedules and directs all Bareos operations. Occasionally, the project refers to the Director as DIR. + +Differential +.. index:: + single: Differential + A backup that includes all files changed since the last Full save started. Note, other backup programs may define this differently. + +File Attributes +.. index:: + single: File Attributes + The File Attributes are all the information necessary about a file to identify it and all its properties such as size, creation date, modification date, permissions, etc. Normally, the attributes are handled entirely by Bareos so that the user never needs to be concerned about them. The attributes do not include the file’s data. + +File daemon +.. index:: + single: File Daemon + The daemon running on the client computer to be backed up. This is also referred to as the File services, and sometimes as the Client services or the FD. + + + +.. _`FileSetDef`: FileSetDef + +FileSet + A FileSet is a Resource contained in a configuration file that defines the files to be backed up. It consists of a list of included files or directories, a list of excluded files, and how the file is to be stored (compression, encryption, signatures). For more details, see the :ref:`DirectorResourceFileSet` in the Director chapter of this document. + +Incremental +.. index:: + single: Incremental + A backup that includes all files changed since the last Full, Differential, or Incremental backup started. It is normally specified on the **Level** directive within the Job resource definition, or in a Schedule resource. + + + +.. _`JobDef`: JobDef + +Job +.. index:: + single: Job + A Bareos Job is a configuration resource that defines the work that Bareos must perform to backup or restore a particular Client. It consists of the **Type** (backup, restore, verify, etc), the **Level** (full, differential, incremental, etc.), the **FileSet**, and **Storage** the files are to be backed up (Storage device, Media Pool). For more details, see the :ref:`DirectorResourceJob` in the Director chapter of this document. + +Monitor +.. index:: + single: Monitor + The program that interfaces to all the daemons allowing the user or system administrator to monitor Bareos status. + +Resource +.. index:: + single: Resource + A resource is a part of a configuration file that defines a specific unit of information that is available to Bareos. It consists of several directives (individual configuration statements). For example, the **Job** resource defines all the properties of a specific Job: name, schedule, Volume pool, backup type, backup level, ... + +Restore +.. index:: + single: Restore + A restore is a configuration resource that describes the operation of recovering a file from backup media. It is the inverse of a save, except that in most cases, a restore will normally have a small set of files to restore, while normally a Save backs up all the files on the system. Of course, after a disk crash, Bareos can be called upon to do a full Restore of all files that were on the system. + +Schedule +.. index:: + single: Schedule + A Schedule is a configuration resource that defines when the Bareos Job will be scheduled for execution. To use the Schedule, the Job resource will refer to the name of the Schedule. For more details, see the :ref:`DirectorResourceSchedule` in the Director chapter of this document. + +Service +.. index:: + single: Service + This is a program that remains permanently in memory awaiting instructions. In Unix environments, services are also known as **daemons**. + +Storage Coordinates +.. index:: + single: Storage Coordinates + The information returned from the Storage Services that uniquely locates a file on a backup medium. It consists of two parts: one part pertains to each file saved, and the other part pertains to the whole Job. Normally, this information is saved in the Catalog so that the user doesn’t need specific knowledge of the Storage Coordinates. The Storage Coordinates include the File Attributes (see above) plus the unique location of the information + on the backup Volume. + +Storage Daemon +.. index:: + single: Storage Daemon + The Storage daemon, sometimes referred to as the SD, is the code that writes the attributes and data to a storage Volume (usually a tape or disk). + +Session +.. index:: + single: Session + Normally refers to the internal conversation between the File daemon and the Storage daemon. The File daemon opens a **session** with the Storage daemon to save a FileSet or to restore it. A session has a one-to-one correspondence to a Bareos Job (see above). + +Verify +.. index:: + single: Verify + A verify is a job that compares the current file attributes to the attributes that have previously been stored in the Bareos Catalog. This feature can be used for detecting changes to critical system files similar to what a file integrity checker like Tripwire does. One of the major advantages of using Bareos to do this is that on the machine you want protected such as a server, you can run just the File daemon, and the Director, Storage daemon, and + Catalog reside on a different machine. As a consequence, if your server is ever compromised, it is unlikely that your verification database will be tampered with. + + Verify can also be used to check that the most recent Job data written to a Volume agrees with what is stored in the Catalog (i.e. it compares the file attributes), \*or it can check the Volume contents against the original files on disk. + +Retention Period +.. index:: + single: Retention Period + There are various kinds of retention periods that Bareos recognizes. The most important are the **File** Retention Period, **Job** Retention Period, and the **Volume** Retention Period. Each of these retention periods applies to the time that specific records will be kept in the Catalog database. This should not be confused with the time that the data saved to a Volume is valid. + + The File Retention Period determines the time that File records are kept in the catalog database. This period is important for two reasons: the first is that as long as File records remain in the database, you can "browse" the database with a console program and restore any individual file. Once the File records are removed or pruned from the database, the individual files of a backup job can no longer be "browsed". The second reason for carefully choosing the File Retention Period is + because the volume of the database File records use the most storage space in the database. As a consequence, you must ensure that regular "pruning" of the database file records is done to keep your database from growing too large. (See the Console **prune** command for more details on this subject). + + The Job Retention Period is the length of time that Job records will be kept in the database. Note, all the File records are tied to the Job that saved those files. The File records can be purged leaving the Job records. In this case, information will be available about the jobs that ran, but not the details of the files that were backed up. Normally, when a Job record is purged, all its File records will also be purged. + + The Volume Retention Period is the minimum of time that a Volume will be kept before it is reused. Bareos will normally never overwrite a Volume that contains the only backup copy of a file. Under ideal conditions, the Catalog would retain entries for all files backed up for all current Volumes. Once a Volume is overwritten, the files that were backed up on that Volume are automatically removed from the Catalog. However, if there is a very large pool of Volumes or a Volume is never + overwritten, the Catalog database may become enormous. To keep the Catalog to a manageable size, the backup information should be removed from the Catalog after the defined File Retention Period. Bareos provides the mechanisms for the catalog to be automatically pruned according to the retention periods defined. + +Scan +.. index:: + single: Scan + A Scan operation causes the contents of a Volume or a series of Volumes to be scanned. These Volumes with the information on which files they contain are restored to the Bareos Catalog. Once the information is restored to the Catalog, the files contained on those Volumes may be easily restored. This function is particularly useful if certain Volumes or Jobs have exceeded their retention period and have been pruned or purged from the Catalog. Scanning data + from Volumes into the Catalog is done by using the **bscan** program. See the :ref:`bscan section ` of the Bareos Utilities chapter of this manual for more details. + +Volume +.. index:: + single: Volume + A Volume is an archive unit, normally a tape or a named disk file where Bareos stores the data from one or more backup jobs. All Bareos Volumes have a software label written to the Volume by Bareos so that it identifies what Volume it is really reading. (Normally there should be no confusion with disk files, but with tapes, it is easy to mount the wrong one.) + +What Bareos is Not +------------------ + +Bareos is a backup, restore and verification program and is not a complete disaster recovery system in itself, but it can be a key part of one if you plan carefully and follow the instructions included in the :ref:`Disaster Recovery ` chapter of this manual. + +Interactions Between the Bareos Services +---------------------------------------- + +The following block diagram shows the typical interactions between the Bareos Services for a backup job. Each block represents in general a separate process (normally a daemon). In general, the Director oversees the flow of information. It also maintains the Catalog. + +|image| + +.. |image| image:: \idir flow + :width: 80.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter01/state.rst b/docs/manuals/en/new_main_reference/source/chapter01/state.rst new file mode 100644 index 00000000000..39a6ebab68e --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter01/state.rst @@ -0,0 +1,178 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _StateChapter: + +The Current State of Bareos +=========================== + +What is Implemented +------------------- + +.. index:: + pair: Implementation; What is implemented + + +- Job Control + + - Network backup/restore with centralized Director. + + - Internal scheduler for automatic :ref:`Job ` execution. + + - Scheduling of multiple Jobs at the same time. + + - You may run one Job at a time or multiple simultaneous Jobs (sometimes called multiplexing). + + - Job sequencing using priorities. + + - :ref:`Console ` interface to the Director allowing complete control. Some GUIs are also available. + +- Security + + - Verification of files previously cataloged, permitting a Tripwire like capability (system break-in detection). + + - CRAM-MD5 password authentication between each component (daemon). + + - Configurable :ref:`TLS (SSL) communications encryption ` between each component. + + - Configurable :ref:`Data (on Volume) encryption ` on a Client by Client basis. + + - Computation of MD5 or SHA1 signatures of the file data if requested. + +- Restore Features + + - Restore of one or more files selected interactively either for the current backup or a backup prior to a specified time and date. + + - Listing and Restoration of files using stand-alone :program:`bls` and :program:`bextract` tool programs. Among other things, this permits extraction of files when Bareos and/or the catalog are not available. Note, the recommended way to restore files is using the restore command in the Console. These programs are designed for use as a last resort. + + - Ability to restore the catalog database rapidly by using bootstrap files (previously saved). + + - Ability to recreate the catalog database by scanning backup Volumes using the :program:`bscan` program. + +- SQL Catalog + + - Catalog database facility for remembering Volumes, Pools, Jobs, and Files backed up. + + - Support for PostgreSQL, MySQL and SQLite Catalog databases. + + - User extensible queries to the PostgreSQL, MySQL and SQLite databases. + +- Advanced Volume and Pool Management + + - Labeled Volumes, preventing accidental overwriting (at least by Bareos). + + - Any number of Jobs and Clients can be backed up to a single Volume. That is, you can backup and restore Linux, Unix and Windows machines to the same Volume. + + - Multi-volume saves. When a Volume is full, **Bareos** automatically requests the next Volume and continues the backup. + + - :ref:`Pool and Volume ` library management providing Volume flexibility (e.g. monthly, weekly, daily Volume sets, Volume sets segregated by Client, ...). + + - Machine independent Volume data format. Linux, Solaris, and Windows clients can all be backed up to the same Volume if desired. + + - The Volume data format is upwards compatible so that old Volumes can always be read. + + - A flexible :ref:`message ` handler including routing of messages from any daemon back to the Director and automatic email reporting. + + - Data spooling to disk during backup with subsequent write to tape from the spooled disk files. This prevents tape "shoe shine" during Incremental/Differential backups. + +- Advanced Support for most Storage Devices + + - Autochanger support using a simple shell interface that can interface to virtually any autoloader program. A script for :program:`mtx` is provided. + + - Support for autochanger barcodes – automatic tape labeling from barcodes. + + - Automatic support for multiple autochanger magazines either using barcodes or by reading the tapes. + + - Support for multiple drive autochangers. + + - Raw device backup/restore. Restore must be to the same device. + + - All Volume blocks contain a data checksum. + + - Migration support – move data from one Pool to another or one Volume to another. + +- Multi-Operating System Support + + - Programmed to handle arbitrarily long filenames and messages. + + - Compression on a file by file basis done by the Client program if requested before network transit. + + - Saves and restores POSIX ACLs and Extended Attributes on most OSes if enabled. + + - Access control lists for Consoles that permit restricting user access to only their data. + + - Support for save/restore of files larger than 2GB. + + - Support ANSI and IBM tape labels. + + - Support for Unicode filenames (e.g. Chinese) on Win32 machines + + - Consistent backup of open files on Win32 systems using Volume Shadow Copy (VSS). + + - Support for path/filename lengths of up to 64K on Win32 machines (unlimited on Unix/Linux machines). + +- Miscellaneous + + - Multi-threaded implementation. + +Advantages Over Other Backup Programs +------------------------------------- + +- Bareos handles multi-volume backups. + +- A full comprehensive SQL standard database of all files backed up. This permits online viewing of files saved on any particular Volume. + +- Automatic pruning of the database (removal of old records) thus simplifying database administration. + +- The modular but integrated design makes Bareos very scalable. + +- Bareos has a built-in Job scheduler. + +- The Volume format is documented and there are simple C programs to read/write it. + +- Bareos uses well defined (IANA registered) TCP/IP ports – no rpcs, no shared memory. + +- Bareos installation and configuration is relatively simple compared to other comparable products. + +- Aside from several GUI administrative interfaces, Bareos has a comprehensive shell administrative interface, which allows the administrator to use tools such as ssh to administrate any part of Bareos from anywhere. + +Current Implementation Restrictions +----------------------------------- + +.. index:: + pair: Restrictions; Current Implementation + + +- + +.. _`MultipleCatalogs`: MultipleCatalogs It is possible to configure the Bareos Director to use multiple Catalogs. However, this is neither advised, nor supported. Multiple catalogs require more management because in general you must know what catalog contains what data, e.g. currently, all Pools are defined in each catalog. + +- Bareos can generally restore any backup made from one client to any other client. However, if the architecture is significantly different (i.e. 32 bit architecture to 64 bit or Win32 to Unix), some restrictions may apply (e.g. Solaris door files do not exist on other Unix/Linux machines; there are reports that Zlib compression written with 64 bit machines does not always read correctly on a 32 bit machine). + +.. _section-DesignLimitations: + +Design Limitations or Restrictions +---------------------------------- + +.. index:: + pair: Restrictions; Design Limitations +.. index:: + pair: Design; Limitations + + +- Names (resource names, volume names, and such) defined in Bareos configuration files are limited to a fixed number of characters. Currently the limit is defined as 127 characters. Note, this does not apply to filenames, which may be arbitrarily long. + +- Command line input to some of the stand alone tools – e.g. :program:`btape`, :program:`bconsole` is restricted to several hundred characters maximum. Normally, this is not a restriction, except in the case of listing multiple Volume names for programs such as :program:`bscan`. To avoid this command line length restriction, please use a **.bsr** file to specify the Volume names. + +- Bareos configuration files for each of the components can be any length. However, the length of an individual line is limited to 500 characters after which it is truncated. If you need lines longer than 500 characters for directives such as ACLs where they permit a list of names are character strings simply specify multiple short lines repeating the directive on each line but with different list values. + +Items to Note +------------- + +.. index:: + single: Items to Note + + +- Bareos’s Differential and Incremental *normal* backups are based on time stamps. Consequently, if you move files into an existing directory or move a whole directory into the backup fileset after a Full backup, those files will probably not be backed up by an Incremental save because they will have old dates. This problem is corrected by using :ref:`Accurate mode ` backups or by explicitly updating the date/time stamp on all moved files. + +- In non Accurate mode, files deleted after a Full save will be included in a restoration. This is typical for most similar backup programs. To avoid this, use :ref:`Accurate mode ` backup. diff --git a/docs/manuals/en/new_main_reference/source/chapter02/install.rst b/docs/manuals/en/new_main_reference/source/chapter02/install.rst new file mode 100644 index 00000000000..2c6b1cfd3ce --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter02/install.rst @@ -0,0 +1,418 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +If you are like me, you want to get Bareos running immediately to get a feel for it, then later you want to go back and read about all the details. This chapter attempts to accomplish just that: get you going quickly without all the details. + +Bareos comes prepackaged for a number of Linux distributions. So the easiest way to get to a running Bareos installation, is to use a platform where prepacked Bareos packages are available. Additional information can be found in the chapter :ref:`Operating Systems `. + +If Bareos is available as a package, only 4 steps are required to get to a running Bareos system: + +#. + + + + :ref:`section-AddSoftwareRepository` + +#. + + + + :ref:`section-InstallBareosPackages` + +#. + + + + :ref:`section-CreateDatabase` + +#. + + + + :ref:`section-StartDaemons` + +This will start a very basic Bareos installation which will regularly backup a directory to disk. In order to fit it to your needs, you’ll have to adapt the configuration and might want to backup other clients. + +.. _section-AddSoftwareRepository: + +Decide about the Bareos release to use +====================================== + +- `http://download.bareos.org/bareos/release/latest/ `_ + +You’ll find Bareos binary package repositories at `http://download.bareos.org/ `_. The latest stable released version is available at `http://download.bareos.org/bareos/release/latest/ `_. + +The public key to verify the repository is also in repository directory (:file:`Release.key` for Debian based distributions, :file:`repodata/repomd.xml.key` for RPM based distributions). + +Section :ref:`section-InstallBareosPackages` describes how to add the software repository to your system. + +.. _section-ChooseDatabaseBackend: + +Decide about the Database Backend +================================= + +Bareos offers the following database backends: + +- PostgreSQL by package **bareos-database-postgresql**. This is the recommended backend. + +- MariaDB/MySQL by package **bareos-database-mysql** + +- | Sqlite by package **bareos-database-sqlite3** + | +.. warning:: + The Sqlite backend is only intended for testing, not for productive use. + +PostgreSQL +^^^^^^^^^^ + +is the default backend. + +MariaDB/MySQL +^^^^^^^^^^^^^ + +backend is also included. + +Sqlite +^^^^^^ + +backend is intended for testing purposes only. + +The Bareos database packages have there dependencies only to the database client packages, therefore the database itself must be installed manually. + +If you do not explicitly choose a database backend, your operating system installer will choose one for you. The default should be PostgreSQL, but depending on your operating system and the already installed packages, this may differ. + +.. _section-InstallBareosPackages: + +Install the Bareos Software Packages +==================================== + +The package **bareos** is only a meta package, that contains dependencies to the main components of Bareos, see :ref:`section-BareosPackages`. If you want to setup a distributed environment (like one Director, separate database server, multiple Storage daemons) you have to choose the corresponding Bareos packages to install on each hosts instead of just installing the **bareos** package. + +Install on RedHat based Linux Distributions +------------------------------------------- + +RHEL\ :math:`\ge`\ 7, CentOS\ :math:`\ge`\ 7, Fedora +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Platform; RHEL +.. index:: + pair: Platform; CentOS +.. index:: + pair: Platform; Fedora + + +Bareos 15.2.0 requires the :ref:`Jansson library ` package. On RHEL 7 it is available through the RHEL Server Optional channel. On CentOS 7 and Fedora is it included on the main repository. + + + + +.. code-block:: sh + :caption: Bareos installation on RHEL ≥ 7 / CentOS ≥ 7 / Fedora + + # + # define parameter + # + + DIST=RHEL_7 + # or + # DIST=CentOS_7 + # DIST=Fedora_26 + # DIST=Fedora_25 + + RELEASE=release/17.2/ + # or + # RELEASE=release/latest/ + # RELEASE=experimental/nightly/ + + # add the Bareos repository + URL=http://download.bareos.org/bareos/$RELEASE/$DIST + wget -O /etc/yum.repos.d/bareos.repo $URL/bareos.repo + + # install Bareos packages + yum install bareos bareos-database-postgresql + +RHEL 6, CentOS 6 +~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Platform; RHEL; 6; + :raw-latex:`\index[general]{Platform!CentOS!6}` + +Bareos 15.2.0 requires the :ref:`Jansson library ` package. This package is available on `EPEL `_ 6. Make sure, it is available on your system. + + + + +.. code-block:: sh + :caption: Bareos installation on RHEL ≥ 6 / CentOS ≥ 6 + + # + # add EPEL repository, if not already present. + # Required for the jansson package. + # + rpm -Uhv https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm + + # + # define parameter + # + + DIST=RHEL_6 + # DIST=CentOS_6 + + RELEASE=release/17.2/ + # or + # RELEASE=release/latest/ + # RELEASE=experimental/nightly/ + + # add the Bareos repository + URL=http://download.bareos.org/bareos/$RELEASE/$DIST + wget -O /etc/yum.repos.d/bareos.repo $URL/bareos.repo + + # install Bareos packages + yum install bareos bareos-database-postgresql + +RHEL 5 +~~~~~~ + + +.. index:: + triple: Platform; RHEL; 5; + + +yum in RHEL 5/CentOS 5 has slightly different behaviour as far as dependency resolving is concerned: it sometimes install a dependent package after the one that has the dependency defined. To make sure that it works, install the desired Bareos database backend package first in a separate step: + + + + +.. code-block:: sh + :caption: Bareos installation on RHEL 5 / CentOS 5 + + # + # define parameter + # + + DIST=RHEL_5 + + RELEASE=release/17.2/ + # or + # RELEASE=release/latest/ + # RELEASE=experimental/nightly/ + + # add the Bareos repository + URL=http://download.bareos.org/bareos/$RELEASE/$DIST + wget -O /etc/yum.repos.d/bareos.repo $URL/bareos.repo + + # install Bareos packages + yum install bareos-database-postgresql + yum install bareos + +Install on SUSE based Linux Distributions +----------------------------------------- + +SUSE Linux Enterprise Server (SLES), openSUSE +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Platform; SLES +.. index:: + pair: Platform; openSUSE + + + + + +.. code-block:: sh + :caption: Bareos installation on SLES / openSUSE + + # + # define parameter + # + + DIST=SLE_12_SP3 + # or + # DIST=SLE_12_SP2 + # DIST=SLE_12_SP1 + # DIST=SLE_11_SP4 + # DIST=openSUSE_Leap_42.3 + # DIST=openSUSE_Leap_42.2 + + RELEASE=release/17.2/ + # or + # RELEASE=release/latest/ + # RELEASE=experimental/nightly/ + + # add the Bareos repository + URL=http://download.bareos.org/bareos/$RELEASE/$DIST + zypper addrepo --refresh $URL/bareos.repo + + # install Bareos packages + zypper install bareos bareos-database-postgresql + +.. _section-InstallBareosPackagesDebian: + +Install on Debian based Linux Distributions +------------------------------------------- + +Debian / Ubuntu +~~~~~~~~~~~~~~~ + +.. index:: + pair: Platform; Debian +.. index:: + pair: Platform; Ubuntu + + +Bareos 15.2.0 requires the :ref:`Jansson library ` package. On Ubuntu is it available in Ubuntu Universe. In Debian, is it included in the main repository. + + + + +.. code-block:: sh + :caption: Bareos installation on Debian / Ubuntu + + # + # define parameter + # + + DIST=Debian_9.0 + # or + # DIST=Debian_8.0 + # DIST=xUbuntu_16.04 + # DIST=xUbuntu_14.04 + # DIST=xUbuntu_12.04 + + RELEASE=release/17.2/ + # or + # RELEASE=release/latest/ + # RELEASE=experimental/nightly/ + + URL=http://download.bareos.org/bareos/$RELEASE/$DIST + + # add the Bareos repository + printf "deb $URL /\n" > /etc/apt/sources.list.d/bareos.list + + # add package key + wget -q $URL/Release.key -O- | apt-key add - + + # install Bareos packages + apt-get update + apt-get install bareos bareos-database-postgresql + +If you prefer using the versions of Bareos directly integrated into the distributions, please note that there are some differences, see :ref:`section-DebianOrgLimitations`. + +Install on Univention Corporate Server +-------------------------------------- + +.. raw:: latex + + :strong:`Univention` + +Bareos offers additional functionality and integration into an Univention Corporate Server environment. Please follow the intructions in :ref:`section-UniventionCorporateServer`. + +If you are not interested in this additional functionality, the commands described in :ref:`section-InstallBareosPackagesDebian` will also work for Univention Corporate Servers. + +.. _section-CreateDatabase: + +Prepare Bareos database +======================= + +We assume that you have already your database installed and basically running. Using the PostgreSQL database backend is recommended. + +The easiest way to set up a database is using an system account that have passwordless local access to the database. Often this is the user **root}` for MySQL and the user :raw-latex:`\user{postgres** for PostgreSQL. + +For details, see chapter :ref:`CatMaintenanceChapter`. + +Debian based Linux Distributions +-------------------------------- + +Since Bareos 14.2.0 the Debian (and Ubuntu) based packages support the **dbconfig-common** mechanism to create and update the Bareos database. + +Follow the instructions during install to configure it according to your needs. + +|image| |image| + +If you decide not to use **dbconfig-common** (selecting :option:`` on the initial dialog), follow the instructions for :ref:`section-CreateDatabaseOtherDistributions`. + +The selectable database backends depend on the **bareos-database-*** packages installed. + +For details see :ref:`section-dbconfig`. + +.. _section-CreateDatabaseOtherDistributions: + +Other Platforms +--------------- + +.. _postgresql-1: + +PostgreSQL +~~~~~~~~~~ + +If your are using PostgreSQL and your PostgreSQL administration user is **postgres** (default), use following commands: + + + + +.. code-block:: sh + :caption: Setup Bareos catalog with PostgreSQL + + su postgres -c /usr/lib/bareos/scripts/create_bareos_database + su postgres -c /usr/lib/bareos/scripts/make_bareos_tables + su postgres -c /usr/lib/bareos/scripts/grant_bareos_privileges + +MySQL/MariaDB +~~~~~~~~~~~~~ + +Make sure, that **root** has direct access to the local MySQL server. Check if the command :program:`mysql` connects to the database without defining the password. This is the default on RedHat and SUSE distributions. On other systems (Debian, Ubuntu), create the file :file:`~/.my.cnf` with your authentication informations: + + + + +.. code-block:: sh + :caption: MySQL credentials file .my.cnf + + [client] + host=localhost + user=root + password=YourPasswordForAccessingMysqlAsRoot + +It is recommended, to secure the Bareos database connection with a password. See :ref:`Catalog Maintenance -- MySQL ` about how to archieve this. For testing, using a password-less MySQL connection is probable okay. Setup the Bareos database tables by following commands: + + + + +.. code-block:: sh + :caption: Setup Bareos catalog with MySQL + + /usr/lib/bareos/scripts/create_bareos_database + /usr/lib/bareos/scripts/make_bareos_tables + /usr/lib/bareos/scripts/grant_bareos_privileges + +As some Bareos updates require a database schema update, therefore the file :file:`/root/.my.cnf` might also be useful in the future. + +.. _section-StartDaemons: + +Start the daemons +================= + + + + +.. code-block:: sh + :caption: Start the Bareos Daemons + + service bareos-dir start + service bareos-sd start + service bareos-fd start + +You will eventually have to allow access to the ports 9101-9103, used by Bareos. + +Now you should be able to access the director using the bconsole. + +When you want to use the bareos-webui, please refer to the chapter :ref:`section-install-webui`. + +.. |image| image:: \idir dbconfig-1-enable + :width: 45.0% +.. |image| image:: \idir dbconfig-2-select-database-type + :width: 45.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter03/webui.rst b/docs/manuals/en/new_main_reference/source/chapter03/webui.rst new file mode 100644 index 00000000000..2c997f37043 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter03/webui.rst @@ -0,0 +1,532 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +This chapter addresses the installation process of the |bareosWebui|. + +Since 15.2.0 |bareosWebui| is part of the Bareos project and available for a number of platforms. + +|image| + +Features +======== + +- Intuitive web interface + +- Multilinugual + +- Can access multiple directors and catalogs + +- Individual accounts and ACL support via Bareos restricted named consoles + +- Tape Autochanger management, with the possibility to label, import/export media and update your autochanger slot status + +- Temporarly enable or disable jobs, clients and schedules and also see their current state + +- Show + + - Detailed information about Jobs, Clients, Filesets, Pools, Volumes, Storages, Schedules, Logs and Director messages + + - Filedaemon, Storage- and Director updates + + - Client, Director, Storage and Scheduler status + +- Backup Jobs + + - Start, cancel, rerun and restore from. + + - Show the file list of backup jobs + +- Restore files by browsing through a filetree of your backup jobs. + + - Merge your backup jobs history and filesets of a client or use a single backup job for restore. + + - Restore files to a different client instead of the origin + +- bconsole interface (limited to non-interactive commands) + +System Requirements +=================== + +- A platform, for which the **bareos-webui** package is available, see :ref:`section-BareosPackages`. + +- A working Bareos environment. + +- |bareosDir| version :math:`>=` |bareosWebui| version. + +- The |bareosWebui| can be installed on any host. It does not have to be installed on the same as the |bareosDir|. + +- The default installation uses an Apache webserver with mod-rewrite, mod-php and mod-setenv. + +- PHP :math:`>`\ = 5.3.23 + +- On SUSE Linux Enterprise 12 you need the additional SUSE Linux Enterprise Module for Web Scripting 12. + +Version :math:`<` 16.2 +---------------------- + +|bareosWebui| 16.2.4 incorporates the required Zend Framework 2 components, no extra Zend Framework installation is required. For older versions of **bareos-webui**, you must install Zend Framework separately. Unfortunately, not all distributions offer Zend Framework 2 packages. The following list shows where to get the Zend Framework 2 package: + +- RHEL, CentOS + + - `https://fedoraproject.org/wiki/EPEL `_ + + - `https://apps.fedoraproject.org/packages/php-ZendFramework2 `_ + +- Fedora + + - `https://apps.fedoraproject.org/packages/php-ZendFramework2 `_ + +- SUSE, Debian, Ubuntu + + - `http://download.bareos.org/bareos `_ + +Also be aware, that older versions of |bareosDir| do not support the :ref:`section-SubdirectoryConfigurationScheme` and therefore Bareos configuration resource files must be included manually. + +Installation +============ + +Adding the Bareos Repository +---------------------------- + +If not already done, add the Bareos repository that is matching your Linux distribution. Please have a look at the chapter :ref:`section-InstallBareosPackages` for more information on how to achieve this. + +Install the bareos-webui package +-------------------------------- + +After adding the repository simply install the bareos-webui package via your package manager. + +- RHEL, CentOS and Fedora + + + + +.. code-block:: sh + :caption: + + yum install bareos-webui + + or + + + + +.. code-block:: sh + :caption: + + dnf install bareos-webui + +- SUSE Linux Enterprise Server (SLES), openSUSE + + + + +.. code-block:: sh + :caption: + + zypper install bareos-webui + +- Debian, Ubuntu + + + + +.. code-block:: sh + :caption: + + apt-get install bareos-webui + +Minimal Configuration +--------------------- + +This assumes, |bareosDir| and |bareosWebui| are installed on the same host. + +#. If you are using SELinux, allow HTTPD scripts and modules make network connections: + + + + +.. code-block:: sh + :caption: + + setsebool -P httpd_can_network_connect on + + For details, see :ref:`section-webui-selinux`. + +#. Restart Apache (to load configuration provided by bareos-webui, see :ref:`section-webui-apache`) + +#. + +.. _`item-webui-create-user}` Use :program:`bconsole` to create a user with name **admin** and password **secret** and permissions defined in **webui-admin`: item-webui-create-user**:sup:`Dir`:sub:`Profile` Use :program:`bconsole` to create a user with name **admin** and password **secret** and permissions defined in :raw-latex:`\resourcename{Dir}{Profile}{webui-admin: + + + + +.. code-block:: sh + :caption: add a named console + + *configure add console name=admin password=secret profile=webui-admin + + Of course, you can choose other names and passwords. For details, see :ref:`section-webui-console`. + +#. Login to http://HOSTNAME/bareos-webui with username and password as created in ``_. + +Configuration Details +--------------------- + +.. _section-webui-console: + +Create a restricted consoles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There is not need, that |bareosWebui| itself provide a user management. Instead it uses so named :sup:`Dir` :strong:`Console` defined in the |bareosDir|. You can have multiple consoles with different names and passwords, sort of like multiple users, each with different privileges. + +At least one :sup:`Dir` :strong:`Console` is required to use the |bareosWebui|. + +To allow a user with name **admin** and password **secret** to access the |bareosDir| with permissions defined in the **webui-admin**:sup:`Dir`:sub:`Profile` (see :ref:`section-webui-profile`), either + +- create a file :file:`/etc/bareos/bareos-dir.d/console/admin.conf` with following content: + + + + +.. code-block:: sh + :caption: bareos-dir console admin + + Console { + Name = "admin" + Password = "secret" + Profile = "webui-admin" + } + + To enable this, reload or restart your |bareosDir|. + +- or use the :program:`bconsole`: + + + + +.. code-block:: sh + :caption: add console + + *configure add console name=admin password=secret profile=webui-admin + +For details, please read :ref:`DirectorResourceConsole`. + +.. _section-webui-profile: + +Configuration of profile resources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The package **bareos-webui** comes with a predefined profile for |bareosWebui|: **webui-admin**:sup:`Dir`:sub:`Profile` . + +If your |bareosWebui| is installed on another system than the |bareosDir|, you have to copy the profile to the |bareosDir|. + +This is the default profile, giving access to all Bareos resources and allowing all commands used by the |bareosWebui|: + + + + +.. code-block:: sh + :caption: bareos-dir profile webui-admin + + Profile { + Name = webui-admin + CommandACL = !.bvfs_clear_cache, !.exit, !.sql, !configure, !create, !delete, !purge, !sqlquery, !umount, !unmount, *all* + Job ACL = *all* + Schedule ACL = *all* + Catalog ACL = *all* + Pool ACL = *all* + Storage ACL = *all* + Client ACL = *all* + FileSet ACL = *all* + Where ACL = *all* + Plugin Options ACL = *all* + } + +The :sup:`Dir` :strong:`Profile` itself does not give any access to the |bareosDir|, but can be used by :sup:`Dir` :strong:`Console`, which do give access to the |bareosDir|, see :ref:`section-webui-console`. + +For details, please read :ref:`DirectorResourceProfile`. + +.. _section-webui-selinux: + +SELinux +~~~~~~~ + +.. index:: + pair: SELinux; bareos-webui + + +To use |bareosDir| on a system with SELinux enabled, permission must be given to HTTPD to make network connections: + + + + +.. code-block:: sh + :caption: + + setsebool -P httpd_can_network_connect on + +Configure your Apache Webserver +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Apache; bareos-webui + + +.. _`section-webui-apache`: section-webui-apache + +The package **bareos-webui** provides a default configuration for Apache. Depending on your distribution, it is installed at :file:`/etc/apache2/conf.d/bareos-webui.conf`, :file:`/etc/httpd/conf.d/bareos-webui.conf` or :file:`/etc/apache2/available-conf/bareos-webui.conf`. + +The required Apache modules, :strong:`setenv`, :strong:`rewrite` and :strong:`php` are enabled via package postinstall script. However, after installing the **bareos-webui** package, you need to restart your Apache webserver manually. + +Configure your /etc/bareos-webui/directors.ini +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Configuration; WebUI + + +.. _`section-webui-configuration-files`: section-webui-configuration-files + +Configure your directors in :file:`/etc/bareos-webui/directors.ini` to match your settings. + +The configuration file :file:`/etc/bareos-webui/directors.ini` should look similar to this: + + + + +.. code-block:: sh + :caption: /etc/bareos-webui/directors.ini + + ; + ; Bareos WebUI Configuration File + ; + ; File: /etc/bareos-webui/directors.ini + ; + + ;------------------------------------------------------------------------------ + ; Section localhost-dir + ;------------------------------------------------------------------------------ + [localhost-dir] + + ; Enable or disable section. Possible values are "yes" or "no", the default is "yes". + enabled = "yes" + + ; Fill in the IP-Address or FQDN of you director. + diraddress = "localhost" + + ; Default value is 9101 + dirport = 9101 + + ; Set catalog to explicit value if you have multiple catalogs + ;catalog = "MyCatalog" + + ; TLS verify peer + ; Possible values: true or false + tls_verify_peer = false + + ; Server can do TLS + ; Possible values: true or false + server_can_do_tls = false + + ; Server requires TLS + ; Possible values: true or false + server_requires_tls = false + + ; Client can do TLS + ; Possible values: true or false + client_can_do_tls = false + + ; Client requires TLS + ; Possible value: true or false + client_requires_tls = false + + ; Path to the certificate authority file + ; E.g. ca_file = "/etc/bareos-webui/tls/BareosCA.crt" + ;ca_file = "" + + ; Path to the cert file which needs to contain the client certificate and the key in PEM encoding + ; E.g. ca_file = "/etc/bareos-webui/tls/restricted-named-console.pem" + ;cert_file = "" + + ; Passphrase needed to unlock the above cert file if set + ;cert_file_passphrase = "" + + ; Allowed common names + ; E.g. allowed_cns = "host1.example.com" + ;allowed_cns = "" + + ;------------------------------------------------------------------------------ + ; Section another-host-dir + ;------------------------------------------------------------------------------ + [another-host-dir] + enabled = "no" + diraddress = "192.168.120.1" + dirport = 9101 + ;catalog = "MyCatalog" + ;tls_verify_peer = false + ;server_can_do_tls = false + ;server_requires_tls = false + ;client_can_do_tls = false + ;client_requires_tls = false + ;ca_file = "" + ;cert_file = "" + ;cert_file_passphrase = "" + ;allowed_cns = "" + +You can add as many directors as you want, also the same host with a different name and different catalog, if you have multiple catalogs. + +Configure your /etc/bareos-webui/configuration.ini +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since 16.2.2 you are able to configure some parameters of the |bareosWebui| to your needs. + + + + +.. code-block:: sh + :caption: /etc/bareos-webui/configuration.ini + + ; + ; Bareos WebUI Configuration File + ; + ; File: /etc/bareos-webui/configuration.ini + ; + + ;------------------------------------------------------------------------------ + ; SESSION SETTINGS + ;------------------------------------------------------------------------------ + ; + [session] + ; Default: 3600 seconds + timeout=3600 + + ;------------------------------------------------------------------------------ + ; DASHBOARD SETTINGS + ;------------------------------------------------------------------------------ + [dashboard] + ; Autorefresh Interval + ; Default: 60000 milliseconds + autorefresh_interval=60000 + + ;------------------------------------------------------------------------------ + ; TABLE SETTINGS + ;------------------------------------------------------------------------------ + [tables] + ; Possible values for pagination + ; Default: 10,25,50,100 + pagination_values=10,25,50,100 + + ; Default number of rows per page + ; for possible values see pagination_values + ; Default: 25 + pagination_default_value=25 + + ; State saving - restore table state on page reload. + ; Default: false + save_previous_state=false + + ;------------------------------------------------------------------------------ + ; VARIOUS SETTINGS + ;------------------------------------------------------------------------------ + [autochanger] + ; Pooltype for label to use as filter. + ; Default: none + labelpooltype=scratch + +Upgrade from 15.2 to 16.2 +========================= + +Console/Profile changes +----------------------- + +The |bareosWebui| Director profile shipped with Bareos 15.2 (**webui**:sup:`Dir`:sub:`Profile` in the file :file:`/etc/bareos/bareos-dir.d/webui-profiles.conf`) is not sufficient to use the |bareosWebui| 16.2. This has several reasons: + +#. The handling of :strong:`Acl` s is more strict in Bareos 16.2 than it has been in Bareos 15.2. Substring matching is no longer enabled, therefore you need to change :strong:`.bvfs_*` to :strong:`.bvfs_.*` in your **Command ACL**:sup:`Dir`:sub:`Profile` to have a proper regular expression. Otherwise the restore module won’t work any longer, especially the file browser. + +#. The |bareosWebui| 16.2 uses following additional commands: + + - .help + + - .schedule + + - .pools + + - import + + - export + + - update + + - release + + - enable + + - disable + +If you used an unmodified :file:`/etc/bareos/bareos-dir.d/webui-profiles.conf` file, the easiest way is to overwrite it with the new profile file :file:`/etc/bareos/bareos-dir.d/profile/webui-admin.conf`. The new **webui-admin**:sup:`Dir`:sub:`Profile` allows all commands, except of the dangerous ones, see :ref:`section-webui-profile`. + +directors.ini +------------- + +Since MyCatalog will be used. Please see :ref:`section-webui-configuration-files` for more details. + +configuration.ini +----------------- + +Since 16.2 the |bareosWebui| introduced an additional configuration file besides the directors.ini file named configuration.ini where you are able to adjust some parameters of the webui to your needs. Please see :ref:`section-webui-configuration-files` for more details. + +Additional information +====================== + +NGINX +----- + +.. index:: + pair: nginx; bareos-webui + + +If you prefer to use |bareosWebui| on Nginx with php5-fpm instead of Apache, a basic working configuration could look like this: + + + + +.. code-block:: sh + :caption: bareos-webui on nginx + + server { + + listen 9100; + server_name bareos; + root /var/www/bareos-webui/public; + + location / { + index index.php; + try_files $uri $uri/ /index.php?$query_string; + } + + location ~ .php$ { + + include snippets/fastcgi-php.conf; + + # php5-cgi alone: + # pass the PHP + # scripts to FastCGI server + # listening on 127.0.0.1:9000 + #fastcgi_pass 127.0.0.1:9000; + + # php5-fpm: + fastcgi_pass unix:/var/run/php5-fpm.sock; + + # APPLICATION_ENV: set to 'development' or 'production' + #fastcgi_param APPLICATION_ENV development; + fastcgi_param APPLICATION_ENV production; + + } + + } + +This will make the |bareosWebui| accessible at http://bareos:9100/ (assuming your DNS resolve the hostname :strong:`bareos` to the NGINX server). + +.. |image| image:: \idir bareos-webui-jobs + :width: 80.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter04/update.rst b/docs/manuals/en/new_main_reference/source/chapter04/update.rst new file mode 100644 index 00000000000..3e234ff0028 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter04/update.rst @@ -0,0 +1,121 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +In most cases, a Bareos update is simply done by a package update of the distribution. Please remind, that Bareos Director and Bareos Storage Daemon must always have the same version. The version of the File Daemon may differ, see chapter about :ref:`backward compatibility `. + +Updating the configuration files +================================ + +When updating Bareos through the distribution packaging mechanism, the existing configuration kept as they are. + +If you don’t want to modify the behavior, there is normally no need to modify the configuration. + +However, in some rare cases, configuration changes are required. These cases are described in the :ref:`Release Notes `. + +With Bareos version 16.2.4 the default configuration uses the :ref:`section-SubdirectoryConfigurationScheme`. This scheme offers various improvements. However, if your are updating from earlier versions, your existing single configuration files (:file:`/etc/bareos/bareos-*.conf`) stay in place and are contentiously used by Bareos. The new default configuration resource files will also be installed (:file:`/etc/bareos/bareos-*.d/*/*.conf`). However, +they will only be used, when the legacy configuration file does not exist. + +See :ref:`section-UpdateToConfigurationSubdirectories` for details and how to migrate to :ref:`section-SubdirectoryConfigurationScheme`. + +Updating the database scheme +============================ + +Sometimes improvements in Bareos make it necessary to update the database scheme. + +.. raw:: latex + + +.. warning:: + If the Bareos catalog database does not have the current schema, the Bareos Director refuses to start. + +Detailed information can then be found in the log file :file:`/var/log/bareos/bareos.log`. + +Take a look into the :ref:`Release Notes ` to see which Bareos updates do require a database scheme update. + +.. raw:: latex + + \warning{Especially the upgrade to Bareos ≥ 17.2.0 restructures the **File} database table. In larger installations this is very time consuming and temporarily doubles the amount of required database disk space.** + +Debian based Linux Distributions +-------------------------------- + +Since Bareos 14.2.0 the Debian (and Ubuntu) based packages support the **dbconfig-common** mechanism to create and update the Bareos database. If this is properly configured, the database schema will be automatically adapted by the Bareos packages. + +.. raw:: latex + + \warning{When using the PostgreSQL backend and updating to Bareos $<$ 14.2.3, it is necessary to manually grant database permissions, normally by using} + + + + +.. code-block:: sh + :caption: + + su - postgres -c /usr/lib/bareos/scripts/grant_bareos_privileges + +For details see :ref:`section-dbconfig`. + +If you disabled the usage of **dbconfig-common**, follow the instructions for :ref:`section-UpdateDatabaseOtherDistributions`. + +.. _section-UpdateDatabaseOtherDistributions: + +Other Platforms +--------------- + +This has to be done as database administrator. On most platforms Bareos knows only about the credentials to access the Bareos database, but not about the database administrator to modify the database schema. + +The task of updating the database schema is done by the script :program:`/usr/lib/bareos/scripts/update_bareos_tables`. + +However, this script requires administration access to the database. Depending on your distribution and your database, this requires different preparations. More details can be found in chapter :ref:`Catalog Maintenance `. + +.. raw:: latex + + \warning{If you're updating to Bareos $<=$ 13.2.3 and have configured the Bareos database during install using Bareos environment variables (\variable{db_name}, \variable{db_user} or \variable{db_password}, see :ref:`CatMaintenanceChapter`), make sure to have these variables defined in the same way when calling the update and grant scripts. Newer versions of Bareos read these variables from the Director configuration file :file:`/etc/bareos/bareos-dir.conf`. However, make sure that the user running the database scripts has read access to this file (or set the environment variables). The **postgres} user normally does not have the required permissions.** + +PostgreSQL +~~~~~~~~~~ + +If your are using PostgreSQL and your PostgreSQL administrator is **postgres** (default), use following commands: + + + + +.. code-block:: sh + :caption: Update PostgreSQL database schema + + su postgres -c /usr/lib/bareos/scripts/update_bareos_tables + su postgres -c /usr/lib/bareos/scripts/grant_bareos_privileges + +The :program:`grant_bareos_privileges` command is required, if new databases tables are introduced. It does not hurt to run it multiple times. + +After this, restart the Bareos Director and verify it starts without problems. + +MySQL/MariaDB +~~~~~~~~~~~~~ + +Make sure, that **root** has direct access to the local MySQL server. Check if the command :program:`mysql` without parameter connects to the database. If not, you may be required to adapt your local MySQL configuration file :file:`~/.my.cnf`. It should look similar to this: + + + + +.. code-block:: sh + :caption: MySQL credentials file .my.cnf + + [client] + host=localhost + user=root + password=YourPasswordForAccessingMysqlAsRoot + +If you are able to connect via the :program:`mysql` to the database, run the following script from the Unix prompt: + + + + +.. code-block:: sh + :caption: Update MySQL database schema + + /usr/lib/bareos/scripts/update_bareos_tables + +Currently on MySQL is it not necessary to run :program:`grant_bareos_privileges`, because access to the database is already given using wildcards. + +After this, restart the Bareos Director and verify it starts without problems. diff --git a/docs/manuals/en/new_main_reference/source/chapter05/quickstart.rst b/docs/manuals/en/new_main_reference/source/chapter05/quickstart.rst new file mode 100644 index 00000000000..24c01f7caf1 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter05/quickstart.rst @@ -0,0 +1,82 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +Understanding Jobs and Schedules +================================ + +.. index:: + pair: Schedule; Understanding Schedules + + +.. _`JobsandSchedules`: JobsandSchedules + +In order to make Bareos as flexible as possible, the directions given to Bareos are specified in several pieces. The main instruction is the job resource, which defines a job. A backup job generally consists of a FileSet, a Client, a Schedule for one or several levels or times of backups, a Pool, as well as additional instructions. Another way of looking at it is the FileSet is what to backup; the Client is who to backup; the Schedule defines when, and the Pool defines where (i.e. what Volume). + +Typically one FileSet/Client combination will have one corresponding job. Most of the directives, such as FileSets, Pools, Schedules, can be mixed and matched among the jobs. So you might have two different Job definitions (resources) backing up different servers using the same Schedule, the same Fileset (backing up the same directories on two machines) and maybe even the same Pools. The Schedule will define what type of backup will run when (e.g. Full on Monday, incremental the rest of the +week), and when more than one job uses the same schedule, the job priority determines which actually runs first. If you have a lot of jobs, you might want to use JobDefs, where you can set defaults for the jobs, which can then be changed in the job resource, but this saves rewriting the identical parameters for each job. In addition to the FileSets you want to back up, you should also have a job that backs up your catalog. + +Finally, be aware that in addition to the backup jobs there are restore, verify, and admin jobs, which have different requirements. + +Understanding Pools, Volumes and Labels +======================================= + +.. index:: + pair: Pools; Understanding +.. index:: + pair: Volumes; Understanding +.. index:: + pair: Label; Understanding Labels + + +.. _`PoolsVolsLabels`: PoolsVolsLabels + +If you have been using a program such as :program:`tar` to backup your system, Pools, Volumes, and labeling may be a bit confusing at first. A Volume is a single physical tape (or possibly a single file) on which Bareos will write your backup data. Pools group together Volumes so that a backup is not restricted to the length of a single Volume (tape). Consequently, rather than explicitly naming Volumes in your Job, you specify a Pool, and Bareos will select the next appendable Volume +from the Pool and mounts it. + +Although the basic Pool options are specified in the Director’s :ref:`Pool ` resource, the real Pool is maintained in the Bareos Catalog. It contains information taken from the Pool resource (configuration file) as well as information on all the Volumes that have been added to the Pool. + +For each Volume, Bareos maintains a fair amount of catalog information such as the first write date/time, the last write date/time, the number of files on the Volume, the number of bytes on the Volume, the number of Mounts, etc. + +Before Bareos will read or write a Volume, the physical Volume must have a Bareos software label so that Bareos can be sure the correct Volume is mounted. Depending on your configuration, this is either done automatically by Bareos or manually using the :strong:`label` command in the Console program. + +The steps for creating a Pool, adding Volumes to it, and writing software labels to the Volumes, may seem tedious at first, but in fact, they are quite simple to do, and they allow you to use multiple Volumes (rather than being limited to the size of a single tape). Pools also give you significant flexibility in your backup process. For example, you can have a "Daily" Pool of Volumes for Incremental backups and a "Weekly" Pool of Volumes for Full backups. By specifying the appropriate Pool in +the daily and weekly backup Jobs, you thereby insure that no daily Job ever writes to a Volume in the Weekly Pool and vice versa, and Bareos will tell you what tape is needed and when. + +For more on Pools, see the :ref:`DirectorResourcePool` section of the Director Configuration chapter, or simply read on, and we will come back to this subject later. + +.. _config: + +Setting Up Bareos Configuration Files +===================================== + +.. index:: + pair: Configuration; Files + + +On Unix, Bareos configuration files are usually located in the :file:`/etc/bareos/` directory and are named accordingly to the programs that use it. Since Bareos 16.2.4}` the default configuration is stored as one file per resource in subdirectories under :file:`bareos-dir.d`, :file:`bareos-sd.d` or :raw-latex:`\directory{bareos-fd.d. For details, see +:ref:`ConfigureChapter` and :ref:`section-SubdirectoryConfigurationScheme`. + +Testing your Configuration Files +================================ + +.. index:: + pair: Testing; Configuration Files + + +You can test if your configuration file is syntactically correct by running the appropriate daemon with the :option:`-t` option. The daemon will process the configuration file and print any error messages then terminate. + +As the |bareosDir| and |bareosSd| runs as user **bareos}`, testing the configuration should be done as :raw-latex:`\user{bareos**. + +This is especially required to test the |bareosDir|, as it also connects to the database and checks if the catalog schema version is correct. Depending on your database, only the **bareos** has permission to access it. + + + + +.. code-block:: sh + :caption: Testing Configuration Files + + su bareos -s /bin/sh -c "/usr/sbin/bareos-dir -t" + su bareos -s /bin/sh -c "/usr/sbin/bareos-sd -t" + bareos-fd -t + bconsole -t + bareos-tray-monitor -t diff --git a/docs/manuals/en/new_main_reference/source/chapter06/tutorial.rst b/docs/manuals/en/new_main_reference/source/chapter06/tutorial.rst new file mode 100644 index 00000000000..5e40b004073 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter06/tutorial.rst @@ -0,0 +1,1075 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _TutorialChapter: + +Tutorial +======== + +.. index:: + single: Tutorial + + +This chapter will guide you through running Bareos. To do so, we assume you have installed Bareos. However, we assume that you have not modified the configuration. The examples in this chapter use the default configuration files and will write the volumes to disk in your :file:`/var/lib/bareos/storage/` directory. + +The general flow of running Bareos is: + +#. Start the Database (if using |postgresql| or |mysql|) + +#. + + + + :ref:`InstallChapter` + +#. Start the Bareos Daemons + +#. Start the Console program to interact with the |bareosDir| + +#. Run a job + +#. Test recovering some files from the Volume just written to ensure the backup is good and that you know how to recover. Better test before disaster strikes + +#. Add a second client. + +Each of these steps is described in more detail below. + +Starting the Database +--------------------- + +If you are using |postgresql| or |mysql| as the Bareos database, you should start it before you install Bareos. If you are using |sqlite| you need do nothing. |sqlite| is automatically started by the |bareosDir|. + +Installing Bareos +----------------- + +For installing Bareos, follow the instructions from the :ref:`InstallChapter` chapter. + +.. _StartDaemon: + +Starting the Daemons +-------------------- + +.. index:: + single: Starting the Daemons +.. index:: + pair: Daemon; Start + + +Assuming you have installed the packages, to start the three daemons, from your installation directory, simply enter: + + + + +.. code-block:: sh + :caption: start services + + service bareos-dir start + service bareos-sd start + service bareos-fd start + +.. _section-TuturialBconsole: + +Using the Director to Query and Start Jobs +------------------------------------------ + +To communicate with the |bareosDir| and to query the state of Bareos or run jobs, the :program:`bconsole` program can be used as a textual interface. Alternatively, for most purposes, also the :ref:`\bareosWebui ` can be used, but for simplicity, here we will describe only the :program:`bconsole` program. + +The :program:`bconsole` runs the Bareos Console program, which connects to the |bareosDir|. Since Bareos is a network program, you can run the Console program anywhere on your network. Most frequently, however, one runs it on the same machine as the |bareosDir|. Normally, the Console program will print something similar to the following: + + + + +.. code-block:: sh + :caption: bconsole + + bconsole + Connecting to Director bareos:9101 + Enter a period to cancel a command. + * + +The asterisk is the console command prompt. + +Type :strong:`help` to see a list of available commands: + + + + +.. code-block:: sh + :caption: help + + *help + Command Description + ======= =========== + add Add media to a pool + autodisplay Autodisplay console messages + automount Automount after label + cancel Cancel a job + create Create DB Pool from resource + delete Delete volume, pool or job + disable Disable a job + enable Enable a job + estimate Performs FileSet estimate, listing gives full listing + exit Terminate Bconsole session + export Export volumes from normal slots to import/export slots + gui Non-interactive gui mode + help Print help on specific command + import Import volumes from import/export slots to normal slots + label Label a tape + list List objects from catalog + llist Full or long list like list command + messages Display pending messages + memory Print current memory usage + mount Mount storage + move Move slots in an autochanger + prune Prune expired records from catalog + purge Purge records from catalog + quit Terminate Bconsole session + query Query catalog + restore Restore files + relabel Relabel a tape + release Release storage + reload Reload conf file + rerun Rerun a job + run Run a job + status Report status + setbandwidth Sets bandwidth + setdebug Sets debug level + setip Sets new client address -- if authorized + show Show resource records + sqlquery Use SQL to query catalog + time Print current time + trace Turn on/off trace to file + unmount Unmount storage + umount Umount - for old-time Unix guys, see unmount + update Update volume, pool or stats + use Use specific catalog + var Does variable expansion + version Print Director version + wait Wait until no jobs are running + +Details of the console program’s commands are explained in the :ref:`section-bconsole` chapter. + +.. _Running: + +Running a Job +------------- + +.. index:: + single: Running a Job + + +At this point, we assume you have done the following: + +- Started the Database + +- Installed Bareos + +- Prepared the database for Bareos + +- Started Bareos Director, Storage Daemon and File Daemon + +- Invoked the Console program with :program:`bconsole` + +Furthermore, we assume for the moment you are using the default configuration files. + +At this point, enter the :strong:`show filesets` and you should get something similar this: + + + + +.. code-block:: sh + :caption: show filesets + + *show filesets + ... + FileSet { + Name = "SelfTest" + Include { + Options { + Signature = MD5 + } + File = "/usr/sbin" + } + } + + FileSet { + Name = "Catalog" + Include { + Options { + Signature = MD5 + } + File = "/var/lib/bareos/bareos.sql" + File = "/etc/bareos" + } + } + ... + +One of the FileSets is the pre-defined **SelfTest}` FileSet that will backup the :file:`/usr/sbin`` directory. For testing purposes, we have chosen a directory of moderate size (about 30 Megabytes) and complexity without being too big. The FileSet :raw-latex:`\fileset{Catalog** is used for backing up Bareos’s catalog and is not of interest to us for the moment. You can change what is backed up by editing the configuration and changing the ``path:File =` line in the +:sup:`Dir` :strong:`FileSet` resource. + +Now is the time to run your first backup job. We are going to backup your Bareos source directory to a File Volume in your :file:`/var/lib/bareos/storage/` directory just to show you how easy it is. Now enter: + + + + +.. code-block:: sh + :caption: status dir + + *status dir + bareos-dir Version: 13.2.0 (09 April 2013) x86_64-pc-linux-gnu debian Debian GNU/Linux 6.0 (squeeze) + Daemon started 23-May-13 13:17. Jobs: run=0, running=0 mode=0 + Heap: heap=270,336 smbytes=59,285 max_bytes=59,285 bufs=239 max_bufs=239 + + Scheduled Jobs: + Level Type Pri Scheduled Name Volume + =================================================================================== + Incremental Backup 10 23-May-13 23:05 BackupClient1 testvol + Full Backup 11 23-May-13 23:10 BackupCatalog testvol + ==== + + Running Jobs: + Console connected at 23-May-13 13:34 + No Jobs running. + ==== + +where the times and the Director’s name will be different according to your setup. This shows that an Incremental job is scheduled to run for the Job **BackupClient1**:sup:`Dir`:sub:`Job` at 1:05am and that at 1:10, a **BackupCatalog**:sup:`Dir`:sub:`Job` is scheduled to run. + +Now enter: + + + + +.. code-block:: sh + :caption: status client + + *status client + Automatically selected Client: bareos-fd + Connecting to Client bareos-fd at bareos:9102 + + bareos-fd Version: 13.2.0 (09 April 2013) x86_64-pc-linux-gnu debian Debian GNU/Linux 6.0 (squeeze) + Daemon started 23-May-13 13:17. Jobs: run=0 running=0. + Heap: heap=135,168 smbytes=26,000 max_bytes=26,147 bufs=65 max_bufs=66 + Sizeof: boffset_t=8 size_t=8 debug=0 trace=0 bwlimit=0kB/s + + Running Jobs: + Director connected at: 23-May-13 13:58 + No Jobs running. + ==== + +In this case, the client is named **bareos-fd**:sup:`Dir`:sub:`Client` your name might be different, but the line beginning with :file:`bareos-fd Version` is printed by your |bareosFd|, so we are now sure it is up and running. + +Finally do the same for your |bareosSd| with: + + + + +.. code-block:: sh + :caption: status storage + + *status storage + Automatically selected Storage: File + Connecting to Storage daemon File at bareos:9103 + + bareos-sd Version: 13.2.0 (09 April 2013) x86_64-pc-linux-gnu debian Debian GNU/Linux 6.0 (squeeze) + Daemon started 23-May-13 13:17. Jobs: run=0, running=0. + Heap: heap=241,664 smbytes=28,574 max_bytes=88,969 bufs=73 max_bufs=74 + Sizes: boffset_t=8 size_t=8 int32_t=4 int64_t=8 mode=0 bwlimit=0kB/s + + Running Jobs: + No Jobs running. + ==== + + Device status: + + Device "FileStorage" (/var/lib/bareos/storage) is not open. + == + ==== + + Used Volume status: + ==== + + ==== + +You will notice that the default |bareosSd| device is named **File**:sup:`Dir`:sub:`Storage` and that it will use device :file:`/var/lib/bareos/storage`, which is not currently open. + +Now, let’s actually run a job with: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: run + + run + +.. raw:: latex + + + +you should get the following output: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: select job + + Automatically selected Catalog: MyCatalog + Using Catalog "MyCatalog" + A job name must be specified. + The defined Job resources are: + 1: BackupClient1 + 2: BackupCatalog + 3: RestoreFiles + Select Job resource (1-3): + +.. raw:: latex + + + +Here, Bareos has listed the three different Jobs that you can run, and you should choose number **1** and type enter, at which point you will get: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: run job + + Run Backup job + JobName: BackupClient1 + Level: Incremental + Client: bareos-fd + Format: Native + FileSet: SelfTest + Pool: Full (From Job resource) + NextPool: *None* (From unknown source) + Storage: File (From Job resource) + When: 2013-05-23 14:50:04 + Priority: 10 + OK to run? (yes/mod/no): + +.. raw:: latex + + + +At this point, take some time to look carefully at what is printed and understand it. It is asking you if it is OK to run a job named **BackupClient1**:sup:`Dir`:sub:`job` with FileSet **SelfTest** as an Incremental job on your Client, and to use Storage **File**:sup:`Dir`:sub:`Storage` and Pool **Full**:sup:`Dir`:sub:`pool` , and finally, it wants to run it now (the current time should be displayed by your console). + +Here we have the choice to run (**yes**), to modify one or more of the above parameters (**mod**), or to not run the job (**no**). Please enter **yes**, at which point you should immediately get the command prompt (an asterisk). + +If you wait a few seconds, then enter the command :strong:`messages` you will get back something like: + +.. raw:: latex + + \TODO{Replace bconsole output by current version of Bareos.} + + + + +.. code-block:: sh + :caption: run + + *messages + 28-Apr-2003 14:30 bareos-sd: Wrote label to prelabeled Volume + "TestVolume001" on device /var/lib/bareos/storage + 28-Apr-2003 14:30 rufus-dir: Bareos 1.30 (28Apr03): 28-Apr-2003 14:30 + JobId: 1 + Job: BackupClient1.2003-04-28_14.22.33 + FileSet: Full Set + Backup Level: Full + Client: bareos-fd + Start time: 28-Apr-2003 14:22 + End time: 28-Apr-2003 14:30 + Files Written: 1,444 + Bytes Written: 38,988,877 + Rate: 81.2 KB/s + Software Compression: None + Volume names(s): TestVolume001 + Volume Session Id: 1 + Volume Session Time: 1051531381 + Last Volume Bytes: 39,072,359 + FD termination status: OK + SD termination status: OK + Termination: Backup OK + 28-Apr-2003 14:30 rufus-dir: Begin pruning Jobs. + 28-Apr-2003 14:30 rufus-dir: No Jobs found to prune. + 28-Apr-2003 14:30 rufus-dir: Begin pruning Files. + 28-Apr-2003 14:30 rufus-dir: No Files found to prune. + 28-Apr-2003 14:30 rufus-dir: End auto prune. + +If you don’t see the output immediately, you can keep entering :strong:`messages` until the job terminates. + +Instead of typing :strong:`messages` multiple times, you can also ask bconsole to wait, until a specific job is finished: + + + + +.. code-block:: sh + :caption: wait + + *wait jobid=1 + +or just :strong:`wait`, which waits for all running jobs to finish. + +Another useful command is :strong:`autodisplay on`. With autodisplay activated, messages will automatically be displayed as soon as they are ready. + +If you do an :program:`ls -l` of your :file:`/var/lib/bareos/storage` directory, you will see that you have the following item- + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: volume + + -rw-r----- 1 bareos bareos 39072153 Apr 28 14:30 Full-001 + +.. raw:: latex + + + +This is the file Volume that you just wrote and it contains all the data of the job just run. If you run additional jobs, they will be appended to this Volume unless you specify otherwise. + +If you would like to stop here, you can simply enter :strong:`quit` in the Console program. + +If you would like to try restoring the files that you just backed up, read the following section. + +.. _`restoring`: restoring + +Restoring Your Files +-------------------- + +.. index:: + single: Restoring Your Files + + +If you have run the default configuration and run the job as demonstrated above, you can restore the backed up files in the Console program by entering: + + + + +.. code-block:: sh + :caption: restore + + *restore all + First you select one or more JobIds that contain files + to be restored. You will be presented several methods + of specifying the JobIds. Then you will be allowed to + select which files from those JobIds are to be restored. + + To select the JobIds, you have the following choices: + 1: List last 20 Jobs run + 2: List Jobs where a given File is saved + 3: Enter list of comma separated JobIds to select + 4: Enter SQL list command + 5: Select the most recent backup for a client + 6: Select backup for a client before a specified time + 7: Enter a list of files to restore + 8: Enter a list of files to restore before a specified time + 9: Find the JobIds of the most recent backup for a client + 10: Find the JobIds for a backup for a client before a specified time + 11: Enter a list of directories to restore for found JobIds + 12: Select full restore to a specified Job date + 13: Cancel + Select item- (1-13): + +As you can see, there are a number of options, but for the current demonstration, please enter **5** to do a restore of the last backup you did, and you will get the following output: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: select resource + + Automatically selected Client: bareos-fd + The defined FileSet resources are: + 1: Catalog + 2: Full Set + Select FileSet resource (1-2): + +.. raw:: latex + + + +As you can see, Bareos knows what client you have, and since there was only one, it selected it automatically. Select **2**, because you want to restore files from the file set. + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: restore filesystem + + +-------+-------+----------+------------+---------------------+---------------+ + | jobid | level | jobfiles | jobbytes | starttime | volumename | + +-------+-------+----------+------------+---------------------+---------------+ + | 1 | F | 166 | 19,069,526 | 2013-05-05 23:05:02 | TestVolume001 | + +-------+-------+----------+------------+---------------------+---------------+ + You have selected the following JobIds: 1 + + Building directory tree for JobId(s) 1 ... +++++++++++++++++++++++++++++++++++++++++ + 165 files inserted into the tree and marked for extraction. + + You are now entering file selection mode where you add (mark) and + remove (unmark) files to be restored. No files are initially added, unless + you used the "all" keyword on the command line. + Enter "done" to leave this mode. + + cwd is: / + $ + +.. raw:: latex + + + +where I have truncated the listing on the right side to make it more readable. + +Then Bareos produced a listing containing all the jobs that form the current backup, in this case, there is only one, and the Storage daemon was also automatically chosen. Bareos then took all the files that were in Job number 1 and entered them into a **directory tree** (a sort of in memory representation of your filesystem). At this point, you can use the :strong:`cd` and :strong:`ls` or :strong:`dir` commands to walk up and down the directory +tree and view what files will be restored. For example, if you enter :strong:`cd /usr/sbin` and then enter :strong:`dir` you will get a listing of all the files in the :file:`/usr/sbin/` directory. On your system, the path might be somewhat different. For more information on this, please refer to the :ref:`Restore Command Chapter ` of this manual for more details. + +To exit this mode, simply enter: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: done + + done + +.. raw:: latex + + + +and you will get the following output: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: job report + + Bootstrap records written to + /home/user/bareos/testbin/working/restore.bsr + The restore job will require the following Volumes: + + TestVolume001 + 1444 files selected to restore. + Run Restore job + JobName: RestoreFiles + Bootstrap: /home/user/bareos/testbin/working/restore.bsr + Where: /tmp/bareos-restores + Replace: always + FileSet: Full Set + Backup Client: rufus-fd + Restore Client: rufus-fd + Storage: File + JobId: *None* + When: 2005-04-28 14:53:54 + OK to run? (yes/mod/no): + Bootstrap records written to /var/lib/bareos/bareos-dir.restore.1.bsr + + The job will require the following + Volume(s) Storage(s) SD Device(s) + =========================================================================== + + TestVolume001 File FileStorage + + Volumes marked with "*" are online. + + + 166 files selected to be restored. + + Run Restore job + JobName: RestoreFiles + Bootstrap: /var/lib/bareos/bareos-dir.restore.1.bsr + Where: /tmp/bareos-restores + Replace: Always + FileSet: Full Set + Backup Client: bareos-fd + Restore Client: bareos-fd + Format: Native + Storage: File + When: 2013-05-23 15:56:53 + Catalog: MyCatalog + Priority: 10 + Plugin Options: *None* + OK to run? (yes/mod/no): + +.. raw:: latex + + + +If you answer **yes** your files will be restored to :file:`/tmp/bareos-restores`. If you want to restore the files to their original locations, you must use the **mod** option and explicitly set **Where:** to nothing (or to /). We recommend you go ahead and answer **yes** and after a brief moment, enter :strong:`messages`, at which point you should get a listing of all the files that were restored as well as a summary of the job that looks similar to this: + +.. raw:: latex + + + + + + +.. code-block:: sh + :caption: job report + + 23-May 15:24 bareos-dir JobId 2: Start Restore Job RestoreFiles.2013-05-23_15.24.01_10 + 23-May 15:24 bareos-dir JobId 2: Using Device "FileStorage" to read. + 23-May 15:24 bareos-sd JobId 2: Ready to read from volume "TestVolume001" on device "FileStorage" (/var/lib/bareos/storage). + 23-May 15:24 bareos-sd JobId 2: Forward spacing Volume "TestVolume001" to file:block 0:194. + 23-May 15:58 bareos-dir JobId 3: Bareos bareos-dir 13.2.0 (09Apr13): + Build OS: x86_64-pc-linux-gnu debian Debian GNU/Linux 6.0 (squeeze) + JobId: 2 + Job: RestoreFiles.2013-05-23_15.58.48_11 + Restore Client: bareos-fd + Start time: 23-May-2013 15:58:50 + End time: 23-May-2013 15:58:52 + Files Expected: 166 + Files Restored: 166 + Bytes Restored: 19,069,526 + Rate: 9534.8 KB/s + FD Errors: 0 + FD termination status: OK + SD termination status: OK + Termination: Restore OK + +.. raw:: latex + + + +After exiting the Console program, you can examine the files in :file:`/tmp/bareos-restores`, which will contain a small directory tree with all the files. Be sure to clean up at the end with: + + + + +.. code-block:: sh + :caption: remove restore directory + + rm -rf /tmp/bareos-restore + +Quitting the Console Program +---------------------------- + +.. index:: + single: Quitting the Console Program + + +Simply enter the command :strong:`quit`. + +.. _SecondClient: + +Adding a Client +--------------- + + + +.. _`section-AddAClient}` :raw-latex:`\index[general]{Client!Adding a Second}` :raw-latex:`\index[general]{Adding a Client`: section-AddAClient}` :raw-latex:`\index[general]{Client!Adding a Second}` :raw-latex:`\index[general]{Adding a Client + +If you have gotten the example shown above to work on your system, you may be ready to add a second Client (|bareosFd|). That is you have a second machine that you would like backed up. Lets assume, following settings about the machine you want to add to your backup environment: + +Hostname (FQDN) + + + \host{client2.example.com} + +IP Address + 192.168.0.2 + +OS + Linux (otherwise the paths may differ) + +For this you have to make changes on the server side (|bareosDir|) and the client side. + +Client: install package +~~~~~~~~~~~~~~~~~~~~~~~ + +See :ref:`InstallChapter` about how to add the Bareos repository. The only part you need installed on the other machine is the **bareos-filedaemon**. + +Director: configure client +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Bareos 16.2.4 offers the :ref:`configure add command ` to add resources to the |bareosDir|. + +Start the :program:`bconsole` and use the :strong:`configure add client` command. Address must be a DNS resolvable name or an IP address. + + + + +.. code-block:: sh + :caption: add a client + + *configure add client name=client2-fd address=192.168.0.2 password=secret + Created resource config file "/etc/bareos/bareos-dir.d/client/client2-fd.conf": + Client { + Name = client2-fd + Address = 192.168.0.2 + Password = secret + } + +This creates two resource configuration files: + +- + + + + :file:`/etc/bareos/bareos-dir.d/client/client2-fd.conf` + +- :file:`/etc/bareos/bareos-dir-export/client/client2-fd/bareos-fd.d/director/bareos-dir.conf` (assuming your director resource is named **bareos-dir**) + +The :file:`/etc/bareos/bareos-dir-export/client/client2-fd/bareos-fd.d/director/bareos-dir.conf` is the required resource needed on the |bareosFd|. You can copy it to the destination: + + + + +.. code-block:: sh + :caption: Copy the bareos-fd director resource to the new client + + scp /etc/bareos/bareos-dir-export/client/client2-fd/bareos-fd.d/director/bareos-dir.conf root@client2.example.com:/etc/bareos/bareos-fd.d/director/ + +Manual configuration +^^^^^^^^^^^^^^^^^^^^ + +Alternatively you can configure your resources manually. On the |bareosDir| create the file + + + + +.. code-block:: sh + :caption: bareos-dir client client2-fd + + Client { + Name = client2-fd + Address = 192.168.0.2 + Password = secret + } + +Reload or restart your |bareosDir|: + + + + +.. code-block:: sh + :caption: reload the Director configuration + + *reload + reloaded + +The corresponding |bareosFd| director resource can be created directly on the client, see below. + +Client: configure +~~~~~~~~~~~~~~~~~ + +The package **bareos-filedaemon** 16.2.4 brings several configuration files: + +- + + + + :file:`/etc/bareos/bareos-fd.d/client/myself.conf` + +- + + + + :file:`/etc/bareos/bareos-fd.d/director/bareos-dir.conf` + +- + + + + :file:`/etc/bareos/bareos-fd.d/director/bareos-mon.conf` + +- + + + + :file:`/etc/bareos/bareos-fd.d/messages/Standard.conf` + +In detail: + +:file:`client/myself.conf` + defines the name of the client. The default is :file:`-fd`. Changes are only required, if you want to use another name or en- or disable special |bareosFd| features. See :ref:`ClientResourceClient`. + +:file:`director/bareos-dir.conf` + gives the |bareosDir| **bareos-dir** full access to this |bareosFd|. During installation, the **Password**:sup:`Fd`:sub:`Director` is set to a random default. Adapt the name and/or the password to your |bareosDir|. (The name **bareos-dir** is the default |bareosDir| name since Bareos 16.2.4.) + +:file:`director/bareos-mon.conf` + gives the |bareosDir| **bareos-mon** restricted access to this |bareosFd|. During installation, the **Password**:sup:`Fd`:sub:`Director` is set to a random value. This resource is intended to be used by the local **bareos-tray-monitor**. + +:file:`messages/Standard.conf` + defines, how messages should be handled. The default sends all relevant messages to the |bareosDir|. + +If your |bareosDir| is named **bareos-dir**, the :file:`/etc/bareos/bareos-fd.d/director/bareos-dir.conf` may already be overwritten by the file you copied from the |bareosDir|. If your Director has another name, an addition resource file will exists. You can define an arbitrary number of |bareosDir|’s in your |bareosFd| configuration. However, normally you will only have one :sup:`Fd` :strong:`Director` with +full control of your |bareosFd| and optional one :sup:`Fd` :strong:`Director` for monitoring (used by the |bareosTrayMonitor|). + +Anyhow, the resource will look similar to this: + + + + +.. code-block:: sh + :caption: bareos-fd director bareos-dir + + Director { + Name = bareos-dir + Password = "[md5]5ebe2294ecd0e0f08eab7690d2a6ee69" + } + +After a restart of the |bareosFd| to reload the configuration this resource allows the access for a |bareosDir| with name **bareos-dir** and password **secret** (stored in MD5 format). + + + + +.. code-block:: sh + :caption: restart bareos-fd + + service bareos-fd restart + +.. _manual-configuration-1: + +Manual configuration +^^^^^^^^^^^^^^^^^^^^ + +If you have not created the :sup:`Fd` :strong:`Director` by :strong:`configure`, you can create it also manually. If your |bareosDir| is also named **bareos-dir**, modify or create the file :file:`/etc/bareos/bareos-fd.d/director/bareos-dir.conf`: + + + + +.. code-block:: sh + :caption: bareos-fd director bareos-dir + + Director { + Name = "bareos-dir" # Name of your Bareos Director + Password = "secret" # Password (cleartext or MD5) must be identical + # to the password of your client reosurce in the Direcotr + # (bareos-dir.d/client/client2-fd.conf) + } + +See the relation between resource names and password of the different Bareos components in :ref:`section-resource-relation`. + +If your are not using the :ref:`section-SubdirectoryConfigurationScheme`, make sure that this resource file gets included in your |bareosFd| configuration. You can verify this by + + + + +.. code-block:: sh + :caption: show how bareos-fd would read the current configuration files + + bareos-fd -xc + +After modifying the file, you have to restart the |bareosFd|: + + + + +.. code-block:: sh + :caption: restart bareos-fd + + service bareos-fd restart + +Director: test client, add a job +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following example show how to + +- Verify the network connection from |bareosDir| to the |bareosFd|. + +- Add a job resource. + +- Dry-run the job (:strong:`estimate listing`). + +- Run the job. + +- Wait for the job to finish. + +- Verify the job. + + + + +.. code-block:: sh + :caption: test the client and add a job resource + + *status client=client2-fd + ... + *configure add job name=client2-job client=client2-fd jobdefs=DefaultJob + Created resource config file "/etc/bareos/bareos-dir.d/job/client2-job.conf": + Job { + Name = client2-job + Client = client2-fd + JobDefs = DefaultJob + } + *estimate listing job=client2-job + ... + *run job=client2-job + ... + *wait jobid=... + ... + *list joblog jobid=... + ... + *list files jobid=... + ... + *list volumes + ... + +Patience When Starting Daemons or Mounting Blank Tapes +------------------------------------------------------ + +When you start the Bareos daemons, the Storage daemon attempts to open all defined storage devices and verify the currently mounted Volume (if configured). Until all the storage devices are verified, the Storage daemon will not accept connections from the Console program. If a tape was previously used, it will be rewound, and on some devices this can take several minutes. As a consequence, you may need to have a bit of patience when first contacting the Storage daemon after starting the daemons. +If you can see your tape drive, once the lights stop flashing, the drive will be ready to be used. + +The same considerations apply if you have just mounted a blank tape in a drive. It can take a minute or two before the drive properly recognizes that the tape is blank. If you attempt to :strong:`mount` the tape with the Console program during this recognition period, it is quite possible that you will hang your SCSI driver. As a consequence, you are again urged to have patience when inserting blank tapes. Let the device settle down before attempting to access it. + +.. _Pool: + +Pools +----- + +.. index:: + pair: Pool; Overview + + +Creating the Pool is automatically done when the |bareosDir| starts, so if you understand Pools, you can skip to the next section. + +When you run a backup job, one of the things that Bareos must know is what Volumes to use. Instead of specifying a Volume (tape) directly, you specify which Pool of Volumes you want Bareos to consult when it wants a Volume for writing backups. Bareos will select the first available Volume from the Pool that is appropriate for the **Storage**:sup:`Dir`:sub:`Job` you have specified for the Job being run. When a volume has filled up with data, Bareos will change its +**VolStatus** from **Append** to **Full**, and then Bareos will use the next volume and so on. If no appendable Volume exists in the Pool, the Director will attempt to recycle an old Volume. For details, please read the :ref:`RecyclingChapter` chapter. + +If there are still no appendable Volumes available, Bareos will send a message requesting the operator to create an appropriate Volume. + +Bareos keeps track of the Pool name, the volumes contained in the Pool, and a number of attributes of each of those Volumes. + +When Bareos starts, it ensures that all Pool resource definitions have been recorded in the catalog. You can verify this by entering: + + + + +.. code-block:: sh + :caption: list pools + + *list pools + +--------+--------------+---------+---------+----------+---------------+ + | PoolId | Name | NumVols | MaxVols | PoolType | LabelFormat | + +--------+--------------+---------+---------+----------+---------------+ + | 1 | Full | 1 | 100 | Backup | Full- | + | 2 | Differential | 0 | 100 | Backup | Differential- | + | 3 | Incremental | 1 | 100 | Backup | Incremental- | + | 4 | Scratch | 0 | 0 | Backup | * | + +--------+--------------+---------+---------+----------+---------------+ + +Other Useful Console Commands +----------------------------- + + +.. index:: + triple: Console; Commands; Useful; + + +help + Show the list all all available commands. + +help list + Show detail information about a specific command, in this case the command :strong:`list`. + +status dir + +.. index:: + triple: Console; Command; status dir; + Print a status of all running jobs and jobs scheduled in the next 24 hours. + +status + +.. index:: + triple: Console; Command; status; + The console program will prompt you to select a daemon type, then will request the daemon’s status. + +status jobid=nn + +.. index:: + triple: Console; Command; status jobid; + Print a status of JobId nn if it is running. The Storage daemon is contacted and requested to print a current status of the job as well. + +list pools + +.. index:: + triple: Console; Command; list pools; + List the pools defined in the Catalog (normally only Default is used). + +list volumes + +.. index:: + triple: Console; Command; list volumes; + Lists all the media defined in the Catalog. + +list jobs + +.. index:: + triple: Console; Command; list jobs; + Lists all jobs in the Catalog that have run. + +list jobid=nn + +.. index:: + triple: Console; Command; list jobid; + Lists JobId nn from the Catalog. + +list jobtotals + +.. index:: + triple: Console; Command; list jobtotals; + Lists totals for all jobs in the Catalog. + +list files jobid=nn + +.. index:: + triple: Console; Command; list files jobid; + List the files that were saved for JobId nn. + +list jobmedia + +.. index:: + triple: Console; Command; list jobmedia; + List the media information for each Job run. + +messages + +.. index:: + triple: Console; Command; messages; + Prints any messages that have been directed to the console. + +quit + +.. index:: + triple: Console; Command; quit; + Exit or quit the console program. + +Most of the commands given above, with the exception of **list**, will prompt you for the necessary arguments if you simply enter the command name. + +The full list of commands is shown in the chapter :ref:`section-ConsoleCommands`. diff --git a/docs/manuals/en/new_main_reference/source/chapter07/critical.rst b/docs/manuals/en/new_main_reference/source/chapter07/critical.rst new file mode 100644 index 00000000000..305c6990459 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter07/critical.rst @@ -0,0 +1,74 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _CriticalChapter: + +Critical Items to Implement Before Production +============================================= + +.. index:: + single: Critical Items to Implement Before Production + + +We recommend you take your time before implementing a production on a Bareos backup system since Bareos is a rather complex program, and if you make a mistake, you may suddenly find that you cannot restore your files in case of a disaster. This is especially true if you have not previously used a major backup product. + +If you follow the instructions in this chapter, you will have covered most of the major problems that can occur. It goes without saying that if you ever find that we have left out an important point, please inform us, so that we can document it to the benefit of everyone. + + + +.. _`Critical`: Critical + +Critical Items +-------------- + +.. index:: + single: Critical Items + + +The following assumes that you have installed Bareos, you more or less understand it, you have at least worked through the tutorial or have equivalent experience, and that you have set up a basic production configuration. If you haven’t done the above, please do so and then come back here. The following is a sort of checklist that points with perhaps a brief explanation of why you should do it. In most cases, you will find the details elsewhere in the manual. The order is more or less the order +you would use in setting up a production system (if you already are in production, use the checklist anyway). + +- Test your tape drive for compatibility with Bareos by using the test command of the :ref:`btape ` program. + +- Test the end of tape handling of your tape drive by using the fill command in the :ref:`btape ` program. + +- Do at least one restore of files. If you backup multiple OS types (Linux, Solaris, HP, MacOS, FreeBSD, Win32, ...), restore files from each system type. The :ref:`Restoring Files ` chapter shows you how. + +- Write a bootstrap file to a separate system for each backup job. See **Write Bootstrap**:sup:`Dir`:sub:`Job` directive and more details are available in the :ref:`BootstrapChapter` chapter. Also, the default :file:`bareos-dir.conf` comes with a Write Bootstrap directive defined. This allows you to recover the state of your system as of the last backup. + +- Backup your catalog. An example of this is found in the default bareos-dir.conf file. The backup script is installed by default and should handle any database, though you may want to make your own local modifications. See also :ref:`Backing Up Your Bareos Database ` for more information. + +- Write a bootstrap file for the catalog. An example of this is found in the default bareos-dir.conf file. This will allow you to quickly restore your catalog in the event it is wiped out – otherwise it is many excruciating hours of work. + +- Make a copy of the bareos-dir.conf, bareos-sd.conf, and bareos-fd.conf files that you are using on your server. Put it in a safe place (on another machine) as these files can be difficult to reconstruct if your server dies. + +- Bareos assumes all filenames are in UTF-8 format. This is important when saving the filenames to the catalog. For Win32 machine, Bareos will automatically convert from Unicode to UTF-8, but on Unix, Linux, \*BSD, and MacOS X machines, you must explicitly ensure that your locale is set properly. Typically this means that the **LANG** environment variable must end in **.UTF-8**. A full example is **en_US.UTF-8**. The exact syntax may vary a bit from OS to OS, and exactly how you define it will + also vary. + + On most modern Win32 machines, you can edit the conf files with **notepad** and choose output encoding UTF-8. + +Recommended Items +----------------- + +.. index:: + single: Recommended Items + + +Although these items may not be critical, they are recommended and will help you avoid problems. + +- Read the :ref:`QuickStartChapter` chapter + +- After installing and experimenting with Bareos, read and work carefully through the examples in the :ref:`TutorialChapter` chapter of this manual. + +- Learn what each of the :ref:`section-Utilities` does. + +- | Set up reasonable retention periods so that your catalog does not grow to be too big. See the following three chapters: + | :ref:`RecyclingChapter`, + | :ref:`DiskChapter`, + | :ref:`PoolsChapter`. + +If you absolutely must implement a system where you write a different tape each night and take it offsite in the morning. We recommend that you do several things: + +- Write a bootstrap file of your backed up data and a bootstrap file of your catalog backup to a external media like CDROM or USB stick, and take that with the tape. If this is not possible, try to write those files to another computer or offsite computer, or send them as email to a friend. If none of that is possible, at least print the bootstrap files and take that offsite with the tape. Having the bootstrap files will make recovery much easier. + +- It is better not to force Bareos to load a particular tape each day. Instead, let Bareos choose the tape. If you need to know what tape to mount, you can print a list of recycled and appendable tapes daily, and select any tape from that list. Bareos may propose a particular tape for use that it considers optimal, but it will accept any valid tape from the correct pool. diff --git a/docs/manuals/en/new_main_reference/source/chapter08/configure.rst b/docs/manuals/en/new_main_reference/source/chapter08/configure.rst new file mode 100644 index 00000000000..2dc46b84299 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter08/configure.rst @@ -0,0 +1,1112 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _ConfigureChapter: + +Customizing the Configuration +============================= + +.. index:: + single: Customizing the Configuration + + +Each Bareos component (Director, Client, Storage, Console) has its own configuration containing a set of resource definitions. These resources are very similar from one service to another, but may contain different directives (records) depending on the component. For example, in the Director configuration, the :ref:`DirectorResourceDirector` defines the name of the Director, a number of global Director parameters and his password. In the File daemon configuration, the +:ref:`ClientResourceDirector` specifies which Directors are permitted to use the File daemon. + +If you install all Bareos daemons (Director, Storage and File Daemon) onto one system, the Bareos package tries its best to generate a working configuration as a basis for your individual configuration. + +The details of each resource and the directives permitted therein are described in the following chapters. + +The following configuration files must be present: + +- :ref:`DirectorChapter` – to define the resources necessary for the Director. You define all the Clients and Storage daemons that you use in this configuration file. + +- :ref:`StoredConfChapter` – to define the resources to be used by each Storage daemon. Normally, you will have a single Storage daemon that controls your disk storage or tape drives. However, if you have tape drives on several machines, you will have at least one Storage daemon per machine. + +- :ref:`FiledConfChapter` – to define the resources for each client to be backed up. That is, you will have a separate Client resource file on each machine that runs a File daemon. + +- :ref:`ConsoleConfChapter` – to define the resources for the Console program (user interface to the Director). It defines which Directors are available so that you may interact with them. + +.. _section-ConfigurationPathLayout: + +Configuration Path Layout +------------------------- + +.. index:: + pair: Configuration; Directories +.. index:: + pair: Configuration; Subdirectories + + +When a Bareos component starts, it reads its configuration. In Bareos :math:`<` 16.2.2 only configuration files (which optionally can include other files) are supported. Since Bareos 16.2.2 also configuration subdirectories are supported. + +Naming +^^^^^^ + +In this section, the following naming is used: + +- :file:`CONFIGDIR` refers to the base configuration directory. Bareos Linux packages use :file:`/etc/bareos/`. + +- A component is one of the following Bareos programs: + + - bareos-dir + + - bareos-sd + + - bareos-fd + + - bareos-traymonitor + + - bconsole + + - bat (only legacy config file: bat.conf) + + - Bareos tools, like :ref:`section-VolumeUtilityCommands` and others. + +- :file:`COMPONENT` refers to one of the listed components. + +.. _section-ConfigurationFileOrConfigurationSubDirectories: + +What configuration will be used? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When starting a Bareos component, it will look for its configuration. Bareos components allow the configuration file/directory to be specified as a command line parameter :file:`-c PATH`. + +- configuration path parameter is not given (default) + + - :file:`CONFIGDIR/COMPONENT.conf` is a file + + - the configuration is read from the file :file:`CONFIGDIR/COMPONENT.conf` + + - :file:`CONFIGDIR/COMPONENT.d/` is a directory + + - the configuration is read from :file:`CONFIGDIR/COMPONENT.d/*/*.conf` (subdirectory configuration) + +- configuration path parameter is given (:file:`-c PATH`) + + - :file:`PATH` is a file + + - the configuration is read from the file specified in :file:`PATH` + + - :file:`PATH` is a directory + + - the configuration is read from :file:`PATH/COMPONENT.d/*/*.conf` (subdirectory configuration) + +As the :file:`CONFIGDIR`` differs between platforms or is overwritten by the path parameter, the documentation will often refer to the configuration without the leading path (e.g. ``path:COMPONENT.d/*/*.conf`` instead of ``path:CONFIGDIR/COMPONENT.d/*/*.conf`). + +|image| + +When subdirectory configuration is used, all files matching :file:`PATH/COMPONENT.d/*/*.conf` will be read, see :ref:`section-ConfigurationSubdirectories`. + +Relation between Bareos components and configuration +'''''''''''''''''''''''''''''''''''''''''''''''''''' + ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| Bareos component | | | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| (default path on Unix) | | | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| (default path on Unix) | | | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| since Bareos :math:`>=` 16.2.2 | | | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bareos-dir | :file:`bareos-dir.conf` | :file:`bareos-dir.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`DirectorChapter` | (:raw-latex:`:file:`/etc/bareos/bareos-dir.conf``) | (:file:`/etc/bareos/bareos-dir.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bareos-sd | :file:`bareos-sd.conf` | :file:`bareos-sd.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`StoredConfChapter` | (:file:`/etc/bareos/bareos-sd.conf`) | (:file:`/etc/bareos/bareos-sd.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bareos-fd | :file:`bareos-fd.conf` | :file:`bareos-fd.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`FiledConfChapter` | (:file:`/etc/bareos/bareos-fd.conf`) | (:file:`/etc/bareos/bareos-fd.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bconsole | :file:`bconsole.conf` | :file:`bconsole.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`ConsoleConfChapter` | (:file:`/etc/bareos/bconsole.conf`) | (:file:`/etc/bareos/bconsole.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bareos-traymonitor | :file:`tray-monitor.conf` | :file:`tray-monitor.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`section-MonitorConfig` | (:file:`/etc/bareos/tray-monitor.conf`) | (:file:`/etc/bareos/tray-monitor.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| bat | :file:`bat.conf` | (not supported) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| | (:file:`/etc/bareos/bat.conf`) | | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| :ref:`section-VolumeUtilityCommands` | :file:`bareos-sd.conf` | :file:`bareos-sd.d` | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ +| (use the bareos-sd configuration) | (:file:`/etc/bareos/bareos-sd.conf`) | (:file:`/etc/bareos/bareos-sd.d/`) | ++------------------------------------------------------+-------------------------------------------+------------------------------------------------+ + +.. _section-SubdirectoryConfigurationScheme: + +Subdirectory Configuration Scheme +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +.. _`section-ConfigurationSubdirectories}` :raw-latex:`\label{ConfigurationIncludeDirectory`: section-ConfigurationSubdirectories}` :raw-latex:`\label{ConfigurationIncludeDirectory + +If the subdirectory configuration is used, instead of a single configuration file, all files matching :file:`COMPONENT.d/*/*.conf` are read as a configuration, see :ref:`section-ConfigurationFileOrConfigurationSubDirectories`. + +Reason for the Subdirectory Configuration Scheme +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Bareos :math:`<` 16.2.2, Bareos uses one configuration file per component. + +Most larger Bareos environments split their configuration into separate files, making it easier to manage the configuration. + +Also some extra packages (bareos-webui, plugins, ...) require a configuration, which must be included into the |bareosDir| or |bareosSd| configuration. The subdirectory approach makes it easier to add or modify the configuration resources of different Bareos packages. + +The Bareos :ref:`configure ` command requires a configuration directory structure, as provided by the subdirectory approach. + +From Bareos 16.2.4 on, new installations will use configuration subdirectories by default. + +.. _section-ConfigurationResourceFileConventions: + +Resource file conventions +^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Each configuration resource has to use its own configuration file. + +- The path of a resource file is :file:`COMPONENT.d/RESOURCETYPE/RESOURCENAME.conf`. + +- The name of the configuration file is identical with the resource name: + + - e.g. + + - :file:`bareos-dir.d/director/bareos-dir.conf` + + - :file:`bareos-dir.d/pool/Full.conf` + + - Exceptions: + + - The resource file :file:`bareos-fd.d/client/myself.conf`` always has the file name ``path:myself.conf`, while the name is normally set to the hostname of the system. + +- Example resource files: + + - Additional packages can contain configuration files that are automatically included. However, most additional configuration resources require configuration. When a resource file requires configuration, it has to be included as an example file: + + - :file:`CONFIGDIR/COMPONENT.d/RESOURCE/NAME.conf.example` + + - For example, the |bareosWebui| entails one config resource and one config resource example for the |bareosDir|: + + - :file:`CONFIGDIR/bareos-director.d/profile/webui-admin.conf` + + - :file:`CONFIGDIR/bareos-director.d/console/admin.conf.example` + +- extbfsection-deleteConfigurationResourceFilesDisable/remove configuration resource files: + + - Normally you should not remove resources that are already in use (jobs, clients, ...). Instead you should disable them by adding the directive :strong:`Enable = no`. Otherwise you take the risk that orphaned entries are kept in the Bareos catalog. However, if a resource has not been used or all references have been cleared from the database, they can also be removed from the configuration. + +.. warning:: + If you want to remove a configuration resource that is part of a Bareos package, + replace the resource configuration file by an empty file. + This prevents the resource from reappearing in the course of a package update. + +Using Subdirectories Configuration Scheme +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +New installation +'''''''''''''''' + +- The Subdirectories Configuration Scheme is used by default from Bareos 16.2.4 onwards. + +- They will be usable immediately after installing a Bareos component. + +- If additional packages entail example configuration files (:file:`NAME.conf.example``), copy them to ``path:NAME.conf`, modify it as required and reload or restart the component. + +.. _section-UpdateToConfigurationSubdirectories: + +Updates from Bareos :math:`<` 16.2.4 +'''''''''''''''''''''''''''''''''''' + +- When updating to a Bareos version containing the Subdirectories Configuration, the existing configuration will not be touched and is still the default configuration. + + - + + + + +.. warning:: + Problems can occur if you have implemented an own wildcard mechanism to load your configuration + from the same subdirectories as used by the new packages (\verb|path:CONFIGDIR/COMPONENT.d/*/*.conf|). + In this case, newly installed configuration resource files can alter + your current configuration by adding resources. + + Best create a copy of your configuration directory before updating Bareos and modify your existing configuration file to use that other directory. + +- As long as the old configuration file (:file:`CONFIGDIR/COMPONENT.conf`) exists, it will be used. + +- The correct way of migrating to the new configuration scheme would be to split the configuration file into resources, store them in the resource directories and then remove the original configuration file. + + - For migrating the |bareosDir| configuration, the script `bareos-migrate-config.sh `_ exists. Being called, it connects via :program:`bconsole` to a running |bareosDir| and creates subdirectories with the resource configuration files. + + + + +.. code-block:: sh + :caption: \bareosMigrateConfigSh + + # prepare temporary directory + mkdir /tmp/baroes-dir.d + cd /tmp/baroes-dir.d + + # download migration script + wget https://raw.githubusercontent.com/bareos/bareos-contrib/master/misc/bareos-migrate-config/bareos-migrate-config.sh + + # execute the script + bash bareos-migrate-config.sh + + # backup old configuration + mv /etc/bareos/bareos-dir.conf /etc/bareos/bareos-dir.conf.bak + mv /etc/bareos/bareos-dir.d /etc/bareos/bareos-dir.d.bak + + # make sure, that all packaged configuration resources exists, + # otherwise they will be added when updating Bareos. + for i in `find /etc/bareos/bareos-dir.d.bak/ -name *.conf -type f -printf "%P\n"`; do touch "$i"; done + + # install newly generated configuration + cp -a /tmp/bareos-dir.d /etc/bareos/ + + + Restart the |bareosDir| and verify your configuration. Also make sure, that all resource configuration files coming from Bareos packages exists, in doubt as empty files, see `remove configuration resource files <#section-deleteConfigurationResourceFiles>`__. + + - Another way, without splitting the configuration into resource files is: + + - + + +.. code-block:: sh + :caption: move configuration to subdirectory + + mkdir CONFIGDIR/COMPONENT.d/migrate && mv CONFIGDIR/COMPONENT.conf CONFIGDIR/COMPONENT.d/migrate + + + - Resources defined in both, the new configuration directory scheme and the old configuration file, must be removed from one of the places, best from the old configuration file, after verifying that the settings are identical with the new settings. + +Configuration File Format +------------------------- + +A configuration file consists of one or more resources (see :ref:`section-ConfigurationResourceFormat`). + +Bareos programs can work with + +- all resources defined in one configuration file + +- configuration files that include other configuration files (see :ref:`Includes`) + +- :ref:`section-ConfigurationSubdirectories`, where each configuration file contains exactly one resource definition + +Character Sets +~~~~~~~~~~~~~~ + +.. index:: + single: Character Sets + Bareos is designed to handle most character sets of the world, US ASCII, German, French, Chinese, ... However, it does this by encoding everything in UTF-8, and it expects all configuration files (including those read on Win32 machines) to be in UTF-8 format. UTF-8 is typically the default on Linux machines, but not on all Unix machines, nor on Windows, so you must take some care to ensure that your locale is set properly before starting Bareos. + + +.. index:: + triple: Windows; Configuration Files; UTF-8; + To ensure that Bareos configuration files can be correctly read including foreign characters, the **LANG** environment variable must end in **.UTF-8**. A full example is **en_US.UTF-8**. The exact syntax may vary a bit from OS to OS, so that the way you have to define it will differ from the example. On most newer Win32 machines you can use :program:`notepad` to edit the conf files, then choose output encoding UTF-8. + +Bareos assumes that all filenames are in UTF-8 format on Linux and Unix machines. On Win32 they are in Unicode (UTF-16) and will hence be automatically converted to UTF-8 format. + +.. _Comments: + +Comments +~~~~~~~~ + +.. index:: + pair: Configuration; Comments + + +When reading a configuration, blank lines are ignored and everything after a hash sign (#) until the end of the line is taken to be a comment. + +Semicolons +~~~~~~~~~~ + +A semicolon (;) is a logical end of line and anything after the semicolon is considered as the next statement. If a statement appears on a line by itself, a semicolon is not necessary to terminate it, so generally in the examples in this manual, you will not see many semicolons. + +.. _Includes: + +Including other Configuration Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Including other Configuration Files +.. index:: + pair: Files; Including other Configuration +.. index:: + pair: Configuration; Including Files + + +If you wish to break your configuration file into smaller pieces, you can do so by including other files using the syntax :strong:`@filename` where :file:`filename` is the full path and filename of another file. The :strong:`@filename` specification can be given anywhere a primitive token would appear. + + + + +.. code-block:: sh + :caption: include a configuration file + + @/etc/bareos/extra/clients.conf + +Since Bareos 16.2.1 wildcards in pathes are supported: + + + + +.. code-block:: sh + :caption: include multiple configuration files + + @/etc/bareos/extra/*.conf + +By using :strong:`@|command` it is also possible to include the output of a script as a configuration: + + + + +.. code-block:: sh + :caption: use the output of a script as configuration + + @|"/etc/bareos/generate_configuration_to_stdout.sh" + +or if a parameter should be used: + + + + +.. code-block:: sh + :caption: use the output of a script with parameter as a configuration + + @|"sh -c '/etc/bareos/generate_client_configuration_to_stdout.sh clientname=client1.example.com'" + +The scripts are called at the start of the daemon. You should use this with care. + +.. _section-ConfigurationResourceFormat: + +Resource +-------- + +.. index:: + pair: Configuration; Resource + + +A resource is defined as the resource type (see :ref:`ResTypes`), followed by an open brace (:file:`{``), a number of :ref:`section-ConfigurationResourceDirective`s, and ended by a closing brace (``path:}`) + +Each resource definition MUST contain a :strong:`Name` directive. It can contain a :strong:`Description` directive. The :strong:`Name` directive is used to uniquely identify the resource. The :strong:`Description` directive can be used during the display of the Resource to provide easier human recognition. For example: + + + + +.. code-block:: sh + :caption: Resource example + + Director { + Name = "bareos-dir" + Description = "Main Bareos Director" + Query File = "/usr/lib/bareos/scripts/query.sql" + } + +defines the Director resource with the name :option:`bareos-dir` and a query file :file:`/usr/lib/bareos/scripts/query.sql`. + +.. index:: + pair: Configuration; Naming Convention + + +When naming resources, for some resource types naming conventions should be applied: + +Client + names should be postfixed with **-fd** + +Storage + names should be postfixed with **-sd** + +Director + names should be postfixed with **-dir** + +These conventions helps a lot when reading log messages. + +.. _section-ConfigurationResourceDirective: + +Resource Directive +~~~~~~~~~~~~~~~~~~ + +Each directive contained within the resource (within the curly braces :file:`{}`) is composed of a :ref:`section-ConfigurationResourceDirectiveKeyword` followed by an equal sign (=) followed by a :ref:`section-ConfigurationResourceDirectiveValue`. The keywords must be one of the known Bareos resource record keywords. + +.. _section-ConfigurationResourceDirectiveKeyword: + +Resource Directive Keyword +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A resource directive keyword is the part before the equal sign (:file:`=`) in a :ref:`section-ConfigurationResourceDirective`. The following sections will list all available directives by Bareos component resources. + +Upper and Lower Case and Spaces +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Case (upper/lower) and spaces are ignored in the resource directive keywords. + +Within the keyword (i.e. before the equal sign), spaces are not significant. Thus the keywords: **name**, **Name**, and **N a m e** are all identical. + +.. _section-ConfigurationResourceDirectiveValue: + +Resource Directive Value +~~~~~~~~~~~~~~~~~~~~~~~~ + +A resource directive value is the part after the equal sign (:file:`=`) in a :ref:`section-ConfigurationResourceDirective`. + +Spaces +^^^^^^ + +Spaces after the equal sign and before the first character of the value are ignored. Other spaces within a value may be significant (not ignored) and may require quoting. + +.. _section-Quotes: + +Quotes +^^^^^^ + +In general, if you want spaces in a name to the right of the first equal sign (=), you must enclose that name within double quotes. Otherwise quotes are not generally necessary because once defined, quoted strings and unquoted strings are all equal. + +Within a quoted string, any character following a backslash (\) is taken as itself (handy for inserting backslashes and double quotes (")). + +:raw-latex:`\warning{If the configure directive is used to define a number, the number is never to be put between surrounding quotes. +This is even true if the number is specified together with its unit, like :option:`365 days`}`. + +Numbers +^^^^^^^ + +Numbers are not to be quoted, see :ref:`section-Quotes`. Also do not prepend numbers by zeros (0), as these are not parsed in the expected manner (write 1 instead of 01). + +Data Types +^^^^^^^^^^ + +.. index:: + single: Data Type + + +.. _`DataTypes`: DataTypes + +When parsing the resource directives, Bareos classifies the data according to the types listed below. + +acl +.. index:: + pair: Data Type; acl + + +.. _`DataTypeAcl}` This directive defines what is permitted to be accessed. It does this by using a list of regular expressions, separated by commas (:strong:`,`) or using multiple directives. If :strong:`!` is prepended, the expression is negated. The special keyword :option:`*all*`: DataTypeAcl` This directive defines what is permitted to be accessed. It does this by using a list of regular expressions, separated by commas (:strong:`,`) or using multiple directives. If :strong:`!` is prepended, the expression is negated. The special keyword :raw-latex:`\parameter{*all* allows unrestricted access. + + Depending on the type of the ACL, the regular expressions can be either resource names, paths or console commands. + + Since Bareos 16.2.4 regular expression are handled more strictly. Before also substring matches has been accepted. + + + +.. _`section-CommandAclExample`: section-CommandAclExample For clarification, we demonstrate the usage of ACLs by some examples for **Command ACL**:sup:`Dir`:sub:`Console` : + + + + +.. code-block:: sh + :caption: Allow only the help command + + Command ACL = help + + + + +.. code-block:: sh + :caption: Allow the help and the list command + + Command ACL = help, list + + + + +.. code-block:: sh + :caption: Allow the help and the (not existing) iDoNotExist command + + Command ACL = help, iDoNotExist + + + + +.. code-block:: sh + :caption: Allow all commands (special keyword) + + Command ACL = *all* + + + + +.. code-block:: sh + :caption: Allow all commands except sqlquery and commands starting with u + + Command ACL = !sqlquery, !u.*, *all* + + Same: + + + + +.. code-block:: sh + :caption: Some as above. Specifying it in multiple lines doesn't change the meaning + + Command ACL = !sqlquery, !u.* + Command ACL = *all* + + + + +.. code-block:: sh + :caption: Additional deny the setip and setdebug commands + + Command ACL = !sqlquery + Command ACL = !u.* + Comamnd ACL = !set(ip|debug) + Comamnd ACL = *all* + + + + \warning{ + ACL checking stops at the first match. So the following definition allows all commands, which might not be what you expected: + } + + + + +.. code-block:: sh + :caption: Wrong: Allows all commands + + # WARNING: this configuration ignores !sqlquery, as *all* is matched before. + Command ACL = *all*, !sqlquery + +auth-type +.. index:: + pair: Data Type; auth-type + + +.. _`DataTypeAuthType`: DataTypeAuthType Specifies the authentication type that must be supplied when connecting to a backup protocol that uses a specific authentication type. Currently only used for :ref:`NDMPResource`. + + The following values are allowed: + + None + - Use no password + + Clear + - Use clear text password + + MD5 + - Use MD5 hashing + +integer +.. index:: + pair: Data Type; integer + + +.. _`DataTypeInteger`: DataTypeInteger A 32 bit integer value. It may be positive or negative. + + Don’t use quotes around the number, see :ref:`section-Quotes`. + +long integer +.. index:: + pair: Data Type; long integer + + +.. _`DataTypeLongInteger`: DataTypeLongInteger A 64 bit integer value. Typically these are values such as bytes that can exceed 4 billion and thus require a 64 bit value. + + Don’t use quotes around the number, see :ref:`section-Quotes`. + +job protocol +.. index:: + pair: Data Type; job protocol + + +.. _`DataTypeJobProtocol`: DataTypeJobProtocol + + The protocol to run a the job. Following protocols are available: + + Native + Native Bareos job protocol. + + NDMP + Deprecated. Alias for NDMP_BAREOS. + + NDMP_BAREOS + Since Bareos 17.2.3. See :ref:`section-NdmpBareos`. + + NDMP_NATIVE + Since Bareos 17.2.3. See :ref:`section-NdmpNative`. + +name +.. index:: + pair: Data Type; name + + +.. _`DataTypeName`: DataTypeName A keyword or name consisting of alphanumeric characters, including the hyphen, underscore, and dollar characters. The first character of a **name** must be a letter. A name has a maximum length currently set to 127 bytes. + + Please note that Bareos resource names as well as certain other names (e.g. Volume names) must contain only letters (including ISO accented letters), numbers, and a few special characters (space, underscore, ...). All other characters and punctuation are invalid. + +password +.. index:: + pair: Data Type; password + + +.. _`DataTypePassword`: DataTypePassword This is a Bareos password and it is stored internally in MD5 hashed format. + +path +.. index:: + pair: Data Type; path + + +.. _`DataTypeDirectory`: DataTypeDirectory A path is either a quoted or non-quoted string. A path will be passed to your standard shell for expansion when it is scanned. Thus constructs such as **$HOME** are interpreted to be their correct values. The path can either reference to a file or a directory. + +positive integer +.. index:: + pair: Data Type; positive integer + + +.. _`DataTypePositiveInteger`: DataTypePositiveInteger A 32 bit positive integer value. + + Don’t use quotes around the number, see :ref:`section-Quotes`. + +speed +.. index:: + pair: Data Type; speed + + +.. _`DataTypeSpeed`: DataTypeSpeed The speed parameter can be specified as k/s, kb/s, m/s or mb/s. + + Don’t use quotes around the parameter, see :ref:`section-Quotes`. + +string +.. index:: + pair: Data Type; string + + +.. _`DataTypeString`: DataTypeString A quoted string containing virtually any character including spaces, or a non-quoted string. A string may be of any length. Strings are typically values that correspond to filenames, directories, or system command names. A backslash (\) turns the next character into itself, so to include a double quote in a string, you precede the double quote with a backslash. Likewise to include a backslash. + +string-list +.. index:: + pair: Data Type; string list + + +.. _`DataTypeStringList`: DataTypeStringList Multiple strings, specified in multiple directives, or in a single directive, separated by commas (**,**). + +strname +.. index:: + pair: Data Type; strname + + +.. _`DataTypeStrname`: DataTypeStrname is similar to a :raw-latex:`\dtName`, except that the name may be quoted and can thus contain additional characters including spaces. + +net-address +.. index:: + pair: Data Type; net-address + + +.. _`DataTypeNetAddress`: DataTypeNetAddress is either a domain name or an IP address specified as a dotted quadruple in string or quoted string format. This directive only permits a single address to be specified. The :raw-latex:`\dtNetPort` must be specificly separated. If multiple net-addresses are needed, please assess if it is also possible to specify :raw-latex:`\dtNetAddresses` (plural). + +net-addresses +.. index:: + pair: Data Type; net-addresses + + +.. _`DataTypeNetAddresses`: DataTypeNetAddresses Specify a set of net-addresses and net-ports. Probably the simplest way to explain this is to show an example: + + + + +.. code-block:: sh + :caption: net-addresses + + Addresses = { + ip = { addr = 1.2.3.4; port = 1205;} + ipv4 = { + addr = 1.2.3.4; port = http;} + ipv6 = { + addr = 1.2.3.4; + port = 1205; + } + ip = { + addr = 1.2.3.4 + port = 1205 + } + ip = { addr = 1.2.3.4 } + ip = { addr = 201:220:222::2 } + ip = { + addr = server.example.com + } + } + + where ip, ip4, ip6, addr, and port are all keywords. Note, that the address can be specified as either a dotted quadruple, or in IPv6 colon notation, or as a symbolic name (only in the ip specification). Also, the port can be specified as a number or as the mnemonic value from the :file:`/etc/services` file. If a port is not specified, the default one will be used. If an ip section is specified, the resolution can be made either by IPv4 or IPv6. If ip4 is specified, then only + IPv4 resolutions will be permitted, and likewise with ip6. + +net-port +.. index:: + pair: Data Type; net-port + + +.. _`DataTypeNetPort`: DataTypeNetPort Specify a network port (a positive integer). + + Don’t use quotes around the parameter, see :ref:`section-Quotes`. + +resource +.. index:: + pair: Data Type; resource + + +.. _`DataTypeRes`: DataTypeRes A resource defines a relation to the name of another resource. + +size +.. index:: + pair: Data Type; size + + +.. _`DataTypeSize}` :raw-latex:`\label{Size1`: DataTypeSize}` :raw-latex:`\label{Size1 A size specified as bytes. Typically, this is a floating point scientific input format followed by an optional modifier. The floating point input is stored as a 64 bit integer value. If a modifier is present, it must immediately follow the value with no intervening spaces. The following modifiers are permitted: + + k + 1,024 (kilobytes) + + kb + 1,000 (kilobytes) + + m + 1,048,576 (megabytes) + + mb + 1,000,000 (megabytes) + + g + 1,073,741,824 (gigabytes) + + gb + 1,000,000,000 (gigabytes) + + Don’t use quotes around the parameter, see :ref:`section-Quotes`. + +time +.. index:: + pair: Data Type; time + + +.. _`DataTypeTime}` :raw-latex:`\label{Time`: DataTypeTime}` :raw-latex:`\label{Time A time or duration specified in seconds. The time is stored internally as a 64 bit integer value, but it is specified in two parts: a number part and a modifier part. The number can be an integer or a floating point number. If it is entered in floating point notation, it will be rounded to the nearest integer. The modifier is mandatory and follows the number part, either with or without + intervening spaces. The following modifiers are permitted: + + seconds +.. index:: + single: seconds + + + minutes +.. index:: + single: minutes + (60 seconds) + + hours +.. index:: + single: hours + (3600 seconds) + + days +.. index:: + single: days + (3600*24 seconds) + + weeks +.. index:: + single: weeks + (3600*24*7 seconds) + + months +.. index:: + single: months + (3600*24*30 seconds) + + quarters +.. index:: + single: quarters + (3600*24*91 seconds) + + years +.. index:: + single: years + (3600*24*365 seconds) + + Any abbreviation of these modifiers is also permitted (i.e. **seconds** may be specified as **sec** or **s**). A specification of **m** will be taken as months. + + The specification of a time may have as many number/modifier parts as you wish. For example: + + + + + + + + 1 week 2 days 3 hours 10 mins + 1 month 2 days 30 sec + + + + + + are valid date specifications. + + Don’t use quotes around the parameter, see :ref:`section-Quotes`. + +audit-command-list +.. index:: + pair: Data Type; audit command list + + +.. _`DataTypeAuditCommandList`: DataTypeAuditCommandList Specifies the commands that should be logged on execution (audited). E.g. + + + + +.. code-block:: sh + :caption: + + Audit Events = label + Audit Events = restore + + Based on the type :raw-latex:`\dtStringList`. + +yes\|no +.. index:: + pair: Data Type; \yesno +.. index:: + pair: Data Type; boolean + + +.. _`DataTypeYesNo}` Either a :option:`yes` or a :option:`no` (or :option:`true` or :option:`false`: DataTypeYesNo` Either a :option:`yes` or a :option:`no` (or :option:`true` or :raw-latex:`\parameter{false). + +.. _VarsChapter: + +Variable Expansion +^^^^^^^^^^^^^^^^^^ + +Depending on the directive, Bareos will expand to the following variables: + +.. _section-VariableExpansionVolumeLabels: + +Variable Expansion on Volume Labels +''''''''''''''''''''''''''''''''''' + +When labeling a new volume (see **Label Format**:sup:`Dir`:sub:`Pool` ), following Bareos internal variables can be used: + ++-----------------------+------------------------------------------+ +| **Internal Variable** | **Description** | ++-----------------------+------------------------------------------+ +| **$Year** | Year | ++-----------------------+------------------------------------------+ +| **$Month** | Month: 1-12 | ++-----------------------+------------------------------------------+ +| **$Day** | Day: 1-31 | ++-----------------------+------------------------------------------+ +| **$Hour** | Hour: 0-24 | ++-----------------------+------------------------------------------+ +| **$Minute** | Minute: 0-59 | ++-----------------------+------------------------------------------+ +| **$Second** | Second: 0-59 | ++-----------------------+------------------------------------------+ +| **$WeekDay** | Day of the week: 0-6, using 0 for Sunday | ++-----------------------+------------------------------------------+ +| **$Job** | Name of the Job | ++-----------------------+------------------------------------------+ +| **$Dir** | Name of the Director | ++-----------------------+------------------------------------------+ +| **$Level** | Job Level | ++-----------------------+------------------------------------------+ +| **$Type** | Job Type | ++-----------------------+------------------------------------------+ +| **$JobId** | JobId | ++-----------------------+------------------------------------------+ +| **$JobName** | unique name of a job | ++-----------------------+------------------------------------------+ +| **$Storage** | Name of the Storage Daemon | ++-----------------------+------------------------------------------+ +| **$Client** | Name of the Clients | ++-----------------------+------------------------------------------+ +| **$NumVols** | Numbers of volumes in the pool | ++-----------------------+------------------------------------------+ +| **$Pool** | Name of the Pool | ++-----------------------+------------------------------------------+ +| **$Catalog** | Name of the Catalog | ++-----------------------+------------------------------------------+ +| **$MediaType** | Type of the media | ++-----------------------+------------------------------------------+ + +Additional, normal environment variables can be used, e.g. **$HOME** oder **$HOSTNAME**. + +With the exception of Job specific variables, you can trigger the variable expansion by using the :ref:`var command `. + +Variable Expansion in Autochanger Commands +'''''''''''''''''''''''''''''''''''''''''' + +At the configuration of autochanger commands the following variables can be used: + ++------------------------------+---------------------+ +| **Variable** | **Description** | ++------------------------------+---------------------+ +| :option:`\%a` | Archive Device Name | ++------------------------------+---------------------+ +| :option:`\%c` | Changer Device Name | ++------------------------------+---------------------+ +| :option:`\%d` | Changer Drive Index | ++------------------------------+---------------------+ +| :option:`\%f` | Client’s Name | ++------------------------------+---------------------+ +| :option:`\%j` | Job Name | ++------------------------------+---------------------+ +| :option:`\%o` | Command | ++------------------------------+---------------------+ +| :option:`\%s` | Slot Base 0 | ++------------------------------+---------------------+ +| :option:`\%S` | Slot Base 1 | ++------------------------------+---------------------+ +| :option:`\%v` | Volume Name | ++------------------------------+---------------------+ + +Variable Expansion in Mount Commands +'''''''''''''''''''''''''''''''''''' + +At the configuration of mount commands the following variables can be used: + ++------------------------------+---------------------+ +| **Variable** | **Description** | ++------------------------------+---------------------+ +| :option:`\%a` | Archive Device Name | ++------------------------------+---------------------+ +| :option:`\%e` | Erase | ++------------------------------+---------------------+ +| :option:`\%n` | Part Number | ++------------------------------+---------------------+ +| :option:`\%m` | Mount Point | ++------------------------------+---------------------+ +| :option:`\%v` | Last Part Name | ++------------------------------+---------------------+ + +Variable Expansion on RunScripts +'''''''''''''''''''''''''''''''' + +Variable Expansion on RunScripts is described at **Run Script**:sup:`Dir`:sub:`Job` . + +Variable Expansion in Mail and Operator Commands +'''''''''''''''''''''''''''''''''''''''''''''''' + +At the configuration of mail and operator commands the following variables can be used: + ++------------------------------+------------------------------------+ +| **Variable** | **Description** | ++------------------------------+------------------------------------+ +| :option:`\%c` | Client’s Name | ++------------------------------+------------------------------------+ +| :option:`\%d` | Director’s Name | ++------------------------------+------------------------------------+ +| :option:`\%e` | Job Exit Code | ++------------------------------+------------------------------------+ +| :option:`\%i` | JobId | ++------------------------------+------------------------------------+ +| :option:`\%j` | Unique Job Id | ++------------------------------+------------------------------------+ +| :option:`\%l` | Job Level | ++------------------------------+------------------------------------+ +| :option:`\%n` | Unadorned Job Name | ++------------------------------+------------------------------------+ +| :option:`\%s` | Since Time | ++------------------------------+------------------------------------+ +| :option:`\%t` | Job Type (Backup, ...) | ++------------------------------+------------------------------------+ +| :option:`\%r` | Recipients | ++------------------------------+------------------------------------+ +| :option:`\%v` | Read Volume Name | ++------------------------------+------------------------------------+ +| :option:`\%V` | Write Volume Name | ++------------------------------+------------------------------------+ +| :option:`\%b` | Job Bytes | ++------------------------------+------------------------------------+ +| :option:`\%B` | Job Bytes in human readable format | ++------------------------------+------------------------------------+ +| :option:`\%F` | Job Files | ++------------------------------+------------------------------------+ + +Resource Types +~~~~~~~~~~~~~~ + +.. index:: + single: Resource Types + + +.. _`ResTypes`: ResTypes + +The following table lists all current Bareos resource types. It shows what resources must be defined for each service (daemon). The default configuration files will already contain at least one example of each permitted resource. + +.. raw:: latex + + \addcontentsline{lot}{table}{Resource Types} + ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| :raw-latex:`\multicolumn{1}{|c|| }{\bf Resource }` | :raw-latex:`\multicolumn{1}{c| }{ \ilink{Director}{DirectorConfChapter} }` | :raw-latex:`\multicolumn{1}{c| }{ \ilink{Client}{FiledConfChapter} }` | :raw-latex:`\multicolumn{1}{c| }{ \ilink{Storage}{StoredConfChapter} }` | :raw-latex:`\multicolumn{1}{c| }{ \ilink{Console}{ConsoleConfChapter} }` | ++====================================================+============================================================================+=======================================================================+=========================================================================+===========================================================================+ +| Autochanger | | | **StorageResourceAutochanger** | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Catalog | **DirectorResourceCatalog** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Client | **DirectorResourceClient** | **ClientResourceClient** | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Console | **DirectorResourceConsole** | | | **ConsoleResourceConsole** | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Device | | | **StorageResourceDevice** | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Director | **DirectorResourceDirector** | **ClientResourceDirector** | **StorageResourceDirector** | **ConsoleResourceDirector** | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| FileSet | **DirectorResourceFileSet** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Job | **DirectorResourceJob** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| JobDefs | **DirectorResourceJobDefs** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Message | **ResourceMessages** | **ResourceMessages** | **ResourceMessages** | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| NDMP | | | **StorageResourceNDMP** | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Pool | **DirectorResourcePool** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Profile | **DirectorResourceProfile** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Schedule | **DirectorResourceSchedule** | | | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ +| Storage | **DirectorResourceStorage** | | **StorageResourceStorage** | | ++----------------------------------------------------+----------------------------------------------------------------------------+-----------------------------------------------------------------------+-------------------------------------------------------------------------+---------------------------------------------------------------------------+ + +.. _Names: + +Names, Passwords and Authorization +---------------------------------- + +.. index:: + single: Passwords + + +In order for one daemon to contact another daemon, it must authorize itself with a password. In most cases, the password corresponds to a particular name, so both the name and the password must match to be authorized. Passwords are plain text, any text. They are not generated by any special process; just use random text. + +The default configuration files are automatically defined for correct authorization with random passwords. If you add to or modify these files, you will need to take care to keep them consistent. + + + +.. _`section-resource-relation`: section-resource-relation + +.. figure:: \idir Conf-Diagram + :alt: Relation between resource names and passwords + :width: 80.0% + + Relation between resource names and passwords + +In the left column, you can see the Director, Storage, and Client resources and their corresponding names and passwords – these are all in :file:`bareos-dir.conf`. In the right column the corresponding values in the Console, Storage daemon (SD), and File daemon (FD) configuration files are shown. + +Please note that the address :strong:`fw-sd`, that appears in the Storage resource of the Director, is passed to the File daemon in symbolic form. The File daemon then resolves it to an IP address. For this reason you must use either an IP address or a resolvable fully qualified name. A name such as :strong:`localhost`, not being a fully qualified name, will resolve in the File daemon to the :strong:`localhost` of the File daemon, which is most likely not what is +desired. The password used for the File daemon to authorize with the Storage daemon is a temporary password unique to each Job created by the daemons and is not specified in any .conf file. + +.. |image| image:: \idir bareos-read-configuration + diff --git a/docs/manuals/en/new_main_reference/source/chapter09/dirdconf.rst b/docs/manuals/en/new_main_reference/source/chapter09/dirdconf.rst new file mode 100644 index 00000000000..dd52e2e1c0d --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter09/dirdconf.rst @@ -0,0 +1,539 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _DirectorChapter: + +Director Configuration +====================== + + + +.. _`DirectorConfChapter}` :raw-latex:`\index[general]{Director!Configuration}` :raw-latex:`\index[general]{Configuration!Director`: DirectorConfChapter}` :raw-latex:`\index[general]{Director!Configuration}` :raw-latex:`\index[general]{Configuration!Director + +Of all the configuration files needed to run Bareos, the Director’s is the most complicated and the one that you will need to modify the most often as you add clients or modify the FileSets. + +For a general discussion of configuration files and resources including the recognized data types see :ref:`ConfigureChapter`. + +.. index:: + single: Resource Types + + +Everything revolves around a job and is tied to a job in one way or another. + +The |bareosDir| knows about following resource types: + +- :ref:`DirectorResourceDirector` – to define the Director’s name and its access password used for authenticating the Console program. Only a single Director resource definition may appear in the Director’s configuration file. + +- :ref:`DirectorResourceJob` – to define the backup/restore Jobs and to tie together the Client, FileSet and Schedule resources to be used for each Job. Normally, you will Jobs of different names corresponding to each client (i.e. one Job per client, but a different one with a different name for each client). + +- :ref:`DirectorResourceJobDefs` – optional resource for providing defaults for Job resources. + +- :ref:`DirectorResourceSchedule` – to define when a Job has to run. You may have any number of Schedules, but each job will reference only one. + +- :ref:`DirectorResourceFileSet` – to define the set of files to be backed up for each Client. You may have any number of FileSets but each Job will reference only one. + +- :ref:`DirectorResourceClient` – to define what Client is to be backed up. You will generally have multiple Client definitions. Each Job will reference only a single client. + +- :ref:`DirectorResourceStorage` – to define on what physical device the Volumes should be mounted. You may have one or more Storage definitions. + +- :ref:`DirectorResourcePool` – to define the pool of Volumes that can be used for a particular Job. Most people use a single default Pool. However, if you have a large number of clients or volumes, you may want to have multiple Pools. Pools allow you to restrict a Job (or a Client) to use only a particular set of Volumes. + +- :ref:`DirectorResourceCatalog` – to define in what database to keep the list of files and the Volume names where they are backed up. Most people only use a single catalog. It is possible, however not adviced and not supported to use multiple catalogs, see :ref:`MultipleCatalogs`. + +- :ref:`DirectorResourceMessages` – to define where error and information messages are to be sent or logged. You may define multiple different message resources and hence direct particular classes of messages to different users or locations (files, ...). + +.. _DirectorResourceDirector: + +Director Resource +----------------- + +.. index:: + single: Director Resource +.. index:: + pair: Resource; Director + + +The Director resource defines the attributes of the Directors running on the network. Only a single Director resource is allowed. + +The following is an example of a valid Director resource definition: + + + + +.. code-block:: sh + :caption: Director Resource example + + Director { + Name = bareos-dir + Password = secretpassword + QueryFile = "/etc/bareos/query.sql" + Maximum Concurrent Jobs = 10 + Messages = Daemon + } + +.. _DirectorResourceJob: + +Job Resource +------------ + + + +.. _`JobResource}` :raw-latex:`\index[general]{Resource!Job}` :raw-latex:`\index[general]{Job!Resource`: JobResource}` :raw-latex:`\index[general]{Resource!Job}` :raw-latex:`\index[general]{Job!Resource + +The Job resource defines a Job (Backup, Restore, ...) that Bareos must perform. Each Job resource definition contains the name of a Client and a FileSet to backup, the Schedule for the Job, where the data are to be stored, and what media Pool can be used. In effect, each Job resource must specify What, Where, How, and When or FileSet, Storage, Backup/Restore/Level, and Schedule respectively. Note, the FileSet must be specified for a restore job for historical reasons, but it is no longer used. + +Only a single type (**Backup**, **Restore**, ...) can be specified for any job. If you want to backup multiple FileSets on the same Client or multiple Clients, you must define a Job for each one. + +Note, you define only a single Job to do the Full, Differential, and Incremental backups since the different backup levels are tied together by a unique Job name. Normally, you will have only one Job per Client, but if a client has a really huge number of files (more than several million), you might want to split it into to Jobs each with a different FileSet covering only part of the total files. + +Multiple Storage daemons are not currently supported for Jobs, so if you do want to use multiple storage daemons, you will need to create a different Job and ensure that for each Job that the combination of Client and FileSet are unique. The Client and FileSet are what Bareos uses to restore a client, so if there are multiple Jobs with the same Client and FileSet or multiple Storage daemons that are used, the restore will not work. This problem can be resolved by defining multiple FileSet +definitions (the names must be different, but the contents of the FileSets may be the same). + +The following is an example of a valid Job resource definition: + + + + +.. code-block:: sh + :caption: Job Resource Example + + Job { + Name = "Minou" + Type = Backup + Level = Incremental # default + Client = Minou + FileSet="Minou Full Set" + Storage = DLTDrive + Pool = Default + Schedule = "MinouWeeklyCycle" + Messages = Standard + } + +.. _DirectorResourceJobDefs: + +JobDefs Resource +---------------- + +.. index:: + pair: Job; JobDefs Resource +.. index:: + pair: Resource; JobDefs + + +The JobDefs resource permits all the same directives that can appear in a Job resource. However, a JobDefs resource does not create a Job, rather it can be referenced within a Job to provide defaults for that Job. This permits you to concisely define several nearly identical Jobs, each one referencing a JobDefs resource which contains the defaults. Only the changes from the defaults need to be mentioned in each Job. + +.. _DirectorResourceSchedule: + +Schedule Resource +----------------- + +.. index:: + pair: Resource; Schedule +.. index:: + pair: Schedule; Resource + + +The Schedule resource provides a means of automatically scheduling a Job as well as the ability to override the default Level, Pool, Storage and Messages resources. If a Schedule resource is not referenced in a Job, the Job can only be run manually. In general, you specify an action to be taken and when. + +Note, the Week of Year specification wnn follows the ISO standard definition of the week of the year, where Week 1 is the week in which the first Thursday of the year occurs, or alternatively, the week which contains the 4th of January. Weeks are numbered w01 to w53. w00 for Bareos is the week that precedes the first ISO week (i.e. has the first few days of the year if any occur before Thursday). w00 is not defined by the ISO specification. A week starts with Monday and ends with Sunday. + +According to the NIST (US National Institute of Standards and Technology), 12am and 12pm are ambiguous and can be defined to anything. However, 12:01am is the same as 00:01 and 12:01pm is the same as 12:01, so Bareos defines 12am as 00:00 (midnight) and 12pm as 12:00 (noon). You can avoid this abiguity (confusion) by using 24 hour time specifications (i.e. no am/pm). + +An example schedule resource that is named **WeeklyCycle** and runs a job with level full each Sunday at 2:05am and an incremental job Monday through Saturday at 2:05am is: + + + + +.. code-block:: sh + :caption: Schedule Example + + Schedule { + Name = "WeeklyCycle" + Run = Level=Full sun at 2:05 + Run = Level=Incremental mon-sat at 2:05 + } + +An example of a possible monthly cycle is as follows: + + + + +.. code-block:: sh + :caption: + + Schedule { + Name = "MonthlyCycle" + Run = Level=Full Pool=Monthly 1st sun at 2:05 + Run = Level=Differential 2nd-5th sun at 2:05 + Run = Level=Incremental Pool=Daily mon-sat at 2:05 + } + +The first of every month: + + + + +.. code-block:: sh + :caption: + + Schedule { + Name = "First" + Run = Level=Full on 1 at 2:05 + Run = Level=Incremental on 2-31 at 2:05 + } + +The last friday of the month (i.e. the last friday in the last week of the month) + + + + +.. code-block:: sh + :caption: + + Schedule { + Name = "Last Friday" + Run = Level=Full last fri at 21:00 + } + +Every 10 minutes: + + + + +.. code-block:: sh + :caption: + + Schedule { + Name = "TenMinutes" + Run = Level=Full hourly at 0:05 + Run = Level=Full hourly at 0:15 + Run = Level=Full hourly at 0:25 + Run = Level=Full hourly at 0:35 + Run = Level=Full hourly at 0:45 + Run = Level=Full hourly at 0:55 + } + +The **modulo scheduler** makes it easy to specify schedules like odd or even days/weeks, or more generally every n days or weeks. It is called modulo scheduler because it uses the modulo to determine if the schedule must be run or not. The second variable behind the slash lets you determine in which cycle of days/weeks a job should be run. The first part determines on which day/week the job should be run first. E.g. if you want to run a backup in a 5-week-cycle, starting on week 3, you set it up +as w03/w05. + + + + +.. code-block:: sh + :caption: Schedule Examples: modulo + + Schedule { + Name = "Odd Days" + Run = 1/2 at 23:10 + } + + Schedule { + Name = "Even Days" + Run = 2/2 at 23:10 + } + + Schedule { + Name = "On the 3rd week in a 5-week-cycle" + Run = w03/w05 at 23:10 + } + + Schedule { + Name = "Odd Weeks" + Run = w01/w02 at 23:10 + } + + Schedule { + Name = "Even Weeks" + Run = w02/w02 at 23:10 + } + +Technical Notes on Schedules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Schedule; Technical Notes on Schedules + + +Internally Bareos keeps a schedule as a bit mask. There are six masks and a minute field to each schedule. The masks are hour, day of the month (mday), month, day of the week (wday), week of the month (wom), and week of the year (woy). The schedule is initialized to have the bits of each of these masks set, which means that at the beginning of every hour, the job will run. When you specify a month for the first time, the mask will be cleared and the bit corresponding to your selected month will +be selected. If you specify a second month, the bit corresponding to it will also be added to the mask. Thus when Bareos checks the masks to see if the bits are set corresponding to the current time, your job will run only in the two months you have set. Likewise, if you set a time (hour), the hour mask will be cleared, and the hour you specify will be set in the bit mask and the minutes will be stored in the minute field. + +For any schedule you have defined, you can see how these bits are set by doing a **show schedules** command in the Console program. Please note that the bit mask is zero based, and Sunday is the first day of the week (bit zero). + +.. _DirectorResourceFileSet: + +FileSet Resource +---------------- + + + +.. _`FileSetResource}` :raw-latex:`\index[general]{Resource!FileSet}` :raw-latex:`\index[general]{FileSet!Resource`: FileSetResource}` :raw-latex:`\index[general]{Resource!FileSet}` :raw-latex:`\index[general]{FileSet!Resource + +The FileSet resource defines what files are to be included or excluded in a backup job. A **FileSet** resource is required for each backup Job. It consists of a list of files or directories to be included, a list of files or directories to be excluded and the various backup options such as compression, encryption, and signatures that are to be applied to each file. + +Any change to the list of the included files will cause Bareos to automatically create a new FileSet (defined by the name and an MD5 checksum of the Include/Exclude contents). Each time a new FileSet is created, Bareos will ensure that the next backup is always a Full save. + +.. _DirectorResourceClient: + +Client Resource +--------------- + +.. index:: + single: Client Resource + + +The Client (or FileDaemon) resource defines the attributes of the Clients that are served by this Director; that is the machines that are to be backed up. You will need one Client resource definition for each machine to be backed up. + +The following is an example of a valid Client resource definition: + + + + +.. code-block:: sh + :caption: Minimal client resource definition in bareos-dir.conf + + Client { + Name = client1-fd + Address = client1.example.com + Password = "secret" + } + +The following is an example of a Quota Configuration in Client resource: + + + + +.. code-block:: sh + :caption: Quota Configuration in Client resource + + Client { + Name = client1-fd + Address = client1.example.com + Password = "secret" + + # Quota + Soft Quota = 50 mb + Soft Quota Grace Period = 2 days + Strict Quotas = Yes + Hard Quota = 150 mb + Quota Include Failed Jobs = yes + } + +.. _DirectorResourceStorage: + +Storage Resource +---------------- + +.. index:: + single: Storage Resource + + +The Storage resource defines which Storage daemons are available for use by the Director. + +The following is an example of a valid Storage resource definition: + + + + +.. code-block:: sh + :caption: Storage resource (tape) example + + Storage { + Name = DLTDrive + Address = lpmatou + Password = storage\_password # password for Storage daemon + Device = "HP DLT 80" # same as Device in Storage daemon + Media Type = DLT8000 # same as MediaType in Storage daemon + } + +.. _DirectorResourcePool: + +Pool Resource +------------- + +.. index:: + single: Pool Resource + + +The Pool resource defines the set of storage Volumes (tapes or files) to be used by Bareos to write the data. By configuring different Pools, you can determine which set of Volumes (media) receives the backup data. This permits, for example, to store all full backup data on one set of Volumes and all incremental backups on another set of Volumes. Alternatively, you could assign a different set of Volumes to each machine that you backup. This is most easily done by defining multiple Pools. + +Another important aspect of a Pool is that it contains the default attributes (Maximum Jobs, Retention Period, Recycle flag, ...) that will be given to a Volume when it is created. This avoids the need for you to answer a large number of questions when labeling a new Volume. Each of these attributes can later be changed on a Volume by Volume basis using the :strong:`update` command in the console program. Note that you must explicitly specify which Pool Bareos is to use with each +Job. Bareos will not automatically search for the correct Pool. + +To use a Pool, there are three distinct steps. First the Pool must be defined in the Director’s configuration. Then the Pool must be written to the Catalog database. This is done automatically by the Director each time that it starts. Finally, if you change the Pool definition in the Director’s configuration file and restart Bareos, the pool will be updated alternatively you can use the :strong:`update pool` console command to refresh the database image. It is this database image +rather than the Director’s resource image that is used for the default Volume attributes. Note, for the pool to be automatically created or updated, it must be explicitly referenced by a Job resource. + +If automatic labeling is not enabled (see :ref:`AutomaticLabeling`) the physical media must be manually labeled. The labeling can either be done with the :strong:`label` command in the console program or using the :program:`btape` program. The preferred method is to use the :strong:`label` command in the console program. Generally, automatic labeling is enabled for **Device Type**:sup:`Sd`:sub:`Device` = **File** +and disabled for **Device Type**:sup:`Sd`:sub:`Device` = **Tape**. + +Finally, you must add Volume names (and their attributes) to the Pool. For Volumes to be used by Bareos they must be of the same **Media Type**:sup:`Sd`:sub:`Device` as the archive device specified for the job (i.e. if you are going to back up to a DLT device, the Pool must have DLT volumes defined since 8mm volumes cannot be mounted on a DLT drive). The **Media Type**:sup:`Sd`:sub:`Device` has particular importance if you are backing up to files. +When running a Job, you must explicitly specify which Pool to use. Bareos will then automatically select the next Volume to use from the Pool, but it will ensure that the **Media Type**:sup:`Sd`:sub:`Device` of any Volume selected from the Pool is identical to that required by the Storage resource you have specified for the Job. + +If you use the :strong:`label` command in the console program to label the Volumes, they will automatically be added to the Pool, so this last step is not normally required. + +It is also possible to add Volumes to the database without explicitly labeling the physical volume. This is done with the :strong:`add` console command. + +As previously mentioned, each time Bareos starts, it scans all the Pools associated with each Catalog, and if the database record does not already exist, it will be created from the Pool Resource definition. If you change the Pool definition, you manually have to call :strong:`update pool` command in the console program to propagate the changes to existing volumes. + +The Pool Resource defined in the Director’s configuration may contain the following directives: + +The following is an example of a valid Pool resource definition: + + + + +.. code-block:: sh + :caption: Pool resource example + + Pool { + Name = Default + Pool Type = Backup + } + +.. _TheScratchPool: + +Scratch Pool +~~~~~~~~~~~~ + +.. index:: + single: Scratch Pool +.. index:: + pair: Pool; Scratch + + +In general, you can give your Pools any name you wish, but there is one important restriction: the Pool named **Scratch**, if it exists behaves like a scratch pool of Volumes in that when Bareos needs a new Volume for writing and it cannot find one, it will look in the Scratch pool, and if it finds an available Volume, it will move it out of the Scratch pool into the Pool currently being used by the job. + +.. _DirectorResourceCatalog: + +Catalog Resource +---------------- + +.. index:: + single: Catalog Resource + + +The Catalog Resource defines what catalog to use for the current job. Currently, Bareos can only handle a single database server (SQLite, MySQL, PostgreSQL) that is defined when configuring **Bareos**. However, there may be as many Catalogs (databases) defined as you wish. For example, you may want each Client to have its own Catalog database, or you may want backup jobs to use one database and verify or restore jobs to use another database. + +Since SQLite is compiled in, it always runs on the same machine as the Director and the database must be directly accessible (mounted) from the Director. However, since both MySQL and PostgreSQL are networked databases, they may reside either on the same machine as the Director or on a different machine on the network. See below for more details. + +The following is an example of a valid Catalog resource definition: + + + + +.. code-block:: sh + :caption: Catalog Resource for Sqlite + + Catalog + { + Name = SQLite + DB Driver = sqlite + DB Name = bareos; + DB User = bareos; + DB Password = "" + } + +or for a Catalog on another machine: + + + + +.. code-block:: sh + :caption: Catalog Resource for remote MySQL + + Catalog + { + Name = MySQL + DB Driver = mysql + DB Name = bareos + DB User = bareos + DB Password = "secret" + DB Address = remote.example.com + DB Port = 1234 + } + +.. _DirectorResourceMessages: + +Messages Resource +----------------- + +.. index:: + single: Messages Resource + + +For the details of the Messages Resource, please see the :ref:`MessagesChapter` of this manual. + +.. _DirectorResourceConsole: + +Console Resource +---------------- + +.. index:: + single: Console Resource +.. index:: + pair: Resource; Console + + +There are three different kinds of consoles, which the administrator or user can use to interact with the Director. These three kinds of consoles comprise three different security levels. + +Default Console +.. index:: + pair: Console; Default Console + the first console type is an :emphasis:`anonymous` or :emphasis:`default` console, which has full privileges. There is no console resource necessary for this type since the password is specified in the Director’s resource and consequently such consoles do not have a name as defined on a :strong:`Name` directive. Typically you would use it only for administrators. + +Named Console +.. index:: + single: Named Console +.. index:: + pair: Console; Named Console +.. index:: + pair: Console; Restricted Console + the second type of console, is a :emphasis:`named` console (also called :emphasis:`Restricted Console`) defined within a Console resource in both the Director’s configuration file and in the Console’s configuration file. Both the names and the passwords in these two entries must match much as is the case for Client programs. + + This second type of console begins with absolutely no privileges except those explicitly specified in the Director’s Console resource. Thus you can have multiple Consoles with different names and passwords, sort of like multiple users, each with different privileges. As a default, these consoles can do absolutely nothing – no commands whatsoever. You give them privileges or rather access to commands and resources by specifying access control lists in the Director’s Console resource. The ACLs + are specified by a directive followed by a list of access names. Examples of this are shown below. + + - The third type of console is similar to the above mentioned one in that it requires a Console resource definition in both the Director and the Console. In addition, if the console name, provided on the **Name**:sup:`Dir`:sub:`Console` directive, is the same as a Client name, that console is permitted to use the :strong:`SetIP` command to change the Address directive in the Director’s client resource to the IP address of the Console. This permits + portables or other machines using DHCP (non-fixed IP addresses) to "notify" the Director of their current IP address. + +The Console resource is optional and need not be specified. The following directives are permitted within these resources: + +The example at :ref:`section-ConsoleAccessExample` shows how to use a console resource for a connection from a client like :program:`bconsole`. + +.. _DirectorResourceProfile: + +Profile Resource +---------------- + +.. index:: + single: Profile Resource +.. index:: + pair: Resource; Profile + + +The Profile Resource defines a set of ACLs. :ref:`DirectorResourceConsole`s can be tight to one or more profiles (**Profile**:sup:`Dir`:sub:`Console` ), making it easier to use a common set of ACLs. + +.. _DirectorResourceCounter: + +Counter Resource +---------------- + +.. index:: + single: Counter Resource + + +The Counter Resource defines a counter variable that can be accessed by variable expansion used for creating Volume labels with the **Label Format**:sup:`Dir`:sub:`Pool` directive. + +.. _SampleDirectorConfiguration: + +Example Director Configuration File +----------------------------------- + +.. index:: + single: Configuration File Example + + +See below an example of a full Director configuration file: + +.. raw:: latex + + .. literalinclude:: ../../main/bareos-dir.conf.in diff --git a/docs/manuals/en/new_main_reference/source/chapter10/storedconf.rst b/docs/manuals/en/new_main_reference/source/chapter10/storedconf.rst new file mode 100644 index 00000000000..f872cb2cbf1 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter10/storedconf.rst @@ -0,0 +1,246 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _StoredConfChapter: + +Storage Daemon Configuration +============================ + +.. index:: + single: Configuration +.. index:: + pair: Storage Daemon; Configuration +.. index:: + pair: Configuration; Storage Daemon + + +The |bareosSd| configuration file has relatively few resource definitions. However, due to the great variation in backup media and system capabilities, the storage daemon must be highly configurable. As a consequence, there are quite a large number of directives in the Device Resource definition that allow you to define all the characteristics of your Storage device (normally a tape drive). Fortunately, with modern storage devices, the defaults are sufficient, and very few directives +are actually needed. + +For a general discussion of configuration file and resources including the data types recognized by **Bareos**, please see the :ref:`Configuration ` chapter of this manual. The following Storage Resource definitions must be defined: + +- :ref:`Storage ` – to define the name of the Storage daemon. + +- :ref:`Director ` – to define the Director’s name and his access password. + +- :ref:`Device ` – to define the characteristics of your storage device (tape drive). + +- :ref:`Messages ` – to define where error and information messages are to be sent. + +Following resources are optional: + +- :ref:`StorageResourceAutochanger` – to define Autochanger devices. + +- :ref:`StorageResourceNDMP` – to define the NDMP authentication context. + +.. _StorageResourceStorage: + +Storage Resource +---------------- + +.. index:: + pair: Resource; Storage +.. index:: + pair: Storage; Resource + + +In general, the properties specified under the Storage resource define global properties of the Storage daemon. Each Storage daemon configuration file must have one and only one Storage resource definition. + +The following is a typical Storage daemon storage resource definition. + + + + +.. code-block:: sh + :caption: Storage daemon storage definition + + # + # "Global" Storage daemon configuration specifications appear + # under the Storage resource. + # + Storage { + Name = "Storage daemon" + Address = localhost + } + +.. _StorageResourceDirector: + +Director Resource +----------------- + +.. index:: + pair: Resource; Director +.. index:: + pair: Director; Resource + + +The Director resource specifies the Name of the Director which is permitted to use the services of the Storage daemon. There may be multiple Director resources. The Director Name and Password must match the corresponding values in the Director’s configuration file. + +The following is an example of a valid Director resource definition: + + + + +.. code-block:: sh + :caption: Storage daemon Director definition + + Director { + Name = MainDirector + Password = my\_secret\_password + } + +.. _NDMPResource: + +NDMP Resource +------------- + + + +.. _`StorageResourceNDMP}` :raw-latex:`\index[sd]{Resource!NDMP}` :raw-latex:`\index[sd]{NDMP!Resource`: StorageResourceNDMP}` :raw-latex:`\index[sd]{Resource!NDMP}` :raw-latex:`\index[sd]{NDMP!Resource + +The NDMP Resource specifies the authentication details of each NDMP client. There may be multiple NDMP resources for a single Storage daemon. In general, the properties specified within the NDMP resource are specific to one client. + +.. _StorageResourceDevice: + +Device Resource +--------------- + +.. index:: + pair: Resource; Device +.. index:: + pair: Device; Resource + + +The Device Resource specifies the details of each device (normally a tape drive) that can be used by the Storage daemon. There may be multiple Device resources for a single Storage daemon. In general, the properties specified within the Device resource are specific to the Device. + +Edit Codes for Mount and Unmount Directives +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Edit Codes for Mount and Unmount Directives}` :raw-latex:`\index[general]{Mount and Unmount: use variables in directives + + +.. _`mountcodes`: mountcodes + +Before submitting the **Mount Command**, or **Unmount Command** directives to the operating system, Bareos performs character substitution of the following characters: + +.. raw:: latex + + + + + + %% = % + %a = Archive device name + %e = erase (set if cannot mount and first part) + %n = part number + %m = mount point + %v = last part name (i.e. filename) + +.. raw:: latex + + + +Devices that require a mount (USB) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Devices that require a mount (USB) + + +.. raw:: latex + + \begin{description} + \item **Requires Mount**:sup:`Sd`:sub:`Device` + You must set this directive to {\bf yes} for removable devices such as + USB unless they are automounted, and to {\bf no} for all other devices + (tapes/files). This directive indicates if the device requires to be + mounted to be read, and if it must be written in a special way. If it + set, **Mount Point**:sup:`Sd`:sub:`Device` , + **Mount Command**:sup:`Sd`:sub:`Device` and + **Unmount Command**:sup:`Sd`:sub:`Device` + directives must also be defined. + + \item **Mount Point**:sup:`Sd`:sub:`Device` + Directory where the device can be mounted. + + \item **Mount Command**:sup:`Sd`:sub:`Device` + Command that must be executed to mount the device. Before the command is + executed, \%a is replaced with the Archive Device, and \%m with the Mount + Point. + + Most frequently, you will define it as follows: + + \ +.. code-block:: sh + :caption: + + Mount Command = "/bin/mount -t iso9660 -o ro %a %m" + \ + + For some media, you may need multiple commands. If so, it is recommended + that you use a shell script instead of putting them all into the Mount + Command. For example, instead of this: + + \ +.. code-block:: sh + :caption: + + Mount Command = "/usr/local/bin/mymount" + \ + + Where that script contains: + + \ +.. code-block:: sh + :caption: + + #!/bin/sh + ndasadmin enable -s 1 -o w + sleep 2 + mount /dev/ndas-00323794-0p1 /backup + \ + + Similar consideration should be given to all other Command parameters. + + \item **Unmount Command**:sup:`Sd`:sub:`Device` + Command that must be executed to unmount the device. Before the command is + executed, \%a is replaced with the Archive Device, and \%m with the Mount + Point. + + Most frequently, you will define it as follows: + + \ +.. code-block:: sh + :caption: + + Unmount Command = "/bin/umount %m" + \ + + If you need to specify multiple commands, create a shell script. + + \end{description} + +.. _MessagesResource1: + +Messages Resource +----------------- + +.. index:: + pair: Resource; Messages +.. index:: + pair: Messages; Resource + + +For a description of the Messages Resource, please see the :ref:`MessagesChapter` chapter of this manual. + +.. _ExampleStorageConfiguration: + +Example Storage Daemon Configuration File +----------------------------------------- + +A example Storage Daemon configuration file might be the following: + +.. raw:: latex + + .. literalinclude:: ../../main/bareos-sd.conf diff --git a/docs/manuals/en/new_main_reference/source/chapter11/filedconf.rst b/docs/manuals/en/new_main_reference/source/chapter11/filedconf.rst new file mode 100644 index 00000000000..1b4d596d87e --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter11/filedconf.rst @@ -0,0 +1,166 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _FiledConfChapter: + +Client/File Daemon Configuration +================================ + +.. index:: + single: Client/File daemon Configuration + + +The Client (or File Daemon) Configuration is one of the simpler ones to specify. Generally, other than changing the Client name so that error messages are easily identified, you will not need to modify the default Client configuration file. + +For a general discussion of configuration file and resources including the data types recognized by **Bareos**, please see the :ref:`Configuration ` chapter of this manual. The following Client Resource definitions must be defined: + +- :ref:`Client ` – to define what Clients are to be backed up. + +- :ref:`Director ` – to define the Director’s name and its access password. + +- :ref:`Messages ` – to define where error and information messages are to be sent. + +.. _ClientResourceClient: + +Client Resource +--------------- + +.. index:: + single: Client Resource + + +The Client Resource (or FileDaemon) resource defines the name of the Client (as used by the Director) as well as the port on which the Client listens for Director connections. + +Start of the Client records. There must be one and only one Client resource in the configuration file, since it defines the properties of the current client program. + +The following is an example of a valid Client resource definition: + +.. raw:: latex + + + + + + Client { # this is me + Name = rufus-fd + } + +.. raw:: latex + + + +.. _ClientResourceDirector: + +Director Resource +----------------- + +.. index:: + single: Director Resource +.. index:: + pair: Resource; Director + + +The Director resource defines the name and password of the Directors that are permitted to contact this Client. + +Thus multiple Directors may be authorized to use this Client’s services. Each Director will have a different name, and normally a different password as well. + +The following is an example of a valid Director resource definition: + +.. raw:: latex + + + + + + # + # List Directors who are permitted to contact the File daemon + # + Director { + Name = HeadMan + Password = very_good # password HeadMan must supply + } + Director { + Name = Worker + Password = not_as_good + Monitor = Yes + } + +.. raw:: latex + + + +.. _MessagesResource3: + +Messages Resource +----------------- + +.. index:: + single: Messages Resource +.. index:: + pair: Resource; Messages + + +Please see the :ref:`Messages Resource ` Chapter of this manual for the details of the Messages Resource. + +There must be at least one Message resource in the Client configuration file. + +.. _SampleClientConfiguration: + +Example Client Configuration File +--------------------------------- + +An example File Daemon configuration file might be the following: + +.. raw:: latex + + + + + + # + # Default Bareos File Daemon Configuration file + # + # For Bareos release 12.4.4 (12 June 2013) + # + # There is not much to change here except perhaps the + # File daemon Name to + # + + # + # List Directors who are permitted to contact this File daemon + # + Director { + Name = bareos-dir + Password = "aEODFz89JgUbWpuG6hP4OTuAoMvfM1PaJwO+ShXGqXsP" + } + + # + # Restricted Director, used by tray-monitor to get the + # status of the file daemon + # + Director { + Name = client1-mon + Password = "8BoVwTju2TQlafdHFExRIJmUcHUMoIyIqPJjbvcSO61P" + Monitor = yes + } + + # + # "Global" File daemon configuration specifications + # + FileDaemon { # this is me + Name = client1-fd + Maximum Concurrent Jobs = 20 + + # remove comment in next line to load plugins from specified directory + # Plugin Directory = /usr/lib64/bareos/plugins + } + + # Send all messages except skipped files back to Director + Messages { + Name = Standard + director = client1-dir = all, !skipped, !restored + } + +.. raw:: latex + + diff --git a/docs/manuals/en/new_main_reference/source/chapter12/messagesres.rst b/docs/manuals/en/new_main_reference/source/chapter12/messagesres.rst new file mode 100644 index 00000000000..488283ac75c --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter12/messagesres.rst @@ -0,0 +1,135 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _MessagesChapter: + +Messages Resource +================= + + + +.. _`ResourceMessages}` :raw-latex:`\index[general]{Resource!Messages}` :raw-latex:`\index[general]{Messages Resource`: ResourceMessages}` :raw-latex:`\index[general]{Resource!Messages}` :raw-latex:`\index[general]{Messages Resource + +The Messages resource defines how messages are to be handled and destinations to which they should be sent. + +Even though each daemon has a full message handler, within the |bareosFd| and the |bareosSd|, you will normally choose to send all the appropriate messages back to the |bareosDir|. This permits all the messages associated with a single Job to be combined in the Director and sent as a single email message to the user, or logged together in a single file. + +Each message that Bareos generates (i.e. that each daemon generates) has an associated type such as INFO, WARNING, ERROR, FATAL, etc. Using the message resource, you can specify which message types you wish to see and where they should be sent. In addition, a message may be sent to multiple destinations. For example, you may want all error messages both logged as well as sent to you in an email. By defining multiple messages resources, you can have different message handling for each type of Job +(e.g. Full backups versus Incremental backups). + +In general, messages are attached to a Job and are included in the Job report. There are some rare cases, where this is not possible, e.g. when no job is running, or if a communications error occurs between a daemon and the director. In those cases, the message may remain in the system, and should be flushed at the end of the next Job. + +The records contained in a Messages resource consist of a **destination** specification followed by a list of **message-types** in the format: :raw-latex:`\index[dir]{Messages!destination}` + +destination = message-type1, message-type2, message-type3, ... + +or for those destinations that need and address specification (e.g. email): + +destination = address = message-type1, message-type2, message-type3, ... + | + | where + + destination + is one of a predefined set of keywords that define where the message is to be sent (**Append**:sup:`Dir`:sub:`Messages` , **Console**:sup:`Dir`:sub:`Messages` , **File**:sup:`Dir`:sub:`Messages` , **Mail**:sup:`Dir`:sub:`Messages` , ...), + + address + varies according to the **destination** keyword, but is typically an email address or a filename, + + :ref:`message-type ` + is one of a predefined set of keywords that define the type of message generated by Bareos: **ERROR**, **WARNING**, **FATAL**, ... + +.. _MessageTypes: + +Message Types +------------- + +For any destination, the **message-type** field is a comma separated list of the following types or classes of messages: + +info + | + | :raw-latex:`\index[general]{Messages!type!info}` General information messages. + +warning + | + | :raw-latex:`\index[general]{Messages!type!warning}` Warning messages. Generally this is some unusual condition but not expected to be serious. + +error + | + | :raw-latex:`\index[general]{Messages!type!error}` Non-fatal error messages. The job continues running. Any error message should be investigated as it means that something went wrong. + +fatal + | + | :raw-latex:`\index[general]{Messages!type!fatal}` Fatal error messages. Fatal errors cause the job to terminate. + +terminate + | + | :raw-latex:`\index[general]{Messages!type!terminate}` Message generated when the daemon shuts down. + +notsaved + | + | :raw-latex:`\index[general]{Messages!type!notsaved}` Files not saved because of some error. Usually because the file cannot be accessed (i.e. it does not exist or is not mounted). + +skipped + | + | :raw-latex:`\index[general]{Messages!type!skipped}` Files that were skipped because of a user supplied option such as an incremental backup or a file that matches an exclusion pattern. This is not considered an error condition such as the files listed for the **notsaved** type because the configuration file explicitly requests these types of files to be skipped. For example, any unchanged file during an incremental backup, or any subdirectory if the no recursion option is specified. + +mount + | + | :raw-latex:`\index[general]{Messages!type!mount}` Volume mount or intervention requests from the Storage daemon. These requests require a specific operator intervention for the job to continue. + +restored + | + | :raw-latex:`\index[general]{Messages!type!restored}` The **ls** style listing generated for each file restored is sent to this message class. + +all + | + | :raw-latex:`\index[general]{Messages!type!all}` All message types. + +security + | + | :raw-latex:`\index[general]{Messages!type!security}` Security info/warning messages principally from unauthorized connection attempts. + +alert + | + | :raw-latex:`\index[general]{Messages!type!alert}` Alert messages. These are messages generated by tape alerts. + +volmgmt + | + | :raw-latex:`\index[general]{Messages!type!volmgmt}` Volume management messages. Currently there are no volume management messages generated. + +audit + | + | :raw-latex:`\index[general]{Messages!type!audit}` :raw-latex:`\index[general]{auditing}` Audit messages. Interacting with the Bareos Director will be audited. Can be configured with in resource **Auditing**:sup:`Dir`:sub:`Director` . + +The following is an example of a valid Messages resource definition, where all messages except files explicitly skipped or daemon termination messages are sent by email to backupoperator@example.com. In addition all mount messages are sent to the operator (i.e. emailed to backupoperator@example.com). Finally all messages other than explicitly skipped files and files saved are sent to the console: + + + + +.. code-block:: sh + :caption: Message resource + + Messages { + Name = Standard + Mail = backupoperator@example.com = all, !skipped, !terminate + Operator = backupoperator@example.com = mount + Console = all, !skipped, !saved + } + +With the exception of the email address, an example Director’s Messages resource is as follows: + + + + +.. code-block:: sh + :caption: Message resource + + Messages { + Name = Standard + Mail Command = "/usr/sbin/bsmtp -h mail.example.com -f \"\(Bareos\) %r\" -s \"Bareos: %t %e of %c %l\" %r" + Operator Command = "/usr/sbin/bsmtp -h mail.example.com -f \"\(Bareos\) %r\" -s \"Bareos: Intervention needed for %j\" %r" + Mail On Error = backupoperator@example.com = all, !skipped, !terminate + Append = "/var/log/bareos/bareos.log" = all, !skipped, !terminate + Operator = backupoperator@example.com = mount + Console = all, !skipped, !saved + } diff --git a/docs/manuals/en/new_main_reference/source/chapter13/consoleconf.rst b/docs/manuals/en/new_main_reference/source/chapter13/consoleconf.rst new file mode 100644 index 00000000000..df4d84ccefa --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter13/consoleconf.rst @@ -0,0 +1,254 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _ConsoleConfChapter: + +Console Configuration +===================== + +.. index:: + single: Console Configuration + + +The Console configuration file is the simplest of all the configuration files, and in general, you should not need to change it except for the password. It simply contains the information necessary to contact the Director or Directors. + +For a general discussion of the syntax of configuration files and their resources including the data types recognized by **Bareos**, please see the :ref:`Configuration ` chapter of this manual. + +The following Console Resource definition must be defined: + +.. _ConsoleResourceDirector: + +Director Resource +----------------- + +.. index:: + single: Director Resource +.. index:: + pair: Resource; Director + + +The Director resource defines the attributes of the Director running on the network. You may have multiple Director resource specifications in a single Console configuration file. If you have more than one, you will be prompted to choose one when you start the **Console** program. + +An actual example might be: + +.. raw:: latex + + + + + + Director { + Name = HeadMan + address = rufus.cats.com + password = xyz1erploit + } + +.. raw:: latex + + + +.. raw:: latex + + \hide{ + \section{ConsoleFont Resource} + \index[general]{Resource!ConsoleFont} + \index[general]{ConsoleFont Resource} + + The ConsoleFont resource is available only in the GNOME version of the + console. It permits you to define the font that you want used to display in + the main listing window. + + \begin{description} + + \item [ConsoleFont] + \index[console]{ConsoleFont} + Start of the ConsoleFont directives. + + \item [Name = {\textless}name{\textgreater}] \\ + \index[console]{Name} + The name of the font. + + \item [Font = {\textless}Pango Font Name{\textgreater}] \\ + \index[console]{Font} + The string value given here defines the desired font. It is specified in the + Pango format. For example, the default specification is: + + + \ + Font = "LucidaTypewriter 9" + \ + + + \end{description} + + Thanks to Phil Stracchino for providing the code for this feature. + + An different example might be: + + + \ + ConsoleFont { + Name = Default + Font = "Monospace 10" + } + \ + + } + +.. _ConsoleResourceConsole: + +Console Resource +---------------- + +.. index:: + single: Console Resource +.. index:: + pair: Resource; Console + + +There are three different kinds of consoles, which the administrator or user can use to interact with the Director. These three kinds of consoles comprise three different security levels. + +- The first console type is an **admin** or **anonymous** or **default** console, which has full privileges. There is no console resource necessary for this type since the password is specified in the Director resource. Typically you would use this console only for administrators. + +- The second type of console is a "named" or "restricted" console defined within a Console resource in both the Director’s configuration file and in the Console’s configuration file. Both the names and the passwords in these two entries must match much as is the case for Client programs. + + This second type of console begins with absolutely no privileges except those explicitly specified in the Director’s Console resource. Note, the definition of what these restricted consoles can do is determined by the Director’s conf file. + + Thus you may define within the Director’s conf file multiple Consoles with different names and passwords, sort of like multiple users, each with different privileges. As a default, these consoles can do absolutely nothing – no commands what so ever. You give them privileges or rather access to commands and resources by specifying access control lists in the Director’s Console resource. This gives the administrator fine grained control over what particular consoles (or users) can do. + +- The third type of console is similar to the above mentioned restricted console in that it requires a Console resource definition in both the Director and the Console. In addition, if the console name, provided on the **Name =** directive, is the same as a Client name, the user of that console is permitted to use the **SetIP** command to change the Address directive in the Director’s client resource to the IP address of the Console. This permits portables or other machines using DHCP + (non-fixed IP addresses) to "notify" the Director of their current IP address. + +The Console resource is optional and need not be specified. However, if it is specified, you can use ACLs (Access Control Lists) in the Director’s configuration file to restrict the particular console (or user) to see only information pertaining to his jobs or client machine. + +You may specify as many Console resources in the console’s conf file. If you do so, generally the first Console resource will be used. However, if you have multiple Director resources (i.e. you want to connect to different directors), you can bind one of your Console resources to a particular Director resource, and thus when you choose a particular Director, the appropriate Console configuration resource will be used. See the "Director" directive in the Console resource described below for more +information. + +Note, the Console resource is optional, but can be useful for restricted consoles as noted above. + +Example Console Configuration File +---------------------------------- + +.. index:: + pair: Configuration; bconsole + + +A Console configuration file might look like this: + + + + +.. code-block:: sh + :caption: bconsole configuration + + Director { + Name = "bareos.example.com-dir" + address = "bareos.example.com" + Password = "PASSWORD" + } + +With this configuration, the console program (e.g. :program:`bconsole`) will try to connect to a |bareosDir| named **bareos.example.com-dir** at the network address :strong:`bareos.example.com` and authenticate to the admin console using the password **PASSWORD**. + +.. _section-ConsoleAccessExample: + +Using Named Consoles +~~~~~~~~~~~~~~~~~~~~ + +The following configuration files were supplied by Phil Stracchino. + +To use named consoles from :program:`bconsole`, use a :file:`bconsole.conf` configuration file like this: + + + + +.. code-block:: sh + :caption: bconsole: restricted-user + + Director { + Name = bareos-dir + Address = myserver + Password = "XXXXXXXXXXX" + } + + Console { + Name = restricted-user + Password = "RUPASSWORD" + } + +Where the Password in the Director section is deliberately incorrect and the Console resource is given a name, in this case :strong:`restricted-user`. Then in the Director configuration (not directly accessible by the user), we define: + + + + +.. code-block:: sh + :caption: bareos-dir console restricted-user + + Console { + Name = restricted-user + Password = "RUPASSWORD" + JobACL = "Restricted Client Save" + ClientACL = restricted-client + StorageACL = main-storage + ScheduleACL = *all* + PoolACL = *all* + FileSetACL = "Restricted Client's FileSet" + CatalogACL = MyCatalog + CommandACL = run + } + +The user login into the Director from his Console will get logged in as **restricted-user**:sup:`Dir`:sub:`Console` and he will only be able to see or access a Job with the name **Restricted Client Save**:sup:`Dir`:sub:`Job` , a Client with the name **restricted-client**:sup:`Dir`:sub:`Client` , a storage device **main-storage**:sup:`Dir`:sub:`Storage` , any Schedule or Pool, a FileSet named +**Restricted Client's FileSet**:sup:`Dir`:sub:`FileSet` , a Catalog named **MyCatalog**:sup:`Dir`:sub:`Catalog` and the only command he can use in the Console is the :strong:`run` command. In other words, this user is rather limited in what he can see and do with Bareos. For details how to configure ACLs, see the :strong:`Acl` data type description. + +The following is an example of a :file:`bconsole.conf` file that can access several Directors and has different Consoles depending on the Director: + + + + +.. code-block:: sh + :caption: bconsole: multiple consoles + + Director { + Name = bareos-dir + Address = myserver + Password = "XXXXXXXXXXX" # no, really. this is not obfuscation. + } + + Director { + Name = SecondDirector + Address = secondserver + Password = "XXXXXXXXXXX" # no, really. this is not obfuscation. + } + + Console { + Name = restricted-user + Password = "RUPASSWORD" + Director = MyDirector + } + + Console { + Name = restricted-user2 + Password = "OTHERPASSWORD" + Director = SecondDirector + } + +The second Director referenced at **secondserver**:sup:`Dir`:sub:`Director` might look like the following: + + + + +.. code-block:: sh + :caption: bareos-dir console restricted-user2 + + Console { + Name = restricted-user2 + Password = "OTHERPASSWORD" + JobACL = "Restricted Client Save" + ClientACL = restricted-client + StorageACL = second-storage + ScheduleACL = *all* + PoolACL = *all* + FileSetACL = "Restricted Client's FileSet" + CatalogACL = RestrictedCatalog + CommandACL = run, restore + WhereACL = "/" + } diff --git a/docs/manuals/en/new_main_reference/source/chapter14/monitorconf.rst b/docs/manuals/en/new_main_reference/source/chapter14/monitorconf.rst new file mode 100644 index 00000000000..661be94b2e9 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter14/monitorconf.rst @@ -0,0 +1,209 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _section-MonitorConfig: + +Monitor Configuration +===================== + +.. index:: + single: Monitor Configuration +.. index:: + pair: Configuration; Monitor + + +The Monitor configuration file is a stripped down version of the Director configuration file, mixed with a Console configuration file. It simply contains the information necessary to contact Directors, Clients, and Storage daemons you want to monitor. + +For a general discussion of configuration file and resources including the data types recognized by **Bareos**, please see the :ref:`Configuration ` chapter of this manual. + +The following Monitor Resource definition must be defined: + +- :ref:`Monitor ` – to define the Monitor’s name used to connect to all the daemons and the password used to connect to the Directors. Note, you must not define more than one Monitor resource in the Monitor configuration file. + +- At least one :ref:`Client `, :ref:`Storage ` or :ref:`Director ` resource, to define the daemons to monitor. + +.. _MonitorResource: + +Monitor Resource +---------------- + +.. index:: + single: Monitor Resource +.. index:: + pair: Resource; Monitor + + +The Monitor resource defines the attributes of the Monitor running on the network. The parameters you define here must be configured as a Director resource in Clients and Storages configuration files, and as a Console resource in Directors configuration files. + +.. _DirectorResource2: + +Director Resource +----------------- + +.. index:: + single: Director Resource +.. index:: + pair: Resource; Director + + +The Director resource defines the attributes of the Directors that are monitored by this Monitor. + +As you are not permitted to define a Password in this resource, to avoid obtaining full Director privileges, you must create a Console resource in the :ref:`Director's configuration ` file, using the Console Name and Password defined in the Monitor resource. To avoid security problems, you should configure this Console resource to allow access to no other daemons, and permit the use of only two commands: **status** and **.status** (see below for an example). + +You may have multiple Director resource specifications in a single Monitor configuration file. + +.. _ClientResource1: + +Client Resource +--------------- + +.. index:: + single: Client Resource + + +The Client resource defines the attributes of the Clients that are monitored by this Monitor. + +You must create a Director resource in the :ref:`Client's configuration ` file, using the Director Name defined in the Monitor resource. To avoid security problems, you should set the **Monitor** directive to **Yes** in this Director resource. + +You may have multiple Director resource specifications in a single Monitor configuration file. + +.. _StorageResource1: + +Storage Resource +---------------- + +.. index:: + single: Storage Resource + + +The Storage resource defines the attributes of the Storages that are monitored by this Monitor. + +You must create a Director resource in the :ref:`Storage's configuration ` file, using the Director Name defined in the Monitor resource. To avoid security problems, you should set the **Monitor** directive to **Yes** in this Director resource. + +You may have multiple Director resource specifications in a single Monitor configuration file. + +Tray Monitor +------------ + +Tray Monitor Security +~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Tray Monitor Security +.. index:: + pair: Security; Tray Monitor + + +There is no security problem in relaxing the permissions on tray-monitor.conf as long as FD, SD and DIR are configured properly, so the passwords contained in this file only gives access to the status of the daemons. It could be a security problem if you consider the status information as potentially dangerous (most people consider this as not being dangerous). + +| Concerning Director’s configuration: +| In tray-monitor.conf, the password in the Monitor resource must point to a restricted console in bareos-dir.conf (see the documentation). So, if you use this password with bconsole, you’ll only have access to the status of the director (commands status and .status). It could be a security problem if there is a bug in the ACL code of the director. + +| Concerning File and Storage Daemons’ configuration: +| In tray-monitor.conf, the Name in the Monitor resource must point to a Director resource in bareos-fd/sd.conf, with the Monitor directive set to **Yes** (see the documentation). It could be a security problem if there is a bug in the code which check if a command is valid for a Monitor (this is very unlikely as the code is pretty simple). + +Example Tray Monitor configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Tray Monitor; Configuration +.. index:: + pair: Configuration; Tray Monitor + + +An example Tray Monitor configuration file might be the following: + + + + +.. code-block:: sh + :caption: Example tray-monitor.conf + + # + # Bareos Tray Monitor Configuration File + # + Monitor { + Name = rufus-mon # password for Directors + Password = "GN0uRo7PTUmlMbqrJ2Gr1p0fk0HQJTxwnFyE4WSST3MWZseR" + RefreshInterval = 10 seconds + } + + Client { + Name = rufus-fd + Address = rufus + FDPort = 9102 # password for FileDaemon + Password = "FYpq4yyI1y562EMS35bA0J0QC0M2L3t5cZObxT3XQxgxppTn" + } + Storage { + Name = rufus-sd + Address = rufus + SDPort = 9103 # password for StorageDaemon + Password = "9usxgc307dMbe7jbD16v0PXlhD64UVasIDD0DH2WAujcDsc6" + } + Director { + Name = rufus-dir + DIRport = 9101 + address = rufus + } + +Example File daemon’s Director record +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + + +.. code-block:: sh + :caption: Example Monitor resource + + # + # Restricted Director, used by tray-monitor to get the + # status of the file daemon + # + Director { + Name = rufus-mon + Password = "FYpq4yyI1y562EMS35bA0J0QC0M2L3t5cZObxT3XQxgxppTn" + Monitor = yes + } + +A full example can be found at :ref:`SampleClientConfiguration`. + +Example Storage daemon’s Director record +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + + +.. code-block:: sh + :caption: Example Monitor resource + + # + # Restricted Director, used by tray-monitor to get the + # status of the storage daemon + # + Director { + Name = rufus-mon + Password = "9usxgc307dMbe7jbD16v0PXlhD64UVasIDD0DH2WAujcDsc6" + Monitor = yes + } + +A full example can be found at :ref:`ExampleStorageConfiguration`. + +Example Director’s Console record +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + + + +.. code-block:: sh + :caption: Example Monitor resource + + # + # Restricted console used by tray-monitor to get the status of the director + # + Console { + Name = Monitor + Password = "GN0uRo7PTUmlMbqrJ2Gr1p0fk0HQJTxwnFyE4WSST3MWZseR" + CommandACL = status, .status + } + +A full example can be found at :ref:`SampleDirectorConfiguration`. diff --git a/docs/manuals/en/new_main_reference/source/chapter15/bconsole.rst b/docs/manuals/en/new_main_reference/source/chapter15/bconsole.rst new file mode 100644 index 00000000000..a81e1b41361 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter15/bconsole.rst @@ -0,0 +1,1936 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _section-bconsole: + +Bareos Console +============== + +.. index:: + single: Bareos Console}` :raw-latex:`\index[general]{bconsole +.. index:: + pair: Command; bconsole + + +The **Bareos Console** (:program:`bconsole`) is a program that allows the user or the System Administrator, to interact with the Bareos Director daemon while the daemon is running. + +The current Bareos Console comes as a shell interface (TTY style). It permit the administrator or authorized users to interact with Bareos. You can determine the status of a particular job, examine the contents of the Catalog as well as perform certain tape manipulations with the Console program. + +Since the Console program interacts with the Director through the network, your Console and Director programs do not necessarily need to run on the same machine. + +In fact, a certain minimal knowledge of the Console program is needed in order for Bareos to be able to write on more than one tape, because when Bareos requests a new tape, it waits until the user, via the Console program, indicates that the new tape is mounted. + +Console Configuration +--------------------- + +.. index:: + pair: Configuration; Console +.. index:: + pair: Configuration; bconsole + + +When the Console starts, it reads a standard Bareos configuration file named **bconsole.conf** unless you specify the **-c** command line option (see below). This file allows default configuration of the Console, and at the current time, the only Resource Record defined is the Director resource, which gives the Console the name and address of the Director. For more information on configuration of the Console program, please see the :ref:`Console Configuration ` +chapter of this document. + +Running the Console Program +--------------------------- + +The console program can be run with the following options: + +.. index:: + single: Command Line Options + + + + + +.. code-block:: sh + :caption: bconsole command line options}{}{bconsole -? + + Usage: bconsole [-s] [-c config_file] [-d debug_level] + -D select a Director + -l list Directors defined + -c set configuration file to file + -d set debug level to + -dt print timestamp in debug output + -n no conio + -s no signals + -u set command execution timeout to seconds + -t test - read configuration and exit + -? print this message. + +After launching the Console program (bconsole), it will prompt you for the next command with an asterisk (*). Generally, for all commands, you can simply enter the command name and the Console program will prompt you for the necessary arguments. Alternatively, in most cases, you may enter the command followed by arguments. The general format is: + +.. raw:: latex + + + + + + [=] [=] ... + +.. raw:: latex + + + +where **command** is one of the commands listed below; **keyword** is one of the keywords listed below (usually followed by an argument); and **argument** is the value. The command may be abbreviated to the shortest unique form. If two commands have the same starting letters, the one that will be selected is the one that appears first in the **help** listing. If you want the second command, simply spell out the full command. None of the keywords following the command may be abbreviated. + +For example: + +.. raw:: latex + + + + + + list files jobid=23 + +.. raw:: latex + + + +will list all files saved for JobId 23. Or: + +.. raw:: latex + + + + + + show pools + +.. raw:: latex + + + +will display all the Pool resource records. + +The maximum command line length is limited to 511 characters, so if you are scripting the console, you may need to take some care to limit the line length. + +Exit the Console Program +~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Command; bconsole; exit; + + +Normally, you simply enter **quit** or **exit** and the Console program will terminate. However, it waits until the Director acknowledges the command. If the Director is already doing a lengthy command (e.g. prune), it may take some time. If you want to immediately terminate the Console program, enter the **.quit** command. + +There is currently no way to interrupt a Console command once issued (i.e. Ctrl-C does not work). However, if you are at a prompt that is asking you to select one of several possibilities and you would like to abort the command, you can enter a period (**.**), and in most cases, you will either be returned to the main command prompt or if appropriate the previous prompt (in the case of nested prompts). In a few places such as where it is asking for a Volume name, the period will be taken to be +the Volume name. In that case, you will most likely be able to cancel at the next prompt. + +Running the Console from a Shell Script +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: Console; Running from a Shell + + +.. _`scripting`: scripting + +You can automate many Console tasks by running the console program from a shell script. For example, if you have created a file containing the following commands: + +.. raw:: latex + + + + + + bconsole -c ./bconsole.conf <] [storage=] [jobid=] + + Normally, the :strong:`label` command is used rather than this command because the :strong:`label` command labels the physical media (tape, disk,, ...) and does the equivalent of the :strong:`add` command. The :strong:`add` command affects only the Catalog and not the physical media (data on Volumes). The physical media must exist and be labeled before use (usually with the :strong:`label` command). This command + can, however, be useful if you wish to add a number of Volumes to the Pool that will be physically labeled at a later time. It can also be useful if you are importing a tape from another site. Please see the :strong:`label` command for the list of legal characters in a Volume name. + +autodisplay + +.. index:: + triple: Console; Command; autodisplay on/off; + This command accepts **on** or **off** as an argument, and turns auto-display of messages on or off respectively. The default for the console program is **off**, which means that you will be notified when there are console messages pending, but they will not automatically be displayed. + + When autodisplay is turned off, you must explicitly retrieve the messages with the **messages** command. When autodisplay is turned on, the messages will be displayed on the console as they are received. + +automount + +.. index:: + triple: Console; Command; automount on/off; + This command accepts **on** or **off** as the argument, and turns auto-mounting of the Volume after a **label** command on or off respectively. The default is **on**. If **automount** is turned off, you must explicitly **mount** tape Volumes after a label command to use it. + +cancel + +.. index:: + triple: Console; Command; cancel jobid; + This command is used to cancel a job and accepts **jobid=nnn** or **job=xxx** as an argument where nnn is replaced by the JobId and xxx is replaced by the job name. If you do not specify a keyword, the Console program will prompt you with the names of all the active jobs allowing you to choose one. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: cancel + + cancel [jobid= job= ujobid=] + + Once a Job is marked to be cancelled, it may take a bit of time (generally within a minute but up to two hours) before the Job actually terminates, depending on what operations it is doing. Don’t be surprised that you receive a Job not found message. That just means that one of the three daemons had already canceled the job. Messages numbered in the 1000’s are from the Director, 2000’s are from the File daemon and 3000’s from the Storage daemon. + + It is possible to cancel multiple jobs at once. Therefore, the following extra options are available for the job-selection: + + - all jobs + + - all jobs with a created state + + - all jobs with a blocked state + + - all jobs with a waiting state + + - all jobs with a running state + + Usage: + + + + +.. code-block:: sh + :caption: cancel all + + cancel all + cancel all state= + + Sometimes the Director already removed the job from its running queue, but the storage daemon still thinks it is doing a backup (or another job) - so you cannot cancel the job from within a console anymore. Therefore it is possible to cancel a job by JobId on the storage daemon. It might be helpful to execute a :strong:`status storage` on the Storage Daemon to make sure what job you want to cancel. + + Usage: + + + + +.. code-block:: sh + :caption: cancel on Storage Daemon + + cancel storage= Jobid= + + This way you can also remove a job that blocks any other jobs from running without the need to restart the whole storage daemon. + +create + +.. index:: + triple: Console; Command; create pool; + This command is not normally used as the Pool records are automatically created by the Director when it starts based on what it finds in the configuration. If needed, this command can be used, to create a Pool record in the database using the Pool resource record defined in the Director’s configuration file. So in a sense, this command simply transfers the information from the Pool resource in the configuration file into the Catalog. + Normally this command is done automatically for you when the Director starts providing the Pool is referenced within a Job resource. If you use this command on an existing Pool, it will automatically update the Catalog to have the same information as the Pool resource. After creating a Pool, you will most likely use the **label** command to label one or more volumes and add their names to the Media database. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: create + + create [pool=] + + When starting a Job, if Bareos determines that there is no Pool record in the database, but there is a Pool resource of the appropriate name, it will create it for you. If you want the Pool record to appear in the database immediately, simply use this command to force it to be created. + +configure + + +.. _`section-bcommandConfigure`: section-bcommandConfigure + + Configures director resources during runtime. The first configure subcommands are :strong:`configure add` and :strong:`configure export`. Other subcommands may follow in later releases. + + configure add + + +.. _`section-bcommandConfigureAdd}` :raw-latex:`\index[general]{Console!Command!configure add`: section-bcommandConfigureAdd}` :raw-latex:`\index[general]{Console!Command!configure add + + This command allows to add resources during runtime. Usage: + + + + +.. code-block:: sh + :caption: configure add usage + + configure add name= = = ... + + Values that must be quoted in the resulting configuration must be added as: + + + + +.. code-block:: sh + :caption: configure add usage with values containing spaces + + configure add name= ="\"\"" ... + + The command generates and loads a new valid resource. As the new resource is also stored at + + + + :file:`/bareos-dir.d//.conf` + + (see :ref:`section-ConfigurationResourceFileConventions`) it is persistent upon reload and restart. + + This feature requires :ref:`section-ConfigurationSubdirectories`. + + All kinds of resources can be added. When adding a client resource, the :ref:`ClientResourceDirector` for the |bareosFd| is also created and stored at: + + + + :file:`/bareos-dir-export/client//bareos-fd.d/director/.conf` + + + + +.. code-block:: sh + :caption: Example: adding a client and a job resource during runtime + + *configure add client name=client2-fd address=192.168.0.2 password=secret + Created resource config file "/etc/bareos/bareos-dir.d/client/client2-fd.conf": + Client { + Name = client2-fd + Address = 192.168.0.2 + Password = secret + } + *configure add job name=client2-job client=client2-fd jobdefs=DefaultJob + Created resource config file "/etc/bareos/bareos-dir.d/job/client2-job.conf": + Job { + Name = client2-job + Client = client2-fd + JobDefs = DefaultJob + } + + These two commands create three resource configuration files: + + - + + + + :file:`/etc/bareos/bareos-dir.d/client/client2-fd.conf` + + - :file:`/etc/bareos/bareos-dir-export/client/client2-fd/bareos-fd.d/director/bareos-dir.conf` (assuming your director resource is named **bareos-dir**) + + - + + + + :file:`/etc/bareos/bareos-dir.d/job/client2-job.conf` + + The files in :file:`bareos-dir-export/client/` directory are not used by the |bareosDir|. However, they can be copied to new clients to configure these clients for the |bareosDir|. + + + + +.. warning:: + Don't be confused by the extensive output of \bcommand{help{configure}. As \bcommand{configure}{add} allows configuring arbitrary resources, the output of \bcommand{help}{configure} lists all the resources, each with all valid directives. The same data is also used for \command{bconsole} command line completion.} + + Available since Bareos 16.2.4. + + configure export + + +.. _`section-bcommandConfigureExport}` :raw-latex:`\index[general]{Console!Command!configure export`: section-bcommandConfigureExport}` :raw-latex:`\index[general]{Console!Command!configure export + + This command allows to export the :sup:`Fd` :strong:`Director` resource for clients already configured in the |bareosDir|. + + Usage: + + + + +.. code-block:: sh + :caption: Export the bareos-fd Director resource for the client bareos-fd + + configure export client=bareos-fd + Exported resource file "/etc/bareos/bareos-dir-export/client/bareos-fd/bareos-fd.d/director/bareos-dir.conf": + Director { + Name = bareos-dir + Password = "[md5]932d1d3ef3c298047809119510f4bee6" + } + + To use it, copy the :sup:`Fd` :strong:`Director` resource file to the client machine (on Linux: to :file:`/etc/bareos/bareos-fd.d/director/`) and restart the |bareosFd|. + + Available since Bareos 16.2.4. + +delete + +.. index:: + triple: Console; Command; delete; + The delete command is used to delete a Volume, Pool or Job record from the Catalog as well as all associated catalog Volume records that were created. This command operates only on the Catalog database and has no effect on the actual data written to a Volume. This command can be dangerous and we strongly recommend that you do not use it unless you know what you are doing. + + If the keyword **Volume** appears on the command line, the named Volume will be deleted from the catalog, if the keyword **Pool** appears on the command line, a Pool will be deleted, and if the keyword **Job** appears on the command line, a Job and all its associated records (File and JobMedia) will be deleted from the catalog. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: delete + + delete pool= + delete volume= pool= + delete JobId= JobId= ... + delete Job JobId=n,m,o-r,t ... + + The first form deletes a Pool record from the catalog database. The second form deletes a Volume record from the specified pool in the catalog database. The third form deletes the specified Job record from the catalog database. The last form deletes JobId records for JobIds n, m, o, p, q, r, and t. Where each one of the n,m,... is, of course, a number. That is a "delete jobid" accepts lists and ranges of jobids. + +disable + +.. index:: + triple: Console; Command; disable; + This command permits you to disable a Job for automatic scheduling. The job may have been previously enabled with the Job resource **Enabled** directive or using the console **enable** command. The next time the Director is reloaded or restarted, the Enable/Disable state will be set to the value in the Job resource (default enabled) as defined in the |bareosDir| configuration. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: disable + + disable job= + +enable + +.. index:: + triple: Console; Command; enable; + This command permits you to enable a Job for automatic scheduling. The job may have been previously disabled with the Job resource **Enabled** directive or using the console **disable** command. The next time the Director is reloaded or restarted, the Enable/Disable state will be set to the value in the Job resource (default enabled) as defined in the |bareosDir| configuration. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: enable + + enable job= + + + +.. _`estimate`: estimate + +estimate + +.. index:: + triple: Console; Command; estimate; + Using this command, you can get an idea how many files will be backed up, or if you are unsure about your Include statements in your FileSet, you can test them without doing an actual backup. The default is to assume a Full backup. However, you can override this by specifying a **level=Incremental** or **level=Differential** on the command line. A Job name must be specified or you will be prompted for one, and optionally a Client and + FileSet may be specified on the command line. It then contacts the client which computes the number of files and bytes that would be backed up. Please note that this is an estimate calculated from the number of blocks in the file rather than by reading the actual bytes. As such, the estimated backup size will generally be larger than an actual backup. + + The ``estimate`` command can use the accurate code to detect changes and give a better estimation. You can set the accurate behavior on command line using ``accurate=yes/no`` or use the Job setting as default value. + + Optionally you may specify the keyword **listing** in which case, all the files to be backed up will be listed. Note, it could take quite some time to display them if the backup is large. The full form is: + + The full form of this command is: + + + + +.. code-block:: sh + :caption: estimate + + estimate job= listing client= accurate= fileset= level= + + Specification of the **job** is sufficient, but you can also override the client, fileset, accurate and/or level by specifying them on the estimate command line. + + As an example, you might do: + + + + +.. code-block:: sh + :caption: estimate: redirected output + + @output /tmp/listing + estimate job=NightlySave listing level=Incremental + @output + + which will do a full listing of all files to be backed up for the Job **NightlySave** during an Incremental save and put it in the file **/tmp/listing**. Note, the byte estimate provided by this command is based on the file size contained in the directory item. This can give wildly incorrect estimates of the actual storage used if there are sparse files on your systems. Sparse files are often found on 64 bit systems for certain system files. The size that is returned is the size Bareos will + backup if the sparse option is not specified in the FileSet. There is currently no way to get an estimate of the real file size that would be found should the sparse option be enabled. + +exit + +.. index:: + triple: Console; Command; exit; + This command terminates the console program. + +export + +.. index:: + triple: Console; Command; export; + The export command is used to export tapes from an autochanger. Most Automatic Tapechangers offer special slots for importing new tape cartridges or exporting written tape cartridges. This can happen without having to set the device offline. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: export + + export storage= srcslots= [dstslots= volume= scan] + + The export command does exactly the opposite of the import command. You can specify which slots should be transferred to import/export slots. The most useful application of the export command is the possibility to automatically transfer the volumes of a certain backup into the import/export slots for external storage. + + To be able to to this, the export command also accepts a list of volume names to be exported. + + Example: + + + + +.. code-block:: sh + :caption: export volume + + export volume=A00020L4|A00007L4|A00005L4 + + Instead of exporting volumes by names you can also select a number of slots via the srcslots keyword and export those to the slots you specify in dstslots. The export command will check if the slots have content (e.g. otherwise there is not much to export) and if there are enough export slots and if those are really import/export slots. + + Example: + + + + +.. code-block:: sh + :caption: export slots + + export srcslots=1-2 dstslots=37-38 + + To automatically export the Volumes used by a certain backup job, you can use the following RunScript in that job: + + + + +.. code-block:: sh + :caption: automatic export + + RunScript { + Console = "export storage=TandbergT40 volume=%V" + RunsWhen = After + RunsOnClient = no + } + + To send an e-mail notification via the Messages resource regarding export tapes you can use the Variable %V substitution in the Messages resource, which is implemented in Bareos 13.2. However, it does also work in earlier releases inside the job resources. So in versions prior to Bareos 13.2 the following workaround can be used: + + + + +.. code-block:: sh + :caption: e-mail notification via messages resource regarding export tapes + + RunAfterJob = "/bin/bash -c \"/bin/echo Remove Tape %V | \ + /usr/sbin/bsmtp -h localhost -f root@localhost -s 'Remove Tape %V' root@localhost \"" + +gui + +.. index:: + triple: Console; Command; gui; + Invoke the non-interactive gui mode. This command is only used when :program:`bconsole` is commanded by an external program. + +help + +.. index:: + triple: Console; Command; help; + This command displays the list of commands available. + +import + +.. index:: + triple: Console; Command; import; + The import command is used to import tapes into an autochanger. Most Automatic Tapechangers offer special slots for importing new tape cartridges or exporting written tape cartridges. This can happen without having to set the device offline. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: import + + import storage= [srcslots= dstslots= volume= scan] + + To import new tapes into the autochanger, you only have to load the new tapes into the import/export slots and call import from the cmdline. + + The import command will automatically transfer the new tapes into free slots of the autochanger. The slots are filled in order of the slot numbers. To import all tapes, there have to be enough free slots to load all tapes. + + Example with a Library with 36 Slots and 3 Import/Export Slots: + + + + +.. code-block:: sh + :caption: import example + + *import storage=TandbergT40 + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "slots" command. + Device "Drive-1" has 39 slots. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "listall" command. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger transfer command. + 3308 Successfully transfered volume from slot 37 to 20. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger transfer command. + 3308 Successfully transfered volume from slot 38 to 21. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger transfer command. + 3308 Successfully transfered volume from slot 39 to 25. + + You can also import certain slots when you don’t have enough free slots in your autochanger to put all the import/export slots in. + + Example with a Library with 36 Slots and 3 Import/Export Slots importing one slot: + + + + +.. code-block:: sh + :caption: import example + + *import storage=TandbergT40 srcslots=37 dstslots=20 + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "slots" command. + Device "Drive-1" has 39 slots. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "listall" command. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger transfer command. + 3308 Successfully transfered volume from slot 37 to 20. + +label + +.. index:: + triple: Console; Command; label; + :raw-latex:`\index[general]{Console!Command!relabel}` This command is used to label physical volumes. The full form of this command is: + + + + +.. code-block:: sh + :caption: label + + label storage= volume= slot= + + If you leave out any part, you will be prompted for it. The media type is automatically taken from the Storage resource definition that you supply. Once the necessary information is obtained, the Console program contacts the specified Storage daemon and requests that the Volume be labeled. If the Volume labeling is successful, the Console program will create a Volume record in the appropriate Pool. + + The Volume name is restricted to letters, numbers, and the special characters hyphen (**-**), underscore (**\_**), colon (**:**), and period (**.**). All other characters including a space are invalid. This restriction is to ensure good readability of Volume names to reduce operator errors. + + Please note, when labeling a blank tape, Bareos will get **read I/O error** when it attempts to ensure that the tape is not already labeled. If you wish to avoid getting these messages, please write an EOF mark on your tape before attempting to label it: + + + + + + + + mt rewind + mt weof + + + + + + The label command can fail for a number of reasons: + + #. The Volume name you specify is already in the Volume database. + + #. The Storage daemon has a tape or other Volume already mounted on the device, in which case you must **unmount** the device, insert a blank tape, then do the **label** command. + + #. The Volume in the device is already a Bareos labeled Volume. (Bareos will never relabel a Bareos labeled Volume unless it is recycled and you use the **relabel** command). + + #. There is no Volume in the drive. + + There are two ways to relabel a volume that already has a Bareos label. The brute force method is to write an end of file mark on the tape using the system **mt** program, something like the following: + + + + + + + + mt -f /dev/st0 rewind + mt -f /dev/st0 weof + + + + + + For a disk volume, you would manually delete the Volume. + + Then you use the **label** command to add a new label. However, this could leave traces of the old volume in the catalog. + + The preferable method to relabel a Volume is to first purge the volume, either automatically, or explicitly with the :strong:`purge` command, then use the :strong:`relabel` command described below. + + If your autochanger has barcode labels, you can label all the Volumes in your autochanger one after another by using the :strong:`label barcodes` command. For each tape in the changer containing a barcode, Bareos will mount the tape and then label it with the same name as the barcode. An appropriate Media record will also be created in the catalog. Any barcode that begins with the same characters as specified on the "CleaningPrefix=xxx" (default is "CLN") directive in the + Director’s Pool resource, will be treated as a cleaning tape, and will not be labeled. However, an entry for the cleaning tape will be created in the catalog. For example with: + + + + +.. code-block:: sh + :caption: Cleaning Tape + + Pool { + Name ... + Cleaning Prefix = "CLN" + } + + Any slot containing a barcode of CLNxxxx will be treated as a cleaning tape and will not be mounted. Note, the full form of the command is: + + + + +.. code-block:: sh + :caption: label + + label storage=xxx pool=yyy slots=1-5,10 barcodes + +list + +.. index:: + triple: Console; Command; list; + The list command lists the requested contents of the Catalog. The various fields of each record are listed on a single line. The various forms of the list command are: + + + + +.. code-block:: sh + :caption: list + + list jobs + list jobid= (list jobid id) + list ujobid= (list job with unique name) + list job= (list all jobs with "job-name") + list jobname= (same as above) + In the above, you can add "limit=nn" to limit the output to nn jobs. + list joblog jobid= (list job output if recorded in the catalog) + list jobmedia + list jobmedia jobid= + list jobmedia job= + list files jobid= + list files job= + list pools + list clients + list jobtotals + list volumes + list volumes jobid= + list volumes pool= + list volumes job= + list volume= + list nextvolume job= + list nextvol job= + list nextvol job= days=nnn + + What most of the above commands do should be more or less obvious. In general if you do not specify all the command line arguments, the command will prompt you for what is needed. + + The :strong:`list nextvol` command will print the Volume name to be used by the specified job. You should be aware that exactly what Volume will be used depends on a lot of factors including the time and what a prior job will do. It may fill a tape that is not full when you issue this command. As a consequence, this command will give you a good estimate of what Volume will be used but not a definitive answer. In addition, this command may have certain side effect because it + runs through the same algorithm as a job, which means it may automatically purge or recycle a Volume. By default, the job specified must run within the next two days or no volume will be found. You can, however, use the **days=nnn** specification to specify up to 50 days. For example, if on Friday, you want to see what Volume will be needed on Monday, for job MyJob, you would use :strong:`list}{nextvol job=MyJob days=3`. + + If you wish to add specialized commands that list the contents of the catalog, you can do so by adding them to the :file:`query.sql` file. However, this takes some knowledge of programming SQL. Please see the :strong:`query` command below for additional information. See below for listing the full contents of a catalog record with the :strong:`llist` command. + + As an example, the command **list pools** might produce the following output: + + + + +.. code-block:: sh + :caption: list pools + + *list pools + +------+---------+---------+---------+----------+-------------+ + | PoId | Name | NumVols | MaxVols | PoolType | LabelFormat | + +------+---------+---------+---------+----------+-------------+ + | 1 | Default | 0 | 0 | Backup | * | + | 2 | Recycle | 0 | 8 | Backup | File | + +------+---------+---------+---------+----------+-------------+ + + As mentioned above, the **list** command lists what is in the database. Some things are put into the database immediately when Bareos starts up, but in general, most things are put in only when they are first used, which is the case for a Client as with Job records, etc. + + Bareos should create a client record in the database the first time you run a job for that client. Doing a **status** will not cause a database record to be created. The client database record will be created whether or not the job fails, but it must at least start. When the Client is actually contacted, additional info from the client will be added to the client record (a "uname -a" output). + + If you want to see what Client resources you have available in your conf file, you use the Console command **show clients**. + +llist + +.. index:: + triple: Console; Command; llist; + The llist or "long list" command takes all the same arguments that the list command described above does. The difference is that the llist command list the full contents of each database record selected. It does so by listing the various fields of the record vertically, with one field per line. It is possible to produce a very large number of output lines with this command. + + If instead of the **list pools** as in the example above, you enter **llist pools** you might get the following output: + + + + +.. code-block:: sh + :caption: llist pools + + *llist pools + PoolId: 1 + Name: Default + NumVols: 0 + MaxVols: 0 + UseOnce: 0 + UseCatalog: 1 + AcceptAnyVolume: 1 + VolRetention: 1,296,000 + VolUseDuration: 86,400 + MaxVolJobs: 0 + MaxVolBytes: 0 + AutoPrune: 0 + Recycle: 1 + PoolType: Backup + LabelFormat: * + + PoolId: 2 + Name: Recycle + NumVols: 0 + MaxVols: 8 + UseOnce: 0 + UseCatalog: 1 + AcceptAnyVolume: 1 + VolRetention: 3,600 + VolUseDuration: 3,600 + MaxVolJobs: 1 + MaxVolBytes: 0 + AutoPrune: 0 + Recycle: 1 + PoolType: Backup + LabelFormat: File + +messages + +.. index:: + triple: Console; Command; messages; + This command causes any pending console messages to be immediately displayed. + +memory + +.. index:: + triple: Console; Command; memory; + Print current memory usage. + +mount + +.. index:: + triple: Console; Command; mount; + The mount command is used to get Bareos to read a volume on a physical device. It is a way to tell Bareos that you have mounted a tape and that Bareos should examine the tape. This command is normally used only after there was no Volume in a drive and Bareos requests you to mount a new Volume or when you have specifically unmounted a Volume with the :strong:`unmount` console command, which causes Bareos to close the drive. + If you have an autoloader, the mount command will not cause Bareos to operate the autoloader unless you specify a **slot** and possibly a **drive**. The various forms of the mount command are: + + + + +.. code-block:: sh + :caption: mount + + mount storage= [slot=] [drive=] + mount [jobid= | job=] + + If you have specified **Automatic Mount**:sup:`Sd`:sub:`Device` = **yes**, under most circumstances, Bareos will automatically access the Volume unless you have explicitly :strong:`unmount`ed it in the Console program. + +move + +.. index:: + triple: Console; Command; move; + The move command allows to move volumes between slots in an autochanger without having to leave the bconsole. + + To move a volume from slot 32 to slots 33, use: + + + + +.. code-block:: sh + :caption: move + + *move storage=TandbergT40 srcslots=32 dstslots=33 + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "slots" command. + Device "Drive-1" has 39 slots. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger "listall" command. + Connecting to Storage daemon TandbergT40 at bareos:9103 ... + 3306 Issuing autochanger transfer command. + 3308 Successfully transfered volume from slot 32 to 33. + +prune + +.. index:: + triple: Console; Command; prune; + + +.. _`ManualPruning`: ManualPruning The Prune command allows you to safely remove expired database records from Jobs, Volumes and Statistics. This command works only on the Catalog database and does not affect data written to Volumes. In all cases, the Prune command applies a retention period to the specified records. You can Prune expired File entries from Job records; you can Prune expired Job records from the database, and you can Prune + both expired Job and File records from specified Volumes. + + + + +.. code-block:: sh + :caption: prune + + prune files [client=] [pool=] [yes] | + jobs [client=] [pool=] [jobtype=] [yes] | + volume [=volume] [pool=] [yes] | + stats [yes] + + For a Volume to be pruned, the volume status must be **Full**, **Used** or **Append** otherwise the pruning will not take place. + +purge + +.. index:: + triple: Console; Command; purge; + + +.. _`bcommandPurge}` The Purge command will delete associated catalog database records from Jobs and Volumes without considering the retention period. This command can be dangerous because you can delete catalog records associated with current backups of files, and we recommend that you do not use it unless you know what you are doing. The permitted forms of :strong:`purge}{`: bcommandPurge` The Purge command will delete associated catalog database records from Jobs and Volumes without considering the retention period. This command can be dangerous because you can delete catalog records associated with current backups of files, and we recommend that you do not use it unless you know what you are doing. The permitted forms of :raw-latex:`\bcommand{purge}{ are: + + + + +.. code-block:: sh + :caption: purge + + purge [files [job= | jobid= | client= | volume=]] | + [jobs [client= | volume=]] | + [volume [=] [storage=] [pool=] [devicetype=] [drive=] [action=]] | + [quota [client=]] + + For the :strong:`purge` command to work on volume catalog database records the volume status must be **Append**, **Full**, **Used** or **Error**. + + The actual data written to the Volume will be unaffected by this command unless you are using the **Action On Purge**:sup:`Dir`:sub:`Pool` = **Truncate** option. + + To ask Bareos to truncate your **Purged** volumes, you need to use the following command in interactive mode: + + + + +.. code-block:: sh + :caption: purge example + + *purge volume action=truncate storage=File pool=Full + + However, normally you should use the :strong:`purge` command only to purge a volume from the catalog and use the :strong:`truncate` command to truncate the volume on the |bareosSd|. + +resolve + +.. index:: + triple: Console; Command; resolve; + In the configuration files, Addresses can (and normally should) be specified as DNS names. As the different components of Bareos will establish network connections to other Bareos components, it is important that DNS name resolution works on involved components and delivers the same results. The :strong:`resolve` command can be used to test DNS resolution of a given hostname on director, storage daemon or client. + + + + +.. code-block:: sh + :caption: resolve example + + *resolve www.bareos.com + bareos-dir resolves www.bareos.com to host[ipv4:84.44.166.242] + + *resolve client=client1-fd www.bareos.com + client1-fd resolves www.bareos.com to host[ipv4:84.44.166.242] + + *resolve storage=File www.bareos.com + bareos-sd resolves www.bareos.com to host[ipv4:84.44.166.242] + +query + +.. index:: + triple: Console; Command; query; + + +.. _`section-bcommandQuery`: section-bcommandQuery This command reads a predefined SQL query from the query file (the name and location of the query file is defined with the QueryFile resource record in the Director’s configuration file). You are prompted to select a query from the file, and possibly enter one or more parameters, then the command is submitted to the Catalog database SQL engine. + +quit +.. index:: + single: quit + This command terminates the console program. The console program sends the **quit** request to the Director and waits for acknowledgment. If the Director is busy doing a previous command for you that has not terminated, it may take some time. You may quit immediately by issuing the **.quit** command (i.e. quit preceded by a period). + +relabel + +.. index:: + triple: Console; Command; relabel; + This command is used to label physical volumes. + + The full form of this command is: + + + + +.. code-block:: sh + :caption: relabel + + relabel storage= oldvolume= volume= pool= [encrypt] + + If you leave out any part, you will be prompted for it. In order for the Volume (old-volume-name) to be relabeled, it must be in the catalog, and the volume status must be marked **Purged** or **Recycle**. This happens automatically as a result of applying retention periods or you may explicitly purge the volume using the :strong:`purge` command. + + Once the volume is physically relabeled, the old data previously written on the Volume is lost and cannot be recovered. + +release + +.. index:: + triple: Console; Command; release; + This command is used to cause the Storage daemon to release (and rewind) the current tape in the drive, and to re-read the Volume label the next time the tape is used. + + + + +.. code-block:: sh + :caption: release + + release storage= + + After a release command, the device is still kept open by Bareos (unless **Always Open**:sup:`Sd`:sub:`Device` = **no**) so it cannot be used by another program. However, with some tape drives, the operator can remove the current tape and to insert a different one, and when the next Job starts, Bareos will know to re-read the tape label to find out what tape is mounted. If you want to be able to use the drive with another program (e.g. :program:`mt`), you + must use the :strong:`unmount` command to cause Bareos to completely release (close) the device. + +reload + +.. index:: + triple: Console; Command; reload; + The reload command causes the Director to re-read its configuration file and apply the new values. The new values will take effect immediately for all new jobs. However, if you change schedules, be aware that the scheduler pre-schedules jobs up to two hours in advance, so any changes that are to take place during the next two hours may be delayed. Jobs that have already been scheduled to run (i.e. surpassed their requested start time) will + continue with the old values. New jobs will use the new values. Each time you issue a reload command while jobs are running, the prior config values will queued until all jobs that were running before issuing the reload terminate, at which time the old config values will be released from memory. The Directory permits keeping up to ten prior set of configurations before it will refuse a reload command. Once at least one old set of config values has been released it will again accept new + reload commands. + + While it is possible to reload the Director’s configuration on the fly, even while jobs are executing, this is a complex operation and not without side effects. Accordingly, if you have to reload the Director’s configuration while Bareos is running, it is advisable to restart the Director at the next convenient opportunity. + +rerun + +.. index:: + triple: Console; Command; rerun; + The rerun command allows you to re-run a Job with exactly the same setting as the original Job. In Bareos, the job configuration is often altered by job overrides. These overrides alter the configuration of the job just for one job run. If because of any reason, a job with overrides fails, it is not easy to restart a new job that is exactly configured as the job that failed. The whole job configuration is automatically set to the defaults + and it is hard to configure everything like it was. + + By using the rerun command, it is much easier to rerun a job exactly as it was configured. You only have to specify the JobId of the failed job. + + + + +.. code-block:: sh + :caption: rerun + + rerun jobid= since_jobid= days= hours= yes + + You can select the jobid(s) to rerun by using one of the selection criteria. Using jobid= will automatically select all jobs failed after and including the given jobid for rerunning. By using days= or hours=, you can select all failed jobids in the last number of days or number of hours respectively for rerunning. + +restore +.. index:: + single: Restore +.. index:: + pair: Console; File Selection + + +.. _`bcommandRestore`: bcommandRestore The restore command allows you to select one or more Jobs (JobIds) to be restored using various methods. Once the JobIds are selected, the File records for those Jobs are placed in an internal Bareos directory tree, and the restore enters a file selection mode that allows you to interactively walk up and down the + file tree selecting individual files to be restored. This mode is somewhat similar to the standard Unix **restore** program’s interactive file selection mode. + + + + +.. code-block:: sh + :caption: restore + + restore storage= client= + where= pool= fileset= + restoreclient= + restorejob= + select current all done + + Where **current**, if specified, tells the restore command to automatically select a restore to the most current backup. If not specified, you will be prompted. The **all** specification tells the restore command to restore all files. If it is not specified, you will be prompted for the files to restore. For details of the **restore** command, please see the :ref:`Restore Chapter ` of this manual. + + The client keyword initially specifies the client from which the backup was made and the client to which the restore will be make. However, if the restoreclient keyword is specified, then the restore is written to that client. + + The restore job rarely needs to be specified, as bareos installations commonly only have a single restore job configured. However, for certain cases, such as a varying list of RunScript specifications, multiple restore jobs may be configured. The restorejob argument allows the selection of one of these jobs. + + For more details, see the :ref:`Restore chapter `. + +run + +.. index:: + triple: Console; Command; run; + This command allows you to schedule jobs to be run immediately. + + The full form of the command is: + + + + +.. code-block:: sh + :caption: run + + run job= client= fileset= + level= storage= where= + when= pool= + pluginoptions= accurate= + comment= spooldata= priority= + jobid= catalog= migrationjob= backupclient= + backupformat= nextpool= since= + verifyjob= verifylist= migrationjob= + yes + + Any information that is needed but not specified will be listed for selection, and before starting the job, you will be prompted to accept, reject, or modify the parameters of the job to be run, unless you have specified **yes**, in which case the job will be immediately sent to the scheduler. + + If you wish to start a job at a later time, you can do so by setting the When time. Use the **mod** option and select **When** (no. 6). Then enter the desired start time in YYYY-MM-DD HH:MM:SS format. + + The spooldata argument of the run command cannot be modified through the menu and is only accessible by setting its value on the intial command line. If no spooldata flag is set, the job, storage or schedule flag is used. + +setbandwidth + +.. index:: + triple: Console; Command; setbandwidth; + This command (12.4.1) is used to limit the bandwidth of a running job or a client. + + + + +.. code-block:: sh + :caption: setbandwidth + + setbandwidth limit= [jobid= | client=] + +setdebug + + +.. _`bcommandSetdebug}` :raw-latex:`\index[general]{Console!Command!setdebug}` :raw-latex:`\index[general]{Debug!setdebug}` :raw-latex:`\index[general]{Debug!Windows}` :raw-latex:`\index[general]{Windows!Debug`: bcommandSetdebug}` :raw-latex:`\index[general]{Console!Command!setdebug}` :raw-latex:`\index[general]{Debug!setdebug}` :raw-latex:`\index[general]{Debug!Windows}` :raw-latex:`\index[general]{Windows!Debug This command is used to set the debug level in each daemon. The form of this command is: + + + + +.. code-block:: sh + :caption: setdebug + + setdebug level=nnn [trace=0/1 client= | dir | director | storage= | all] + + Each of the daemons normally has debug compiled into the program, but disabled. There are two ways to enable the debug output. + + One is to add the **-d nnn** option on the command line when starting the daemon. The **nnn** is the debug level, and generally anything between 50 and 200 is reasonable. The higher the number, the more output is produced. The output is written to standard output. + + The second way of getting debug output is to dynamically turn it on using the Console using the :program:`setdebug level=nnn` command. If none of the options are given, the command will prompt you. You can selectively turn on/off debugging in any or all the daemons (i.e. it is not necessary to specify all the components of the above command). + + If trace=1 is set, then tracing will be enabled, and the daemon will be placed in trace mode, which means that all debug output as set by the debug level will be directed to his trace file in the current directory of the daemon. When tracing, each debug output message is appended to the trace file. You must explicitly delete the file when you are done. + + + + +.. code-block:: sh + :caption: set Director debug level to 100 and get messages written to his trace file + + *setdebug level=100 trace=1 dir + level=100 trace=1 hangup=0 timestamp=0 tracefilename=/var/lib/bareos/bareos-dir.example.com.trace + +setip + + +.. _`bcommandSetIP}` :raw-latex:`\index[general]{Console!Command!setip`: bcommandSetIP}` :raw-latex:`\index[general]{Console!Command!setip Sets new client address – if authorized. + + A console is authorized to use the **SetIP** command only if it has a Console resource definition in both the Director and the Console. In addition, if the console name, provided on the **Name =** directive, must be the same as a Client name, the user of that console is permitted to use the **SetIP** command to change the Address directive in the Director’s client resource to the IP address of the Console. This permits portables or other machines using DHCP (non-fixed IP addresses) to + "notify" the Director of their current IP address. + +show + +.. index:: + triple: Console; Command; show; + The show command will list the Director’s resource records as defined in the Director’s configuration. This command is used mainly for debugging purposes by developers. The following keywords are accepted on the show command line: catalogs, clients, counters, devices, directors, filesets, jobs, messages, pools, schedules, storages, all, help. Please don’t confuse this command with the **list**, which displays the contents of the catalog. + +sqlquery + +.. index:: + triple: Console; Command; sqlquery; + The sqlquery command puts the Console program into SQL query mode where each line you enter is concatenated to the previous line until a semicolon (;) is seen. The semicolon terminates the command, which is then passed directly to the SQL database engine. When the output from the SQL engine is displayed, the formation of a new SQL command begins. To terminate SQL query mode and return to the Console command prompt, you enter a period (.) + in column 1. + + Using this command, you can query the SQL catalog database directly. Note you should really know what you are doing otherwise you could damage the catalog database. See the **query** command below for simpler and safer way of entering SQL queries. + + Depending on what database engine you are using (MySQL, PostgreSQL or SQLite), you will have somewhat different SQL commands available. For more detailed information, please refer to the MySQL, PostgreSQL or SQLite documentation. + +status + +.. index:: + triple: Console; Command; status; + + + This command will display the status of all components. For the director, it will display the next jobs that are scheduled during the next 24 hours as well as the status of currently running jobs. For the Storage Daemon, you will have drive status or autochanger content. The File Daemon will give you information about current jobs like average speed or file accounting. The full form of this command is: + + + + +.. code-block:: sh + :caption: status + + status [all | dir= | director | scheduler | schedule= | + client= | storage= slots | subscriptions] + + If you do a **status dir**, the console will list any currently running jobs, a summary of all jobs scheduled to be run in the next 24 hours, and a listing of the last ten terminated jobs with their statuses. The scheduled jobs summary will include the Volume name to be used. You should be aware of two things: 1. to obtain the volume name, the code goes through the same code that will be used when the job runs, but it does not do pruning nor recycling of Volumes; 2. The Volume listed is at + best a guess. The Volume actually used may be different because of the time difference (more durations may expire when the job runs) and another job could completely fill the Volume requiring a new one. + + In the Running Jobs listing, you may find the following types of information: + + + + +.. code-block:: sh + :caption: + + 2507 Catalog MatouVerify.2004-03-13_05.05.02 is waiting execution + 5349 Full CatalogBackup.2004-03-13_01.10.00 is waiting for higher + priority jobs to finish + 5348 Differe Minou.2004-03-13_01.05.09 is waiting on max Storage jobs + 5343 Full Rufus.2004-03-13_01.05.04 is running + + Looking at the above listing from bottom to top, obviously JobId 5343 (Rufus) is running. JobId 5348 (Minou) is waiting for JobId 5343 to finish because it is using the Storage resource, hence the "waiting on max Storage jobs". JobId 5349 has a lower priority than all the other jobs so it is waiting for higher priority jobs to finish, and finally, JobId 2507 (MatouVerify) is waiting because only one job can run at a time, hence it is simply "waiting execution" + + If you do a **status dir**, it will by default list the first occurrence of all jobs that are scheduled today and tomorrow. If you wish to see the jobs that are scheduled in the next three days (e.g. on Friday you want to see the first occurrence of what tapes are scheduled to be used on Friday, the weekend, and Monday), you can add the **days=3** option. Note, a **days=0** shows the first occurrence of jobs scheduled today only. If you have multiple run statements, the first occurrence of + each run statement for the job will be displayed for the period specified. + + If your job seems to be blocked, you can get a general idea of the problem by doing a **status dir**, but you can most often get a much more specific indication of the problem by doing a **status storage=xxx**. For example, on an idle test system, when I do **status storage=File**, I get: + + + + +.. code-block:: sh + :caption: status storage + + *status storage=File + Connecting to Storage daemon File at 192.168.68.112:8103 + + rufus-sd Version: 1.39.6 (24 March 2006) i686-pc-linux-gnu redhat (Stentz) + Daemon started 26-Mar-06 11:06, 0 Jobs run since started. + + Running Jobs: + No Jobs running. + ==== + + Jobs waiting to reserve a drive: + ==== + + Terminated Jobs: + JobId Level Files Bytes Status Finished Name + ====================================================================== + 59 Full 234 4,417,599 OK 15-Jan-06 11:54 usersave + ==== + + Device status: + Autochanger "DDS-4-changer" with devices: + "DDS-4" (/dev/nst0) + Device "DDS-4" (/dev/nst0) is mounted with Volume="TestVolume002" + Pool="*unknown*" + Slot 2 is loaded in drive 0. + Total Bytes Read=0 Blocks Read=0 Bytes/block=0 + Positioned at File=0 Block=0 + + Device "File" (/tmp) is not open. + ==== + + In Use Volume status: + ==== + + Now, what this tells me is that no jobs are running and that none of the devices are in use. Now, if I **unmount** the autochanger, which will not be used in this example, and then start a Job that uses the File device, the job will block. When I re-issue the status storage command, I get for the Device status: + + + + +.. code-block:: sh + :caption: status storage + + *status storage=File + ... + Device status: + Autochanger "DDS-4-changer" with devices: + "DDS-4" (/dev/nst0) + Device "DDS-4" (/dev/nst0) is not open. + Device is BLOCKED. User unmounted. + Drive 0 is not loaded. + + Device "File" (/tmp) is not open. + Device is BLOCKED waiting for media. + ==== + ... + + Now, here it should be clear that if a job were running that wanted to use the Autochanger (with two devices), it would block because the user unmounted the device. The real problem for the Job I started using the "File" device is that the device is blocked waiting for media – that is Bareos needs you to label a Volume. + + The command :strong:`status scheduler` (12.4.4) can be used to check when a certain schedule will trigger. This gives more information than :strong:`status director`. + + Called without parameters, :strong:`status scheduler` shows a preview for all schedules for the next 14 days. It first shows a list of the known schedules and the jobs that will be triggered by these jobs, and next, a table with date (including weekday), schedule name and applied overrides is displayed: + + + + +.. code-block:: sh + :caption: status scheduler + + *status scheduler + Scheduler Jobs: + + Schedule Jobs Triggered + =========================================================== + WeeklyCycle + BackupClient1 + + WeeklyCycleAfterBackup + BackupCatalog + + ==== + + Scheduler Preview for 14 days: + + Date Schedule Overrides + ============================================================== + Di 04-Jun-2013 21:00 WeeklyCycle Level=Incremental + Di 04-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Mi 05-Jun-2013 21:00 WeeklyCycle Level=Incremental + Mi 05-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Do 06-Jun-2013 21:00 WeeklyCycle Level=Incremental + Do 06-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Fr 07-Jun-2013 21:00 WeeklyCycle Level=Incremental + Fr 07-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Sa 08-Jun-2013 21:00 WeeklyCycle Level=Differential + Mo 10-Jun-2013 21:00 WeeklyCycle Level=Incremental + Mo 10-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Di 11-Jun-2013 21:00 WeeklyCycle Level=Incremental + Di 11-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Mi 12-Jun-2013 21:00 WeeklyCycle Level=Incremental + Mi 12-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Do 13-Jun-2013 21:00 WeeklyCycle Level=Incremental + Do 13-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Fr 14-Jun-2013 21:00 WeeklyCycle Level=Incremental + Fr 14-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + Sa 15-Jun-2013 21:00 WeeklyCycle Level=Differential + Mo 17-Jun-2013 21:00 WeeklyCycle Level=Incremental + Mo 17-Jun-2013 21:10 WeeklyCycleAfterBackup Level=Full + ==== + + :strong:`status scheduler` accepts the following parameters: + + client=clientname + shows only the schedules that affect the given client. + + job=jobname + shows only the schedules that affect the given job. + + schedule=schedulename + shows only the given schedule. + + days=number + of days shows only the number of days in the scheduler preview. Positive numbers show the future, negative numbers show the past. days can be combined with the other selection criteria. days= can be combined with the other selection criteria. + + In case you are running a maintained version of Bareos, the command :strong:`status subscriptions` (12.4.4) can help you to keep the overview over the subscriptions that are used. + + To enable this functionality, just add the configuration **Subscriptions**:sup:`Dir`:sub:`Director` directive and specify the number of subscribed clients, for example: + + + + +.. code-block:: sh + :caption: enable subscription check + + Director { + ... + Subscriptions = 50 + } + + Using the console command :strong:`status subscriptions`, the status of the subscriptions can be checked any time interactively: + + + + +.. code-block:: sh + :caption: status subscriptions + + *status subscriptions + Ok: available subscriptions: 8 (42/50) (used/total) + + Also, the number of subscriptions is checked after every job. If the number of clients is bigger than the configured limit, a Job warning is created a message like this: + + + + +.. code-block:: sh + :caption: subscriptions warning + + JobId 7: Warning: Subscriptions exceeded: (used/total) (51/50) + + Please note: Nothing else than the warning is issued, no enforcement on backup, restore or any other operation will happen. + + Setting the value for **Subscriptions**:sup:`Dir`:sub:`Director` = **0** disables this functionality: + + + + +.. code-block:: sh + :caption: disable subscription check + + Director { + ... + Subscriptions = 0 + } + + Not configuring the directive at all also disables it, as the default value for the Subscriptions directive is zero. + +time + +.. index:: + triple: Console; Command; time; + The time command shows the current date, time and weekday. + +trace + +.. index:: + triple: Console; Command; trace; + Turn on/off trace to file. + +truncate +.. index:: + pair: Disk; Freeing disk space +.. index:: + pair: Disk; Freeing disk space + + +.. _`bcommandTruncate`: bcommandTruncate + + If the status of a volume is **Purged**, it normally still contains data, even so it can not easily be accessed. + + + + +.. code-block:: sh + :caption: truncate + + truncate volstatus=Purged [storage=] [pool=] [volume=] [yes] + + When using a disk volume (and other volume types also) the volume file still resides on the |bareosSd|. If you want to reclaim disk space, you can use the :strong:`truncate}{volstatus=Purged` command. When used on a volume, it rewrites the header and by this frees the rest of the disk space. + + If the volume you want to get rid of has not the **Purged** status, you first have to use the :strong:`prune volume` or even the :strong:`purge volume` command to free the volume from all remaining jobs. + + This command is available since Bareos 16.2.5. + +umount + +.. index:: + triple: Console; Command; umount; + Alias for :strong:`unmount`. + +unmount + +.. index:: + triple: Console; Command; unmount; + This command causes the indicated Bareos Storage daemon to unmount the specified device. The forms of the command are the same as the mount command: + + + + +.. code-block:: sh + :caption: unmount + + unmount storage= [drive=] + unmount [jobid= | job=] + + Once you unmount a storage device, Bareos will no longer be able to use it until you issue a mount command for that device. If Bareos needs to access that device, it will block and issue mount requests periodically to the operator. + + If the device you are unmounting is an autochanger, it will unload the drive you have specified on the command line. If no drive is specified, it will assume drive 1. + + In most cases, it is preferable to use the :strong:`release` instead. + +update + +.. index:: + triple: Console; Command; update; + + +.. _`UpdateCommand`: UpdateCommand This command will update the catalog for either a specific Pool record, a Volume record, or the Slots in an autochanger with barcode capability. In the case of updating a Pool record, the new information will be automatically taken from the corresponding Director’s configuration resource record. It can be used to increase the maximum number of volumes permitted or to set a maximum number of volumes. The + following main keywords may be specified: + + - volume + + - pool + + - slots + + - iobid + + - stats + + In the case of updating a Volume (:strong:`update volume`), you will be prompted for which value you wish to change. The following Volume parameters may be changed: + + + + + + + + Volume Status + Volume Retention Period + Volume Use Duration + Maximum Volume Jobs + Maximum Volume Files + Maximum Volume Bytes + Recycle Flag + Recycle Pool + Slot + InChanger Flag + Pool + Volume Files + Volume from Pool + All Volumes from Pool + All Volumes from all Pools + + + + + + For slots :strong:`update slots`, Bareos will obtain a list of slots and their barcodes from the Storage daemon, and for each barcode found, it will automatically update the slot in the catalog Media record to correspond to the new value. This is very useful if you have moved cassettes in the magazine, or if you have removed the magazine and inserted a different one. As the slot of each Volume is updated, the InChanger flag for that Volume will also be set, and any other + Volumes in the Pool that were last mounted on the same Storage device will have their InChanger flag turned off. This permits Bareos to know what magazine (tape holder) is currently in the autochanger. + + If you do not have barcodes, you can accomplish the same thing by using the :strong:`update slots scan` command. The :option:`scan` keyword tells Bareos to physically mount each tape and to read its VolumeName. + + For Pool :strong:`update pool`, Bareos will move the Volume record from its existing pool to the pool specified. + + For **Volume from Pool**, **All Volumes from Pool** and **All Volumes from all Pools**, the following values are updated from the Pool record: Recycle, RecyclePool, VolRetention, VolUseDuration, MaxVolJobs, MaxVolFiles, and MaxVolBytes. + + For updating the statistics, use :strong:`updates stats`, see :ref:`section-JobStatistics`. + + The full form of the update command with all command line arguments is: + + + + +.. code-block:: sh + :caption: update + + update volume= [volstatus=] + [volretention=] [pool=] + [recycle=] [slot=] [inchanger=] | + pool= [maxvolbytes=] [maxvolfiles=] + [maxvoljobs=][enabled=] [recyclepool=] + [actiononpurge=] | + slots [storage=] [scan] | + jobid= [jobname=] [starttime=] + [client=] [filesetid=] + [jobtype=] | + stats [days=] + +use + +.. index:: + triple: Console; Command; use; + This command allows you to specify which Catalog database to use. Normally, you will be using only one database so this will be done automatically. In the case that you are using more than one database, you can use this command to switch from one to another. + + + + +.. code-block:: sh + :caption: use + + use [catalog=] + +var + + +.. _`var}` :raw-latex:`\index[general]{Console!Command!var}` This command takes a string or quoted string and does variable expansion on it mostly the same way variable expansion is done on the **Label Format**:sup:`Dir`:sub:`Pool` string. The difference between the :strong:`var}{`: var` :raw-latex:`\index[general]{Console!Command!var}` This command takes a string or quoted string and does variable expansion on it mostly the same way variable expansion is done on the **Label Format**:sup:`Dir`:sub:`Pool` string. The difference between the :raw-latex:`\bcommand{var}{ command and the actual **Label Format**:sup:`Dir`:sub:`Pool` process is that during the var command, no job is running so dummy values are + used in place of Job specific variables. + +version + +.. index:: + triple: Console; Command; version; + The command prints the Director’s version. + +wait + +.. index:: + triple: Console; Command; wait; + The wait command causes the Director to pause until there are no jobs running. This command is useful in a batch situation such as regression testing where you wish to start a job and wait until that job completes before continuing. This command now has the following options: + + + + +.. code-block:: sh + :caption: wait + + wait [jobid=] [jobuid=] [job=] + + If specified with a specific JobId, ... the wait command will wait for that particular job to terminate before continuing. + +.. _dotcommands: + +Special dot (.) Commands +~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Console; Command; . commands; + + +There is a list of commands that are prefixed with a period (.). These commands are intended to be used either by batch programs or graphical user interface front-ends. They are not normally used by interactive users. For details, see :raw-latex:`\bareosDeveloperGuideDotCommands`. + +.. _atcommands: + +Special At (@) Commands +~~~~~~~~~~~~~~~~~~~~~~~ + +Normally, all commands entered to the Console program are immediately forwarded to the Director, which may be on another machine, to be executed. However, there is a small list of **at** commands, all beginning with an at character (@), that will not be sent to the Director, but rather interpreted by the Console program directly. Note, these commands are implemented only in the TTY console program and not in the Bat Console. These commands are: + +@input + :raw-latex:`\index[general]{Console!Command!\at{}input {\textless}filename{\textgreater}}` Read and execute the commands contained in the file specified. + +@output + :raw-latex:`\index[general]{Console!Command!\at{}output {\textless}filename{\textgreater} {\textless}w{\textbar}a{\textgreater}}` Send all following output to the filename specified either overwriting the file (w) or appending to the file (a). To redirect the output to the terminal, simply enter **@output** without a filename specification. WARNING: be careful not to overwrite a valid file. A typical example during a regression test might be: + + + + + + + + @output /dev/null + commands ... + @output + + + + + +@tee + :raw-latex:`\index[general]{Console!Command!\at{}tee {\textless}filename{\textgreater} {\textless}w{\textbar}a{\textgreater}}` Send all subsequent output to both the specified file and the terminal. It is turned off by specifying **@tee** or **@output** without a filename. + +@sleep + :raw-latex:`\index[general]{Console!Command!\at{}sleep {\textless}seconds{\textgreater}}` Sleep the specified number of seconds. + +@time + :raw-latex:`\index[general]{Console!Command!\at{}time}` Print the current time and date. + +@version + :raw-latex:`\index[general]{Console!Command!\at{}version}` Print the console’s version. + +@quit + :raw-latex:`\index[general]{Console!Command!\at{}quit}` quit + +@exit + :raw-latex:`\index[general]{Console!Command!\at{}exit}` quit + +@# anything + :raw-latex:`\index[general]{Console!Command!\at{}\# anything}` Comment + +@help + :raw-latex:`\index[general]{Console!Command!\at{}help}` Get the list of every special @ commands. + +@separator + :raw-latex:`\index[general]{Console!Command!\at{}separator}` When using bconsole with readline, you can set the command separator to one of those characters to write commands who require multiple input on one line, or to put multiple commands on a single line. + + + + !$%&'()*+,-/:;<>?[]^`{|}~ + + Note, if you use a semicolon (;) as a separator character, which is common, you will not be able to use the **sql** command, which requires each command to be terminated by a semicolon. + +Adding Volumes to a Pool +------------------------ + +.. index:: + pair: Console; Adding a Volume to a Pool + + +.. raw:: latex + + \TODO{move to another chapter} + +If you have used the **label** command to label a Volume, it will be automatically added to the Pool, and you will not need to add any media to the pool. + +Alternatively, you may choose to add a number of Volumes to the pool without labeling them. At a later time when the Volume is requested by **Bareos** you will need to label it. + +Before adding a volume, you must know the following information: + +#. The name of the Pool (normally "Default") + +#. The Media Type as specified in the Storage Resource in the Director’s configuration file (e.g. "DLT8000") + +#. The number and names of the Volumes you wish to create. + +For example, to add media to a Pool, you would issue the following commands to the console program: + +.. raw:: latex + + + + + + *add + Enter name of Pool to add Volumes to: Default + Enter the Media Type: DLT8000 + Enter number of Media volumes to create. Max=1000: 10 + Enter base volume name: Save + Enter the starting number: 1 + 10 Volumes created in pool Default + * + +.. raw:: latex + + + +To see what you have added, enter: + +.. raw:: latex + + + + + + *list media pool=Default + +-------+----------+---------+---------+-------+------------------+ + | MedId | VolumeNa | MediaTyp| VolStat | Bytes | LastWritten | + +-------+----------+---------+---------+-------+------------------+ + | 11 | Save0001 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 12 | Save0002 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 13 | Save0003 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 14 | Save0004 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 15 | Save0005 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 16 | Save0006 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 17 | Save0007 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 18 | Save0008 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 19 | Save0009 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + | 20 | Save0010 | DLT8000 | Append | 0 | 0000-00-00 00:00 | + +-------+----------+---------+---------+-------+------------------+ + * + +.. raw:: latex + + + +Notice that the console program automatically appended a number to the base Volume name that you specify (Save in this case). If you don’t want it to append a number, you can simply answer 0 (zero) to the question "Enter number of Media volumes to create. Max=1000:", and in this case, it will create a single Volume with the exact name you specify. diff --git a/docs/manuals/en/new_main_reference/source/chapter16/restore.rst b/docs/manuals/en/new_main_reference/source/chapter16/restore.rst new file mode 100644 index 00000000000..233feb4027a --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter16/restore.rst @@ -0,0 +1,785 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _RestoreChapter: + +The Restore Command +=================== + +.. index:: + single: Restore + + +General +------- + +Below, we will discuss restoring files with the Console **restore** command, which is the recommended way of doing restoring files. It is not possible to restore files by automatically starting a job as you do with Backup, Verify, ... jobs. However, in addition to the console restore command, there is a standalone program named **bextract**, which also permits restoring files. For more information on this program, please see the :ref:`Bareos Utility Programs ` chapter of +this manual. We don’t particularly recommend the **bextract** program because it lacks many of the features of the normal Bareos restore, such as the ability to restore Win32 files to Unix systems, and the ability to restore access control lists (ACL). As a consequence, we recommend, wherever possible to use Bareos itself for restores as described below. + +You may also want to look at the **bls** program in the same chapter, which allows you to list the contents of your Volumes. Finally, if you have an old Volume that is no longer in the catalog, you can restore the catalog entries using the program named **bscan**, documented in the same :ref:`Bareos Utility Programs ` chapter. + +In general, to restore a file or a set of files, you must run a **restore** job. That is a job with **Type = Restore**. As a consequence, you will need a predefined **restore** job in your **bareos-dir.conf** (Director’s config) file. The exact parameters (Client, FileSet, ...) that you define are not important as you can either modify them manually before running the job or if you use the **restore** command, explained below, Bareos will automatically set them for you. In fact, you can no +longer simply run a restore job. You must use the restore command. + +Since Bareos is a network backup program, you must be aware that when you restore files, it is up to you to ensure that you or Bareos have selected the correct Client and the correct hard disk location for restoring those files. **Bareos** will quite willingly backup client A, and restore it by sending the files to a different directory on client B. Normally, you will want to avoid this, but assuming the operating systems are not too different in their file structures, this should work perfectly +well, if so desired. By default, Bareos will restore data to the same Client that was backed up, and those data will be restored not to the original places but to **/tmp/bareos-restores**. This is configured in the default restore command resource in bareos-dir.conf. You may modify any of these defaults when the restore command prompts you to run the job by selecting the **mod** option. + +The Restore Command +------------------- + + +.. index:: + triple: Console; Command; restore; + + +Since Bareos maintains a catalog of your files and on which Volumes (disk or tape), they are stored, it can do most of the bookkeeping work, allowing you simply to specify what kind of restore you want (current, before a particular date), and what files to restore. Bareos will then do the rest. + +This is accomplished using the **restore** command in the Console. First you select the kind of restore you want, then the JobIds are selected, the File records for those Jobs are placed in an internal Bareos directory tree, and the restore enters a file selection mode that allows you to interactively walk up and down the file tree selecting individual files to be restored. This mode is somewhat similar to the standard Unix **restore** program’s interactive file selection mode. + +If a Job’s file records have been pruned from the catalog, the **restore** command will be unable to find any files to restore. Bareos will ask if you want to restore all of them or if you want to use a regular expression to restore only a selection while reading media. See :ref:`FileRegex option ` and below for more details on this. + +Within the Console program, after entering the **restore** command, you are presented with the following selection prompt: + + + + +.. code-block:: sh + :caption: restore + + * restore + First you select one or more JobIds that contain files + to be restored. You will be presented several methods + of specifying the JobIds. Then you will be allowed to + select which files from those JobIds are to be restored. + + To select the JobIds, you have the following choices: + 1: List last 20 Jobs run + 2: List Jobs where a given File is saved + 3: Enter list of comma separated JobIds to select + 4: Enter SQL list command + 5: Select the most recent backup for a client + 6: Select backup for a client before a specified time + 7: Enter a list of files to restore + 8: Enter a list of files to restore before a specified time + 9: Find the JobIds of the most recent backup for a client + 10: Find the JobIds for a backup for a client before a specified time + 11: Enter a list of directories to restore for found JobIds + 12: Select full restore to a specified Job date + 13: Cancel + Select item- (1-13): + +There are a lot of options, and as a point of reference, most people will want to select item 5 (the most recent backup for a client). The details of the above options are: + +- Item 1 will list the last 20 jobs run. If you find the Job you want, you can then select item 3 and enter its JobId(s). + +- Item 2 will list all the Jobs where a specified file is saved. If you find the Job you want, you can then select item 3 and enter the JobId. + +- Item 3 allows you the enter a list of comma separated JobIds whose files will be put into the directory tree. You may then select which files from those JobIds to restore. Normally, you would use this option if you have a particular version of a file that you want to restore and you know its JobId. The most common options (5 and 6) will not select a job that did not terminate normally, so if you know a file is backed up by a Job that failed (possibly because of a system crash), you can access + it through this option by specifying the JobId. + +- Item 4 allows you to enter any arbitrary SQL command. This is probably the most primitive way of finding the desired JobIds, but at the same time, the most flexible. Once you have found the JobId(s), you can select item 3 and enter them. + +- Item 5 will automatically select the most recent Full backup and all subsequent incremental and differential backups for a specified Client. These are the Jobs and Files which, if reloaded, will restore your system to the most current saved state. It automatically enters the JobIds found into the directory tree in an optimal way such that only the most recent copy of any particular file found in the set of Jobs will be restored. This is probably the most convenient of all the above options to + use if you wish to restore a selected Client to its most recent state. + + There are two important things to note. First, this automatic selection will never select a job that failed (terminated with an error status). If you have such a job and want to recover one or more files from it, you will need to explicitly enter the JobId in item 3, then choose the files to restore. + + If some of the Jobs that are needed to do the restore have had their File records pruned, the restore will be incomplete. Bareos currently does not correctly detect this condition. You can however, check for this by looking carefully at the list of Jobs that Bareos selects and prints. If you find Jobs with the JobFiles column set to zero, when files should have been backed up, then you should expect problems. + + If all the File records have been pruned, Bareos will realize that there are no file records in any of the JobIds chosen and will inform you. It will then propose doing a full restore (non-selective) of those JobIds. This is possible because Bareos still knows where the beginning of the Job data is on the Volumes, even if it does not know where particular files are located or what their names are. + +- Item 6 allows you to specify a date and time, after which Bareos will automatically select the most recent Full backup and all subsequent incremental and differential backups that started before the specified date and time. + +- Item 7 allows you to specify one or more filenames (complete path required) to be restored. Each filename is entered one at a time or if you prefix a filename with the less-than symbol (<) Bareos will read that file and assume it is a list of filenames to be restored. If you prefix the filename with a question mark (?), then the filename will be interpreted as an SQL table name, and Bareos will include the rows of that table in the list to be restored. The table must contain the JobId in the + first column and the FileIndex in the second column. This table feature is intended for external programs that want to build their own list of files to be restored. The filename entry mode is terminated by entering a blank line. + +- Item 8 allows you to specify a date and time before entering the filenames. See Item 7 above for more details. + +- Item 9 allows you find the JobIds of the most recent backup for a client. This is much like option 5 (it uses the same code), but those JobIds are retained internally as if you had entered them manually. You may then select item 11 (see below) to restore one or more directories. + +- Item 10 is the same as item 9, except that it allows you to enter a before date (as with item 6). These JobIds will then be retained internally. + +.. index:: + single: Restore Directories + + +- Item 11 allows you to enter a list of JobIds from which you can select directories to be restored. The list of JobIds can have been previously created by using either item 9 or 10 on the menu. You may then enter a full path to a directory name or a filename preceded by a less than sign (<). The filename should contain a list of directories to be restored. All files in those directories will be restored, but if the directory contains subdirectories, nothing will be restored in the subdirectory + unless you explicitly enter its name. + +- Item 12 is a full restore to a specified job date. + +- Item 13 allows you to cancel the restore command. + +As an example, suppose that we select item 5 (restore to most recent state). If you have not specified a client=xxx on the command line, it it will then ask for the desired Client, which on my system, will print all the Clients found in the database as follows: + + + + +.. code-block:: sh + :caption: restore: select client + + Select item- (1-13): 5 + Defined clients: + 1: Rufus + 2: Matou + 3: Polymatou + 4: Minimatou + 5: Minou + 6: MatouVerify + 7: PmatouVerify + 8: RufusVerify + 9: Watchdog + Select Client (File daemon) resource (1-9): 1 + +The listed clients are only examples, yours will look differently. If you have only one Client, it will be automatically selected. In this example, I enter 1 for **Rufus** to select the Client. Then Bareos needs to know what FileSet is to be restored, so it prompts with: + +.. raw:: latex + + + + + + The defined FileSet resources are: + 1: Full Set + 2: Other Files + Select FileSet resource (1-2): + +.. raw:: latex + + + +If you have only one FileSet defined for the Client, it will be selected automatically. I choose item 1, which is my full backup. Normally, you will only have a single FileSet for each Job, and if your machines are similar (all Linux) you may only have one FileSet for all your Clients. + +At this point, Bareos has all the information it needs to find the most recent set of backups. It will then query the database, which may take a bit of time, and it will come up with something like the following. Note, some of the columns are truncated here for presentation: + +.. raw:: latex + + + + + + +-------+------+----------+-------------+-------------+------+-------+------------+ + | JobId | Levl | JobFiles | StartTime | VolumeName | File | SesId |VolSesTime | + +-------+------+----------+-------------+-------------+------+-------+------------+ + | 1,792 | F | 128,374 | 08-03 01:58 | DLT-19Jul02 | 67 | 18 | 1028042998 | + | 1,792 | F | 128,374 | 08-03 01:58 | DLT-04Aug02 | 0 | 18 | 1028042998 | + | 1,797 | I | 254 | 08-04 13:53 | DLT-04Aug02 | 5 | 23 | 1028042998 | + | 1,798 | I | 15 | 08-05 01:05 | DLT-04Aug02 | 6 | 24 | 1028042998 | + +-------+------+----------+-------------+-------------+------+-------+------------+ + You have selected the following JobId: 1792,1792,1797 + Building directory tree for JobId 1792 ... + Building directory tree for JobId 1797 ... + Building directory tree for JobId 1798 ... + cwd is: / + $ + +.. raw:: latex + + + +Depending on the number of **JobFiles** for each JobId, the :emphasis:`Building directory tree ...` can take a bit of time. If you notice ath all the JobFiles are zero, your Files have probably been pruned and you will not be able to select any individual files – it will be restore everything or nothing. + +In our example, Bareos found four Jobs that comprise the most recent backup of the specified Client and FileSet. Two of the Jobs have the same JobId because that Job wrote on two different Volumes. The third Job was an incremental backup to the previous Full backup, and it only saved 254 Files compared to 128,374 for the Full backup. The fourth Job was also an incremental backup that saved 15 files. + +Next Bareos entered those Jobs into the directory tree, with no files marked to be restored as a default, tells you how many files are in the tree, and tells you that the current working directory (**cwd**) is /. Finally, Bareos prompts with the dollar sign ($) to indicate that you may enter commands to move around the directory tree and to select files. + +If you want all the files to automatically be marked when the directory tree is built, you could have entered the command **restore all**, or at the $ prompt, you can simply enter **mark \***. + +Instead of choosing item 5 on the first menu (Select the most recent backup for a client), if we had chosen item 3 (Enter list of JobIds to select) and we had entered the JobIds **1792,1797,1798** we would have arrived at the same point. + +One point to note, if you are manually entering JobIds, is that you must enter them in the order they were run (generally in increasing JobId order). If you enter them out of order and the same file was saved in two or more of the Jobs, you may end up with an old version of that file (i.e. not the most recent). + +Directly entering the JobIds can also permit you to recover data from a Job that wrote files to tape but that terminated with an error status. + +While in file selection mode, you can enter **help** or a question mark (?) to produce a summary of the available commands: + +.. raw:: latex + + + + + + Command Description + ======= =========== + cd change current directory + count count marked files in and below the cd + dir long list current directory, wildcards allowed + done leave file selection mode + estimate estimate restore size + exit same as done command + find find files, wildcards allowed + help print help + ls list current directory, wildcards allowed + lsmark list the marked files in and below the cd + mark mark dir/file to be restored recursively in dirs + markdir mark directory name to be restored (no files) + pwd print current working directory + unmark unmark dir/file to be restored recursively in dir + unmarkdir unmark directory name only no recursion + quit quit and do not do restore + ? print help + +.. raw:: latex + + + +As a default no files have been selected for restore (unless you added **all** to the command line. If you want to restore everything, at this point, you should enter **mark \***, and then **done** and Bareos will write the bootstrap records to a file and request your approval to start a restore job. + +If you do not enter the above mentioned **mark \*** command, you will start with an empty state. Now you can simply start looking at the tree and **mark** particular files or directories you want restored. It is easy to make a mistake in specifying a file to mark or unmark, and Bareos’s error handling is not perfect, so please check your work by using the **ls** or **dir** commands to see what files are actually selected. Any selected file has its name preceded by an asterisk. + +To check what is marked or not marked, enter the **count** command, which displays: + +.. raw:: latex + + + + + + 128401 total files. 128401 marked to be restored. + +.. raw:: latex + + + +Each of the above commands will be described in more detail in the next section. We continue with the above example, having accepted to restore all files as Bareos set by default. On entering the **done** command, Bareos prints: + +.. raw:: latex + + + + + + Run Restore job + JobName: RestoreFiles + Bootstrap: /var/lib/bareos/client1.restore.3.bsr + Where: /tmp/bareos-restores + Replace: Always + FileSet: Full Set + Backup Client: client1 + Restore Client: client1 + Format: Native + Storage: File + When: 2013-06-28 13:30:08 + Catalog: MyCatalog + Priority: 10 + Plugin Options: *None* + OK to run? (yes/mod/no): + +.. raw:: latex + + + +Please examine each of the items very carefully to make sure that they are correct. In particular, look at **Where**, which tells you where in the directory structure the files will be restored, and **Client**, which tells you which client will receive the files. Note that by default the Client which will receive the files is the Client that was backed up. These items will not always be completed with the correct values depending on which of the restore options you chose. You can change any of +these default items by entering **mod** and responding to the prompts. + +The above assumes that you have defined a **Restore** Job resource in your Director’s configuration file. Normally, you will only need one Restore Job resource definition because by its nature, restoring is a manual operation, and using the Console interface, you will be able to modify the Restore Job to do what you want. + +An example Restore Job resource definition is given below. + +Returning to the above example, you should verify that the Client name is correct before running the Job. However, you may want to modify some of the parameters of the restore job. For example, in addition to checking the Client it is wise to check that the Storage device chosen by Bareos is indeed correct. Although the **FileSet** is shown, it will be ignored in restore. The restore will choose the files to be restored either by reading the **Bootstrap** file, or if not specified, it will +restore all files associated with the specified backup **JobId** (i.e. the JobId of the Job that originally backed up the files). + +Finally before running the job, please note that the default location for restoring files is **not** their original locations, but rather the directory **/tmp/bareos-restores**. You can change this default by modifying your **bareos-dir.conf** file, or you can modify it using the **mod** option. If you want to restore the files to their original location, you must have **Where** set to nothing or to the root, i.e. **/**. + +If you now enter **yes**, Bareos will run the restore Job. + +Selecting Files by Filename +--------------------------- + +.. index:: + pair: Restore; by filename + + +If you have a small number of files to restore, and you know the filenames, you can either put the list of filenames in a file to be read by Bareos, or you can enter the names one at a time. The filenames must include the full path and filename. No wild cards are used. + +To enter the files, after the **restore**, you select item number 7 from the prompt list: + + + + +.. code-block:: sh + :caption: restore list of files + + * restore + First you select one or more JobIds that contain files + to be restored. You will be presented several methods + of specifying the JobIds. Then you will be allowed to + select which files from those JobIds are to be restored. + + To select the JobIds, you have the following choices: + 1: List last 20 Jobs run + 2: List Jobs where a given File is saved + 3: Enter list of comma separated JobIds to select + 4: Enter SQL list command + 5: Select the most recent backup for a client + 6: Select backup for a client before a specified time + 7: Enter a list of files to restore + 8: Enter a list of files to restore before a specified time + 9: Find the JobIds of the most recent backup for a client + 10: Find the JobIds for a backup for a client before a specified time + 11: Enter a list of directories to restore for found JobIds + 12: Select full restore to a specified Job date + 13: Cancel + Select item- (1-13): 7 + +which then prompts you for the client name: + +.. raw:: latex + + + + + + Defined Clients: + 1: client1 + 2: Tibs + 3: Rufus + Select the Client (1-3): 3 + +.. raw:: latex + + + +Of course, your client list will be different, and if you have only one client, it will be automatically selected. And finally, Bareos requests you to enter a filename: + +.. raw:: latex + + + + + + Enter filename: + +.. raw:: latex + + + +At this point, you can enter the full path and filename + +.. raw:: latex + + + + + + Enter filename: /etc/resolv.conf + Enter filename: + +.. raw:: latex + + + +as you can see, it took the filename. If Bareos cannot find a copy of the file, it prints the following: + +.. raw:: latex + + + + + + Enter filename: junk filename + No database record found for: junk filename + Enter filename: + +.. raw:: latex + + + +If you want Bareos to read the filenames from a file, you simply precede the filename with a less-than symbol (<). + +It is possible to automate the selection by file by putting your list of files in say **/tmp/file-list**, then using the following command: + +.. raw:: latex + + + + + + restore client=client1 file=` section. + +- **restorejob=jobname** – Pre-chooses a restore job. Bareos can be configured with multiple restore jobs ("Type = Restore" in the job definition). This allows the specification of different restore properties, including a set of RunScripts. When more than one job of this type is configured, during restore, Bareos will ask for a user selection interactively, or use the given restorejob. + +Using File Relocation +--------------------- + +.. index:: + pair: File Relocation; using + + +.. _`filerelocation}` :raw-latex:`\label{restorefilerelocation`: filerelocation}` :raw-latex:`\label{restorefilerelocation + +Introduction +~~~~~~~~~~~~ + +The **where=** option is simple, but not very powerful. With file relocation, Bareos can restore a file to the same directory, but with a different name, or in an other directory without recreating the full path. + +You can also do filename and path manipulations, such as adding a suffix to all your files, renaming files or directories, etc. Theses options will overwrite **where=** option. + +For example, many users use OS snapshot features so that file ``/home/eric/mbox`` will be backed up from the directory ``/.snap/home/eric/mbox``, which can complicate restores. If you use **where=/tmp**, the file will be restored to ``/tmp/.snap/home/eric/mbox`` and you will have to move the file to ``/home/eric/mbox.bkp`` by hand. + +However, case, you could use the **strip_prefix=/.snap** and **add_suffix=.bkp** options and Bareos will restore the file to its original location – that is ``/home/eric/mbox``. + +To use this feature, there are command line options as described in the :ref:`restore section ` of this manual; you can modify your restore job before running it; or you can add options to your restore job in as described in **Strip Prefix**:sup:`Dir`:sub:`Job` and **Add Prefix**:sup:`Dir`:sub:`Job` . + + + + Parameters to modify: + 1: Level + 2: Storage + ... + 10: File Relocation + ... + Select parameter to modify (1-12): + + + This will replace your current Where value + 1: Strip prefix + 2: Add prefix + 3: Add file suffix + 4: Enter a regexp + 5: Test filename manipulation + 6: Use this ? + Select parameter to modify (1-6): + +.. _regexwhere: + +RegexWhere Format +~~~~~~~~~~~~~~~~~ + +The format is very close to that used by sed or Perl (``s/replace this/by that/``) operator. A valid regexwhere expression has three fields : + +- a search expression (with optional submatch) + +- a replacement expression (with optionnal back references $1 to $9) + +- a set of search options (only case-insensitive “i” at this time) + +Each field is delimited by a separator specified by the user as the first character of the expression. The separator can be one of the following: + + + + = / ! ; % : , ~ # = & + +You can use several expressions separated by a commas. + +Examples +^^^^^^^^ + ++----------------------+-----------------------+-----------------------------------+--------------------------------+ +| Orignal filename | New filename | RegexWhere | Comments | ++======================+=======================+===================================+================================+ +| ``c:/system.ini`` | ``c:/system.old.ini`` | ``/.ini$/.old.ini/`` | $ matches end of name | ++----------------------+-----------------------+-----------------------------------+--------------------------------+ +| ``/prod/u01/pdata/`` | ``/rect/u01/rdata`` | ``/prod/rect/,/pdata/rdata/`` | uses two regexp | ++----------------------+-----------------------+-----------------------------------+--------------------------------+ +| ``/prod/u01/pdata/`` | ``/rect/u01/rdata`` | ``!/prod/!/rect/!,/pdata/rdata/`` | use ``!`` as separator | ++----------------------+-----------------------+-----------------------------------+--------------------------------+ +| ``C:/WINNT`` | ``d:/WINNT`` | ``/c:/d:/i`` | case insensitive pattern match | ++----------------------+-----------------------+-----------------------------------+--------------------------------+ + +Restoring Directory Attributes +------------------------------ + +.. index:: + single: Restoring Directory Attributes + + +Depending how you do the restore, you may or may not get the directory entries back to their original state. Here are a few of the problems you can encounter, and for same machine restores, how to avoid them. + +- You backed up on one machine and are restoring to another that is either a different OS or doesn’t have the same users/groups defined. Bareos does the best it can in these situations. Note, Bareos has saved the user/groups in numeric form, which means on a different machine, they may map to different user/group names. + +- You are restoring into a directory that is already created and has file creation restrictions. Bareos tries to reset everything but without walking up the full chain of directories and modifying them all during the restore, which Bareos does and will not do, getting permissions back correctly in this situation depends to a large extent on your OS. + +- You are doing a recursive restore of a directory tree. In this case Bareos will restore a file before restoring the file’s parent directory entry. In the process of restoring the file Bareos will create the parent directory with open permissions and ownership of the file being restored. Then when Bareos tries to restore the parent directory Bareos sees that it already exists (Similar to the previous situation). If you had set the Restore job’s "Replace" property to "never" then Bareos will + not change the directory’s permissions and ownerships to match what it backed up, you should also notice that the actual number of files restored is less then the expected number. If you had set the Restore job’s "Replace" property to "always" then Bareos will change the Directory’s ownership and permissions to match what it backed up, also the actual number of files restored should be equal to the expected number. + +- You selected one or more files in a directory, but did not select the directory entry to be restored. In that case, if the directory is not on disk Bareos simply creates the directory with some default attributes which may not be the same as the original. If you do not select a directory and all its contents to be restored, you can still select items within the directory to be restored by individually marking those files, but in that case, you should individually use the "markdir" command to + select all higher level directory entries (one at a time) to be restored if you want the directory entries properly restored. + +.. _section-RestoreOnWindows: + +Restoring on Windows +-------------------- + +.. index:: + single: Restoring on Windows +.. index:: + pair: Windows; Restoring on + + +If you are restoring on Windows systems, Bareos will restore the files with the original ownerships and permissions as would be expected. This is also true if you are restoring those files to an alternate directory (using the Where option in restore). However, if the alternate directory does not already exist, the Bareos File daemon (Client) will try to create it. In some cases, it may not create the directories, and if it does since the File daemon runs under the SYSTEM account, the directory +will be created with SYSTEM ownership and permissions. In this case, you may have problems accessing the newly restored files. + +To avoid this problem, you should create any alternate directory before doing the restore. Bareos will not change the ownership and permissions of the directory if it is already created as long as it is not one of the directories being restored (i.e. written to tape). + +The default restore location is **/tmp/bareos-restores/** and if you are restoring from drive **E:**, the default will be **/tmp/bareos-restores/e/**, so you should ensure that this directory exists before doing the restore, or use the **mod** option to select a different **where** directory that does exist. + +Some users have experienced problems restoring files that participate in the Active Directory. They also report that changing the userid under which Bareos (bareos-fd.exe) runs, from SYSTEM to a Domain Admin userid, resolves the problem. + +Restore Errors +-------------- + +.. index:: + single: Restore Errors + + +There are a number of reasons why there may be restore errors or warning messages. Some of the more common ones are: + +file count mismatch + This can occur for the following reasons: + + - You requested Bareos not to overwrite existing or newer files. + + - A Bareos miscount of files/directories. This is an on-going problem due to the complications of directories, soft/hard link, and such. Simply check that all the files you wanted were actually restored. + +file size error + When Bareos restores files, it checks that the size of the restored file is the same as the file status data it saved when starting the backup of the file. If the sizes do not agree, Bareos will print an error message. This size mismatch most often occurs because the file was being written as Bareos backed up the file. In this case, the size that Bareos restored will be greater than the status size. This often happens with log files. + + If the restored size is smaller, then you should be concerned about a possible tape error and check the Bareos output as well as your system logs. + +Example Restore Job Resource +---------------------------- + +.. index:: + pair: Resource; Example Restore Job + + +.. raw:: latex + + + + + + Job { + Name = "RestoreFiles" + Type = Restore + Client = Any-client + FileSet = "Any-FileSet" + Storage = Any-storage + Where = /tmp/bareos-restores + Messages = Standard + Pool = Default + } + +.. raw:: latex + + + +If **Where** is not specified, the default location for restoring files will be their original locations. + +.. _`Selection`: Selection + +File Selection Commands +----------------------- + +.. index:: + single: File Selection Commands + + +After you have selected the Jobs to be restored and Bareos has created the in-memory directory tree, you will enter file selection mode as indicated by the dollar sign (**$**) prompt. While in this mode, you may use the commands listed above. The basic idea is to move up and down the in memory directory structure with the **cd** command much as you normally do on the system. Once you are in a directory, you may select the files that you want restored. As a default no files are marked to be +restored. If you wish to start with all files, simply enter: **cd /** and **mark \***. Otherwise proceed to select the files you wish to restore by marking them with the **mark** command. The available commands are: + +cd + +.. index:: + triple: Console; File Selection; cd; + The **cd** command changes the current directory to the argument specified. It operates much like the Unix **cd** command. Wildcard specifications are not permitted. + + Note, on Windows systems, the various drives (c:, d:, ...) are treated like a directory within the file tree while in the file selection mode. As a consequence, you must do a **cd c:** or possibly in some cases a **cd C:** (note upper case) to get down to the first directory. + +dir + +.. index:: + triple: Console; File Selection; dir; + The **dir** command is similar to the **ls** command, except that it prints it in long format (all details). This command can be a bit slower than the **ls** command because it must access the catalog database for the detailed information for each file. + +estimate + +.. index:: + triple: Console; File Selection; estimate; + The **estimate** command prints a summary of the total files in the tree, how many are marked to be restored, and an estimate of the number of bytes to be restored. This can be useful if you are short on disk space on the machine where the files will be restored. + +find + +.. index:: + triple: Console; File Selection; find; + The **find** command accepts one or more arguments and displays all files in the tree that match that argument. The argument may have wildcards. It is somewhat similar to the Unix command **find / -name arg**. + +ls + +.. index:: + triple: Console; File Selection; ls; + The **ls** command produces a listing of all the files contained in the current directory much like the Unix **ls** command. You may specify an argument containing wildcards, in which case only those files will be listed. + + Any file that is marked to be restored will have its name preceded by an asterisk (****). Directory names will be terminated with a forward slash (**/**) to distinguish them from filenames. + +lsmark + +.. index:: + triple: Console; File Selection; lsmark; + The **lsmark** command is the same as the **ls** except that it will print only those files marked for extraction. The other distinction is that it will recursively descend into any directory selected. + +mark + +.. index:: + triple: Console; File Selection; mark; + The **mark** command allows you to mark files to be restored. It takes a single argument which is the filename or directory name in the current directory to be marked for extraction. The argument may be a wildcard specification, in which case all files that match in the current directory are marked to be restored. If the argument matches a directory rather than a file, then the directory and all the files contained in that directory + (recursively) are marked to be restored. Any marked file will have its name preceded with an asterisk (****) in the output produced by the **ls** or **dir** commands. Note, supplying a full path on the mark command does not work as expected to select a file or directory in the current directory. Also, the **mark** command works on the current and lower directories but does not touch higher level directories. + + After executing the **mark** command, it will print a brief summary: + + + + + + + + No files marked. + + + + + + If no files were marked, or: + + + + + + + + nn files marked. + + + + + + if some files are marked. + +unmark + +.. index:: + triple: Console; File Selection; unmark; + The **unmark** is identical to the **mark** command, except that it unmarks the specified file or files so that they will not be restored. Note: the **unmark** command works from the current directory, so it does not unmark any files at a higher level. First do a **cd /** before the **unmark \*** command if you want to unmark everything. + +pwd + +.. index:: + triple: Console; File Selection; pwd; + The **pwd** command prints the current working directory. It accepts no arguments. + +count + +.. index:: + triple: Console; File Selection; count; + The **count** command prints the total files in the directory tree and the number of files marked to be restored. + +done + +.. index:: + triple: Console; File Selection; done; + This command terminates file selection mode. + +exit + +.. index:: + triple: Console; File Selection; exit; + This command terminates file selection mode (the same as done). + +quit + +.. index:: + triple: Console; File Selection; quit; + This command terminates the file selection and does not run the restore job. + +help + +.. index:: + triple: Console; File Selection; help; + This command prints a summary of the commands available. + +? + +.. index:: + triple: Console; File Selection; ?; + This command is the same as the **help** command. + +If your filename contains some weird caracters, you can use ``?``, ``*`` or \\\. For example, if your filename contains a \\, you can use \\\\\. + + + + * mark weird_file\\\\with-backslash diff --git a/docs/manuals/en/new_main_reference/source/chapter17/disk.rst b/docs/manuals/en/new_main_reference/source/chapter17/disk.rst new file mode 100644 index 00000000000..062e4b4c369 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter17/disk.rst @@ -0,0 +1,510 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + + + +.. _`DiskChapter}` :raw-latex:`\index[general]{Volume!Management}` :raw-latex:`\index[general]{Disk Volumes`: DiskChapter}` :raw-latex:`\index[general]{Volume!Management}` :raw-latex:`\index[general]{Disk Volumes + +This chapter presents most all the features needed to do Volume management. Most of the concepts apply equally well to both tape and disk Volumes. However, the chapter was originally written to explain backing up to disk, so you will see it is slanted in that direction, but all the directives presented here apply equally well whether your volume is disk or tape. + +If you have a lot of hard disk storage or you absolutely must have your backups run within a small time window, you may want to direct Bareos to backup to disk Volumes rather than tape Volumes. This chapter is intended to give you some of the options that are available to you so that you can manage either disk or tape volumes. + +Key Concepts and Resource Records +================================= + + +.. index:: + triple: Volume; Management; Key Concepts and Resource Records; + + +Getting Bareos to write to disk rather than tape in the simplest case is rather easy. In the Storage daemon’s configuration file, you simply define an **Archive Device**:sup:`Sd`:sub:`Device` to be a directory. The default directory to store backups on disk is :file:`/var/lib/bareos/storage`: + +.. raw:: latex + + + + + + Device { + Name = FileBackup + Media Type = File + Archive Device = /var/lib/bareos/storage + Random Access = Yes; + AutomaticMount = yes; + RemovableMedia = no; + AlwaysOpen = no; + } + +.. raw:: latex + + + +Assuming you have the appropriate :strong:`Storage` resource in your Director’s configuration file that references the above Device resource, + +.. raw:: latex + + + + + + Storage { + Name = FileStorage + Address = ... + Password = ... + Device = FileBackup + Media Type = File + } + +.. raw:: latex + + + +Bareos will then write the archive to the file **/var/lib/bareos/storage/** where is the volume name of a Volume defined in the Pool. For example, if you have labeled a Volume named **Vol001**, Bareos will write to the file **/var/lib/bareos/storage/Vol001**. Although you can later move the archive file to another directory, you should not rename it or it will become unreadable by Bareos. This is because each archive has the filename as part of the internal label, and +the internal label must agree with the system filename before Bareos will use it. + +Although this is quite simple, there are a number of problems. The first is that unless you specify otherwise, Bareos will always write to the same volume until you run out of disk space. This problem is addressed below. + +In addition, if you want to use concurrent jobs that write to several different volumes at the same time, you will need to understand a number of other details. An example of such a configuration is given at the end of this chapter under :ref:`ConcurrentDiskJobs`. + +Pool Options to Limit the Volume Usage +-------------------------------------- + +.. index:: + pair: Pool; Options to Limit the Volume Usage + + +Some of the options you have, all of which are specified in the Pool record, are: + +- **Maximum Volume Jobs**:sup:`Dir`:sub:`Pool` : write only the specified number of jobs on each Volume. + +- **Maximum Volume Bytes**:sup:`Dir`:sub:`Pool` : limit the maximum size of each Volume. + + Note, if you use disk volumes you should probably limit the Volume size to some reasonable value. If you ever have a partial hard disk failure, you are more likely to be able to recover more data if they are in smaller Volumes. + +- **Volume Use Duration**:sup:`Dir`:sub:`Pool` : restrict the time between first and last data written to Volume. + +Note that although you probably would not want to limit the number of bytes on a tape as you would on a disk Volume, the other options can be very useful in limiting the time Bareos will use a particular Volume (be it tape or disk). For example, the above directives can allow you to ensure that you rotate through a set of daily Volumes if you wish. + +As mentioned above, each of those directives is specified in the Pool or Pools that you use for your Volumes. In the case of **Maximum Volume Jobs**:sup:`Dir`:sub:`Pool` , **Maximum Volume Bytes**:sup:`Dir`:sub:`Pool` and **Volume Use Duration**:sup:`Dir`:sub:`Pool` , you can actually specify the desired value on a Volume by Volume basis. The value specified in the Pool record becomes the default when labeling new Volumes. Once a +Volume has been created, it gets its own copy of the Pool defaults, and subsequently changing the Pool will have no effect on existing Volumes. You can either manually change the Volume values, or refresh them from the Pool defaults using the :strong:`update volume` command in the Console. As an example of the use of one of the above, suppose your Pool resource contains: + + + + +.. code-block:: sh + :caption: Volume Use Duration + + Pool { + Name = File + Pool Type = Backup + Volume Use Duration = 23h + } + +then if you run a backup once a day (every 24 hours), Bareos will use a new Volume for each backup, because each Volume it writes can only be used for 23 hours after the first write. Note, setting the use duration to 23 hours is not a very good solution for tapes unless you have someone on-site during the weekends, because Bareos will want a new Volume and no one will be present to mount it, so no weekend backups will be done until Monday morning. + +.. _AutomaticLabeling: + +Automatic Volume Labeling +------------------------- + +.. index:: + pair: Label; Automatic Volume Labeling + +.. index:: + triple: Volume; Labeling; Automatic; + + +Use of the above records brings up another problem – that of labeling your Volumes. For automated disk backup, you can either manually label each of your Volumes, or you can have Bareos automatically label new Volumes when they are needed. + +Please note that automatic Volume labeling can also be used with tapes, but it is not nearly so practical since the tapes must be pre-mounted. This requires some user interaction. Automatic labeling from templates does NOT work with autochangers since Bareos will not access unknown slots. There are several methods of labeling all volumes in an autochanger magazine. For more information on this, please see the :ref:`AutochangersChapter` chapter. + +Automatic Volume labeling is enabled by making a change to both the :sup:`Dir` :strong:`Pool` resource and to the :sup:`Sd` :strong:`Device` resource shown above. In the case of the Pool resource, you must provide Bareos with a label format that it will use to create new names. In the simplest form, the label format is simply the Volume name, to which Bareos will append a four digit number. This number starts at 0001 and is incremented for each Volume the catalog +contains. Thus if you modify your Pool resource to be: + + + + +.. code-block:: sh + :caption: Label Format + + Pool { + Name = File + Pool Type = Backup + Volume Use Duration = 23h + Label Format = "Vol" + } + +Bareos will create Volume names Vol0001, Vol0002, and so on when new Volumes are needed. Much more complex and elaborate labels can be created using variable expansion defined in the :ref:`Variable Expansion ` chapter of this manual. + +The second change that is necessary to make automatic labeling work is to give the Storage daemon permission to automatically label Volumes. Do so by adding **Label Media**:sup:`Sd`:sub:`Device` = yes to the :strong:`Device` resource as follows: + + + + +.. code-block:: sh + :caption: Label Media = yes + + Device { + Name = File + Media Type = File + Archive Device = /var/lib/bareos/storage/ + Random Access = yes + Automatic Mount = yes + Removable Media = no + Always Open = no + Label Media = yes + } + +See **Label Format**:sup:`Dir`:sub:`Pool` for details about the labeling format. + +Restricting the Number of Volumes and Recycling +----------------------------------------------- + +.. index:: + single: Restricting the Number of Volumes and Recycling + + +Automatic labeling discussed above brings up the problem of Volume management. With the above scheme, a new Volume will be created every day. If you have not specified Retention periods, your Catalog will continue to fill keeping track of all the files Bareos has backed up, and this procedure will create one new archive file (Volume) every day. + +The tools Bareos gives you to help automatically manage these problems are the following: + +- **File Retention**:sup:`Dir`:sub:`Client` : catalog file record retention period. + +- **Job Retention**:sup:`Dir`:sub:`Client` : catalog job record retention period. + +- **Auto Prune**:sup:`Dir`:sub:`Client` = yes: permit the application of the above two retention periods. + +- + + + + **Volume Retention**:sup:`Dir`:sub:`Pool` + +- **Auto Prune**:sup:`Dir`:sub:`Pool` = yes: permit the application of the **Volume Retention**:sup:`Dir`:sub:`Pool` period. + +- **Recycle**:sup:`Dir`:sub:`Pool` = yes: permit automatic recycling of Volumes whose Volume retention period has expired. + +- **Recycle Oldest Volume**:sup:`Dir`:sub:`Pool` = yes: prune the oldest volume in the Pool, and if all files were pruned, recycle this volume and use it. + +- **Recycle Current Volume**:sup:`Dir`:sub:`Pool` = yes: prune the currently mounted volume in the Pool, and if all files were pruned, recycle this volume and use it. + +- | **Purge Oldest Volume**:sup:`Dir`:sub:`Pool` = yes: permits a forced recycling of the oldest Volume when a new one is needed. + | +.. warning:: + This record ignores retention periods! We highly + recommend not to use this record, but instead use **Recycle Oldest Volume**:sup:`Dir`:sub:`Pool` . + +- **Maximum Volumes**:sup:`Dir`:sub:`Pool` : limit the number of Volumes that can be created. + +The first three records (**File Retention**:sup:`Dir`:sub:`Client` , **Job Retention**:sup:`Dir`:sub:`Client` and **Auto Prune**:sup:`Dir`:sub:`Client` ) determine the amount of time that Job and File records will remain in your Catalog and they are discussed in detail in the :ref:`Automatic Volume Recycling ` chapter. + +**Volume Retention**:sup:`Dir`:sub:`Pool` , **Auto Prune**:sup:`Dir`:sub:`Pool` and **Recycle**:sup:`Dir`:sub:`Pool` determine how long Bareos will keep your Volumes before reusing them and they are also discussed in detail in the :ref:`Automatic Volume Recycling ` chapter. + +The **Maximum Volumes**:sup:`Dir`:sub:`Pool` record can also be used in conjunction with the **Volume Retention**:sup:`Dir`:sub:`Pool` period to limit the total number of archive Volumes that Bareos will create. By setting an appropriate **Volume Retention**:sup:`Dir`:sub:`Pool` period, a Volume will be purged just before it is needed and thus Bareos can cycle through a fixed set of Volumes. Cycling through a fixed set of +Volumes can also be done by setting **Purge Oldest Volume**:sup:`Dir`:sub:`Pool` = yes or **Recycle Current Volume**:sup:`Dir`:sub:`Pool` = yes. In this case, when Bareos needs a new Volume, it will prune the specified volume. + +Concurrent Disk Jobs +==================== + +.. index:: + single: Concurrent Disk Jobs + + +.. _`ConcurrentDiskJobs}` Above, we discussed how you could have a single device named **FileBackup`: ConcurrentDiskJobs**:sup:`Sd`:sub:`Device` Above, we discussed how you could have a single device named :raw-latex:`\resourcename{Sd}{Device}{FileBackup that writes to volumes in :file:`/var/lib/bareos/storage/`. You can, in fact, run multiple concurrent jobs using the Storage definition given with this example, and all the jobs will simultaneously write into the Volume that is being written. + +Now suppose you want to use multiple Pools, which means multiple Volumes, or suppose you want each client to have its own Volume and perhaps its own directory such as **/home/bareos/client1** and **/home/bareos/client2** ... . With the single Storage and Device definition above, neither of these two is possible. Why? Because Bareos disk storage follows the same rules as tape devices. Only one Volume can be mounted on any Device at any time. If you want to simultaneously write multiple Volumes, +you will need multiple Device resources in your |bareosSd| configuration and thus multiple Storage resources in your |bareosDir| configuration. + +Okay, so now you should understand that you need multiple Device definitions in the case of different directories or different Pools, but you also need to know that the catalog data that Bareos keeps contains only the Media Type and not the specific storage device. This permits a tape for example to be re-read on any compatible tape drive. The compatibility being determined by the Media Type (**Media Type**:sup:`Dir`:sub:`Storage` and +**Media Type**:sup:`Sd`:sub:`Device` ). The same applies to disk storage. Since a volume that is written by a Device in say directory :file:`/home/bareos/backups`` cannot be read by a Device with an **Archive Device**:sup:`Sd`:sub:`Device` = ``path:/home/bareos/client1`, you will not be able to restore all your files if you give both those devices **Media Type**:sup:`Sd`:sub:`Device` = File. During the restore, Bareos will +simply choose the first available device, which may not be the correct one. If this is confusing, just remember that the Directory has only the Media Type and the Volume name. It does not know the **Archive Device**:sup:`Sd`:sub:`Device` (or the full path) that is specified in the |bareosSd|. Thus you must explicitly tie your Volumes to the correct Device by using the Media Type. + +Example for two clients, separate devices and recycling +------------------------------------------------------- + +The following example is not very practical, but can be used to demonstrate the proof of concept in a relatively short period of time. + +The example consists of a two clients that are backed up to a set of 12 Volumes for each client into different directories on the Storage machine. Each Volume is used (written) only once, and there are four Full saves done every hour (so the whole thing cycles around after three hours). + +What is key here is that each physical device on the |bareosSd| has a different Media Type. This allows the Director to choose the correct device for restores. + +The |bareosDir| configuration is as follows: + + + + +.. code-block:: sh + :caption: + + Director { + Name = bareos-dir + QueryFile = "/usr/lib/bareos/scripts/query.sql" + Password = "" + } + + Schedule { + Name = "FourPerHour" + Run = Level=Full hourly at 0:05 + Run = Level=Full hourly at 0:20 + Run = Level=Full hourly at 0:35 + Run = Level=Full hourly at 0:50 + } + + FileSet { + Name = "Example FileSet" + Include { + Options { + compression=GZIP + signature=SHA1 + } + File = /etc + } + } + + Job { + Name = "RecycleExample" + Type = Backup + Level = Full + Client = client1-fd + FileSet= "Example FileSet" + Messages = Standard + Storage = FileStorage + Pool = Recycle + Schedule = FourPerHour + } + + Job { + Name = "RecycleExample2" + Type = Backup + Level = Full + Client = client2-fd + FileSet= "Example FileSet" + Messages = Standard + Storage = FileStorage2 + Pool = Recycle2 + Schedule = FourPerHour + } + + Client { + Name = client1-fd + Address = client1.example.com + Password = client1_password + } + + Client { + Name = client2-fd + Address = client2.example.com + Password = client2_password + } + + Storage { + Name = FileStorage + Address = bareos-sd.example.com + Password = local_storage_password + Device = RecycleDir + Media Type = File + } + + Storage { + Name = FileStorage2 + Address = bareos-sd.example.com + Password = local_storage_password + Device = RecycleDir2 + Media Type = File1 + } + + Catalog { + Name = MyCatalog + ... + } + + Messages { + Name = Standard + ... + } + + Pool { + Name = Recycle + Pool Type = Backup + Label Format = "Recycle-" + Auto Prune = yes + Use Volume Once = yes + Volume Retention = 2h + Maximum Volumes = 12 + Recycle = yes + } + + Pool { + Name = Recycle2 + Pool Type = Backup + Label Format = "Recycle2-" + Auto Prune = yes + Use Volume Once = yes + Volume Retention = 2h + Maximum Volumes = 12 + Recycle = yes + } + +and the |bareosSd| configuration is: + + + + +.. code-block:: sh + :caption: + + Storage { + Name = bareos-sd + Maximum Concurrent Jobs = 10 + } + + Director { + Name = bareos-dir + Password = local_storage_password + } + + Device { + Name = RecycleDir + Media Type = File + Archive Device = /home/bareos/backups + LabelMedia = yes; + Random Access = Yes; + AutomaticMount = yes; + RemovableMedia = no; + AlwaysOpen = no; + } + + Device { + Name = RecycleDir2 + Media Type = File2 + Archive Device = /home/bareos/backups2 + LabelMedia = yes; + Random Access = Yes; + AutomaticMount = yes; + RemovableMedia = no; + AlwaysOpen = no; + } + + Messages { + Name = Standard + director = bareos-dir = all + } + +With a little bit of work, you can change the above example into a weekly or monthly cycle (take care about the amount of archive disk space used). + +.. _section-MultipleStorageDevices: + +Using Multiple Storage Devices +------------------------------ + +.. index:: + single: Multiple Storage Devices +.. index:: + pair: Storage Device; Multiple + + +Bareos treats disk volumes similar to tape volumes as much as it can. This means that you can only have a single Volume mounted at one time on a disk as defined in your :sup:`Sd` :strong:`Device` resource. + +If you use Bareos without :ref:`section-DataSpooling`, multiple concurrent backup jobs can be written to a Volume using interleaving. However, interleaving has disadvantages, see :ref:`section-Interleaving`. + +Also the :sup:`Sd` :strong:`Device` will be in use. If there are other jobs, requesting other Volumes, these jobs have to wait. + +On a tape (or autochanger), this is a physical limitation of the hardware. However, when using disk storage, this is only a limitation of the software. + +To enable Bareos to run concurrent jobs (on disk storage), define as many :sup:`Sd` :strong:`Device` as concurrent jobs should run. All these :sup:`Sd` :strong:`Device`s can use the same **Archive Device**:sup:`Sd`:sub:`Device` directory. Set **Maximum Concurrent Jobs**:sup:`Sd`:sub:`Device` = 1 for all these devices. + +Example: use four storage devices pointing to the same directory +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: \bareosDir configuration: using 4 storage devices + + Director { + Name = bareos-dir.example.com + QueryFile = "/usr/lib/bareos/scripts/query.sql" + Maximum Concurrent Jobs = 10 + Password = "" + } + + Storage { + Name = File + Address = bareos-sd.bareos.com + Password = "" + Device = FileStorage1 + Device = FileStorage2 + Device = FileStorage3 + Device = FileStorage4 + # number of devices = Maximum Concurrent Jobs + Maximum Concurrent Jobs = 4 + Media Type = File + } + + [...] + + + + +.. code-block:: sh + :caption: \bareosSd configuraton: using 4 storage devices + + Storage { + Name = bareos-sd.example.com + # any number >= 4 + Maximum Concurrent Jobs = 20 + } + + Director { + Name = bareos-dir.example.com + Password = "" + } + + Device { + Name = FileStorage1 + Media Type = File + Archive Device = /var/lib/bareos/storage + LabelMedia = yes + Random Access = yes + AutomaticMount = yes + RemovableMedia = no + AlwaysOpen = no + Maximum Concurrent Jobs = 1 + } + + Device { + Name = FileStorage2 + Media Type = File + Archive Device = /var/lib/bareos/storage + LabelMedia = yes + Random Access = yes + AutomaticMount = yes + RemovableMedia = no + AlwaysOpen = no + Maximum Concurrent Jobs = 1 + } + + Device { + Name = FileStorage3 + Media Type = File + Archive Device = /var/lib/bareos/storage + LabelMedia = yes + Random Access = yes + AutomaticMount = yes + RemovableMedia = no + AlwaysOpen = no + Maximum Concurrent Jobs = 1 + } + + Device { + Name = FileStorage4 + Media Type = File + Archive Device = /var/lib/bareos/storage + LabelMedia = yes + Random Access = yes + AutomaticMount = yes + RemovableMedia = no + AlwaysOpen = no + Maximum Concurrent Jobs = 1 + } diff --git a/docs/manuals/en/new_main_reference/source/chapter17/recycling.rst b/docs/manuals/en/new_main_reference/source/chapter17/recycling.rst new file mode 100644 index 00000000000..96fd4f42fa3 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter17/recycling.rst @@ -0,0 +1,557 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _RecyclingChapter: + +Automatic Volume Recycling +========================== + +.. index:: + pair: Recycle; Automatic Volume + +.. index:: + triple: Volume; Recycle; Automatic; + + +By default, once Bareos starts writing a Volume, it can append to the volume, but it will not overwrite the existing data thus destroying it. However when Bareos recycles a Volume, the Volume becomes available for being reused and Bareos can at some later time overwrite the previous contents of that Volume. Thus all previous data will be lost. If the Volume is a tape, the tape will be rewritten from the beginning. If the Volume is a disk file, the file will be truncated before being rewritten. + +You may not want Bareos to automatically recycle (reuse) tapes. This would require a large number of tapes though, and in such a case, it is possible to manually recycle tapes. For more on manual recycling, see the :ref:`manualrecycling` chapter. + +Most people prefer to have a Pool of tapes that are used for daily backups and recycled once a week, another Pool of tapes that are used for Full backups once a week and recycled monthly, and finally a Pool of tapes that are used once a month and recycled after a year or two. With a scheme like this, the number of tapes in your pool or pools remains constant. + +By properly defining your Volume Pools with appropriate Retention periods, Bareos can manage the recycling (such as defined above) automatically. + +Automatic recycling of Volumes is controlled by four records in the :sup:`Dir` :strong:`Pool` resource definition. These four records are: + +- **Auto Prune**:sup:`Dir`:sub:`Pool` = yes + +- + + + + **Volume Retention**:sup:`Dir`:sub:`Pool` + +- **Recycle**:sup:`Dir`:sub:`Pool` = yes + +- + + + + **Recycle Pool**:sup:`Dir`:sub:`Pool` + +The above three directives are all you need assuming that you fill each of your Volumes then wait the Volume Retention period before reusing them. If you want Bareos to stop using a Volume and recycle it before it is full, you can use one or more additional directives such as: + +- + + + + **Volume Use Duration**:sup:`Dir`:sub:`Pool` + +- + + + + **Maximum Volume Jobs**:sup:`Dir`:sub:`Pool` + +- + + + + **Maximum Volume Bytes**:sup:`Dir`:sub:`Pool` + +Please see below and the :ref:`Basic Volume Management ` chapter of this manual for complete examples. + +Automatic recycling of Volumes is performed by Bareos only when it wants a new Volume and no appendable Volumes are available in the Pool. It will then search the Pool for any Volumes with the **Recycle** flag set and the Volume Status is **Purged**. At that point, it will choose the oldest purged volume and recycle it. + +If there are no volumes with status **Purged**, then the recycling occurs in two steps: + +#. The Catalog for a Volume must be pruned of all Jobs (i.e. Purged). + +#. The actual recycling of the Volume. + +Only Volumes marked **Full** or **Used** will be considerd for pruning. The Volume will be purged if the Volume Retention = **** period has expired. When a Volume is marked as **Purged**, it means that no Catalog records reference that Volume and the Volume can be recycled. + +Until recycling actually occurs, the Volume data remains intact. If no Volumes can be found for recycling for any of the reasons stated above, Bareos will request operator intervention (i.e. it will ask you to label a new volume). + +A key point mentioned above, that can be a source of frustration, is that Bareos will only recycle purged Volumes if there is no other appendable Volume available. Otherwise, it will always write to an appendable Volume before recycling even if there are Volume marked as Purged. This preserves your data as long as possible. So, if you wish to :emphasis:`force` Bareos to use a purged Volume, you must first ensure that no other Volume in the Pool is marked **Append**. If necessary, you +can manually set a volume to **Full**. The reason for this is that Bareos wants to preserve the data on your old tapes (even though purged from the catalog) as long as absolutely possible before overwriting it. There are also a number of directives such as Volume Use Duration = **** that will automatically mark a volume as **Used** and thus no longer appendable. + +.. _AutoPruning: + +Automatic Pruning +----------------- + +.. index:: + pair: Automatic; Pruning +.. index:: + pair: Pruning; Automatic + + +As Bareos writes files to tape, it keeps a list of files, jobs, and volumes in a database called the catalog. Among other things, the database helps Bareos to decide which files to back up in an incremental or differential backup, and helps you locate files on past backups when you want to restore something. However, the catalog will grow larger and larger as time goes on, and eventually it can become unacceptably large. + +Bareos’s process for removing entries from the catalog is called Pruning. The default is Automatic Pruning, which means that once an entry reaches a certain age (e.g. 30 days old) it is removed from the catalog. Note that Job records that are required for current restore and File records are needed for VirtualFull and Accurate backups won’t be removed automatically. + +Once a job has been pruned, you can still restore it from the backup tape, but one additional step is required: scanning the volume with :program:`bscan`. + +The alternative to Automatic Pruning is Manual Pruning, in which you explicitly tell Bareos to erase the catalog entries for a volume. You’d usually do this when you want to reuse a Bareos volume, because there’s no point in keeping a list of files that USED TO BE on a tape. Or, if the catalog is starting to get too big, you could prune the oldest jobs to save space. Manual pruning is done with the :ref:`prune command ` in the console. + +Pruning Directives +------------------ + +.. index:: + pair: Pruning; Directives + + +There are three pruning durations. All apply to catalog database records and not to the actual data in a Volume. The pruning (or retention) durations are for: Volumes (Media records), Jobs (Job records), and Files (File records). The durations inter-depend because if Bareos prunes a Volume, it automatically removes all the Job records, and all the File records. Also when a Job record is pruned, all the File records for that Job are also pruned (deleted) from the catalog. + +Having the File records in the database means that you can examine all the files backed up for a particular Job. They take the most space in the catalog (probably 90-95% of the total). When the File records are pruned, the Job records can remain, and you can still examine what Jobs ran, but not the details of the Files backed up. In addition, without the File records, you cannot use the Console restore command to restore the files. + +When a Job record is pruned, the Volume (Media record) for that Job can still remain in the database, and if you do a :strong:`list volumes`, you will see the volume information, but the Job records (and its File records) will no longer be available. + +In each case, pruning removes information about where older files are, but it also prevents the catalog from growing to be too large. You choose the retention periods in function of how many files you are backing up and the time periods you want to keep those records online, and the size of the database. It is possible to re-insert the records (with 98% of the original data) by using :program:`bscan` to scan in a whole Volume or any part of the volume that you want. + +By setting **Auto Prune**:sup:`Dir`:sub:`Pool` = yes you will permit the |bareosDir| to automatically prune all Volumes in the Pool when a Job needs another Volume. Volume pruning means removing records from the catalog. It does not shrink the size of the Volume or affect the Volume data until the Volume gets overwritten. When a Job requests another volume and there are no Volumes with Volume status **Append** available, Bareos will +begin volume pruning. This means that all Jobs that are older than the Volume Retention = **** period will be pruned from every Volume that has Volume status **Full** or **Used** and has Recycle = **yes**. Pruning consists of deleting the corresponding Job, File, and JobMedia records from the catalog database. No change to the physical data on the Volume occurs during the pruning process. When all +files are pruned from a Volume (i.e. no records in the catalog), the Volume will be marked as **Purged** implying that no Jobs remain on the volume. The Pool records that control the pruning are described below. + +.. raw:: latex + + \begin{description} + + \item **Auto Prune**:sup:`Dir`:sub:`Pool` = yes: + when running a Job and it needs a new Volume but no appendable volumes are available, apply the Volume retention period. + At that point, + Bareos will prune all Volumes that can be pruned in an + attempt to find a usable volume. If during the autoprune, all files are + pruned from the Volume, it will be marked with Volume status \volumestatus{Purged}. + + Note, that although the File and Job records may be + pruned from the catalog, a Volume will only be marked \volumestatus{Purged} (and hence + ready for recycling) if the Volume status is \volumestatus{Append}, \volumestatus{Full}, \volumestatus{Used}, or \volumestatus{Error}. + If the Volume has another status, such as \volumestatus{Archive}, \volumestatus{Read-Only}, \volumestatus{Disabled}, + \volumestatus{Busy} or \volumestatus{Cleaning}, the Volume status will not be changed to \volumestatus{Purged}. + + \item **Volume Retention**:sup:`Dir`:sub:`Pool` + defines the length of time that Bareos will + guarantee that the Volume is not reused counting from the time the last + job stored on the Volume terminated. A key point is that this time + period is not even considered as long at the Volume remains appendable. + The Volume Retention period count down begins only when the \volumestatus{Append} + status has been changed to some other status (\volumestatus{Full}, \volumestatus{Used}, \volumestatus{Purged}, ...). + + When this time period expires and if **Auto Prune**:sup:`Dir`:sub:`Pool` = yes + and a new Volume is needed, but no appendable Volume is available, + Bareos will prune (remove) Job records that are older than the specified + \volumeparameter{Volume Retention}{} period. + + The \volumeparameter{Volume Retention}{} period takes precedence over any **Job Retention**:sup:`Dir`:sub:`Client` + period you have specified in the Client resource. It should also be + noted, that the \volumeparameter{Volume Retention}{} period is obtained by reading the + Catalog Database Media record rather than the Pool resource record. + This means that if you change the **Volume Retention**:sup:`Dir`:sub:`Pool` in the Pool resource + record, you must ensure that the corresponding change is made in the + catalog by using the \bcommand{update}{pool} command. Doing so will insure + that any new Volumes will be created with the changed \volumeparameter{Volume Retention}{} + period. Any existing Volumes will have their own copy of the \volumeparameter{Volume Retention}{} + period that can only be changed on a Volume by Volume basis + using the \bcommand{update}{volume} command. + + When all file catalog entries are removed from the volume, its Volume status is + set to \volumestatus{Purged}. The files remain physically on the Volume until the + volume is overwritten. + + \item **Recycle**:sup:`Dir`:sub:`Pool` + defines whether or not the particular Volume can be + recycled (i.e. rewritten). If Recycle is set to :option:`no`, + then even if Bareos prunes all the Jobs on the volume and it + is marked \volumestatus{Purged}, it will not consider the tape for recycling. If + Recycle is set to :option:`yes` and all Jobs have been pruned, the volume + status will be set to \volumestatus{Purged} and the volume may then be reused + when another volume is needed. If the volume is reused, it is relabeled + with the same Volume Name, however all previous data will be lost. + \end{description} + +Recycling Algorithm +------------------- + +.. index:: + pair: Algorithm; Recycling +.. index:: + pair: Recycle; Algorithm + + +.. _`RecyclingAlgorithm}` :raw-latex:`\label{Recycling`: RecyclingAlgorithm}` :raw-latex:`\label{Recycling + +After all Volumes of a Pool have been pruned (as mentioned above, this happens when a Job needs a new Volume and no appendable Volumes are available), Bareos will look for the oldest Volume that is **Purged** (all Jobs and Files expired), and if the Recycle = **yes** for that Volume, Bareos will relabel it and write new data on it. + +As mentioned above, there are two key points for getting a Volume to be recycled. First, the Volume must no longer be marked **Append** (there are a number of directives to automatically make this change), and second since the last write on the Volume, one or more of the Retention periods must have expired so that there are no more catalog backup job records that reference that Volume. Once both those conditions are satisfied, the volume can be marked +**Purged** and hence recycled. + +The full algorithm that Bareos uses when it needs a new Volume is: :raw-latex:`\index[general]{New Volume Algorithm}` :raw-latex:`\index[general]{Algorithm!New Volume}` + +The algorithm described below assumes that :strong:`Auto Prune` is enabled, that Recycling is turned on, and that you have defined appropriate Retention periods or used the defaults for all these items. + +#. If the request is for an Autochanger device, look only for Volumes in the Autochanger (i.e. with InChanger set and that have the correct Storage device). + +#. Search the Pool for a Volume with Volume status=**Append** (if there is more than one, the Volume with the oldest date last written is chosen. If two have the same date then the one with the lowest MediaId is chosen). + +#. Search the Pool for a Volume with Volume status=**Recycle** and the InChanger flag is set true (if there is more than one, the Volume with the oldest date last written is chosen. If two have the same date then the one with the lowest MediaId is chosen). + +#. Try recycling any purged Volumes. + +#. Prune volumes applying Volume retention period (Volumes with VolStatus Full, Used, or Append are pruned). Note, even if all the File and Job records are pruned from a Volume, the Volume will not be marked Purged until the Volume retention period expires. + +#. Search the Pool for a Volume with VolStatus=Purged + +#. If a Pool named **Scratch**:sup:`Dir`:sub:`pool` exists, search for a Volume and if found move it to the current Pool for the Job and use it. Note, when the Scratch Volume is moved into the current Pool, the basic Pool defaults are applied as if it is a newly labeled Volume (equivalent to an :strong:`update volume from pool` command). + +#. If we were looking for Volumes in the Autochanger, go back to step 2 above, but this time, look for any Volume whether or not it is in the Autochanger. + +#. Attempt to create a new Volume if automatic labeling enabled. If the maximum number of Volumes specified for the pool is reached, no new Volume will be created. + +#. Prune the oldest Volume if **Recycle Oldest Volume**:sup:`Dir`:sub:`Pool` =yes (the Volume with the oldest LastWritten date and VolStatus equal to Full, Recycle, Purged, Used, or Append is chosen). This record ensures that all retention periods are properly respected. + +#. Purge the oldest Volume if **Purge Oldest Volume**:sup:`Dir`:sub:`Pool` =yes (the Volume with the oldest LastWritten date and VolStatus equal to Full, Recycle, Purged, Used, or Append is chosen). +.. warning:: + We strongly recommend against the use of \configdirective{Purge Oldest Volume} as it can quite easily lead to loss of current backup + data. + +#. Give up and ask operator. + +The above occurs when Bareos has finished writing a Volume or when no Volume is present in the drive. + +On the other hand, if you have inserted a different Volume after the last job, and Bareos recognizes the Volume as valid, it will request authorization from the Director to use this Volume. In this case, if you have set **Recycle Current Volume**:sup:`Dir`:sub:`Pool` = yes and the Volume is marked as Used or Full, Bareos will prune the volume and if all jobs were removed during the pruning (respecting the retention periods), the Volume will be recycled and used. + +The recycling algorithm in this case is: + +- If the Volume status is **Append** or **Recycle**, the volume will be used. + +- If **Recycle Current Volume**:sup:`Dir`:sub:`Pool` = yes and the volume is marked **Full** or **Used**, Bareos will prune the volume (applying the retention period). If all Jobs are pruned from the volume, it will be recycled. + +This permits users to manually change the Volume every day and load tapes in an order different from what is in the catalog, and if the volume does not contain a current copy of your backup data, it will be used. + +A few points from Alan Brown to keep in mind: + +- If **Maximum Volumes**:sup:`Dir`:sub:`Pool` is not set, Bareos will prefer to demand new volumes over forcibly purging older volumes. + +- If volumes become free through pruning and the Volume retention period has expired, then they get marked as **Purged** and are immediately available for recycling - these will be used in preference to creating new volumes. + +Recycle Status +-------------- + +.. index:: + single: Recycle Status + + +Each Volume inherits the Recycle status (yes or no) from the Pool resource record when the Media record is created (normally when the Volume is labeled). This Recycle status is stored in the Media record of the Catalog. Using the Console program, you may subsequently change the Recycle status for each Volume. For example in the following output from **list volumes**: + +.. raw:: latex + + + + + + +----------+-------+--------+---------+------------+--------+-----+ + | VolumeNa | Media | VolSta | VolByte | LastWritte | VolRet | Rec | + +----------+-------+--------+---------+------------+--------+-----+ + | File0001 | File | Full | 4190055 | 2002-05-25 | 14400 | 1 | + | File0002 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0003 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0004 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0005 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0006 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0007 | File | Purged | 1896466 | 2002-05-26 | 14400 | 1 | + +----------+-------+--------+---------+------------+--------+-----+ + +.. raw:: latex + + + +all the volumes are marked as recyclable, and the last Volume, **File0007** has been purged, so it may be immediately recycled. The other volumes are all marked recyclable and when their Volume Retention period (14400 seconds or four hours) expires, they will be eligible for pruning, and possibly recycling. Even though Volume **File0007** has been purged, all the data on the Volume is still recoverable. A purged Volume simply means that there are no entries in the Catalog. Even if the Volume +Status is changed to **Recycle**, the data on the Volume will be recoverable. The data is lost only when the Volume is re-labeled and re-written. + +To modify Volume **File0001** so that it cannot be recycled, you use the **update volume pool=File** command in the console program, or simply **update** and Bareos will prompt you for the information. + +.. raw:: latex + + + + + + +----------+------+-------+---------+-------------+-------+-----+ + | VolumeNa | Media| VolSta| VolByte | LastWritten | VolRet| Rec | + +----------+------+-------+---------+-------------+-------+-----+ + | File0001 | File | Full | 4190055 | 2002-05-25 | 14400 | 0 | + | File0002 | File | Full | 1897236 | 2002-05-26 | 14400 | 1 | + | File0003 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0004 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0005 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0006 | File | Full | 1896460 | 2002-05-26 | 14400 | 1 | + | File0007 | File | Purged| 1896466 | 2002-05-26 | 14400 | 1 | + +----------+------+-------+---------+-------------+-------+-----+ + +.. raw:: latex + + + +In this case, **File0001** will never be automatically recycled. The same effect can be achieved by setting the Volume Status to Read-Only. + +As you have noted, the Volume Status (VolStatus) column in the catalog database contains the current status of the Volume, which is normally maintained automatically by Bareos. To give you an idea of some of the values it can take during the life cycle of a Volume, here is a picture created by Arno Lehmann: + +.. raw:: latex + + + + + + A typical volume life cycle is like this: + + because job count or size limit exceeded + Append --------------------------------------> Used/Full + ^ | + | First Job writes to Retention time passed | + | the volume and recycling takes | + | place | + | v + Recycled <-------------------------------------- Purged + Volume is selected for reuse + +.. raw:: latex + + + +Daily, Weekly, Monthly Tape Usage Example +----------------------------------------- + +This example is meant to show you how one could define a fixed set of volumes that Bareos will rotate through on a regular schedule. There are an infinite number of such schemes, all of which have various advantages and disadvantages. + +We start with the following assumptions: + +- A single tape has more than enough capacity to do a full save. + +- There are ten tapes that are used on a daily basis for incremental backups. They are prelabeled Daily1 ... Daily10. + +- There are four tapes that are used on a weekly basis for full backups. They are labeled Week1 ... Week4. + +- There are 12 tapes that are used on a monthly basis for full backups. They are numbered Month1 ... Month12 + +- A full backup is done every Saturday evening (tape inserted Friday evening before leaving work). + +- No backups are done over the weekend (this is easy to change). + +- The first Friday of each month, a Monthly tape is used for the Full backup. + +- Incremental backups are done Monday - Friday (actually Tue-Fri mornings). + +We start the system by doing a Full save to one of the weekly volumes or one of the monthly volumes. The next morning, we remove the tape and insert a Daily tape. Friday evening, we remove the Daily tape and insert the next tape in the Weekly series. Monday, we remove the Weekly tape and re-insert the Daily tape. On the first Friday of the next month, we insert the next Monthly tape in the series rather than a Weekly tape, then continue. When a Daily tape finally fills up, **Bareos** will +request the next one in the series, and the next day when you notice the email message, you will mount it and **Bareos** will finish the unfinished incremental backup. + +What does this give? Well, at any point, you will have the last complete Full save plus several Incremental saves. For any given file you want to recover (or your whole system), you will have a copy of that file every day for at least the last 14 days. For older versions, you will have at least three and probably four Friday full saves of that file, and going back further, you will have a copy of that file made on the beginning of the month for at least a year. + +So you have copies of any file (or your whole system) for at least a year, but as you go back in time, the time between copies increases from daily to weekly to monthly. + +What would the Bareos configuration look like to implement such a scheme? + +.. raw:: latex + + + + + + Schedule { + Name = "NightlySave" + Run = Level=Full Pool=Monthly 1st sat at 03:05 + Run = Level=Full Pool=Weekly 2nd-5th sat at 03:05 + Run = Level=Incremental Pool=Daily tue-fri at 03:05 + } + Job { + Name = "NightlySave" + Type = Backup + Level = Full + Client = LocalMachine + FileSet = "File Set" + Messages = Standard + Storage = DDS-4 + Pool = Daily + Schedule = "NightlySave" + } + # Definition of file storage device + Storage { + Name = DDS-4 + Address = localhost + SDPort = 9103 + Password = XXXXXXXXXXXXX + Device = FileStorage + Media Type = 8mm + } + FileSet { + Name = "File Set" + Include { + Options { + signature=MD5 + } + File = fffffffffffffffff + } + Exclude { File=*.o } + } + Pool { + Name = Daily + Pool Type = Backup + AutoPrune = yes + VolumeRetention = 10d # recycle in 10 days + Maximum Volumes = 10 + Recycle = yes + } + Pool { + Name = Weekly + Use Volume Once = yes + Pool Type = Backup + AutoPrune = yes + VolumeRetention = 30d # recycle in 30 days (default) + Recycle = yes + } + Pool { + Name = Monthly + Use Volume Once = yes + Pool Type = Backup + AutoPrune = yes + VolumeRetention = 365d # recycle in 1 year + Recycle = yes + } + +.. raw:: latex + + + +.. _PruningExample: + +Automatic Pruning and Recycling Example +--------------------------------------- + +.. index:: + pair: Automatic; Pruning and Recycling Example +.. index:: + pair: Example; Automatic Pruning and Recycling + +.. index:: + triple: Pruning; Automatic; Example; + :raw-latex:`\index[general]{Recycle!Automatic!Example}` + +Perhaps the best way to understand the various resource records that come into play during automatic pruning and recycling is to run a Job that goes through the whole cycle. If you add the following resources to your Director’s configuration file: + +.. raw:: latex + + + + + + Schedule { + Name = "30 minute cycle" + Run = Level=Full Pool=File Messages=Standard Storage=File + hourly at 0:05 + Run = Level=Full Pool=File Messages=Standard Storage=File + hourly at 0:35 + } + Job { + Name = "Filetest" + Type = Backup + Level = Full + Client=XXXXXXXXXX + FileSet="Test Files" + Messages = Standard + Storage = File + Pool = File + Schedule = "30 minute cycle" + } + # Definition of file storage device + Storage { + Name = File + Address = XXXXXXXXXXX + SDPort = 9103 + Password = XXXXXXXXXXXXX + Device = FileStorage + Media Type = File + } + FileSet { + Name = "File Set" + Include { + Options { + signature=MD5 + } + File = fffffffffffffffff + } + Exclude { File=*.o } + } + Pool { + Name = File + Use Volume Once = yes + Pool Type = Backup + LabelFormat = "File" + AutoPrune = yes + VolumeRetention = 4h + Maximum Volumes = 12 + Recycle = yes + } + +.. raw:: latex + + + +Where you will need to replace the **ffffffffff**\ ’s by the appropriate files to be saved for your configuration. For the FileSet Include, choose a directory that has one or two megabytes maximum since there will probably be approximately eight copies of the directory that **Bareos** will cycle through. + +In addition, you will need to add the following to your Storage daemon’s configuration file: + +.. raw:: latex + + + + + + Device { + Name = FileStorage + Media Type = File + Archive Device = /tmp + LabelMedia = yes; + Random Access = Yes; + AutomaticMount = yes; + RemovableMedia = no; + AlwaysOpen = no; + } + +.. raw:: latex + + + +With the above resources, Bareos will start a Job every half hour that saves a copy of the directory you chose to /tmp/File0001 ... /tmp/File0012. After 4 hours, Bareos will start recycling the backup Volumes (/tmp/File0001 ...). You should see this happening in the output produced. Bareos will automatically create the Volumes (Files) the first time it uses them. + +To turn it off, either delete all the resources you’ve added, or simply comment out the **Schedule** record in the **Job** resource. + +.. _manualrecycling: + +Manually Recycling Volumes +-------------------------- + +.. index:: + pair: Recycle; Manual + + +Although automatic recycling of Volumes is implemented (see the :ref:`RecyclingChapter` chapter of this manual), you may want to manually force reuse (recycling) of a Volume. + +Assuming that you want to keep the Volume name, but you simply want to write new data on the tape, the steps to take are: + +- Use the :strong:`update volume` command in the Console to ensure that Recycle = **yes**. + +- Use the :strong:`purge jobs volume` command in the Console to mark the Volume as **Purged**. Check by using :strong:`list volumes`. + +Once the Volume is marked Purged, it will be recycled the next time a Volume is needed. + +If you wish to reuse the tape by giving it a new name, use the :strong:`relabel` instead of the :strong:`purge` command. + +.. raw:: latex + + +.. warning:: + The :strong:`delete` command can be dangerous. Once it is + done, to recover the File records, you must either restore your database as it + was before the :strong:`delete` command or use the :ref:`bscan` utility program to + scan the tape and recreate the database entries. diff --git a/docs/manuals/en/new_main_reference/source/chapter18/pools.rst b/docs/manuals/en/new_main_reference/source/chapter18/pools.rst new file mode 100644 index 00000000000..a8a2c0bc9b4 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter18/pools.rst @@ -0,0 +1,382 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _PoolsChapter: + +Automated Disk Backup +===================== + +.. index:: + single: Automated Disk Backup}` :raw-latex:`\index[general]{Pool + + +If you manage five or ten machines and have a nice tape backup, you don’t need Pools, and you may wonder what they are good for. In this chapter, you will see that Pools can help you optimize disk storage space. The same techniques can be applied to a shop that has multiple tape drives, or that wants to mount various different Volumes to meet their needs. + +The rest of this chapter will give an example involving backup to disk Volumes, but most of the information applies equally well to tape Volumes. + +Given is a scenario, where the size of a full backup is about 15GB. + +It is required, that backup data is available for six months. Old files should be available on a daily basis for a week, a weekly basis for a month, then monthly for six months. In addition, offsite capability is not needed. The daily changes amount to about 300MB on the average, or about 2GB per week. + +As a consequence, the total volume of data they need to keep to meet their needs is about 100GB (15GB x 6 + 2GB x 5 + 0.3 x 7) = 102.1GB. + +The chosen solution was to use a 120GB hard disk – far less than 1/10th the price of a tape drive and the cassettes to handle the same amount of data, and to have the backup software write to disk files. + +The rest of this chapter will explain how to setup Bareos so that it would automatically manage a set of disk files with the minimum sysadmin intervention. + +.. _OverallDesign: + +Overall Design +-------------- + +Getting Bareos to write to disk rather than tape in the simplest case is rather easy. + +One needs to consider about what happens if we have only a single large Bareos Volume defined on our hard disk. Everything works fine until the Volume fills, then Bareos will ask you to mount a new Volume. This same problem applies to the use of tape Volumes if your tape fills. Being a hard disk and the only one you have, this will be a bit of a problem. It should be obvious that it is better to use a number of smaller Volumes and arrange for Bareos to automatically recycle them so that the disk +storage space can be reused. + +As mentioned, the solution is to have multiple Volumes, or files on the disk. To do so, we need to limit the use and thus the size of a single Volume, by time, by number of jobs, or by size. Any of these would work, but we chose to limit the use of a single Volume by putting a single job in each Volume with the exception of Volumes containing Incremental backup where there will be 6 jobs (a week’s worth of data) per volume. The details of this will be discussed shortly. This is a single client +backup, so if you have multiple clients you will need to multiply those numbers by the number of clients, or use a different system for switching volumes, such as limiting the volume size. + +.. raw:: latex + + \TODO{This chapter will get rewritten. Instead of limiting a Volume to one job, we will utilize \variable{Max Use Duration = 24 hours}. This prevents problems when adding more clients, because otherwise each job has to run seperat.} + +The next problem to resolve is recycling of Volumes. As you noted from above, the requirements are to be able to restore monthly for 6 months, weekly for a month, and daily for a week. So to simplify things, why not do a Full save once a month, a Differential save once a week, and Incremental saves daily. Now since each of these different kinds of saves needs to remain valid for differing periods, the simplest way to do this (and possibly the only) is to have a separate Pool for each backup +type. + +The decision was to use three Pools: one for Full saves, one for Differential saves, and one for Incremental saves, and each would have a different number of volumes and a different Retention period to accomplish the requirements. + + + +.. _`FullPool`: FullPool + +Full Pool +~~~~~~~~~ + +.. index:: + single: Full Pool + + +Putting a single Full backup on each Volume, will require six Full save Volumes, and a retention period of six months. The Pool needed to do that is: + + + + +.. code-block:: sh + :caption: Full-Pool + + Pool { + Name = Full-Pool + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 6 months + Maximum Volume Jobs = 1 + Label Format = Full- + Maximum Volumes = 9 + } + +Since these are disk Volumes, no space is lost by having separate Volumes for each backup (done once a month in this case). The items to note are the retention period of six months (i.e. they are recycled after six months), that there is one job per volume (Maximum Volume Jobs = 1), the volumes will be labeled Full-0001, ... Full-0006 automatically. One could have labeled these manually from the start, but why not use the features of Bareos. + +Six months after the first volume is used, it will be subject to pruning and thus recycling, so with a maximum of 9 volumes, there should always be 3 volumes available (note, they may all be marked used, but they will be marked purged and recycled as needed). + +If you have two clients, you would want to set **Maximum Volume Jobs** to 2 instead of one, or set a limit on the size of the Volumes, and possibly increase the maximum number of Volumes. + + + +.. _`DiffPool`: DiffPool + +Differential Pool +~~~~~~~~~~~~~~~~~ + +.. index:: + single: Differential Pool + + +For the Differential backup Pool, we choose a retention period of a bit longer than a month and ensure that there is at least one Volume for each of the maximum of five weeks in a month. So the following works: + + + + +.. code-block:: sh + :caption: Differential Pool + + Pool { + Name = Diff-Pool + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 40 days + Maximum Volume Jobs = 1 + Label Format = Diff- + Maximum Volumes = 10 + } + +As you can see, the Differential Pool can grow to a maximum of 9 volumes, and the Volumes are retained 40 days and thereafter they can be recycled. Finally there is one job per volume. This, of course, could be tightened up a lot, but the expense here is a few GB which is not too serious. + +If a new volume is used every week, after 40 days, one will have used 7 volumes, and there should then always be 3 volumes that can be purged and recycled. + +See the discussion above concering the Full pool for how to handle multiple clients. + + + +.. _`IncPool`: IncPool + +Incremental Pool +~~~~~~~~~~~~~~~~ + +.. index:: + single: Incremental Pool +.. index:: + pair: Pool; Incremental + + +Finally, here is the resource for the Incremental Pool: + + + + +.. code-block:: sh + :caption: Incremental Pool + + Pool { + Name = Inc-Pool + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 20 days + Maximum Volume Jobs = 6 + Label Format = Inc- + Maximum Volumes = 7 + } + +We keep the data for 20 days rather than just a week as the needs require. To reduce the proliferation of volume names, we keep a week’s worth of data (6 incremental backups) in each Volume. In practice, the retention period should be set to just a bit more than a week and keep only two or three volumes instead of five. Again, the lost is very little and as the system reaches the full steady state, we can adjust these values so that the total disk usage doesn’t exceed the disk capacity. + +If you have two clients, the simplest thing to do is to increase the maximum volume jobs from 6 to 12. As mentioned above, it is also possible limit the size of the volumes. However, in that case, you will need to have a better idea of the volume or add sufficient volumes to the pool so that you will be assured that in the next cycle (after 20 days) there is at least one volume that is pruned and can be recycled. + +Configuration Files +------------------- + +The following example shows you the actual files used, with only a few minor modifications to simplify things. + +The Director’s configuration file is as follows: + + + + +.. code-block:: sh + :caption: bareos-dir.conf + + Director { # define myself + Name = bareos-dir + QueryFile = "/usr/lib/bareos/scripts/query.sql" + Maximum Concurrent Jobs = 1 + Password = "*** CHANGE ME ***" + Messages = Standard + } + + JobDefs { + Name = "DefaultJob" + Type = Backup + Level = Incremental + Client = bareos-fd + FileSet = "Full Set" + Schedule = "WeeklyCycle" + Storage = File + Messages = Standard + Pool = Inc-Pool + Full Backup Pool = Full-Pool + Incremental Backup Pool = Inc-Pool + Differential Backup Pool = Diff-Pool + Priority = 10 + Write Bootstrap = "/var/lib/bareos/%c.bsr" + } + + Job { + Name = client + Client = client-fd + JobDefs = "DefaultJob" + FileSet = "Full Set" + } + + # Backup the catalog database (after the nightly save) + Job { + Name = "BackupCatalog" + Client = client-fd + JobDefs = "DefaultJob" + Level = Full + FileSet="Catalog" + Schedule = "WeeklyCycleAfterBackup" + # This creates an ASCII copy of the catalog + # Arguments to make_catalog_backup.pl are: + # make_catalog_backup.pl + RunBeforeJob = "/usr/lib/bareos/scripts/make_catalog_backup.pl MyCatalog" + # This deletes the copy of the catalog + RunAfterJob = "/usr/lib/bareos/scripts/delete_catalog_backup" + # This sends the bootstrap via mail for disaster recovery. + # Should be sent to another system, please change recipient accordingly + Write Bootstrap = "|/usr/sbin/bsmtp -h localhost -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" root@localhost" + Priority = 11 # run after main backup + } + + # Standard Restore template, to be changed by Console program + Job { + Name = "RestoreFiles" + Type = Restore + Client = client-fd + FileSet="Full Set" + Storage = File + Messages = Standard + Pool = Default + Where = /tmp/bareos-restores + } + + # List of files to be backed up + FileSet { + Name = "Full Set" + Include = { + Options { + signature=SHA1; + compression=GZIP9 + } + File = / + File = /usr + File = /home + File = /boot + File = /var + File = /opt + } + Exclude = { + File = /proc + File = /tmp + File = /.journal + File = /.fsck + ... + } + } + + Schedule { + Name = "WeeklyCycle" + Run = Level=Full 1st sun at 2:05 + Run = Level=Differential 2nd-5th sun at 2:05 + Run = Level=Incremental mon-sat at 2:05 + } + + # This schedule does the catalog. It starts after the WeeklyCycle + Schedule { + Name = "WeeklyCycleAfterBackup" + Run = Level=Full sun-sat at 2:10 + } + + # This is the backup of the catalog + FileSet { + Name = "Catalog" + Include { + Options { + signature = MD5 + } + File = "/var/lib/bareos/bareos.sql" # database dump + File = "/etc/bareos" # configuration + } + } + + Client { + Name = client-fd + Address = client + FDPort = 9102 + Password = " *** CHANGE ME ***" + AutoPrune = yes # Prune expired Jobs/Files + Job Retention = 6 months + File Retention = 60 days + } + + Storage { + Name = File + Address = localhost + Password = " *** CHANGE ME ***" + Device = FileStorage + Media Type = File + } + + Catalog { + Name = MyCatalog + dbname = bareos; user = bareos; password = "" + } + + Pool { + Name = Full-Pool + Pool Type = Backup + Recycle = yes # automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 6 months + Maximum Volume Jobs = 1 + Label Format = Full- + Maximum Volumes = 9 + } + + Pool { + Name = Inc-Pool + Pool Type = Backup + Recycle = yes # automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 20 days + Maximum Volume Jobs = 6 + Label Format = Inc- + Maximum Volumes = 7 + } + + Pool { + Name = Diff-Pool + Pool Type = Backup + Recycle = yes + AutoPrune = yes + Volume Retention = 40 days + Maximum Volume Jobs = 1 + Label Format = Diff- + Maximum Volumes = 10 + } + + Messages { + Name = Standard + mailcommand = "bsmtp -h mail.domain.com -f \"\(Bareos\) %r\" + -s \"Bareos: %t %e of %c %l\" %r" + operatorcommand = "bsmtp -h mail.domain.com -f \"\(Bareos\) %r\" + -s \"Bareos: Intervention needed for %j\" %r" + mail = root@domain.com = all, !skipped + operator = root@domain.com = mount + console = all, !skipped, !saved + append = "/home/bareos/bin/log" = all, !skipped + } + +and the Storage daemon’s configuration file is: + + + + +.. code-block:: sh + :caption: bareos-sd.conf + + Storage { # definition of myself + Name = bareos-sd + } + + Director { + Name = bareos-dir + Password = " *** CHANGE ME ***" + } + + Device { + Name = FileStorage + Media Type = File + Archive Device = /var/lib/bareos/storage + LabelMedia = yes; # lets Bareos label unlabeled media + Random Access = yes; + AutomaticMount = yes; # when device opened, read it + RemovableMedia = no; + AlwaysOpen = no; + } + + Messages { + Name = Standard + director = bareos-dir = all + } diff --git a/docs/manuals/en/new_main_reference/source/chapter19/autochangers.rst b/docs/manuals/en/new_main_reference/source/chapter19/autochangers.rst new file mode 100644 index 00000000000..2d7f86e0da9 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter19/autochangers.rst @@ -0,0 +1,899 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _AutochangersChapter: + +Autochanger Support +=================== + +.. index:: + pair: Support; Autochanger +.. index:: + pair: Autochanger; Support + + +Bareos provides autochanger support for reading and writing tapes. In order to work with an autochanger, Bareos requires a number of things, each of which is explained in more detail after this list: + +- The package **bareos-storage-tape** must be installed. + +- A script that actually controls the autochanger according to commands sent by Bareos. Bareos contains the script :program:`mtx-changer`, that utilize the command :program:`mtx`. It’s config file is normally located at :file:`/etc/bareos/mtx-changer.conf` + +- That each Volume (tape) to be used must be defined in the Catalog and have a Slot number assigned to it so that Bareos knows where the Volume is in the autochanger. This is generally done with the :strong:`label` command, but can also done after the tape is labeled using the :strong:`update slots` command. See below for more details. You must pre-label the tapes manually before using them. + +- Modifications to your Storage daemon’s Device configuration resource to identify that the device is a changer, as well as a few other parameters. + +- You need to ensure that your Storage daemon has access permissions to both the tape drive and the control device. On Linux, the system user **bareos}` is added to the groups :raw-latex:`\group{disk}` and :raw-latex:`\group{tape**, so that it should have the permission to access the library. + +- Set **Auto Changer**:sup:`Dir`:sub:`Storage` = **yes**. + +Bareos uses its own :program:`mtx-changer` script to interface with a program that actually does the tape changing. Thus in principle, :program:`mtx-changer` can be adapted to function with any autochanger program, or you can call any other script or program. The current version of :program:`mtx-changer` works with the :program:`mtx` program. FreeBSD users might need to adapt this script to use :program:`chio`. For more details, refer +to the :ref:`Testing Autochanger ` chapter. + +Bareos also supports autochangers with barcode readers. This support includes two Console commands: :strong:`label barcodes` and :strong:`update slots`. For more details on these commands, see the chapter about :ref:`Barcodes`. + +Current Bareos autochanger support does not include cleaning, stackers, or silos. Stackers and silos are not supported because Bareos expects to be able to access the Slots randomly. However, if you are very careful to setup Bareos to access the Volumes in the autochanger sequentially, you may be able to make Bareos work with stackers (gravity feed and such). + +In principle, if :program:`mtx` will operate your changer correctly, then it is just a question of adapting the :program:`mtx-changer` script (or selecting one already adapted) for proper interfacing. + +If you are having troubles, please use the **auto** command in the :program:`btape` program to test the functioning of your autochanger with Bareos. Please remember, that on most distributions, the |bareosSd| runs as user **bareos}` and not as :raw-latex:`\user{root**. You will need to ensure that the Storage daemon has sufficient permissions to access the autochanger. + +Some users have reported that the the Storage daemon blocks under certain circumstances in trying to mount a volume on a drive that has a different volume loaded. As best we can determine, this is simply a matter of waiting a bit. The drive was previously in use writing a Volume, and sometimes the drive will remain BLOCKED for a good deal of time (up to 7 minutes on a slow drive) waiting for the cassette to rewind and to unload before the drive can be used with a different Volume. + +.. _SCSI devices: + +Knowing What SCSI Devices You Have +---------------------------------- + +.. index:: + single: SCSI devices +.. index:: + pair: Devices; SCSI +.. index:: + pair: Devices; Detecting + + +Linux +~~~~~ + +Under Linux, you can + +.. raw:: latex + + + + + + cat /proc/scsi/scsi + +.. raw:: latex + + + +to see what SCSI devices you have available. You can also: + +.. raw:: latex + + + + + + cat /proc/scsi/sg/device_hdr /proc/scsi/sg/devices + +.. raw:: latex + + + +to find out how to specify their control address (**/dev/sg0** for the first, **/dev/sg1** for the second, ...) on the **Changer Device**:sup:`Sd`:sub:`Autochanger` Bareos directive. + +You can also use the excellent **lsscsi** tool. :raw-latex:`` + + + + $ lsscsi -g + [1:0:2:0] tape SEAGATE ULTRIUM06242-XXX 1619 /dev/st0 /dev/sg9 + [1:0:14:0] mediumx STK L180 0315 /dev/sch0 /dev/sg10 + [2:0:3:0] tape HP Ultrium 3-SCSI G24S /dev/st1 /dev/sg11 + [3:0:0:0] enclosu HP A6255A HP04 - /dev/sg3 + [3:0:1:0] disk HP 36.4G ST336753FC HP00 /dev/sdd /dev/sg4 + +.. raw:: latex + + + +FreeBSD +~~~~~~~ + +Under FreeBSD, use the following command to list the SCSI devices as well as the :file:`/dev/passN` that you will use on the Bareos **Changer Device**:sup:`Sd`:sub:`Autochanger` directive: + +.. raw:: latex + + + + + + camcontrol devlist + +.. raw:: latex + + + +Please check that your Storage daemon has permission to access this device. + +The following tip for FreeBSD users comes from Danny Butroyd: on reboot Bareos will NOT have permission to control the device :file:`/dev/pass0` (assuming this is your changer device). To get around this just edit the :file:`/etc/devfs.conf` file and add the following to the bottom: :raw-latex:`` + + + + own pass0 root:bareos + perm pass0 0666 + own nsa0.0 root:bareos + perm nsa0.0 0666 + +.. raw:: latex + + + +This gives the bareos group permission to write to the nsa0.0 device too just to be on the safe side. To bring these changes into effect just run:- + + + + +.. code-block:: sh + :caption: + + /etc/rc.d/devfs restart + +Basically this will stop you having to manually change permissions on these devices to make Bareos work when operating the AutoChanger after a reboot. + +Solaris +~~~~~~~ + +On Solaris, the changer device will typically be some file under :file:`/dev/rdsk`. + +Slots +----- + +.. index:: + single: Slots + + +.. _`Slots`: Slots + +To properly address autochangers, Bareos must know which Volume is in each **slot** of the autochanger. Slots are where the changer cartridges reside when not loaded into the drive. Bareos numbers these slots from one to the number of cartridges contained in the autochanger. + +Bareos will not automatically use a Volume in your autochanger unless it is labeled and the slot number is stored in the catalog and the Volume is marked as InChanger. This is because it must know where each volume is to be able to load the volume. For each Volume in your changer, you will, using the Console program, assign a slot. This information is kept in Bareos’s catalog database along with the other data for the volume. If no slot is given, or the slot is set to zero, Bareos will not +attempt to use the autochanger even if all the necessary configuration records are present. When doing a :strong:`mount` command on an autochanger, you must specify which slot you want mounted. If the drive is loaded with a tape from another slot, it will unload it and load the correct tape, but normally, no tape will be loaded because an :strong:`unmount` command causes Bareos to unload the tape in the drive. + +You can check if the Slot number and InChanger flag by: + + + + +.. code-block:: sh + :caption: list volumes + + *list volumes + +.. _section-MultipleDevices: + +Multiple Devices +---------------- + +.. index:: + single: Multiple Devices + + +Some autochangers have more than one read/write device (drive). The :ref:`Autochanger resource ` permits you to group Device resources, where each device represents a drive. The Director may still reference the Devices (drives) directly, but doing so, bypasses the proper functioning of the drives together. Instead, the Director (in the Storage resource) should reference the Autochanger resource name. Doing so permits the Storage daemon to ensure that only one drive +uses the mtx-changer script at a time, and also that two drives don’t reference the same Volume. + +Multi-drive requires the use of the **Drive Index**:sup:`Sd`:sub:`Device` directive. Drive numbers or the Device Index are numbered beginning at zero, which is the default. To use the second Drive in an autochanger, you need to define a second Device resource, set the **Drive Index**:sup:`Sd`:sub:`Device` and set the **Archive Device**:sup:`Sd`:sub:`Device` . + +As a default, Bareos jobs will prefer to write to a Volume that is already mounted. If you have a multiple drive autochanger and you want Bareos to write to more than one Volume in the same Pool at the same time, you will need to set **Prefer Mounted Volumes**:sup:`Dir`:sub:`Job` = **no**. This will cause the Storage daemon to maximize the use of drives. + +Device Configuration Records +---------------------------- + +.. index:: + single: Device Configuration Records + + +Configuration of autochangers within Bareos is done in the Device resource of the Storage daemon. + +Following records control how Bareos uses the autochanger: + +**Autochanger**:sup:`Sd`:sub:`Device` + Specifies if the current device belongs to an autochanger resource. + +**Changer Command**:sup:`Sd`:sub:`Autochanger` (**Changer Command**:sup:`Sd`:sub:`Device` ) +**Changer Device**:sup:`Sd`:sub:`Autochanger` (**Changer Device**:sup:`Sd`:sub:`Device` ) +**Drive Index**:sup:`Sd`:sub:`Device` + Individual driver number, starting at 0. + +**Maximum Changer Wait**:sup:`Sd`:sub:`Device` + +Specifying Slots When Labeling +------------------------------ + +.. index:: + single: Specifying Slots When Labeling +.. index:: + pair: Label; Specifying Slots When Labeling + + +.. _`SpecifyingSlots`: SpecifyingSlots + +If you add an **Autochanger = yes** record to the Storage resource in your Director’s configuration file, the Bareos Console will automatically prompt you for the slot number when the Volume is in the changer when you **add** or **label** tapes for that Storage device. If your **mtx-changer** script is properly installed, Bareos will automatically load the correct tape during the label command. + +You must also set **Autochanger = yes** in the Storage daemon’s Device resource as we have described above in order for the autochanger to be used. Please see **Auto Changer**:sup:`Dir`:sub:`Storage` and **Autochanger**:sup:`Sd`:sub:`Device` for more details on these records. + +Thus all stages of dealing with tapes can be totally automated. It is also possible to set or change the Slot using the **update** command in the Console and selecting **Volume Parameters** to update. + +Even though all the above configuration statements are specified and correct, Bareos will attempt to access the autochanger only if a **slot** is non-zero in the catalog Volume record (with the Volume name). + +If your autochanger has barcode labels, you can label all the Volumes in your autochanger one after another by using the :strong:`label barcodes` command. For each tape in the changer containing a barcode, Bareos will mount the tape and then label it with the same name as the barcode. An appropriate Media record will also be created in the catalog. Any barcode that begins with the same characters as specified on the "CleaningPrefix=xxx" command, will be treated as a cleaning tape, +and will not be labeled. For example with: + +.. raw:: latex + + + + + + Pool { + Name ... + Cleaning Prefix = "CLN" + } + +.. raw:: latex + + + +Any slot containing a barcode of CLNxxxx will be treated as a cleaning tape and will not be mounted. + +Changing Cartridges +------------------- + +.. index:: + pair: Cartridges; Changing + If you wish to insert or remove cartridges in your autochanger or you manually run the **mtx** program, you must first tell Bareos to release the autochanger by doing: + +.. raw:: latex + + + + + + unmount + (change cartridges and/or run mtx) + mount + +.. raw:: latex + + + +If you do not do the unmount before making such a change, Bareos will become completely confused about what is in the autochanger and may stop function because it expects to have exclusive use of the autochanger while it has the drive mounted. + +Dealing with Multiple Magazines +------------------------------- + +.. index:: + pair: Magazines; Dealing with Multiple + + +If you have several magazines or if you insert or remove cartridges from a magazine, you should notify Bareos of this. By doing so, Bareos will as a preference, use Volumes that it knows to be in the autochanger before accessing Volumes that are not in the autochanger. This prevents unneeded operator intervention. + +If your autochanger has barcodes (machine readable tape labels), the task of informing Bareos is simple. Every time, you change a magazine, or add or remove a cartridge from the magazine, simply use following commands in the Console program: + +.. raw:: latex + + + + + + unmount + (remove magazine) + (insert new magazine) + update slots + mount + +.. raw:: latex + + + +This will cause Bareos to request the autochanger to return the current Volume names in the magazine. This will be done without actually accessing or reading the Volumes because the barcode reader does this during inventory when the autochanger is first turned on. Bareos will ensure that any Volumes that are currently marked as being in the magazine are marked as no longer in the magazine, and the new list of Volumes will be marked as being in the magazine. In addition, the Slot numbers of the +Volumes will be corrected in Bareos’s catalog if they are incorrect (added or moved). + +If you do not have a barcode reader on your autochanger, you have several alternatives. + +#. You can manually set the Slot and InChanger flag using the **update volume** command in the Console (quite painful). + +#. You can issue a + + + + + + + + update slots scan + + + + + + command that will cause Bareos to read the label on each of the cartridges in the magazine in turn and update the information (Slot, InChanger flag) in the catalog. This is quite effective but does take time to load each cartridge into the drive in turn and read the Volume label. + +.. raw:: latex + + \hide{ + % unwanted, commented out + \section{Simulating Barcodes in your Autochanger} + \index[general]{Autochanger!Simulating Barcodes in your} + \index[general]{Simulating Barcodes in your Autochanger} + \label{simulating} + + You can simulate barcodes in your autochanger by making the {\bf mtx-changer} + script return the same information that an autochanger with barcodes would do. + This is done by commenting out the one and only line in the {\bf list)} case, + which is: + + + \ + ${MTX} -f $ctl status | grep " *Storage Element [0-9]*:.*Full" | awk "{print \$3 \$4}" | sed "s/Full *\(:VolumeTag=\)*//" + \ + + + at approximately line 99 by putting a \# in column one of that line, or by + simply deleting it. Then in its place add a new line that prints the contents + of a file. For example: + + + \ + cat /etc/bareos/changer.volumes + \ + + + Be sure to include a full path to the file, which can have any name. The + contents of the file must be of the following format: + + + \ + 1:Volume1 + 2:Volume2 + 3:Volume3 + ... + \ + + + Where the 1, 2, 3 are the slot numbers and Volume1, Volume2, ... are the + Volume names in those slots. You can have multiple files that represent the + Volumes in different magazines, and when you change magazines, simply copy the + contents of the correct file into your {\bf /etc/bareos/changer.volumes} file. + There is no need to stop and start Bareos when you change magazines, simply + put the correct data in the file, then run the {\bf update slots} command, and + your autochanger will appear to Bareos to be an autochanger with barcodes. + } + +Update Slots Command +-------------------- + + +.. index:: + triple: Console; Command; update slots; + + +.. _`updateslots`: updateslots + +If you change only one cartridge in the magazine, you may not want to scan all Volumes, so the **update slots** command (as well as the **update slots scan** command) has the additional form: + +.. raw:: latex + + + + + + update slots=n1,n2,n3-n4, ... + +.. raw:: latex + + + +where the keyword **scan** can be appended or not. The n1,n2, ... represent Slot numbers to be updated and the form n3-n4 represents a range of Slot numbers to be updated (e.g. 4-7 will update Slots 4,5,6, and 7). + +This form is particularly useful if you want to do a scan (time expensive) and restrict the update to one or two slots. + +For example, the command: + +.. raw:: latex + + + + + + update slots=1,6 scan + +.. raw:: latex + + + +will cause Bareos to load the Volume in Slot 1, read its Volume label and update the Catalog. It will do the same for the Volume in Slot 6. The command: + +.. raw:: latex + + + + + + update slots=1-3,6 + +.. raw:: latex + + + +will read the barcoded Volume names for slots 1,2,3 and 6 and make the appropriate updates in the Catalog. If you don’t have a barcode reader the above command will not find any Volume names so will do nothing. + +Using the Autochanger +--------------------- + +.. index:: + pair: Autochanger; Using the + + +.. _`using`: using + +Let’s assume that you have properly defined the necessary Storage daemon Device records, and you have added the **Autochanger = yes** record to the Storage resource in your Director’s configuration file. + +Now you fill your autochanger with say six blank tapes. + +What do you do to make Bareos access those tapes? + +One strategy is to prelabel each of the tapes. Do so by starting Bareos, then with the Console program, enter the **label** command: + +.. raw:: latex + + + + + + ./bconsole + Connecting to Director rufus:8101 + 1000 OK: rufus-dir Version: 1.26 (4 October 2002) + *label + +.. raw:: latex + + + +it will then print something like: + +.. raw:: latex + + + + + + Using default Catalog name=BackupDB DB=bareos + The defined Storage resources are: + 1: Autochanger + 2: File + Select Storage resource (1-2): 1 + +.. raw:: latex + + + +I select the autochanger (1), and it prints: + +.. raw:: latex + + + + + + Enter new Volume name: TestVolume1 + Enter slot (0 for none): 1 + +.. raw:: latex + + + +where I entered **TestVolume1** for the tape name, and slot **1** for the slot. It then asks: + +.. raw:: latex + + + + + + Defined Pools: + 1: Default + 2: File + Select the Pool (1-2): 1 + +.. raw:: latex + + + +I select the Default pool. This will be automatically done if you only have a single pool, then Bareos will proceed to unload any loaded volume, load the volume in slot 1 and label it. In this example, nothing was in the drive, so it printed: + +.. raw:: latex + + + + + + Connecting to Storage daemon Autochanger at localhost:9103 ... + Sending label command ... + 3903 Issuing autochanger "load slot 1" command. + 3000 OK label. Volume=TestVolume1 Device=/dev/nst0 + Media record for Volume=TestVolume1 successfully created. + Requesting mount Autochanger ... + 3001 Device /dev/nst0 is mounted with Volume TestVolume1 + You have messages. + * + +.. raw:: latex + + + +You may then proceed to label the other volumes. The messages will change slightly because Bareos will unload the volume (just labeled TestVolume1) before loading the next volume to be labeled. + +Once all your Volumes are labeled, Bareos will automatically load them as they are needed. + +To "see" how you have labeled your Volumes, simply enter the **list volumes** command from the Console program, which should print something like the following: + +.. raw:: latex + + + + + + *{\bf list volumes} + Using default Catalog name=BackupDB DB=bareos + Defined Pools: + 1: Default + 2: File + Select the Pool (1-2): 1 + +-------+----------+--------+---------+-------+--------+----------+-------+------+ + | MedId | VolName | MedTyp | VolStat | Bites | LstWrt | VolReten | Recyc | Slot | + +-------+----------+--------+---------+-------+--------+----------+-------+------+ + | 1 | TestVol1 | DDS-4 | Append | 0 | 0 | 30672000 | 0 | 1 | + | 2 | TestVol2 | DDS-4 | Append | 0 | 0 | 30672000 | 0 | 2 | + | 3 | TestVol3 | DDS-4 | Append | 0 | 0 | 30672000 | 0 | 3 | + | ... | + +-------+----------+--------+---------+-------+--------+----------+-------+------+ + +.. raw:: latex + + + +Barcode Support +--------------- + +.. index:: + single: Barcode Support + + +.. _`Barcodes`: Barcodes + +Bareos provides barcode support with two Console commands, **label barcodes** and **update slots**. + +The **label barcodes** will cause Bareos to read the barcodes of all the cassettes that are currently installed in the magazine (cassette holder) using the **mtx-changer** **list** command. Each cassette is mounted in turn and labeled with the same Volume name as the barcode. + +The **update slots** command will first obtain the list of cassettes and their barcodes from **mtx-changer**. Then it will find each volume in turn in the catalog database corresponding to the barcodes and set its Slot to correspond to the value just read. If the Volume is not in the catalog, then nothing will be done. This command is useful for synchronizing Bareos with the current magazine in case you have changed magazines or in case you have moved cassettes from one slot to another. If the +autochanger is empty, nothing will be done. + +The **Cleaning Prefix** statement can be used in the Pool resource to define a Volume name prefix, which if it matches that of the Volume (barcode) will cause that Volume to be marked with a VolStatus of **Cleaning**. This will prevent Bareos from attempting to write on the Volume. + +Use bconsole to display Autochanger content +------------------------------------------- + +The **status slots storage=xxx** command displays autochanger content. + +.. raw:: latex + + + + + + Slot | Volume Name | Status | Type | Pool | Loaded | + ------+-----------------+----------+-------------------+----------------+---------| + 1 | 00001 | Append | DiskChangerMedia | Default | 0 | + 2 | 00002 | Append | DiskChangerMedia | Default | 0 | + 3*| 00003 | Append | DiskChangerMedia | Scratch | 0 | + 4 | | | | | 0 | + +.. raw:: latex + + + +If you see a **** near the slot number, you have to run **update slots** command to synchronize autochanger content with your catalog. + +Bareos Autochanger Interface +---------------------------- + +.. index:: + pair: Autochanger; Interface + + +.. _`autochanger-interface`: autochanger-interface + +Bareos calls the autochanger script that you specify on the **Changer Command** statement. Normally this script will be the **mtx-changer** script that we provide, but it can in fact be any program. The only requirement for the script is that it must understand the commands that Bareos uses, which are **loaded**, **load**, **unload**, **list**, and **slots**. In addition, each of those commands must return the information in the precise format as specified below: + +.. raw:: latex + + + + + + - Currently the changer commands used are: + loaded -- returns number of the slot that is loaded, base 1, + in the drive or 0 if the drive is empty. + load -- loads a specified slot (note, some autochangers + require a 30 second pause after this command) into + the drive. + unload -- unloads the device (returns cassette to its slot). + list -- returns one line for each cassette in the autochanger + in the format :. Where + the {\bf slot} is the non-zero integer representing + the slot number, and {\bf barcode} is the barcode + associated with the cassette if it exists and if you + autoloader supports barcodes. Otherwise the barcode + field is blank. + slots -- returns total number of slots in the autochanger. + +.. raw:: latex + + + +Bareos checks the exit status of the program called, and if it is zero, the data is accepted. If the exit status is non-zero, Bareos will print an error message and request the tape be manually mounted on the drive. + +Tapespeed and blocksizes +------------------------ + +.. index:: + pair: Tuning; Tape +.. index:: + pair: Tuning; blocksize +.. index:: + pair: Tape; speed + :raw-latex:`\index[general]{Blocksize!optimize}` + +.. _`Tapespeed and blocksizes}` :raw-latex:`\label{setblocksizes`: Tapespeed and blocksizes}` :raw-latex:`\label{setblocksizes + +The :raw-latex:`\bareosWhitepaperTapeSpeedTuning `shows that the two parameters :strong:`Maximum File Size` and :strong:`Maximum Block Size` of the device have significant influence on the tape speed. + +While it is no problem to change the **Maximum File Size**:sup:`Sd`:sub:`Device` parameter, unfortunately it is not possible to change the **Maximum Block Size**:sup:`Sd`:sub:`Device` parameter, because the previously written tapes would become unreadable in the new setup. It would require that the **Maximum Block Size**:sup:`Sd`:sub:`Device` parameter is switched back to the old value to be able to read the old volumes, but of +course then the new volumes would be unreadable. + +Why is that the case? + +The problem is that Bareos writes the label block (header) in the same block size that is configured in the **Maximum Block Size**:sup:`Sd`:sub:`Device` parameter in the device. Per default, this value is 63k, so usually a tape written by Bareos looks like this: + + + + |------------------- + |label block (63k)| + |------------------- + |data block 1(63k)| + |data block 2(63k)| + |... | + |data block n(63k)| + -------------------- + +Setting the maximum block size to e.g. 512k, would lead to the following: + + + + |------------------- + |label block (512k)| + |------------------- + |data block 1(512k)| + |data block 2(512k)| + |... | + |data block n(512k)| + -------------------- + +As you can see, every block is written with the maximum block size, also the label block. + +The problem that arises here is that reading a block header with a wrong block size causes a read error which is interpreted as an non-existent label by Bareos. + +This is a potential source of data loss, because in normal operation, Bareos refuses to relabel an already labeled volume to be sure to not overwrite data that is still needed. If Bareos cannot read the volume label, this security mechanism does not work and you might label tapes already labeled accidentally. + +To solve this problem, the block size handling was changed in Bareos 14.2.0 in the following way: + +- The tape label block is always written in the standard 63k (64512) block size. + +- The following blocks are then written in the block size configured in the :strong:`Maximum Block Size` directive. + +- To be able to change the block size in an existing environment, it is now possible to set the **Maximum Block Size**:sup:`Dir`:sub:`Pool` and **Minimum Block Size**:sup:`Dir`:sub:`Pool` in the pool resource. This setting is automatically promoted to each medium in that pool as usual (i.e. when a medium is labeled for that pool or if a volume is transferred to that pool from the scratch pool). When a volume is mounted, the volume’s block size is + used to write and read the data blocks that follow the header block. + +The following picture shows the result: + + + + |--------------------------------| + |label block (label block size) | + |--------------------------------| + |data block 1(maximum block size)| + |data block 2(maximum block size)| + |... | + |data block n(maximum block size)| + ---------------------------------| + +We have a label block with a certain size (63k per default to be compatible to old installations), and the following data blocks are written with another blocksize. + +This approach has the following advantages: + +- If nothing is configured, existing installations keep on working without problems. + +- If you want to switch an existing installation that uses the default block size and move to a new (usually bigger) block size, you can do that easily by creating a new pool, where **Maximum Block Size**:sup:`Dir`:sub:`Pool` is set to the new value that you wish to use in the future: + + + + +.. code-block:: sh + :caption: Pool Ressource: setting Maximum Block Size + + Pool { + Name = LTO-4-1M + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 1 Month # How long should the Full Backups be kept? (#06) + Maximum Block Size = 1048576 + Recycle Pool = Scratch + } + +Now configure your backups that they will write into the newly defined pool in the future, and your backups will be written with the new block size. + +Your existing tapes can be automatically transferred to the new pool when they expire via the :ref:`Scratch Pool ` mechanism. When a tape in your old pool expires, it is transferred to the scratch pool if you set **Recycle Pool = Scratch**. When your new pool needs a new volume, it will get it from the scratch pool and apply the new pool’s properties to that tape which also include **Maximum Block Size**:sup:`Dir`:sub:`Pool` and +**Minimum Block Size**:sup:`Dir`:sub:`Pool` . + +This way you can smoothly switch your tapes to a new block size while you can still restore the data on your old tapes at any time. + +Possible Problems +~~~~~~~~~~~~~~~~~ + +There is only one case where the new block handling will cause problems, and this is if you have used bigger block sizes already in your setup. As we now defined the label block to always be 63k, all labels will not be readable. + +To also solve this problem, the directive **Label Block Size**:sup:`Sd`:sub:`Device` can be used to define a different label block size. That way, everything should work smoothly as all label blocks will be readable again. + +How can I find out which block size was used when the tape was written? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +At least on Linux, you can see if Bareos tries to read the blocks with the wrong block size. In that case, you get a kernel message like the following in your system’s messages: + + + + [542132.410170] st1: Failed to read 1048576 byte block with 64512 byte transfer. + +Here, the block was written with 1M block size but we only read 64k. + +.. _direct-access-to-volumes-with-non-default-blocksizes: + +Direct access to Volumes with with non-default block sizes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: bls; block size +.. index:: + pair: bextract; block size + +.. index:: + triple: Command; bls; block size; + :raw-latex:`\index[general]{Command!bextract!block size}` + +:program:`bls` and :program:`bextract` can directly access Bareos volumes without catalog database. This means that these programs don’t have information about the used block size. + +To be able to read a volume written with an arbitrary block size, you need to set the **Label Block Size**:sup:`Sd`:sub:`Device` (to be able to to read the label block) and the **Maximum Block Size**:sup:`Sd`:sub:`Device` (to be able to read the data blocks) setting in the device definition used by those tools to be able to open the medium. + +Example using :program:`bls` with a tape that was written with another blocksize than the :raw-latex:`\variable{DEFAULT_BLOCK_SIZE}` (63k), but with the default label block size of 63k: + + + + +.. code-block:: sh + :caption: bls with non-default block size + + bls FC-Drive-1 -V A00007L4 + bls: butil.c:289-0 Using device: "FC-Drive-1" for reading. + 25-Feb 12:47 bls JobId 0: No slot defined in catalog (slot=0) for Volume "A00007L4" on "FC-Drive-1" (/dev/tape/by-id/scsi-350011d00018a5f03-nst). + 25-Feb 12:47 bls JobId 0: Cartridge change or "update slots" may be required. + 25-Feb 12:47 bls JobId 0: Ready to read from volume "A00007L4" on device "FC-Drive-1" (/dev/tape/by-id/scsi-350011d00018a5f03-nst). + 25-Feb 12:47 bls JobId 0: Error: block.c:1004 Read error on fd=3 at file:blk 0:1 on device "FC-Drive-1" (/dev/tape/by-id/scsi-350011d00018a5f03-nst). ERR=Cannot allocate memory. + Bareos status: file=0 block=1 + Device status: ONLINE IM_REP_EN file=0 block=2 + 0 files found. + +As can be seen, :program:`bls` manages to read the label block as it knows what volume is mounted (Ready to read from volume :option:`A00007L4`), but fails to read the data blocks. + + + + +.. code-block:: sh + :caption: dmesg + + dmesg + [...] + st2: Failed to read 131072 byte block with 64512 byte transfer. + [...] + +This shows that the block size for the data blocks that we need is 131072. + +Now we have to set this block size in the :file:`bareos-sd.conf`, device resource as **Maximum Block Size**:sup:`Sd`:sub:`Device` : + + + + +.. code-block:: sh + :caption: Storage Device Resource: setting Maximum Block Size + + Device { + Name = FC-Drive-1 + Drive Index = 0 + Media Type = LTO-4 + Archive Device = /dev/tape/by-id/scsi-350011d00018a5f03-nst + AutomaticMount = yes + AlwaysOpen = yes + RemovableMedia = yes + RandomAccess = no + AutoChanger = yes + Maximum Block Size = 131072 + } + +Now we can call bls again, and everything works as expected: + + + + +.. code-block:: sh + :caption: bls with non-default block size + + bls FC-Drive-1 -V A00007L4 + bls: butil.c:289-0 Using device: "FC-Drive-1" for reading. + 25-Feb 12:49 bls JobId 0: No slot defined in catalog (slot=0) for Volume "A00007L4" on "FC-Drive-1" (/dev/tape/by-id/scsi-350011d00018a5f03-nst). + 25-Feb 12:49 bls JobId 0: Cartridge change or "update slots" may be required. + 25-Feb 12:49 bls JobId 0: Ready to read from volume "A00007L4" on device "FC-Drive-1" (/dev/tape/by-id/scsi-350011d00018a5f03-nst). + bls JobId 203: [...] + +How to configure the block sizes in your environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following chart shows how to set the directives for **maximum block size** and **label block size** depending on how your current setup is: + +|image| + +Tape Drive Cleaning +------------------- + +Bareos has no build-in functionality for tape drive cleaning. Fortunately this is not required as most modern tape libraries have build in auto-cleaning functionality. This functionality might require an empty tape drive, so the tape library gets aware, that it is currently not used. However, by default Bareos keeps tapes in the drives, in case the same tape is required again. + +The directive **Cleaning Prefix**:sup:`Dir`:sub:`Pool` is only used for making sure that Bareos does not try to write backups on a cleaning tape. + +If your tape libraries auto-cleaning won’t work when there are tapes in the drives, it’s probably best to set up an admin job that removes the tapes from the drives. This job has to run, when no other backups do run. A job definition for an admin job to do that may look like this: + + + + +.. code-block:: sh + :caption: bareos-dir job ReleaseAllTapeDrives + + Job { + Name = ReleaseAllTapeDrives + JobDefs = DefaultJob + Schedule = "WeeklyCycleAfterBackup" + Type = Admin + Priority = 200 + + RunScript { + Runs When = Before + Runs On Client = no + Console = "release storage=Tape alldrives" + } + } + +Replace **Tape**:sup:`Dir`:sub:`Storage` by the storage name of your tape library. Use the highest **Priority**:sup:`Dir`:sub:`Job` value to make sure no other jobs are running. In the default configuration for example, the **CatalogBackup**:sup:`Dir`:sub:`job` job has Priority = 100. The higher the number, the lower the job priority. + +.. |image| image:: \idir blocksize-decisionchart + diff --git a/docs/manuals/en/new_main_reference/source/chapter20/tape-without-autochanger.rst b/docs/manuals/en/new_main_reference/source/chapter20/tape-without-autochanger.rst new file mode 100644 index 00000000000..a8fd27ee5da --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter20/tape-without-autochanger.rst @@ -0,0 +1,416 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _StrategiesChapter: + +Using Tape Drives without Autochanger +===================================== + +.. index:: + single: Backup Strategies + + +Although Recycling and Backing Up to Disk Volume have been discussed in previous chapters, this chapter is meant to give you an overall view of possible backup strategies and to explain their advantages and disadvantages. + +.. _`Simple`: Simple + +Simple One Tape Backup +---------------------- + +.. index:: + single: One Tape Backup + + +Probably the simplest strategy is to back everything up to a single tape and insert a new (or recycled) tape when it fills and Bareos requests a new one. + +Advantages +~~~~~~~~~~ + +- The operator intervenes only when a tape change is needed (e.g. once a month). + +- There is little chance of operator error because the tape is not changed daily. + +- A minimum number of tapes will be needed for a full restore. Typically the best case will be one tape and worst two. + +- You can easily arrange for the Full backup to occur a different night of the month for each system, thus load balancing and shortening the backup time. + +Disadvantages +~~~~~~~~~~~~~ + +- If your site burns down, you will lose your current backups + +- After a tape fills and you have put in a blank tape, the backup will continue, and this will generally happen during working hours. + +Practical Details +~~~~~~~~~~~~~~~~~ + +This system is very simple. When the tape fills and Bareos requests a new tape, you **unmount** the tape from the Console program, insert a new tape and **label** it. In most cases after the label, Bareos will automatically mount the tape and resume the backup. Otherwise, you simply **mount** the tape. + +Using this strategy, one typically does a Full backup once a week followed by daily Incremental backups. To minimize the amount of data written to the tape, one can do a Full backup once a month on the first Sunday of the month, a Differential backup on the 2nd-5th Sunday of the month, and incremental backups the rest of the week. + +.. _`Manual`: Manual + +Manually Changing Tapes +----------------------- + +.. index:: + pair: Tape; Manually Changing + + +If you use the strategy presented above, Bareos will ask you to change the tape, and you will **unmount** it and then remount it when you have inserted the new tape. + +If you do not wish to interact with Bareos to change each tape, there are several ways to get Bareos to release the tape: + +- In your Storage daemon’s Device resource, set :strong:`AlwaysOpen = no`. In this case, Bareos will release the tape after every job. If you run several jobs, the tape will be rewound and repositioned to the end at the beginning of every job. This is not very efficient, but does let you change the tape whenever you want. + +- Use a **RunAfterJob** statement to run a script after your last job. This could also be an **Admin** job that runs after all your backup jobs. The script could be something like: + + + + + + + + #!/bin/sh + bconsole < + Always Incremental Keep Number = + ... + } + +**Accurate**:sup:`Dir`:sub:`Job` = **yes** + is required to detect deleted files and prevent that they are kept in the consolidated backup jobs. + +**Always Incremental**:sup:`Dir`:sub:`Job` = **yes** + enables the Always Incremental feature. + +**Always Incremental Job Retention**:sup:`Dir`:sub:`Job` + set the age where incrementals of this job will be kept, older jobs will be consolidated. + +**Always Incremental Keep Number**:sup:`Dir`:sub:`Job` + sets the number of incrementals that will be kept without regarding the age. This should make sure that a certain history of a job will be kept even if the job is not executed for some time. + +**Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` + is described later, see :ref:`section-AlwaysIncrementalMaxFullAge`. + +Consolidate Job +~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: bareos-dir job Consolidate + + Job { + Name = "Consolidate" + Type = "Consolidate" + Accurate = "yes" + JobDefs = "DefaultJob" + } + +**Type**:sup:`Dir`:sub:`Job` = **Consolidate** + configures a job to be a consolidate job. This type have been introduced with the Always Incremental feature. When used, it automatically trigger the consolidation of incremental jobs that need to be consolidated. + +**Accurate**:sup:`Dir`:sub:`Job` = **yes** + let the generated virtual backup job keep the accurate information. + +**Max Full Consolidations**:sup:`Dir`:sub:`Job` + is described later, see :ref:`section-MaxFullConsolidations`. + +The **Consolidate**:sup:`Dir`:sub:`job` job evaluates all jobs configured with **Always Incremental**:sup:`Dir`:sub:`Job` = **yes**. When a job is selected for consolidation, all job runs are taken into account, independent of the pool and storage where they are located. + +The always incremental jobs need to be executed during the backup window (usually at night), while the consolidation jobs should be scheduled during the daytime when no backups are executed. + +.. raw:: latex + + +.. warning:: + All Bareos job resources have some required directives, e.g. **Client**:sup:`Dir`:sub:`Job` . + Even so, none other than the mentioned directives are evaluated by a \resourceDirectiveValue{Dir{Job}{Type}{Consolidate}, + they still have to be defined. + Normally all required directives are already set in \resourceDirectiveValue{Dir}{Job}{Job Defs}{DefaultJob}. + If not, you have to add them. You can use arbitrary, but valid values.} + +Storages and Pools +~~~~~~~~~~~~~~~~~~ + +For the Always Incremental Backup Scheme at least two storages are needed. See :ref:`section-MultipleStorageDevices` how to setup multiple storages. + + + + +.. code-block:: sh + :caption: bareos-dir pool AI-Incremental + + Pool { + Name = AI-Incremental + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + Auto Prune = yes # Prune expired volumes + Volume Retention = 360 days # How long should jobs be kept? + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Incremental-" + Volume Use Duration = 23h + Storage = File1 + Next Pool = AI-Consolidated # consolidated jobs go to this pool + } + + + + +.. code-block:: sh + :caption: bareos-dir pool AI-Consolidated + + Pool { + Name = AI-Consolidated + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + Auto Prune = yes # Prune expired volumes + Volume Retention = 360 days # How long should jobs be kept? + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Consolidated-" + Volume Use Duration = 23h + Storage = File2 + Next Pool = AI-Longterm # copy jobs write to this pool + } + + + + +.. code-block:: sh + :caption: bareos-dir pool AI-Longterm + + Pool { + Name = AI-Longterm + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + Auto Prune = yes # Prune expired volumes + Volume Retention = 10 years # How long should jobs be kept? + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Label Format = "AI-Longterm-" + Volume Use Duration = 23h + Storage = File1 + } + +**AI-Longterm**:sup:`Dir`:sub:`pool` is optional and will be explained in :ref:`section-AlwaysIncrementalLongTermStorage`. + +How it works +------------ + +The following configuration extract shows how a client backup is configured for always incremental Backup. The Backup itself is scheduled every night to run as incremental backup, while the consolidation is scheduled to run every day. + + + + +.. code-block:: sh + :caption: bareos-dir job BackupClient1 + + Job { + Name = "BackupClient1" + JobDefs = "DefaultJob" + + # Always incremental settings + AlwaysIncremental = yes + AlwaysIncrementalJobRetention = 7 days + + Accurate = yes + + Pool = AI-Incremental + Full Backup Pool = AI-Consolidated + } + + + + +.. code-block:: sh + :caption: bareos-dir job Consolidate + + Job { + Name = "Consolidate" + Type = "Consolidate" + Accurate = "yes" + JobDefs = "DefaultJob" + } + +The following image shows the available backups for each day: + +|image| + +- The backup cycle starts with a full backup of the client. + +- Every day a incremental backup is done and is additionally available. + +- When the age of the oldest incremental reaches **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` , the consolidation job consolidates the oldest incremental with the full backup before to a new full backup. + +This can go on more or less forever and there will be always an incremental history of **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` . + +The following plot shows what happens if a job is not run for a certain amount of time. + +|image| + +As can be seen, the nightly consolidation jobs still go on consolidating until the last incremental is too old and then only one full backup is left. This is usually not what is intended. + +For this reason, the directive **Always Incremental Keep Number**:sup:`Dir`:sub:`Job` is available which sets the minimum number of incrementals that should be kept even if they are older than **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` . + +Setting **Always Incremental Keep Number**:sup:`Dir`:sub:`Job` to 7 in our case leads to the following result: + +|image| + +**Always Incremental Keep Number**:sup:`Dir`:sub:`Job` incrementals are always kept, and when the backup starts again the consolidation of old incrementals starts again. + +Enhancements for the Always Incremental Backup Scheme +----------------------------------------------------- + +Besides the available backups at each point in time which we have considered until now, the amount of data being moved during the backups is another very important aspect. + +We will have a look at this aspect in the following pictures: + +The basic always incremental scheme +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The basic always incremental scheme does an incremental backup from the client daily which is relatively small and as such is very good. + +During the consolidation, each day the full backup is consolidated with the oldest incremental backup, which means that more or less the full amount of data being stored on the client is moved. Although this consolidation only is performed locally on the storage daemon without client interaction, it is still an enormous amount of data being touched and can take an considerable amount of time. + +If all clients use the "always incremental" backup scheme, this means that the complete data being stored in the backup system needs to be moved every day! + +This is usually only feasible in relatively small environments. + +The following figure shows the Data Volume being moved during the normal always incremental scheme. + +- The red bar shows the amount of the first full backup being copied from the client. + +- The blue bars show the amount of the daily incremental backups. They are so little that the can be barely seen. + +- The green bars show the amount of data being moved every day during the consolidation jobs. + +|image| + +.. _section-AlwaysIncrementalMaxFullAge: + +Always Incremental Max Full Age +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To be able to cope with this problem, the directive **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` was added. When **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` is configured, in daily operation the Full Backup is left untouched while the incrementals are consolidated as usual. Only if the Full Backup is older than **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` , the full backup will also be part of +the consolidation. + +Depending on the setting of the **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` , the amount of daily data being moved can be reduced without losing the advantages of the always incremental Backup Scheme. + +**Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` must be larger than **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` . + +The resulting interval between full consolidations when running daily backups and daily consolidations is **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` - **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` . + +.. raw:: latex + + \centering + +.. figure:: \idir always-incremental-jobdata-AlwaysIncrementalMaxFullAge_21_days + :alt: Data Volume being moved with "Always Incremental Max Full Age" + + Data Volume being moved with "Always Incremental Max Full Age" + +.. raw:: latex + + \centering + +.. figure:: \idir always-incremental-jobs_available-AlwaysIncrementalMaxFullAge_21_days + :alt: Jobs Available with "Always Incremental Max Full Age" + + Jobs Available with "Always Incremental Max Full Age" + +.. _section-MaxFullConsolidations: + +Max Full Consolidations +~~~~~~~~~~~~~~~~~~~~~~~ + +When the **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` of many clients is set to the same value, it is probable that all full backups will reach the **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` at once and so consolidation jobs including the full backup will be started for all clients at once. This would again mean that the whole data being stored from all clients will be moved in one day. + +The following figure shows the amount of data being copied by the virtual jobs that do the consolidation when having 3 identically configured backup jobs: + +|image| + +As can be seen, virtual jobs including the full are triggered for all three clients at the same time. + +This is of course not desirable so the directive **Max Full Consolidations**:sup:`Dir`:sub:`Job` was introduced. + +**Max Full Consolidations**:sup:`Dir`:sub:`Job` needs to be configured in the **Type**:sup:`Dir`:sub:`Job` = **Consolidate** job: + + + + +.. code-block:: sh + :caption: bareos-dir job Consolidate + + Job { + Name = "Consolidate" + Type = "Consolidate" + Accurate = "yes" + JobDefs = "DefaultJob" + + Max Full Consolidations = 1 + } + +If **Max Full Consolidations**:sup:`Dir`:sub:`Job` is configured, the consolidation job will not start more than the specified Consolidations that include the Full Backup. + +This leads to a better load balancing of full backup consolidations over different days. The value should configured so that the consolidation jobs are completed before the next normal backup run starts. + +The number of always incremental jobs, the interval that the jobs are triggered and the setting of **Always Incremental Max Full Age**:sup:`Dir`:sub:`Job` influence the value that makes sense for **Max Full Consolidations**:sup:`Dir`:sub:`Job` . + +.. raw:: latex + + \centering + +.. figure:: \idir jobdata_multiple_clients_maxfullconsilidate + :alt: Data Volume being moved with Max Full Consolidations = 1 + + Data Volume being moved with Max Full Consolidations = 1 + +.. raw:: latex + + \centering + +.. figure:: \idir jobs_available_multiple_clients_maxfullconsolidate + :alt: Jobs Available with Max Full Consolidations = 1 + + Jobs Available with Max Full Consolidations = 1 + +.. _section-AlwaysIncrementalLongTermStorage: + +Long Term Storage of Always Incremental Jobs +-------------------------------------------- + +What is missing in the always incremental backup scheme in comparison to the traditional "Incremental Differential Full" scheme is the option to store a certain job for a longer time. + +When using always incremental, the usual maximum age of data stored during the backup cycle is **Always Incremental Job Retention**:sup:`Dir`:sub:`Job` . + +Usually, it is desired to be able to store a certain backup for a longer time, e.g. monthly a backup should be kept for half a year. + +There are two options to achieve this goal. + +Copy Jobs +~~~~~~~~~ + +The configuration of archiving via copy job is simple, just configure a copy job that copies over the latest full backup at that point in time. + +As all full backups go into the **AI-Consolidated**:sup:`Dir`:sub:`pool` , we just copy all uncopied backups in the **AI-Consolidated**:sup:`Dir`:sub:`pool` to a longterm pool: + + + + +.. code-block:: sh + :caption: bareos-dir job CopyLongtermFull + + Job { + Name = "CopyLongtermFull" + Schedule = LongtermFull + Type = Copy + Level = Full + Pool = AI-Consolidated + Selection Type = PoolUncopiedJobs + Messages = Standard + } + +As can be seen in the plot, the copy job creates a copy of the current full backup that is available and is already 7 days old. + +|image| + +The other disadvantage is, that it copies all jobs, not only the virtual full jobs. It also includes the virtual incremental jobs from this pool. + +Virtual Full Jobs +~~~~~~~~~~~~~~~~~ + +The alternative to Copy Jobs is creating a virtual Full Backup Job when the data should be stored in a long-term pool. + + + + +.. code-block:: sh + :caption: bareos-dir job VirtualLongtermFull + + Job { + Name = "VirtualLongtermFull" + Client = bareos-fd + FileSet = SelfTest + Schedule = LongtermFull + Type = Backup + Level = VirtualFull + Pool = AI-Consolidated + Messages = Standard + + Priority = 13 # run after Consolidate + Run Script { + console = "update jobid=%i jobtype=A" + Runs When = After + Runs On Client = No + Runs On Failure = No + } + } + +To make sure the longterm **Level**:sup:`Dir`:sub:`Job` = **VirtualFull** is not taken as base for the next incrementals, the job type of the copied job is set to **Type**:sup:`Dir`:sub:`Job` = **Archive** with the **Run Script**:sup:`Dir`:sub:`Job` . + +As can be seen on the plot, the **Level**:sup:`Dir`:sub:`Job` = **VirtualFull** archives the current data, i.e. it consolidates the full and all incrementals that are currently available. + +|image| + +How to manually transfer data/volumes +===================================== + +The always incremental backup scheme minimizes the amount of data that needs to be transferred over the wire. + +This makes it possible to backup big filesystems over small bandwidths. + +The only challenge is to do the first full backup. + +The easiest way to transfer the data is to copy it to a portable data medium (or even directly store it on there) and import the data into the local bareos catalog as if it was backed up from the original client. + +This can be done in two ways + +#. Install a storage daemon in the remote location that needs to be backed up and connect it to the main director. This makes it easy to make a local backup in the remote location and then transfer the volumes to the local storage. For this option the communication between the local director and the remote storage daemon needs to be possible. + + |image| + +#. Install a director and a storage daemon in the remote location. This option means that the backup is done completely independent from the local director and only the volume is then transferred and needs to be imported afterwards. + + |image| + +Import Data from a Remote Storage Daemon +---------------------------------------- + +First setup client, fileset, job and schedule as needed for a always incremental backup of the remote client. + +Run the first backup but make sure that you choose the remote storage to be used. + + + + +.. code-block:: sh + :caption: run + + *run job=BackupClient-remote level=Full storage=File-remote + +Transport the volumes that were used for that backup over to the local storage daemon and make them available to the local storage daemon. This can be either by putting the tapes into the local changer or by storing the file volumes into the local file volume directory. + +If copying a volume to the local storage directory make sure that the file rights are correct. + +Now tell the director that the volume now belongs to the local storage daemon. + +List volumes shows that the volumes used still belong to the remote storage: + + + + +.. code-block:: sh + :caption: list volumes + + *list volumes + ..... + Pool: Full + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+-------------+ + | MediaId | VolumeName | VolStatus | Enabled | VolBytes | VolFiles | VolRetention | Recycle | Slot | InChanger | MediaType | LastWritten | Storage | + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+-------------+ + | 1 | Full-0001 | Append | 1 | 38600329 | 0 | 31536000 | 1 | 0 | 0 | File | 2016-07-28 14:00:47 | File-remote | + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+-------------+ + +Use :strong:`update volume` to set the right storage and check with list volumes that it worked: + + + + +.. code-block:: sh + :caption: update volume + + *update volume=Full-0001 storage=File + *list volumes + ... + Pool: Full + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+---------+ + | MediaId | VolumeName | VolStatus | Enabled | VolBytes | VolFiles | VolRetention | Recycle | Slot | InChanger | MediaType | LastWritten | Storage | + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+---------+ + | 1 | Full-0001 | Append | 1 | 38600329 | 0 | 31536000 | 1 | 0 | 0 | File | 2016-07-28 14:00:47 | File | + +---------+------------+-----------+---------+----------+----------+--------------+---------+------+-----------+-----------+---------------------+---------+ + +Now the remote storage daemon can be disabled as it is not needed anymore. + +The next incremental run will take the previously taken full backup as reference. + +Import Data from a Independent Remote Full Bareos Installation +-------------------------------------------------------------- + +If a network connection between the local director and the remote storage daemon is not possible, it is also an option to setup a fully functional Bareos installation remotely and then to import the created volumes. Of course the network connection between the |bareosDir| and the |bareosFd| is needed in any case to make the incremental backups possible. + +- Configure the connection from local |bareosDir| to remote |bareosFd|, give the remote client the same name as it was when the data was backed up. + +- Add the Fileset created on remote machine to local machine. + +- Configure the Job that should backup the remote client with the fileset. + +- Run :strong:`estimate listing` on the remote backup job. + +- Run :strong:`list filesets` to make sure the fileset was added to the catalog. + +Then we need to create a backup on the remote machine onto a portable disk which we can then import into our local installation. + +On remote machine: + +- Install full Bareos server on remote server (sd, fd, dir). Using the Sqlite backend is sufficient. + +- Add the client to the remote backup server. + +- Add fileset which the client will be backed up. + +- Add Pool with name **transfer**:sup:`Dir`:sub:`pool` where the data will be written to. + +- create job that will backup the remote client with the remote fileset into the new pool + +- Do the local backup using the just created Pool and Filesets. + +Transport the newly created volume over to the director machine (e.g. via external harddrive) and store the file where the device stores its files (e.g. /var/lib/bareos/storage) + +Shutdown Director on local director machine. + +Import data form volume via :program:`bscan`, you need to set which database backend is used: :program:`bscan -B sqlite3 FileStorage -V Transfer-0001 -s -S` + +If the import was successfully completed, test if an incremental job really only backs up the minimum amount of data. + +.. |image| image:: \idir inc-diff-full-jobs_available +.. |image| image:: \idir inc-diff-full-jobdata +.. |image| image:: \idir inc-diff-full-jobs_available-zoom +.. |image| image:: \idir always-incremental +.. |image| image:: \idir always-incremental-with-pause-7days-retention-no-keep +.. |image| image:: \idir always-incremental-with-pause-7days-retention-7days-keep +.. |image| image:: \idir always-incremental-jobdata +.. |image| image:: \idir jobdata_multiple_clients +.. |image| image:: \idir always-incremental-copy-job-archiving +.. |image| image:: \idir always-incremental-virtualfull-job-archiving +.. |image| image:: \idir ai-transfer-first-backup2 +.. |image| image:: \idir ai-transfer-first-backup3 + diff --git a/docs/manuals/en/new_main_reference/source/chapter24/basejob.rst b/docs/manuals/en/new_main_reference/source/chapter24/basejob.rst new file mode 100644 index 00000000000..b5fdea0792c --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter24/basejob.rst @@ -0,0 +1,56 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +File Deduplication using Base Jobs +================================== + +.. index:: + single: Base Jobs}` :raw-latex:`\index[general]{File Deduplication + + +.. _`basejobs`: basejobs A base job is sort of like a Full save except that you will want the FileSet to contain only files that are unlikely to change in the future (i.e. a snapshot of most of your system after installing it). After the base job has been run, when you are doing a Full save, you specify one or more Base jobs to be used. All files that have been backed up in the Base job/jobs but not +modified will then be excluded from the backup. During a restore, the Base jobs will be automatically pulled in where necessary. + +Imagine having 100 nearly identical Windows or Linux machine containing the OS and user files. Now for the OS part, a Base job will be backed up once, and rather than making 100 copies of the OS, there will be only one. If one or more of the systems have some files updated, no problem, they will be automatically backuped. + +A new Job directive :strong:`Base=JobX,JobY,...` permits to specify the list of files that will be used during Full backup as base. + + + + Job { + Name = BackupLinux + Level= Base + ... + } + + Job { + Name = BackupZog4 + Base = BackupZog4, BackupLinux + Accurate = yes + ... + } + +In this example, the job ``BackupZog4`` will use the most recent version of all files contained in ``BackupZog4`` and ``BackupLinux`` jobs. Base jobs should have run with :strong:`Level=Base` to be used. + +By default, Bareos will compare permissions bits, user and group fields, modification time, size and the checksum of the file to choose between the current backup and the BaseJob file list. You can change this behavior with the ``BaseJob`` FileSet option. This option works like the :strong:`Verify`, that is described in the :ref:`FileSet ` chapter. + + + + FileSet { + Name = Full + Include = { + Options { + BaseJob = pmugcs5 + Accurate = mcs + Verify = pin5 + } + File = / + } + } + +.. raw:: latex + + +.. warning:: + The current implementation doesn't permit to scan + volume with \command{bscan. The result wouldn't permit to restore files easily.} diff --git a/docs/manuals/en/new_main_reference/source/chapter26/plugins.rst b/docs/manuals/en/new_main_reference/source/chapter26/plugins.rst new file mode 100644 index 00000000000..dc7e4cff700 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter26/plugins.rst @@ -0,0 +1,624 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +Plugins +======= + +.. index:: + single: Plugin + + +.. _`section-plugins`: section-plugins + +The functionality of Bareos can be extended by plugins. They do exists plugins for the different daemons (Director, Storage- and File-Daemon). + +To use plugins, they must be enabled in the configuration (:strong:`Plugin Directory` and optionally :strong:`Plugin Names`). + +If a :strong:`Plugin Directory` is specified :strong:`Plugin Names` defines, which plugins get loaded. + +If :strong:`Plugin Names` is not defined. + +.. _fdPlugins: + +File Daemon Plugins +------------------- + +File Daemon plugins are configured by the :strong:`Plugin` directive of a :ref:`File Set `. + +.. raw:: latex + + +.. warning:: + Currently the plugin command is being stored as part of the backup. The restore command in your directive should be flexible enough if things might change in future, otherwise you could run into trouble. + +.. _bpipe: + +bpipe Plugin +~~~~~~~~~~~~ + +.. index:: + pair: Plugin; bpipe + + +The bpipe plugin is a generic pipe program, that simply transmits the data from a specified program to Bareos for backup, and from Bareos to a specified program for restore. The purpose of the plugin is to provide an interface to any system program for backup and restore. That allows you, for example, to do database backups without a local dump. By using different command lines to bpipe, you can backup any kind of data (ASCII or binary) depending on the program called. + +On Linux, the Bareos bpipe plugin is part of the **bareos-filedaemon** package and is therefore installed on any system running the filedaemon. + +The bpipe plugin is so simple and flexible, you may call it the "Swiss Army Knife" of the current existing plugins for Bareos. + +The bpipe plugin is specified in the Include section of your Job’s FileSet resource in your :file:`bareos-dir.conf`. + + + + +.. code-block:: sh + :caption: bpipe fileset + + FileSet { + Name = "MyFileSet" + Include { + Options { + signature = MD5 + compression = gzip + } + Plugin = "bpipe:file=:reader=:writer= + } + } + +The syntax and semantics of the Plugin directive require the first part of the string up to the colon to be the name of the plugin. Everything after the first colon is ignored by the File daemon but is passed to the plugin. Thus the plugin writer may define the meaning of the rest of the string as he wishes. The full syntax of the plugin directive as interpreted by the bpipe plugin is: + + + + +.. code-block:: sh + :caption: bpipe directive + + Plugin = ":file=:reader=:writer=" + +plugin + is the name of the plugin with the trailing -fd.so stripped off, so in this case, we would put bpipe in the field. + +filepath + specifies the namespace, which for bpipe is the pseudo path and filename under which the backup will be saved. This pseudo path and filename will be seen by the user in the restore file tree. For example, if the value is :strong:`/MySQL/mydump.sql`, the data backed up by the plugin will be put under that :emphasis:`pseudo` path and filename. You must be careful to choose a naming convention that is unique to avoid a conflict with a path and filename that actually + exists on your system. + +readprogram + for the bpipe plugin specifies the "reader" program that is called by the plugin during backup to read the data. bpipe will call this program by doing a popen on it. + +writeprogram + for the bpipe plugin specifies the "writer" program that is called by the plugin during restore to write the data back to the filesystem. + +Please note that the two items above describing the "reader" and "writer", these programs are "executed" by Bareos, which means there is no shell interpretation of any command line arguments you might use. If you want to use shell characters (redirection of input or output, ...), then we recommend that you put your command or commands in a shell script and execute the script. In addition if you backup a file with reader program, when running the writer program during the restore, Bareos will not +automatically create the path to the file. Either the path must exist, or you must explicitly do so with your command or in a shell script. + +See the examples about :ref:`backup-postgresql` and :ref:`backup-mysql`. + +PGSQL Plugin +~~~~~~~~~~~~ + +See chapter :ref:`backup-postgresql-plugin`. + +MySQL Plugin +~~~~~~~~~~~~ + +See the chapters :ref:`backup-mysql-xtrabackup` and :ref:`backup-mysql-python`. + +MSSQL Plugin +~~~~~~~~~~~~ + +See chapter :ref:`MSSQL`. + +LDAP Plugin +~~~~~~~~~~~ + +.. index:: + pair: Plugin; ldap + + +This plugin is intended to backup (and restore) the contents of a LDAP server. It uses normal LDAP operation for this. The package **bareos-filedaemon-ldap-python-plugin** (15.2.0) contains an example configuration file, that must be adapted to your envirnoment. + +Cephfs Plugin +~~~~~~~~~~~~~ + +.. index:: + pair: Ceph; Cephfs Plugin + + +Opposite to the :ref:`Rados Backend ` that is used to store data on a CEPH Object Store, this plugin is intended to backup a CEPH Object Store via the Cephfs interface to other media. The package **bareos-filedaemon-ceph-plugin** (15.2.0) contains an example configuration file, that must be adapted to your envirnoment. + +Rados Plugin +~~~~~~~~~~~~ + +.. index:: + pair: Ceph; Rados Plugin + + +Opposite to the :ref:`Rados Backend ` that is used to store data on a CEPH Object Store, this plugin is intended to backup a CEPH Object Store via the Rados interface to other media. The package **bareos-filedaemon-ceph-plugin** (15.2.0) contains an example configuration file, that must be adapted to your envirnoment. + +GlusterFS Plugin +~~~~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; glusterfs +.. index:: + pair: GlusterFS; Plugin + + +Opposite to the :ref:`GFAPI Backend ` that is used to store data on a Gluster system, this plugin is intended to backup data from a Gluster system to other media. The package **bareos-filedaemon-glusterfs-plugin** (15.2.0) contains an example configuration file, that must be adapted to your envirnoment. + +python-fd Plugin +~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Plugin; Python; File Daemon; + + +The **python-fd** plugin behaves similar to the :ref:`director-python-plugin`. Base plugins and an example get installed via the package bareos-filedaemon-python-plugin. Configuration is done in the :ref:`DirectorResourceFileSet` on the director. + +We basically distinguish between command-plugin and option-plugins. + +Command Plugins +^^^^^^^^^^^^^^^ + +Command plugins are used to replace or extend the FileSet definition in the File Section. If you have a command-plugin, you can use it like in this example: + + + + +.. code-block:: sh + :caption: bareos-dir.conf: Python FD command plugins + + FileSet { + Name = "mysql" + Include { + Options { + Signature = MD5 # calculate md5 checksum per file + } + File = "/etc" + Plugin = "python:module_path=/usr/lib/bareos/plugins:module_name=bareos-fd-mysql" + } + } + +.. index:: + pair: MySQL; Backup + This example uses the :ref:`MySQL plugin ` to backup MySQL dumps in addition to :file:`/etc`. + +Option Plugins +^^^^^^^^^^^^^^ + +Option plugins are activated in the Options resource of a FileSet definition. + +Example: + + + + +.. code-block:: sh + :caption: bareos-dir.conf: Python FD option plugins + + FileSet { + Name = "option" + Include { + Options { + Signature = MD5 # calculate md5 checksum per file + Plugin = "python:module_path=/usr/lib/bareos/plugins:module_name=bareos-fd-file-interact" + } + File = "/etc" + File = "/usr/lib/bareos/plugins" + } + } + +This plugin bareos-fd-file-interact from `https://github.com/bareos/bareos-contrib/tree/master/fd-plugins/options-plugin-sample `_ has a method that is called before and after each file that goes into the backup, it can be used as a template for whatever plugin wants to interact with files before or after backup. + +.. _sdPlugins: + +Storage Daemon Plugins +---------------------- + +.. _plugin-autoxflate-sd: + +autoxflate-sd +~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; autoxflate-sd + + +This plugin is part of the **bareos-storage** package. + +The autoxflate-sd plugin can inflate (decompress) and deflate (compress) the data being written to or read from a device. It can also do both. + +|image| + +Therefore the autoxflate plugin inserts a inflate and a deflate function block into the stream going to the device (called OUT) and coming from the device (called IN). + +Each stream passes first the inflate function block, then the deflate function block. + +The inflate blocks are controlled by the setting of the **Auto Inflate**:sup:`Sd`:sub:`Device` directive. + +The deflate blocks are controlled by the setting of the **Auto Deflate**:sup:`Sd`:sub:`Device` , **Auto Deflate Algorithm**:sup:`Sd`:sub:`Device` and **Auto Deflate Level**:sup:`Sd`:sub:`Device` directives. + +The inflate blocks, if enabled, will uncompress data if it is compressed using the algorithm that was used during compression. + +The deflate blocks, if enabled, will compress uncompressed data with the algorithm and level configured in the according directives. + +The series connection of the inflate and deflate function blocks makes the plugin very flexible. + +Szenarios where this plugin can be used are for example: + +- client computers with weak cpus can do backups without compression and let the sd do the compression when writing to disk + +- compressed backups can be recompressed to a different compression format (e.g. gzip -> lzo) using migration jobs + +- client backups can be compressed with compression algorithms that the client itself does not support + +Multi-core cpus will be utilized when using parallel jobs as the compression is done in each jobs’ thread. + +When the autoxflate plugin is configured, it will write some status information into the joblog. + + + + {used compression algorithm} + autodeflation: compressor on device FileStorage is FZ4H + + + + {configured inflation and deflation blocks} + autoxflate-sd.c: FileStorage OUT:[SD->inflate=yes->deflate=yes->DEV] IN:[DEV->inflate=yes->deflate=yes->SD] + + + + {overall deflation/inflation ratio} + autoxflate-sd.c: deflate ratio: 50.59% + +Additional **Auto XFlate On Replication**:sup:`Sd`:sub:`Storage` can be configured at the Storage resource. + +scsicrypto-sd +~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; scsicrypto-sd + + +This plugin is part of the **bareos-storage-tape** package. + +General +^^^^^^^ + +.. _LTOHardwareEncryptionGeneral: + +LTO Hardware Encryption +''''''''''''''''''''''' + +Modern tape-drives, for example LTO (from LTO4 onwards) support hardware encryption. There are several ways of using encryption with these drives. The following three types of key management are available for encrypting drives. The transmission of the keys to the volumes is accomplished by either of the three: + +- A backup application that supports Application Managed Encryption (AME) + +- A tape library that supports Library Managed Encryption (LME) + +- A Key Management Appliance (KMA) + +We added support for Application Managed Encryption (AME) scheme, where on labeling a crypto key is generated for a volume and when the volume is mounted, the crypto key is loaded. When finally the volume is unmounted, the key is cleared from the memory of the Tape Drive using the SCSI SPOUT command set. + +If you have implemented Library Managed Encryption (LME) or a Key Management Appliance (KMA), there is no need to have support from Bareos on loading and clearing the encryption keys, as either the Library knows the per volume encryption keys itself, or it will ask the KMA for the encryption key when it needs it. For big installations you might consider using a KMA, but the Application Managed Encryption implemented in Bareos should also scale rather well and have a low overhead as the keys are +only loaded and cleared when needed. + +The scsicrypto-sd plugin +'''''''''''''''''''''''' + +The :program:`scsicrypto-sd` hooks into the :strong:`unload`, :strong:`label read`, :strong:`label write` and :strong:`label verified` events for loading and clearing the key. It checks whether it it needs to clear the drive by either using an internal state (if it loaded a key before) or by checking the state of a special option that first issues an encrytion status query. If there is a connection to the director +and the volume information is not available, it will ask the director for the data on the currently loaded volume. If no connection is available, a cache will be used which should contain the most recently mounted volumes. If an encryption key is available, it will be loaded into the drive’s memory. + +Changes in the director +''''''''''''''''''''''' + +The director has been extended with additional code for handling hardware data encryption. The extra keyword **encrypt** on the label of a volume will force the director to generate a new semi-random passphrase for the volume, which will be stored in the database as part of the media information. + +A passphrase is always stored in the database base64-encoded. When a so called **Key Encryption Key** is set in the config of the director, the passphrase is first wrapped using RFC3394 key wrapping and then base64-encoded. By using key wrapping, the keys in the database are safe against people sniffing the info, as the data is still encrypted using the Key Encryption Key (which in essence is just an extra passphrase of the same length as the volume passphrases used). + +When the storage daemon needs to mount the volume, it will ask the director for the volume information and that protocol is extended with the exchange of the base64-wrapped encryption key (passphrase). The storage daemon provides an extra config option in which it records the Key Encryption Key of the particular director, and as such can unwrap the key sent into the original passphrase. + +As can be seen from the above info we don’t allow the user to enter a passphrase, but generate a semi-random passphrase using the openssl random functions (if available) and convert that into a readable ASCII stream of letters, numbers and most other characters, apart from the quotes and space etc. This will produce much stronger passphrases than when requesting the info from a user. As we store this information in the database, the user never has to enter these passphrases. + +The volume label is written in unencrypted form to the volume, so we can always recognize a Bareos volume. When the key is loaded onto the drive, we set the decryption mode to mixed, so we can read both unencrypted and encrypted data from the volume. When no key or the wrong key has been loaded, the drive will give an IO error when trying to read the volume. For disaster recovery you can store the Key Encryption Key and the content of the wrapped encryption keys somewhere safe and the +:ref:`bscrypto ` tool together with the scsicrypto-sd plugin can be used to get access to your volumes, in case you ever lose your complete environment. + +If you don’t want to use the scsicrypto-sd plugin when doing DR and you are only reading one volume, you can also set the crypto key using the bscrypto tool. Because we use the mixed decryption mode, in which you can read both encrypted and unencrypted data from a volume, you can set the right encryption key before reading the volume label. + +If you need to read more than one volume, you better use the scsicrypto-sd plugin with tools like bscan/bextract, as the plugin will then auto-load the correct encryption key when it loads the volume, similiarly to what the storage daemon does when performing backups and restores. + +The volume label is unencrypted, so a volume can also be recognized by a non-encrypted installation, but it won’t be able to read the actual data from it. Using an encrypted volume label doesn’t add much security (there is no security-related info in the volume label anyhow) and it makes it harder to recognize either a labeled volume with encrypted data or an unlabeled new volume (both would return an IO-error on read of the label.) + +Configuration +^^^^^^^^^^^^^ + +SCSI crypto setup +''''''''''''''''' + +The initial setup of SCSI crypto looks something like this: + +- Generate a Key Encryption Key e.g. + + + + +.. code-block:: sh + :caption: + + bscrypto -g - + +For details see :ref:`bscrypto `. + +Security Setup +'''''''''''''' + +Some security levels need to be increased for the storage daemon to be able to use the low level SCSI interface for setting and getting the encryption status on a tape device. + +The following additional security is needed for the following operating systems: + +Linux (SG_IO ioctl interface): + + +The user running the storage daemon needs the following additional capabilities: :raw-latex:`\index[sd]{Platform!Linux!Privileges}` + +- :option:`CAP_SYS_RAWIO` (see capabilities(7)) + + - On older kernels you might need :option:`CAP_SYS_ADMIN`. Try :option:`CAP_SYS_RAWIO` first and if that doesn’t work try :option:`CAP_SYS_ADMIN` + +- If you are running the storage daemon as another user than root (which has the :option:`CAP_SYS_RAWIO` capability), you need to add it to the current set of capabilities. + +- If you are using systemd, you could add this additional capability to the CapabilityBoundingSet parameter. + + - For systemd add the following to the bareos-sd.service: :option:`Capabilities=cap_sys_rawio+ep` + +You can also set up the extra capability on :program:`bscrypto` and :program:`bareos-sd` by running the following commands: + + + + +.. code-block:: sh + :caption: + + setcap cap_sys_rawio=ep bscrypto + setcap cap_sys_rawio=ep bareos-sd + +Check the setting with + + + + +.. code-block:: sh + :caption: + + getcap -v bscrypto + getcap -v bareos-sd + +:program:`getcap` and :program:`setcap` are part of libcap-progs. + +If :program:`bareos-sd` does not have the appropriate capabilities, all other tape operations may still work correctly, but you will get :emphasis:`Unable to perform SG\_IO ioctl` errors. + +Solaris (USCSI ioctl interface): + + +The user running the storage daemon needs the following additional privileges: :raw-latex:`\index[sd]{Platform!Solaris!Privileges}` + +- :option:`PRIV_SYS_DEVICES` (see privileges(5)) + +If you are running the storage daemon as another user than root (which has the :option:`PRIV_SYS_DEVICES` privilege), you need to add it to the current set of privileges. This can be set up by setting this either as a project for the user, or as a set of extra privileges in the SMF definition starting the storage daemon. The SMF setup is the cleanest one. + +For SMF make sure you have something like this in the instance block: + + + + +.. code-block:: sh + :caption: + + + +Changes in bareos-sd.conf +''''''''''''''''''''''''' + +- Set the Key Encryption Key + + - **Key Encryption Key**:sup:`Sd`:sub:`Director` = :strong:`passphrase` + +- Enable the loading of storage daemon plugins + + - **Plugin Directory**:sup:`Sd`:sub:`Storage` = :file:`path_to_sd_plugins` + +- Enable the SCSI encryption option + + - **Drive Crypto Enabled**:sup:`Sd`:sub:`Device` = yes + +- Enable this, if you want the plugin to probe the encryption status of the drive when it needs to clear a pending key + + - **Query Crypto Status**:sup:`Sd`:sub:`Device` = yes + +Changes in bareos-dir.conf +'''''''''''''''''''''''''' + +- Set the Key Encryption Key + + - **Key Encryption Key**:sup:`Dir`:sub:`Director` = :strong:`passphrase` + +Testing +^^^^^^^ + +Restart the Storage Daemon and the Director. After this you can label new volumes with the encrypt option, e.g. + + + + +.. code-block:: sh + :caption: + + label slots=1-5 barcodes encrypt + +Disaster Recovery +^^^^^^^^^^^^^^^^^ + +For Disaster Recovery (DR) you need the following information: + +- Actual bareos-sd.conf with config options enabled as described above, including, among others, a definition of a director with the Key Encryption Key used for creating the encryption keys of the volumes. + +- The actual keys used for the encryption of the volumes. + +This data needs to be availabe as a so called crypto cache file which is used by the plugin when no connection to the director can be made to do a lookup (most likely on DR). + +Most of the times the needed information, e.g. the bootstrap info, is available on recently written volumes and most of the time the encryption cache will contain the most recent data, so a recent copy of the :file:`bareos-sd..cryptoc` file in the working directory is enough most of the time. You can also save the info from database in a safe place and use bscrypto to populate this info (VolumeName ->EncryptKey) into the crypto cache file used by +:program:`bextract` and :program:`bscan`. You can use :program:`bscrypto` with the following flags to create a new or update an existing crypto cache file e.g.: + + + + +.. code-block:: sh + :caption: + + bscrypto -p /var/lib/bareos/bareos-sd..cryptoc + +- A valid BSR file containing the location of the last safe of the database makes recovery much easier. Adding a post script to the database save job could collect the needed info and make sure its stored somewhere safe. + +- Recover the database in the normal way e.g. for postgresql: + + + + +.. code-block:: sh + :caption: + + bextract -D -c bareos-sd.conf -V \ /dev/nst0 /tmp -b bootstrap.bsr + /usr/lib64/bareos/create_bareos_database + /usr/lib64/bareos/grant_bareos_privileges + psql bareos < /tmp/var/lib/bareos/bareos.sql + +Or something similar (change paths to follow where you installed the software or where the package put it). + +**Note:** As described at the beginning of this chapter, there are different types of key management, AME, LME and KMA. If the Library is set up for LME or KMA, it probably won’t allow our AME setup and the scsi-crypto plugin will fail to set/clear the encryption key. To be able to use AME you need to :emphasis:`Modify Encryption Method` and set it to something like :emphasis:`Application Managed`. If you decide to use LME or KMA you don’t have to bother with the whole setup +of AME which may for big libraries be easier, although the overhead of using AME even for very big libraries should be minimal. + +scsitapealert-sd +~~~~~~~~~~~~~~~~ + +.. index:: + pair: Plugin; scsitapealert-sd + + +This plugin is part of the **bareos-storage-tape** package. + +python-sd Plugin +~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Plugin; Python; Storage Daemon; + + +The **python-sd** plugin behaves similar to the :ref:`director-python-plugin`. + +.. _dirPlugins: + +Director Plugins +---------------- + +.. _director-python-plugin: + +python-dir Plugin +~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Plugin; Python; Director; + + +The **python-dir** plugin is intended to extend the functionality of the Bareos Director by Python code. A working example is included. + +- install the **bareos-director-python-plugin** package + +- change to the Bareos plugin directory (:file:`/usr/lib/bareos/plugins/` or :file:`/usr/lib64/bareos/plugins/`) + +- copy :file:`bareos-dir.py.template` to :file:`bareos-dir.py` + +- activate the plugin in the Bareos Director configuration + +- restart the Bareos Director + +- change :file:`bareos-dir.py` as required + +- restart the Bareos Director + +Loading plugins +^^^^^^^^^^^^^^^ + +Since 14.4.0}` multiple Python plugins can be loaded and plugin names can be arbitrary. Before this, the Python plugin always loads the file :raw-latex:`\file{bareos-dir.py. + +The director plugins are configured in the Job-Resource (or JobDefs resource). To load a Python plugin you need + +- pointing to your plugin directory (needs to be enabled in the Director resource, too + +- Your plugin (without the suffix .py) + +- default is ’0’, you can leave this, as long as you only have 1 Director Python plugin. If you have more than 1, start with instance=0 and increment the instance for each plugin. + +- You can add plugin specific option key-value pairs, each pair separated by ’:’ key=value + +Single Python Plugin Loading Example: + + + + +.. code-block:: sh + :caption: bareos-dir.conf: Single Python Plugin Loading Example + + Director { + # ... + # Plugin directory + Plugin Directory = /usr/lib64/bareos/plugins + # Load the python plugin + Plugin Names = "python" + } + + JobDefs { + Name = "DefaultJob" + Type = Backup + # ... + # Load the class based plugin with testoption=testparam + Dir Plugin Options = "python:instance=0:module_path=/usr/lib64/bareos/plugins:module_name=bareos-dir-class-plugins:testoption=testparam + # ... + } + +Multiple Python Plugin Loading Example: + + + + +.. code-block:: sh + :caption: bareos-dir.conf: Multiple Python Plugin Loading Example + + Director { + # ... + # Plugin directory + Plugin Directory = /usr/lib64/bareos/plugins + # Load the python plugin + Plugin Names = "python" + } + + JobDefs { + Name = "DefaultJob" + Type = Backup + # ... + # Load the class based plugin with testoption=testparam + Dir Plugin Options = "python:instance=0:module_path=/usr/lib64/bareos/plugins:module_name=bareos-dir-class-plugins:testoption=testparam1 + Dir Plugin Options = "python:instance=1:module_path=/usr/lib64/bareos/plugins:module_name=bareos-dir-class-plugins:testoption=testparam2 + # ... + } + +Write your own Python Plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some plugin examples are available on `https://github.com/bareos/bareos-contrib `_. The class-based approach lets you easily reuse stuff already defined in the baseclass BareosDirPluginBaseclass, which ships with the **bareos-director-python-plugin** package. The examples contain the plugin bareos-dir-nsca-sender, that submits the results and performance data of a backup job directly to Icinga:raw-latex:`\index[general]{Icinga}` or +Nagios:raw-latex:`\index[general]{Nagios|see{Icinga}}` using the NSCA protocol. + +.. |image| image:: \idir autoxflate-functionblocks + :width: 80.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter27/win32.rst b/docs/manuals/en/new_main_reference/source/chapter27/win32.rst new file mode 100644 index 00000000000..799e936e453 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter27/win32.rst @@ -0,0 +1,717 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _Win32Chapter: + +The Windows Version of Bareos +============================= + + + +.. _`section-windows}` :raw-latex:`\index[general]{Platform!Windows}` :raw-latex:`\index[general]{Windows`: section-windows}` :raw-latex:`\index[general]{Platform!Windows}` :raw-latex:`\index[general]{Windows + +The Windows version of Bareos is a native Win32 port, but there are very few source code changes to the Unix code, which means that the Windows version is for the most part running code that has long proved stable on Unix systems. + +Chapter :ref:`SupportedOSes` shows, what Windows versions are supported. + +The Bareos component that is most often used in Windows is the File daemon or Client program. As a consequence, when we speak of the Windows version of Bareos below, we are mostly referring to the File daemon (client). + +Once installed Bareos normally runs as a system service. This means that it is immediately started by the operating system when the system is booted, and runs in the background even if there is no user logged into the system. + +.. _Windows:Installation: + +Windows Installation +-------------------- + + + +.. _`Windows:Configuration:Files}` :raw-latex:`\index[general]{Installation!Windows}` :raw-latex:`\index[general]{Windows!File Daemon!Installation`: Windows:Configuration:Files}` :raw-latex:`\index[general]{Installation!Windows}` :raw-latex:`\index[general]{Windows!File Daemon!Installation + +Normally, you will install the Windows version of Bareos from the binaries. The **winbareos** binary packages are provided under `http://download.bareos.org/bareos/release/latest/windows `_. Additionally, there are `OPSI `_ packages available under `http://download.bareos.org/bareos/release/latest/windows/opsi `_. + +This install is standard Windows .exe that runs an install wizard using the NSIS Free Software installer, so if you have already installed Windows software, it should be very familiar to you. Providing you do not already have Bareos installed, the installer installs the binaries and dlls in :file:`C:\Program Files\Bareos` and the configuration files in :file:`C:\ProgramData\Bareos` (for Windows XP and older: +:file:`C:\Documents and Settings\All Users\Application Data\Bareos`). + +In addition, the **Start:raw-latex:`\-{\textgreater}`All Programs:raw-latex:`\-{\textgreater}`Bareos** menu item will be created during the installation, and on that menu, you will find items for editing the configuration files, displaying the document, and starting a user interface. + +During installation you can decide, what Bareos components you want to install. + +Typically, you only want to install the Bareos Client (|bareosFd|) and optionally some interface tools on a Windows system. Normally, we recommend to let the server components run on a Linux or other Unix system. However, it is possible, to run the |bareosDir|, |bareosSd| and |bareosWebui| on a Windows systems. You should be aware about following limitations: + +.. raw:: latex + + \limitation{Windows}{\bareosDir does not support MySQL database backend}{ + When running the \bareosDir on Windows, only PostgreSQL (and SQLite) database backends are supported. + SQLite is best suited for test environments. + } + +.. raw:: latex + + \limitation{Windows}{\bareosSd only support backup to disk, not to tape}{ + } + +.. raw:: latex + + \limitation{Windows}{The default installation of \bareosWebui is only suitable for local access}{% + Normally the \bareosWebui is running on a Apache server on Linux. + While it is possible, to run the \bareosWebui under Apache or another Webserver which supports PHP under Windows, + the configuration shipped the the \package{winbareos} package uses the PHP internal webserver. + This is okay for local access, but not suitable for being accessed via the network. + To guarantee this, it is configured to only listen locally (\verb|URL:http://localhost:9100|). + } + +Graphical Installation +~~~~~~~~~~~~~~~~~~~~~~ + +Here are the important steps. + +- You must be logged in as an Administrator to the local machine to do a correct installation, if not, please do so before continuing. + +- For a standard installation you may only select the "Tray-Monitor" and the "Open Firewall for Client" as additional optional components. + + |image| + +- You need to fill in the name of your bareos director in the client configuration dialogue and the FQDN or ip address of your client. + + |image| + +- Add the client resource to your Bareos Director Configuration and a job resource for the client as it is also described in the default bareos-dir.conf + + |image| + +Command Line (Silent) Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Silent installation is possible since 12.4.4. All inputs that are given during interactive install can now directly be configured on the commandline, so that an automatic silent install is possible. + +Commandline Switches +'''''''''''''''''''' + +/? + shows the list of available parameters. + +/S + sets the installer to silent. The Installation is done without user interaction. This switch is also available for the uninstaller. + +/CLIENTADDRESS + network address of the client + +/CLIENTNAME + sets the name of the client resource + +/CLIENTMONITORPASSWORD + sets the password for monitor access + +/CLIENTPASSWORD + sets the password to access the client + +/DBADMINUSER=user + sets the database admin user, default=postgres. 14.2.1 + +/DBADMINPASSWORD=password + sets the database admin password, default=\ *none*. 14.2.1 + +/DIRECTORADDRESS + sets network address of the director for bconsole or bat access + +/DIRECTORNAME + sets the name of the director to access the client and of the director to accessed by bconsole and bat + +/DIRECTORPASSWORD + set the password to access the director + +/SILENTKEEPCONFIG + keep configuration files on silent uninstall and use exinsting config files during silent install. 12.4.4 + +/INSTALLDIRECTOR + install the Bareos Director (and bconsole). 14.2.1 + +/INSTALLSTORAGE + install the Bareos Storage Daemon. 14.2.1 + +/WRITELOGS + makes also non-debug installer write a log file. 14.2.1 + +/D=:file:`C:\specify\installation\directory` + (Important: It has to be the last option!) + +By setting the Installation Parameters via commandline and using the silent installer, you can install the bareos client without having to do any configuration after the installation e.g. as follows: + +.. raw:: latex + + + + + + c:\winbareos.exe /S /CLIENTNAME=hostname-fd /CLIENTPASSWORD="verysecretpassword" /DIRECTORNAME=bareos-dir + +.. raw:: latex + + + +DBADMINUSER and DBADMINPASSWORD are used to create the bareos databases. If login is not possible silent installer will abort + +Dealing with Windows Problems +----------------------------- + +.. index:: + pair: Problem; Windows +.. index:: + pair: Windows; Dealing with Problems + + +Antivirus Program +~~~~~~~~~~~~~~~~~ + +If you are not using the portable option, and you have **Enable VSS**:sup:`Dir`:sub:`FileSet` (Volume Shadow Copy) enabled in the |bareosDir| and you experience problems with Bareos not being able to open files, it is most likely that you are running an antivirus program that blocks Bareos from doing certain operations. In this case, disable the antivirus program and try another backup. If it succeeds, either get a different (better) antivirus program or use +something like **Client Run Before Job**:sup:`Dir`:sub:`Job` /**Client Run Before Job**:sup:`Dir`:sub:`Job` to turn off the antivirus program while the backup is running. + +If turning off anti-virus software does not resolve your VSS problems, you might have to turn on VSS debugging. The following link describes how to do this: `http://support.microsoft.com/kb/887013/en-us `_. + +Enable Debuggging +~~~~~~~~~~~~~~~~~ + +In case of problems, you can enable the creation of log files. For this you have to use the :program:`bconsole` :ref:`setdebug ` command: + + + + +.. code-block:: sh + :caption: Enable debug + + *setdebug client=bareos-fd level=200 trace=1 + Connecting to Client bareos-fd at bareos.example.com:9102 + 2000 OK setdebug=200 trace=1 hangup=0 tracefile=c:\bareos-fd.trace + +.. _Compatibility: + +Windows Compatibility Considerations +------------------------------------ + +.. index:: + pair: Windows; Compatibility Considerations + + +Exclusively Opened Files +~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are not using the :ref:`VSS` option and if any applications are running during the backup and they have files opened exclusively, Bareos will not be able to backup those files, so be sure you close your applications (or tell your users to close their applications) before the backup. Fortunately, most Microsoft applications do not open files exclusively so that they can be backed up. However, you will need to experiment. In any case, if Bareos cannot open the file, it will +print an error message, so you will always know which files were not backed up. If Volume Shadow Copy Service is enabled, Bareos is able to backing up any file. + +Backing up the Windows Registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +During backup, Bareos doesn’t know about the system registry, so you will either need to write it out to an ASCII file using :program:`regedit /e` or use a program specifically designed to make a copy or backup the registry. + +Windows Reparse Points +~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \sinceVersion{fd}{Windows!Reparse points}{12.4.5} + +.. index:: + pair: Windows; Symbolic links +.. index:: + pair: Windows; Junction points +.. index:: + pair: Windows; Volume Mount Points (VMP) + + +Besides normal files and directories, Windows filesystems also support special files, called "Reparse Points". Bareos can handle the following types of Reparse points: + +- Symbolic links to directories + +- Symbolic links to files + +- Junction Points + +- Volume Mount Points + +The Volume Mount Points are a special case of a Junction Point. To make things easier, in the following when talking about Junction Points, we mean only the Junction Points that are not Volume Mount Points. + +The Symbolic Links and the Junction Points are comparable to Symbolic Links in Unix/Linux. They are files that point to another location in the filesystem. + +Symbolic Links and Junction Points can be created with the Windows commandline command :program:`mklink`. + +When doing a directory listing in the commandline (cmd) in Windows, it shows the filetypes JUNCTION, SYMLINK or SYMLINKD and the target between the square brackets: + + + + +.. code-block:: sh + :caption: special files + + C:\linktest>dir + Volume in drive C has no label. + Volume Serial Number is C8A3-971F + + Directory of C:\linktest + + 08/07/2014 03:05 PM . + 08/07/2014 03:05 PM .. + 08/07/2014 02:59 PM dirlink [C:\Program Files\Bareos] + 08/07/2014 03:02 PM filelink [C:\Program Files\Bareos\bareos-dir.exe] + 08/07/2014 03:00 PM junction [C:\Program Files\Bareos] + 08/07/2014 03:05 PM volumemountpoint [\??\Volume{e960247d-09a1-11e3-93ec-005056add71d}\] + 1 File(s) 0 bytes + 5 Dir(s) 90,315,137,024 bytes free + +Symbolic Links. Directory Symbolic Links, and Junctions that are not a Volume MountPoint are treated by Bareos as symbolic links and are backed up and restored as they are, so the object is restored and points to where it pointed when it was backed up. + +Volume Mount Points are different. They allow to mount a harddisk partition as a subfolder of a drive instead of a driveletter. + +When backing up a Volume Mount Point, it is backed up as directory. + +If :strong:`OneFS` is set to yes (default), the Volume Mount Point (VMP) is backed up as directory but the content of the VMP will not be backed up. Also, the Joblog will contain a message like this: + + + + {Warning on Volume Moint Point and OneFS=yes} + C:/linktest/vmp is a different filesystem. Will not descend from C:/linktest into it. + +This is the normal behavior of the :strong:`OneFS` option. + +If OneFS is set to no, the filedaemon will change into the VMP as if it was a normal directory and will backup all files found inside of the VMP. + +VMPs and VSS Snapshots +^^^^^^^^^^^^^^^^^^^^^^ + +As Virtual Mount Points mounts another Volume into the current filesystem, it is desired that if the content of the VMP will be backed up during the backup (:strong:`onefs = no`), we also want to have this volume snapshotted via VSS. + +To achieve this, we now automatically check every volume added to the VSS snapshotset if it contains VMPs, and add the volumes mounted by those VMPs to the vss snapshotset recursively. + +Volumes can be mounted nested and multiple times, but can only be added to the snapshotset once. This is the reason why the number of vmps can be greater than the number of volumes added for the volume mount points. + +The Job Log will show how many VMPs were found like this: + + + + {Volume Mount Points are added automatically to VSS snapshots (if onefs=no)} + Volume Mount Points found: 7, added to snapshotset: 5 + +Accordingly, if OneFS is set to yes, we do not need to handle Volume Mount Points this way. If OneFS is set to yes (default), the joblog will contain the following information: + + + + {Volume Mount Points are ignored on VSS snapshots (if onefs=yes)} + VolumeMountpoints are not processed as onefs = yes. + +Hard Links +~~~~~~~~~~ + +Windows also supports hard links, even so they are seldom used. These are treated as normal files and will be restored as individual files (which will not be hardlinks again) + +.. _FilesNotToBackup: + +FilesNotToBackup Registry Key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \sinceVersion{fd}{Windows!FilesNotToBackup}{14.2.0} + +.. index:: + pair: Windows; Exclude Files from Backup + + +Windows supports a special Registry Key that specifies the names of the files and directories that backup applications should not backup or restore. + +The full path to this registry key is **HKEY_LOCAL_MACHINE:raw-latex:`\SYSTEM`:raw-latex:`\CurrentControlSet`:raw-latex:`\Control`:raw-latex:`\BackupRestore`:raw-latex:`\FilesNotToBackup`** + +Bareos automatically converts these entries to wildcards which will be automatically excluded from backup. + +The backup log shows a short information about the creation of the exludes like this: + + + + {Excludes according to the FilesNotToBackup registry key} + Created 28 wildcard excludes from FilesNotToBackup Registry key + +More details can be found if the filedaemon is run in debug mode inside of the :file:`bareos-fd.trace` logfile. Each entry and the resulting wildcard are logged. + + + + {translation between registry key FilesNotToBackup and Bareos Exclude FileSet} + client-win-fd: win32.c:465-0 (1) "WER" : + client-win-fd: win32.c:482-0 "C:\ProgramData\Microsoft\Windows\WER\* /s" + client-win-fd: win32.c:527-0 -> "C:/ProgramData/Microsoft/Windows/WER/*" + client-win-fd: win32.c:465-0 (2) "Kernel Dumps" : + client-win-fd: win32.c:482-0 "C:\Windows\Minidump\* /s" + client-win-fd: win32.c:527-0 -> "C:/Windows/Minidump/*" + client-win-fd: win32.c:482-0 "C:\Windows\memory.dmp" + client-win-fd: win32.c:527-0 -> "C:/Windows/memory.dmp" + client-win-fd: win32.c:465-0 (3) "Power Management" : + client-win-fd: win32.c:482-0 "\hiberfil.sys" + client-win-fd: win32.c:527-0 -> "[A-Z]:/hiberfil.sys" + client-win-fd: win32.c:465-0 (4) "MS Distributed Transaction Coordinator" : + client-win-fd: win32.c:482-0 "C:\Windows\system32\MSDtc\MSDTC.LOG" + client-win-fd: win32.c:527-0 -> "C:/Windows/system32/MSDtc/MSDTC.LOG" + client-win-fd: win32.c:482-0 "C:\Windows\system32\MSDtc\trace\dtctrace.log" + client-win-fd: win32.c:527-0 -> "C:/Windows/system32/MSDtc/trace/dtctrace.log" + +It is possible to disable this functionality by setting the FileSet option :strong:`AutoExclude` to no. + +The JobLog will then show the following informational line: + + + + {AutoExclude disabled} + Fileset has autoexclude disabled, ignoring FilesNotToBackup Registry key + +For more details about the Windows registry key see `http://msdn.microsoft.com/en-us/library/windows/desktop/bb891959%28v=vs.85%29.aspx#filesnottobackup `_. + +Windows dedup support +~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \sinceVersion{fd}{Windows!dedupclication}{12.4.5} + +Windows 2012 has dedup support which needs handling. + +Store all file attributes +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \sinceVersion{fd}{Windows!file attributes}{12.4.5} + +Windows has gathered quite some special specific file flags over the years but not all are saved during backup so some are never restored by the restore process. The most important ones are the ARCHIVE flag which is "misused" by some programs for storing some special information. Others that are known not to be stored are the COMPRESSED flag which means that a restored file looses it and will be restored as an uncompressed file. + +Support for Windows EFS filesystems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \sinceVersion{fd}{Windows!Encrypted Filesystems (EFS)}{12.4.5} + +Windows has support for a so called EFS filesystem. This is an encrypted filesystem, to be able to backup the data and to restore it we need to use a special API. With this API you in essence export the data on backup and import it on restore. This way you never have access to the unencrypted data but just import and export the encrypted data. This is the cleanest way of handling encryption by just seeing the data as some opaque data and not try to do anything special with it. + +Volume Shadow Copy Service (VSS) +-------------------------------- + +.. index:: + pair: Windows; Volume Shadow Copy Service +.. index:: + pair: Windows; VSS + + +.. _`VSS`: VSS + +VSS is available since Windows XP. From the perspective of a backup-solution for Windows, this is an extremely important step. VSS allows Bareos to backup open files and even to interact with applications like RDBMS to produce consistent file copies. VSS aware applications are called VSS Writers, they register with the OS so that when Bareos wants to do a Snapshot, the OS will notify the register Writer programs, which may then create a consistent state in their application, which will be backed +up. Examples for these writers are "MSDE" (Microsoft database engine), "Event Log Writer", "Registry Writer" plus 3rd party-writers. If you have a non-vss aware application a shadow copy is still generated and the open files can be backed up, but there is no guarantee that the file is consistent. + +Bareos produces a message from each of the registered writer programs when it is doing a VSS backup so you know which ones are correctly backed up. + +Technically Bareos creates a shadow copy as soon as the backup process starts. It does then backup all files from the shadow copy and destroys the shadow copy after the backup process. Please have in mind, that VSS creates a snapshot and thus backs up the system at the state it had when starting the backup. It will disregard file changes which occur during the backup process. + +VSS can be turned on by placing an + +.. index:: + single: Enable VSS +.. index:: + pair: VSS; Enable + + + + + Enable VSS = yes + +in your FileSet resource. + +The VSS aware File daemon has the letters VSS on the signon line that it produces when contacted by the console. For example: + + + + Tibs-fd Version: 1.37.32 (22 July 2005) VSS Windows XP MVS NT 5.1.2600 + +the VSS is shown in the line above. This only means that the File daemon is capable of doing VSS not that VSS is turned on for a particular backup. There are two ways of telling if VSS is actually turned on during a backup. The first is to look at the status output for a job, e.g.: :raw-latex:`` + + + + Running Jobs: + JobId 1 Job NightlySave.2005-07-23_13.25.45 is running. + VSS Backup Job started: 23-Jul-05 13:25 + Files=70,113 Bytes=3,987,180,650 Bytes/sec=3,244,247 + Files Examined=75,021 + Processing file: c:/Documents and Settings/user/My Documents/My Pictures/Misc1/Sans titre - 39.pdd + SDReadSeqNo=5 fd=352 + +.. raw:: latex + + + +Here, you see under Running Jobs that JobId 1 is "VSS Backup Job started ..." This means that VSS is enabled for that job. If VSS is not enabled, it will simply show "Backup Job started ..." without the letters VSS. + +The second way to know that the job was backed up with VSS is to look at the Job Report, which will look something like the following: :raw-latex:`` + + + + 23-Jul 13:25 rufus-dir: Start Backup JobId 1, Job=NightlySave.2005-07-23_13.25.45 + 23-Jul 13:26 rufus-sd: Wrote label to prelabeled Volume "TestVolume001" on device "DDS-4" (/dev/nst0) + 23-Jul 13:26 rufus-sd: Spooling data ... + 23-Jul 13:26 Tibs: Generate VSS snapshots. Driver="VSS WinXP", Drive(s)="C" + 23-Jul 13:26 Tibs: VSS Writer: "MSDEWriter", State: 1 (VSS_WS_STABLE) + 23-Jul 13:26 Tibs: VSS Writer: "Microsoft Writer (Bootable State)", State: 1 (VSS_WS_STABLE) + 23-Jul 13:26 Tibs: VSS Writer: "WMI Writer", State: 1 (VSS_WS_STABLE) + 23-Jul 13:26 Tibs: VSS Writer: "Microsoft Writer (Service State)", State: 1 (VSS_WS_STABLE) + +.. raw:: latex + + + +In the above Job Report listing, you see that the VSS snapshot was generated for drive C (if other drives are backed up, they will be listed on the :raw-latex:`\bconsoleOutput{Drive(s)="C"}` line. You also see the reports from each of the writer program. Here they all report VSS_WS_STABLE, which means that you will get a consistent snapshot of the data handled by that writer. + +VSS Problems +~~~~~~~~~~~~ + + +.. index:: + triple: Windows; Problem; VSS; + :raw-latex:`\index[general]{Windows!VSS!Problem}` :raw-latex:`\index[fd]{Windows!Problem!VSS}` :raw-latex:`\index[general]{Problem!Windows!VSS}` + +If you are experiencing problems such as VSS hanging on MSDE, first try running **vssadmin** to check for problems, then try running **ntbackup** which also uses VSS to see if it has similar problems. If so, you know that the problem is in your Windows machine and not with Bareos. + +The FD hang problems were reported with **MSDEwriter** when: + +- a local firewall locked local access to the MSDE TCP port (MSDEwriter seems to use TCP/IP and not Named Pipes). + +- msdtcs was installed to run under "localsystem": try running msdtcs under networking account (instead of local system) (com+ seems to work better with this configuration). + +Windows Firewalls +----------------- + +.. index:: + pair: Firewall; Windows +.. index:: + pair: Windows; Firewall + + +The Windows buildin Firewall is enabled since Windows version WinXP SP2. The Bareos installer opens the required network ports for Bareos. However, if you are using another Firewall, you might need to manually open the Bareos network ports. The |bareosFd| listens on 9102/TCP. + +Network TCP Port +~~~~~~~~~~~~~~~~ + +If you want to see if the File daemon has properly opened the port and is listening, you can enter the following command in a shell window: + + + + +.. code-block:: sh + :caption: + + netstat -an | findstr 910[123] + +Windows Restore Problems +------------------------ + +.. index:: + pair: Problem; Windows Restore +.. index:: + pair: Windows; Restore Problem + + +Please see the :ref:`section-RestoreOnWindows` chapter for problems that you might encounter doing a restore. + +Windows Backup Problems +----------------------- + +.. index:: + pair: Problem; Windows Backup +.. index:: + pair: Windows; Backup Problems + + +If during a Backup, you get the message: **ERR=Access is denied** and you are using the portable option, you should try both adding both the non-portable (backup API) and the Volume Shadow Copy options to your Director’s conf file. + +In the Options resource: :raw-latex:`` + + + + portable = no + +.. raw:: latex + + + +In the FileSet resource: :raw-latex:`` + + + + enablevss = yes + +.. raw:: latex + + + +In general, specifying these two options should allow you to backup any file on a Windows system. However, in some cases, if users have allowed to have full control of their folders, even system programs such a Bareos can be locked out. In this case, you must identify which folders or files are creating the problem and do the following: + +#. Grant ownership of the file/folder to the Administrators group, with the option to replace the owner on all child objects. + +#. Grant full control permissions to the Administrators group, and change the user’s group to only have Modify permission to the file/folder and all child objects. + +Thanks to Georger Araujo for the above information. + +Windows Ownership and Permissions Problems +------------------------------------------ + +.. index:: + pair: Problem; Windows Ownership and Permissions +.. index:: + pair: Windows; Ownership and Permissions Problems + + +If you restore files backed up from Windows to an alternate directory, Bareos may need to create some higher level directories that were not saved (or restored). In this case, the File daemon will create them under the SYSTEM account because that is the account that Bareos runs under as a service and with full access permission. However, there may be cases where you have problems accessing those files even if you run as administrator. In principle, Microsoft supplies you with the way to cease +the ownership of those files and thus change the permissions. However, a much better solution to working with and changing Win32 permissions is the program **SetACL**, which can be found at `http://setacl.sourceforge.net/ `_. + +If you have not installed Bareos while running as Administrator and if Bareos is not running as a Process with the userid (User Name) SYSTEM, then it is very unlikely that it will have sufficient permission to access all your files. + +Some users have experienced problems restoring files that participate in the Active Directory. They also report that changing the userid under which Bareos (bareos-fd.exe) runs, from SYSTEM to a Domain Admin userid, resolves the problem. + +.. raw:: latex + + \hide{ + \section{Fixing the Windows Boot Record} + \index[general]{Windows!Fixing the Boot Record} + + An effective way to restore a Windows backup is to install Windows on a different + hard drive and restore the backup. Then run the + recovery CD and run + + \ + diskpart + select disk 0 + select part 1 + active + exit + + bootrec /rebuldbcd + bootrec /fixboot + bootrec /fixmbr + \ + } + +Advanced Windows Configuration +------------------------------ + +Windows Service +~~~~~~~~~~~~~~~ + +The |bareosFd| (and also the |bareosDir| and |bareosSd|) is started as a Windows service. + +This is configured in the Registry at + +- :file:`Computer\HKEY\_LOCAL\_MACHINE\SYSTEM\CurrentControlSet\services\Bareos-fd` + +You can use the command :program:`regedit` to modify the settings. + +E.g. to always start Bareos in debug mode, modify :file:`Computer\HKEY\_LOCAL\_MACHINE\SYSTEM\CurrentControlSet\services\Bareos-fd`` ``path:ImagePath` from + +.. raw:: latex + + + + + + "C:\Program Files\Bareos\bareos-fd.exe" /service + +to + +.. raw:: latex + + + + + + "C:\Program Files\Bareos\bareos-fd.exe" /service -d200 + +After restarting the service, you will find a file called :file:`C:\bareos-fd.trace` which will contain the debug output created by the daemon. + +Installing multiple Windows filedaemon services +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is possible to run multiple |bareosFd| instances on Windows. To achieve this, you need to create a service for each instance, and a configuration file that at least has a individual fd port for each instance. + +To create two bareos-fd services, you can call the following service create calls on the commandline on windows as administrator: + + + + +.. code-block:: sh + :caption: + + sc create bareosfd2 binpath="\"C:\Program Files\Bareos\bareos-fd.exe\" /service -c \"C:\ProgramData\Bareos\bareos-fd2.conf\"" depend= "tcpip/afd" + sc create bareosfd3 binpath="\"C:\Program Files\Bareos\bareos-fd.exe\" /service -c \"C:\ProgramData\Bareos\bareos-fd3.conf\"" depend= "tcpip/afd" + +This will create two |bareosFd| services, one with the name bareosfd2 and the second with the name bareosfd3. + +The configuration files for the two services are :file:`bareos-fd.conf` and :file:`bareos-fd2.conf`, and need to have different network settings. + +The services can be started by calling + + + + +.. code-block:: sh + :caption: + + sc start bareosfd2 + +and + + + + +.. code-block:: sh + :caption: + + sc start bareosfd3 + +Windows Specific Command Line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Windows; File Daemon; Command Line Options; + + +These options are not normally seen or used by the user, and are documented here only for information purposes. At the current time, to change the default options, you must either manually run **Bareos** or you must manually edit the system registry and modify the appropriate entries. + +In order to avoid option clashes between the options necessary for **Bareos** to run on Windows and the standard Bareos options, all Windows specific options are signaled with a forward slash character (/), while as usual, the standard Bareos options are signaled with a minus (-), or a minus minus (``--``). All the standard Bareos options can be used on the Windows version. In addition, the following Windows only options are implemented: + +/service + Start Bareos as a service + +/run + Run the Bareos application + +/install + Install Bareos as a service in the system registry + +/remove + Uninstall Bareos from the system registry + +/about + Show the Bareos about dialogue box + +/status + Show the Bareos status dialogue box + +/events + Show the Bareos events dialogue box (not yet implemented) + +/kill + Stop any running **Bareos** + +/help + Show the Bareos help dialogue box + +It is important to note that under normal circumstances the user should never need to use these options as they are normally handled by the system automatically once Bareos is installed. However, you may note these options in some of the .bat files that have been created for your use. + +.. |image| image:: \idir win-install-1 + :width: 80.0% +.. |image| image:: \idir win-install-2 + :width: 80.0% +.. |image| image:: \idir win-install-3 + :width: 80.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter28/client-initiated-connection.rst b/docs/manuals/en/new_main_reference/source/chapter28/client-initiated-connection.rst new file mode 100644 index 00000000000..8654324c640 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter28/client-initiated-connection.rst @@ -0,0 +1,88 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _section-ClientInitiatedConnection: + +Client Initiated Connection +=========================== + +The |bareosDir| knows, when it is required to talk to a client (|bareosFd|). Therefore, by defaults, the |bareosDir| connects to the clients. + +However, there are setups where this can cause problems, as this means that: + +- The client must be reachable by its configured **Address**:sup:`Dir`:sub:`Client` . Address can be the DNS name or the IP address. (For completeness: there are potential workarounds by using the :ref:`setip ` command.) + +- The |bareosDir| must be able to connect to the |bareosFd| over the network. + +To circumvent these problems, since Bareos 16.2.2 it is possible to let the |bareosFd| initiate the network connection to the |bareosDir|. + +Which address the client connects to reach the |bareosDir| is configured in the **Address**:sup:`Fd`:sub:`Director` directive. + +To additional allow this connection direction use: + +- **Connection From Client To Director**:sup:`Dir`:sub:`Client` = yes + +- **Heartbeat Interval**:sup:`Dir`:sub:`Client` = 60 ``#`` to keep the network connection established + +- **Connection From Client To Director**:sup:`Fd`:sub:`Director` = yes + +To only allow Connection From the Client to the Director use: + +- **Connection From Director To Client**:sup:`Dir`:sub:`Client` = no + +- **Connection From Client To Director**:sup:`Dir`:sub:`Client` = yes + +- **Heartbeat Interval**:sup:`Dir`:sub:`Client` = 60 ``#`` to keep the network connection established + +- **Connection From Director To Client**:sup:`Fd`:sub:`Director` = no + +- **Connection From Client To Director**:sup:`Fd`:sub:`Director` = yes + +Using Client Initiated Connections has disadvantages. Without Client Initiated Connections the |bareosDir| only establishes a network connection when this is required. With Client Initiated Connections, the |bareosFd| connects to the |bareosDir| and the |bareosDir| keeps these connections open. The command :strong:`status dir` will show all waiting connections: + + + + +.. code-block:: sh + :caption: show waiting client connections + + *status dir + ... + Client Initiated Connections (waiting for jobs): + Connect time Protocol Authenticated Name + ==================================================================================================== + 19-Apr-16 21:50 54 1 client1.example.com + ... + ==== + +When both connection directions are allowed, the |bareosDir| + +#. checks, if there is a waiting connection from this client. + +#. tries to connect to the client (using the usual timeouts). + +#. waits for a client connection to appear (using the same timeout as when trying to connect to a client). + +If none of this worked, the job fails. + +When a waiting connection is used for a job, the |bareosFd| will detect this and creates an additional connection. This is required, to keep the client responsive for additional commands, like :strong:`cancel`. + +To get feedback in case the |bareosFd| fails to connect to the |bareosDir|, consider configuring |bareosFd| to log in a local file. This can be archived by adding the line + +.. raw:: latex + + \configline{Append = "/var/log/bareos/bareos-fd.log" = all, !skipped, !restored} + +to the default message resource **Standard**:sup:`Fd`:sub:`Messages` : + + + + +.. code-block:: sh + :caption: bareos-fd messages Standard + + Messages { + Name = Standard + Director = bareos-dir = all, !skipped, !restored + Append = "/var/log/bareos/bareos-fd.log" = all, !skipped, !restored + } diff --git a/docs/manuals/en/new_main_reference/source/chapter28/lanaddress.rst b/docs/manuals/en/new_main_reference/source/chapter28/lanaddress.rst new file mode 100644 index 00000000000..bb2a2c78aa6 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter28/lanaddress.rst @@ -0,0 +1,111 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _LanAddress: + +Using different IP Adresses for SD – FD Communication +===================================================== + +.. index:: + single: Lan Address + + +Bareos supports network topologies where the |bareosFd| and |bareosSd| are situated inside of a LAN, but the |bareosDir| is outside of that LAN in the Internet and accesses the |bareosFd| and |bareosSd| via SNAT / port forwarding. + +Consider the following scheme: + + + + +.. code-block:: sh + :caption: + + /-------------------\ + | | LAN 10.0.0.1/24 + | | + | FD_LAN SD_LAN | + | .10 .20 | + | | + \___________________/ + | + NAT Firewall + FD: 8.8.8.10 -> 10.0.0.10 + SD: 8.8.8.20 -> 10.0.0.20 + | + /-------------------\ + | | + | | WAN / Internet + | DIR | + | 8.8.8.100 | + | | + | FD_WAN SD_WAN | + | .30 .40 | + \___________________/ + +The |bareosDir| can access the :strong:`FD_LAN` via the IP 8.8.8.10, which is forwarded to the IP 10.0.0.10 inside of the LAN. + +The |bareosDir| can access the :strong:`SD_LAN` via the IP 8.8.8.20 which is forwarded to the IP 10.0.0.20 inside of the LAN. + +There is also a |bareosFd| and a |bareosSd| outside of the LAN, which have the IPs 8.8.8.30 and 8.8.8.40 + +All resources are configured so that the :strong:`Address` directive gets the address where the |bareosDir| can reach the daemons. + +Additionally, devices being in the LAN get the LAN address configured in the :strong:`Lan Address` directive. The configuration looks as follows: + + + + +.. code-block:: sh + :caption: bareos-dir client FD\_LAN + + Client { + Name = FD_LAN + Address = 8.8.8.10 + LanAddress = 10.0.0.10 + ... + } + + + + +.. code-block:: sh + :caption: bareos-dir client SD\_LAN + + Storage { + Name = SD_LAN + Address = 8.8.8.20 + LanAddress = 10.0.0.20 + ... + } + + + + +.. code-block:: sh + :caption: bareos-dir client FD\_WAN + + Client { + Name = FD_WAN + Address = 8.8.8.30 + ... + } + + + + +.. code-block:: sh + :caption: bareos-dir client SD\_WAN + + Storage { + Name = SD_WAN + Address = 8.8.8.40 + ... + } + +This way, backups and restores from each |bareosFd| using each |bareosSd| are possible as long as the firewall allows the needed network connections. + +The |bareosDir| simply checks if both the involved |bareosFd| and |bareosSd| both have a :strong:`Lan Address` (**Lan Address**:sup:`Dir`:sub:`Client` and **Lan Address**:sup:`Dir`:sub:`Storage` ) configured. + +In that case, the initiating daemon is ordered to connect to the :strong:`Lan Address` instead of the :strong:`Address`. In active client mode, the |bareosFd| connects to the |bareosSd|, in passive client mode (see :ref:`PassiveClient`) the |bareosSd| connects to the |bareosFd|. + +If only one or none of the involved |bareosFd| and |bareosSd| have a :strong:`Lan Address` configured, the :strong:`Address` is used as connection target for the initiating daemon. diff --git a/docs/manuals/en/new_main_reference/source/chapter28/passiveclient.rst b/docs/manuals/en/new_main_reference/source/chapter28/passiveclient.rst new file mode 100644 index 00000000000..e75bee6d53d --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter28/passiveclient.rst @@ -0,0 +1,67 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _PassiveClient: + +Passive Clients +=============== + +The normal way of initializing the data channel (the channel where the backup data itself is transported) is done by the |bareosFd| (client) that connects to the |bareosSd|. + +In many setups, this can cause problems, as this means that: + +- The client must be able to resolve the name of the |bareosSd| (often not true, you have to do tricks with the hosts file) + +- The client must be allowed to create a new connection. + +- The client must be able to connect to the |bareosSd| over the network (often difficult over NAT or Firewall) + +By using Passive Client, the initialization of the datachannel is reversed, so that the storage daemon connects to the |bareosFd|. This solves almost every problem created by firewalls, NAT-gateways and resolving issues, as + +- The |bareosSd| initiates the connection, and thus can pass through the same or similar firewall rules that the director already has to access the |bareosFd|. + +- The client never initiates any connection, thus can be completely firewalled. + +- The client never needs any name resolution and is totally independent from any resolving issues. + +|image| + +Usage +----- + +To use this new feature, just configure **Passive**:sup:`Dir`:sub:`Client` =yes in the client definition of the |bareosDir|: + + + + +.. code-block:: sh + :caption: Enable passive mode in bareos-dir.conf + + Client { + Name = client1-fd + Password = "secretpassword" + Passive = yes + [...] + } + +Also, prior to bareos version 15, you need to set **Compatible**:sup:`Fd`:sub:`Client` =no in the :file:`bareos-fd.conf` configuration file. Since Bareos Version 15, the compatible option is set to no per default and does not need to be specified anymore. + + + + +.. code-block:: sh + :caption: Disable compatible mode for the \bareosFd in bareos-fd.conf + + Director { + Name = bareos-dir + Password = "secretpassword" + } + + Client { + Name = client1-fd + [...] + Compatible = no + } + +.. |image| image:: \idir passive-client-communication + :width: 60.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter29/tls.rst b/docs/manuals/en/new_main_reference/source/chapter29/tls.rst new file mode 100644 index 00000000000..5fe7e273a4f --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter29/tls.rst @@ -0,0 +1,302 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _CommEncryption: + +Transport Encryption +==================== + + + +.. _`section-TransportEncryption}` :raw-latex:`\index[general]{Communications Encryption}` :raw-latex:`\index[general]{Encryption!Communication}` :raw-latex:`\index[general]{Encryption!Transport}` :raw-latex:`\index[general]{Transport Encryption}` :raw-latex:`\index[general]{TLS}` :raw-latex:`\index[general]{SSL`: section-TransportEncryption}` :raw-latex:`\index[general]{Communications Encryption}` :raw-latex:`\index[general]{Encryption!Communication}` :raw-latex:`\index[general]{Encryption!Transport}` :raw-latex:`\index[general]{Transport Encryption}` :raw-latex:`\index[general]{TLS}` :raw-latex:`\index[general]{SSL + +Bareos TLS (Transport Layer Security) is built-in network encryption code to provide secure network transport similar to that offered by :program:`stunnel` or :program:`ssh`. The data written to Volumes by the Storage daemon is not encrypted by this code. For data encryption, please see the :ref:`DataEncryption` chapter. + +The initial Bacula encryption implementation has been written by Landon Fuller. + +Supported features of this code include: + +- Client/Server TLS Requirement Negotiation + +- TLSv1 Connections with Server and Client Certificate Validation + +- Forward Secrecy Support via Diffie-Hellman Ephemeral Keying + +This document will refer to both :emphasis:`server` and :emphasis:`client` contexts. These terms refer to the accepting and initiating peer, respectively. + +Diffie-Hellman anonymous ciphers are not supported by this code. The use of DH anonymous ciphers increases the code complexity and places explicit trust upon the two-way CRAM-MD5 implementation. CRAM-MD5 is subject to known plaintext attacks, and it should be considered considerably less secure than PKI certificate-based authentication. + +.. _TlsDirectives: + +TLS Configuration Directives +---------------------------- + +Additional configuration directives have been added to all the daemons (Director, File daemon, and Storage daemon) as well as the various different Console programs. These directives are defined as follows: + +.. raw:: latex + + \begin{description} + \directive{dir}{TLS Enable}{yes{\textbar}no}{}{no}{}% + Enable TLS support. Without setting \configdirective{TLS Require}=yes, + the connection can fall back to unencrypted connection, + if the other side does not support TLS. + + \directive{dir}{TLS Require}{yes{\textbar}no}{}{no}{}% + Require TLS connections. + If TLS is not required, + then Bareos will connect with other daemons either with or without TLS depending + on what the other daemon requests. + If TLS is required, + then Bareos will refuse any connection that does not use TLS. + \configdirective{TLS Require}=yes implicitly sets \configdirective{TLS Enable}=yes. + + \directive{dir}{TLS Certificate}{filename}{}{}{}% + The full path and filename of a PEM encoded TLS certificate. It can be + used as either a client or server certificate. + It is used because PEM files are base64 encoded and hence ASCII + text based rather than binary. + They may also contain encrypted information. + + \directive{dir}{TLS Key}{filename}{}{}{}% + The full path and filename of a PEM encoded TLS private key. It must + correspond to the certificate specified in the \configdirective{TLS Certificate} configuration directive. + + \directive{dir}{TLS Verify Peer}{yes{\textbar}no}{}{}{}% + Request and verify the peers certificate. + + In server context, unless the \configdirective{TLS Allowed CN} configuration directive is specified, + any client certificate signed by a known-CA will be accepted. + + In client context, the server certificate CommonName attribute is checked against + the \configdirective{Address} and \configdirective{TLS Allowed CN} configuration directives. + + + \directive{dir}{TLS Allowed CN}{stringlist}{}{}{}% + Common name attribute of allowed peer certificates. + If \configdirective{TLS Verify Peer}=yes, all connection request certificates + will be checked against this list. + + This directive may be specified more than once. + + + \directive{dir}{TLS CA Certificate File}{filename}{}{}{}% + The full path and filename specifying a + PEM encoded TLS CA certificate(s). Multiple certificates are + permitted in the file. + + In a client context, one of + \configdirective{TLS CA Certificate File} or \configdirective{TLS CA Certificate Dir} + is required. + + In a server context, it is only required if \configdirective{TLS Verify Peer} is used. + + \directive{dir}{TLS CA Certificate Dir}{directory}{}{}{}% + Full path to TLS CA certificate directory. In the current implementation, + certificates must be stored PEM encoded with OpenSSL-compatible hashes, + which is the subject name's hash and an extension of {\bf .0}. + + In a client context, one of + \configdirective{TLS CA Certificate File} or \configdirective{TLS CA Certificate Dir} + is required. + + In a server context, it is only required if \configdirective{TLS Verify Peer} is used. + + + \directive{dir}{TLS DH File}{filename}{}{}{}% + Path to PEM encoded Diffie-Hellman parameter file. If this directive is + specified, DH key exchange will be used for the ephemeral keying, allowing + for forward secrecy of communications. DH key exchange adds an additional + level of security because the key used for encryption/decryption by the + server and the client is computed on each end and thus is never passed over + the network if Diffie-Hellman key exchange is used. Even if DH key + exchange is not used, the encryption/decryption key is always passed + encrypted. This directive is only valid within a server context. + + To generate the parameter file, you + may use openssl: + + \ +.. code-block:: sh + :caption: create DH key + + openssl dhparam -out dh1024.pem -5 1024 + \ + + \end{description} + +Getting TLS Certificates +------------------------ + +To get a trusted certificate (CA or Certificate Authority signed certificate), you will either need to purchase certificates signed by a commercial CA or become a CA yourself, and thus you can sign all your own certificates. + +Bareos is known to work well with RSA certificates. + +You can use programs like `xca `_ or TinyCA to easily manage your own CA with a Graphical User Interface. + +Example TLS Configuration Files +------------------------------- + +.. index:: + single: TLS Configuration Files + + +An example of the TLS portions of the configuration files are listed below. + +Another example can be found at `Bareos Regression Testing Base Configuration `_. + +Bareos Director +~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: bareos-dir director bareos-dir + + Director { # define myself + Name = bareos-dir + ... + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + # This is a server certificate, used for incoming + # (console) connections. + TLS Certificate = /etc/bareos/tls/bareos-dir.example.com-cert.pem + TLS Key = /etc/bareos/tls/bareos-dir.example.com-key.pem + TLS Verify Peer = yes + TLS Allowed CN = "bareos@backup1.example.com" + TLS Allowed CN = "administrator@example.com" + } + + + + +.. code-block:: sh + :caption: bareos-dir storage File + + Storage { + Name = File + Address = bareos-sd1.example.com + ... + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + # This is a client certificate, used by the director to + # connect to the storage daemon + TLS Certificate = /etc/bareos/tls/bareos-dir.example.com-cert.pem + TLS Key = /etc/bareos/tls/bareos-dir.example.com-key.pem + TLS Allowed CN = bareos-sd1.example.com + } + + + + +.. code-block:: sh + :caption: bareos-dir client client1-fd + + Client { + Name = client1-fd + Address = client1.example.com + ... + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + TLS Certificate = "/etc/bareos/tls/bareos-dir.example.com-cert.pem" + TLS Key = "/etc/bareos/tls/bareos-dir.example.com-key.pem" + TLS Allowed CN = client1.example.com + } + +Bareos Storage Daemon +~~~~~~~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: bareos-sd storage bareos-sd1 + + Storage { + Name = bareos-sd1 + ... + # These TLS configuration options are used for incoming + # file daemon connections. Director TLS settings are handled + # in Director resources. + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + # This is a server certificate. It is used by connecting + # file daemons to verify the authenticity of this storage daemon + TLS Certificate = /etc/bareos/tls/bareos-sd1.example.com-cert.pem + TLS Key = /etc/bareos/tls/bareos-sd1.example.com-key.pem + # Peer verification must be disabled, + # or all file daemon CNs must be listed in "TLS Allowed CN". + # Peer validity is verified by the storage connection cookie + # provided to the File Daemon by the Director. + TLS Verify Peer = no + } + + + + +.. code-block:: sh + :caption: bareos-sd director bareos-dir + + Director { + Name = bareos-dir + ... + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + # This is a server certificate. It is used by the connecting + # director to verify the authenticity of this storage daemon + TLS Certificate = /etc/bareos/tls/bareos-sd1.example.com-cert.pem + TLS Key = /etc/bareos/tls/bareos-sd1.example.com-key.pem + # Require the connecting director to provide a certificate + # with the matching CN. + TLS Verify Peer = yes + TLS Allowed CN = "bareos-dir.example.com" + } + +Bareos File Daemon +~~~~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: bareos-fd client myself + + Client { + Name = client1-fd + ... + # you need these TLS entries so the SD and FD can + # communicate + TLS Enable = yes + TLS Require = yes + + TLS CA Certificate File = /etc/bareos/tls/ca.pem + TLS Certificate = /etc/bareos/tls/client1.example.com-cert.pem + TLS Key = /etc/bareos/tls/client1.example.com-key.pem + + TLS Allowed CN = bareos-sd1.example.com + } + + + + +.. code-block:: sh + :caption: bareos-fd director bareos-dir + + Director { + Name = bareos-dir + ... + TLS Enable = yes + TLS Require = yes + TLS CA Certificate File = /etc/bareos/tls/ca.pem + # This is a server certificate. It is used by connecting + # directors to verify the authenticity of this file daemon + TLS Certificate = /etc/bareos/tls/client11.example.com-cert.pem + TLS Key = /etc/bareos/tls/client1.example.com-key.pem + TLS Verify Peer = yes + # Allow only the Director to connect + TLS Allowed CN = "bareos-dir.example.com" + } diff --git a/docs/manuals/en/new_main_reference/source/chapter30/dataencryption.rst b/docs/manuals/en/new_main_reference/source/chapter30/dataencryption.rst new file mode 100644 index 00000000000..5682f1db16d --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter30/dataencryption.rst @@ -0,0 +1,156 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _DataEncryption: + +Data Encryption +=============== + +.. index:: + single: Data Encryption +.. index:: + pair: Encryption; Data + + +Bareos permits file data encryption and signing within the File Daemon (or Client) prior to sending data to the Storage Daemon. Upon restoration, file signatures are validated and any mismatches are reported. At no time does the Director or the Storage Daemon have access to unencrypted file contents. + +.. raw:: latex + + +.. warning:: + These feature is only available, if Bareos is build against OpenSSL. + +It is very important to specify what this implementation does NOT do: + +- The implementation does not encrypt file metadata such as file path names, permissions, ownership and extended attributes. However, Mac OS X resource forks are encrypted. + +Encryption and signing are implemented using RSA private keys coupled with self-signed x509 public certificates. This is also sometimes known as PKI or Public Key Infrastructure. + +Each File Daemon should be given its own unique private/public key pair. In addition to this key pair, any number of "Master Keys" may be specified – these are key pairs that may be used to decrypt any backups should the File Daemon key be lost. Only the Master Key’s public certificate should be made available to the File Daemon. Under no circumstances should the Master Private Key be shared or stored on the Client machine. + +The Master Keys should be backed up to a secure location, such as a CD placed in a in a fire-proof safe or bank safety deposit box. The Master Keys should never be kept on the same machine as the Storage Daemon or Director if you are worried about an unauthorized party compromising either machine and accessing your encrypted backups. + +While less critical than the Master Keys, File Daemon Keys are also a prime candidate for off-site backups; burn the key pair to a CD and send the CD home with the owner of the machine. + +.. raw:: latex + + \warning{If you lose your encryption keys, backups will be unrecoverable. + {\bf always} store a copy of your master keys in a secure, off-site location.} + +The basic algorithm used for each backup session (Job) is: + +#. The File daemon generates a session key. + +#. The FD encrypts that session key via PKE for all recipients (the file daemon, any master keys). + +#. The FD uses that session key to perform symmetric encryption on the data. + +Encryption Technical Details +---------------------------- + +.. index:: + pair: Encryption; Technical Details + + +The implementation uses 128bit AES-CBC, with RSA encrypted symmetric session keys. The RSA key is user supplied. If you are running OpenSSL :math:`>=` 0.9.8, the signed file hash uses SHA-256, otherwise SHA-1 is used. + +End-user configuration settings for the algorithms are not currently exposed, only the algorithms listed above are used. However, the data written to Volume supports arbitrary symmetric, asymmetric, and digest algorithms for future extensibility, and the back-end implementation currently supports: + + + + Symmetric Encryption: + - 128, 192, and 256-bit AES-CBC + - Blowfish-CBC + + Asymmetric Encryption (used to encrypt symmetric session keys): + - RSA + + Digest Algorithms: + - MD5 + - SHA1 + - SHA256 + - SHA512 + +The various algorithms are exposed via an entirely re-usable, OpenSSL-agnostic API (ie, it is possible to drop in a new encryption backend). The Volume format is DER-encoded ASN.1, modeled after the Cryptographic Message Syntax from RFC 3852. Unfortunately, using CMS directly was not possible, as at the time of coding a free software streaming DER decoder/encoder was not available. + +Generating Private/Public Encryption Keys +----------------------------------------- + +.. index:: + pair: Encryption; Generating Private/Public Encryption Keypairs + + +Generate a Master Key Pair with: + +.. raw:: latex + + + + + + openssl genrsa -out master.key 2048 + openssl req -new -key master.key -x509 -out master.cert + +.. raw:: latex + + + +Generate a File Daemon Key Pair for each FD: + +.. raw:: latex + + + + + + openssl genrsa -out fd-example.key 2048 + openssl req -new -key fd-example.key -x509 -out fd-example.cert + cat fd-example.key fd-example.cert >fd-example.pem + +.. raw:: latex + + + +Note, there seems to be a lot of confusion around the file extensions given to these keys. For example, a .pem file can contain all the following: private keys (RSA and DSA), public keys (RSA and DSA) and (x509) certificates. It is the default format for OpenSSL. It stores data Base64 encoded DER format, surrounded by ASCII headers, so is suitable for text mode transfers between systems. A .pem file may contain any number of keys either public or private. We use it in cases where there is both a +public and a private key. + +Above we have used the .cert extension to refer to X509 certificate encoding that contains only a single public key. + +Example Data Encryption Configurations (bareos-fd.conf) +------------------------------------------------------- + +.. index:: + pair: Example; Data Encryption Configuration File + + +.. raw:: latex + + .. literalinclude:: ../../main/config/FdClientPki.conf + +Decrypting with a Master Key +---------------------------- + +.. index:: + single: Decrypting with a Master Key +.. index:: + pair: Encryption; Decrypting with a Master Key + + +It is preferable to retain a secure, non-encrypted copy of the client’s own encryption keypair. However, should you lose the client’s keypair, recovery with the master keypair is possible. + +You must: + +- Concatenate the master private and public key into a single keypair file, ie: + + + + cat master.key master.cert > master.keypair + + +- Set the PKI Keypair statement in your bareos configuration file: + + + + PKI Keypair = master.keypair + +- Start the restore. The master keypair will be used to decrypt the file data. diff --git a/docs/manuals/en/new_main_reference/source/chapter31/ndmp.rst b/docs/manuals/en/new_main_reference/source/chapter31/ndmp.rst new file mode 100644 index 00000000000..5677f96c72e --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter31/ndmp.rst @@ -0,0 +1,2158 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. index:: + pair: NDMP; Overview + + +NDMP Basics +=========== + +NDMP + +- is the abbreviation for Network Data Management Protocol. + +- is a protocol that transports data between Network Attached Storages (NAS) and backup devices. + +- is widely used by storage product vendors and OS vendors like NetApp, Isilon, EMC, Oracle. + +- information is available at `http://www.ndmp.org/ `_. + +- version is currently (2016) NDMP Version 4. + +- uses TCP/IP and XDR (External Data Representation) for the communication. + +The Bareos NDMP implementation is based on the NDMJOB NDMP reference implementation of Traakan, Inc., Los Altos, CA which has a BSD style license (2 clause one) with some enhancements. + +In NDMP, there are different components (called :emphasis:`agents`) involved in doing backups. The most important agents are: + +Data Management Agent (DMA) + is the part that controls the NDMP backup or recover operation. + +Data Agent + (or Primary Storage System) is the part that reads the data from the Filesystem during Backup and writes data to the Filesystem during recover. + +Tape Agent + (or Secondary Storage System) is the part that writes NDMP blocks to the Tape during backup and reads them during recover. + +Robot Agent + is the part that controls the media changer. It loads/unloads tapes and gets the inventory of the Changer. The use of a robot agent is optional. It is possible to run backups on a single tape drive. + +All elements involved talk to each other via the NDMP protocol which is usually transported via TCP/IP port 10000. + +The :raw-latex:`\DataManagementAgent `is part of the Backup Application. + +The :raw-latex:`\DataAgent `is part of the (NAS)-System being backed up and recovered. + +The :raw-latex:`\TapeAgent `and :raw-latex:`\RobotAgent `can + +- run on the system being backed up + +- run as part of the backup application + +- or even run independently on a third system + +This flexibility leads to different topologies how NDMP backups can be done. + +NDMP Topologies +--------------- + +When looking at the different topologies, the location of the :raw-latex:`\RobotAgent `is not specially considered, as the data needed to control the robot is minimal compared to the backup data. + +So the parts considered are + +- the :raw-latex:`\DataManagementAgent `controlling the operation + +- the :raw-latex:`\DataAgent `as source of the backup data and + +- the :raw-latex:`\TapeAgent `as destination of the backup data + +The :raw-latex:`\DataManagementAgent `always controls both :raw-latex:`\DataAgent `and :raw-latex:`\TapeAgent `over the Network via NDMP. + +The :raw-latex:`\TapeAgent `can either + +- run on a separate system + +- run on the same system + +as the Data Agent. + +NDMP 3-way Backup: Data Agent and Tape Agent running on different systems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + --+--------------- NETWORK ----+-------------------+---- + | -->---->---->-->-->-->\ | //==>==>==>==>\\ | + | / | | || (2) || | + | | | | || || | + /----------\ /----------\ /----------\ + | | | | | | + | DMA | DISK====>| DATA | | Tape |====>TAPE DRIVE + | | (1) | Agent | | Agent | (3) + \----------/ \----------/ \----------/ + +The data path consists of three ways + +- From Disk to Data Agent (1) + +- From Data Agent over the Network to the Tape Agent (2) + +- From Tape Agent to the Tape (3) + +and is called NDMP 3-way Backup. + +NDMP 2-way Backup: Data Agent and Tape Agent running on the same system +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + --+--------------- NETWORK ----+--------- + | -->---->---->-->-->-->\ | + | / | | + | | | | + /----------\ /----------\ + | | | Data | + | DMA | DISK====>| Agent | + | | (1) | | + \----------/ | Tape | (2) + | Agent |====>TAPE DRIVE + \----------/ + +:raw-latex:`\DataAgent `and :raw-latex:`\TapeAgent `are both part of the same process on the system, so the data path consists of two ways: + +- From Disk to Data Agent (1) + +- From Tape Agent to the Tape (2) + +and is called NDMP 2-way Backup, also sometimes referred as NDMP local backup. + +Properties of the different NDMP Backup topologies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NDMP 3-way backup: + +- The data can be send to a different location over the network + +- No need to attach a tape drive to the NAS system. + +- The backup speed is usually slower than with 2-way backup as the data is being sent over the network and processed multiple times. + +NDMP 2-way backup: + +- The data is directly copied from the NAS system to the Tape + +- Usually the fastest way to do a NDMP backup + +- tape drives need to be attached to the NAS System + +NDMP Backup in Bareos +===================== + +Bareos offers two types of NDMP integration: + +NDMP_NATIVE +NDMP_BAREOS + +In both cases, + +- |bareosDir| acts as :raw-latex:`\DataManagementAgent`. + +- The :raw-latex:`\DataAgent `is part of the storage system and must be provided by the storage vendor. + +The main difference is which :raw-latex:`\TapeAgent `is used. + +When using NDMP_BAREOS, the |bareosSd| acts as :raw-latex:`\TapeAgent`. + +When using NDMP_NATIVE, the :raw-latex:`\TapeAgent `must be provided by some other systems. Some storage vendors provide it with there storages, or offer it as an option, e.g. Isilon with their :emphasis:`Isilon Backup Accelerator`. + ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| | :raw-latex:`\NdmpBareos ` | :raw-latex:`\NdmpNative ` | ++==============================================================================+===================================================+===================================================+ +| :raw-latex:`\DataManagementAgent ` | |bareosDir| | |bareosDir| | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| :raw-latex:`\TapeAgent ` | |bareosSd| | external | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| requires external :raw-latex:`\TapeAgent ` | | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| backup to tape (and VTL) | :raw-latex:`\bcheckmark ` | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| backup to other **Device Type**:sup:`Sd`:sub:`Device` | :raw-latex:`\bcheckmark ` | | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| 2-way backup | | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| 3-way backup | :raw-latex:`\bcheckmark ` | untested | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| Full Backups | :raw-latex:`\bcheckmark ` | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| Differential Backups | :raw-latex:`\bcheckmark ` | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| Incremental Backups | **section-NdmpBackupLevel** (8) | **section-NdmpBackupLevel** (8) | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| Single File Restore | :raw-latex:`\bcheckmark ` | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| DAR | | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| DDAR | | :raw-latex:`\bcheckmark ` | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ +| .. raw:: latex | :raw-latex:`\bcheckmark ` | | +| | | | +| \ilink{Copy and Migration jobs}{MigrationChapter} | | | ++------------------------------------------------------------------------------+---------------------------------------------------+---------------------------------------------------+ + +.. _section-NdmpBareos: + +NDMP_BAREOS +=========== + +Bareos implements the :raw-latex:`\DataManagementAgent `inside of the |bareosDir| and a :raw-latex:`\TapeAgent `in the |bareosSd|. + +The :raw-latex:`\TapeAgent `in the |bareosSd| emulates a NDMP tape drive that has an infinite tape. Because of the infinite tape, no :raw-latex:`\RobotAgent `is required and therefore not implemented. The blocks being written to the NDMP tape are wrapped into a normal Bareos backup stream and then stored into the volumes managed by Bareos. + +There is always a pair of storage resource definitions: + +- a conventional Bareos storage resource and + +- a NDMP storage resource + +These two are linked together. Data that is received by the :raw-latex:`\TapeAgent `inside of the |bareosSd| is then stored as Bareos backup stream inside of the paired conventional Bareos storage resource. + +On restore, the data is read by the conventional resource, and then recovered as NDMP stream from the NDMP resource. + +.. raw:: latex + + \centering + +.. figure:: \idir ndmp-backup + :alt: Relationship between Bareos and NDMP components + + Relationship between Bareos and NDMP components + +Example Setup for NDMP_BAREOS backup +------------------------------------ + + +.. index:: + triple: NDMP; Example; NDMP\_BAREOS; + + +This example starts from a clean default Bareos installation. + +Enable NDMP on your storage appliance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The storage appliance needs to be configured to allow NDMP connections. Therefore usually the NDMP service needs to be enabled and configured with a username and password. + +Bareos Director: Configure NDMP Client Resource +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add a Client resource to the |bareosDir| configuration and configure it to access your NDMP storage system (Primary Storage System/:raw-latex:`\DataAgent`). + +- **Protocol**:sup:`Dir`:sub:`Client` must be either NDMPv2, NDMPv3 or NDMPv4. + +- **Port**:sup:`Dir`:sub:`Client` is set to the NDMP Port (usually 10000). + +- **Username**:sup:`Dir`:sub:`Client` and **Password**:sup:`Dir`:sub:`Client` are used for the authentication against the NDMP Storage System. + +- **Auth Type**:sup:`Dir`:sub:`Client` is either Cleartext or MD5. NDMP supports both. + +In our example we connect to a Isilon storage appliance emulator: + + + + +.. code-block:: sh + :caption: + + Client { + Name = ndmp-client + Address = isilon.example.com + Port = 10000 # Default port for NDMP + Protocol = NDMPv4 # Need to specify protocol before password as protocol determines password encoding used + Auth Type = Clear # Cleartext Authentication + Username = "ndmpadmin" # username of the NDMP user on the DATA AGENT e.g. storage box being backuped. + Password = "secret" # password of the NDMP user on the DATA AGENT e.g. storage box being backuped. + } + +Verify, that you can access your Primary Storage System via Bareos: + + + + +.. code-block:: sh + :caption: verify connection to NDMP Primary Storage System + + *status client=ndmp-client + + Data Agent isilon.example.com NDMPv4 + Host info + hostname isilonsim-1 + os_type Isilon OneFS + os_vers v7.1.1.5 + hostid 005056ad8483ba43cc55a711cd384506e3c1 + + Server info + vendor Isilon + product Isilon NDMP + revision 2.2 + auths (2) NDMP4_AUTH_TEXT NDMP4_AUTH_MD5 + + Connection types + addr_types (2) NDMP4_ADDR_TCP NDMP4_ADDR_LOCAL + + Backup type info of tar format + attrs 0x7fe + set FILESYSTEM=/ifs + set FILES= + set EXCLUDE= + set PER_DIRECTORY_MATCHING=N + set HIST=f + set DIRECT=N + set LEVEL=0 + set UPDATE=Y + set RECURSIVE=Y + set ENCODING=UTF-8 + set ENFORCE_UNIQUE_NODE=N + set PATHNAME_SEPARATOR=/ + set DMP_NAME= + set BASE_DATE=0 + set NDMP_UNICODE_FH=N + + Backup type info of dump format + attrs 0x7fe + set FILESYSTEM=/ifs + set FILES= + set EXCLUDE= + set PER_DIRECTORY_MATCHING=N + set HIST=f + set DIRECT=N + set LEVEL=0 + set UPDATE=Y + set RECURSIVE=Y + set ENCODING=UTF-8 + set ENFORCE_UNIQUE_NODE=N + set PATHNAME_SEPARATOR=/ + set DMP_NAME= + set BASE_DATE=0 + set NDMP_UNICODE_FH=N + + File system /ifs + physdev OneFS + unsupported 0x0 + type NFS + status + space 12182519808 total, 686768128 used, 11495751680 avail + inodes 17664000 total, 16997501 used + set MNTOPTS= + set MNTTIME=00:00:00 00:00:00 + +This output shows that the access to the storage appliance was successful. + +.. _section-ndmp-sd-configure: + +Bareos Storage Daemon: Configure NDMP +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \subsubsubsection{Enabling NDMP} + +To enable the NDMP :raw-latex:`\TapeAgent `inside of the |bareosSd|, set **NDMP Enable**:sup:`Sd`:sub:`Storage` =yes: + + + + +.. code-block:: sh + :caption: enable NDMP in \bareosSd + + # + # Default SD config block: enable the NDMP protocol, + # otherwise it won't listen on port 10000. + # + Storage { + Name = .... + ... + NDMP Enable = yes + } + +.. raw:: latex + + \subsubsubsection{Add a NDMP resource} + +Additionally, we need to define the access credentials for our NDMP TAPE AGENT (Secondary Storage) inside of this Storage Daemon. + +These are configured by adding a NDMP resource to :file:`bareos-sd.conf`: + + + + +.. code-block:: sh + :caption: + + # + # This resource gives the DMA in the Director access to the Bareos SD via the NDMP protocol. + # This option is used via the NDMP protocol to open the right TAPE AGENT connection to your + # Bareos SD via the NDMP protocol. The initialization of the SD is done via the native protocol + # and is handled via the PairedStorage keyword. + # + Ndmp { + Name = bareos-dir-isilon + Username = ndmpadmin + Password = test + AuthType = Clear + } + +Username and Password can be anything, but they will have to match the settings in the |bareosDir| NDMP Storage resource we configure next. + +Now restart the |bareosSd|. If everything is correct, the |bareosSd| starts and listens now on the usual port (9103) and additionally on port 10000 (ndmp). + + + + +.. code-block:: sh + :caption: + + netstat -lntp | grep bareos-sd + tcp 0 0 0.0.0.0:9103 0.0.0.0:* LISTEN 10661/bareos-sd + tcp 0 0 0.0.0.0:10000 0.0.0.0:* LISTEN 10661/bareos-sd + +Bareos Director: Configure a Paired Storage +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For NDMP Backups, we always need two storages that are paired together. The default configuration already has a Storage **File**:sup:`Dir`:sub:`Storage` defined: + + + + +.. code-block:: sh + :caption: + + Storage { + Name = File + Address = bareos + Password = "pNZ3TvFAL/t+MyOIQo58p5B/oB79SFncdAmLXKHa9U59" + Device = FileStorage + Media Type = File + } + +We now add a paired storage to the already existing **File**:sup:`Dir`:sub:`` storage: + + + + +.. code-block:: sh + :caption: + + # + # Same storage daemon but via NDMP protocol. + # We link via the PairedStorage config option the Bareos SD + # instance definition to a NDMP TAPE AGENT. + # + Storage { + Name = NDMPFile + Address = bareos + Port = 10000 + Protocol = NDMPv4 + Auth Type = Clear + Username = ndmpadmin + Password = "test" + Device = FileStorage + Media Type = File + PairedStorage = File + } + +The settings of Username and Password need to match the settings of the |bareosSd|’s NDMP resource we added before. The address will be used by the storage appliance’s NDMP Daemon to connect to the |bareosSd| via NDMP. Make sure that the Storage appliance can resolve the name or use an IP address. + +Now save the director resource and restart the |bareosDir|. Verify that the configuration is correct: + + + + +.. code-block:: sh + :caption: verify connection to the \bareosSd + + *status storage=NDMPFile + Connecting to Storage daemon File at bareos:9103 + + bareos-sd Version: 15.2.2 (16 November 2015) x86_64-redhat-linux-gnu redhat Red Hat Enterprise Linux Server release 7.0 (Maipo) + Daemon started 14-Jan-16 10:10. Jobs: run=0, running=0. + Heap: heap=135,168 smbytes=34,085 max_bytes=91,589 bufs=75 max_bufs=77 + Sizes: boffset_t=8 size_t=8 int32_t=4 int64_t=8 mode=0 bwlimit=0kB/s + + Running Jobs: + No Jobs running. + ==== + + Jobs waiting to reserve a drive: + ==== + + Terminated Jobs: + ==== + + Device status: + + Device "FileStorage" (/var/lib/bareos/storage) is not open. + == + ==== + + Used Volume status: + ==== + + ==== + + * + +The output looks the same, as if a :strong:`status storage=File` would have been called. + +.. _section-NdmpFileset: + +Bareos Director: Configure NDMP Fileset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To specify what files and directories from the storage appliance should be backed up, a Fileset needs to be specified. In our example, we decided to backup :file:`/ifs/home` directory. + +The specified directory needs to be a filesystem or a subdirectory of a filesystem which can be accessed by NDMP. Which filesystems are available is showed in the :strong:`status client` output of the NDMP client. + +.. index:: + pair: NDMP; Environment variables + Additionally, NDMP can be configured via NDMP environment variables. These can be specified in the Options Block of the Fileset with the :strong:`Meta` keyword. Which variables are available is partly depending on the NDMP implementation of the Storage Appliance. + + + + +.. code-block:: sh + :caption: NDMP Fileset + + Fileset { + Name = "NDMP Fileset" + Include { + Options { + meta = "BUTYPE=DUMP" + meta = "USE_TBB_IF_AVAILABLE=y" + meta = "FH_REPORT_FULL_DIRENTS=y" + meta = "RESTORE_HARDLINK_BY_TABLE=y" + } + File = /ifs/home + } + } + +.. raw:: latex + + +.. warning:: + Normally (**Protocol**:sup:`Dir`:sub:`Client` =Native) Filesets get handled by the \bareosFd. When connecting directly to a NDMP Clients (**Protocol**:sup:`Dir`:sub:`Client` =NDMP*), no \bareosFd is involved and therefore most Fileset options can't be used. Instead, parameters are handled via \configdirective{Options - Meta from **Include**:sup:`Dir`:sub:`FileSet` .} + +.. raw:: latex + + \warning{Avoid using multiple **Include**:sup:`Dir`:sub:`FileSet` \configdirective{File} directives. + The \bareosDir would try to handle them by running multiple NDMP jobs in a single Bareos job. + Even if this is working fine during backup, restore jobs will cause trouble.} + +Some NDMP environment variables are set automatically by the DMA in the |bareosDir|. The following environment variables are currently set automatically: + +FILESYSTEM + is set to the **Include**:sup:`Dir`:sub:`FileSet` :strong:`File` directive. + +HIST + | = Y + | Specifies the file history format: + + Y + Specifies the default file history format determined by your NDMP backup settings. + + N + Disables file history. Without file hostory, single file restore is not possible with Bareos. + + Some NDMP environments (eg. Isilon OneFS) allow additional parameter: + + F + Specifies path-based file history. This is the most efficient with Bareos. + + D + Specifies directory or node file history. + +LEVEL + is set accordingly to :ref:`section-NdmpBackupLevel`. + +PREFIX +TYPE + is set accordingly to BUTYPE. Default :emphasis:`DUMP`. + +UPDATE + = Y + +.. raw:: latex + + \subsubsubsection{Example NDMP Fileset to backup a subset of a NDMP filesystem} + +The following fileset is intended to backup all files and directories matching :file:`/ifs/home/users/a*`. It has been tested against Isilon OneFS 7.2.0.1. See :raw-latex:`\externalReferenceIsilonNdmpEnvironmentVariables `for details about the supported NDMP environment variables. Excludes are not used in this example. + + + + +.. code-block:: sh + :caption: NDMP Fileset Isilon Include/Exclude + + Fileset { + Name = "isilon_fileset_home_a" + Include { + Options { + meta = "BUTYPE=DUMP" + meta = "USE_TBB_IF_AVAILABLE=y" + + # + # EXCLUDE + # + #meta = "EXCLUDE=[b-z]*" + + # + # INCLUDE + # + meta = "FILES=a*" + } + File = /ifs/home/users + } + } + +Bareos Director: Configure NDMP Jobs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To do NDMP backups and restores, some special settings need to be configured. We define special Backup and Restore jobs for NDMP. + + + + +.. code-block:: sh + :caption: NDMP backup job + + Job { + Name = "ndmp-backup-job" + Type = Backup + Protocol = NDMP_BAREOS + Level = Incremental + Client = ndmp-client + Backup Format = dump + FileSet = "NDMP Fileset" + Storage = NDMPFile + Pool = Full + Messages = Standard + } + + + + +.. code-block:: sh + :caption: NDMP restore job + + Job { + Name = "ndmp-restore-job" + Type = Restore + Protocol = NDMP_BAREOS + Client = ndmp-client + Backup Format = dump + FileSet = "NDMP Fileset" + Storage = NDMPFile + Pool = Full + Messages = Standard + Where = / + } + +- **Backup Format**:sup:`Dir`:sub:`Job` =dump is used in our example. Other Backup Formats have other advantages/disadvantages. + +.. raw:: latex + + \centering + +.. figure:: \idir ndmp-cfg + :alt: NDMP configuration overview + + NDMP configuration overview + +Run NDMP Backup +--------------- + +Now we are ready to do our first NDMP backup: + + + + +.. code-block:: sh + :caption: run NDMP backup + + *run job=ndmp-backup-job + Using Catalog "MyCatalog" + Run Backup job + JobName: ndmp-backup-job + Level: Incremental + Client: ndmp-client + Format: dump + FileSet: NDMP Fileset + Pool: Full (From Job resource) + Storage: NDMPFile (From Job resource) + When: 2016-01-14 10:48:04 + Priority: 10 + OK to run? (yes/mod/no): yes + Job queued. JobId=1 + *wait jobid=1 + JobId=1 + JobStatus=OK (T) + *list joblog jobid=1 + 2016-01-14 10:57:53 bareos-dir JobId 1: Start NDMP Backup JobId 1, Job=NDMPJob.2016-01-14_10.57.51_04 + 2016-01-14 10:57:53 bareos-dir JobId 1: Created new Volume "Full-0001" in catalog. + 2016-01-14 10:57:53 bareos-dir JobId 1: Using Device "FileStorage" to write. + 2016-01-14 10:57:53 bareos-dir JobId 1: Opening tape drive LPDA-DEJC-ENJL-AHAI-JCBD-LICP-LKHL-IEDK@/ifs/home%0 read/write + 2016-01-14 10:57:53 bareos-sd JobId 1: Labeled new Volume "Full-0001" on device "FileStorage" (/var/lib/bareos/storage). + 2016-01-14 10:57:53 bareos-sd JobId 1: Wrote label to prelabeled Volume "Full-0001" on device "FileStorage" (/var/lib/bareos/storage) + 2016-01-14 10:57:53 bareos-dir JobId 1: Commanding tape drive to rewind + 2016-01-14 10:57:53 bareos-dir JobId 1: Waiting for operation to start + 2016-01-14 10:57:53 bareos-dir JobId 1: Async request NDMP4_LOG_MESSAGE + 2016-01-14 10:57:53 bareos-dir JobId 1: Operation started + 2016-01-14 10:57:53 bareos-dir JobId 1: Monitoring backup + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: 'Filetransfer: Transferred 5632 bytes in 0.087 seconds throughput of 63.133 KB/s' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: 'Filetransfer: Transferred 5632 total bytes ' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: 'CPU user=0.016416 sys=0.029437 ft=0.077296 cdb=0.000000' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: 'maxrss=14576 in=13 out=22 vol=155 inv=72' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: ' + Objects (scanned/included): + ---------------------------- + Regular Files: (1/1) + Sparse Files: (0/0) + Stub Files: (0/0) + Directories: (2/2) + ADS Entries: (0/0) + ADS Containers: (0/0) + Soft Links: (0/0) + Hard Links: (0/0) + Block Device: (0/0) + Char Device: (0/0) + FIFO: (0/0) + Socket: (0/0) + Whiteout: (0/0) + Unknown: (0/0)' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: ' + Dir Depth (count) + ---------------------------- + Total Dirs: 2 + Max Depth: 1 + + File Size (count) + ---------------------------- + == 0 0 + <= 8k 1 + <= 64k 0 + <= 1M 0 + <= 20M 0 + <= 100M 0 + <= 1G 0 + > 1G 0 + ------------------------- + Total Files: 1 + Total Bytes: 643 + Max Size: 643 + Mean Size: 643' + 2016-01-14 10:57:53 bareos-dir JobId 1: LOG_MESSAGE: ' + File History + ---------------------------- + Num FH_HIST_FILE messages: 0 + Num FH_HIST_DIR messages: 6 + Num FH_HIST_NODE messages: 3' + 2016-01-14 10:57:54 bareos-dir JobId 1: Async request NDMP4_NOTIFY_MOVER_HALTED + 2016-01-14 10:57:54 bareos-dir JobId 1: DATA: bytes 2053KB MOVER: written 2079KB record 33 + 2016-01-14 10:57:54 bareos-dir JobId 1: Operation done, cleaning up + 2016-01-14 10:57:54 bareos-dir JobId 1: Waiting for operation to halt + 2016-01-14 10:57:54 bareos-dir JobId 1: Commanding tape drive to NDMP9_MTIO_EOF 2 times + 2016-01-14 10:57:54 bareos-dir JobId 1: Commanding tape drive to rewind + 2016-01-14 10:57:54 bareos-dir JobId 1: Closing tape drive LPDA-DEJC-ENJL-AHAI-JCBD-LICP-LKHL-IEDK@/ifs/home%0 + 2016-01-14 10:57:54 bareos-dir JobId 1: Operation halted, stopping + 2016-01-14 10:57:54 bareos-dir JobId 1: Operation ended OKAY + 2016-01-14 10:57:54 bareos-sd JobId 1: Elapsed time=00:00:01, Transfer rate=2.128 M Bytes/second + 2016-01-14 10:57:54 bareos-dir JobId 1: Bareos bareos-dir 15.2.2 (16Nov15): + Build OS: x86_64-redhat-linux-gnu redhat Red Hat Enterprise Linux Server release 7.0 (Maipo) + JobId: 1 + Job: ndmp-backup-job.2016-01-14_10.57.51_04 + Backup Level: Full + Client: "ndmp-client" + FileSet: "NDMP Fileset" 2016-01-14 10:57:51 + Pool: "Full" (From Job resource) + Catalog: "MyCatalog" (From Client resource) + Storage: "NDMPFile" (From Job resource) + Scheduled time: 14-Jan-2016 10:57:51 + Start time: 14-Jan-2016 10:57:53 + End time: 14-Jan-2016 10:57:54 + Elapsed time: 1 sec + Priority: 10 + NDMP Files Written: 3 + SD Files Written: 1 + NDMP Bytes Written: 2,102,784 (2.102 MB) + SD Bytes Written: 2,128,987 (2.128 MB) + Rate: 2102.8 KB/s + Volume name(s): Full-0001 + Volume Session Id: 4 + Volume Session Time: 1452764858 + Last Volume Bytes: 2,131,177 (2.131 MB) + Termination: Backup OK + +We have successfully created our first NDMP backup. + +Let us have a look what files are in our backup: + + + + +.. code-block:: sh + :caption: list the files of the backup job + + *list files jobid=1 + /@NDMP/ifs/home%0 + /ifs/home/ + /ifs/home/admin/ + /ifs/home/admin/.zshrc + +The real backup data is stored in the file :file:`/@NDMP/ifs/home%0` (we will refer to it as :emphasis:`NDMP main backup file` or :emphasis:`main backup file` later on). One NDMP main backup file is created for every directory specified in the used Fileset. The other files show the file history and are hardlinks to the backup file. + +Run NDMP Restore +---------------- + +Now that we have a NDMP backup, we of course also want to restore some data from the backup. If the backup we just did saved the Filehistory, we are able to select single files for restore. Otherwise, we will only be able to restore the whole backup. + +Full Restore +~~~~~~~~~~~~ + +Either select all files or the main backup file (:file:`/@NDMP/ifs/home%0`). If file history is not included in the backup job, than only the main backup file is available. + +Restore files to original path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: + + *restore jobid=1 + You have selected the following JobId: 1 + + Building directory tree for JobId(s) 1 ... + 2 files inserted into the tree. + + You are now entering file selection mode where you add (mark) and + remove (unmark) files to be restored. No files are initially added, unless + you used the "all" keyword on the command line. + Enter "done" to leave this mode. + + cwd is: / + $ mark /ifs/home/admin/.zshrc + $ done + Bootstrap records written to /var/lib/bareos/bareos-dir.restore.1.bsr + + The job will require the following + Volume(s) Storage(s) SD Device(s) + =========================================================================== + + Full-0001 File FileStorage + + Volumes marked with "*" are online. + + + 1 file selected to be restored. + + The defined Restore Job resources are: + 1: RestoreFiles + 2: ndmp-restore-job + Select Restore Job (1-2): 2 + Defined Clients: + 1: bareos-fd + 2: ndmp-client + Select the Client (1-2): 2 + Run Restore job + JobName: ndmp-backup-job + Bootstrap: /var/lib/bareos/bareos-dir.restore.1.bsr + Where: / + Replace: Always + FileSet: NDMP Fileset + Backup Client: ndmp-client + Restore Client: ndmp-client + Format: dump + Storage: File + When: 2016-01-14 11:04:46 + Catalog: MyCatalog + Priority: 10 + Plugin Options: *None* + OK to run? (yes/mod/no): yes + Job queued. JobId=2 + *wait jobid=2 + JobId=2 + JobStatus=OK (T) + *list joblog jobid=2 + 14-Jan 11:04 bareos-dir JobId 2: Start Restore Job ndmp-backup-job.2016-01-14_11.04.53_05 + 14-Jan 11:04 bareos-dir JobId 2: Using Device "FileStorage" to read. + 14-Jan 11:04 bareos-dir JobId 2: Opening tape drive KKAE-IMLO-NHJD-GOCO-GJCO-GEHB-BODL-ADNG@/ifs/home read-only + 14-Jan 11:04 bareos-dir JobId 2: Commanding tape drive to rewind + 14-Jan 11:04 bareos-dir JobId 2: Waiting for operation to start + 14-Jan 11:04 bareos-sd JobId 2: Ready to read from volume "Full-0001" on device "FileStorage" (/var/lib/bareos/storage). + 14-Jan 11:04 bareos-sd JobId 2: Forward spacing Volume "Full-0001" to file:block 0:194. + 14-Jan 11:04 bareos-dir JobId 2: Async request NDMP4_LOG_MESSAGE + 14-Jan 11:04 bareos-dir JobId 2: Operation started + 14-Jan 11:04 bareos-dir JobId 2: Monitoring recover + 14-Jan 11:04 bareos-dir JobId 2: DATA: bytes 0KB MOVER: read 0KB record 0 + 14-Jan 11:04 bareos-dir JobId 2: LOG_MESSAGE: 'Filetransfer: Transferred 1048576 bytes in 0.135 seconds throughput of 7557.139 KB/s' + 14-Jan 11:04 bareos-dir JobId 2: OK: /admin/.zshrc + 14-Jan 11:04 bareos-dir JobId 2: LOG_MESSAGE: ' + Objects: + ---------------------------- + Regular Files: (1) + Stub Files: (0) + Directories: (0) + ADS Entries: (0) + Soft Links: (0) + Hard Links: (0) + Block Device: (0) + Char Device: (0) + FIFO: (0) + Socket: (0) + Unknown: (0)' + 14-Jan 11:04 bareos-dir JobId 2: LOG_MESSAGE: ' + File Size (count) + ---------------------------- + == 0 0 + <= 8k 1 + <= 64k 0 + <= 1M 0 + <= 20M 0 + <= 100M 0 + <= 1G 0 + > 1G 0 + ------------------------- + Total Files: 1 + Total Bytes: 643 + Max Size: 643 + Mean Size: 643' + 14-Jan 11:04 bareos-dir JobId 2: Async request NDMP4_NOTIFY_MOVER_PAUSED + 14-Jan 11:04 bareos-dir JobId 2: DATA: bytes 1024KB MOVER: read 2079KB record 32 + 14-Jan 11:04 bareos-dir JobId 2: Mover paused, reason=NDMP9_MOVER_PAUSE_EOF + 14-Jan 11:04 bareos-dir JobId 2: End of tapes + 14-Jan 11:04 bareos-dir JobId 2: DATA: bytes 1024KB MOVER: read 2079KB record 32 + 14-Jan 11:04 bareos-dir JobId 2: Operation done, cleaning up + 14-Jan 11:04 bareos-dir JobId 2: Waiting for operation to halt + 14-Jan 11:04 bareos-dir JobId 2: Commanding tape drive to rewind + 14-Jan 11:04 bareos-dir JobId 2: Closing tape drive KKAE-IMLO-NHJD-GOCO-GJCO-GEHB-BODL-ADNG@/ifs/home + 14-Jan 11:04 bareos-dir JobId 2: Operation halted, stopping + 14-Jan 11:04 bareos-dir JobId 2: Operation ended OKAY + 14-Jan 11:04 bareos-dir JobId 2: LOG_FILE messages: 1 OK, 0 ERROR, total 1 of 1 + 14-Jan 11:04 bareos-dir JobId 2: Bareos bareos-dir 15.2.2 (16Nov15): + Build OS: x86_64-redhat-linux-gnu redhat Red Hat Enterprise Linux Server release 7.0 (Maipo) + JobId: 2 + Job: ndmp-backup-job.2016-01-14_11.04.53_05 + Restore Client: ndmp-client + Start time: 14-Jan-2016 11:04:55 + End time: 14-Jan-2016 11:04:57 + Elapsed time: 2 secs + Files Expected: 1 + Files Restored: 1 + Bytes Restored: 1,048,576 + Rate: 524.3 KB/s + SD termination status: OK + Termination: Restore OK + +.. _section-ndmp-where: + +Restore files to different path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The restore location is determined by the **Where**:sup:`Dir`:sub:`Job` setting of the restore job. In NDMP, this parameter works in a special manner, the prefix can be either :emphasis:`relative` to the filesystem or :emphasis:`absolute`. If a prefix is set in form of a directory (like :file:`/bareos-restores`), it will be a relative prefix and will be added between the filesystem and the filename. This is needed to make sure that the +data is restored in a different directory, but into the same filesystem. If the prefix is set with a leading caret (^), it will be an absolute prefix and will be put at the front of the restore path. This is needed if the restored data should be stored into a different filesystem. + +Example: + ++---------------------------------+---------------------------------------+-------------------------------------------------+ +| original file name | where | restored file | ++=================================+=======================================+=================================================+ +| :file:`/ifs/home/admin/.zshrc`` | ``path:/bareos-restores`` | ``path:/ifs/home/bareos-restores/admin/.zshrc` | ++---------------------------------+---------------------------------------+-------------------------------------------------+ +| :file:`/ifs/home/admin/.zshrc`` | ^\ ``path:/ifs/data/bareos-restores`` | ``path:/ifs/data/bareos-restores/admin/.zshrc` | ++---------------------------------+---------------------------------------+-------------------------------------------------+ + +NDMP Copy Jobs +-------------- + +.. index:: + pair: Copy; NDMP +.. index:: + pair: NDMP; Copy jobs + + +To be able to do copy jobs, we need to have a second storage resource where we can copy the data to. Depending on your requirements, this resource can be added to the existing |bareosSd| (e.g. **autochanger-0**:sup:`Sd`:sub:`Storage` for tape based backups) or to an additional |bareosSd|. + +We set up an additional |bareosSd| on a host named :strong:`bareos-sd2.example.com` with the default **FileStorage**:sup:`Sd`:sub:`Storage` device. + +When this is done, add a second storage resource **File2**:sup:`Dir`:sub:`Storage` to the :file:`bareos-dir.conf`: + + + + +.. code-block:: sh + :caption: Storage resource File2 + + Storage { + Name = File2 + Address = bareos-sd2.example.com + Password = + Device = FileStorage + Media Type = File + } + +Copy Jobs copy data from one pool to another (see :ref:`MigrationChapter`). So we need to define a pool where the copies will be written to: + +Add a Pool that the copies will run to: + + + + +.. code-block:: sh + :caption: Pool resource Copy + + # + # Copy Destination Pool + # + Pool { + Name = Copy + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + AutoPrune = yes # Prune expired volumes + Volume Retention = 365 days # How long should the Full Backups be kept? (#06) + Maximum Volume Bytes = 50G # Limit Volume size to something reasonable + Maximum Volumes = 100 # Limit number of Volumes in Pool + Label Format = "Copy-" # Volumes will be labeled "Full-" + Storage = File2 # Pool belongs to Storage File2 + } + +Then we need to define the just defined pool as the **Next Pool**:sup:`Dir`:sub:`Pool` of the pool that actually holds the data to be copied. + +In our case this is the **Full**:sup:`Dir`:sub:`Pool` Pool: + + + + +.. code-block:: sh + :caption: add Next Pool setting + + # + # Full Pool definition + # + Pool { + Name = Full + [...] + Next Pool = Copy # <- this line needs to be added! + } + +Finally, we need to define a Copy Job that will select the jobs that are in the **Full**:sup:`Dir`:sub:`Pool` pool and copy them over to the **Copy**:sup:`Dir`:sub:`Pool` pool reading the data via the **File**:sup:`Dir`:sub:`Storage` Storage and writing the data via the **File2**:sup:`Dir`:sub:`Storage` Storage: + + + + +.. code-block:: sh + :caption: NDMP copy job + + Job { + Name = NDMPCopy + Type = Copy + Messages = Standard + Selection Type = PoolUncopiedJobs + Pool = Full + Storage = NDMPFile + } + +After restarting the director and storage daemon, we can run the Copy job: + + + + +.. code-block:: sh + :caption: run copy job + + *run job=NDMPCopy + Run Copy job + JobName: NDMPCopy + Bootstrap: *None* + Pool: Full (From Job resource) + NextPool: Copy (From unknown source) + Write Storage: File2 (From Storage from Run NextPool override) + JobId: *None* + When: 2016-01-21 09:19:49 + Catalog: MyCatalog + Priority: 10 + OK to run? (yes/mod/no): yes + Job queued. JobId=74 + *wait jobid=74 + JobId=74 + JobStatus=OK (T) + *list joblog jobid=74 + 21-Jan 09:19 bareos-dir JobId 74: The following 1 JobId was chosen to be copied: 73 + 21-Jan 09:19 bareos-dir JobId 74: Automatically selected Catalog: MyCatalog + 21-Jan 09:19 bareos-dir JobId 74: Using Catalog "MyCatalog" + 21-Jan 09:19 bareos-dir JobId 75: Copying using JobId=73 Job=NDMPJob.2016-01-21_09.18.50_49 + 21-Jan 09:19 bareos-dir JobId 75: Bootstrap records written to /var/lib/bareos/bareos-dir.restore.20.bsr + 21-Jan 09:19 bareos-dir JobId 74: Job queued. JobId=75 + 21-Jan 09:19 bareos-dir JobId 74: Copying JobId 75 started. + 21-Jan 09:19 bareos-dir JobId 74: Bareos bareos-dir 15.2.2 (16Nov15): + Build OS: x86_64-redhat-linux-gnu redhat Red Hat Enterprise Linux Server release 7.0 (Maipo) + Current JobId: 74 + Current Job: NDMPCopy.2016-01-21_09.19.50_50 + Catalog: "MyCatalog" (From Default catalog) + Start time: 21-Jan-2016 09:19:52 + End time: 21-Jan-2016 09:19:52 + Elapsed time: 0 secs + Priority: 10 + Termination: Copying -- OK + + 21-Jan 09:19 bareos-dir JobId 75: Start Copying JobId 75, Job=NDMPCopy.2016-01-21_09.19.52_51 + 21-Jan 09:19 bareos-dir JobId 75: Using Device "FileStorage" to read. + 21-Jan 09:19 bareos-dir JobId 76: Using Device "FileStorage2" to write. + 21-Jan 09:19 bareos-sd JobId 75: Ready to read from volume "Full-0001" on device "FileStorage" (/var/lib/bareos/storage). + 21-Jan 09:19 bareos-sd JobId 76: Volume "Copy-0004" previously written, moving to end of data. + 21-Jan 09:19 bareos-sd JobId 76: Ready to append to end of Volume "Copy-0004" size=78177310 + 21-Jan 09:19 bareos-sd JobId 75: Forward spacing Volume "Full-0001" to file:block 0:78177310. + 21-Jan 09:19 bareos-sd JobId 75: End of Volume at file 0 on device "FileStorage" (/var/lib/bareos/storage), Volume "Full-0001" + 21-Jan 09:19 bareos-sd JobId 75: End of all volumes. + 21-Jan 09:19 bareos-sd JobId 76: Elapsed time=00:00:01, Transfer rate=64.61 K Bytes/second + 21-Jan 09:19 bareos-dir JobId 75: Bareos bareos-dir 15.2.2 (16Nov15): + Build OS: x86_64-redhat-linux-gnu redhat Red Hat Enterprise Linux Server release 7.0 (Maipo) + Prev Backup JobId: 73 + Prev Backup Job: NDMPJob.2016-01-21_09.18.50_49 + New Backup JobId: 76 + Current JobId: 75 + Current Job: NDMPCopy.2016-01-21_09.19.52_51 + Backup Level: Incremental + Client: ndmp-client + FileSet: "NDMP Fileset" + Read Pool: "Full" (From Job resource) + Read Storage: "NDMPFile" (From Job resource) + Write Pool: "Copy" (From Job Pool's NextPool resource) + Write Storage: "File2" (From Storage from Pool's NextPool resource) + Next Pool: "Copy" (From Job Pool's NextPool resource) + Catalog: "MyCatalog" (From Default catalog) + Start time: 21-Jan-2016 09:19:54 + End time: 21-Jan-2016 09:19:54 + Elapsed time: 0 secs + Priority: 10 + SD Files Written: 1 + SD Bytes Written: 64,614 (64.61 KB) + Rate: 0.0 KB/s + Volume name(s): Copy-0004 + Volume Session Id: 43 + Volume Session Time: 1453307753 + Last Volume Bytes: 78,242,384 (78.24 MB) + SD Errors: 0 + SD termination status: OK + Termination: Copying OK + +Now we successfully copied over the NDMP job. + +.. raw:: latex + + \warning{\bcommand{list}{jobs} will only show the number of main backup files as JobFiles. However, with \bcommand{list}{files jobid=...} all files are visible.} + +Restore to NDMP Primary Storage System +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unfortunately, we are not able to restore the copied data to our NDMP storage. If we try we get this message: + + + + {} + 21-Jan 09:21 bareos-dir JobId 77: Fatal error: Read storage File2 doesn't point to storage definition with paired storage option. + +To be able to do NDMP operations from the storage that was used to store the copies, we need to define a NDMP storage that is paired with it. The definition is very similar to our **NDMPFile**:sup:`Dir`:sub:`Storage` Storage, as we want to restore the data to the same NDMP Storage system: + + + + +.. code-block:: sh + :caption: add paired Storage resource for File2 + + Storage { + Name = NDMPFile2 + Address = bareos-sd2.example.com + Port = 10000 + Protocol = NDMPv4 + Auth Type = Clear + Username = ndmpadmin + Password = "test" + Device = FileStorage2 + Media Type = File + PairedStorage = File2 + } + +Also we have to configure NDMP on the |bareosSd| :strong:`bareos-sd2.example.com`. For this follow the instruction from :ref:`section-ndmp-sd-configure`. + +After this, a restore from :strong:`bareos-sd2.example.com` directly to the NDMP Primary Storage System is possible. + +Limitations +----------- + +This list the specific limitiations of the NDMP_BAREOS protocol. For limitation for all Bareos NDMP implementation, see :ref:`section-NdmpCommonLimitations`. + +.. _section-ndmp-filehistory: + +NDMP Job limitations when scanning in volumes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + pair: NDMP; File History + + +For NDMP jobs, all data is stored into a single big file. The file and directory information (File History in NDMP Terms) is stored as hardlinks to this big file. + +.. raw:: latex + + \limitation*{NDMP}{File information are not available in the Bareos backup stream}{% + As hardlink information is only stored in the Bareos database, but not int the backup stream itself, it is not possible to recover the file history information from the NDMP stream with \command{bscan}. + + As storing the database dump for disaster recovery and storing the bootstrap file offsite is recommended anyway (see :ref:`section-before-disaster`), this should be not a big problem in correctly setup environments. + + For the same reason, the information about the number of files of a job (e.g. JobFiles with \bcommand{list}{jobs} command) is limited to the number of NDMP backup files in copied jobs. + } + +Restore always transfers the full main backup file to the Primary Storage System +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Contrary to :raw-latex:`\NdmpNative`, the :raw-latex:`\NdmpBareos `implementation do not support NDMP :emphasis:`Direct Access Restore` (DAR). + +On restore, the full main backup file (:file:`@NDMP/...%.`) is always transfered back to the Primary Storage System, together with a description, what files to restore. + +The reason for this is that the Primary Storage System handles the backup data by itself. Bareos will not modify the backup data it receives from the Primary Storage System. + +.. _section-NdmpNative: + +NDMP_NATIVE +=========== + +The NDMP_NATIVE protocol is implemented since Bareos 17.2.3. + +Bareos implements the :raw-latex:`\DataManagementAgent `inside of the |bareosDir| and is the only Bareos Daemon involved in the backups. + +When using NDMP_NATIVE, the :raw-latex:`\TapeAgent `must be provided by some other systems. Some storage vendors provide it with there storages, or offer it as an option, e.g. Isilon with there :emphasis:`Isilon Backup Accelerator`. + +Example Setup for NDMP_NATIVE backup +------------------------------------ + + +.. index:: + triple: NDMP; Example; NDMP\_NATIVE; + + +Configure a NDMP Client +~~~~~~~~~~~~~~~~~~~~~~~ + +This defines the connection to the NDMP Data Agent. + + + + +.. code-block:: sh + :caption: bareos-dir Client isilon + + Client { + Name = isilon + Address = isilon.example.com + Port = 10000 + Protocol = NDMPv4 + Auth Type = MD5 + Username = "ndmpadmin" + Password = "secret" + Maximum Concurrent Jobs = 1 + } + +Verify, that you can access your Primary Storage System (:raw-latex:`\TapeAgent`) via Bareos: + + + + +.. code-block:: sh + :caption: status ndmp client + + *status client=isilon + + Data Agent isilon.example.com NDMPv4 + Host info + hostname isilon + os_type Isilon OneFS + os_vers v7.2.1.4 + hostid xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + + Server info + vendor Isilon + product Isilon NDMP + revision 2.2.1 + auths (2) NDMP4_AUTH_TEXT NDMP4_AUTH_MD5 + + Connection types + addr_types (2) NDMP4_ADDR_TCP NDMP4_ADDR_LOCAL + + Backup type info of tar format + attrs 0x7fe + set FILESYSTEM=/ifs + set FILES= + set EXCLUDE= + set PER_DIRECTORY_MATCHING=N + set HIST=f + set DIRECT=N + set LEVEL=0 + set UPDATE=Y + set RECURSIVE=Y + set ENCODING=UTF-8 + set ENFORCE_UNIQUE_NODE=N + set PATHNAME_SEPARATOR=/ + set DMP_NAME= + set BASE_DATE=0 + set NDMP_UNICODE_FH=N + + Backup type info of dump format + attrs 0x7fe + set FILESYSTEM=/ifs + set FILES= + set EXCLUDE= + set PER_DIRECTORY_MATCHING=N + set HIST=f + set DIRECT=N + set LEVEL=0 + set UPDATE=Y + set RECURSIVE=Y + set ENCODING=UTF-8 + set ENFORCE_UNIQUE_NODE=N + set PATHNAME_SEPARATOR=/ + set DMP_NAME= + set BASE_DATE=0 + set NDMP_UNICODE_FH=N + + File system /ifs + physdev OneFS + unsupported 0x0 + type NFS + status + space 224681156345856 total, 126267678720 used, 224554888667136 avail + inodes 324102912000 total, 323964781836 used + set MNTOPTS= + set MNTTIME=00:00:00 00:00:00 + +Configure a NDMP Fileset +~~~~~~~~~~~~~~~~~~~~~~~~ + +This determines what filesystem to backup and configures the NDMP environment to use in the meta options for it. + + + + +.. code-block:: sh + :caption: bareos-dir Fileset isilon + + Fileset { + Name = "isilon" + Include { + Options { + meta = "HIST=F" + meta = "DIRECT=Y" + meta = "RECURSIVE=Y" + meta = "BUTYPE=DUMP" + } + File = /ifs/home + } + } + +The setting of :strong:`"DIRECT = Y"` is required for Direct Access Recovery. + +For more information, see :ref:`section-NdmpFileset`. + +Configure a NDMP Storage +~~~~~~~~~~~~~~~~~~~~~~~~ + +This defines now to connect to the Tape and Robot Agents and what devices to use. + +As we do not yet now the device names, we can put a placeholder string in **Device**:sup:`Dir`:sub:`Storage` and **NDMP Changer Device**:sup:`Dir`:sub:`Storage` : + + + + +.. code-block:: sh + :caption: bareos-dir Storage isilon + + Storage { + Name = isilon + Address = isilon.example.com + Port = 10000 + Protocol = NDMPv4 + Auth Type = MD5 + Username = "ndmpadmin" + Password = "secret" + Maximum Concurrent Jobs = 1 + Autochanger = yes + MediaType = NDMP-Tape + + Device = unknown # use "status storage" to determine the tape device + NDMP Changer Device = unknown # use "status storage" to determine the changer device + } + +Verify that the connection to the NDMP Tape Agent and Robot Agent work, by running the :strong:`status storage` command. + +The :raw-latex:`\TapeAgent `will return information about the available tape drives. The :raw-latex:`\RobotAgent `will return information about the available tape changer device. + + + + +.. code-block:: sh + :caption: status ndmp storage (Tape Agent and Robot Agent) + + *status storage=isilon + Tape Agent isilon.bareos.com NDMPv4 + Host info + hostname isilon + os_type Isilon OneFS + os_vers v7.2.1.4 + hostid abcdefg + + Server info + vendor Isilon + product Isilon NDMP + revision 2.2.1 + auths (2) NDMP4_AUTH_TEXT NDMP4_AUTH_MD5 + + Connection types + addr_types (2) NDMP4_ADDR_TCP NDMP4_ADDR_LOCAL + + tape HP Ultrium 5-SCSI I30Z + device HP-TLD-004-01 + attr 0x4 + set EXECUTE_CDB=t + set SERIAL_NUMBER=123456 + + tape HP Ultrium 5-SCSI I30Z + device HP-TLD-004-02 + attr 0x4 + set EXECUTE_CDB=t + set SERIAL_NUMBER=1234567 + + Robot Agent isilon.bareos.com NDMPv4 + Host info + hostname isilon + os_type Isilon OneFS + os_vers v7.2.1.4 + hostid 001517db7e38f40dbb4dfc0b823f29a31e09 + + Server info + vendor Isilon + product Isilon NDMP + revision 2.2.1 + auths (2) NDMP4_AUTH_TEXT NDMP4_AUTH_MD5 + + scsi QUANTUM Scalar i6000 605A + device mc001 + set SERIAL_NUMBER=VL002CX1252BVE01177 + +The interesting parts of the output is the device information both of the :raw-latex:`\TapeAgent `and :raw-latex:`\RobotAgent`. + +As each NDMP backup or recovery operation always involves exactly one tape and at one robot agent. + +We now know the device names and can configure what robot and what tape to use when this storage is used by bareos by updating the **isilon**:sup:`Sd`:sub:`Storage` resource: + + + + +.. code-block:: sh + :caption: bareos-dir Storage isilon + + Storage { + Name = isilon + Address = isilon.example.com + Port = 10000 + Protocol = NDMPv4 + Auth Type = MD5 + Username = "ndmpadmin" + Password = "secret" + Maximum Concurrent Jobs = 1 + Autochanger = yes + MediaType = NDMP-Tape + + Device = HP-TLD-004-01 + NDMP Changer Device = mc001 + } + +Configure a Pool for the NDMP Tapes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + +.. code-block:: sh + :caption: bareos-dir Pool NDMP-Tape + + Pool { + Name = NDMP-Tape + Pool Type = Backup + Recycle = yes # Bareos can automatically recycle Volumes + Auto Prune = yes # Prune expired volumes + Volume Retention = 365 days # How long should the Full Backups be kept? + } + +Configure NDMP Jobs +~~~~~~~~~~~~~~~~~~~ + +To be able to do scheduled backups, we need to configure a backup job that will use the NDMP client and NDMP storage resources: + + + + +.. code-block:: sh + :caption: bareos-dir Job ndmp-native-backup-job + + Job { + Name = ndmp-native-backup-job + type = backup + protocol = NDMP_NATIVE + level = incremental + client = isilon + storage = isilon + backup format = dump + messages = Standard + Pool = NDMP-Tape + save file history = yes + FileSet = isilon + } + +As we also need to be able to do a restore of the backuped data, we also need to define an adequate restore job: + + + + +.. code-block:: sh + :caption: bareos-dir Job ndmp-native-restore-job + + Job{ + Name = ndmp-restore + type = restore + protocol = NDMP_NATIVE + client = isilon + backup format = dump + fileset = isilon + storage = isilon + pool = NDMP-Tape + Messages = Standard + where = / + } + +Label Tapes +----------- + +Before we can really start do do backups, first we need to label the tapes that should be used. + +First we check if our robot has tapes with barcodes by running status slots: + + + + +.. code-block:: sh + :caption: status storage=isilon slots + + *status slots + Slot | Volume Name | Status | Media Type | Pool | + ------+------------------+-----------+----------------+--------------------------| + 1@| ? | ? | ? | ? | + 2@| ? | ? | ? | ? | + 3@| ? | ? | ? | ? | + 4@| ? | ? | ? | ? | + [...] + 251*| BT0001 | ? | ? | ? | + 252*| BT0002 | ? | ? | ? | + 253*| BT0003 | ? | ? | ? | + 254*| BT0004 | ? | ? | ? | + 255*| BT0005 | ? | ? | ? | + 256*| BT0006 | ? | ? | ? | + 257*| BT0007 | ? | ? | ? | + [...] + +Now we can label these tapes and add them to the pool that we have created for NDMP Tapes: + + + + +.. code-block:: sh + :caption: label barcodes + + *label storage=isilon barcodes slots=251-257 + Automatically selected Storage: isilon + Select Drive: + 1: Drive 0 + 2: Drive 1 + Select drive (1-12): 1 + get ndmp_vol_list... + The following Volumes will be labeled: + Slot Volume + ============== + 251 BT0001 + 252 BT0002 + 253 BT0003 + 254 BT0004 + 255 BT0005 + 256 BT0006 + 257 BT0007 + Do you want to label these Volumes? (yes|no): yes + Defined Pools: + 1: Scratch + 2: NDMP-Tape + 3: Incremental + 4: Full + 5: Differential + Select the Pool (1-5): 2 + ndmp_send_label_request: VolumeName=BT0001 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0001", Slot 251 successfully created. + ndmp_send_label_request: VolumeName=BT0002 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0002", Slot 252 successfully created. + ndmp_send_label_request: VolumeName=BT0003 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0003", Slot 253 successfully created. + ndmp_send_label_request: VolumeName=BT0004 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0004", Slot 254 successfully created. + ndmp_send_label_request: VolumeName=BT0005 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0005", Slot 255 successfully created. + ndmp_send_label_request: VolumeName=BT0006 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0006", Slot 256 successfully created. + ndmp_send_label_request: VolumeName=BT0007 MediaType=NDMP-Tape PoolName=NDMP-Tape drive=0 + Catalog record for Volume "BT0007", Slot 257 successfully created. + +We have now 7 volumes in our NDMP-Tape Pool that were labeled and can be used for NDMP Backups. + +Run NDMP_NATIVE Backup +---------------------- + + + + +.. code-block:: sh + :caption: run backup job + + *run job=ndmp-native-backup-job yes + JobId 1: Start NDMP Backup JobId 1, Job=ndmp.2017-04-07_01.40.31_10 + JobId 1: Using Data host isilon.bareos.com + JobId 1: Using Tape host:device isilon.bareos.com:HP-TLD-004-01 + JobId 1: Using Robot host:device isilon.bareos.com:mc001 + JobId 1: Using Tape record size 64512 + JobId 1: Found volume for append: BT0001 + JobId 1: Commanding robot to load slot @4146 into drive @256 + JobId 1: robot moving @4146 to @256 + JobId 1: robot move OK @4146 to @256 + JobId 1: Opening tape drive HP-TLD-004-01 read/write + JobId 1: Commanding tape drive to rewind + JobId 1: Checking tape label, expect 'BT0001' + JobId 1: Reading label + JobId 1: Commanding tape drive to rewind + JobId 1: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 1: Waiting for operation to start + JobId 1: Operation started + JobId 1: Monitoring backup + JobId 1: DATA: bytes 3703831KB MOVER: written 3703644KB record 58788 + JobId 1: LOG_MESSAGE: 'End of medium reached.' + JobId 1: DATA: bytes 4834614KB MOVER: written 4834053KB record 76731 + JobId 1: Mover paused, reason=NDMP9_MOVER_PAUSE_EOM + JobId 1: Operation requires next tape + JobId 1: At EOM, not writing filemarks + JobId 1: Commanding tape drive to rewind + JobId 1: Closing tape drive HP-TLD-004-01 + JobId 1: Commanding robot to unload drive @256 to slot @4146 + JobId 1: robot moving @256 to @4146 + JobId 1: robot move OK @256 to @4146 + JobId 1: Found volume for append: BT0002 + JobId 1: Commanding robot to load slot @4147 into drive @256 + JobId 1: robot moving @4147 to @256 + JobId 1: robot move OK @4147 to @256 + JobId 1: Opening tape drive HP-TLD-004-01 read/write + JobId 1: Commanding tape drive to rewind + JobId 1: Checking tape label, expect 'BT0002' + JobId 1: Reading label + JobId 1: Commanding tape drive to rewind + JobId 1: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 1: Operation resuming + JobId 1: DATA: bytes 6047457KB MOVER: written 6047244KB record 95988 + JobId 1: LOG_MESSAGE: 'End of medium reached.' + JobId 1: DATA: bytes 9668679KB MOVER: written 9668106KB record 153462 + JobId 1: Mover paused, reason=NDMP9_MOVER_PAUSE_EOM + JobId 1: Operation requires next tape + JobId 1: At EOM, not writing filemarks + JobId 1: Commanding tape drive to rewind + JobId 1: Closing tape drive HP-TLD-004-01 + JobId 1: Commanding robot to unload drive @256 to slot @4147 + JobId 1: robot moving @256 to @4147 + JobId 1: robot move OK @256 to @4147 + JobId 1: Found volume for append: BT0003 + JobId 1: Commanding robot to load slot @4148 into drive @256 + JobId 1: robot moving @4148 to @256 + JobId 1: robot move OK @4148 to @256 + JobId 1: Opening tape drive HP-TLD-004-01 read/write + JobId 1: Commanding tape drive to rewind + JobId 1: Checking tape label, expect 'BT0003' + JobId 1: Reading label + JobId 1: Commanding tape drive to rewind + JobId 1: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 1: Operation resuming + JobId 1: LOG_MESSAGE: 'Filetransfer: Transferred 10833593344 bytes in 87.187 seconds throughput of 121345.079 KB/s' + JobId 1: LOG_MESSAGE: 'Filetransfer: Transferred 10833593344 total bytes ' + JobId 1: LOG_MESSAGE: 'CPU user=0.528118 sys=54.575536 ft=87.182576 cdb=0.000000' + JobId 1: LOG_MESSAGE: 'maxrss=171972 in=1323908 out=17 vol=199273 inv=5883' + JobId 1: LOG_MESSAGE: ' + Objects (scanned/included): + ---------------------------- + Regular Files: (2765/2765) + Sparse Files: (0/0) + Stub Files: (0/0) + Directories: (447/447) + ADS Entries: (0/0) + ADS Containers: (0/0) + Soft Links: (0/0) + Hard Links: (0/0) + Block Device: (0/0) + Char Device: (0/0) + FIFO: (0/0) + Socket: (0/0) + Whiteout: (0/0) + Unknown: (0/0)' + JobId 1: LOG_MESSAGE: ' + Dir Depth (count) + ---------------------------- + Total Dirs: 447 + Max Depth: 10 + + File Size (count) + ---------------------------- + == 0 14 + <= 8k 1814 + <= 64k 658 + <= 1M 267 + <= 20M 10 + <= 100M 0 + <= 1G 0 + > 1G 2 + ------------------------- + Total Files: 2765 + Total Bytes: 10827843824 + Max Size: 5368709120 + Mean Size: 3916037' + JobId 1: LOG_MESSAGE: ' + File History + ---------------------------- + Num FH_HIST_FILE messages: 3212 + Num FH_HIST_DIR messages: 0 + Num FH_HIST_NODE messages: 0' + JobId 1: Async request NDMP4_NOTIFY_MOVER_HALTED + JobId 1: DATA: bytes 10581729KB MOVER: written 10581732KB record 167964 + JobId 1: Operation done, cleaning up + JobId 1: Waiting for operation to halt + JobId 1: Commanding tape drive to NDMP9_MTIO_EOF 2 times + JobId 1: Commanding tape drive to rewind + JobId 1: Closing tape drive HP-TLD-004-01 + JobId 1: Commanding robot to unload drive @256 to slot @4148 + JobId 1: robot moving @256 to @4148 + JobId 1: robot move OK @256 to @4148 + JobId 1: Operation halted, stopping + JobId 1: Operation ended OKAY + JobId 1: ERR-CONN NDMP4_CONNECT_CLOSE exchange-failed + JobId 1: media #1 BT0001+1/4834053K@4146 + JobId 1: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 1: media used=Y written=Y eof=N eom=Y io_error=N + JobId 1: label read=Y written=N io_error=N mismatch=N + JobId 1: fm_error=N nb_determined=Y nb_aligned=N + JobId 1: slot empty=N bad=N missing=N + JobId 1: media #2 BT0002+1/4834053K@4147 + JobId 1: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 1: media used=Y written=Y eof=N eom=Y io_error=N + JobId 1: label read=Y written=N io_error=N mismatch=N + JobId 1: fm_error=N nb_determined=Y nb_aligned=N + JobId 1: slot empty=N bad=N missing=N + JobId 1: media #3 BT0003+1/913626K@4148 + JobId 1: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 1: media used=Y written=Y eof=N eom=N io_error=N + JobId 1: label read=Y written=N io_error=N mismatch=N + JobId 1: fm_error=N nb_determined=Y nb_aligned=N + JobId 1: slot empty=N bad=N missing=N + JobId 1: Media: BT0001+1/4834053K@251 + JobId 1: Media: BT0002+1/4834053K@252 + JobId 1: Media: BT0003+1/913626K@253 + JobId 1: ndmp_fhdb_lmdb.c:675 Now processing lmdb database + JobId 1: ndmp_fhdb_lmdb.c:679 Processing lmdb database done + JobId 1: Bareos bareos-dir 17.2.3: + Build OS: x86_64-unknown-linux-gnu redhat Red Hat Enterprise Linux Server release 6.8 (Santiago) + JobId: 1 + Job: ndmp.2017-04-07_01.40.31_10 + Backup Level: Full + Client: "isilon" + FileSet: "isilon" 2017-04-07 01:40:31 + Pool: "NDMP-Tape" (From Job resource) + Catalog: "MyCatalog" (From Client resource) + Storage: "isilon" (From Job resource) + Scheduled time: 07-Apr-2017 01:40:31 + Start time: 07-Apr-2017 01:40:33 + End time: 07-Apr-2017 01:42:03 + Elapsed time: 1 min 30 secs + Priority: 10 + NDMP Files Written: 3,212 + NDMP Bytes Written: 10,835,690,496 (10.83 GB) + Rate: 120396.6 KB/s + Volume name(s): BT0001|BT0002|BT0003 + Volume Session Id: 0 + Volume Session Time: 0 + Last Volume Bytes: 935,553,024 (935.5 MB) + Termination: Backup OK + +Run NDMP_NATIVE Restore +----------------------- + +Now we want to restore some files from the backup we just did: + + + + +.. code-block:: sh + :caption: run ndmp restore job + + *restore + [...] + + cwd is: / + : mark /ifs/home/testdata/git/bareos/src/console/bconsole + 1 file marked. + : mark /ifs/home/testdatrandom5G-2 + 1 file marked. + $ done + Connecting to Director bareos:9101 + 1000 OK: bareos-dir Version: 17.2.3 + Enter a period to cancel a command. + list joblog jobid=2 + Automatically selected Catalog: MyCatalog + Using Catalog "MyCatalog" + JobId 2: Start Restore Job ndmp-restore.2017-04-07_01.48.23_13 + JobId 2: Namelist add: node:6033532893, info:5464882688, name:"/ifs/home/testdata/random5G-2" + JobId 2: Namelist add: node:6033077461, info:40076288, name:"/ifs/home/testdata/git/bareos/src/console/bconsole" + JobId 2: Record size is 64512 + JobId 2: Media: BT0001+1/4834053K@251 + JobId 2: Media: BT0002+1/4834053K@252 + JobId 2: Media: BT0003+1/913626K@253 + JobId 2: Logical slot for volume BT0001 is 251 + JobId 2: Physical(NDMP) slot for volume BT0001 is 4146 + JobId 2: Media Index of volume BT0001 is 1 + JobId 2: Logical slot for volume BT0002 is 252 + JobId 2: Physical(NDMP) slot for volume BT0002 is 4147 + JobId 2: Media Index of volume BT0002 is 2 + JobId 2: Logical slot for volume BT0003 is 253 + JobId 2: Physical(NDMP) slot for volume BT0003 is 4148 + JobId 2: Media Index of volume BT0003 is 3 + JobId 2: Commanding robot to load slot @4146 into drive @256 + JobId 2: robot moving @4146 to @256 + JobId 2: robot move OK @4146 to @256 + JobId 2: Opening tape drive HP-TLD-004-01 read-only + JobId 2: Commanding tape drive to rewind + JobId 2: Checking tape label, expect 'BT0001' + JobId 2: Reading label + JobId 2: Commanding tape drive to rewind + JobId 2: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 2: Waiting for operation to start + JobId 2: Operation started + JobId 2: Monitoring recover + JobId 2: DATA: bytes 0KB MOVER: read 0KB record 0 + JobId 2: DATA: bytes 11KB MOVER: read 11KB record 622 + JobId 2: Mover paused, reason=NDMP9_MOVER_PAUSE_SEEK + JobId 2: Operation requires a different tape + JobId 2: Commanding tape drive to rewind + JobId 2: Closing tape drive HP-TLD-004-01 + JobId 2: Commanding robot to unload drive @256 to slot @4146 + JobId 2: robot moving @256 to @4146 + JobId 2: robot move OK @256 to @4146 + JobId 2: Commanding robot to load slot @4147 into drive @256 + JobId 2: robot moving @4147 to @256 + JobId 2: robot move OK @4147 to @256 + JobId 2: Opening tape drive HP-TLD-004-01 read-only + JobId 2: Commanding tape drive to rewind + JobId 2: Checking tape label, expect 'BT0002' + JobId 2: Reading label + JobId 2: Commanding tape drive to rewind + JobId 2: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 2: Operation resuming + JobId 2: DATA: bytes 79884KB MOVER: read 79884KB record 85979 + JobId 2: DATA: bytes 201740KB MOVER: read 201740KB record 87914 + JobId 2: DATA: bytes 321548KB MOVER: read 321548KB record 89815 + JobId 2: DATA: bytes 440332KB MOVER: read 440332KB record 91701 + JobId 2: DATA: bytes 556044KB MOVER: read 556044KB record 93538 + JobId 2: DATA: bytes 674828KB MOVER: read 674828KB record 95423 + JobId 2: DATA: bytes 796684KB MOVER: read 796684KB record 97357 + JobId 2: DATA: bytes 915468KB MOVER: read 915468KB record 99243 + JobId 2: DATA: bytes 1036300KB MOVER: read 1036300KB record 101161 + JobId 2: DATA: bytes 1157132KB MOVER: read 1157132KB record 103079 + JobId 2: DATA: bytes 1277964KB MOVER: read 1277964KB record 104997 + JobId 2: DATA: bytes 1398796KB MOVER: read 1398796KB record 106915 + JobId 2: DATA: bytes 1518604KB MOVER: read 1518604KB record 108816 + JobId 2: DATA: bytes 1622028KB MOVER: read 1622028KB record 110458 + JobId 2: DATA: bytes 1741836KB MOVER: read 1741836KB record 112360 + JobId 2: DATA: bytes 1859596KB MOVER: read 1859596KB record 114229 + JobId 2: DATA: bytes 1981452KB MOVER: read 1981452KB record 116163 + JobId 2: DATA: bytes 2094092KB MOVER: read 2094092KB record 117951 + JobId 2: DATA: bytes 2207756KB MOVER: read 2207756KB record 119755 + JobId 2: DATA: bytes 2328588KB MOVER: read 2328588KB record 121673 + JobId 2: DATA: bytes 2448396KB MOVER: read 2448396KB record 123575 + JobId 2: DATA: bytes 2569228KB MOVER: read 2569228KB record 125493 + JobId 2: DATA: bytes 2689036KB MOVER: read 2689036KB record 127395 + JobId 2: DATA: bytes 2810892KB MOVER: read 2810892KB record 129329 + JobId 2: DATA: bytes 2926604KB MOVER: read 2926604KB record 131165 + JobId 2: DATA: bytes 3043340KB MOVER: read 3043340KB record 133018 + JobId 2: DATA: bytes 3163148KB MOVER: read 3163148KB record 134920 + JobId 2: DATA: bytes 3279884KB MOVER: read 3279884KB record 136773 + JobId 2: DATA: bytes 3400716KB MOVER: read 3400716KB record 138691 + JobId 2: DATA: bytes 3518476KB MOVER: read 3518476KB record 140560 + JobId 2: DATA: bytes 3636236KB MOVER: read 3636236KB record 142429 + JobId 2: DATA: bytes 3757068KB MOVER: read 3757068KB record 144347 + JobId 2: DATA: bytes 3877900KB MOVER: read 3877900KB record 146265 + JobId 2: DATA: bytes 3994636KB MOVER: read 3994636KB record 148118 + JobId 2: DATA: bytes 4116492KB MOVER: read 4116492KB record 150053 + JobId 2: DATA: bytes 4237324KB MOVER: read 4237324KB record 151971 + JobId 2: DATA: bytes 4331317KB MOVER: read 4331317KB record 153462 + JobId 2: Mover paused, reason=NDMP9_MOVER_PAUSE_SEEK + JobId 2: Operation requires a different tape + JobId 2: Commanding tape drive to rewind + JobId 2: Closing tape drive HP-TLD-004-01 + JobId 2: Commanding robot to unload drive @256 to slot @4147 + JobId 2: robot moving @256 to @4147 + JobId 2: robot move OK @256 to @4147 + JobId 2: Commanding robot to load slot @4148 into drive @256 + JobId 2: robot moving @4148 to @256 + JobId 2: robot move OK @4148 to @256 + JobId 2: Opening tape drive HP-TLD-004-01 read-only + JobId 2: Commanding tape drive to rewind + JobId 2: Checking tape label, expect 'BT0003' + JobId 2: Reading label + JobId 2: Commanding tape drive to rewind + JobId 2: Commanding tape drive to NDMP9_MTIO_FSF 1 times + JobId 2: Operation resuming + JobId 2: DATA: bytes 4424716KB MOVER: read 4424716KB record 154945 + JobId 2: DATA: bytes 4544524KB MOVER: read 4544524KB record 156847 + JobId 2: DATA: bytes 4663308KB MOVER: read 4663308KB record 158732 + JobId 2: DATA: bytes 4781068KB MOVER: read 4781068KB record 160601 + JobId 2: DATA: bytes 4902924KB MOVER: read 4902924KB record 162536 + JobId 2: DATA: bytes 5022732KB MOVER: read 5022732KB record 164437 + JobId 2: DATA: bytes 5138444KB MOVER: read 5138444KB record 166274 + JobId 2: OK: /testdata/git/bareos/src/console/bconsole + JobId 2: OK: /testdata/random5G-2 + JobId 2: LOG_MESSAGE: 'Filetransfer: Transferred 5368721181 bytes in 223.436 seconds throughput of 23464.803 KB/s' + JobId 2: LOG_MESSAGE: ' + Objects: + ---------------------------- + Regular Files: (2) + Stub Files: (0) + Directories: (0) + ADS Entries: (0) + Soft Links: (0) + Hard Links: (0) + Block Device: (0) + Char Device: (0) + FIFO: (0) + Socket: (0) + Unknown: (0)' + JobId 2: LOG_MESSAGE: ' + File Size (count) + ---------------------------- + == 0 0 + <= 8k 1 + <= 64k 0 + <= 1M 0 + <= 20M 0 + <= 100M 0 + <= 1G 0 + > 1G 1 + ------------------------- + Total Files: 2 + Total Bytes: 5368716925 + Max Size: 5368709120 + Mean Size: 2684358462' + JobId 2: Async request NDMP4_NOTIFY_MOVER_HALTED + JobId 2: DATA: bytes 5242893KB MOVER: read 5242893KB record 167932 + JobId 2: Operation done, cleaning up + JobId 2: Waiting for operation to halt + JobId 2: Commanding tape drive to rewind + JobId 2: Closing tape drive HP-TLD-004-01 + JobId 2: Commanding robot to unload drive @256 to slot @4148 + JobId 2: robot moving @256 to @4148 + JobId 2: robot move OK @256 to @4148 + JobId 2: Operation halted, stopping + JobId 2: Operation ended OKAY + JobId 2: ERR-CONN NDMP4_CONNECT_CLOSE exchange-failed + JobId 2: LOG_FILE messages: 2 OK, 0 ERROR, total 2 of 2 + JobId 2: media #1 BT0001+1/4834053K@4146 + JobId 2: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 2: media used=Y written=N eof=N eom=N io_error=N + JobId 2: label read=Y written=N io_error=N mismatch=N + JobId 2: fm_error=N nb_determined=N nb_aligned=N + JobId 2: slot empty=N bad=N missing=N + JobId 2: media #2 BT0002+1/4834053K@4147 + JobId 2: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 2: media used=Y written=N eof=N eom=N io_error=N + JobId 2: label read=Y written=N io_error=N mismatch=N + JobId 2: fm_error=N nb_determined=N nb_aligned=N + JobId 2: slot empty=N bad=N missing=N + JobId 2: media #3 BT0003+1/911610K@4148 + JobId 2: valid label=Y filemark=Y n_bytes=Y slot=Y + JobId 2: media used=Y written=N eof=N eom=N io_error=N + JobId 2: label read=Y written=N io_error=N mismatch=N + JobId 2: fm_error=N nb_determined=Y nb_aligned=N + JobId 2: slot empty=N bad=N missing=N + JobId 2: Bareos bareos-dir 17.2.3: + Build OS: x86_64-unknown-linux-gnu redhat Red Hat Enterprise Linux Server release 6.8 (Santiago) + JobId: 2 + Job: ndmp-restore.2017-04-07_01.48.23_13 + Restore Client: isilon + Start time: 07-Apr-2017 01:48:25 + End time: 07-Apr-2017 01:52:11 + Elapsed time: 3 mins 46 secs + Files Expected: 2 + Files Restored: 1 + Bytes Restored: 5,368,722,944 + Rate: 23755.4 KB/s + +.. _limitations-1: + +Limitations +----------- + +.. raw:: latex + + \limitation*{NDMP\_NATIVE}{Only use the first tape drive will be used}{% + A NDMP job only uses a single tape drive. Currently, a Bareos job always uses the first defined tape drive of the \TapeAgent.} + +NDMP Common +=========== + +This section contains additional information about the Bareos NDMP implementation that are valid for all Bareos NDMP protocols. + +.. _section-NdmpBackupLevel: + +NDMP Backup Level +----------------- + +.. index:: + pair: NDMP; Level + + +The trailing number in the main backup file (after the :file:`%` character) indicates the NDMP backup level: + ++-------+-------------------------------------------+ +| Level | Description | ++=======+===========================================+ +| 0 | Full NDMP backup. | ++-------+-------------------------------------------+ +| 1 | Differential or first Incremental backup. | ++-------+-------------------------------------------+ +| 2-9 | second to ninth Incremental backup. | ++-------+-------------------------------------------+ + +Differential Backups +^^^^^^^^^^^^^^^^^^^^ + +are supported. The NDMP backup level will be 1, visible as trailing number in the backup file (:file:`/@NDMP/ifs/home%1`). + +Incremental Backups +^^^^^^^^^^^^^^^^^^^ + +are supported. The NDMP backup level will increment with each run, until a Full (0) or Differential (1) will be made. The maximum backup level will be 9. Additional Incremental backups will result in a failed job and the message: + + + + {} + 2016-01-21 13:35:51 bareos-dir JobId 12: Fatal error: NDMP dump format doesn't support more than 8 incrementals, please run a Differential or a Full Backup + +NDMP Debugging +-------------- + +To debug the NDMP backups, these settings can be adapted: + +- + + + + **NDMP Snooping**:sup:`Dir`:sub:`Director` + +- + + + + **NDMP Log Level**:sup:`Dir`:sub:`Director` + +- + + + + **NDMP Log Level**:sup:`Dir`:sub:`Client` + +- + + + + **NDMP Snooping**:sup:`Sd`:sub:`Storage` + +- + + + + **NDMP Log Level**:sup:`Sd`:sub:`Storage` + +This will create a lot of debugging output that will help to find the problem during NDMP backups. + +.. _section-NdmpCommonLimitations: + +Bareos NDMP Common Limitations +------------------------------ + +NDMP Fileset limitations +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \limitation*{NDMP}{A NDMP fileset should only contain a single File directive and Meta options}{% + Using multiple **Include**:sup:`Dir`:sub:`FileSet` \configdirective{File} directives should be avoided. + The \bareosDir would try to handle them by running multiple NDMP jobs in a single Bareos job. + Even if this is working fine during backup, restore jobs will cause trouble. + + Normally (**Protocol**:sup:`Dir`:sub:`Client` =Native) Filesets get handled by the \bareosFd. When connecting directly to a NDMP Clients (**Protocol**:sup:`Dir`:sub:`Client` =NDMP*), no \bareosFd is involved and therefore most Fileset options can't be used. Instead, parameters are handled via \configdirective{Options - Meta} from **Include**:sup:`Dir`:sub:`FileSet` . + } + +Single file restore on incremental backups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \limitation*{NDMP}{No single file restore on merged backups}{% + Unfortunately, it is currently (bareos-15.2.2) not possible to restore a chain of Full and Incremental backups at once. + The workaround for that problem is to restore the full backup and each incremental each in a single restore operation. + } + +Temporary memory mapped database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw:: latex + + \limitation*{NDMP}{64-bit system recommended}{% + The \bareosDir uses a memory mapped database (LMBD) to temporarily store NDMP file information. + On some 32-bit systems the default **File History Size**:sup:`Dir`:sub:`Job` requires a larger memory area than available. + In this case, you either have to lower the **File History Size**:sup:`Dir`:sub:`Job` + or preferably run the \bareosDir on a 64-bit system. + } + +Tested Environments +------------------- + +Bareos NDMP support have been tested against: + ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ +| Vendor | Product | NDMP Subsystem | Bareos version | :raw-latex:`\TapeAgent ` | Features | Remarks | ++============+===============================+======================+================+==============================+==========+====================================+ +| Isilon | Isilon OneFS v7.2.1.4 | Isilon NDMP 2.2.1 | bareos-17.2.3 | Isilon Backup Accelerator | | Protocol: :raw-latex:`\NdmpNative` | ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ +| Isilon | Isilon OneFS v7.2.0.1 | Isilon NDMP 2.2 | bareos-16.2.6 | |bareosSd| | | | ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ +| Isilon | Isilon OneFS v7.1.1.5 | Isilon NDMP 2.2 | bareos-15.2.2 | |bareosSd| | | | ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ +| NetApp | | Release 8.2.3 7-Mode | bareos-15.2.2 | |bareosSd| | | | ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ +| Oracle/Sun | ZFS Storage Appliance, OS 8.3 | | bareos-15.2.2 | |bareosSd| | | | ++------------+-------------------------------+----------------------+----------------+------------------------------+----------+------------------------------------+ diff --git a/docs/manuals/en/new_main_reference/source/chapter32/catmaintenance.rst b/docs/manuals/en/new_main_reference/source/chapter32/catmaintenance.rst new file mode 100644 index 00000000000..0cc467c971e --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter32/catmaintenance.rst @@ -0,0 +1,1513 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _CatMaintenanceChapter: + +Catalog Maintenance +=================== + +.. index:: + single: Catalog Maintenance + + +Catalog Database +---------------- + +Bareos stores its catalog in a database. Different database backends are offered: + +- PostgreSQL (recommended) + +- MariaDB/MySQL (see :ref:`section-MysqlSupport`) + +- Sqlite (only for testing) + +What database will be used, can be configured in the |bareosDir| configuration, see the :ref:`DirectorResourceCatalog`. + +The database often runs on the same server as the |bareosDir|. However, it is also possible to run it on a different system. This might require some more manual configuration, a PostgreSQL example can be found in :ref:`catalog-maintenance-remote-psql`. + +dbconfig-common (Debian) +~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Platform; Debian; dbconfig-common; + :raw-latex:`\index[general]{Platform!Ubuntu!dbconfig-common}` + +.. _`section-dbconfig`: section-dbconfig + +Since Bareos 14.2.0 the Debian (and Ubuntu) based packages support the **dbconfig-common** mechanism to create and update the Bareos database, according to the user choices. + +The first choice is, if **dbconfig-common** should be used at all. If you decide against it, the database must be configured manually, see :ref:`CatMaintenanceManualConfiguration`. + +If you decided to use **dbconfig-common**, the next question will only be asked, if more than one Bareos database backend (**bareos-database-***) is installed. If this is the case, select the database backend you want to use. + +|image| |image| + +Depending on the selected database backend, more questions about how to access the database will be asked. Often, the default values are suitable. + +The **dbconfig-common** configuration (and credentials) is done by the **bareos-database-common** package. Settings are stored in the file :file:`/etc/dbconfig-common/bareos-database-common.conf`. + +The Bareos database backend will get automatically configured in :file:`/etc/bareos/bareos-dir.d/catalog/MyCatalog.conf`. If the Server is not running locally you need to specify **DB Address**:sup:`Dir`:sub:`Catalog` in the catalog ressource. A later reconfiguration might require manual adapt changes. + +.. raw:: latex + + +.. warning:: + When using the PostgreSQL backend and updating to Bareos $<$ 14.2.3, it is necessary to manually grant database permissions (\command{grant_bareos_privileges), normally by} + + + + +.. code-block:: sh + :caption: + + su - postgres -c /usr/lib/bareos/scripts/grant_bareos_privileges + +For details see chapter :ref:`CatMaintenanceManualConfiguration`. + +.. _CatMaintenanceManualConfiguration: + +Manual Configuration +~~~~~~~~~~~~~~~~~~~~ + +Bareos comes with a number of scripts to prepare and update the databases. All these scripts are located in the Bareos script directory, normally at :file:`/usr/lib/bareos/scripts/`. + ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| **Script** | **Stage** | **Description** | ++===============================================+================+=====================================================+ +| :file:`create_bareos_database` | installation | create Bareos database | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`make_bareos_tables` | installation | create Bareos tables | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`grant_bareos_privileges` | installation | grant database access privileges | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`update_bareos_tables` [-f] | update | update the database schema | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`drop_bareos_tables` | deinstallation | remove Bareos database tables | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`drop_bareos_database` | deinstallation | remove Bareos database | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`make_catalog_backup.pl` | backup | backup the Bareos database, default on Linux | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`make_catalog_backup` | backup | backup the Bareos database for systems without Perl | ++-----------------------------------------------+----------------+-----------------------------------------------------+ +| :file:`delete_catalog_backup` | backup helper | remove the temporary Bareos database backup file | ++-----------------------------------------------+----------------+-----------------------------------------------------+ + +The database preparation scripts have following configuration options: + +db_type + - command line parameter $1 + + - **DB Driver**:sup:`Dir`:sub:`Catalog` from the configuration + + - installed database backends + + - fallback: postgresql + +db_name + - environment variable :raw-latex:`\variable{db_name}` + + - **DB Name**:sup:`Dir`:sub:`Catalog` from the configuration + + - default: bareos + +db_user + - environment variable :raw-latex:`\variable{db_user}` + + - **DB User**:sup:`Dir`:sub:`Catalog` from the configuration + + - default: bareos + +db_password + - environment variable :raw-latex:`\variable{db_password}` + + - **DB Password**:sup:`Dir`:sub:`Catalog` from the configuration + + - default: *none* + +Reading the settings from the configuration require read permission for the current user. The normal PostgreSQL administrator user (**postgres}`) don’t have these permissions. So if you plan to use non-default database settings, you might add the user :raw-latex:`**postgres**` to the group :raw-latex:`\group{bareos**. + +The database preparation scripts need to have password-less administrator access to the database. Depending on the distribution you’re using, this require additional configuration. See the following section about howto achieve this for the different database systems. + +To view and test the currently configured settings, use following commands: + + + + +.. code-block:: sh + :caption: Show current database configuration + + /usr/sbin/bareos-dbcheck -B + catalog=MyCatalog + db_name=bareos + db_driver=mysql + db_user=bareos + db_password=YourPassword + db_address= + db_port=0 + db_socket= + db_type=MySQL + working_dir=/var/lib/bareos + + + + +.. code-block:: sh + :caption: Test the database connection. Example: wrong password + + /usr/sbin/bareos-dir -t -f -d 500 + [...] + bareos-dir: mysql.c:204-0 Error 1045 (28000): Access denied for user 'bareos'@'localhost' (using password: YES) + bareos-dir: dird.c:1114-0 Could not open Catalog "MyCatalog", database "bareos". + bareos-dir: dird.c:1119-0 mysql.c:200 Unable to connect to MySQL server. + Database=bareos User=bareos + MySQL connect failed either server not running or your authorization is incorrect. + bareos-dir: mysql.c:239-0 closedb ref=0 connected=0 db=0 + 25-Apr 16:25 bareos-dir ERROR TERMINATION + Please correct the configuration in /etc/bareos/bareos-dir.d/*/*.conf + +PostgreSQL +^^^^^^^^^^ + +On most distributions, PostgreSQL uses ident to allow access to the database system. The database administrator account is the Unix user **postgres**. Normally, this user can access the database without password, as the ident mechanism is used to identify the user. + +If this works on your system can be verified by + + + + +.. code-block:: sh + :caption: Access the local PostgreSQL database + + su - postgres + psql + +If your database is configured to require a password, this must be definied in the file `:file:`~/.pgpass` `_ in the following syntax: :option:`HOST:PORT:DATABASE:USER:PASSWORD`, e.g. + + + + +.. code-block:: sh + :caption: PostgreSQL access credentials + + localhost:*:bareos:bareos:secret + +The permission of this file must be 0600 (:program:`chmod 0600 ~/.pgpass`). + +Again, verify that you have specified the correct settings by calling the :program:`psql` command. If this connects you to the database, your credentials are good. Exit the PostgreSQL client and run the Bareos database preparation scripts: + + + + +.. code-block:: sh + :caption: Setup Bareos catalog database + + su - postgres + /usr/lib/bareos/scripts/create_bareos_database + /usr/lib/bareos/scripts/make_bareos_tables + /usr/lib/bareos/scripts/grant_bareos_privileges + +The encoding of the bareos database must be :option:`SQL_ASCII`. The command :program:`create_bareos_database` automatically creates the database with this encoding. This can be verified by the command :program:`psql -l`, which shows information about existing databases: + + + + +.. code-block:: sh + :caption: List existing databases + + psql -l + List of databases + Name | Owner | Encoding + -----------+----------+----------- + bareos | postgres | SQL_ASCII + postgres | postgres | UTF8 + template0 | postgres | UTF8 + template1 | postgres | UTF8 + (4 rows) + +The owner of the database may vary. The Bareos database maintance scripts don’t change the default owner of the Bareos database, so it stays at the PostgreSQL administration user. The :program:`grant_bareos_privileges` script grant the required permissions to the Bareos database user. In contrast, when installing (not updating) using :ref:`dbconfig `, the database owner will be identical with the Bareos database user. + +By default, using PostgreSQL ident, a Unix user can access a database of the same name. Therefore the user **bareos** can access the database :file:`bareos`. + + + + +.. code-block:: sh + :caption: Verify Bareos database on PostgreSQL as Unix user bareos (bareos-13.2.3) + + root@linux:~# su - bareos -s /bin/sh + bareos@linux:~# psql + Welcome to psql 8.3.23, the PostgreSQL interactive terminal. + + Type: \copyright for distribution terms + \h for help with SQL commands + \? for help with psql commands + \g or terminate with semicolon to execute query + \q to quit + + bareos=> \dt + List of relations + Schema | Name | Type | Owner + --------+------------------------+-------+---------- + public | basefiles | table | postgres + public | cdimages | table | postgres + public | client | table | postgres + public | counters | table | postgres + public | device | table | postgres + public | devicestats | table | postgres + public | file | table | postgres + public | filename | table | postgres + public | fileset | table | postgres + public | job | table | postgres + public | jobhisto | table | postgres + public | jobmedia | table | postgres + public | jobstats | table | postgres + public | location | table | postgres + public | locationlog | table | postgres + public | log | table | postgres + public | media | table | postgres + public | mediatype | table | postgres + public | ndmpjobenvironment | table | postgres + public | ndmplevelmap | table | postgres + public | path | table | postgres + public | pathhierarchy | table | postgres + public | pathvisibility | table | postgres + public | pool | table | postgres + public | quota | table | postgres + public | restoreobject | table | postgres + public | status | table | postgres + public | storage | table | postgres + public | unsavedfiles | table | postgres + public | version | table | postgres + (30 rows) + + bareos=> select * from Version; + versionid + ----------- + 2002 + (1 row) + + bareos=> \du + List of roles + Role name | Superuser | Create role | Create DB | Connections | Member of + ---------------+-----------+-------------+-----------+-------------+----------- + bareos | no | no | no | no limit | {} + postgres | yes | yes | yes | no limit | {} + (2 rows) + + bareos=> \dp + Access privileges for database "bareos" + Schema | Name | Type | Access privileges + --------+-----------------------------------+----------+-------------------------------------- + public | basefiles | table | {root=arwdxt/root,bareos=arwdxt/root} + public | basefiles_baseid_seq | sequence | {root=rwU/root,bareos=rw/root} + ... + + bareos=> + +.. _catalog-maintenance-remote-psql: + +Remote PostgreSQL Database +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When configuring bareos with a remote database, your first step is to check the connection from the |bareosDir| host into the database. A functional connection can be verified by + + + + +.. code-block:: sh + :caption: Access the remote PostgreSQL database + + su - postgres + psql --host bareos-database.example.com + +With a correct configuration you can access the database, if it fails you need to correct the PostgreSQL servers configuration files. + +One way to manually create the database would be calling the bareos database preparation scripts with the :option:`--host` option, explained later. How ever, it is advised to use the **dbconfig-common**. Both methods require you to add the database hostname/address as **DB Address**:sup:`Dir`:sub:`Catalog` . + +If you’re using **dbconfig-common** you should choose :option:`New Host`, enter the hostname or the local address followed by the password. As **dbconfig-common** uses the :option:`ident` authentication by default the first try to connect will fail. Don’t be bothered by that. Choose :option:`Retry` when prompted. From there, read carefully and configure the database to your needs. The authentication should be set +to password, as the ident method will not work with a remote server. Set the user and administrator according to your PostgreSQL servers settings. + +Set the PostgreSQL server IP as **DB Address**:sup:`Dir`:sub:`Catalog` in :ref:`DirectorResourceCatalog`. You can also customize other parameters or use the defaults. A quick check should display your recent changes: + + + + +.. code-block:: sh + :caption: Show current database configuration + + /usr/sbin/bareos-dbcheck -B + catalog=MyCatalog + db_name=bareos + db_driver=postgresql + db_user=bareos + db_password=secret + db_address=bareos-database.example.com + db_port=0 + db_socket= + db_type=PostgreSQL + working_dir=/var/lib/bareos + +If **dbconfig-common** did not succeed or you choosed not to use it, run the Bareos database preparation scripts with: + + + + +.. code-block:: sh + :caption: Setup Bareos catalog database + + su - postgres + /usr/lib/bareos/scripts/create_bareos_database --host=bareos-database.example.com + /usr/lib/bareos/scripts/make_bareos_tables --host=bareos-database.example.com + /usr/lib/bareos/scripts/grant_bareos_privileges --host=bareos-database.example.com + +.. _catalog-maintenance-mysql: + +MySQL +^^^^^ + +MySQL user authentication is username, password and host-based. The database administrator is the user **root**. + +On some distributions access to the MySQL database is allowed password-less as database user **root**, on other distributions, a password is required. On productive systems you normally want to have password secured access. + +The bareos database preparation scripts require password-less access to the database. To guarantee this, create a MySQL credentials file `:file:`~/.my.cnf` `_ with the credentials of the database administrator: + + + + +.. code-block:: sh + :caption: MySQL credentials file .my.cnf + + [client] + host=localhost + user=root + password=YourPasswordForAccessingMysqlAsRoot + +Alternatively you can specifiy your database password by adding it to the file :file:`/etc/my.cnf`. + +Verify that you have specified the correct settings by calling the :program:`mysql` command. If this connects you to the database, your credentials are good. Exit the MySQL client. + +For the Bareos database connection, you should specify a database password. Otherwise the Bareos database user gets the permission to connect without password. This is not recommended. Choose a database password and add it into the Bareos Director configuration file :raw-latex:`:file:`/etc/bareos/bareos-dir.conf``: + + + + +.. code-block:: sh + :caption: Bareos catalog configuration + + ... + # + # Generic catalog service + # + Catalog { + Name = MyCatalog + dbdriver = "mysql" + dbname = "bareos" + dbuser = "bareos" + dbpassword = "YourSecretPassword" + } + ... + +After this, run the Bareos database preparation scripts. For Bareos :math:`<=` 13.2.2, the database password must be specified as environment variable :raw-latex:`\variable{db_password}`. From 13.2.3 the database password is read from the configuration, if no environment variable is given. + + + + +.. code-block:: sh + :caption: Setup Bareos catalog database + + export db_password=YourSecretPassword + /usr/lib/bareos/scripts/create_bareos_database + /usr/lib/bareos/scripts/make_bareos_tables + /usr/lib/bareos/scripts/grant_bareos_privileges + +After this, you can use the :program:`mysql` command to verify that your database setup is okay and works with your the Bareos database user. The result should look similar as this (here Bareos 13.2 is used on SLES11): + + + + +.. code-block:: sh + :caption: Verify Bareos database on MySQL + + root@linux:~# mysql --user=bareos --password=YourSecretPassword bareos + Welcome to the MySQL monitor. Commands end with ; or \g. + Your MySQL connection id is 162 + Server version: 5.5.32 SUSE MySQL package + + Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + + Oracle is a registered trademark of Oracle Corporation and/or its + affiliates. Other names may be trademarks of their respective + owners. + + Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. + + mysql> show tables; + +--------------------+ + | Tables_in_bareos | + +--------------------+ + | BaseFiles | + | CDImages | + | Client | + | Counters | + | Device | + | DeviceStats | + | File | + | FileSet | + | Filename | + | Job | + | JobHisto | + | JobMedia | + | JobStats | + | Location | + | LocationLog | + | Log | + | Media | + | MediaType | + | NDMPJobEnvironment | + | NDMPLevelMap | + | Path | + | PathHierarchy | + | PathVisibility | + | Pool | + | Quota | + | RestoreObject | + | Status | + | Storage | + | UnsavedFiles | + | Version | + +--------------------+ + 30 rows in set (0.00 sec) + + mysql> describe Job; + +-----------------+---------------------+------+-----+---------+----------------+ + | Field | Type | Null | Key | Default | Extra | + +-----------------+---------------------+------+-----+---------+----------------+ + | JobId | int(10) unsigned | NO | PRI | NULL | auto_increment | + | Job | tinyblob | NO | | NULL | | + | Name | tinyblob | NO | MUL | NULL | | + | Type | binary(1) | NO | | NULL | | + | Level | binary(1) | NO | | NULL | | + | ClientId | int(11) | YES | | 0 | | + | JobStatus | binary(1) | NO | | NULL | | + | SchedTime | datetime | YES | | NULL | | + | StartTime | datetime | YES | | NULL | | + | EndTime | datetime | YES | | NULL | | + | RealEndTime | datetime | YES | | NULL | | + | JobTDate | bigint(20) unsigned | YES | | 0 | | + | VolSessionId | int(10) unsigned | YES | | 0 | | + | VolSessionTime | int(10) unsigned | YES | | 0 | | + | JobFiles | int(10) unsigned | YES | | 0 | | + | JobBytes | bigint(20) unsigned | YES | | 0 | | + | ReadBytes | bigint(20) unsigned | YES | | 0 | | + | JobErrors | int(10) unsigned | YES | | 0 | | + | JobMissingFiles | int(10) unsigned | YES | | 0 | | + | PoolId | int(10) unsigned | YES | | 0 | | + | FileSetId | int(10) unsigned | YES | | 0 | | + | PriorJobId | int(10) unsigned | YES | | 0 | | + | PurgedFiles | tinyint(4) | YES | | 0 | | + | HasBase | tinyint(4) | YES | | 0 | | + | HasCache | tinyint(4) | YES | | 0 | | + | Reviewed | tinyint(4) | YES | | 0 | | + | Comment | blob | YES | | NULL | | + +-----------------+---------------------+------+-----+---------+----------------+ + 27 rows in set (0,00 sec) + + mysql> select * from Version; + +-----------+ + | VersionId | + +-----------+ + | 2002 | + +-----------+ + 1 row in set (0.00 sec) + + mysql> exit + Bye + +Modify database credentials +''''''''''''''''''''''''''' + +If you want to change the Bareos database credentials, do the following: + +- stop the Bareos director + +- modify the configuration + +- rerun the grant script :program:`grant_bareos_privileges` (or modify database user directly) + +- start the Bareos director + +Modify the configuration, set a new password: + + + + +.. code-block:: sh + :caption: bareos-dir Catalog MyCatalog + + Catalog { + Name = MyCatalog + dbdriver = "mysql" + dbname = "bareos" + dbuser = "bareos" + dbpassword = "MyNewSecretPassword" + } + +Rerun the Bareos grant script :program:`grant_bareos_privileges` ... + + + + +.. code-block:: sh + :caption: Modify database privileges + + export db_password=MyNewSecretPassword + /usr/lib/bareos/scripts/grant_bareos_privileges + +.. raw:: latex + + \hide{ + % table and commands have changed in MySQL 5.7.6. + % As things now get more complicated (different command for different version) + % we skip this topic here. + ... or modify the database users directly: + + \ +.. code-block:: sh + :caption: Show Bareos database users + + mysql + mysql> SELECT user,host,password FROM mysql.user WHERE user='bareos'; + +--------+-----------+-------------------------------------------+ + | user | host | password | + +--------+-----------+-------------------------------------------+ + | bareos | 127.0.0.1 | *CD8C42695AC221807E2BA599FC392C650155C16C | + | bareos | localhost | *CD8C42695AC221807E2BA599FC392C650155C16C | + | bareos | ::1 | *CD8C42695AC221807E2BA599FC392C650155C16C | + +--------+-----------+-------------------------------------------+ + 3 rows in set (0.00 sec) + + mysql> UPDATE mysql.user SET Password=PASSWORD('MyNewSecretPassword') where User='bareos'; + Query OK, 3 rows affected (0.00 sec) + Rows matched: 3 Changed: 3 Warnings: 0 + + mysql> FLUSH PRIVILEGES; + Query OK, 0 rows affected (0.00 sec) + + mysql> SELECT user,host,password FROM mysql.user WHERE user='bareos'; + +--------+-----------+-------------------------------------------+ + | user | host | password | + +--------+-----------+-------------------------------------------+ + | bareos | 127.0.0.1 | *2119D34B0C0F7452E952EE3A73A7CAA30C1B1852 | + | bareos | localhost | *2119D34B0C0F7452E952EE3A73A7CAA30C1B1852 | + | bareos | ::1 | *2119D34B0C0F7452E952EE3A73A7CAA30C1B1852 | + +--------+-----------+-------------------------------------------+ + 3 rows in set (0.00 sec) + + mysql> + \ + } + +Sqlite +^^^^^^ + +There are different versions of Sqlite available. When we use the term Sqlite, we will always refer to Sqlite3. + +Sqlite is a file based database. Access via network connection is not supported. Because its setup is easy, it is a good database for testing. However please don’t use it in a production environment. + +Sqlite stores a database in a single file. Bareos creates this file at :file:`/var/lib/bareos/bareos.db`. + +Sqlite does not offer access permissions. The only permissions that do apply are the Unix file permissions. + +The database is accessable by following command: + + + + +.. code-block:: sh + :caption: Verify Bareos database on Sqlite3 (bareos-13.2.3) + + sqlite3 /var/lib/bareos/bareos.db + SQLite version 3.7.6.3 + Enter ".help" for instructions + Enter SQL statements terminated with a ";" + sqlite> .tables + BaseFiles Filename Media Pool + CDImages Job MediaType Quota + Client JobHisto NDMPJobEnvironment RestoreObject + Counters JobMedia NDMPLevelMap Status + Device JobStats NextId Storage + DeviceStats Location Path UnsavedFiles + File LocationLog PathHierarchy Version + FileSet Log PathVisibility + sqlite> select * from Version; + 2002 + sqlite> + +Retention Periods +----------------- + +Database Size +~~~~~~~~~~~~~ + +.. index:: + single: Database Size + + +As mentioned above, if you do not do automatic pruning, your Catalog will grow each time you run a Job. Normally, you should decide how long you want File records to be maintained in the Catalog and set the **File Retention** period to that time. Then you can either wait and see how big your Catalog gets or make a calculation assuming approximately 154 bytes for each File saved and knowing the number of Files that are saved during each backup and the number of Clients you backup. + +For example, suppose you do a backup of two systems, each with 100,000 files. Suppose further that you do a Full backup weekly and an Incremental every day, and that the Incremental backup typically saves 4,000 files. The size of your database after a month can roughly be calculated as: + +.. raw:: latex + + + + + + Size = 154 * No. Systems * (100,000 * 4 + 10,000 * 26) + +.. raw:: latex + + + +where we have assumed four weeks in a month and 26 incremental backups per month. This would give the following: + +.. raw:: latex + + + + + + Size = 154 * 2 * (100,000 * 4 + 10,000 * 26) = 203,280,000 bytes + +.. raw:: latex + + + +So for the above two systems, we should expect to have a database size of approximately 200 Megabytes. Of course, this will vary according to how many files are actually backed up. + +You will note that the File table (containing the file attributes) make up the large bulk of the number of records as well as the space used. As a consequence, the most important Retention period will be the **File Retention** period. + +Without proper setup and maintenance, your Catalog may continue to grow indefinitely as you run Jobs and backup Files, and/or it may become very inefficient and slow. How fast the size of your Catalog grows depends on the number of Jobs you run and how many files they backup. By deleting records within the database, you can make space available for the new records that will be added during the next Job. By constantly deleting old expired records (dates older than the Retention period), your +database size will remain constant. + +Setting Retention Periods +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Setting Retention Periods +.. index:: + pair: Periods; Setting Retention + + +.. _`Retention`: Retention + +Bareos uses three Retention periods: the **File Retention** period, the **Job Retention** period, and the **Volume Retention** period. Of these three, the File Retention period is by far the most important in determining how large your database will become. + +The **File Retention** and the **Job Retention** are specified in each Client resource as is shown below. The **Volume Retention** period is specified in the Pool resource, and the details are given in the next chapter of this manual. + +.. raw:: latex + + \begin{description} + + \item [File Retention = {\textless}time-period-specification{\textgreater}] + \index[general]{File Retention} + \index[general]{Retention!File} + The File Retention record defines the length of time that Bareos will keep + File records in the Catalog database. When this time period expires, and if + {\bf AutoPrune} is set to {\bf yes}, Bareos will prune (remove) File records + that are older than the specified File Retention period. The pruning will + occur at the end of a backup Job for the given Client. Note that the Client + database record contains a copy of the File and Job retention periods, but + Bareos uses the current values found in the Director's Client resource to do + the pruning. + + Since File records in the database account for probably 80 percent of the + size of the database, you should carefully determine exactly what File + Retention period you need. Once the File records have been removed from + the database, you will no longer be able to restore individual files + in a Job. However, as long as the + Job record still exists, you will be able to restore all files in the + job. + + Retention periods are specified in seconds, but as a convenience, there are + a number of modifiers that permit easy specification in terms of minutes, + hours, days, weeks, months, quarters, or years on the record. See the + \ilink{Configuration chapter}{Time} of this manual for additional details + of modifier specification. + + The default File retention period is 60 days. + + \item [Job Retention = {\textless}time-period-specification{\textgreater}] + \index[general]{Job!Retention} + \index[general]{Retention!Job} + The Job Retention record defines the length of time that {\bf Bareos} + will keep Job records in the Catalog database. When this time period + expires, and if {\bf AutoPrune} is set to {\bf yes} Bareos will prune + (remove) Job records that are older than the specified Job Retention + period. Note, if a Job record is selected for pruning, all associated File + and JobMedia records will also be pruned regardless of the File Retention + period set. As a consequence, you normally will set the File retention + period to be less than the Job retention period. + + As mentioned above, once the File records are removed from the database, + you will no longer be able to restore individual files from the Job. + However, as long as the Job record remains in the database, you will be + able to restore all the files backuped for the Job. + As a consequence, it is generally a good idea to retain the Job + records much longer than the File records. + + The retention period is specified in seconds, but as a convenience, there + are a number of modifiers that permit easy specification in terms of + minutes, hours, days, weeks, months, quarters, or years. + See the \ilink{Configuration chapter}{Time} of this manual for additional details of + modifier specification. + + The default Job Retention period is 180 days. + + \item **Auto Prune**:sup:`Dir`:sub:`Client` + \index[general]{AutoPrune} + \index[general]{Job!Retention!AutoPrune} + If set to {\bf yes}, + Bareos will automatically apply + the File retention period and the Job retention period for the Client at the + end of the Job. + If you turn this off by setting it to {\bf no}, your Catalog will grow each + time you run a Job. + \end{description} + +.. _section-JobStatistics: + +Job Statistics +^^^^^^^^^^^^^^ + +.. index:: + single: Statistics +.. index:: + pair: Job; Statistics + + +Bareos catalog contains lot of information about your IT infrastructure, how many files, their size, the number of video or music files etc. Using Bareos catalog during the day to get them permit to save resources on your servers. + +In this chapter, you will find tips and information to measure Bareos efficiency and report statistics. + +If you want to have statistics on your backups to provide some Service Level Agreement indicators, you could use a few SQL queries on the Job table to report how many: + +- jobs have run + +- jobs have been successful + +- files have been backed up + +- ... + +However, these statistics are accurate only if your job retention is greater than your statistics period. Ie, if jobs are purged from the catalog, you won’t be able to use them. + +Now, you can use the :strong:`update}{stats [days=num]` console command to fill the JobHistory table with new Job records. If you want to be sure to take in account only good jobs, ie if one of your important job has failed but you have fixed the problem and restarted it on time, you probably want to delete the first bad job record and keep only the successful one. For that simply let your staff do the job, and update JobHistory table after two or three days depending on your +organization using the :option:`[days=num]` option. + +These statistics records aren’t used for restoring, but mainly for capacity planning, billings, etc. + +The **Statistics Retention**:sup:`Dir`:sub:`Director` defines the length of time that Bareos will keep statistics job records in the Catalog database after the Job End time. This information is stored in the ``JobHistory`` table. When this time period expires, and if user runs :strong:`prune stats` command, Bareos will prune (remove) Job records that are older than the specified period. + +You can use the following Job resource in your nightly **BackupCatalog**:sup:`Dir`:sub:`job` job to maintain statistics. + + + + +.. code-block:: sh + :caption: bareos-dir Job BackupCatalog + + Job { + Name = BackupCatalog + ... + RunScript { + Console = "update stats days=3" + Console = "prune stats yes" + RunsWhen = After + RunsOnClient = no + } + } + +.. _postgresql-1: + +PostgreSQL +---------- + +.. index:: + single: PostgreSQL + + +Compacting Your PostgreSQL Database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Database; PostgreSQL; Compacting; + + +.. _`CompactingPostgres`: CompactingPostgres + +Over time, as noted above, your database will tend to grow until Bareos starts deleting old expired records based on retention periods. After that starts, it is expected that the database size remains constant, provided that the amount of clients and files being backed up is constant. + +Note that PostgreSQL uses multiversion concurrency control (MVCC), so that an UPDATE or DELETE of a row does not immediately remove the old version of the row. Space occupied by outdated or deleted row versions is only reclaimed for reuse by new rows when running **VACUUM**. Such outdated or deleted row versions are also referred to as *dead tuples*. + +Since PostgreSQL Version 8.3, autovacuum is enabled by default, so that setting up a cron job to run VACUUM is not necesary in most of the cases. Note that there are two variants of VACUUM: standard VACUUM and VACUUM FULL. Standard VACUUM only marks old row versions for reuse, it does not free any allocated disk space to the operating system. Only VACUUM FULL can free up disk space, but it requires exclusive table locks so that it can not be used in parallel with production database operations +and temporarily requires up to as much additional disk space that the table being processed occupies. + +All database programs have some means of writing the database out in ASCII format and then reloading it. Doing so will re-create the database from scratch producing a compacted result, so below, we show you how you can do this for PostgreSQL. + +For a PostgreSQL database, you could write the Bareos database as an ASCII file (:file:`bareos.sql`) then reload it by doing the following: + + + + +.. code-block:: sh + :caption: + + pg_dump -c bareos > bareos.sql + cat bareos.sql | psql bareos + rm -f bareos.sql + +Depending on the size of your database, this will take more or less time and a fair amount of disk space. For example, you can :program:`cd` to the location of the Bareos database (typically :file:`/var/lib/pgsql/data` or possible :file:`/usr/local/pgsql/data`) and check the size. + +Except from special cases PostgreSQL does not need to be dumped/restored to keep the database efficient. A normal process of vacuuming will prevent the database from getting too large. If you want to fine-tweak the database storage, commands such as VACUUM, VACUUM FULL, REINDEX, and CLUSTER exist specifically to keep you from having to do a dump/restore. + +More details on this subject can be found in the PostgreSQL documentation. The page `http://www.postgresql.org/docs/ `_ contains links to the documentation for all PostgreSQL versions. The section *Routine Vacuuming* explains how VACUUM works and why it is required, see `http://www.postgresql.org/docs/current/static/routine-vacuuming.html `_ for the current PostgreSQL version. + +.. _PostgresSize: + +What To Do When The Database Keeps Growing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Especially when a high number of files are beeing backed up or when working with high retention periods, it is probable that autovacuuming will not work. When starting to use Bareos with an empty Database, it is normal that the file table and other tables grow, but the growth rate should drop as soon as jobs are deleted by retention or pruning. The file table is usually the largest table in Bareos. + +The reason for autovacuuming not beeing triggered is then probably the default setting of ``autovacuum_vacuum_scale_factor = 0.2``, the current value can be shown with the following query or looked up in ``postgresql.conf``: + + + + +.. code-block:: sh + :caption: SQL statement to show the autovacuum\_vacuum\_scale\_factor parameter + + bareos=# show autovacuum_vacuum_scale_factor; + autovacuum_vacuum_scale_factor + -------------------------------- + 0.2 + (1 row) + +In essence, this means that a VACUUM is only triggered when 20% of table size are obsolete. Consequently, the larger the table is, the less frequently VACUUM will be triggered by autovacuum. This make sense because vacuuming has a performance impact. While it is possible to override the autovacuum parameters on a table-by-table basis, it can then still be triggered at any time. + +To learn more details about autovacuum see `http://www.postgresql.org/docs/current/static/routine-vacuuming.html#AUTOVACUUM `_ + +The following example shows how to configure running VACUUM on the file table by using an admin-job in Bareos. The job will be scheduled to run at a time that should not run in parallel with normal backup jobs, here by scheduling it to run after the BackupCatalog job. + +First step is to check the amount of dead tuples and if autovacuum triggers VACUUM: + + + + +.. code-block:: sh + :caption: Check dead tuples and vacuuming on PostgreSQL + + bareos=# SELECT relname, n_dead_tup, last_vacuum, last_autovacuum, last_analyze, last_autoanalyze + FROM pg_stat_user_tables WHERE n_dead_tup > 0 ORDER BY n_dead_tup DESC; + -[ RECORD 1 ]----+------------------------------ + relname | file + n_dead_tup | 2955116 + last_vacuum | + last_autovacuum | + last_analyze | + last_autoanalyze | + -[ RECORD 2 ]----+------------------------------ + relname | log + n_dead_tup | 111298 + last_vacuum | + last_autovacuum | + last_analyze | + last_autoanalyze | + -[ RECORD 3 ]----+------------------------------ + relname | job + n_dead_tup | 1785 + last_vacuum | + last_autovacuum | 2015-01-08 01:13:20.70894+01 + last_analyze | + last_autoanalyze | 2014-12-27 18:00:58.639319+01 + ... + +In the above example, the file table has a high number of dead tuples and it has not been vacuumed. Same for the log table, but the dead tuple count is not very high. On the job table autovacuum has been triggered. + +Note that the statistics views in PostgreSQL are not persistent, their values are reset on restart of the PostgreSQL service. + +To setup a scheduled admin job for vacuuming the file table, the following must be done: + +#. | Create a file with the SQL statements for example + | ``/usr/local/lib/bareos/scripts/postgresql_file_table_maintenance.sql`` + | with the following content: + + + + +.. code-block:: sh + :caption: SQL Script for vacuuming the file table on PostgreSQL + + \t \x + SELECT relname, n_dead_tup, last_vacuum, last_autovacuum, last_analyze, last_autoanalyze + FROM pg_stat_user_tables WHERE relname='file'; + VACUUM VERBOSE ANALYZE file; + SELECT relname, n_dead_tup, last_vacuum, last_autovacuum, last_analyze, last_autoanalyze + FROM pg_stat_user_tables WHERE relname='file'; + \t \x + SELECT table_name, + pg_size_pretty(pg_total_relation_size(table_name)) AS total_sz, + pg_size_pretty(pg_total_relation_size(table_name) - pg_relation_size(table_name)) AS idx_sz + FROM ( SELECT ('"' || relname || '"' ) AS table_name + FROM pg_stat_user_tables WHERE relname != 'batch' ) AS all_tables + ORDER BY pg_total_relation_size(table_name) DESC LIMIT 5; + + The SELECT statements are for informational purposes only, the final statement shows the total and index disk usage of the 5 largest tables. + +#. | Create a shell script that runs the SQL statements, for example + | ``/usr/local/lib/bareos/scripts/postgresql_file_table_maintenance.sh`` + | with the following content: + + + + +.. code-block:: sh + :caption: SQL Script for vacuuming the file table on PostgreSQL + + #!/bin/sh + psql bareos < /usr/local/lib/bareos/scripts/postgresql_file_table_maintenance.sql + +#. As in PostgreSQL only the database owner or a database superuser is allowed to run VACUUM, the script will be run as the ``postgres`` user. To permit the ``bareos`` user to run the script via ``sudo``, write the following sudo rule to a file by executing ``visudo -f /etc/sudoers.d/bareos_postgres_vacuum``: + + + + +.. code-block:: sh + :caption: sudo rule for allowing bareos to run a script as postgres + + bareos ALL = (postgres) NOPASSWD: /usr/local/lib/bareos/scripts/postgresql_file_table_maintenance.sh + + and make sure that ``/etc/sudoers`` includes it, usually by the line :raw-latex:`` + + + + #includedir /etc/sudoers.d + + + + + +#. Create the following admin job in the director configuration + + + + +.. code-block:: sh + :caption: SQL Script for vacuuming the file table on PostgreSQL + + # PostgreSQL file table maintenance job + Job { + Name = FileTableMaintJob + JobDefs = DefaultJob + Schedule = "WeeklyCycleAfterBackup" + Type = Admin + Priority = 20 + + RunScript { + RunsWhen = Before + RunsOnClient = no + Fail Job On Error = yes + Command = "sudo -u postgres /usr/local/lib/bareos/scripts/postgresql_file_table_maintenance.sh" + } + } + + In this example the job will be run by the schedule WeeklyCycleAfterBackup, the ``Priority`` should be set to a higher value than ``Priority`` in the BackupCatalog job. + +.. _RepairingPSQL: + +Repairing Your PostgreSQL Database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Repairing Your PostgreSQL Database + + +The same considerations apply as for :ref:`RepairingMySQL`. Consult the PostgreSQL documents for how to repair the database. + +For Bareos specific problems, consider using :ref:`bareos-dbcheck` program. + +MySQL/MariaDB +------------- + +.. index:: + single: MySQL + + +MySQL/MariaDB Support +~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: MariaDB|see{MySQL} + + +.. _`section-MysqlSupport`: section-MysqlSupport + +As MariaDB is a fork of MySQL, we use MySQL as synonym for MariaDB and fully support it. We test our packages against the preferred MySQL fork that a distribution provides. + +.. raw:: latex + + \hide{ + supporting MySQL >= 5.7 since bareos-17.2.4 + \limitation{MySQL}{MySQL ≥ 5.7 not supported}{% + MySQL 5.7 did change it behavior in some respects. The result is, that the Bareos database creation scripts do not work any more. + For the time being, we advise to use MariaDB instead, which is also the default on most Linux distributions. + See \ticket{705}. + } + } + +Compacting Your MySQL Database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +.. index:: + triple: Database; MySQL; Compacting; + + +.. _`CompactingMySQL`: CompactingMySQL + +Over time, as noted above, your database will tend to grow. Even though Bareos regularly prunes files, **MySQL** does not automatically reuse the space, and instead continues growing. + +It is assumed that you are using the **InnoDB** database engine (which is the default since MySQL Version 5.5). + +It is recommended that you use the **OPTIMIZE TABLE** and **ANALYZE TABLE** statements regularly. This is to make sure that all indices are up to date and to recycle space inside the database files. + +You can do this via the **mysqlcheck** command: :raw-latex:`` + + + + mysqlcheck -a -o -A + +.. raw:: latex + + + +Please note that the database files are never shrunk by **MySQL**. If you really need to shrink the database files, you need to recreate the database. This only works if you use per-table tablespaces by setting the **innodb_file_per_table** configuration option. See :raw-latex:`\elink{http://dev.mysql.com/doc/refman/5.5/en/innodb-multiple-tablespaces.html} +{http://dev.mysql.com/doc/refman/5.5/en/innodb-multiple-tablespaces.html}` for details. + +.. raw:: latex + + \hide{ + + \ + mysqldump -f --opt bareos > bareos.sql + mysql bareos < bareos.sql + rm -f bareos.sql + \ + + + Depending on the size of your database, this will take more or less time and a + fair amount of disk space. + } + +Repairing Your MySQL Database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: Repairing Your MySQL Database + + +.. _`RepairingMySQL`: RepairingMySQL + +If you find that you are getting errors writing to your MySQL database, or Bareos hangs each time it tries to access the database, you should consider running MySQL’s database check and repair routines. + +This can be done by running the :program:`mysqlcheck` command: :raw-latex:`` + + + + mysqlcheck --all-databases + +.. raw:: latex + + + +If the errors you are getting are simply SQL warnings, then you might try running :program:`bareos-dbcheck` before (or possibly after) using the MySQL database repair program. It can clean up many of the orphaned record problems, and certain other inconsistencies in the Bareos database. + +A typical cause of MySQL database problems is if your partition fills. In such a case, you will need to create additional space on the partition. + +MySQL Table is Full +~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: MySQL Table is Full + + +If you are running into the error **The table ’File’ is full ...**, it is probably because on version 4.x MySQL, the table is limited by default to a maximum size of 4 GB and you have probably run into the limit. The solution can be found at: :raw-latex:`\elink{http://dev.mysql.com/doc/refman/5.0/en/full-table.html} +{http://dev.mysql.com/doc/refman/5.0/en/full-table.html}` + +You can display the maximum length of your table with: + +.. raw:: latex + + + + + + mysql bareos + SHOW TABLE STATUS FROM bareos like "File"; + +.. raw:: latex + + + +If the column labeled "Max_data_length" is around 4Gb, this is likely to be the source of your problem, and you can modify it with: + +.. raw:: latex + + + + + + mysql bareos + ALTER TABLE File MAX_ROWS=281474976710656; + +.. raw:: latex + + + +MySQL Server Has Gone Away +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. index:: + single: MySQL Server Has Gone Away + If you are having problems with the MySQL server disconnecting or with messages saying that your MySQL server has gone away, then please read the MySQL documentation, which can be found at: + +.. raw:: latex + + \elink{http://dev.mysql.com/doc/refman/5.0/en/gone-away.html} + {http://dev.mysql.com/doc/refman/5.0/en/gone-away.html} + +MySQL Temporary Tables +~~~~~~~~~~~~~~~~~~~~~~ + +When doing backups with large numbers of files, MySQL creates some temporary tables. When these tables are small they can be held in system memory, but as they approach some size, they spool off to disk. The default location for these temp tables is /tmp. Once that space fills up, Bareos daemons such as the Storage daemon doing spooling can get strange errors. E.g. + +.. raw:: latex + + + + + + Fatal error: spool.c:402 Spool data read error. + Fatal error: backup.c:892 Network send error to SD. ERR=Connection reset by + peer + +.. raw:: latex + + + +What you need to do is setup MySQL to use a different (larger) temp directory, which can be set in the /etc/my.cnf with these variables set: + +.. raw:: latex + + + + + + tmpdir=/path/to/larger/tmpdir + bdb_tmpdir=/path/to/larger/tmpdir + +.. raw:: latex + + + +MySQL: Lock Wait Timeout +~~~~~~~~~~~~~~~~~~~~~~~~ + +In large environments, the Bareos |mysql| backend may run in a lock wait timeout. This can be seen as Bareos message, e.g.: + + + + {Bareos error message because of \mysql lock time timeout} + Fatal error: sql_create.c:899 Fill File table Query failed: INSERT INTO File (FileIndex, JobId, PathId, FilenameId, LStat, MD5, DeltaSeq) SELECT batch.FileIndex, batch.JobId, Path.PathId, Filename.FilenameId,batch.LStat, batch.MD5, batch.DeltaSeq FROM batch JOIN Path ON (batch.Path = Path.Path) JOIN Filename ON (batch.Name = Filename.Name): ERR=Lock wait timeout exceeded; try restarting transaction + +In this case the |mysql| :strong:`innodb_lock_wait_timeout` must be increased. A value of 300 should be sufficient. + + + + +.. code-block:: sh + :caption: /etc/my.cnf.d/server.cnf + + ... + [mysqld] + innodb_lock_wait_timeout = 300 + ... + +.. raw:: latex + + \hide{ + \label{DatabasePerformance} + \section{Database Performance Issues} + \index[general]{Database Performance Issues} + \index[general]{Performance!Database} + + There are a considerable number of ways each of the databases can be + tuned to improve the performance. Going from an untuned database to one + that is properly tuned can make a difference of a factor of 100 or more + in the time to insert or search for records. + + For each of the databases, you may get significant improvements by adding + additional indexes. The comments in the Bareos make\_xxx\_tables give some + indications as to what indexes may be appropriate. Please see below + for specific instructions on checking indexes. + + For MySQL, what is very important is to use the examine the + my.cnf file (usually in /etc/my.cnf). + You may obtain significant performances by switching to + the my-large.cnf or my-huge.cnf files that come with the MySQL source + code. + + For SQLite3, one significant factor in improving the performance is + to ensure that there is a "PRAGMA synchronous = NORMAL;" statement. + This reduces the number of times that the database flushes the in memory + cache to disk. There are other settings for this PRAGMA that can + give even further performance improvements at the risk of a database + corruption if your system crashes. + + For PostgreSQL, you might want to consider turning fsync off. Of course + doing so can cause corrupted databases in the event of a machine crash. + There are many different ways that you can tune PostgreSQL, the + following document discusses a few of them: + \elink{ + http://www.varlena.com/varlena/GeneralBits/Tidbits/perf.html} + {http://www.varlena.com/varlena/GeneralBits/Tidbits/perf.html}. + + There is also a PostgreSQL FAQ question number 3.3 that may + answer some of your questions about how to improve performance + of the PostgreSQL engine: + \elink{ + http://www.postgresql.org/docs/faqs.FAQ.html\#3.3} + {http://www.postgresql.org/docs/faqs.FAQ.html\#3.3}. + % TODO: verify above is correct. is this okay for book? + + Also for PostgreSQL, look at what "effective\_cache\_size". For a 2GB memory + machine, you probably want to set it at 131072, but don't set it too high. + In addition, for a 2GB system, work\_mem = 256000 and + maintenance\_work\_mem = 256000 seem to be reasonable values. Make + sure your checkpoint\_segments is set to at least 8. + } + +.. raw:: latex + + \hide{ + \section{Performance Issues Indexes} + \index[general]{Database Performance Issues Indexes} + \index[general]{Performance!Database} + + \TODO{This chapter needs verification/updating.} + + One of the most important considerations for improving performance on + the Bareos database is to ensure that it has all the appropriate indexes. + Several users have reported finding that their database did not have + all the indexes in the default configuration. In addition, you may + find that because of your own usage patterns, you need additional indexes. + + The most important indexes for performance are the two indexes on the + {\bf File} table. The first index is on {\bf FileId} and is automatically + made because it is the unique key used to access the table. The other + one is the (JobId, PathId, Filename) index. If these Indexes + are not present, your performance may suffer a lot. + + \subsection{PostgreSQL Indexes} + On PostgreSQL, you can check to see if you have the proper indexes using + the following commands: + + + \ + psql bareos + select * from pg_indexes where tablename='file'; + \ + + + If the indexes are not present, especially the JobId index, you can + create them with the following commands: + + + \ + psql bareos + CREATE INDEX file_jobid_idx on file (jobid); + \ + + + Make sure that you doesn't have an index on File (filenameid, pathid). + + \subsection{MySQL Indexes} + On MySQL, you can check if you have the proper indexes by: + + + \ + mysql bareos + show index from File; + \ + + + If the indexes are not present, especially the JobId index, you can + create them with the following commands: + + + \ + mysql bareos + CREATE INDEX file_jobid_idx on File (JobId); + \ + + + Though normally not a problem, you should ensure that the indexes + defined for Filename and Path are both set to 255 characters. Some users + reported performance problems when their indexes were set to 50 characters. + To check, do: + + + \ + mysql bareos + show index from Filename; + show index from Path; + \ + + + and what is important is that for Filename, you have an index with + Key\_name "Name" and Sub\_part "255". For Path, you should have a Key\_name + "Path" and Sub\_part "255". If one or the other does not exist or the + Sub\_part is less that 255, you can drop and recreate the appropriate + index with: + + + \ + mysql bareos + DROP INDEX Path on Path; + CREATE INDEX Path on Path (Path(255)); + + DROP INDEX Name on Filename; + CREATE INDEX Name on Filename (Name(255)); + \ + + + + \subsection{SQLite Indexes} + On SQLite, you can check if you have the proper indexes by: + + + \ + sqlite /bareos.db + select * from sqlite_master where type='index' and tbl_name='File'; + \ + + + If the indexes are not present, especially the JobId index, you can + create them with the following commands: + + + \ + sqlite /bareos.db + CREATE INDEX file_jobid_idx on File (JobId); + \ + + } + +Backing Up Your Bareos Database +------------------------------- + +.. index:: + pair: Backup; Bareos database +.. index:: + pair: Backup; Catalog +.. index:: + pair: Database; Backup Bareos database + + +.. _`BackingUpBareos`: BackingUpBareos + +If ever the machine on which your Bareos database crashes, and you need to restore from backup tapes, one of your first priorities will probably be to recover the database. Although Bareos will happily backup your catalog database if it is specified in the FileSet, this is not a very good way to do it, because the database will be saved while Bareos is modifying it. Thus the database may be in an instable state. Worse yet, you will backup the database before all the Bareos updates have been +applied. + +To resolve these problems, you need to backup the database after all the backup jobs have been run. In addition, you will want to make a copy while Bareos is not modifying it. To do so, you can use two scripts provided in the release **make_catalog_backup** and **delete_catalog_backup**. These files will be automatically generated along with all the other Bareos scripts. The first script will make an ASCII copy of your Bareos database into **bareos.sql** in the working directory you specified in +your configuration, and the second will delete the **bareos.sql** file. + +The basic sequence of events to make this work correctly is as follows: + +- Run all your nightly backups + +- After running your nightly backups, run a Catalog backup Job + +- The Catalog backup job must be scheduled after your last nightly backup + +- You use **Run Before Job**:sup:`Dir`:sub:`Job` to create the ASCII backup file and **Run After Job**:sup:`Dir`:sub:`Job` to clean up + +Assuming that you start all your nightly backup jobs at 1:05 am (and that they run one after another), you can do the catalog backup with the following additional Director configuration statements: + + + + +.. code-block:: sh + :caption: bareos-dir Job BackupCatalog + + # Backup the catalog database (after the nightly save) + Job { + Name = "BackupCatalog" + Type = Backup + Client=rufus-fd + FileSet="Catalog" + Schedule = "WeeklyCycleAfterBackup" + Storage = DLTDrive + Messages = Standard + Pool = Default + # This creates an ASCII copy of the catalog + # Arguments to make_catalog_backup.pl are: + # make_catalog_backup.pl + RunBeforeJob = "/usr/lib/bareos/scripts/make_catalog_backup.pl MyCatalog" + # This deletes the copy of the catalog + RunAfterJob = "/usr/lib/bareos/scripts/delete_catalog_backup" + # This sends the bootstrap via mail for disaster recovery. + # Should be sent to another system, please change recipient accordingly + Write Bootstrap = "|/usr/sbin/bsmtp -h localhost -f \"\(Bareos\) \" -s \"Bootstrap for Job %j\" root@localhost" + } + + + + +.. code-block:: sh + :caption: bareos-dir Schedule WeeklyCycleAfterBackup + + # This schedule does the catalog. It starts after the WeeklyCycle + Schedule { + Name = "WeeklyCycleAfterBackup" + Run = Level=Full sun-sat at 1:10 + } + + + + +.. code-block:: sh + :caption: bareos-dir FileSet Catalog + + # This is the backup of the catalog + FileSet { + Name = "Catalog" + Include { + Options { + signature=MD5 + } + File = "/var/lib/bareos/bareos.sql" # database dump + File = "/etc/bareos" # configuration + } + } + +It is preferable to write/send the :ref:`bootstrap ` file to another computer. It will allow you to quickly recover the database backup should that be necessary. If you do not have a bootstrap file, it is still possible to recover your database backup, but it will be more work and take longer. + +.. |image| image:: \idir dbconfig-1-enable + :width: 45.0% +.. |image| image:: \idir dbconfig-2-select-database-type + :width: 45.0% diff --git a/docs/manuals/en/new_main_reference/source/chapter33/security.rst b/docs/manuals/en/new_main_reference/source/chapter33/security.rst new file mode 100644 index 00000000000..73dc1e50141 --- /dev/null +++ b/docs/manuals/en/new_main_reference/source/chapter33/security.rst @@ -0,0 +1,350 @@ +.. ATTENTION do not edit this file manually. + It was automatically converted from the corresponding .tex file + +.. _SecurityChapter: + +Bareos Security Issues +====================== + +.. index:: + single: Security + + +- Security means being able to restore your files, so read the :ref:`Critical Items Chapter ` of this manual. + +- The clients (**bareos-fd**) must run as root to be able to access all the system files. + +- It is not necessary to run the Director as root. + +- It is not necessary to run the Storage daemon as root, but you must ensure that it can open the tape drives, which are often restricted to root access by default. In addition, if you do not run the Storage daemon as root, it will not be able to automatically set your tape drive parameters on most OSes since these functions, unfortunately require root access. + +- You should restrict access to the Bareos configuration files, so that the passwords are not world-readable. The **Bareos** daemons are password protected using CRAM-MD5 (i.e. the password is not sent across the network). This will ensure that not everyone can access the daemons. It is a reasonably good protection, but can be cracked by experts. + +- If you are using the recommended ports 9101, 9102, and 9103, you will probably want to protect these ports from external access using a firewall and/or using tcp wrappers (**etc/hosts.allow**). + +- By default, all data that is sent across the network is unencrypted. However, Bareos does support TLS (transport layer security) and can encrypt transmitted data. Please read the :ref:`TLS (SSL) Communications Encryption ` section of this manual. + +- You should ensure that the Bareos working directories are readable and writable only by the Bareos daemons. + +- The default Bareos :program:`grant_bareos_privileges` script grants all permissions to use the MySQL (and PostgreSQL) database without a password. If you want security, please tighten this up! + +- Don’t forget that Bareos is a network program, so anyone anywhere on the network with the console program and the Director’s password can access Bareos and the backed up data. + +- You can restrict what IP addresses Bareos will bind to by using the appropriate **DirAddress**, **FDAddress**, or **SDAddress** records in the respective daemon configuration files. + +.. _wrappers: + +Configuring and Testing TCP Wrappers +------------------------------------ + +.. index:: + single: TCP Wrappers +.. index:: + pair: Wrappers; TCP + :raw-latex:`\index[general]{libwrappers}` + +The TCP wrapper functionality is available on different platforms. Be default, it is activated on Bareos for Linux. With this enabled, you may control who may access your daemons. This control is done by modifying the file: **/etc/hosts.allow**. The program name that **Bareos** uses when applying these access restrictions is the name you specify in the daemon configuration file (see below for examples). You must not use the **twist** option in your **/etc/hosts.allow** or it will terminate the +Bareos daemon when a connection is refused. + +.. raw:: latex + + \hide{ + Dan Langille has provided the following information on configuring and + testing TCP wrappers with Bareos. + + If you read hosts\_options(5), you will see an option called twist. This + option replaces the current process by an instance of the specified shell + command. Typically, something like this is used: + + + \ + ALL : ALL \ + : severity auth.info \ + : twist /bin/echo "You are not welcome to use %d from %h." + \ + + + The libwrap code tries to avoid {\bf twist} if it runs in a resident process, + but that test will not protect the first hosts\_access() call. This will + result in the process (e.g. bareos-fd, bareos-sd, bareos-dir) being terminated + if the first connection to their port results in the twist option being + invoked. The potential, and I stress potential, exists for an attacker to + prevent the daemons from running. This situation is eliminated if your + /etc/hosts.allow file contains an appropriate rule set. The following example + is sufficient: + + + \ + undef-fd : localhost : allow + undef-sd : localhost : allow + undef-dir : localhost : allow + undef-fd : ALL : deny + undef-sd : ALL : deny + undef-dir : ALL : deny + \ + + + You must adjust the names to be the same as the Name directives found + in each of the daemon configuration files. They are, in general, not the + same as the binary daemon names. It is not possible to use the + daemon names because multiple daemons may be running on the same machine + but with different configurations. + + In these examples, the Director is undef-dir, the + Storage Daemon is undef-sd, and the File Daemon is undef-fd. Adjust to suit + your situation. The above example rules assume that the SD, FD, and DIR all + reside on the same box. If you have a remote FD client, then the following + rule set on the remote client will suffice: + + + \ + undef-fd : director.example.org : allow + undef-fd : ALL : deny + \ + + + where director.example.org is the host which will be contacting the client + (ie. the box on which the Bareos Director daemon runs). The use of "ALL : + deny" ensures that the twist option (if present) is not invoked. To properly + test your configuration, start the daemon(s), then attempt to connect from an + IP address which should be able to connect. You should see something like + this: + + + \ + $ telnet undef 9103 + Trying 192.168.0.56... + Connected to undef.example.org. + Escape character is '^]'. + Connection closed by foreign host. + $ + \ + + + This is the correct response. If you see this: + + + \ + $ telnet undef 9103 + Trying 192.168.0.56... + Connected to undef.example.org. + Escape character is '^]'. + You are not welcome to use undef-sd from xeon.example.org. + Connection closed by foreign host. + $ + \ + + + then twist has been invoked and your configuration is not correct and you need + to add the deny statement. It is important to note that your testing must + include restarting the daemons after each connection attempt. You can also + tcpdchk(8) and tcpdmatch(8) to validate your /etc/hosts.allow rules. Here is a + simple test using tcpdmatch: + + + \ + $ tcpdmatch undef-dir xeon.example.org + warning: undef-dir: no such process name in /etc/inetd.conf + client: hostname xeon.example.org + client: address 192.168.0.18 + server: process undef-dir + matched: /etc/hosts.allow line 40 + option: allow + access: granted + \ + + + If you are running Bareos as a standalone daemon, the warning above can be + safely ignored. Here is an example which indicates that your rules are missing + a deny statement and the twist option has been invoked. + + + \ + $ tcpdmatch undef-dir 10.0.0.1 + warning: undef-dir: no such process name in /etc/inetd.conf + client: address 10.0.0.1 + server: process undef-dir + matched: /etc/hosts.allow line 91 + option: severity auth.info + option: twist /bin/echo "You are not welcome to use + undef-dir from 10.0.0.1." + access: delegated + \ + + } + +.. raw:: latex + + \hide{ + \section{Running as non-root} + \index[general]{Running as non-root} + + Security advice from Dan Langille: + % TODO: don't use specific name + + % TODO: don't be too specific on operating system + + % TODO: maybe remove personalization? + + It is a good idea to run daemons with the lowest possible privileges. In + other words, if you can, don't run applications as root which do not have to + be root. The Storage Daemon and the Director Daemon do not need to be root. + The File Daemon needs to be root in order to access all files on your system. + In order to run as non-root, you need to create a user and a group. Choosing + {\tt bareos} as both the user name and the group name sounds like a good idea + to me. + + The FreeBSD port creates this user and group for you. + Here is what those entries looked like on my FreeBSD laptop: + + + \ + bareos:*:1002:1002::0:0:Bareos Daemon:/var/db/bareos:/sbin/nologin + \ + + + I used vipw to create this entry. I selected a User ID and Group ID of 1002 + as they were unused on my system. + + I also created a group in /etc/group: + + + \ + bareos:*:1002: + \ + + + The bareos user (as opposed to the Bareos daemon) will have a home directory + of {\tt /var/db/bareos} which is the default location for the Bareos + database. + + Now that you have both a bareos user and a bareos group, you can secure the + bareos home directory by issuing this command: + + + \ + chown -R bareos:bareos /var/db/bareos/ + \ + + + This ensures that only the bareos user can access this directory. It also + means that if we run the Director and the Storage daemon as bareos, those + daemons also have restricted access. This would not be the case if they were + running as root. + + It is important to note that the storage daemon actually needs to be in the + operator group for normal access to tape drives etc (at least on a FreeBSD + system, that's how things are set up by default) Such devices are normally + chown root:operator. It is easier and less error prone to make Bareos a + member of that group than it is to play around with system permissions. + + Starting the Bareos daemons + + To start the bareos daemons on a FreeBSD system, issue the following command: + + + \ + /usr/local/etc/rc.d/bareos-dir start + /usr/local/etc/rc.d/bareos-sd start + /usr/local/etc/rc.d/bareos-fd start + \ + + + To confirm they are all running: + + + \ + $ ps auwx | grep bareos + root 63418 0.0 0.3 1856 1036 ?? Ss 4:09PM 0:00.00 + /usr/local/sbin/bareos-fd -v -c /usr/local/etc/bareos-fd.conf + bareos 63416 0.0 0.3 2040 1172 ?? Ss 4:09PM 0:00.01 + /usr/local/sbin/bareos-sd -v -c /usr/local/etc/bareos-sd.conf + bareos 63422 0.0 0.4 2360 1440 ?? Ss 4:09PM 0:00.00 + /usr/local/sbin/bareos-dir -v -c /usr/local/etc/bareos-dir.conf + \ + + } + +.. _section-SecureEraseCommand: + +Secure Erase Command +-------------------- + +From `https://en.wikipedia.org/w/index.php?title=Data_erasure&oldid=675388437 `_: + + Strict industry standards and government regulations are in place that force organizations to mitigate the risk of unauthorized exposure of confidential corporate and government data. Regulations in the United States include HIPAA (Health Insurance Portability and Accountability Act); FACTA (The Fair and Accurate Credit Transactions Act of 2003); GLB (Gramm-Leach Bliley); Sarbanes-Oxley Act (SOx); and Payment Card Industry Data Security Standards (PCI DSS) and the Data Protection Act in the + United Kingdom. Failure to comply can result in fines and damage to company reputation, as well as civil and criminal liability. + +Bareos supports the secure erase of files that usually are simply deleted. Bareos uses an external command to do the secure erase itself. + +This makes it easy to choose a tool that meets the secure erase requirements. + +To configure this functionality, a new configuration directive with the name :strong:`Secure Erase Command` has been introduced. + +This directive is optional and can be configured in: + +- + + + + **Secure Erase Command**:sup:`Dir`:sub:`Director` + +- + + + + **Secure Erase Command**:sup:`Sd`:sub:`Storage` + +- + + + + **Secure Erase Command**:sup:`Fd`:sub:`Client` + +This directive configures the secure erase command globally for the daemon it was configured in. + +If set, the secure erase command is used to delete files instead of the normal delete routine. + +If files are securely erased during a job, the secure delete command output will be shown in the job log. + + + + +.. code-block:: sh + :caption: bareos.log + + 08-Sep 12:58 win-fd JobId 10: secure_erase: executing C:/cygwin64/bin/shred.exe "C:/temp/bareos-restores/C/Program Files/Bareos/Plugins/bareos_fd_consts.py" + 08-Sep 12:58 win-fd JobId 10: secure_erase: executing C:/cygwin64/bin/shred.exe "C:/temp/bareos-restores/C/Program Files/Bareos/Plugins/bareos_sd_consts.py" + 08-Sep 12:58 win-fd JobId 10: secure_erase: executing C:/cygwin64/bin/shred.exe "C:/temp/bareos-restores/C/Program Files/Bareos/Plugins/bpipe-fd.dll" + +The current status of the secure erase command is also shown in the output of status director, status client and status storage. + +If the secure erase command is configured, the current value is printed. + +Example: + + + + +.. code-block:: sh + :caption: + + * status dir + backup1.example.com-dir Version: 15.3.0 (24 August 2015) x86_64-suse-linux-gnu suse openSUSE 13.2 (Harlequin) (x86_64) + Daemon started 08-Sep-15 12:50. Jobs: run=0, running=0 mode=0 db=sqlite3 + Heap: heap=290,816 smbytes=89,166 max_bytes=89,166 bufs=334 max_bufs=335 + secure erase command='/usr/bin/wipe -V' + +Example for Secure Erase Command Settings: + +Linux: + + + \configdirective{Secure Erase Command = "/usr/bin/wipe -V"} + +Windows: + + + \configdirective{Secure Erase Command = "C:/cygwin64/bin/shred.exe"} + +Our tests with the :program:`sdelete` command was not successful, as :program:`sdelete` seems to stay active in the background.