diff --git a/.travis.yml b/.travis.yml index 8d34767778d..d069342de60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ +os: + - linux +dist: trusty +sudo: required language: cpp compiler: - gcc - - clang -sudo: required # This project also uses Coverity Scan https://scan.coverity.com/ # However, the Travis coverity scan addon (as of 19.02.2014) does not fit our needs, @@ -29,47 +31,28 @@ env: - COVERITY_SCAN_BUILD="curl -s $COVERITY_SCAN_BUILD_URL | bash" # -- END Coverity Scan ENV - matrix: - - DB=postgresql - - DB=mysql - - DB=sqlite3 - - DB=postgresql COVERITY_SCAN=1 - matrix: - # covertiy scan should only run once and it might fail, - # because the number of times its runs is limited per week. - # We only check when compiled with gcc. - exclude: - - compiler: clang - env: DB=postgresql COVERITY_SCAN=1 - allow_failures: - - env: DB=postgresql COVERITY_SCAN=1 + include: + - env: DB=postgresql + - env: DB=mysql + - env: DB=sqlite3 + - env: DB=postgresql COVERITY_SCAN=1 + - env: DB=postgresql + compiler: clang + allow_failures: + - env: DB=postgresql COVERITY_SCAN=1 + + before_install: # install build dependencies - # use files instead of shell variables, because travis has some problems supporting variables - - sudo apt-get -qq update - - dpkg-checkbuilddeps 2> /tmp/dpkg-builddeps || true - - cat /tmp/dpkg-builddeps - - sed -e "s/^.*:.*:\s//" -e "s/\s([^)]*)//g" /tmp/dpkg-builddeps > /tmp/build_depends - - echo "additional packages required for building:"; cat /tmp/build_depends - - yes "" | sudo xargs --arg-file /tmp/build_depends apt-get -q --assume-no install fakeroot - - dpkg -l + - test/travis_before_install.sh before_script: # changelog file is required (and normally generated by OBS) - cp -a platforms/packaging/bareos.changes debian/changelog - # build Debian packages - - if [ -z "${COVERITY_SCAN}" ]; then fakeroot debian/rules binary; else debian/rules override_dh_auto_configure; eval "$COVERITY_SCAN_BUILD"; fi - # create Debian package repository - - cd .. - - if [ -z "${COVERITY_SCAN}" ]; then dpkg-scanpackages . /dev/null | gzip > Packages.gz; fi - - if [ -z "${COVERITY_SCAN}" ]; then printf 'deb file:%s /\n' $PWD > /tmp/bareos.list; fi - - if [ -z "${COVERITY_SCAN}" ]; then sudo cp /tmp/bareos.list /etc/apt/sources.list.d/bareos.list; fi - - cd - - # install Bareos packages - - if [ -z "${COVERITY_SCAN}" ]; then sudo apt-get -qq update; fi - - if [ -z "${COVERITY_SCAN}" ]; then sudo apt-get install -y --force-yes bareos bareos-database-$DB; fi + # build and install Bareos packages + - test/travis_before_script.sh script: # run test script diff --git a/autoconf/config.h.in b/autoconf/config.h.in index b35f7eb5780..36f3b92e917 100644 --- a/autoconf/config.h.in +++ b/autoconf/config.h.in @@ -580,7 +580,7 @@ /* Define to 1 if you have the `mempcpy' function. */ #undef HAVE_MEMPCPY -/* Define to 1 if you have a working `mmap' system call. */ +/* Define to 1 if you have the `mmap' function. */ #undef HAVE_MMAP /* Define to 1 if you have the header file. */ @@ -886,6 +886,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_IOCTL_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_MMAN_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MTIO_H diff --git a/autoconf/configure.in b/autoconf/configure.in index 04f094dcd56..9a77eb325d3 100644 --- a/autoconf/configure.in +++ b/autoconf/configure.in @@ -860,9 +860,10 @@ AC_HEADER_DIRENT AC_CHECK_HEADER(glob.h, [AC_DEFINE(HAVE_GLOB_H, 1, [Define to 1 if you have the header file.])] , ) AC_CHECK_HEADER(poll.h, [AC_DEFINE(HAVE_POLL_H, 1, [Define to 1 if you have the header file.])] , ) AC_CHECK_HEADER(sys/poll.h, [AC_DEFINE(HAVE_SYS_POLL_H, 1, [Define to 1 if you have the header file.])] , ) +AC_CHECK_HEADER(sys/mman.h, [AC_DEFINE(HAVE_SYS_MMAN_H, 1, [Define to 1 if you have the header file.])] , ) AC_CHECK_FUNCS(glob strcasecmp select poll setenv putenv tcgetattr) AC_CHECK_FUNCS(lstat lchown lchmod utimes lutimes futimes futimens fchmod fchown) -AC_CHECK_FUNCS(nanosleep nl_langinfo) +AC_CHECK_FUNCS(mmap nanosleep nl_langinfo) AC_CHECK_HEADERS(varargs.h) @@ -4551,14 +4552,18 @@ AC_SUBST_FILE(DEBIAN_CONTROL_STORAGE_PYTHON_PLUGIN) AC_SUBST_FILE(DEBIAN_CONTROL_DIRECTOR_PYTHON_PLUGIN) dnl build a list of storage backends we need to build. +BUILD_SD_BACKENDS="" if test x$use_libtool != xno; then - BUILD_SD_BACKENDS="libbareossd-fifo.la libbareossd-gentape.la libbareossd-tape.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-fifo.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-gentape.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-tape.la" if test X"$have_glusterfs" = "Xyes" ; then BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-gfapi.la" fi if test X"$have_droplet" = "Xyes" ; then + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-chunked.la" BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-object.la" fi @@ -4573,13 +4578,10 @@ if test x$use_libtool != xno; then if test X"$have_elasto" = "Xyes" ; then BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-elasto.la" fi -else - BUILD_SD_BACKENDS="" fi AC_SUBST(BUILD_SD_BACKENDS) - dnl Insanity check if test "x${subsysdir}" = "x${sbindir}" ; then echo " " diff --git a/configure b/configure index 2c4bffe4f59..16a13735de0 100755 --- a/configure +++ b/configure @@ -23559,6 +23559,14 @@ $as_echo "#define HAVE_SYS_POLL_H 1" >>confdefs.h fi +ac_fn_c_check_header_mongrel "$LINENO" "sys/mman.h" "ac_cv_header_sys_mman_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_mman_h" = xyes; then : + +$as_echo "#define HAVE_SYS_MMAN_H 1" >>confdefs.h + +fi + + for ac_func in glob strcasecmp select poll setenv putenv tcgetattr do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` @@ -23583,7 +23591,7 @@ _ACEOF fi done -for ac_func in nanosleep nl_langinfo +for ac_func in mmap nanosleep nl_langinfo do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -33805,14 +33813,18 @@ fi +BUILD_SD_BACKENDS="" if test x$use_libtool != xno; then - BUILD_SD_BACKENDS="libbareossd-fifo.la libbareossd-gentape.la libbareossd-tape.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-fifo.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-gentape.la" + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-tape.la" if test X"$have_glusterfs" = "Xyes" ; then BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-gfapi.la" fi if test X"$have_droplet" = "Xyes" ; then + BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-chunked.la" BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-object.la" fi @@ -33827,13 +33839,10 @@ if test x$use_libtool != xno; then if test X"$have_elasto" = "Xyes" ; then BUILD_SD_BACKENDS="${BUILD_SD_BACKENDS} libbareossd-elasto.la" fi -else - BUILD_SD_BACKENDS="" fi - if test "x${subsysdir}" = "x${sbindir}" ; then echo " " echo " " diff --git a/platforms/packaging/bareos.spec b/platforms/packaging/bareos.spec index 93566f47ea3..18a73ad1206 100644 --- a/platforms/packaging/bareos.spec +++ b/platforms/packaging/bareos.spec @@ -43,6 +43,7 @@ Vendor: The Bareos Team %define build_sqlite3 1 %define check_cmocka 1 %define glusterfs 0 +%define objectstorage 0 %define have_git 1 %define ceph 0 %define install_suse_fw 0 @@ -121,6 +122,10 @@ BuildRequires: systemd-rpm-macros %{?systemd_requires} %endif +%if 0%{?objectstorage} +BuildRequires: libdroplet-devel +%endif + %if 0%{?glusterfs} BuildRequires: glusterfs-devel glusterfs-api-devel %endif @@ -305,6 +310,15 @@ Requires(pre): shadow-utils Requires: bareos-tools %endif +%if 0%{?objectstorage} +%package storage-object +Summary: Object Storage support for the Bareos Storage daemon +Group: Productivity/Archiving/Backup +Requires: %{name}-common = %{version} +Requires: %{name}-storage = %{version} +Requires: libdroplet-common +%endif + %if 0%{?glusterfs} %package storage-glusterfs Summary: GlusterFS support for the Bareos Storage daemon @@ -565,6 +579,13 @@ This package contains the Storage Daemon This package contains the Storage Daemon tape support (Bareos service to read and write data from/to tape media) +%if 0%{?objectstorage} +%description storage-object +%{dscr} + +This package contains the Storage backend for Object Storage. +%endif + %if 0%{?glusterfs} %description storage-glusterfs %{dscr} @@ -991,6 +1012,15 @@ echo "This is a meta package to install a full bareos system" > %{buildroot}%{_d %attr(0640, %{director_daemon_user}, %{daemon_group}) %{_sysconfdir}/bareos/bareos-dir.d/storage/NULL.conf.example %attr(0640, %{storage_daemon_user}, %{daemon_group}) %{_sysconfdir}/bareos/bareos-sd.d/device/NULL.conf.example +%if 0%{?objectstorage} +%files storage-object +%defattr(-, root, root) +%{backend_dir}/libbareossd-chunked*.so +%{backend_dir}/libbareossd-object*.so +%{_sysconfdir}/bareos/bareos-dir.d/storage/Object.conf.example +%{_sysconfdir}/bareos/bareos-sd.d/device/ObjectStorage.conf.example +%endif + %if 0%{?glusterfs} %files storage-glusterfs %defattr(-, root, root) diff --git a/platforms/win32/winbareos.nsi b/platforms/win32/winbareos.nsi index ce1df2d6158..8bbb7560351 100644 --- a/platforms/win32/winbareos.nsi +++ b/platforms/win32/winbareos.nsi @@ -34,6 +34,38 @@ BrandingText "Bareos Installer" !define PRODUCT_DIR_REGKEY "Software\Microsoft\Windows\CurrentVersion\App Paths\bareos-fd.exe" !define PRODUCT_UNINST_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" !define PRODUCT_UNINST_ROOT_KEY "HKLM" +!define INSTALLER_HELP "[/S silent install]$\r$\n\ +[/SILENTKEEPCONFIG keep config on silent upgrades]$\r$\n\ +[/WRITELOGS log to INSTDIR\install.log]$\r$\n\ +[/D=installation directory (HAS TO BE THE LAST OPTION !)]$\r$\n\ +$\r$\n\ +[/CLIENTNAME=name]$\r$\n\ +[/CLIENTPASSWORD=password]$\r$\n\ +[/CLIENTADDRESS=network address]$\r$\n\ +[/CLIENTMONITORPASSWORD=password]$\r$\n\ +[/CLIENTCOMPATIBLE=compatible mode <0=no,1=yes>]$\r$\n\ +$\r$\n\ +[/DIRECTORADDRESS=network address]$\r$\n\ +[/DIRECTORNAME=name]$\r$\n\ +[/DIRECTORPASSWORD=password]$\r$\n\ +$\r$\n\ +[/INSTALLDIRECTOR]$\r$\n\ +[/DBDRIVER=database driver ]$\r$\n\ +[/DBADMINUSER=database user (only Postgres)]$\r$\n\ +[/DBADMINPASSWORD=database password (only Postgres)]$\r$\n\ +[/INSTALLWEBUI requires 'Visual C++ Redistributable for Visual Studio 2012 x86', sets /INSTALLDIRECTOR]$\r$\n\ +[/WEBUILISTENADDRESS=network address, default 127.0.0.1]$\r$\n\ +[/WEBUILISTENPORT=network port, default 9100]$\r$\n\ +[/WEBUILOGIN=login name, default admin]$\r$\n\ +[/WEBUIPASSWORD=password, default admin]$\r$\n\ +$\r$\n\ +[/INSTALLSTORAGE]$\r$\n\ +[/STORAGENAME=name]$\r$\n\ +[/STORAGEPASSWORD=password]$\r$\n\ +[/STORAGEADDRESS=network address]$\r$\n\ +[/STORAGEMONITORPASSWORD=password]$\r$\n\ +$\r$\n\ +[/? (this help dialog)]" SetCompressor lzma @@ -233,9 +265,8 @@ ${EndIf} # See http://nsis.sourceforge.net/Tutorial:_Using_labels_in_macro%27s - StrCmp $WriteLogs "yes" 0 +2 - LogEx::Init false $INSTDIR\sql.log - StrCmp $WriteLogs "yes" 0 +2 + StrCmp $WriteLogs "yes" 0 +3 + LogEx::Init false $INSTDIR\sql.log LogEx::Write "PostgresPath=$PostgresPath" @@ -1290,42 +1321,28 @@ Function .onInit ${GetParameters} $cmdLineParams ClearErrors + # + # enable logging? + # + StrCpy $WriteLogs "yes" + ${GetOptions} $cmdLineParams "/WRITELOGS" $R0 + IfErrors 0 +2 # error is set if NOT found + StrCpy $WriteLogs "no" + ClearErrors + +!If ${WIN_DEBUG} == yes + StrCpy $WriteLogs "yes" +!EndIf + + StrCmp $WriteLogs "yes" 0 +3 + LogSet on # enable nsis-own logging to $INSTDIR\install.log, needs INSTDIR defined + LogText "Logging started, INSTDIR is $INSTDIR" + # /? param (help) ClearErrors ${GetOptions} $cmdLineParams '/?' $R0 IfErrors +3 0 - MessageBox MB_OK|MB_ICONINFORMATION "[/CLIENTNAME=Name of the client ressource]$\r$\n\ - [/CLIENTPASSWORD=Password to access the client]$\r$\n\ - [/CLIENTADDRESS=Network Address of the client]$\r$\n\ - [/CLIENTMONITORPASSWORD=Password for monitor access]$\r$\n\ - [/CLIENTCOMPATIBLE=(0/1) client compatible setting (0=no,1=yes)]$\r$\n\ - $\r$\n\ - [/DIRECTORADDRESS=Network Address of the Director (for bconsole)]$\r$\n\ - [/DIRECTORNAME=Name of Director to access the client and of the Director accessed by bconsole]$\r$\n\ - [/DIRECTORPASSWORD=Password to access Director]$\r$\n\ - $\r$\n\ - [/STORAGENAME=Name of the storage ressource]$\r$\n\ - [/STORAGEPASSWORD=Password to access the storage]$\r$\n\ - [/STORAGEADDRESS=Network Address of the storage]$\r$\n\ - [/STORAGEMONITORPASSWORD=Password for monitor access]$\r$\n\ - $\r$\n\ - [/INSTALLDIRECTOR Installs Director and Components, needs postgresql installed locally! ]$\r$\n\ - [/INSTALLWEBUI Installs Bareos WebUI Components, REQUIRES Visual C++ Redistributable for Visual Studio 2012 x86, implicitly sets /INSTALLDIRECTOR]$\r$\n\ - [/WEBUILISTENADDRESS=webui listen address, default 127.0.0.1]$\r$\n\ - [/WEBUILISTENPORT=webui listen port, default 9100]$\r$\n\ - [/WEBUILOGIN=Login Name for WebUI, default admin]$\r$\n\ - [/WEBUIPASSWORD=Password for WebUI, default admin]$\r$\n\ - [/DBDRIVER=Database Driver , postgresql is default if not specified]$\r$\n\ - [/DBADMINUSER=Database Admin User (not needed for sqlite3)]$\r$\n\ - [/DBADMINPASSWORD=Database Admin Password (not needed for sqlite3)]$\r$\n\ - [/INSTALLSTORAGE Installs Storage Daemon and Components]$\r$\n\ - $\r$\n\ - [/S silent install without user interaction]$\r$\n\ - (deletes config files on uinstall, moves existing config files away and uses newly new ones)$\r$\n\ - [/SILENTKEEPCONFIG keep configuration files on silent uninstall and use existing config files during silent install]$\r$\n\ - [/D=C:\specify\installation\directory (! HAS TO BE THE LAST OPTION !)$\r$\n\ - [/? (this help dialog)] $\r$\n\ - [/WRITELOGS lets the installer create log files in INSTDIR]" + MessageBox MB_OK|MB_ICONINFORMATION "${INSTALLER_HELP}" Abort # Check if this is Windows NT. @@ -1337,6 +1354,7 @@ Function .onInit # Check if we are installing on 64Bit, then do some settings ${If} ${RunningX64} # 64Bit OS + LogText "Windows 64 bit" ${If} ${BIT_WIDTH} == '32' MessageBox MB_OK|MB_ICONSTOP "You are running a 32 Bit installer on a 64 Bit OS.$\r$\nPlease use the 64 Bit installer." Abort @@ -1348,6 +1366,7 @@ Function .onInit SetRegView 64 ${EnableX64FSRedirection} ${Else} # 32Bit OS + LogText "Windows 32 bit" ${If} ${BIT_WIDTH} == '64' MessageBox MB_OK|MB_ICONSTOP "You are running a 64 Bit installer on a 32 Bit OS.$\r$\nPlease use the 32 Bit installer." Abort @@ -1357,7 +1376,6 @@ Function .onInit !insertmacro getPostgresVars - # # UPGRADE: if already installed allow to uninstall installed version # inspired by http://nsis.sourceforge.net/Auto-uninstall_old_before_installing_new @@ -1373,18 +1391,22 @@ Function .onInit # StrCmp $R0 "" done + LogText "Prior Bareos version installed: $0" + # # As versions before 12.4.5 cannot keep the config files during silent install, we do not support upgrading here # ${VersionCompare} "12.4.5" "$0" $1 ${select} $1 ${case} 1 - MessageBox MB_OK|MB_ICONSTOP "Upgrade from version $0 is not supported.$\r$\nPlease uninstall and then install again." + MessageBox MB_OK|MB_ICONSTOP "Upgrade from version $0 is not supported.$\r$\n \ + Please uninstall and then install again." Abort ${endselect} strcpy $Upgrading "yes" ${StrRep} $INSTDIR $R0 "uninst.exe" "" # find current INSTDIR by cutting uninst.exe out of uninstall string + LogText "INSTDIR is now $INSTDIR" MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION "${PRODUCT_NAME} version $0 is already installed in $\r$\n \ '$INSTDIR' on your system.$\r$\n$\n \ @@ -1392,7 +1414,7 @@ Function .onInit `Cancel` cancels this upgrade. $\r$\n $\r$\n \ It is recommended that you make a copy of your configuration files before upgrading.$\r$\n \ " \ - /SD IDCANCEL IDOK uninst + /SD IDOK IDOK uninst Abort uninst: @@ -1403,33 +1425,35 @@ uninst: # StrCpy $6 "$0" 6 StrCmp $6 "16.2.4" config_backup_workaround - StrCmp $6 "16.2.5" config_backup_workaround no_config_backup_workaround + StrCmp $6 "16.2.5" config_backup_workaround + Goto no_config_backup_workaround config_backup_workaround: - CreateDirectory "$PLUGINSDIR\config-backup" - CopyFiles "$APPDATA\${PRODUCT_NAME}\*.conf" "$PLUGINSDIR\config-backup" + IfFileExists "$APPDATA\${PRODUCT_NAME}\*.conf" 0 no_config_backup_workaround + LogText "config_backup_workaround" + CreateDirectory "$PLUGINSDIR\config-backup" + CopyFiles "$APPDATA\${PRODUCT_NAME}\*.conf" "$PLUGINSDIR\config-backup" + ClearErrors no_config_backup_workaround: - ClearErrors - # run the uninstaller + # run the uninstaller in Silent mode. + # Keep Configuration files, Do not copy the uninstaller to a temp file. ExecWait '$R0 /S /SILENTKEEPCONFIG _?=$INSTDIR' - ;Silent Uninstall, Keep Configuration files, Do not copy the uninstaller to a temp file - IfErrors no_remove_uninstaller done no_remove_uninstaller: - MessageBox MB_OK|MB_ICONEXCLAMATION "Error during uninstall of ${PRODUCT_NAME} version $0. Aborting" FileOpen $R1 $TEMP\abortreason.txt w FileWrite $R1 "Error during uninstall of ${PRODUCT_NAME} version $0. Aborting" FileClose $R1 - abort + Abort done: - # config backup workaround: restore config files - IfFileExists "$PLUGINSDIR\config-backup\*.conf" 0 +2 + IfFileExists "$PLUGINSDIR\config-backup\*.conf" 0 +3 + LogText "restore config-backup from workaround" CopyFiles "$PLUGINSDIR\config-backup\*.conf" "$APPDATA\${PRODUCT_NAME}" + ClearErrors ${GetOptions} $cmdLineParams "/CLIENTNAME=" $ClientName ClearErrors @@ -1496,13 +1520,6 @@ done: strcmp $DbDriver "" +1 +2 StrCpy $DbDriver "postgresql" - StrCpy $WriteLogs "yes" - ${GetOptions} $cmdLineParams "/WRITELOGS" $R0 - IfErrors 0 +2 # error is set if NOT found - StrCpy $WriteLogs "no" - ClearErrors - - StrCpy $InstallDirector "yes" ${GetOptions} $cmdLineParams "/INSTALLDIRECTOR" $R0 IfErrors 0 +2 # error is set if NOT found @@ -1776,14 +1793,6 @@ ${EndIf} StrCpy $DbAdminPassword "" -!If ${WIN_DEBUG} == yes - StrCpy $WriteLogs "yes" -!EndIf - - StrCmp $WriteLogs "yes" 0 +3 - LogSet on # enable nsis-own logging to $INSTDIR\install.log, needs INSTDIR defined - LogText "Logging started, INSTDIR is $INSTDIR" - FunctionEnd diff --git a/src/cats/sql_create.c b/src/cats/sql_create.c index 84f9828ec3b..a419f94e4ec 100644 --- a/src/cats/sql_create.c +++ b/src/cats/sql_create.c @@ -869,6 +869,7 @@ bool B_DB::write_batch_file_records(JCR *jcr) } jcr->JobStatus = JobStatus; /* reset entry status */ + Jmsg0(jcr, M_INFO, 0, "Insert of attributes batch table done\n"); retval = true; diff --git a/src/filed/accurate.c b/src/filed/accurate.c index fc17754bbf5..2bab65b6c7f 100644 --- a/src/filed/accurate.c +++ b/src/filed/accurate.c @@ -224,8 +224,9 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt) * Backup only the attribute stream */ if (statc.st_mode != ff_pkt->statp.st_mode) { - Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %x File: %x\n", - fname, (uint32_t)statc.st_mode, (uint32_t)ff_pkt->statp.st_mode); + Dmsg3(dbglvl-1, "%s st_mode differ. Cat: %04o File: %04o\n", + fname, (uint32_t)(statc.st_mode & ~S_IFMT), + (uint32_t)(ff_pkt->statp.st_mode & ~S_IFMT)); status = true; } break; diff --git a/src/filed/fd_plugins.c b/src/filed/fd_plugins.c index 7dfb2a3e58a..fdab61d1979 100644 --- a/src/filed/fd_plugins.c +++ b/src/filed/fd_plugins.c @@ -718,8 +718,7 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level) } if (sp.type == 0) { - Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no type in startBackupFile packet.\n"), - cmd); + Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no type in startBackupFile packet.\n"), cmd); goto bail_out; } @@ -945,8 +944,7 @@ int plugin_estimate(JCR *jcr, FF_PKT *ff_pkt, bool top_level) } if (sp.type == 0) { - Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no type in startBackupFile packet.\n"), - cmd); + Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no type in startBackupFile packet.\n"), cmd); goto bail_out; } @@ -1852,7 +1850,7 @@ static int my_plugin_bopen(BFILE *bfd, const char *fname, int flags, mode_t mode struct io_pkt io; JCR *jcr = bfd->jcr; - Dmsg1(dbglvl, "plugin_bopen flags=%x\n", flags); + Dmsg1(dbglvl, "plugin_bopen flags=%08o\n", flags); if (!jcr->plugin_ctx) { return 0; } diff --git a/src/findlib/bfile.c b/src/findlib/bfile.c index ba9d4dd3a0b..f52fcffd701 100644 --- a/src/findlib/bfile.c +++ b/src/findlib/bfile.c @@ -760,7 +760,7 @@ static inline int bopen_nonencrypted(BFILE *bfd, const char *fname, int flags, m int bopen(BFILE *bfd, const char *fname, int flags, mode_t mode, dev_t rdev) { - Dmsg4(100, "bopen: fname %s, flags %d, mode %d, rdev %u\n", fname, flags, mode, rdev); + Dmsg4(100, "bopen: fname %s, flags %08o, mode %04o, rdev %u\n", fname, flags, (mode & ~S_IFMT), rdev); /* * If the FILE_ATTRIBUTES_DEDUPED_ITEM bit is set this is a deduped file @@ -1098,7 +1098,7 @@ bool is_restore_stream_supported(int stream) int bopen(BFILE *bfd, const char *fname, int flags, mode_t mode, dev_t rdev) { - Dmsg4(100, "bopen: fname %s, flags %d, mode %d, rdev %u\n", fname, flags, mode, rdev); + Dmsg4(100, "bopen: fname %s, flags %08o, mode %04o, rdev %u\n", fname, flags, (mode & ~S_IFMT), rdev); if (bfd->cmd_plugin && plugin_bopen) { Dmsg1(400, "call plugin_bopen fname=%s\n", fname); diff --git a/src/findlib/create_file.c b/src/findlib/create_file.c index 18efa658922..814ac6f0596 100644 --- a/src/findlib/create_file.c +++ b/src/findlib/create_file.c @@ -83,7 +83,7 @@ int create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace) } new_mode = attr->statp.st_mode; - Dmsg3(200, "type=%d newmode=%x file=%s\n", attr->type, new_mode, attr->ofname); + Dmsg3(200, "type=%d newmode=%04o file=%s\n", attr->type, (new_mode & ~S_IFMT), attr->ofname); parent_mode = S_IWUSR | S_IXUSR | new_mode; gid = attr->statp.st_gid; uid = attr->statp.st_uid; @@ -300,7 +300,7 @@ int create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace) if (is_bopen(bfd)) { Qmsg1(jcr, M_ERROR, 0, _("bpkt already open fid=%d\n"), bfd->fid); } - Dmsg2(400, "open %s flags=0x%x\n", attr->ofname, flags); + Dmsg2(400, "open %s flags=%08o\n", attr->ofname, flags); if ((bopen(bfd, attr->ofname, flags, 0, 0)) < 0) { berrno be; be.set_errno(bfd->berrno); @@ -416,7 +416,7 @@ int create_file(JCR *jcr, ATTR *attr, BFILE *bfd, int replace) */ case FT_DIRBEGIN: case FT_DIREND: - Dmsg2(200, "Make dir mode=%o dir=%s\n", new_mode, attr->ofname); + Dmsg2(200, "Make dir mode=%04o dir=%s\n", (new_mode & ~S_IFMT), attr->ofname); if (!makepath(attr, attr->ofname, new_mode, parent_mode, uid, gid, 0)) { return CF_ERROR; } diff --git a/src/include/version.h b/src/include/version.h index efff5e47280..0e3dbbc16a8 100644 --- a/src/include/version.h +++ b/src/include/version.h @@ -1,16 +1,19 @@ #undef VERSION -#define VERSION "17.2.4" -#define BDATE "21 Sep 2017" -#define LSMDATE "21Sep17" +#define VERSION "16.2.7" +#define BDATE "09 October 2017" +#define LSMDATE "09Oct17" #define PROG_COPYRIGHT "Copyright (C) %d-2012 Free Software Foundation Europe e.V.\n" \ ++ "Copyright (C) 2010-2017 Planets Communications B.V.\n" \ "Copyright (C) 2013-2017 Bareos GmbH & Co. KG\n" #define BYEAR "2017" /* year for copyright messages in programs */ /* BAREOS® - Backup Archiving REcovery Open Sourced - Copyright (C) 2000-2017 Free Software Foundation Europe e.V. + Copyright (C) 2000-2013 Free Software Foundation Europe e.V. + Copyright (C) 2010-2017 Planets Communications B.V. + Copyright (C) 2013-2017 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public diff --git a/src/lib/Makefile.in b/src/lib/Makefile.in index 84b644090a0..5e0f97290e8 100644 --- a/src/lib/Makefile.in +++ b/src/lib/Makefile.in @@ -44,11 +44,11 @@ INCLUDE_FILES = ../include/baconfig.h ../include/bareos.h \ bsock_tcp.h bsock_udt.h bsr.h btime.h btimers.h cbuf.h \ crypto.h crypto_cache.h devlock.h dlist.h fnmatch.h \ guid_to_name.h htable.h ini.h lex.h lib.h lockmgr.h \ - md5.h mem_pool.h message.h mntent_cache.h parse_conf.h \ - plugins.h protos.h queue.h rblist.h runscript.h rwlock.h \ - scsi_crypto.h scsi_lli.h scsi_tapealert.h sellist.h \ - serial.h sha1.h smartall.h status.h tls.h tree.h var.h \ - watchdog.h workq.h + md5.h mem_pool.h message.h mntent_cache.h ordered_cbuf.h \ + parse_conf.h plugins.h protos.h queue.h rblist.h \ + runscript.h rwlock.h scsi_crypto.h scsi_lli.h \ + scsi_tapealert.h sellist.h serial.h sha1.h smartall.h \ + status.h tls.h tree.h var.h waitq.h watchdog.h workq.h # # libbareos @@ -61,7 +61,7 @@ LIBBAREOS_SRCS = address_conf.c alist.c attr.c attribs.c base64.c \ crypto_cache.c crypto_gnutls.c crypto_none.c crypto_nss.c \ crypto_openssl.c crypto_wrap.c daemon.c devlock.c dlist.c \ edit.c fnmatch.c guid_to_name.c hmac.c htable.c jcr.c json.c \ - lockmgr.c md5.c mem_pool.c message.c mntent_cache.c \ + lockmgr.c md5.c mem_pool.c message.c mntent_cache.c ordered_cbuf.c \ output_formatter.c passphrase.c path_list.c plugins.c poll.c \ priv.c queue.c rblist.c runscript.c rwlock.c scan.c scsi_crypto.c \ scsi_lli.c scsi_tapealert.c sellist.c serial.c sha1.c signal.c \ diff --git a/src/lib/btimers.c b/src/lib/btimers.c index e6965c16f87..d69314c8b35 100644 --- a/src/lib/btimers.c +++ b/src/lib/btimers.c @@ -130,22 +130,25 @@ static void callback_child_timer(watchdog_t *self) */ btimer_t *start_thread_timer(JCR *jcr, pthread_t tid, uint32_t wait) { + char ed1[50]; btimer_t *wid; + wid = btimer_start_common(wait); if (wid == NULL) { Dmsg1(dbglvl, "start_thread_timer return NULL from common. wait=%d.\n", wait); return NULL; } + wid->type = TYPE_PTHREAD; wid->tid = tid; wid->jcr = jcr; - wid->wd->callback = callback_thread_timer; wid->wd->one_shot = true; wid->wd->interval = wait; register_watchdog(wid->wd); - Dmsg3(dbglvl, "Start thread timer %p tid %p for %d secs.\n", wid, tid, wait); + Dmsg3(dbglvl, "Start thread timer %p tid %s for %d secs.\n", + wid, edit_pthread(tid, ed1, sizeof(ed1)), wait); return wid; } @@ -158,14 +161,18 @@ btimer_t *start_thread_timer(JCR *jcr, pthread_t tid, uint32_t wait) */ btimer_t *start_bsock_timer(BSOCK *bsock, uint32_t wait) { + char ed1[50]; btimer_t *wid; + if (wait <= 0) { /* wait should be > 0 */ return NULL; } + wid = btimer_start_common(wait); if (wid == NULL) { return NULL; } + wid->type = TYPE_BSOCK; wid->tid = pthread_self(); wid->bsock = bsock; @@ -176,8 +183,8 @@ btimer_t *start_bsock_timer(BSOCK *bsock, uint32_t wait) wid->wd->interval = wait; register_watchdog(wid->wd); - Dmsg4(dbglvl, "Start bsock timer %p tid=%p for %d secs at %d\n", wid, - wid->tid, wait, time(NULL)); + Dmsg4(dbglvl, "Start bsock timer %p tid=%s for %d secs at %d\n", + wid, edit_pthread(wid->tid, ed1, sizeof(ed1)), wait, time(NULL)); return wid; } @@ -187,11 +194,15 @@ btimer_t *start_bsock_timer(BSOCK *bsock, uint32_t wait) */ void stop_bsock_timer(btimer_t *wid) { + char ed1[50]; + if (wid == NULL) { Dmsg0(900, "stop_bsock_timer called with NULL btimer_id\n"); return; } - Dmsg3(dbglvl, "Stop bsock timer %p tid=%p at %d.\n", wid, wid->tid, time(NULL)); + + Dmsg3(dbglvl, "Stop bsock timer %p tid=%s at %d.\n", + wid, edit_pthread(wid->tid, ed1, sizeof(ed1)), time(NULL)); stop_btimer(wid); } @@ -201,11 +212,15 @@ void stop_bsock_timer(btimer_t *wid) */ void stop_thread_timer(btimer_t *wid) { + char ed1[50]; + if (wid == NULL) { Dmsg0(dbglvl, "stop_thread_timer called with NULL btimer_id\n"); return; } - Dmsg2(dbglvl, "Stop thread timer %p tid=%p.\n", wid, wid->tid); + + Dmsg2(dbglvl, "Stop thread timer %p tid=%s.\n", + wid, edit_pthread(wid->tid, ed1, sizeof(ed1))); stop_btimer(wid); } @@ -220,12 +235,14 @@ static void destructor_thread_timer(watchdog_t *self) static void callback_thread_timer(watchdog_t *self) { + char ed1[50]; btimer_t *wid = (btimer_t *)self->data; Dmsg4(dbglvl, "thread timer %p kill %s tid=%p at %d.\n", self, - wid->type == TYPE_BSOCK ? "bsock" : "thread", wid->tid, time(NULL)); + wid->type == TYPE_BSOCK ? "bsock" : "thread", + edit_pthread(wid->tid, ed1, sizeof(ed1)), time(NULL)); if (wid->jcr) { - Dmsg2(dbglvl, "killed jid=%u Job=%s\n", wid->jcr->JobId, wid->jcr->Job); + Dmsg2(dbglvl, "killed JobId=%u Job=%s\n", wid->jcr->JobId, wid->jcr->Job); } if (wid->type == TYPE_BSOCK && wid->bsock) { diff --git a/src/lib/cbuf.c b/src/lib/cbuf.c index cf974676cce..59a14da3f14 100644 --- a/src/lib/cbuf.c +++ b/src/lib/cbuf.c @@ -31,7 +31,7 @@ /* * Initialize a new circular buffer. */ -int circbuf::init() +int circbuf::init(int capacity) { if (pthread_mutex_init(&m_lock, NULL) != 0) { return -1; @@ -51,7 +51,11 @@ int circbuf::init() m_next_in = 0; m_next_out = 0; m_size = 0; - m_capacity = QSIZE; + m_capacity = capacity; + if (m_data) { + free(m_data); + } + m_data = (void **)malloc(m_capacity * sizeof(void *)); return 0; } @@ -64,6 +68,10 @@ void circbuf::destroy() pthread_cond_destroy(&m_notempty); pthread_cond_destroy(&m_notfull); pthread_mutex_destroy(&m_lock); + if (m_data) { + free(m_data); + m_data = NULL; + } } /* @@ -86,9 +94,9 @@ int circbuf::enqueue(void *data) m_next_in %= m_capacity; /* - * Let a waiting consumer know there is data. + * Let any waiting consumer know there is data. */ - pthread_cond_signal(&m_notempty); + pthread_cond_broadcast(&m_notempty); pthread_mutex_unlock(&m_lock); @@ -100,7 +108,7 @@ int circbuf::enqueue(void *data) */ void *circbuf::dequeue() { - void *data; + void *data = NULL; if (pthread_mutex_lock(&m_lock) != 0) { return NULL; @@ -117,10 +125,7 @@ void *circbuf::dequeue() * When we are requested to flush and there is no data left return NULL. */ if (empty() && m_flush) { - m_flush = false; - pthread_mutex_unlock(&m_lock); - - return NULL; + goto bail_out; } data = m_data[m_next_out++]; @@ -128,10 +133,11 @@ void *circbuf::dequeue() m_next_out %= m_capacity; /* - * Let a waiting producer know there is room. + * Let all waiting producers know there is room. */ - pthread_cond_signal(&m_notfull); + pthread_cond_broadcast(&m_notfull); +bail_out: pthread_mutex_unlock(&m_lock); return data; @@ -169,12 +175,15 @@ int circbuf::flush() return -1; } + /* + * Set the flush flag. + */ m_flush = true; /* - * Let a waiting consumer know there will be no more data. + * Let all waiting consumers know there will be no more data. */ - pthread_cond_signal(&m_notempty); + pthread_cond_broadcast(&m_notempty); pthread_mutex_unlock(&m_lock); diff --git a/src/lib/cbuf.h b/src/lib/cbuf.h index 7680d757c55..1251530f04f 100644 --- a/src/lib/cbuf.h +++ b/src/lib/cbuf.h @@ -30,6 +30,7 @@ #define QSIZE 10 /**< # of pointers in the queue */ class circbuf : public SMARTALLOC { +private: int m_size; int m_next_in; int m_next_out; @@ -38,12 +39,12 @@ class circbuf : public SMARTALLOC { pthread_mutex_t m_lock; /**< Lock the structure */ pthread_cond_t m_notfull; /**< Full -> not full condition */ pthread_cond_t m_notempty; /**< Empty -> not empty condition */ - void *m_data[QSIZE]; /**< Circular buffer of pointers */ + void **m_data; /**< Circular buffer of pointers */ public: - circbuf(); + circbuf(int capacity = QSIZE); ~circbuf(); - int init(); + int init(int capacity); void destroy(); int enqueue(void *data); void *dequeue(); @@ -51,15 +52,17 @@ class circbuf : public SMARTALLOC { int flush(); bool full() { return m_size == m_capacity; }; bool empty() { return m_size == 0; }; + bool is_flushing() { return m_flush; }; int capacity() const { return m_capacity; }; }; /** * Constructor */ -inline circbuf::circbuf() +inline circbuf::circbuf(int capacity) { - init(); + m_data = NULL; + init(capacity); } /** diff --git a/src/lib/edit.c b/src/lib/edit.c index 3e6667d1cc3..71bcca03d91 100644 --- a/src/lib/edit.c +++ b/src/lib/edit.c @@ -29,7 +29,11 @@ #include "bareos.h" #include -/* We assume ASCII input and don't worry about overflow */ +#define DEFAULT_FORMAT_LENGTH 27 + +/* + * We assume ASCII input and don't worry about overflow + */ uint64_t str_to_uint64(const char *str) { const char *p = str; @@ -38,16 +42,20 @@ uint64_t str_to_uint64(const char *str) if (!p) { return 0; } + while (B_ISSPACE(*p)) { p++; } + if (*p == '+') { p++; } + while (B_ISDIGIT(*p)) { value = B_TIMES10(value) + *p - '0'; p++; } + return value; } @@ -60,50 +68,62 @@ int64_t str_to_int64(const char *str) if (!p) { return 0; } + while (B_ISSPACE(*p)) { p++; } + if (*p == '+') { p++; } else if (*p == '-') { negative = true; p++; } + value = str_to_uint64(p); if (negative) { value = -value; } + return value; } /* - * Edit an integer number with commas, the supplied buffer - * must be at least 27 bytes long. The incoming number - * is always widened to 64 bits. + * Edit an integer number with commas, the supplied buffer must be at least + * DEFAULT_FORMAT_LENGTH bytes long. The incoming number is always widened to 64 bits. */ char *edit_uint64_with_commas(uint64_t val, char *buf) { edit_uint64(val, buf); + return add_commas(buf, buf); } /* - * Edit an integer into "human-readable" format with four or fewer - * significant digits followed by a suffix that indicates the scale - * factor. The buf array inherits a 27 byte minimim length - * requirement from edit_unit64_with_commas(), although the output - * string is limited to eight characters. + * Edit an integer into "human-readable" format with four or fewer significant digits + * followed by a suffix that indicates the scale factor. The buf array inherits a + * DEFAULT_FORMAT_LENGTH byte minimum length requirement from edit_unit64_with_commas(), + * although the output string is limited to eight characters. */ char *edit_uint64_with_suffix(uint64_t val, char *buf) { int commas = 0; char *c, mbuf[50]; - const char *suffix[] = - { "", "K", "M", "G", "T", "P", "E", "Z", "Y", "FIX ME" }; + static const char *suffix[] = { + "", + "K", + "M", + "G", + "T", + "P", + "E", + "Z", + "Y", + "FIX ME" + }; int suffixes = sizeof(suffix) / sizeof(*suffix); edit_uint64_with_commas(val, mbuf); - if ((c = strchr(mbuf, ',')) != NULL) { commas++; *c++ = '.'; @@ -111,49 +131,53 @@ char *edit_uint64_with_suffix(uint64_t val, char *buf) commas++; *c++ = '\0'; } - mbuf[5] = '\0'; // drop this to get '123.456 TB' rather than '123.4 TB' + mbuf[5] = '\0'; /* Drop this to get '123.456 TB' rather than '123.4 TB' */ + } + + if (commas >= suffixes) { + commas = suffixes - 1; } + bsnprintf(buf, DEFAULT_FORMAT_LENGTH, "%s %s", mbuf, suffix[commas]); - if (commas >= suffixes) - commas = suffixes - 1; - bsnprintf(buf, 27, "%s %s", mbuf, suffix[commas]); return buf; } /* - * Edit an integer number, the supplied buffer - * must be at least 27 bytes long. The incoming number - * is always widened to 64 bits. + * Edit an integer number, the supplied buffer must be at least DEFAULT_FORMAT_LENGTH bytes long. + * The incoming number is always widened to 64 bits. + * Replacement for sprintf(buf, "%" llu, val) */ char *edit_uint64(uint64_t val, char *buf) { - /* - * Replacement for sprintf(buf, "%" llu, val) - */ char mbuf[50]; - mbuf[sizeof(mbuf)-1] = 0; - int i = sizeof(mbuf)-2; /* edit backward */ + mbuf[sizeof(mbuf) - 1] = 0; + int i = sizeof(mbuf) - 2; /* Edit backward */ + if (val == 0) { mbuf[i--] = '0'; } else { while (val != 0) { - mbuf[i--] = "0123456789"[val%10]; + mbuf[i--] = "0123456789"[val % 10]; val /= 10; } } - bstrncpy(buf, &mbuf[i+1], 27); + bstrncpy(buf, &mbuf[i + 1], DEFAULT_FORMAT_LENGTH); + return buf; } +/* + * Edit an integer number, the supplied buffer must be at least DEFAULT_FORMAT_LENGTH bytes long. + * The incoming number is always widened to 64 bits. + * Replacement for sprintf(buf, "%" llu, val) + */ char *edit_int64(int64_t val, char *buf) { - /* - * Replacement for sprintf(buf, "%" llu, val) - */ char mbuf[50]; bool negative = false; - mbuf[sizeof(mbuf)-1] = 0; - int i = sizeof(mbuf)-2; /* edit backward */ + mbuf[sizeof(mbuf) - 1] = 0; + int i = sizeof(mbuf) - 2; /* Edit backward */ + if (val == 0) { mbuf[i--] = '0'; } else { @@ -162,31 +186,31 @@ char *edit_int64(int64_t val, char *buf) val = -val; } while (val != 0) { - mbuf[i--] = "0123456789"[val%10]; + mbuf[i--] = "0123456789"[val % 10]; val /= 10; } } if (negative) { mbuf[i--] = '-'; } - bstrncpy(buf, &mbuf[i+1], 27); + bstrncpy(buf, &mbuf[i + 1], DEFAULT_FORMAT_LENGTH); + return buf; } /* - * Edit an integer number with commas, the supplied buffer - * must be at least 27 bytes long. The incoming number - * is always widened to 64 bits. + * Edit an integer number with commas, the supplied buffer must be at least DEFAULT_FORMAT_LENGTH + * bytes long. The incoming number is always widened to 64 bits. */ char *edit_int64_with_commas(int64_t val, char *buf) { edit_int64(val, buf); + return add_commas(buf, buf); } /* - * Given a string "str", separate the numeric part into - * str, and the modifier into mod. + * Given a string "str", separate the numeric part into str, and the modifier into mod. */ static bool get_modifier(char *str, char *num, int num_len, char *mod, int mod_len) { @@ -195,19 +219,22 @@ static bool get_modifier(char *str, char *num, int num_len, char *mod, int mod_l strip_trailing_junk(str); len = strlen(str); - for (i=0; i (num_end - num_begin + 1)) { num_len = num_end - num_begin + 1; @@ -215,29 +242,37 @@ static bool get_modifier(char *str, char *num, int num_len, char *mod, int mod_l if (num_len == 0) { return false; } - /* Eat any spaces in front of modifier */ - for ( ; i (mod_end - mod_begin + 1)) { mod_len = mod_end - mod_begin + 1; } + Dmsg5(900, "str=%s: num_beg=%d num_end=%d mod_beg=%d mod_end=%d\n", - str, num_begin, num_end, mod_begin, mod_end); + str, num_begin, num_end, mod_begin, mod_end); bstrncpy(num, &str[num_begin], num_len); bstrncpy(mod, &str[mod_begin], mod_len); + if (!is_a_number(num)) { return false; } + bstrncpy(str, &str[mod_end], len); Dmsg2(900, "num=%s mod=%s\n", num, mod); @@ -247,7 +282,7 @@ static bool get_modifier(char *str, char *num, int num_len, char *mod, int mod_l /* * Convert a string duration to utime_t (64 bit seconds) * Returns false: if error - true: if OK, and value stored in value + * true: if OK, and value stored in value */ bool duration_to_utime(char *str, utime_t *value) { @@ -289,12 +324,15 @@ bool duration_to_utime(char *str, utime_t *value) if (!get_modifier(str, num_str, sizeof(num_str), mod_str, sizeof(mod_str))) { return false; } - /* Now find the multiplier corresponding to the modifier */ + + /* + * Now find the multiplier corresponding to the modifier + */ mod_len = strlen(mod_str); if (mod_len == 0) { - i = 1; /* default to seconds */ + i = 1; /* Default to seconds */ } else { - for (i=0; mod[i]; i++) { + for (i = 0; mod[i]; i++) { if (bstrncasecmp(mod_str, mod[i], mod_len)) { break; } @@ -303,15 +341,19 @@ bool duration_to_utime(char *str, utime_t *value) return false; } } + Dmsg2(900, "str=%s: mult=%d\n", num_str, mult[i]); errno = 0; val = strtod(num_str, NULL); + if (errno != 0 || val < 0) { return false; } + total += val * mult[i]; } *value = (utime_t)total; + return true; } @@ -321,13 +363,25 @@ bool duration_to_utime(char *str, utime_t *value) char *edit_utime(utime_t val, char *buf, int buf_len) { char mybuf[200]; - static const int32_t mult[] = {60*60*24*365, 60*60*24*30, 60*60*24, 60*60, 60}; - static const char *mod[] = {"year", "month", "day", "hour", "min"}; + static const int32_t mult[] = { + 60 * 60 * 24 * 365, + 60 *60 * 24 *30, + 60 *60 * 24, + 60 * 60, + 60 + }; + static const char *mod[] = { + "year", + "month", + "day", + "hour", + "min" + }; int i; uint32_t times; *buf = 0; - for (i=0; i<5; i++) { + for (i = 0; i < 5; i++) { times = (uint32_t)(val / mult[i]); if (times > 0) { val = val - (utime_t)times * mult[i]; @@ -335,12 +389,29 @@ char *edit_utime(utime_t val, char *buf, int buf_len) bstrncat(buf, mybuf, buf_len); } } + if (val == 0 && strlen(buf) == 0) { bstrncat(buf, "0 secs", buf_len); } else if (val != 0) { bsnprintf(mybuf, sizeof(mybuf), "%d sec%s", (uint32_t)val, val>1?"s":""); bstrncat(buf, mybuf, buf_len); } + + return buf; +} + +char *edit_pthread(pthread_t val, char *buf, int buf_len) +{ + int i; + char mybuf[3]; + unsigned char *ptc = (unsigned char *)(void *)(&val); + + bstrncpy(buf, "0x", buf_len); + for (i = sizeof(val); i; --i) { + bsnprintf(mybuf, sizeof(mybuf), "%02x", (unsigned)(ptc[i])); + bstrncat(buf, mybuf, buf_len); + } + return buf; } @@ -350,23 +421,28 @@ static bool strunit_to_uint64(char *str, uint64_t *value, const char **mod) double val; char mod_str[20]; char num_str[50]; - const int64_t mult[] = {1, /* byte */ - 1024, /* kilobyte */ - 1000, /* kb kilobyte */ - 1048576, /* megabyte */ - 1000000, /* mb megabyte */ - 1073741824, /* gigabyte */ - 1000000000}; /* gb gigabyte */ + static const int64_t mult[] = { + 1, /* Byte */ + 1024, /* kiloByte */ + 1000, /* KiB KiloByte */ + 1048576, /* MegaByte */ + 1000000, /* MiB MegaByte */ + 1073741824, /* GigaByte */ + 1000000000 /* GiB GigaByte */ + }; if (!get_modifier(str, num_str, sizeof(num_str), mod_str, sizeof(mod_str))) { return 0; } - /* Now find the multiplier corresponding to the modifier */ + + /* + * Now find the multiplier corresponding to the modifier + */ mod_len = strlen(mod_str); if (mod_len == 0) { - i = 0; /* default with no modifier = 1 */ + i = 0; /* Default with no modifier = 1 */ } else { - for (i=0; mod[i]; i++) { + for (i = 0; mod[i]; i++) { if (bstrncasecmp(mod_str, mod[i], mod_len)) { break; } @@ -375,43 +451,68 @@ static bool strunit_to_uint64(char *str, uint64_t *value, const char **mod) return false; } } + Dmsg2(900, "str=%s: mult=%d\n", str, mult[i]); errno = 0; val = strtod(num_str, NULL); + if (errno != 0 || val < 0) { return false; } *value = (utime_t)(val * mult[i]); + return true; } /* * Convert a size in bytes to uint64_t * Returns false: if error - true: if OK, and value stored in value + * true: if OK, and value stored in value */ bool size_to_uint64(char *str, uint64_t *value) { - /* first item * not used */ - static const char *mod[] = {"*", "k", "kb", "m", "mb", "g", "gb", NULL}; + /* + * First item * not used + */ + static const char *mod[] = { + "*", + "k", + "kb", + "m", + "mb", + "g", + "gb", + NULL + }; + return strunit_to_uint64(str, value, mod); } /* * Convert a speed in bytes/s to uint64_t * Returns false: if error - true: if OK, and value stored in value + * true: if OK, and value stored in value */ bool speed_to_uint64(char *str, uint64_t *value) { - /* first item * not used */ - static const char *mod[] = {"*", "k/s", "kb/s", "m/s", "mb/s", NULL}; + /* + * First item * not used + */ + static const char *mod[] = { + "*", + "k/s", + "kb/s", + "m/s", + "mb/s", + NULL + }; + return strunit_to_uint64(str, value, mod); } /* * Check if specified string is a number or not. - * Taken from SQLite, cool, thanks. + * Taken from SQLite, cool, thanks. */ bool is_a_number(const char *n) { @@ -420,19 +521,23 @@ bool is_a_number(const char *n) if( *n == '-' || *n == '+' ) { n++; } + while (B_ISDIGIT(*n)) { digit_seen = true; n++; } + if (digit_seen && *n == '.') { n++; while (B_ISDIGIT(*n)) { n++; } } + if (digit_seen && (*n == 'e' || *n == 'E') && (B_ISDIGIT(n[1]) || ((n[1]=='-' || n[1] == '+') && B_ISDIGIT(n[2])))) { - n += 2; /* skip e- or e+ or e digit */ + n += 2; /* Skip e- or e+ or e digit */ while (B_ISDIGIT(*n)) { n++; } } + return digit_seen && *n==0; } @@ -443,6 +548,7 @@ bool is_a_number_list(const char *n) { bool previous_digit = false; bool digit_seen = false; + while (*n) { if (B_ISDIGIT(*n)) { previous_digit=true; @@ -454,6 +560,7 @@ bool is_a_number_list(const char *n) } n++; } + return digit_seen && *n==0; } @@ -472,8 +579,7 @@ bool is_an_integer(const char *n) /* * Check if BAREOS Resoure Name is valid - */ -/* + * * Check if the Volume name has legal characters * If ua is non-NULL send the message */ @@ -499,7 +605,7 @@ bool is_name_valid(const char *name, POOLMEM *&msg) /* * Restrict the characters permitted in the Volume name */ - for (p=name; *p; p++) { + for (p = name; *p; p++) { if (B_ISALPHA(*p) || B_ISDIGIT(*p) || strchr(accept, (int)(*p))) { continue; } @@ -508,6 +614,7 @@ bool is_name_valid(const char *name, POOLMEM *&msg) } return false; } + len = p - name; if (len >= MAX_NAME_LENGTH) { if (msg) { @@ -515,6 +622,7 @@ bool is_name_valid(const char *name, POOLMEM *&msg) } return false; } + if (len == 0) { if (msg) { Mmsg(msg, _("Volume name must be at least one character long.\n")); @@ -538,8 +646,7 @@ bool is_name_valid(const char *name) } /* - * Add commas to a string, which is presumably - * a number. + * Add commas to a string, which is presumably a number. */ char *add_commas(char *val, char *buf) { @@ -559,11 +666,11 @@ char *add_commas(char *val, char *buf) q = p + nc; *q-- = *p--; for ( ; nc; nc--) { - for (i=0; i < 3; i++) { + for (i = 0; i < 3; i++) { *q-- = *p--; } *q-- = ','; } + return buf; } - diff --git a/src/lib/jcr.c b/src/lib/jcr.c index 9dc514198ff..00bfe8c87bc 100644 --- a/src/lib/jcr.c +++ b/src/lib/jcr.c @@ -1296,7 +1296,7 @@ void dbg_jcr_add_hook(dbg_jcr_hook_t *hook) */ void dbg_print_jcr(FILE *fp) { - char buf1[128], buf2[128], buf3[128], buf4[128]; + char ed1[50], buf1[128], buf2[128], buf3[128], buf4[128]; if (!jcrs) { return; } @@ -1304,23 +1304,12 @@ void dbg_print_jcr(FILE *fp) fprintf(fp, "Attempt to dump current JCRs. njcrs=%d\n", jcrs->size()); for (JCR *jcr = (JCR *)jcrs->first(); jcr ; jcr = (JCR *)jcrs->next(jcr)) { -#ifdef HAVE_WIN32 - fprintf(fp, "threadid=%p JobId=%d JobStatus=%c jcr=%p name=%s\n", - (void *)&jcr->my_thread_id, (int)jcr->JobId, - jcr->JobStatus, jcr, jcr->Job); - fprintf(fp, "threadid=%p killable=%d JobId=%d JobStatus=%c " - "jcr=%p name=%s\n", - (void *)&jcr->my_thread_id, jcr->is_killable(), - (int)jcr->JobId, jcr->JobStatus, jcr, jcr->Job); -#else - fprintf(fp, "threadid=%p JobId=%d JobStatus=%c jcr=%p name=%s\n", - (void *)jcr->my_thread_id, (int)jcr->JobId, - jcr->JobStatus, jcr, jcr->Job); - fprintf(fp, "threadid=%p killable=%d JobId=%d JobStatus=%c " - "jcr=%p name=%s\n", - (void *)jcr->my_thread_id, jcr->is_killable(), + fprintf(fp, "threadid=%s JobId=%d JobStatus=%c jcr=%p name=%s\n", + edit_pthread(jcr->my_thread_id, ed1, sizeof(ed1)), (int)jcr->JobId, jcr->JobStatus, jcr, jcr->Job); -#endif + fprintf(fp, "threadid=%s killable=%d JobId=%d JobStatus=%c jcr=%p name=%s\n", + edit_pthread(jcr->my_thread_id, ed1, sizeof(ed1)), + jcr->is_killable(), (int)jcr->JobId, jcr->JobStatus, jcr, jcr->Job); fprintf(fp, "\tuse_count=%i\n", jcr->use_count()); fprintf(fp, "\tJobType=%c JobLevel=%c\n", jcr->getJobType(), jcr->getJobLevel()); diff --git a/src/lib/lockmgr.c b/src/lib/lockmgr.c index b6c14d40961..f5dc15a75b9 100644 --- a/src/lib/lockmgr.c +++ b/src/lib/lockmgr.c @@ -248,8 +248,6 @@ static bool contains_cycle(dlist *g) return false; } -/****************************************************************/ - class lmgr_thread_t: public SMARTALLOC { public: @@ -275,14 +273,18 @@ class lmgr_thread_t: public SMARTALLOC } void _dump(FILE *fp) { - fprintf(fp, "threadid=%p max=%i current=%i\n", - (void *)thread_id, max, current); - for(int i=0; i<=current; i++) { + char ed1[50]; + + fprintf(fp, "threadid=%s max=%i current=%i\n", + edit_pthread(thread_id, ed1, sizeof(ed1)), max, current); + + for (int i = 0; i <= current; i++) { fprintf(fp, " lock=%p state=%s priority=%i %s:%i\n", lock_list[i].lock, - (lock_list[i].state=='W')?"Wanted ":"Granted", + (lock_list[i].state=='W') ? "Wanted " : "Granted", lock_list[i].priority, - lock_list[i].file, lock_list[i].line); + lock_list[i].file, + lock_list[i].line); } } diff --git a/src/lib/ordered_cbuf.c b/src/lib/ordered_cbuf.c new file mode 100644 index 00000000000..9b76300b533 --- /dev/null +++ b/src/lib/ordered_cbuf.c @@ -0,0 +1,409 @@ +/* + BAREOS® - Backup Archiving REcovery Open Sourced + + Copyright (C) 2016-2017 Planets Communications B.V. + + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation and included + in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. +*/ +/* + * Marco van Wieringen, December 2016. + */ + +/* + * Ordered Circular buffer used for producer/consumer problem with pthreads. + */ +#include "bareos.h" +#include "ordered_cbuf.h" + +/* + * Initialize a new ordered circular buffer. + */ +int ordered_circbuf::init(int capacity) +{ + struct ocbuf_item *item = NULL; + + if (pthread_mutex_init(&m_lock, NULL) != 0) { + return -1; + } + + if (pthread_cond_init(&m_notfull, NULL) != 0) { + pthread_mutex_destroy(&m_lock); + return -1; + } + + if (pthread_cond_init(&m_notempty, NULL) != 0) { + pthread_cond_destroy(&m_notfull); + pthread_mutex_destroy(&m_lock); + return -1; + } + + m_size = 0; + m_capacity = capacity; + m_reserved = 0; + if (m_data) { + m_data->destroy(); + delete m_data; + } + m_data = New(dlist(item, &item->link)); + + return 0; +} + +/* + * Destroy a ordered circular buffer. + */ +void ordered_circbuf::destroy() +{ + pthread_cond_destroy(&m_notempty); + pthread_cond_destroy(&m_notfull); + pthread_mutex_destroy(&m_lock); + if (m_data) { + m_data->destroy(); + delete m_data; + } +} + +/* + * Enqueue a new item into the ordered circular buffer. + */ +void *ordered_circbuf::enqueue(void *data, + uint32_t data_size, + int compare(void *item1, void *item2), + void update(void *item1, void *item2), + bool use_reserved_slot, + bool no_signal) +{ + struct ocbuf_item *new_item, *item; + + if (pthread_mutex_lock(&m_lock) != 0) { + return NULL; + } + + /* + * See if we should use a reserved slot and there are actually slots reserved. + */ + if (!use_reserved_slot || !m_reserved) { + /* + * Wait while the buffer is full. + */ + while (full()) { + pthread_cond_wait(&m_notfull, &m_lock); + } + } + + /* + * Decrease the number of reserved slots if we should use a reserved slot. + * We do this even when we don't really add a new item to the ordered + * circular list to keep the reserved slot counting consistent. + */ + if (use_reserved_slot) { + m_reserved--; + } + + /* + * Binary insert the data into the ordered circular buffer. If the + * item returned is not our new_item it means there is already an + * entry with the same keys on the ordered circular list. We then + * just call the update function callback which should perform the + * right actions to update the already existing item with the new + * data in the new item. The compare function callback is used to binary + * insert the item at the right location in the ordered circular list. + */ + new_item = (struct ocbuf_item *)malloc(sizeof(struct ocbuf_item)); + new_item->data = data; + new_item->data_size = data_size; + + item = (struct ocbuf_item *)m_data->binary_insert(new_item, compare); + if (item == new_item) { + m_size++; + } else { + /* + * Update the data on the ordered circular list with the new data. + * e.g. replace the old with the new data but don't allocate a new + * item on the ordered circular list. + */ + update(item, new_item); + + /* + * Release the unused ocbuf_item. + */ + free(new_item); + + /* + * Update data to point to the data that was attached to the original ocbuf_item. + */ + data = item->data; + } + + /* + * See if we need to signal any workers that work is available or not. + */ + if (!no_signal) { + /* + * Let any waiting consumer know there is data. + */ + pthread_cond_broadcast(&m_notempty); + } + + pthread_mutex_unlock(&m_lock); + + /* + * Return the data that is current e.g. either the new data passed in or + * the already existing data on the ordered circular list. + */ + return data; +} + +/* + * Dequeue an item from the ordered circular buffer. + */ +void *ordered_circbuf::dequeue(bool reserve_slot, + bool requeued, + struct timespec *ts, + int timeout) +{ + void *data = NULL; + struct ocbuf_item *item; + + if (pthread_mutex_lock(&m_lock) != 0) { + return NULL; + } + + /* + * Wait while there is nothing in the buffer + */ + while ((requeued || empty()) && !m_flush) { + /* + * The requeued state is only valid one time so clear it. + */ + requeued = false; + + /* + * See if we should block indefinitely or wake up + * after the given timer has expired and calculate + * the next time we need to wakeup. This way we check + * after the timer expired if there is work to be done + * this is something we need if the worker threads can + * put work back onto the circular queue and uses + * enqueue with the no_signal flag set. + */ + if (ts) { + pthread_cond_timedwait(&m_notempty, &m_lock, ts); + + /* + * See if there is really work to be done. + * We could be woken by the broadcast but some other iothread + * could take the work as we have to wait to reacquire the m_lock. + * Only one thread will be in the critical section and be able to + * hold the lock. + */ + if (empty() && !m_flush) { + struct timeval tv; + struct timezone tz; + + /* + * Calculate the next absolute timeout if we find + * out there is no work to be done. + */ + gettimeofday(&tv, &tz); + ts->tv_nsec = tv.tv_usec * 1000; + ts->tv_sec = tv.tv_sec + timeout; + + continue; + } + } else { + pthread_cond_wait(&m_notempty, &m_lock); + + /* + * See if there is really work to be done. + * We could be woken by the broadcast but some other iothread + * could take the work as we have to wait to reacquire the m_lock. + * Only one thread will be in the critical section and be able to + * hold the lock. + */ + if (empty() && !m_flush) { + continue; + } + } + } + + /* + * When we are requested to flush and there is no data left return NULL. + */ + if (empty() && m_flush) { + goto bail_out; + } + + /* + * Get the first item from the dlist and remove it. + */ + item = (struct ocbuf_item *)m_data->first(); + if (!item) { + goto bail_out; + } + + m_data->remove(item); + m_size--; + + /* + * Let all waiting producers know there is room. + */ + pthread_cond_broadcast(&m_notfull); + + /* + * Extract the payload and drop the placeholder. + */ + data = item->data; + free(item); + + /* + * Increase the reserved slot count when we are asked to reserve the slot. + */ + if (reserve_slot) { + m_reserved++; + } + +bail_out: + pthread_mutex_unlock(&m_lock); + + return data; +} + +/* + * Peek on the buffer for a certain item. + * We return a copy of the data on the ordered circular buffer. + * Any pointers in that data may become invallid after its returned + * to the calling function. As such you should not rely on the data. + */ +void *ordered_circbuf::peek(enum oc_peek_types type, + void *data, + int callback(void *item1, void *item2)) +{ + void *retval = NULL; + struct ocbuf_item *item; + + if (pthread_mutex_lock(&m_lock) != 0) { + return NULL; + } + + /* + * There is nothing to be seen on an empty ordered circular buffer. + */ + if (empty()) { + goto bail_out; + } + + /* + * Depending on the peek type start somewhere on the ordered list and + * walk forward or back. + */ + switch (type) { + case PEEK_FIRST: + item = (struct ocbuf_item *)m_data->first(); + while (item) { + if (callback(item->data, data) == 0) { + retval = malloc(item->data_size); + memcpy(retval, item->data, item->data_size); + goto bail_out; + } + + item = (struct ocbuf_item *)m_data->next(item); + } + break; + case PEEK_LAST: + item = (struct ocbuf_item *)m_data->last(); + while (item) { + if (callback(item->data, data) == 0) { + retval = malloc(item->data_size); + memcpy(retval, item->data, item->data_size); + goto bail_out; + } + + item = (struct ocbuf_item *)m_data->prev(item); + } + break; + case PEEK_LIST: + item = (struct ocbuf_item *)m_data->first(); + while (item) { + callback(item->data, data); + item = (struct ocbuf_item *)m_data->next(item); + } + break; + default: + goto bail_out; + } + +bail_out: + pthread_mutex_unlock(&m_lock); + + return retval; +} + +/* + * Unreserve a slot which was reserved by dequeue(). + */ +int ordered_circbuf::unreserve_slot() +{ + int retval = -1; + + if (pthread_mutex_lock(&m_lock) != 0) { + goto bail_out; + } + + /* + * Make sure any slots are still reserved. Otherwise people + * are playing games and should pay the price for doing so. + */ + if (m_reserved) { + m_reserved--; + + /* + * Let all waiting producers know there is room. + */ + pthread_cond_broadcast(&m_notfull); + + retval = 0; + } + pthread_mutex_unlock(&m_lock); + +bail_out: + return retval; +} + +/* + * Flush the ordered circular buffer. + * Any waiting consumer will be wakened and will see we are in flush state. + */ +int ordered_circbuf::flush() +{ + if (pthread_mutex_lock(&m_lock) != 0) { + return -1; + } + + /* + * Set the flush flag. + */ + m_flush = true; + + /* + * Let all waiting consumers know there will be no more data. + */ + pthread_cond_broadcast(&m_notempty); + + pthread_mutex_unlock(&m_lock); + + return 0; +} diff --git a/src/lib/ordered_cbuf.h b/src/lib/ordered_cbuf.h new file mode 100644 index 00000000000..3279c14a774 --- /dev/null +++ b/src/lib/ordered_cbuf.h @@ -0,0 +1,94 @@ +/* + BAREOS® - Backup Archiving REcovery Open Sourced + + Copyright (C) 2016-2017 Planets Communications B.V. + + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation and included + in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. +*/ +/* + * Marco van Wieringen, December 2016. + */ + +/* + * Ordered Circular buffer used for producer/consumer problem with pthread. + */ + +#define OQSIZE 10 /* # of pointers in the queue */ + +enum oc_peek_types { + PEEK_FIRST = 0, + PEEK_LAST, + PEEK_LIST +}; + +struct ocbuf_item { + dlink link; + uint32_t data_size; + void *data; +}; + +class ordered_circbuf : public SMARTALLOC { +private: + int m_size; + int m_capacity; + int m_reserved; + bool m_flush; + pthread_mutex_t m_lock; /* Lock the structure */ + pthread_cond_t m_notfull; /* Full -> not full condition */ + pthread_cond_t m_notempty; /* Empty -> not empty condition */ + dlist *m_data; /* Circular buffer of pointers */ + +public: + ordered_circbuf(int capacity = OQSIZE); + ~ordered_circbuf(); + int init(int capacity); + void destroy(); + void *enqueue(void *data, + uint32_t data_size, + int compare(void *item1, void *item2), + void update(void *item1, void *item2), + bool use_reserved_slot = false, + bool no_signal = false); + void *dequeue(bool reserve_slot = false, + bool requeued = false, + struct timespec *ts = NULL, + int timeout = 300); + void *peek(enum oc_peek_types type, + void *data, + int callback(void *item1, void *item2)); + int unreserve_slot(); + int flush(); + bool full() { return m_size == (m_capacity - m_reserved); }; + bool empty() { return m_size == 0; }; + bool is_flushing() { return m_flush; }; + int capacity() const { return m_capacity; }; +}; + +/* + * Constructor + */ +inline ordered_circbuf::ordered_circbuf(int capacity) +{ + init(capacity); +} + +/* + * Destructor + */ +inline ordered_circbuf::~ordered_circbuf() +{ + destroy(); +} diff --git a/src/lib/protos.h b/src/lib/protos.h index e9e3d044c4a..e18aa04bc91 100644 --- a/src/lib/protos.h +++ b/src/lib/protos.h @@ -235,6 +235,7 @@ bool duration_to_utime(char *str, utime_t *value); bool size_to_uint64(char *str, uint64_t *value); bool speed_to_uint64(char *str, uint64_t *value); char *edit_utime(utime_t val, char *buf, int buf_len); +char *edit_pthread(pthread_t val, char *buf, int buf_len); bool is_a_number(const char *num); bool is_a_number_list(const char *n); bool is_an_integer(const char *n); diff --git a/src/lib/res.c b/src/lib/res.c index c9c53a66d25..7d853eb4738 100644 --- a/src/lib/res.c +++ b/src/lib/res.c @@ -43,7 +43,7 @@ extern CONFIG *my_config; /* Our Global config */ * Define the Union of all the common resource structure definitions. */ union URES { - MSGSRES res_msgs; + MSGSRES res_msgs; RES hdr; }; @@ -56,18 +56,23 @@ void b_LockRes(const char *file, int line) int errstat; #ifdef TRACE_RES + char ed1[50]; + Pmsg4(000, "LockRes locked=%d w_active=%d at %s:%d\n", res_locked, my_config->m_res_lock.w_active, file, line); - if (res_locked) { - Pmsg2(000, "LockRes writerid=%lu myid=%lu\n", - my_config->m_res_lock.writer_id, pthread_self()); - } + if (res_locked) { + Pmsg2(000, "LockRes writerid=%lu myid=%s\n", + my_config->m_res_lock.writer_id, + edit_pthread(pthread_self(), ed1, sizeof(ed1))); + } #endif + if ((errstat = rwl_writelock(&my_config->m_res_lock)) != 0) { Emsg3(M_ABORT, 0, _("rwl_writelock failure at %s:%d: ERR=%s\n"), file, line, strerror(errstat)); } + res_locked++; } diff --git a/src/plugins/filed/cephfs-fd.c b/src/plugins/filed/cephfs-fd.c index 81aa038c103..5805786cb3e 100644 --- a/src/plugins/filed/cephfs-fd.c +++ b/src/plugins/filed/cephfs-fd.c @@ -123,6 +123,7 @@ struct plugin_ctx { int32_t backup_level; /* Backup level e.g. Full/Differential/Incremental */ utime_t since; /* Since time for Differential/Incremental */ char *plugin_options; /* Options passed to plugin */ + char *plugin_definition; /* Previous plugin definition passed to plugin */ char *conffile; /* Configfile to read to be able to connect to CEPHFS */ char *basedir; /* Basedir to start backup in */ char flags[FOPTS_BYTES]; /* Bareos internal flags */ @@ -308,6 +309,10 @@ static bRC freePlugin(bpContext *ctx) free(p_ctx->conffile); } + if (p_ctx->plugin_definition) { + free(p_ctx->plugin_definition); + } + if (p_ctx->plugin_options) { free(p_ctx->plugin_options); } @@ -826,6 +831,22 @@ static bRC parse_plugin_definition(bpContext *ctx, void *value) return bRC_Error; } + /* + * See if we already got some plugin definition before and its exactly the same. + */ + if (p_ctx->plugin_definition) { + if (bstrcmp(p_ctx->plugin_definition, (char *)value)) { + return bRC_OK; + } + + free(p_ctx->plugin_definition); + } + + /* + * Keep track of the last processed plugin definition. + */ + p_ctx->plugin_definition = bstrdup((char *)value); + keep_existing = (p_ctx->plugin_options) ? true : false; /* @@ -941,6 +962,14 @@ static bRC connect_to_cephfs(bpContext *ctx) int status; plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext; + /* + * If we get called and we already have a handle to cephfs we should tear it down. + */ + if (p_ctx->cmount) { + ceph_shutdown(p_ctx->cmount); + p_ctx->cmount = NULL; + } + status = ceph_create(&p_ctx->cmount, NULL); if (status < 0) { berrno be; @@ -983,6 +1012,16 @@ static bRC setup_backup(bpContext *ctx, void *value) return bRC_Error; } + /* + * If we are already having a handle to cepfs and we are getting the + * same plugin definition there is no need to tear down the whole stuff and + * setup exactly the same. + */ + if (p_ctx->cmount && + bstrcmp((char *)value, p_ctx->plugin_definition)) { + return bRC_OK; + } + if (connect_to_cephfs(ctx) != bRC_OK) { return bRC_Error; } @@ -1013,11 +1052,17 @@ static bRC setup_restore(bpContext *ctx, void *value) return bRC_Error; } - if (connect_to_cephfs(ctx) != bRC_OK) { - return bRC_Error; + /* + * If we are already having a handle to cepfs and we are getting the + * same plugin definition there is no need to tear down the whole stuff and + * setup exactly the same. + */ + if (p_ctx->cmount && + bstrcmp((char *)value, p_ctx->plugin_definition)) { + return bRC_OK; } - return bRC_OK; + return connect_to_cephfs(ctx); } /** diff --git a/src/plugins/filed/gfapi-fd.c b/src/plugins/filed/gfapi-fd.c index d424c145a59..a10588937b2 100644 --- a/src/plugins/filed/gfapi-fd.c +++ b/src/plugins/filed/gfapi-fd.c @@ -1,7 +1,7 @@ /* BAREOS® - Backup Archiving REcovery Open Sourced - Copyright (C) 2014-2016 Planets Communications B.V. + Copyright (C) 2014-2017 Planets Communications B.V. Copyright (C) 2014-2016 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or @@ -122,6 +122,7 @@ struct plugin_ctx { int32_t backup_level; /* Backup level e.g. Full/Differential/Incremental */ utime_t since; /* Since time for Differential/Incremental */ char *plugin_options; /* Options passed to plugin */ + char *plugin_definition; /* Previous plugin definition passed to plugin */ char *gfapi_volume_spec; /* Unparsed Gluster volume specification */ char *transport; /* Gluster transport protocol to management server */ char *servername; /* Gluster management server */ @@ -136,6 +137,7 @@ struct plugin_ctx { char *next_xattr_name; /* Next xattr name to process */ bool crawl_fs; /* Use local fs crawler to find files to backup */ char *gf_file_list; /* File with list of files generated by glusterfind to backup */ + bool is_accurate; /* Backup has accurate option enabled */ POOLMEM *cwd; /* Current Working Directory */ POOLMEM *next_filename; /* Next filename to save */ POOLMEM *link_target; /* Target symlink points to */ @@ -158,6 +160,7 @@ enum plugin_argument_type { argument_none, argument_volume_spec, argument_snapdir, + argument_basedir, argument_gf_file_list }; @@ -169,6 +172,7 @@ struct plugin_argument { static plugin_argument plugin_arguments[] = { { "volume", argument_volume_spec }, { "snapdir", argument_snapdir }, + { "basedir", argument_basedir }, { "gffilelist", argument_gf_file_list }, { NULL, argument_none } }; @@ -439,6 +443,10 @@ static bRC freePlugin(bpContext *ctx) free(p_ctx->gfapi_volume_spec); } + if (p_ctx->plugin_definition) { + free(p_ctx->plugin_definition); + } + if (p_ctx->plugin_options) { free(p_ctx->plugin_options); } @@ -654,8 +662,10 @@ static bRC get_next_file_to_backup(bpContext *ctx) continue; } *bp++ = '\0'; - urllib_unquote_plus(p_ctx->next_filename); - bfuncs->ClearSeenBitmap(ctx, false, p_ctx->next_filename); + if (p_ctx->is_accurate) { + urllib_unquote_plus(p_ctx->next_filename); + bfuncs->ClearSeenBitmap(ctx, false, p_ctx->next_filename); + } bstrinlinecpy(p_ctx->next_filename, bp); urllib_unquote_plus(p_ctx->next_filename); break; @@ -663,9 +673,11 @@ static bRC get_next_file_to_backup(bpContext *ctx) /* * DELETE means we clear the seen bitmap for this file and continue. */ - bstrinlinecpy(p_ctx->next_filename, p_ctx->next_filename + gf_mapping->compare_size); - urllib_unquote_plus(p_ctx->next_filename); - bfuncs->ClearSeenBitmap(ctx, false, p_ctx->next_filename); + if (p_ctx->is_accurate) { + bstrinlinecpy(p_ctx->next_filename, p_ctx->next_filename + gf_mapping->compare_size); + urllib_unquote_plus(p_ctx->next_filename); + bfuncs->ClearSeenBitmap(ctx, false, p_ctx->next_filename); + } continue; default: Jmsg(ctx, M_ERROR, "Unrecognized glusterfind entry %s\n", p_ctx->next_filename); @@ -1056,6 +1068,25 @@ static bRC parse_plugin_definition(bpContext *ctx, void *value) return bRC_Error; } + /* + * See if we already got some plugin definition before and its exactly the same. + */ + if (p_ctx->plugin_definition) { + if (bstrcmp(p_ctx->plugin_definition, (char *)value)) { + return bRC_OK; + } + + free(p_ctx->plugin_definition); + } + + /* + * Keep track of the last processed plugin definition. + */ + p_ctx->plugin_definition = bstrdup((char *)value); + + /* + * Keep overrides passed in via pluginoptions. + */ keep_existing = (p_ctx->plugin_options) ? true : false; /* @@ -1123,6 +1154,9 @@ static bRC parse_plugin_definition(bpContext *ctx, void *value) case argument_snapdir: str_destination = &p_ctx->snapdir; break; + case argument_basedir: + str_destination = &p_ctx->basedir; + break; case argument_gf_file_list: str_destination = &p_ctx->gf_file_list; break; @@ -1321,7 +1355,7 @@ static inline bool parse_gfapi_devicename(char *devicename, /* * Validate URI. */ - if (!bp || !*bp == '/') { + if (!bp || *(bp + 1) != '/') { goto bail_out; } @@ -1458,6 +1492,14 @@ static bRC connect_to_gluster(bpContext *ctx, bool is_backup) return bRC_Error; } + /* + * If we get called and we already have a handle to gfapi we should tear it down. + */ + if (p_ctx->glfs) { + glfs_fini(p_ctx->glfs); + p_ctx->glfs = NULL; + } + p_ctx->glfs = glfs_new(p_ctx->volumename); if (!p_ctx->glfs) { goto bail_out; @@ -1504,12 +1546,23 @@ static bRC connect_to_gluster(bpContext *ctx, bool is_backup) */ static bRC setup_backup(bpContext *ctx, void *value) { + bRC retval = bRC_Error; plugin_ctx *p_ctx = (plugin_ctx *)ctx->pContext; if (!p_ctx || !value) { goto bail_out; } + /* + * If we are already having a handle to gfapi and we are getting the + * same plugin definition there is no need to tear down the whole stuff and + * setup exactly the same. + */ + if (p_ctx->glfs && + bstrcmp((char *)value, p_ctx->plugin_definition)) { + return bRC_OK; + } + if (connect_to_gluster(ctx, true) != bRC_OK) { goto bail_out; } @@ -1518,6 +1571,16 @@ static bRC setup_backup(bpContext *ctx, void *value) * See if we use an external list with files to backup or should crawl the filesystem ourself. */ if (p_ctx->gf_file_list) { + int accurate; + + /* + * Get the setting for accurate for this Job. + */ + bfuncs->getBareosValue(ctx, bVarAccurate, (void *)&accurate); + if (accurate) { + p_ctx->is_accurate = true; + } + p_ctx->crawl_fs = false; if ((p_ctx->file_list_handle = fopen(p_ctx->gf_file_list, "r")) == (FILE *)NULL) { Jmsg(ctx, M_FATAL, "Failed to open %s for reading files to backup\n", p_ctx->gf_file_list); @@ -1525,21 +1588,23 @@ static bRC setup_backup(bpContext *ctx, void *value) goto bail_out; } - /* - * Mark all files as seen from the previous backup when this is a incremental or differential backup. - * The entries we get from glusterfind are only the changes since that earlier backup. - */ - switch (p_ctx->backup_level) { - case L_INCREMENTAL: - case L_DIFFERENTIAL: - if (bfuncs->SetSeenBitmap(ctx, true, NULL) != bRC_OK) { - Jmsg(ctx, M_FATAL, "Failed to enable all entries in Seen bitmap, not an accurate backup ?\n"); - Dmsg(ctx, dbglvl, "Failed to enable all entries in Seen bitmap, not an accurate backup ?\n"); - goto bail_out; + if (p_ctx->is_accurate) { + /* + * Mark all files as seen from the previous backup when this is a incremental or differential backup. + * The entries we get from glusterfind are only the changes since that earlier backup. + */ + switch (p_ctx->backup_level) { + case L_INCREMENTAL: + case L_DIFFERENTIAL: + if (bfuncs->SetSeenBitmap(ctx, true, NULL) != bRC_OK) { + Jmsg(ctx, M_FATAL, "Failed to enable all entries in Seen bitmap, not an accurate backup ?\n"); + Dmsg(ctx, dbglvl, "Failed to enable all entries in Seen bitmap, not an accurate backup ?\n"); + goto bail_out; + } + break; + default: + break; } - break; - default: - break; } /* @@ -1547,9 +1612,24 @@ static bRC setup_backup(bpContext *ctx, void *value) * As we need to get it from the gfflilelist we use get_next_file_to_backup() * to do the setup for us it retrieves the entry and does a setup of filetype etc. */ - if (get_next_file_to_backup(ctx) == bRC_Error) { + switch (get_next_file_to_backup(ctx)) { + case bRC_OK: + /* + * get_next_file_to_backup() normally returns bRC_More to indicate that there are + * more files to backup. But when using glusterfind we use an external filelist which + * could be empty in that special case we get bRC_OK back from get_next_file_to_backup() + * and then only in setup_backup() we return bRC_Skip which will skip processing of any + * more files to backup. + */ + retval = bRC_Skip; + break; + case bRC_Error: Jmsg(ctx, M_FATAL, "Failed to get first file to backup\n"); Dmsg(ctx, dbglvl, "Failed to get first file to backup\n"); + goto bail_out; + default: + retval = bRC_OK; + break; } } else { p_ctx->crawl_fs = true; @@ -1589,12 +1669,12 @@ static bRC setup_backup(bpContext *ctx, void *value) } else { pm_strcpy(p_ctx->next_filename, "/"); } - } - return bRC_OK; + retval = bRC_OK; + } bail_out: - return bRC_Error; + return retval; } /** @@ -1608,11 +1688,17 @@ static bRC setup_restore(bpContext *ctx, void *value) return bRC_Error; } - if (connect_to_gluster(ctx, false) != bRC_OK) { - return bRC_Error; + /* + * If we are already having a handle to gfapi and we are getting the + * same plugin definition there is no need to tear down the whole stuff and + * setup exactly the same. + */ + if (p_ctx->glfs && + bstrcmp((char *)value, p_ctx->plugin_definition)) { + return bRC_OK; } - return bRC_OK; + return connect_to_gluster(ctx, false); } /** @@ -1632,6 +1718,13 @@ static bRC pluginIO(bpContext *ctx, struct io_pkt *io) switch(io->func) { case IO_OPEN: + /* + * Close the gfd when it was not closed before. + */ + if (p_ctx->gfd) { + glfs_close(p_ctx->gfd); + } + if (io->flags & (O_CREAT | O_WRONLY)) { p_ctx->gfd = glfs_creat(p_ctx->glfs, io->fname, io->flags, io->mode); } else { diff --git a/src/plugins/filed/python-fd.c b/src/plugins/filed/python-fd.c index f64586a23cf..b87ab2975d1 100644 --- a/src/plugins/filed/python-fd.c +++ b/src/plugins/filed/python-fd.c @@ -3332,10 +3332,10 @@ static PyObject *PyStatPacket_repr(PyStatPacket *self) PyObject *s; POOL_MEM buf(PM_MESSAGE); - Mmsg(buf, "StatPacket(dev=%ld, ino=%lld, mode=%d, nlink=%d, " + Mmsg(buf, "StatPacket(dev=%ld, ino=%lld, mode=%04o, nlink=%d, " "uid=%ld, gid=%ld, rdev=%ld, size=%lld, " "atime=%ld, mtime=%ld, ctime=%ld, blksize=%ld, blocks=%lld)", - self->dev, self->ino, self->mode, self->nlink, + self->dev, self->ino, (self->mode & ~S_IFMT), self->nlink, self->uid, self->gid, self->rdev, self->size, self->atime, self->mtime, self->ctime, self->blksize, self->blocks); @@ -3660,10 +3660,10 @@ static PyObject *PyIoPacket_repr(PyIoPacket *self) PyObject *s; POOL_MEM buf(PM_MESSAGE); - Mmsg(buf, "IoPacket(func=%d, count=%ld, flags=%ld, mode=%ld, " + Mmsg(buf, "IoPacket(func=%d, count=%ld, flags=%ld, mode=%04o, " "buf=\"%s\", fname=\"%s\", status=%ld, io_errno=%ld, lerror=%ld, " "whence=%ld, offset=%lld, win32=%d)", - self->func, self->count, self->flags, self->mode, + self->func, self->count, self->flags, (self->mode & ~S_IFMT), PyGetByteArrayValue(self->buf), self->fname, self->status, self->io_errno, self->lerror, self->whence, self->offset, self->win32); diff --git a/src/stored/Makefile.in b/src/stored/Makefile.in index b94404dec53..7956c7cc4ba 100644 --- a/src/stored/Makefile.in +++ b/src/stored/Makefile.in @@ -28,6 +28,7 @@ first_rule: all dummy: AVAILABLE_DEVICE_API_SRCS = cephfs_device.c \ + chunked_device.c \ elasto_device.c \ gfapi_device.c \ object_store_device.c \ diff --git a/src/stored/backends/Makefile.in b/src/stored/backends/Makefile.in index 9f5f6084308..f0c6c6ba5bc 100644 --- a/src/stored/backends/Makefile.in +++ b/src/stored/backends/Makefile.in @@ -32,6 +32,9 @@ RADOS_LIBS = @RADOS_STRIPER_LIBS@ @RADOS_LIBS@ CHEPHFS_SRCS = cephfs_device.c CHEPHFS_LOBJS = $(CHEPHFS_SRCS:.c=.lo) +CHUNKED_SRCS = chunked_device.c +CHUNKED_LOBJS = $(CHUNKED_SRCS:.c=.lo) + ELASTO_SRCS = elasto_device.c ELASTO_LOBJS = $(ELASTO_SRCS:.c=.lo) @@ -80,6 +83,7 @@ STORED_RESTYPES = autochanger device director ndmp messages storage $(ELASTO_LOBJS): @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(ELASTO_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) + if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi $(CHEPHFS_LOBJS): @echo "Compiling $(@:.lo=.c)" @@ -94,6 +98,7 @@ $(GFAPI_LOBJS): $(OBJECT_LOBJS): @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(DROPLET_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) + if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi $(RADOS_LOBJS): @echo "Compiling $(@:.lo=.c)" @@ -110,6 +115,11 @@ libbareossd-cephfs.la: Makefile $(CHEPHFS_LOBJS) $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -L../../lib -o $@ $(CHEPHFS_LOBJS) -export-dynamic -rpath $(backenddir) -release $(LIBBAREOSSD_LT_RELEASE) \ -soname libbareossd-cephfs-$(LIBBAREOSSD_LT_RELEASE).so $(CEPHFS_LIBS) -lbareos +libbareossd-chunked.la: Makefile $(CHUNKED_LOBJS) + @echo "Making $@ ..." + $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -L../../lib -o $@ $(CHUNKED_LOBJS) -export-dynamic -rpath $(backenddir) -release $(LIBBAREOSSD_LT_RELEASE) \ + -soname libbareossd-chunked-$(LIBBAREOSSD_LT_RELEASE).so -lbareos + libbareossd-elasto.la: Makefile $(ELASTO_LOBJS) @echo "Making $@ ..." $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -L../../lib -o $@ $(ELASTO_LOBJS) -export-dynamic -rpath $(backenddir) -release $(LIBBAREOSSD_LT_RELEASE) \ @@ -120,10 +130,10 @@ libbareossd-gfapi.la: Makefile $(GFAPI_LOBJS) $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -L../../lib -o $@ $(GFAPI_LOBJS) -export-dynamic -rpath $(backenddir) -release $(LIBBAREOSSD_LT_RELEASE) \ -soname libbareossd-gfapi-$(LIBBAREOSSD_LT_RELEASE).so $(GLUSTER_LIBS) -lbareos -libbareossd-object.la: Makefile $(OBJECT_LOBJS) +libbareossd-object.la: Makefile libbareossd-chunked.la $(OBJECT_LOBJS) @echo "Making $@ ..." $(LIBTOOL_LINK) $(CXX) $(DEFS) $(DEBUG) $(LDFLAGS) -L../../lib -o $@ $(OBJECT_LOBJS) -export-dynamic -rpath $(backenddir) -release $(LIBBAREOSSD_LT_RELEASE) \ - -soname libbareossd-object-$(LIBBAREOSSD_LT_RELEASE).so $(DROPLET_LIBS) -lbareos + -soname libbareossd-object-$(LIBBAREOSSD_LT_RELEASE).so $(DROPLET_LIBS) libbareossd-chunked.la -lbareos libbareossd-rados.la: Makefile $(RADOS_LOBJS) @echo "Making $@ ..." diff --git a/src/stored/backends/chunked_device.c b/src/stored/backends/chunked_device.c new file mode 100644 index 00000000000..0f256861bc4 --- /dev/null +++ b/src/stored/backends/chunked_device.c @@ -0,0 +1,1095 @@ +/* + BAREOS® - Backup Archiving REcovery Open Sourced + + Copyright (C) 2015-2017 Planets Communications B.V. + + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation and included + in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. +*/ +/* + * Chunked volume device abstraction. + * + * Marco van Wieringen, February 2015 + */ + +#include "bareos.h" + +#if defined(HAVE_OBJECTSTORE) +#include "stored.h" +#include "chunked_device.h" + +#ifdef HAVE_MMAP +#ifdef HAVE_SYS_MMAN_H +#include +#endif +#endif + +/* + * This implements a device abstraction that provides so called chunked + * volumes. These chunks are kept in memory and flushed to the backing + * store when requested. This class fully abstracts the chunked volumes + * for the upper level device. The stacking for this device type is: + * + * :: + * | + * v + * chunked_device:: + * | + * v + * DEVICE:: + * + * The public interfaces exported from this device are: + * + * setup_chunk() - Setup a chunked volume for reading or writing. + * read_chunked() - Read a chunked volume. + * write_chunked() - Write a chunked volume. + * close_chunk() - Close a chunked volume. + * truncate_chunked_volume() - Truncate a chunked volume. + * chunked_volume_size() - Get the current size of a volume. + * load_chunk() - Make sure we have the right chunk in memory. + * + * It also demands that the inheriting class implements the + * following methods: + * + * flush_remote_chunk() - Flush a chunk to the remote backing store. + * read_remote_chunk() - Read a chunk from the remote backing store. + * chunked_remote_volume_size - Return the current size of a volume. + * truncate_remote_chunked_volume() - Truncate a chunked volume on the + * remote backing store. + */ + +/* + * Actual thread runner that processes IO request from circular buffer. + */ +static void *io_thread(void *data) +{ + char ed1[50]; + chunked_device *dev = (chunked_device *)data; + + /* + * Dequeue from the circular buffer until we are done. + */ + while (1) { + if (!dev->dequeue_chunk()) { + break; + } + } + + Dmsg1(100, "Stopping IO-thread threadid=%s\n", + edit_pthread(pthread_self(), ed1, sizeof(ed1))); + + return NULL; +} + +/* + * Allocate a new chunk buffer. + */ +char *chunked_device::allocate_chunkbuffer() +{ + char *buffer = NULL; + +#ifdef HAVE_MMAP + if (m_use_mmap) { + buffer = (char *)::mmap(NULL, m_current_chunk->chunk_size, + (PROT_READ | PROT_WRITE), + (MAP_SHARED | MAP_ANONYMOUS), + -1, 0); + Dmsg1(100, "Mapped %ld bytes for chunk buffer\n", m_current_chunk->chunk_size); + } else { +#endif + buffer = (char *)malloc(m_current_chunk->chunk_size); +#ifdef HAVE_MMAP + } +#endif + + Dmsg2(100, "New allocated buffer of %d bytes at %p\n", m_current_chunk->chunk_size, buffer); + + return buffer; +} + +/* + * Free a chunk buffer. + */ +void chunked_device::free_chunkbuffer(char *buffer) +{ + Dmsg2(100, "Freeing buffer of %d bytes at %p\n", m_current_chunk->chunk_size, buffer); + +#ifdef HAVE_MMAP + if (m_use_mmap) { + ::munmap(buffer, m_current_chunk->chunk_size); + Dmsg1(100, "Unmapped %ld bytes used as chunk buffer\n", m_current_chunk->chunk_size); + } else { +#endif + free(buffer); + + /* + * As we released a big memory chunk let the garbage collector run. + */ + garbage_collect_memory(); +#ifdef HAVE_MMAP + } +#endif +} + +/* + * Free a chunk_io_request. + */ +void chunked_device::free_chunk_io_request(chunk_io_request *request) +{ + Dmsg2(100, "Freeing chunk io request of %d bytes at %p\n", sizeof(chunk_io_request), request); + + if (request->release) { + free_chunkbuffer(request->buffer); + } + free((void *)request->volname); + free(request); +} + +/* + * Start the io-threads that are used for uploading. + */ +bool chunked_device::start_io_threads() +{ + char ed1[50]; + uint8_t thread_nr; + pthread_t thread_id; + thread_handle *handle; + + /* + * Create a new ordered circular buffer for exchanging chunks between + * the producer (the storage driver) and multiple consumers (io-threads). + */ + if (m_io_slots) { + m_cb = New(ordered_circbuf(m_io_threads * m_io_slots)); + } else { + m_cb = New(ordered_circbuf(m_io_threads * OQSIZE)); + } + + /* + * Start all IO threads and keep track of their thread ids in m_thread_ids. + */ + if (!m_thread_ids) { + m_thread_ids = New(alist(10, owned_by_alist)); + } + + for (thread_nr = 1; thread_nr <= m_io_threads; thread_nr++) { + if (pthread_create(&thread_id, NULL, io_thread, (void *)this)) { + return false; + } + + handle = (thread_handle *)malloc(sizeof(thread_handle)); + memset(handle, 0, sizeof(thread_handle)); + handle->type = WAIT_JOIN_THREAD; + memcpy(&handle->thread_id, &thread_id, sizeof(pthread_t)); + m_thread_ids->append(handle); + + Dmsg1(100, "Started new IO-thread threadid=%s\n", + edit_pthread(thread_id, ed1, sizeof(ed1))); + } + + m_io_threads_started = true; + + return true; +} + +/* + * Stop the io-threads that are used for uploading. + */ +void chunked_device::stop_threads() +{ + char ed1[50]; + thread_handle *handle; + + /* + * Tell all IO threads that we flush the circular buffer. + * As such they will get a NULL chunk_io_request back and exit. + */ + m_cb->flush(); + + /* + * Wait for all threads to exit. + */ + if (m_thread_ids) { + foreach_alist(handle, m_thread_ids) { + switch (handle->type) { + case WAIT_CANCEL_THREAD: + Dmsg1(100, "Canceling thread with threadid=%s\n", + edit_pthread(handle->thread_id, ed1, sizeof(ed1))); + pthread_cancel(handle->thread_id); + break; + case WAIT_JOIN_THREAD: + Dmsg1(100, "Waiting to join with threadid=%s\n", + edit_pthread(handle->thread_id, ed1, sizeof(ed1))); + pthread_join(handle->thread_id, NULL); + break; + default: + break; + } + } + + m_thread_ids->destroy(); + delete m_thread_ids; + m_thread_ids = NULL; + } +} + +/* + * Call back function for comparing two chunk_io_requests. + */ +static int compare_chunk_io_request(void *item1, void *item2) +{ + ocbuf_item *ocbuf1 = (ocbuf_item *)item1; + ocbuf_item *ocbuf2 = (ocbuf_item *)item2; + chunk_io_request *chunk1 = (chunk_io_request *)ocbuf1->data; + chunk_io_request *chunk2 = (chunk_io_request *)ocbuf2->data; + + /* + * Same volume name ? + */ + if (bstrcmp(chunk1->volname, chunk2->volname)) { + /* + * Compare on chunk number. + */ + if (chunk1->chunk == chunk2->chunk) { + return 0; + } else { + return (chunk1->chunk < chunk2->chunk) ? -1 : 1; + } + } else { + return strcmp(chunk1->volname, chunk2->volname); + } +} + +/* + * Call back function for updating two chunk_io_requests. + */ +static void update_chunk_io_request(void *item1, void *item2) +{ + ocbuf_item *ocbuf1 = (ocbuf_item *)item1; + ocbuf_item *ocbuf2 = (ocbuf_item *)item2; + chunk_io_request *chunk1 = (chunk_io_request *)ocbuf1->data; + chunk_io_request *chunk2 = (chunk_io_request *)ocbuf2->data; + + /* + * See if the new chunk_io_request has more bytes then + * the chunk_io_request currently on the ordered circular + * buffer. We can only have multiple chunk_io_requests for + * the same chunk of a volume when a chunk was not fully + * filled by one backup Job and a next one writes data to + * the chunk before its being flushed to backing store. This + * means all pointers are the same only the wbuflen and the + * release flag of the chunk_io_request differ. So we only + * copy those two fields and not the others. + */ + if (chunk2->buffer == chunk1->buffer && + chunk2->wbuflen > chunk1->wbuflen) { + chunk1->wbuflen = chunk2->wbuflen; + chunk1->release = chunk2->release; + } + chunk2->release = false; +} + +/* + * Enqueue a chunk flush request onto the ordered circular buffer. + */ +bool chunked_device::enqueue_chunk(chunk_io_request *request) +{ + chunk_io_request *new_request, + *enqueued_request; + + Dmsg2(100, "Enqueueing chunk %d of volume %s\n", request->chunk, request->volname); + + if (!m_io_threads_started) { + if (!start_io_threads()) { + return false; + } + } + + new_request = (chunk_io_request *)malloc(sizeof(chunk_io_request)); + memset(new_request, 0, sizeof(chunk_io_request)); + new_request->volname = bstrdup(request->volname); + new_request->chunk = request->chunk; + new_request->buffer = request->buffer; + new_request->wbuflen = request->wbuflen; + new_request->release = request->release; + + Dmsg2(100, "Allocated chunk io request of %d bytes at %p\n", sizeof(chunk_io_request), new_request); + + /* + * Enqueue the item onto the ordered circular buffer. + * This returns either the same request as we passed + * in or the previous flush request for the same chunk. + */ + enqueued_request = (chunk_io_request *)m_cb->enqueue(new_request, + sizeof(chunk_io_request), + compare_chunk_io_request, + update_chunk_io_request, + false, /* use_reserved_slot */ + false /* no_signal */); + + /* + * Compare the return value from the enqueue. + */ + if (enqueued_request && enqueued_request != new_request) { + free_chunk_io_request(new_request); + } + + return (enqueued_request) ? true : false; +} + +/* + * Dequeue a chunk flush request from the ordered circular buffer and process it. + */ +bool chunked_device::dequeue_chunk() +{ + char ed1[50]; + struct timeval tv; + struct timezone tz; + struct timespec ts; + bool requeued = false; + chunk_io_request *new_request; + + /* + * Loop while we are not done either due to the ordered circular buffer being flushed + * some fatal error or successfully dequeueing a chunk flush request. + */ + while (1) { + /* + * See if we are in the flushing state then we just return and exit the io-thread. + */ + if (m_cb->is_flushing()) { + return false; + } + + /* + * Calculate the next absolute timeout if we find out there is no work to be done. + */ + gettimeofday(&tv, &tz); + ts.tv_nsec = tv.tv_usec * 1000; + ts.tv_sec = tv.tv_sec + DEFAULT_RECHECK_INTERVAL; + + /* + * Dequeue the next item from the ordered circular buffer and reserve the slot as we + * might need to put this item back onto the ordered circular buffer if we fail to + * flush it to the remote backing store. Also let the dequeue wake up every + * DEFAULT_RECHECK_INTERVAL seconds to retry failed previous uploads. + */ + new_request = (chunk_io_request *)m_cb->dequeue(true, /* reserve_slot we may need to enqueue the request */ + requeued, /* request is requeued due to failure ? */ + &ts, DEFAULT_RECHECK_INTERVAL); + if (!new_request) { + return false; + } + + Dmsg3(100, "Flushing chunk %d of volume %s by thread %s\n", + new_request->chunk, new_request->volname, + edit_pthread(pthread_self(), ed1, sizeof(ed1))); + + if (!flush_remote_chunk(new_request)) { + chunk_io_request *enqueued_request; + + /* + * We failed to flush the chunk to the backing store + * so enqueue it again using the reserved slot by dequeue() + * but don't signal the workers otherwise we would try uploading + * the same chunk again and again by different io-threads. + * As we set the requeued flag to the dequeue method on the ordered circular buffer + * we will not try dequeueing any new item either until a new item is put + * onto the ordered circular buffer or after the retry interval has expired. + */ + Dmsg2(100, "Enqueueing chunk %d of volume %s for retry of upload later\n", + new_request->chunk, new_request->volname); + + /* + * Enqueue the item onto the ordered circular buffer. + * This returns either the same request as we passed + * in or the previous flush request for the same chunk. + */ + enqueued_request = (chunk_io_request *)m_cb->enqueue(new_request, + sizeof(chunk_io_request), + compare_chunk_io_request, + update_chunk_io_request, + true, /* use_reserved_slot */ + true /* no_signal */); + /* + * See if the enqueue succeeded. + */ + if (!enqueued_request) { + return false; + } + + /* + * Compare the return value from the enqueue against our new_request. + * If it is different there was already a chunk io request for the + * same chunk on the ordered circular buffer. + */ + if (enqueued_request != new_request) { + free_chunk_io_request(new_request); + } + + requeued = true; + continue; + } + + /* + * Unreserve the slot on the ordered circular buffer reserved by dequeue(). + */ + m_cb->unreserve_slot(); + + /* + * Processed the chunk so clean it up now. + */ + free_chunk_io_request(new_request); + + return true; + } +} + +/* + * Internal method for flushing a chunk to the backing store. + * The retry logic is in the io-threads but if those are not + * used we give this one try and otherwise drop the chunk and + * return an IO error to the upper level callers. That way the + * volume will go into error. + */ +bool chunked_device::flush_chunk(bool release_chunk, bool move_to_next_chunk) +{ + bool retval = false; + chunk_io_request request; + + /* + * Calculate in which chunk we are currently. + */ + request.chunk = m_current_chunk->start_offset / m_current_chunk->chunk_size; + request.volname = m_current_volname; + request.buffer = m_current_chunk->buffer; + request.wbuflen = m_current_chunk->buflen; + request.release = release_chunk; + + if (m_io_threads) { + retval = enqueue_chunk(&request); + } else { + retval = flush_remote_chunk(&request); + } + + /* + * Clear the need flushing flag. + */ + m_current_chunk->need_flushing = false; + + /* + * Change to the next chunk ? + */ + if (move_to_next_chunk) { + /* + * If we enqueued the data we need to allocate a new buffer. + */ + if (m_io_threads) { + m_current_chunk->buffer = allocate_chunkbuffer(); + } + m_current_chunk->start_offset += m_current_chunk->chunk_size; + m_current_chunk->end_offset = m_current_chunk->start_offset + (m_current_chunk->chunk_size - 1); + m_current_chunk->buflen = 0; + } else { + /* + * If we enqueued the data we need to allocate a new buffer. + */ + if (release_chunk && m_io_threads) { + m_current_chunk->buffer = NULL; + } + } + + if (!retval) { + Dmsg1(100, "%s", errmsg); + } + + return retval; +} + +/* + * Internal method for reading a chunk from the backing store. + */ +bool chunked_device::read_chunk() +{ + chunk_io_request request; + + /* + * Calculate in which chunk we are currently. + */ + request.chunk = m_current_chunk->start_offset / m_current_chunk->chunk_size; + request.volname = m_current_volname; + request.buffer = m_current_chunk->buffer; + request.wbuflen = m_current_chunk->chunk_size; + request.rbuflen = &m_current_chunk->buflen; + request.release = false; + + m_current_chunk->end_offset = m_current_chunk->start_offset + (m_current_chunk->chunk_size - 1); + + if (!read_remote_chunk(&request)) { + /* + * If the chunk doesn't exist on the backing store it has a size of 0 bytes. + */ + m_current_chunk->buflen = 0; + return false; + } + + return true; +} + +/* + * Setup a chunked volume for reading or writing. + */ +void chunked_device::setup_chunk(int flags) +{ + if (!m_current_chunk) { + m_current_chunk = (chunk_descriptor *)malloc(sizeof(chunk_descriptor)); + memset(m_current_chunk, 0, sizeof(chunk_descriptor)); + if (m_chunk_size > DEFAULT_CHUNK_SIZE) { + m_current_chunk->chunk_size = m_chunk_size; + } else { + m_current_chunk->chunk_size = DEFAULT_CHUNK_SIZE; + } + m_current_chunk->start_offset = -1; + m_current_chunk->end_offset = -1; + } + + /* + * Reopen of a device. + */ + if (m_current_chunk->opened) { + /* + * Invalidate chunk. + */ + m_current_chunk->buflen = 0; + m_current_chunk->start_offset = -1; + m_current_chunk->end_offset = -1; + } + + if (flags & O_RDWR) { + m_current_chunk->writing = true; + } + + m_current_chunk->opened = true; + m_current_chunk->chunk_setup = false; + + /* + * We need to limit the maximum size of a chunked volume to MAX_CHUNKS * chunk_size). + */ + if (max_volume_size == 0 || max_volume_size > (uint64_t)(MAX_CHUNKS * m_current_chunk->chunk_size)) { + max_volume_size = MAX_CHUNKS * m_current_chunk->chunk_size; + } + + /* + * On open set begin offset to 0. + */ + m_offset = 0; + + /* + * On open we are no longer at the End of the Media. + */ + m_end_of_media = false; + + /* + * Keep track of the volume currently mounted. + */ + if (m_current_volname) { + free(m_current_volname); + } + + m_current_volname = bstrdup(getVolCatName()); +} + +/* + * Read a chunked volume. + */ +ssize_t chunked_device::read_chunked(int fd, void *buffer, size_t count) +{ + ssize_t retval = 0; + + if (m_current_chunk->opened) { + ssize_t wanted_offset; + ssize_t bytes_left; + + /* + * Shortcut logic see if m_end_of_media is set then we are at the End of the Media + */ + if (m_end_of_media) { + goto bail_out; + } + + /* + * If we are starting reading without the chunk being setup it means we + * are start reading at the beginning of the file otherwise the d_lseek method + * would have read in the correct chunk. + */ + if (!m_current_chunk->chunk_setup) { + m_current_chunk->start_offset = 0; + + /* + * See if we have to allocate a new buffer. + */ + if (!m_current_chunk->buffer) { + m_current_chunk->buffer = allocate_chunkbuffer(); + } + + if (!read_chunk()) { + retval = -1; + goto bail_out; + } + m_current_chunk->chunk_setup = true; + } + + /* + * See if we can fulfill the wanted read from the current chunk. + */ + if (m_current_chunk->start_offset <= m_offset && + m_current_chunk->end_offset >= (boffset_t)((m_offset + count) - 1)) { + wanted_offset = (m_offset % m_current_chunk->chunk_size); + + bytes_left = MIN((ssize_t)count, (m_current_chunk->buflen - wanted_offset)); + Dmsg2(200, "Reading %d bytes at offset %d from chunk buffer\n", bytes_left, wanted_offset); + + if (bytes_left < 0) { + retval = -1; + goto bail_out; + } + + if (bytes_left > 0) { + memcpy(buffer, m_current_chunk->buffer + wanted_offset, bytes_left); + } + m_offset += bytes_left; + retval = bytes_left; + goto bail_out; + } else { + ssize_t offset = 0; + + /* + * We cannot fulfill the read from the current chunk, see how much + * is available and return that and see if by reading the next chunk + * we can fulfill the whole read. + */ + while (retval < (ssize_t)count) { + /* + * See how much is left in this chunk. + */ + wanted_offset = (m_offset % m_current_chunk->chunk_size); + bytes_left = MIN((ssize_t)count, (m_current_chunk->buflen - wanted_offset)); + + if (bytes_left > 0) { + Dmsg2(200, "Reading %d bytes at offset %d from chunk buffer\n", bytes_left, wanted_offset); + + memcpy(buffer, m_current_chunk->buffer + wanted_offset, bytes_left); + m_offset += bytes_left; + offset += bytes_left; + retval += bytes_left; + } + + /* + * Read in the next chunk. + */ + m_current_chunk->start_offset += m_current_chunk->chunk_size; + if (!read_chunk()) { + switch (dev_errno) { + case EIO: + /* + * If the are no more chunks to read we return only the bytes available. + * We also set m_end_of_media as we are at the end of media. + */ + m_end_of_media = true; + goto bail_out; + default: + retval = -1; + goto bail_out; + } + } else { + bytes_left = MIN((boffset_t)(count - retval), m_current_chunk->buflen); + + if (bytes_left > 0) { + Dmsg2(200, "Reading %d bytes at offset %d from chunk buffer\n", bytes_left, 0); + + memcpy((char *)buffer + offset, m_current_chunk->buffer, bytes_left); + m_offset += bytes_left; + retval += bytes_left; + } + } + } + } + } else { + errno = EBADF; + retval = -1; + } + +bail_out: + return retval; +} + +/* + * Write a chunked volume. + */ +ssize_t chunked_device::write_chunked(int fd, const void *buffer, size_t count) +{ + ssize_t retval = 0; + + if (m_current_chunk->opened) { + ssize_t wanted_offset; + + /* + * If we are starting writing without the chunk being setup it means we + * are start writing to an empty file because otherwise the d_lseek method + * would have read in the correct chunk. + */ + if (!m_current_chunk->chunk_setup) { + m_current_chunk->start_offset = 0; + m_current_chunk->end_offset = (m_current_chunk->chunk_size - 1); + m_current_chunk->buflen = 0; + m_current_chunk->chunk_setup = true; + + /* + * See if we have to allocate a new buffer. + */ + if (!m_current_chunk->buffer) { + m_current_chunk->buffer = allocate_chunkbuffer(); + } + } + + /* + * See if we can write the whole data inside the current chunk. + */ + if (m_current_chunk->start_offset <= m_offset && + m_current_chunk->end_offset >= (boffset_t)((m_offset + count) - 1)) { + + wanted_offset = (m_offset % m_current_chunk->chunk_size); + + Dmsg2(200, "Writing %d bytes at offset %d in chunk buffer\n", count, wanted_offset); + + memcpy(m_current_chunk->buffer + wanted_offset, buffer, count); + + m_offset += count; + if ((wanted_offset + count) > m_current_chunk->buflen) { + m_current_chunk->buflen = wanted_offset + count; + } + m_current_chunk->need_flushing = true; + retval = count; + } else { + ssize_t bytes_left; + ssize_t offset = 0; + + /* + * Things don't fit so first write as many bytes as can be written into + * the current chunk and then flush it and write the next bytes into the + * next chunk. + */ + while (retval < (ssize_t)count) { + /* + * See how much is left in this chunk. + */ + wanted_offset = (m_offset % m_current_chunk->chunk_size); + bytes_left = ((m_current_chunk->end_offset - (m_current_chunk->start_offset + wanted_offset)) + 1); + + if (bytes_left > 0) { + Dmsg2(200, "Writing %d bytes at offset %d in chunk buffer\n", bytes_left, wanted_offset); + + memcpy(m_current_chunk->buffer + wanted_offset, buffer, bytes_left); + m_offset += bytes_left; + if ((wanted_offset + bytes_left) > m_current_chunk->buflen) { + m_current_chunk->buflen = wanted_offset + bytes_left; + } + m_current_chunk->need_flushing = true; + retval += bytes_left; + + /* + * Keep track of the number of bytes we already consumed. + */ + offset += bytes_left; + } + + /* + * Flush out the current chunk. + */ + if (!flush_chunk(true /* release */, true /* move_to_next_chunk */)) { + retval = -1; + goto bail_out; + } + + bytes_left = MIN((boffset_t)(count - retval), ((m_current_chunk->end_offset - m_current_chunk->start_offset) + 1)); + if (bytes_left > 0) { + Dmsg2(200, "Writing %d bytes at offset %d in chunk buffer\n", bytes_left, 0); + + memcpy(m_current_chunk->buffer, (char *)buffer + offset, bytes_left); + m_current_chunk->buflen = bytes_left; + m_current_chunk->need_flushing = true; + m_offset += bytes_left; + retval += bytes_left; + } + } + } + } else { + errno = EBADF; + retval = -1; + } + +bail_out: + return retval; +} + +/* + * Close a chunked volume. + */ +int chunked_device::close_chunk() +{ + int retval = -1; + + if (m_current_chunk->opened) { + if (m_current_chunk->need_flushing) { + if (flush_chunk(true /* release */, false /* move_to_next_chunk */)) { + retval = 0; + } else { + dev_errno = EIO; + } + } + + /* + * Invalidate chunk. + */ + m_current_chunk->writing = false; + m_current_chunk->opened = false; + m_current_chunk->chunk_setup = false; + m_current_chunk->buflen = 0; + m_current_chunk->start_offset = -1; + m_current_chunk->end_offset = -1; + } else { + errno = EBADF; + } + + return retval; +} + +/* + * Truncate a chunked volume. + */ +bool chunked_device::truncate_chunked_volume(DCR *dcr) +{ + if (m_current_chunk->opened) { + if (!truncate_remote_chunked_volume(dcr)) { + return false; + } + + /* + * Reinitialize the initial chunk. + */ + m_current_chunk->start_offset = 0; + m_current_chunk->end_offset = (m_current_chunk->chunk_size - 1); + m_current_chunk->buflen = 0; + m_current_chunk->chunk_setup = true; + m_current_chunk->need_flushing = false; + + /* + * Reinitialize the volume name on a relabel we could get a new name. + */ + if (m_current_volname) { + free(m_current_volname); + } + + m_current_volname = bstrdup(getVolCatName()); + } + + return true; +} + +static int compare_volume_name(void *item1, void *item2) +{ + const char *volname = (const char *)item2; + chunk_io_request *request = (chunk_io_request *)item1; + + return strcmp(request->volname, volname); +} + +/* + * Get the current size of a volume. + */ +ssize_t chunked_device::chunked_volume_size() +{ + /* + * See if we are using io-threads or not and the ordered circbuf is created and not empty. + */ + if (m_io_threads > 0 && m_cb && !m_cb->empty()) { + char *volname; + chunk_io_request *request; + + volname = getVolCatName(); + + /* + * Peek on the ordered circular queue if there are any pending IO-requests + * for this volume. If there are use that as the indication of the size of + * the volume and don't contact the remote storage as there is still data + * inflight and as such we need to look at the last chunk that is still not + * uploaded of the volume. + */ + request = (chunk_io_request *)m_cb->peek(PEEK_LAST, volname, compare_volume_name); + if (request) { + ssize_t retval; + + /* + * Calculate the size of the volume based on the last chunk inflight . + */ + retval = (request->chunk * m_current_chunk->chunk_size) + request->wbuflen; + + /* + * The peek method gives us a cloned chunk_io_request with pointers to + * the original chunk_io_request. We just need to free the structure not + * the content so we call free() here and not free_chunk_io_request() ! + */ + free(request); + + return retval; + } + } + + /* + * Get the actual length by contacting the remote backing store. + */ + return chunked_remote_volume_size(); +} + +/* + * Make sure we have the right chunk in memory. + */ +bool chunked_device::load_chunk() +{ + boffset_t start_offset; + + start_offset = (m_offset / m_current_chunk->chunk_size) * m_current_chunk->chunk_size; + + /* + * See if we have to allocate a new buffer. + */ + if (!m_current_chunk->buffer) { + m_current_chunk->buffer = allocate_chunkbuffer(); + } + + /* + * If the wrong chunk is loaded populate the chunk buffer with the right data. + */ + if (start_offset != m_current_chunk->start_offset) { + m_current_chunk->buflen = 0; + m_current_chunk->start_offset = start_offset; + if (!read_chunk()) { + switch (dev_errno) { + case EIO: + if (m_current_chunk->writing) { + m_current_chunk->end_offset = start_offset + (m_current_chunk->chunk_size - 1); + } + break; + default: + return false; + } + } + } + m_current_chunk->chunk_setup = true; + + return true; +} + +static int list_io_request(void *request, void *data) +{ + chunk_io_request *io_request = (chunk_io_request *)request; + bsdDevStatTrig *dst = (bsdDevStatTrig *)data; + POOL_MEM status(PM_MESSAGE); + + status.bsprintf(" /%s/%04d - %ld\n", io_request->volname, io_request->chunk, io_request->wbuflen); + dst->status_length = pm_strcat(dst->status, status.c_str()); + + return 0; +} + +/* + * Return specific device status information. + */ +bool chunked_device::device_status(bsdDevStatTrig *dst) +{ + /* + * See if we are using io-threads or not and the ordered circbuf is created and not empty. + */ + dst->status_length = 0; + if (m_io_threads > 0 && m_cb) { + if (!m_cb->empty()) { + dst->status_length = pm_strcpy(dst->status, _("Pending IO flush requests:\n")); + + /* + * Peek on the ordered circular queue and list all pending requests. + */ + m_cb->peek(PEEK_LIST, dst, list_io_request); + } else { + dst->status_length = pm_strcpy(dst->status, _("No Pending IO flush requests\n")); + } + } + + return (dst->status_length > 0); +} + +chunked_device::~chunked_device() +{ + if (m_thread_ids) { + stop_threads(); + } + + if (m_cb) { + /* + * If there is any work on the ordered circular buffer remove it. + */ + if (!m_cb->empty()) { + chunk_io_request *request; + do { + request = (chunk_io_request *)m_cb->dequeue(); + if (request) { + request->release = true; + free_chunk_io_request(request); + } + } while (!m_cb->empty()); + } + + delete m_cb; + m_cb = NULL; + } + + if (m_current_chunk) { + if (m_current_chunk->buffer) { + free_chunkbuffer(m_current_chunk->buffer); + } + free(m_current_chunk); + m_current_chunk = NULL; + } + + if (m_current_volname) { + free(m_current_volname); + } +} + +chunked_device::chunked_device() +{ + m_current_volname = NULL; + m_current_chunk = NULL; + m_io_threads = 0; + m_io_slots = 0; + m_chunk_size = 0; + m_io_threads_started = false; + m_end_of_media = false; + m_cb = NULL; + m_io_threads = 0; + m_chunk_size = 0; + m_offset = 0; + m_use_mmap = false; +} +#endif /* HAVE_OBJECTSTORE */ diff --git a/src/stored/backends/chunked_device.h b/src/stored/backends/chunked_device.h new file mode 100644 index 00000000000..53a24bee5ed --- /dev/null +++ b/src/stored/backends/chunked_device.h @@ -0,0 +1,156 @@ +/* + BAREOS® - Backup Archiving REcovery Open Sourced + + Copyright (C) 2015-2017 Planets Communications B.V. + + This program is Free Software; you can redistribute it and/or + modify it under the terms of version three of the GNU Affero General Public + License as published by the Free Software Foundation, which is + listed in the file LICENSE. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. +*/ +/* + * Chunked device device abstraction. + * + * Marco van Wieringen, February 2015 + */ + +#ifndef CHUNKED_DEVICE_H +#define CHUNKED_DEVICE_H + +/* + * Let io-threads check for work every 300 seconds. + */ +#define DEFAULT_RECHECK_INTERVAL 300 + +/* + * Chunk the volume into chunks of this size. + * This is the lower limit used the exact chunksize is + * configured as a device option. + */ +#define DEFAULT_CHUNK_SIZE 10 * 1024 * 1024 + +/* + * Maximum number of chunks per volume. + * When you change this make sure you update the %04d format + * used in the code to format the chunk numbers e.g. 0000-9999 + */ +#define MAX_CHUNKS 10000 + +enum thread_wait_type { + WAIT_CANCEL_THREAD, /* Perform a pthread_cancel() on exit. */ + WAIT_JOIN_THREAD /* Perform a pthread_join() on exit. */ +}; + +struct thread_handle { + thread_wait_type type; /* See WAIT_*_THREAD thread_wait_type enum */ + pthread_t thread_id; /* Actual threadid */ +}; + +struct chunk_io_request { + const char *volname; /* VolumeName */ + uint16_t chunk; /* Chunk number */ + char *buffer; /* Data */ + uint32_t wbuflen; /* Size of the actual valid data in the chunk (Write) */ + uint32_t *rbuflen; /* Size of the actual valid data in the chunk (Read) */ + bool release; /* Should we release the data to which the buffer points ? */ +}; + +struct chunk_descriptor { + ssize_t chunk_size; /* Total size of the memory chunk */ + char *buffer; /* Data */ + uint32_t buflen; /* Size of the actual valid data in the chunk */ + boffset_t start_offset; /* Start offset of the current chunk */ + boffset_t end_offset; /* End offset of the current chunk */ + bool need_flushing; /* Data is dirty and needs flushing to backing store */ + bool chunk_setup; /* Chunk is initialized and ready for use */ + bool writing; /* We are currently writing */ + bool opened; /* An open call was done */ +}; + +#include "lib/ordered_cbuf.h" + +class chunked_device: public DEVICE { +private: + /* + * Private Members + */ + bool m_io_threads_started; + bool m_end_of_media; + char *m_current_volname; + ordered_circbuf *m_cb; + alist *m_thread_ids; + chunk_descriptor *m_current_chunk; + + /* + * Private Methods + */ + char *allocate_chunkbuffer(); + void free_chunkbuffer(char *buffer); + void free_chunk_io_request(chunk_io_request *request); + bool start_io_threads(); + void stop_threads(); + bool enqueue_chunk(chunk_io_request *request); + bool flush_chunk(bool release_chunk, bool move_to_next_chunk); + bool read_chunk(); + +protected: + /* + * Protected Members + */ + uint8_t m_io_threads; + uint8_t m_io_slots; + uint64_t m_chunk_size; + boffset_t m_offset; + bool m_use_mmap; + + /* + * Protected Methods + */ + void setup_chunk(int flags); + ssize_t read_chunked(int fd, void *buffer, size_t count); + ssize_t write_chunked(int fd, const void *buffer, size_t count); + int close_chunk(); + bool truncate_chunked_volume(DCR *dcr); + ssize_t chunked_volume_size(); + bool load_chunk(); + + /* + * Methods implemented by inheriting class. + */ + virtual bool flush_remote_chunk(chunk_io_request *request) = 0; + virtual bool read_remote_chunk(chunk_io_request *request) = 0; + virtual ssize_t chunked_remote_volume_size() = 0; + virtual bool truncate_remote_chunked_volume(DCR *dcr) = 0; + +public: + /* + * Public Methods + */ + chunked_device(); + virtual ~chunked_device(); + + bool dequeue_chunk(); + bool device_status(bsdDevStatTrig *dst); + + /* + * Interface from DEVICE + */ + virtual int d_close(int fd) = 0; + virtual int d_open(const char *pathname, int flags, int mode) = 0; + virtual int d_ioctl(int fd, ioctl_req_t request, char *mt = NULL) = 0; + virtual boffset_t d_lseek(DCR *dcr, boffset_t offset, int whence) = 0; + virtual ssize_t d_read(int fd, void *buffer, size_t count) = 0; + virtual ssize_t d_write(int fd, const void *buffer, size_t count) = 0; + virtual bool d_truncate(DCR *dcr) = 0; +}; +#endif /* CHUNKED_DEVICE_H */ diff --git a/src/stored/backends/object_store_device.c b/src/stored/backends/object_store_device.c index 5ccae516d3a..c4a424933fb 100644 --- a/src/stored/backends/object_store_device.c +++ b/src/stored/backends/object_store_device.c @@ -1,7 +1,7 @@ /* BAREOS® - Backup Archiving REcovery Open Sourced - Copyright (C) 2014-2014 Planets Communications B.V. + Copyright (C) 2014-2017 Planets Communications B.V. Copyright (C) 2014-2014 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or @@ -21,6 +21,18 @@ */ /* * Marco van Wieringen, February 2014 + * Object Storage API device abstraction. + * + * Stacking is the following: + * + * object_store_device:: + * | + * v + * chunked_device:: + * | + * v + * DEVICE:: + * */ /** * @file @@ -31,6 +43,7 @@ #ifdef HAVE_OBJECTSTORE #include "stored.h" +#include "chunked_device.h" #include "object_store_device.h" /** @@ -39,7 +52,14 @@ enum device_option_type { argument_none = 0, argument_profile, - argument_bucket + argument_location, + argument_canned_acl, + argument_storage_class, + argument_bucket, + argument_chunksize, + argument_iothreads, + argument_ioslots, + argument_mmap }; struct device_option { @@ -50,7 +70,14 @@ struct device_option { static device_option device_options[] = { { "profile=", argument_profile, 8 }, + { "location=", argument_location, 9 }, + { "acl=", argument_canned_acl, 4 }, + { "storageclass=", argument_storage_class, 13 }, { "bucket=", argument_bucket, 7 }, + { "chunksize=", argument_chunksize, 10 }, + { "iothreads=", argument_iothreads, 10 }, + { "ioslots=", argument_ioslots, 8 }, + { "mmap", argument_mmap, 4 }, { NULL, argument_none } }; @@ -75,6 +102,8 @@ static void object_store_logfunc(dpl_ctx_t *ctx, dpl_log_level_t level, const ch case DPL_ERROR: Emsg1(M_ERROR, 0, "%s\n", message); break; + default: + break; } } @@ -87,12 +116,26 @@ static inline int droplet_errno_to_system_errno(dpl_status_t status) case DPL_ENOENT: errno = ENOENT; break; + case DPL_ETIMEOUT: + errno = ETIMEDOUT; + case DPL_ENOMEM: + errno = ENOMEM; + break; case DPL_EIO: errno = EIO; break; case DPL_ENAMETOOLONG: errno = ENAMETOOLONG; break; + case DPL_ENOTDIR: + errno = ENOTDIR; + break; + case DPL_ENOTEMPTY: + errno = ENOTEMPTY; + break; + case DPL_EISDIR: + errno = EISDIR; + break; case DPL_EEXIST: errno = EEXIST; break; @@ -100,46 +143,398 @@ static inline int droplet_errno_to_system_errno(dpl_status_t status) errno = EPERM; break; default: + errno = EINVAL; break; } - return -1; + return errno; } /** - * Open a volume using libdroplet. + * Generic callback for the walk_dpl_directory() function. + * + * Returns true - abort loop + * false - continue loop */ -int object_store_device::d_open(const char *pathname, int flags, int mode) +typedef bool (*t_call_back)(dpl_dirent_t *dirent, dpl_ctx_t *ctx, + const char *dirname, void *data); + +/* + * Callback for getting the total size of a chunked volume. + */ +static bool chunked_volume_size_callback(dpl_dirent_t *dirent, dpl_ctx_t *ctx, + const char *dirname, void *data) +{ + ssize_t *volumesize = (ssize_t *)data; + + /* + * Make sure it starts with [0-9] e.g. a volume chunk. + */ + if (*dirent->name >= '0' && *dirent->name <= '9') { + *volumesize = *volumesize + dirent->size; + } + + return false; +} + +/* + * Callback for truncating a chunked volume. + */ +static bool chunked_volume_truncate_callback(dpl_dirent_t *dirent, dpl_ctx_t *ctx, + const char *dirname, void *data) +{ + dpl_status_t status; + + /* + * Make sure it starts with [0-9] e.g. a volume chunk. + */ + if (*dirent->name >= '0' && *dirent->name <= '9') { + status = dpl_unlink(ctx, dirent->name); + + switch (status) { + case DPL_SUCCESS: + break; + default: + return true; + } + } + + return false; +} + +/* + * Generic function that walks a dirname and calls the callback + * function for each entry it finds in that directory. + */ +static bool walk_dpl_directory(dpl_ctx_t *ctx, const char *dirname, t_call_back callback, void *data) +{ + void *dir_hdl; + dpl_status_t status; + dpl_dirent_t dirent; + + if (dirname) { + status = dpl_chdir(ctx, dirname); + + switch (status) { + case DPL_SUCCESS: + break; + default: + return false; + } + } + + status = dpl_opendir(ctx, ".", &dir_hdl); + + switch (status) { + case DPL_SUCCESS: + break; + default: + return false; + } + + while (!dpl_eof(dir_hdl)) { + status = dpl_readdir(dir_hdl, &dirent); + + switch (status) { + case DPL_SUCCESS: + break; + default: + dpl_closedir(dir_hdl); + return false; + } + + /* + * Skip '.' and '..' + */ + if (bstrcmp(dirent.name, ".") || + bstrcmp(dirent.name, "..")) { + continue; + } + + if (callback(&dirent, ctx, dirname, data)) { + break; + } + } + + dpl_closedir(dir_hdl); + + if (dirname) { + status = dpl_chdir(ctx, "/"); + + switch (status) { + case DPL_SUCCESS: + break; + default: + return false; + } + } + + return true; +} + +/* + * Internal method for flushing a chunk to the backing store. + * This does the real work either by being called from a + * io-thread or directly blocking the device. + */ +bool object_store_device::flush_remote_chunk(chunk_io_request *request) { + bool retval = false; dpl_status_t status; - dpl_vfile_flag_t dpl_flags; dpl_option_t dpl_options; + dpl_sysmd_t *sysmd = NULL; + POOL_MEM chunk_dir(PM_FNAME), + chunk_name(PM_FNAME); -#if 1 - Mmsg1(errmsg, _("Object Storage devices are not yet supported, please disable %s\n"), dev_name); - return -1; -#endif + Mmsg(chunk_dir, "/%s", request->volname); + Mmsg(chunk_name, "%s/%04d", chunk_dir.c_str(), request->chunk); + + Dmsg1(100, "Flushing chunk %s\n", chunk_name.c_str()); + + /* + * Check on the remote backing store if the chunk already exists. + * We only upload this chunk if it is bigger then the chunk that exists + * on the remote backing store. When using io-threads it could happen + * that there are multiple flush requests for the same chunk when a + * chunk is reused in a next backup job. We only want the chunk with + * the biggest amount of valid data to persist as we only append to + * chunks. + */ + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + chunk_name.c_str(), /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ + + switch (status) { + case DPL_SUCCESS: + if (sysmd->size > request->wbuflen) { + retval = true; + goto bail_out; + } + break; + default: + /* + * Check on the remote backing store if the chunkdir exists. + */ + dpl_sysmd_free(sysmd); + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + chunk_dir.c_str(), /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ + + switch (status) { + case DPL_SUCCESS: + break; + case DPL_ENOENT: + case DPL_FAILURE: + /* + * Make sure the chunk directory with the name of the volume exists. + */ + dpl_sysmd_free(sysmd); + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_mkdir(m_ctx, /* context */ + chunk_dir.c_str(), /* locator */ + NULL, /* metadata */ + sysmd);/* sysmd */ + + switch (status) { + case DPL_SUCCESS: + break; + default: + Mmsg2(errmsg, _("Failed to create direcory %s using dpl_mkdir(): ERR=%s.\n"), + chunk_dir.c_str(), dpl_status_str(status)); + dev_errno = droplet_errno_to_system_errno(status); + goto bail_out; + } + break; + default: + break; + } + break; + } + + /* + * Create some options for libdroplet. + * + * DPL_OPTION_NOALLOC - we provide the buffer to copy the data into + * no need to let the library allocate memory we + * need to free after copying the data. + */ + memset(&dpl_options, 0, sizeof(dpl_options)); + dpl_options.mask |= DPL_OPTION_NOALLOC; + + dpl_sysmd_free(sysmd); + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_fput(m_ctx, /* context */ + chunk_name.c_str(), /* locator */ + &dpl_options, /* options */ + NULL, /* condition */ + NULL, /* range */ + NULL, /* metadata */ + sysmd, /* sysmd */ + (char *)request->buffer, /* data_buf */ + request->wbuflen); /* data_len */ + + switch (status) { + case DPL_SUCCESS: + break; + default: + Mmsg2(errmsg, _("Failed to flush %s using dpl_fput(): ERR=%s.\n"), + chunk_name.c_str(), dpl_status_str(status)); + dev_errno = droplet_errno_to_system_errno(status); + goto bail_out; + } + + retval = true; + +bail_out: + if (sysmd) { + dpl_sysmd_free(sysmd); + } + + return retval; +} + +/* + * Internal method for reading a chunk from the remote backing store. + */ +bool object_store_device::read_remote_chunk(chunk_io_request *request) +{ + bool retval = false; + dpl_status_t status; + dpl_option_t dpl_options; + dpl_range_t dpl_range; + dpl_sysmd_t *sysmd = NULL; + POOL_MEM chunk_name(PM_FNAME); + + Mmsg(chunk_name, "/%s/%04d", request->volname, request->chunk); + Dmsg1(100, "Reading chunk %s\n", chunk_name.c_str()); + + /* + * See if chunk exists. + */ + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + chunk_name.c_str(), /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ + + switch (status) { + case DPL_SUCCESS: + break; + default: + Mmsg1(errmsg, _("Failed to open %s doesn't exist\n"), chunk_name.c_str()); + Dmsg1(100, "%s", errmsg); + dev_errno = EIO; + goto bail_out; + } + + if (sysmd->size > request->wbuflen) { + Mmsg3(errmsg, _("Failed to read %s (%ld) to big to fit in chunksize of %ld bytes\n"), + chunk_name.c_str(), sysmd->size, request->wbuflen); + Dmsg1(100, "%s", errmsg); + dev_errno = EINVAL; + goto bail_out; + } + + /* + * Create some options for libdroplet. + * + * DPL_OPTION_NOALLOC - we provide the buffer to copy the data into + * no need to let the library allocate memory we + * need to free after copying the data. + */ + memset(&dpl_options, 0, sizeof(dpl_options)); + dpl_options.mask |= DPL_OPTION_NOALLOC; + + dpl_range.start = 0; + dpl_range.end = sysmd->size; + *request->rbuflen = sysmd->size; + dpl_sysmd_free(sysmd); + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_fget(m_ctx, /* context */ + chunk_name.c_str(), /* locator */ + &dpl_options, /* options */ + NULL, /* condition */ + &dpl_range, /* range */ + (char **)&request->buffer, /* data_bufp */ + request->rbuflen, /* data_lenp */ + NULL, /* metadatap */ + sysmd); /* sysmdp */ + + switch (status) { + case DPL_SUCCESS: + break; + case DPL_ENOENT: + Mmsg1(errmsg, _("Failed to open %s doesn't exist\n"), chunk_name.c_str()); + Dmsg1(100, "%s", errmsg); + dev_errno = EIO; + goto bail_out; + default: + Mmsg2(errmsg, _("Failed to read %s using dpl_fget(): ERR=%s.\n"), + chunk_name.c_str(), dpl_status_str(status)); + dev_errno = droplet_errno_to_system_errno(status); + goto bail_out; + } + + retval = true; + +bail_out: + if (sysmd) { + dpl_sysmd_free(sysmd); + } + + return retval; +} + +/* + * Internal method for truncating a chunked volume on the remote backing store. + */ +bool object_store_device::truncate_remote_chunked_volume(DCR *dcr) +{ + POOL_MEM chunk_dir(PM_FNAME); + + Mmsg(chunk_dir, "/%s", getVolCatName()); + if (!walk_dpl_directory(m_ctx, chunk_dir.c_str(), chunked_volume_truncate_callback, NULL)) { + return false; + } + + return true; +} + +/* + * Initialize backend. + */ +bool object_store_device::initialize() +{ + dpl_status_t status; /* * Initialize the droplet library when its not done previously. */ P(mutex); if (droplet_reference_count == 0) { + dpl_set_log_func(object_store_logfunc); + status = dpl_init(); - if (status != DPL_SUCCESS) { + switch (status) { + case DPL_SUCCESS: + break; + default: V(mutex); - return -1; + goto bail_out; } - - dpl_set_log_func(object_store_logfunc); - droplet_reference_count++; } + droplet_reference_count++; V(mutex); if (!m_object_configstring) { int len; - char *bp, *next_option; bool done; + uint64_t value; + char *bp, *next_option; if (!dev_options) { Mmsg0(errmsg, _("No device options configured\n")); @@ -163,14 +558,56 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) */ if (bstrncasecmp(bp, device_options[i].name, device_options[i].compare_size)) { switch (device_options[i].type) { - case argument_profile: - m_profile = bp + device_options[i].compare_size; + case argument_profile: { + char *profile; + + /* + * Strip any .profile prefix from the libdroplet profile name. + */ + profile = bp + device_options[i].compare_size; + len = strlen(profile); + if (len > 8 && bstrcasecmp(profile + (len - 8), ".profile")) { + profile[len - 8] = '\0'; + } + m_profile = profile; + done = true; + break; + } + case argument_location: + m_location = bp + device_options[i].compare_size; + done = true; + break; + case argument_canned_acl: + m_canned_acl = bp + device_options[i].compare_size; + done = true; + break; + case argument_storage_class: + m_storage_class = bp + device_options[i].compare_size; done = true; break; case argument_bucket: m_object_bucketname = bp + device_options[i].compare_size; done = true; break; + case argument_chunksize: + size_to_uint64(bp + device_options[i].compare_size, &value); + m_chunk_size = value; + done = true; + break; + case argument_iothreads: + size_to_uint64(bp + device_options[i].compare_size, &value); + m_io_threads = value & 0xFF; + done = true; + break; + case argument_ioslots: + size_to_uint64(bp + device_options[i].compare_size, &value); + m_io_slots = value & 0xFF; + done = true; + break; + case argument_mmap: + m_use_mmap = true; + done = true; + break; default: break; } @@ -191,14 +628,6 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) Emsg0(M_FATAL, 0, errmsg); goto bail_out; } - - /* - * Strip any .profile prefix from the libdroplet profile name. - */ - len = strlen(m_profile); - if (len > 8 && bstrcasecmp(m_profile + (len - 8), ".profile")) { - m_profile[len - 8] = '\0'; - } } /* @@ -206,18 +635,54 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) */ if (!m_ctx) { char *bp; + POOL_MEM temp(PM_NAME); + + /* + * Setup global sysmd settings which are cloned for each operation. + */ + memset(&m_sysmd, 0, sizeof(m_sysmd)); + if (m_location) { + pm_strcpy(temp, m_location); + m_sysmd.mask |= DPL_SYSMD_MASK_LOCATION_CONSTRAINT; + m_sysmd.location_constraint = dpl_location_constraint(temp.c_str()); + if (m_sysmd.location_constraint == -1) { + Mmsg2(errmsg, _("Illegal location argument %s for device %s%s\n"), temp.c_str(), dev_name); + goto bail_out; + } + } + + if (m_canned_acl) { + pm_strcpy(temp, m_canned_acl); + m_sysmd.mask |= DPL_SYSMD_MASK_CANNED_ACL; + m_sysmd.canned_acl = dpl_canned_acl(temp.c_str()); + if (m_sysmd.canned_acl == -1) { + Mmsg2(errmsg, _("Illegal canned_acl argument %s for device %s%s\n"), temp.c_str(), dev_name); + goto bail_out; + } + } + + if (m_storage_class) { + pm_strcpy(temp, m_storage_class); + m_sysmd.mask |= DPL_SYSMD_MASK_STORAGE_CLASS; + m_sysmd.storage_class = dpl_storage_class(temp.c_str()); + if (m_sysmd.storage_class == -1) { + Mmsg2(errmsg, _("Illegal storage_class argument %s for device %s%s\n"), temp.c_str(), dev_name); + goto bail_out; + } + } /* * See if this is a path. */ - bp = strrchr(m_object_configstring, '/'); + pm_strcpy(temp, m_profile); + bp = strrchr(temp.c_str(), '/'); if (!bp) { /* * Only a profile name. */ - m_ctx = dpl_ctx_new(NULL, m_object_configstring); + m_ctx = dpl_ctx_new(NULL, temp.c_str()); } else { - if (bp == m_object_configstring) { + if (bp == temp.c_str()) { /* * Profile in root of filesystem */ @@ -227,7 +692,7 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) * Profile somewhere else. */ *bp++ = '\0'; - m_ctx = dpl_ctx_new(m_object_configstring, bp); + m_ctx = dpl_ctx_new(temp.c_str(), bp); } } @@ -236,13 +701,15 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) */ if (!m_ctx) { Mmsg1(errmsg, _("Failed to create a new context using config %s\n"), dev_options); - return -1; + Dmsg1(100, "%s", errmsg); + goto bail_out; } /* * Login if that is needed for this backend. */ status = dpl_login(m_ctx); + switch (status) { case DPL_SUCCESS: break; @@ -252,76 +719,42 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) */ break; default: - Mmsg2(errmsg, _("Failed to login for voume %s using dpl_login(): ERR=%s.\n"), + Mmsg2(errmsg, _("Failed to login for volume %s using dpl_login(): ERR=%s.\n"), getVolCatName(), dpl_status_str(status)); - return -1; + Dmsg1(100, "%s", errmsg); + goto bail_out; } /* * If a bucketname was defined set it in the context. */ if (m_object_bucketname) { - m_ctx->cur_bucket = m_object_bucketname; + m_ctx->cur_bucket = bstrdup(m_object_bucketname); } } - /* - * See if we don't have a file open already. - */ - if (m_vfd) { - dpl_close(m_vfd); - m_vfd = NULL; - } + return true; - /* - * Create some options for libdroplet. - * - * DPL_OPTION_NOALLOC - we provide the buffer to copy the data into - * no need to let the library allocate memory we - * need to free after copying the data. - */ - memset(&dpl_options, 0, sizeof(dpl_options)); - dpl_options.mask |= DPL_OPTION_NOALLOC; +bail_out: + return false; +} - if (flags & O_CREAT) { - dpl_flags = DPL_VFILE_FLAG_CREAT | DPL_VFILE_FLAG_RDWR; - status = dpl_open(m_ctx, /* context */ - getVolCatName(), /* locator */ - dpl_flags, /* flags */ - &dpl_options, /* options */ - NULL, /* condition */ - NULL, /* metadata */ - NULL, /* sysmd */ - NULL, /* query_params */ - NULL, /* stream_status */ - &m_vfd); - } else { - dpl_flags = DPL_VFILE_FLAG_RDWR; - status = dpl_open(m_ctx, /* context */ - getVolCatName(), /* locator */ - dpl_flags, /* flags */ - &dpl_options, /* options */ - NULL, /* condition */ - NULL, /* metadata */ - NULL, /* sysmd */ - NULL, /* query_params */ - NULL, /* stream_status */ - &m_vfd); - } +/* + * Open a volume using libdroplet. + */ +int object_store_device::d_open(const char *pathname, int flags, int mode) +{ + int retval = -1; - switch (status) { - case DPL_SUCCESS: - m_offset = 0; - return 0; - default: - Mmsg2(errmsg, _("Failed to open %s using dpl_open(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - m_vfd = NULL; - return droplet_errno_to_system_errno(status); + if (!initialize()) { + goto bail_out; } + setup_chunk(flags); + retval = 0; + bail_out: - return -1; + return retval; } /** @@ -329,26 +762,7 @@ int object_store_device::d_open(const char *pathname, int flags, int mode) */ ssize_t object_store_device::d_read(int fd, void *buffer, size_t count) { - if (m_vfd) { - unsigned int buflen; - dpl_status_t status; - - buflen = count; - status = dpl_pread(m_vfd, count, m_offset, (char **)&buffer, &buflen); - - switch (status) { - case DPL_SUCCESS: - m_offset += buflen; - return buflen; - default: - Mmsg2(errmsg, _("Failed to read %s using dpl_read(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - return droplet_errno_to_system_errno(status); - } - } else { - errno = EBADF; - return -1; - } + return read_chunked(fd, buffer, count); } /** @@ -356,43 +770,12 @@ ssize_t object_store_device::d_read(int fd, void *buffer, size_t count) */ ssize_t object_store_device::d_write(int fd, const void *buffer, size_t count) { - if (m_vfd) { - dpl_status_t status; - - status = dpl_pwrite(m_vfd, (char *)buffer, count, m_offset); - switch (status) { - case DPL_SUCCESS: - m_offset += count; - return count; - default: - Mmsg2(errmsg, _("Failed to write %s using dpl_write(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - return droplet_errno_to_system_errno(status); - } - } else { - errno = EBADF; - return -1; - } + return write_chunked(fd, buffer, count); } int object_store_device::d_close(int fd) { - if (m_vfd) { - dpl_status_t status; - - status = dpl_close(m_vfd); - switch (status) { - case DPL_SUCCESS: - m_vfd = NULL; - return 0; - default: - m_vfd = NULL; - return droplet_errno_to_system_errno(status); - } - } else { - errno = EBADF; - return -1; - } + return close_chunk(); } int object_store_device::d_ioctl(int fd, ioctl_req_t request, char *op) @@ -401,33 +784,66 @@ int object_store_device::d_ioctl(int fd, ioctl_req_t request, char *op) } /** - * Open a directory on the object store and find out size information for a file. + * Open a directory on the object store and find out size information for a volume. */ -static inline size_t object_store_get_file_size(dpl_ctx_t *ctx, const char *filename) +ssize_t object_store_device::chunked_remote_volume_size() { - void *dir_hdl; dpl_status_t status; - dpl_dirent_t dirent; - size_t filesize = -1; + ssize_t volumesize = 0; + dpl_sysmd_t *sysmd = NULL; + POOL_MEM chunk_dir(PM_FNAME); + + Mmsg(chunk_dir, "/%s", getVolCatName()); + + /* + * FIXME: With the current version of libdroplet a dpl_getattr() on a directory + * fails with DPL_ENOENT even when the directory does exist. All other + * operations succeed and as walk_dpl_directory() does a dpl_chdir() anyway + * that will fail if the directory doesn't exist for now we should be + * mostly fine. + */ + +#if 0 + /* + * First make sure that the chunkdir exists otherwise it makes little sense to scan it. + */ + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + chunk_dir.c_str(), /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ - status = dpl_opendir(ctx, ".", &dir_hdl); switch (status) { case DPL_SUCCESS: + /* + * Make sure the filetype is a directory and not a file. + */ + if (sysmd->ftype != DPL_FTYPE_DIR) { + volumesize = -1; + goto bail_out; + } break; + case DPL_ENOENT: + volumesize = -1; + goto bail_out; default: - return -1; + break; } +#endif - while (!dpl_eof(dir_hdl)) { - if (bstrcasecmp(dirent.name, filename)) { - filesize = dirent.size; - break; - } + if (!walk_dpl_directory(m_ctx, chunk_dir.c_str(), chunked_volume_size_callback, &volumesize)) { + volumesize = -1; + goto bail_out; } - dpl_closedir(dir_hdl); +bail_out: + if (sysmd) { + dpl_sysmd_free(sysmd); + } - return filesize; + Dmsg2(100, "Volume size of volume %s, %lld\n", chunk_dir.c_str(), volumesize); + + return volumesize; } boffset_t object_store_device::d_lseek(DCR *dcr, boffset_t offset, int whence) @@ -440,11 +856,14 @@ boffset_t object_store_device::d_lseek(DCR *dcr, boffset_t offset, int whence) m_offset += offset; break; case SEEK_END: { - size_t filesize; + ssize_t volumesize; + + volumesize = chunked_volume_size(); + + Dmsg1(100, "Current volumesize: %lld\n", volumesize); - filesize = object_store_get_file_size(m_ctx, getVolCatName()); - if (filesize >= 0) { - m_offset = filesize + offset; + if (volumesize >= 0) { + m_offset = volumesize + offset; } else { return -1; } @@ -454,78 +873,25 @@ boffset_t object_store_device::d_lseek(DCR *dcr, boffset_t offset, int whence) return -1; } + if (!load_chunk()) { + return -1; + } + return m_offset; } bool object_store_device::d_truncate(DCR *dcr) { - /* - * libdroplet doesn't have a truncate function so unlink the volume and create a new empty one. - */ - if (m_vfd) { - dpl_status_t status; - dpl_vfile_flag_t dpl_flags; - dpl_option_t dpl_options; - - status = dpl_close(m_vfd); - switch (status) { - case DPL_SUCCESS: - m_vfd = NULL; - break; - default: - Mmsg2(errmsg, _("Failed to close %s using dpl_close(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - return false; - } - - status = dpl_unlink(m_ctx, getVolCatName()); - switch (status) { - case DPL_SUCCESS: - break; - default: - Mmsg2(errmsg, _("Failed to unlink %s using dpl_unlink(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - return false; - } - - /* - * Create some options for libdroplet. - * - * DPL_OPTION_NOALLOC - we provide the buffer to copy the data into - * no need to let the library allocate memory we - * need to free after copying the data. - */ - memset(&dpl_options, 0, sizeof(dpl_options)); - dpl_options.mask |= DPL_OPTION_NOALLOC; - - dpl_flags = DPL_VFILE_FLAG_CREAT | DPL_VFILE_FLAG_RDWR; - status = dpl_open(m_ctx, /* context */ - getVolCatName(), /* locator */ - dpl_flags, /* flags */ - &dpl_options, /* options */ - NULL, /* condition */ - NULL, /* metadata */ - NULL, /* sysmd */ - NULL, /* query_params */ - NULL, /* stream_status */ - &m_vfd); - - switch (status) { - case DPL_SUCCESS: - break; - default: - Mmsg2(errmsg, _("Failed to open %s using dpl_open(): ERR=%s.\n"), - getVolCatName(), dpl_status_str(status)); - return false; - } - } - - return true; + return truncate_chunked_volume(dcr); } object_store_device::~object_store_device() { if (m_ctx) { + if (m_object_bucketname && m_ctx->cur_bucket) { + free(m_ctx->cur_bucket); + m_ctx->cur_bucket = NULL; + } dpl_ctx_free(m_ctx); m_ctx = NULL; } @@ -546,6 +912,9 @@ object_store_device::object_store_device() { m_object_configstring = NULL; m_object_bucketname = NULL; + m_location = NULL; + m_canned_acl = NULL; + m_storage_class = NULL; m_ctx = NULL; } diff --git a/src/stored/backends/object_store_device.d/bareos-dir.d/storage/Object.conf.example b/src/stored/backends/object_store_device.d/bareos-dir.d/storage/Object.conf.example new file mode 100644 index 00000000000..2a0c0a55dea --- /dev/null +++ b/src/stored/backends/object_store_device.d/bareos-dir.d/storage/Object.conf.example @@ -0,0 +1,7 @@ +Storage { + Name = ObjectS3 + Address = "Replace this by the Bareos Storage Daemon FQDN or IP address" + Password = "Replace this by the Bareos Storage Daemon director password" + Device = ObjectStorage + Media Type = S3_File1 +} diff --git a/src/stored/backends/object_store_device.d/bareos-sd.d/device/ObjectStorage.conf.example b/src/stored/backends/object_store_device.d/bareos-sd.d/device/ObjectStorage.conf.example new file mode 100644 index 00000000000..dd4d748b098 --- /dev/null +++ b/src/stored/backends/object_store_device.d/bareos-sd.d/device/ObjectStorage.conf.example @@ -0,0 +1,26 @@ +Device { + Name = ObjectStorage + Media Type = S3_File1 + Archive Device = Object S3 Storage + # + # Config options: + # profile= - Droplet profile to use either absolute PATH or logical name (e.g. ~/.droplet/.profile + # location= - AWS location (e.g. us-east etc.) + # acl= - Canned ACL + # storageclass= - Storage Class to use. + # bucket= - Bucket to store objects in. + # chunksize= - Size of Volume Chunks (default = 10 Mb) + # iothreads= - Number of IO-threads to use for uploads (use blocking uploads if not set.) + # ioslots= - Number of IO-slots per IO-thread (default 10) + # mmap - Use mmap to allocate Chunk memory instead of malloc(). + # + Device Options = "profile=/etc/bareos/bareos-sd.d/.objectstorage/objectstorage.profile,bucket=bareos,iothreads=2" + Device Type = object + LabelMedia = yes # lets Bareos label unlabeled media + Random Access = yes + AutomaticMount = yes # when device opened, read it + RemovableMedia = no + AlwaysOpen = no + Description = "Object S3 device. A connecting Director must have the same Name and MediaType." + Maximum File Size = 200000000 # 200 MB (Allows for seeking to small portions of the Volume) +} diff --git a/src/stored/backends/object_store_device.h b/src/stored/backends/object_store_device.h index d80912ed736..dfa1077a18d 100644 --- a/src/stored/backends/object_store_device.h +++ b/src/stored/backends/object_store_device.h @@ -1,7 +1,7 @@ /* BAREOS® - Backup Archiving REcovery Open Sourced - Copyright (C) 2014-2014 Planets Communications B.V. + Copyright (C) 2014-2017 Planets Communications B.V. Copyright (C) 2014-2014 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or @@ -31,23 +31,44 @@ #include #include -class object_store_device: public DEVICE { +class object_store_device: public chunked_device { private: + /* + * Private Members + */ char *m_object_configstring; - char *m_profile; - char *m_object_bucketname; + const char *m_profile; + const char *m_location; + const char *m_canned_acl; + const char *m_storage_class; + const char *m_object_bucketname; dpl_ctx_t *m_ctx; - dpl_vfile_t *m_vfd; - boffset_t m_offset; + dpl_sysmd_t m_sysmd; + + /* + * Private Methods + */ + bool initialize(); + + /* + * Interface from chunked_device + */ + bool flush_remote_chunk(chunk_io_request *request); + bool read_remote_chunk(chunk_io_request *request); + ssize_t chunked_remote_volume_size(); + bool truncate_remote_chunked_volume(DCR *dcr); public: + /* + * Public Methods + */ object_store_device(); ~object_store_device(); /* * Interface from DEVICE */ - int d_close(int); + int d_close(int fd); int d_open(const char *pathname, int flags, int mode); int d_ioctl(int fd, ioctl_req_t request, char *mt = NULL); boffset_t d_lseek(DCR *dcr, boffset_t offset, int whence); diff --git a/src/stored/backends/unix_tape_device.c b/src/stored/backends/unix_tape_device.c index acae7bc4c28..336628c824b 100644 --- a/src/stored/backends/unix_tape_device.c +++ b/src/stored/backends/unix_tape_device.c @@ -22,6 +22,19 @@ */ /* * Marco van Wieringen, December 2013 + * + * UNIX Tape API device abstraction. + * + * Stacking is the following: + * + * unix_tape_device:: + * | + * v + * generic_tape_device:: + * | + * v + * DEVICE:: + * */ /** * @file diff --git a/src/stored/dev.c b/src/stored/dev.c index 7be89f41a84..02e70daeb3f 100644 --- a/src/stored/dev.c +++ b/src/stored/dev.c @@ -80,6 +80,7 @@ #include "backends/gfapi_device.h" #endif #ifdef HAVE_OBJECTSTORE +#include "backends/chunked_device.h" #include "backends/object_store_device.h" #endif #ifdef HAVE_RADOS @@ -163,8 +164,8 @@ static inline DEVICE *m_init_dev(JCR *jcr, DEVRES *device, bool new_init) device->dev_type = B_FIFO_DEV; } else if (!bit_is_set(CAP_REQMOUNT, device->cap_bits)) { Jmsg2(jcr, M_ERROR, 0, - _("%s is an unknown device type. Must be tape or directory, st_mode=%x\n"), - device->device_name, statp.st_mode); + _("%s is an unknown device type. Must be tape or directory, st_mode=%04o\n"), + device->device_name, (statp.st_mode & ~S_IFMT)); return NULL; } } @@ -588,7 +589,7 @@ bool DEVICE::open(DCR *dcr, int omode) */ clone_bits(ST_MAX, preserve, state); - Dmsg2(100, "preserve=0x%x fd=%d\n", preserve, m_fd); + Dmsg2(100, "preserve=%08o fd=%d\n", preserve, m_fd); return m_fd >= 0; } @@ -653,7 +654,7 @@ void DEVICE::open_device(DCR *dcr, int omode) /* * If creating file, give 0640 permissions */ - Dmsg3(100, "open disk: mode=%s open(%s, 0x%x, 0640)\n", mode_to_str(omode), + Dmsg3(100, "open disk: mode=%s open(%s, %08o, 0640)\n", mode_to_str(omode), archive_name.c_str(), oflags); if ((m_fd = d_open(archive_name.c_str(), oflags, 0640)) < 0) { diff --git a/src/stored/dev.h b/src/stored/dev.h index 6349eb6357e..264ce0ae2c7 100644 --- a/src/stored/dev.h +++ b/src/stored/dev.h @@ -252,11 +252,21 @@ struct BLOCKSIZES { uint32_t min_block_size; }; -class DEVRES; /* Device resource defined in stored_conf.h */ +class DEVRES; /* Forward reference Device resource defined in stored_conf.h */ class DCR; /* Forward reference */ class VOLRES; /* Forward reference */ /** + * Device specific status information either returned via DEVICE::device_status() + * method of via bsdEventDriveStatus and bsdEventVolumeStatus plugin events. + */ +typedef struct DevStatTrigger { + DEVRES *device; + POOLMEM *status; + int status_length; +} bsdDevStatTrig; + +/* * Device structure definition. * * There is one of these for each physical device. Everything here is "global" to @@ -506,6 +516,7 @@ class DEVICE: public SMARTALLOC { virtual bool reposition(DCR *dcr, uint32_t rfile, uint32_t rblock); virtual bool mount_backend(DCR *dcr, int timeout) { return true; }; virtual bool unmount_backend(DCR *dcr, int timeout) { return true; }; + virtual bool device_status(bsdDevStatTrig *dst) { return false; }; boffset_t lseek(DCR *dcr, boffset_t offset, int whence) { return d_lseek(dcr, offset, whence); }; bool truncate(DCR *dcr) { return d_truncate(dcr); }; diff --git a/src/stored/lock.c b/src/stored/lock.c index 0a50228a6b8..5667dde175b 100644 --- a/src/stored/lock.c +++ b/src/stored/lock.c @@ -392,14 +392,13 @@ void DEVICE::rLock(bool locked) num_waiting++; /* indicate that I am waiting */ while (blocked()) { int status; -#ifndef HAVE_WIN32 - /* - * thread id on Win32 may be a struct - */ - Dmsg3(sd_dbglvl, "rLock blked=%s no_wait=%p me=%p\n", print_blocked(), - no_wait_id, pthread_self()); -#endif - if ((status = pthread_cond_wait(&this->wait, &m_mutex)) != 0) { + char ed1[50], ed2[50]; + + Dmsg3(sd_dbglvl, "rLock blked=%s no_wait=%s me=%s\n", + print_blocked(), + edit_pthread(no_wait_id, ed1, sizeof(ed1)), + edit_pthread(pthread_self(), ed2, sizeof(ed2))); + if ((status = pthread_cond_wait(&wait, &m_mutex)) != 0) { berrno be; this->Unlock(); Emsg1(M_ABORT, 0, _("pthread_cond_wait failure. ERR=%s\n"), diff --git a/src/stored/mount.c b/src/stored/mount.c index 877cf5bc772..6ba50a03928 100644 --- a/src/stored/mount.c +++ b/src/stored/mount.c @@ -692,7 +692,7 @@ bool DCR::is_eod_valid() char ed1[50], ed2[50]; boffset_t pos; - pos = dev->lseek(dcr, (boffset_t)0, SEEK_END); + pos = dev->lseek(dcr, (boffset_t)0, SEEK_CUR); if (dev->VolCatInfo.VolCatBytes == (uint64_t)pos) { Jmsg(jcr, M_INFO, 0, _("Ready to append to end of Volume \"%s\"" " size=%s\n"), diff --git a/src/stored/sd_plugins.h b/src/stored/sd_plugins.h index 6020c1fe48d..e61357ed534 100644 --- a/src/stored/sd_plugins.h +++ b/src/stored/sd_plugins.h @@ -200,13 +200,6 @@ typedef struct s_sdpluginFuncs { #define sdplug_func(plugin) ((psdFuncs *)(plugin->pfuncs)) #define sdplug_info(plugin) ((genpInfo *)(plugin->pinfo)) -class DEVRES; -typedef struct s_sdbareosDevStatTrigger { - DEVRES *device; - POOLMEM *status; - int status_length; -} bsdDevStatTrig; - #ifdef __cplusplus } #endif diff --git a/src/stored/status.c b/src/stored/status.c index fe28934b804..272732d63a2 100644 --- a/src/stored/status.c +++ b/src/stored/status.c @@ -54,10 +54,6 @@ static void sendit(const char *msg, int len, STATUS_PKT *sp); static void sendit(POOL_MEM &msg, int len, STATUS_PKT *sp); static void sendit(const char *msg, int len, void *arg); -static void trigger_device_status_hook(JCR *jcr, - DEVRES *device, - STATUS_PKT *sp, - bsdEventType eventType); static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp); static void send_device_status(DEVICE *dev, STATUS_PKT *sp); static void list_terminated_jobs(STATUS_PKT *sp); @@ -222,6 +218,49 @@ static bool need_to_list_device(const char *devicenames, DEVRES *device) return true; } +/* + * Trigger the specific eventtype to get status information from any plugin that + * registered the event to return specific device information. + */ +static void trigger_device_status_hook(JCR *jcr, + DEVRES *device, + STATUS_PKT *sp, + bsdEventType eventType) +{ + bsdDevStatTrig dst; + + dst.device = device; + dst.status = get_pool_memory(PM_MESSAGE); + dst.status_length = 0; + + if (generate_plugin_event(jcr, eventType, &dst) == bRC_OK) { + if (dst.status_length > 0) { + sendit(dst.status, dst.status_length, sp); + } + } + free_pool_memory(dst.status); +} + +/* + * Ask the device if it want to log something specific in the status overview. + */ +static void get_device_specific_status(DEVRES *device, + STATUS_PKT *sp) +{ + bsdDevStatTrig dst; + + dst.device = device; + dst.status = get_pool_memory(PM_MESSAGE); + dst.status_length = 0; + + if (device->dev->device_status(&dst)) { + if (dst.status_length > 0) { + sendit(dst.status, dst.status_length, sp); + } + } + free_pool_memory(dst.status); +} + static void list_devices(JCR *jcr, STATUS_PKT *sp, const char *devicenames) { int len; @@ -308,7 +347,9 @@ static void list_devices(JCR *jcr, STATUS_PKT *sp, const char *devicenames) sendit(msg, len, sp); } + get_device_specific_status(device, sp); trigger_device_status_hook(jcr, device, sp, bsdEventDriveStatus); + send_blocked_status(dev, sp); if (dev->can_append()) { @@ -354,6 +395,8 @@ static void list_devices(JCR *jcr, STATUS_PKT *sp, const char *devicenames) len = Mmsg(msg, _("\nDevice \"%s\" is not open or does not exist.\n"), device->name()); sendit(msg, len, sp); } + + get_device_specific_status(device, sp); } if (!sp->api) { @@ -514,25 +557,6 @@ static void list_status_header(STATUS_PKT *sp) } } -static void trigger_device_status_hook(JCR *jcr, - DEVRES *device, - STATUS_PKT *sp, - bsdEventType eventType) -{ - bsdDevStatTrig dst; - - dst.device = device; - dst.status = get_pool_memory(PM_MESSAGE); - dst.status_length = 0; - - if (generate_plugin_event(jcr, eventType, &dst) == bRC_OK) { - if (dst.status_length > 0) { - sendit(dst.status, dst.status_length, sp); - } - } - free_pool_memory(dst.status); -} - static void send_blocked_status(DEVICE *dev, STATUS_PKT *sp) { int len; diff --git a/src/win32/stored/backends/win32_tape_device.c b/src/win32/stored/backends/win32_tape_device.c index 132ed6f3015..ece58fd543b 100644 --- a/src/win32/stored/backends/win32_tape_device.c +++ b/src/win32/stored/backends/win32_tape_device.c @@ -24,6 +24,16 @@ * Kern Sibbald, MM * Robert Nelson, May, 2006 * Extracted from other source files Marco van Wieringen, December 2013 + * + * Stacking is the following: + * + * win32_tape_device:: + * | + * v + * generic_tape_device:: + * | + * v + * DEVICE:: */ /** * @file diff --git a/test/travis_before_install.sh b/test/travis_before_install.sh new file mode 100755 index 00000000000..bca62bd2811 --- /dev/null +++ b/test/travis_before_install.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +sudo apt-get -qq update +dpkg-checkbuilddeps 2> /tmp/dpkg-builddeps || true +cat /tmp/dpkg-builddeps +sed -e "s/^.*:.*:\s//" -e "s/\s([^)]*)//g" /tmp/dpkg-builddeps > /tmp/build_depends +echo "additional packages required for building:"; cat /tmp/build_depends +sudo xargs --arg-file /tmp/build_depends apt-get -q --assume-yes install fakeroot +dpkg -l + diff --git a/test/travis_before_script.sh b/test/travis_before_script.sh new file mode 100755 index 00000000000..d2b9d07c1a0 --- /dev/null +++ b/test/travis_before_script.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +print_header() +{ + TEXT="$1" + printf "#\n" + printf "# %s\n" "$TEXT" + printf "#\n" +} + +if [ "${COVERITY_SCAN}" ]; then + # run configure with default options + debian/rules override_dh_auto_configure + eval "$COVERITY_SCAN_BUILD" +else + print_header "build Bareos packages" + fakeroot debian/rules binary + + print_header "create Debian package repository" + cd .. + dpkg-scanpackages . > Packages + gzip --keep Packages + ls -la Packages* + printf 'deb file:%s /\n' $PWD > /tmp/bareos.list + sudo cp /tmp/bareos.list /etc/apt/sources.list.d/bareos.list + cd - + + print_header "install Bareos packages" + sudo apt-get -qq update + sudo apt-get install -y --force-yes bareos bareos-database-$DB +fi +