Basic webDAV support for backups + Backup storage stats

createlinks

@@ -30,6 +30,8 @@ my $event = "nethserver-backup-data-update";
 
 event_actions($event, 'initialize-default-databases' => '00');
 templates2events("/etc/cron.d/backup-data",  $event);
+templates2events("/etc/davfs2/davfs2.conf",  $event);
+templates2events("/etc/davfs2/secrets",  $event);
 
 
 #--------------------------------------------------
@@ -38,7 +40,8 @@ templates2events("/etc/cron.d/backup-data",  $event);
 
 $event = "nethserver-backup-data-save";
 templates2events("/etc/cron.d/backup-data",  $event);
-
+templates2events("/etc/davfs2/davfs2.conf",  $event);
+templates2events("/etc/davfs2/secrets",  $event);
 
 #--------------------------------------------------
 # actions for nethserver-backup-data-uninstall event
@@ -56,6 +59,7 @@ event_actions($event,
    'mount-usb' => '70',
    'mount-cifs' => '70',
    'mount-nfs' => '70',
+   'mount-webdav' => '70',
 );
 
 #--------------------------------------------------
@@ -65,9 +69,11 @@ event_actions($event,
 $event = "post-backup-data";
 event_actions($event, 
    'cleanup-data-duplicity' => '70',
+   'disk-usage' => '70',
    'umount-usb' => '80',
    'umount-cifs' => '80',
    'umount-nfs' => '80',
+   'umount-webdav' => '80',
 );
 
 
@@ -80,6 +86,7 @@ event_actions($event,
    'mount-usb' => '10',
    'mount-cifs' => '10',
    'mount-nfs' => '10',
+   'mount-webdav' => '10',
 );
 
 #--------------------------------------------------
@@ -91,6 +98,7 @@ event_actions($event,
    'umount-usb' => '90',
    'umount-cifs' => '90',
    'umount-nfs' => '90',
+   'umount-webdav' => '90',
 );
 
 exit;

nethserver-backup-data.spec

@@ -8,7 +8,7 @@ URL: %{url_prefix}/%{name}
 
 BuildArch: noarch
 BuildRequires: nethserver-devtools
-Requires: cifs-utils, nfs-utils, duplicity
+Requires: cifs-utils, nfs-utils, duplicity, davfs2
 Requires: nethserver-backup-config
 
 %description
@@ -39,6 +39,7 @@ rm -rf %{buildroot}
 
 
 %changelog
+
 * Tue Jan 03 2017 Giacomo Sanchietti <giacomo.sanchietti@nethesis.it> - 1.2.3-1
 - Backup-data not run after system crash - Bug NethServer/dev#5179
 

root/etc/e-smith/db/configuration/defaults/backup-data/WebDAVLogin

@@ -0,0 +1 @@
+

root/etc/e-smith/db/configuration/defaults/backup-data/WebDAVPassword

@@ -0,0 +1 @@
+

root/etc/e-smith/db/configuration/defaults/backup-data/WebDAVUrl

@@ -0,0 +1 @@
+

root/etc/e-smith/events/actions/backup-data-duplicity

@@ -92,7 +92,7 @@ print $tmp "- **\n";
 $opts .= " --exclude=$cacheDir --include-filelist $tmp";
 $opts .= " --volsize $volSize ";
 
-if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') ) {
+if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') || ($VFSType eq 'webdav') ) {
    mkdir "$mntdir/$systemname";
    $dst = " file://$mntdir/$systemname ";
 } else {
@@ -107,7 +107,7 @@ $opts .= " --no-encryption ";
 # set duplicity cache dir
 $opts .= " --archive-dir $cacheDir ";
 
-# remove stale duplicyty lockfile
+# remove stale duplicity lockfile
 unlink glob "$cacheDir/*/lockfile.lock";
 
 # Debug 

root/etc/e-smith/events/actions/cleanup-data-duplicity

@@ -46,7 +46,7 @@ my $dst = "";
 
 our $backup = new NethServer::BackupData();
 
-if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') ) {
+if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') || ($VFSType eq 'webdav') ) {
    mkdir "$mntdir/$systemname";
    $dst = " file://$mntdir/$systemname ";
 } else {

root/etc/e-smith/events/actions/disk-usage

@@ -0,0 +1,65 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright (C) 2017 Nethesis S.r.l.
+# http://www.nethesis.it - support@nethesis.it
+#
+# This script is part of NethServer.
+#
+# NethServer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License,
+# or any later version.
+#
+# NethServer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NethServer.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+use strict;
+use Errno;
+use esmith::ConfigDB;
+use NethServer::BackupData;
+use JSON;
+
+my $confdb = esmith::ConfigDB->open;
+my $backupwk = $confdb->get('backup-data') or die "No backup-data db entry found\n";
+my $VFSType = $backupwk->prop('VFSType') || 'UNKNOWN';
+my $mntdir = $backupwk->prop('Mount') || '/mnt/backup';
+my $duLogFile = '/var/lib/nethserver/backup/disk_usage';
+my $err = 0;
+my %du;
+
+my $backup = new NethServer::BackupData();
+
+
+if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') || ($VFSType eq 'webdav') ) {
+    if ($backup->is_mounted($mntdir)) {
+
+        # get disk usage stats
+        if ( @du{qw(source size used avail pcent target)} = split(" ", `/bin/df -P "$mntdir" 2>/dev/null | grep "$mntdir"`) ) {
+            delete @du{'source', 'target'};
+            $du{'pcent'} = sprintf "%.2f", $du{'used'}/$du{'size'}*100;
+            $du{'pcent'} += 0;
+
+            if ( $du{'used'} lt 0 || $du{'avail'} lt 0 ) {
+                warn("[WARNING] Unable to determine backup storage capacity.\n");
+                @du{'size', 'used', 'avail', 'pcent'} = undef;
+            }
+
+            # output disk usage to file
+            open(FILE, ">$duLogFile");
+            print FILE encode_json(\%du);
+            close(FILE);
+        }
+    }
+} else {
+   $backup->logger("ERROR","No valid VFSType ($VFSType) - 1");
+   exit 1;
+}
+
+exit 0;

root/etc/e-smith/events/actions/mount-webdav

@@ -0,0 +1,83 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright (C) 2017 Nethesis S.r.l.
+# http://www.nethesis.it - support@nethesis.it
+#
+# Original work:
+# copyright (C) 2006-2007 Jean-Paul Leclere <jean-paul@leclere.org>
+# copyright (C) 2007 Charlie Brady <charlieb@e-smith.com>
+#
+# This script is part of NethServer.
+#
+# NethServer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License,
+# or any later version.
+#
+# NethServer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NethServer.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+use strict;
+use Errno;
+use File::Temp;
+use esmith::ConfigDB;
+use NethServer::BackupData();
+use File::Path qw(make_path);
+
+sub ldie;
+
+my $confdb = esmith::ConfigDB->open;
+my $backupwk = $confdb->get('backup-data') or die "No backup-data db entry found\n";
+my $webdavurl = $backupwk->prop('WebDAVUrl');
+my $login = $backupwk->prop('WebDAVLogin');
+my $password = $backupwk->prop('WebDAVPassword');
+my $VFSType = $backupwk->prop('VFSType') || 'UNKNOWN';
+our $mntdir = $backupwk->prop('Mount') || '/mnt/backup';
+my $err = 0;
+my $mntdone = 0;
+
+exit 0 unless ($VFSType eq 'webdav');
+
+our $b = new NethServer::BackupData();
+
+# creating mount directory
+if ( ! -d $mntdir ) {
+    make_path($mntdir);
+}
+
+# mounting backup directory if not already mounted
+if (!$b->is_mounted("$webdavurl") && !$b->is_mounted($mntdir)) {
+
+    # mount the backup dir
+    $err = qx(/bin/mount -t davfs "$webdavurl" $mntdir 2>&1);
+    if ($err) {
+        $err =~s/\n/ /g;
+        # workaround: do not fail on minor warnings (e.g. mtab write). Future upstream davfs2 releases will make this unnecessary.
+        if ($err !~m/warning/i) {
+          ldie("Error while mounting $webdavurl : " . $err);
+        }
+        warn($err);
+    }
+}
+
+$mntdone = 1;
+
+# verify $mntdir has an entry in /proc/mounts
+
+if (!$b->is_mounted($mntdir)) {
+    ldie("Backup directory is not mounted");
+}
+
+sub ldie 
+{
+    my $errmsg = shift;
+    system("/bin/umount", "$mntdir") if $mntdone;
+    $b->bad_exit($errmsg);
+}

root/etc/e-smith/events/actions/restore-config-duplicity

@@ -48,7 +48,7 @@ my $backup = new NethServer::BackupData();
 $backup->set_log_file('/var/log/restore-data.log');
 
 
-if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') ) {
+if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') || ($VFSType eq 'webdav') ) {
    $src = " file://$mntdir/$systemname ";
    my $status = system("/etc/e-smith/events/actions/mount-$VFSType");
    if ($status) {

root/etc/e-smith/events/actions/restore-data-duplicity

@@ -69,7 +69,7 @@ my $backup = new NethServer::BackupData();
 $backup->set_log_file('/var/log/restore-data.log');
 
 
-if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') ) {
+if ( ($VFSType eq 'usb') || ($VFSType eq 'nfs') || ($VFSType eq 'cifs') || ($VFSType eq 'webdav') ) {
    $src = " file://$mntdir/$systemname ";
 } else {
    $backup->bad_exit("No valid VFSType ($VFSType)",1);

root/etc/e-smith/events/actions/umount-webdav

@@ -0,0 +1,44 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright (C) 2017 Nethesis S.r.l.
+# http://www.nethesis.it - support@nethesis.it
+# 
+# This script is part of NethServer.
+# 
+# NethServer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License,
+# or any later version.
+# 
+# NethServer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with NethServer.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+use strict;
+use Errno;
+use esmith::ConfigDB;
+use NethServer::BackupData;
+
+my $confdb = esmith::ConfigDB->open;
+my $backupwk = $confdb->get('backup-data') or die "No backup-data db entry found\n";
+my $VFSType = $backupwk->prop('VFSType') || 'UNKNOWN';
+my $webdavurl = $backupwk->prop('WebDAVUrl');
+my $mntdir = $backupwk->prop('Mount') || '/mnt/backup';
+my $err = 0;
+
+exit 0 unless ($VFSType eq 'webdav');
+
+my $b = new NethServer::BackupData();
+
+# check if backp directory is mounted and umount
+if ($b->is_mounted("$webdavurl") or $b->is_mounted($mntdir)) {
+    system("/bin/umount", "$mntdir");
+}
+
+exit 0;

root/etc/e-smith/templates.metadata/etc/davfs2/secrets

@@ -0,0 +1,3 @@
+PERMS=0600
+UID="root"
+GID="root"

root/etc/e-smith/templates/etc/davfs2/davfs2.conf/00template_vars

@@ -0,0 +1,5 @@
+{
+  %backup = $DB->get('backup-data')->props;
+  $OUT = '';
+  ''; # expand to an empty string
+}

root/etc/e-smith/templates/etc/davfs2/davfs2.conf/10base

@@ -0,0 +1,31 @@
+#
+# 10base
+#
+
+# davfs2 configuration file 2012-07-19
+# version 9
+# ------------------------------------
+
+# Copyright (C) 2006, 2007, 2008, 2009, 2012 Werner Baumann
+
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved.
+
+
+# Please read the davfs2.conf (5) man page for a description of the
+# configuration options and syntax rules.
+
+
+# Available options and default values
+# ====================================
+
+# General Options
+# ---------------
+
+# dav_user        davfs2            # system wide config file only
+# dav_group       davfs2            # system wide config file only
+# kernel_fs       fuse
+# buf_size        16                 # KiByte
+
+

root/etc/e-smith/templates/etc/davfs2/davfs2.conf/20webdav

@@ -0,0 +1,31 @@
+#
+# 20webdav
+#
+
+# WebDAV Related Options
+# ----------------------
+
+# use_proxy       1                 # system wide config file only
+# proxy                             # system wide config file only
+# servercert
+# clientcert
+# secrets         ~/.davfs2/secrets # user config file only
+# ask_auth        1
+# use_locks       1
+# lock_owner      <user-name>
+# lock_timeout    1800              # seconds
+# lock_refresh    60                # seconds
+# use_expect100   0
+# if_match_bug    0
+# drop_weak_etags 0
+# allow_cookie    0
+# precheck        1
+# ignore_dav_header 0
+# server_charset
+# connect_timeout 10                # seconds
+# read_timeout    30                # seconds
+# retry           30                # seconds
+# max_retry       300               # seconds
+# add_header
+
+