Skip to content

Commit

Permalink
added backup status (ok, ko, uploading, cleaning), recursive path now…
Browse files Browse the repository at this point in the history 
… working (except for gdrive with _ instead of folders), guess date from filename now the default one (fallback to remote modified date), source default value is jeedom backup path
  • Loading branch information
@fensoft
fensoft committed Oct 6, 2017
1 parent 66af442 commit d7e4f3f
Show file tree
Hide file tree
Showing 8 changed files with 183 additions and 86 deletions.
179 changes: 116 additions & 63 deletions core/class/datatransfert.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,37 +16,30 @@
* along with Jeedom. If not, see <http://www.gnu.org/licenses/>.
*/

/* * ***************************Includes********************************* */
require_once dirname(__FILE__) . '/../../../../core/php/core.inc.php';
require_once dirname(__FILE__) . '/../../core/php/datatransfert.inc.php';

class datatransfert extends eqLogic {
/* * *************************Attributs****************************** */


/* * ***********************Methode static*************************** */


public static function dependancy_info() {
$return = array();
$return['log'] = __CLASS__ . '_update';
$return['progress_file'] = jeedom::getTmpFolder(__CLASS__) . '_progress';
$state = '';
if (file_exists(dirname(__FILE__) . "/../../external/rclone/rclone")) {
$state = 'ok';
} else {
$state = 'nok';
}
$return['state'] = $state;
return $return;
}
public static function dependancy_info() {
$return = array();
$return['log'] = __CLASS__ . '_update';
$return['progress_file'] = jeedom::getTmpFolder(__CLASS__) . '_progress';
$state = '';
if (file_exists(dirname(__FILE__) . "/../../external/rclone/rclone")) {
$state = 'ok';
} else {
$state = 'nok';
}
$return['state'] = $state;
return $return;
}

public static function dependancy_install() {
log::remove(__CLASS__ . '_update');
$cmd = dirname(__FILE__) . '/../../external/rclone/download.sh';
$cmd .= ' ' . jeedom::getTmpFolder(__CLASS__) . '_progress';
return array('script' => $cmd, 'log' => log::getPathToLog(__CLASS__ . '_update'));
}
public static function dependancy_install() {
log::remove(__CLASS__ . '_update');
$cmd = dirname(__FILE__) . '/../../external/rclone/download.sh';
$cmd .= ' ' . jeedom::getTmpFolder(__CLASS__) . '_progress';
return array('script' => $cmd, 'log' => log::getPathToLog(__CLASS__ . '_update'));
}

public static function supportedProtocol() {
$return = array();
Expand All @@ -57,59 +50,119 @@ public static function supportedProtocol() {
return $return;
}

/* * *********************Methode d'instance************************* */

/* * **********************Getteur Setteur*************************** */
public function postSave() {
foreach ($this->getCmd() as $cmd) {
if (strpos($cmd->getName(), "_status") !== false)
continue;
$id = $cmd->getName() . "_status";
$logic = cmd::byEqLogicIdAndLogicalId($this->getId(), $id);
if (!is_object($logic)) {
$cmd = new cmd();
$cmd->setEventOnly(1);
$cmd->setIsHistorized(1);
$cmd->setOrder(count($this->getCmd()));
$cmd->setEqLogic_id($this->getId());
$cmd->setEqType('datatransfertInfo');
$cmd->setLogicalId($id);
$cmd->setName($cmd->getLogicalId());
$cmd->setType('info');
$cmd->setSubType('string');
$cmd->setIsVisible(false);
$cmd->save();
$logic = $cmd;
}
}
}

public function setUploadStatus($name, $status) {
$logic = cmd::byEqLogicIdAndLogicalId($this->getId(), $name . "_status");
if ($logic)
$logic->event($status);
else
\log::add('datatransfert', 'info', "missing: " . $name);
}
}

class datatransfertCmd extends cmd {
/* * *************************Attributs****************************** */


/* * ***********************Methode static*************************** */

public static function orderFile($a, $b) {
if ($a['datetime'] == $b['datetime']) {
return 0;
}
return ($a['datetime'] < $b['datetime']) ? +1 : -1;
}

/* * *********************Methode d'instance************************* */
public static function getDirContents($dir, &$results = array()){
$files = scandir($dir);
foreach($files as $key => $value){
$path = realpath($dir."/".$value);
if(!is_dir($path)) {
$results[] = $path;
} else if($value != "." && $value != "..") {
self::getDirContents($path, $results);
//$results[] = $path;
}
}

return $results;
}

public function ls($dir, $filter) {
$lst = array();
self::getDirContents($dir, $lst);
$res = array();
foreach ($lst as $file) {
if (fnmatch($filter, $file))
array_push($res, str_replace($dir . "/", "", $file));
}
return $res;
}

public function execute($_options = null) {
$eqLogic = $this->getEqLogic();
$protocol = $eqLogic->getConfiguration('protocol');
include_file('core', $protocol . '.protocol', 'php', 'datatransfert');
$class = call_user_func('DataTransfert\\' . $protocol . '::withEqLogic', $eqLogic);
$cible = $this->getConfiguration('cible');
$source = calculPath($this->getConfiguration('source'));
$filelist = array();
$filter_recentfile = $this->getConfiguration('filter_recentfile');
if ($this->getConfiguration('filter_recentfile') != '') {
foreach (ls($source, $this->getConfiguration('filter_file', '*')) as $file) {
$filelist[] = array(
'file' => $file,
'datetime' => filemtime($source . '/' . $file)
);
try {
$eqLogic = $this->getEqLogic();
$eqLogic->setUploadStatus($this->getName(), "uploading");
$protocol = $eqLogic->getConfiguration('protocol');
include_file('core', $protocol . '.protocol', 'php', 'datatransfert');
$class = call_user_func('DataTransfert\\' . $protocol . '::withEqLogic', $eqLogic);
$cible = $this->getConfiguration('cible');
$source = calculPath($this->getConfiguration('source'));
$res = array();
$filter_recentfile = $this->getConfiguration('filter_recentfile');
//$this->ls($source, $this->getConfiguration('filter_file', '*'));
if ($this->getConfiguration('filter_recentfile') != '') {
$filelist = array();
foreach ($this->ls($source, $this->getConfiguration('filter_file', '*')) as $file) {
$filelist[] = array(
'file' => $file,
'datetime' => filemtime($source . '/' . $file)
);
}
usort($filelist, 'datatransfertCmd::orderFile');
foreach (array_slice($filelist, 0, $this->getConfiguration('filter_recentfile')) as $file)
array_push($res, $file['file']);
} else {
$res = $this->ls($source, $this->getConfiguration('filter_file', '*'));
}
usort($filelist, 'datatransfertCmd::orderFile');
$filelist = array_slice($filelist, 0, $this->getConfiguration('filter_recentfile'));
foreach ($filelist as $file) {
\log::add('datatransfert', 'info', "uploading " . $source . "/" . $file['file'] . " to " . $cible . "/" . $file['file']);
$class->put($source . "/" . $file['file'], $cible . "/" . $file['file']);
\log::add('datatransfert', 'info', "upload " . $source . "/" . $file['file'] . " to " . $cible . "/" . $file['file'] . " complete !");
}
} else {
foreach (ls($source, $this->getConfiguration('filter_file', '*')) as $file) {
$class->put($source . '/' . $file, $cible . '/' . $file);
foreach ($res as $file) {
\log::add('datatransfert', 'info', "uploading " . $source . "/" . $file . " to " . $cible . "/" . $file);
if (dirname($file) != "")
$class->mkdir(dirname($cible . "/" . $file));
$class->put($source . "/" . $file, $cible . "/" . $file);
\log::add('datatransfert', 'info', "upload " . $source . "/" . $file . " to " . $cible . "/" . $file . " complete !");
}
$eqLogic->setUploadStatus($this->getName(), "cleaning");
if ($this->getConfiguration('remove_old') != "")
$class->removeOlder($cible, $this->getConfiguration('remove_old'));
$eqLogic->setUploadStatus($this->getName(), "ok");
} catch (Exception $e) {
$eqLogic->setUploadStatus($this->getName(), "ko");
throw $e;
}
if ($this->getConfiguration('remove_old') != "")
$class->removeOlder($cible, $this->getConfiguration('remove_old'));
}

/* * **********************Getteur Setteur*************************** */
}

class datatransfertInfo extends cmd {
}
class datatransfertInfoCmd extends cmd {
}
?>
46 changes: 35 additions & 11 deletions core/php/datatransfert.inc.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ function timesort($a, $b)
}

class DataTransfert {
static function guessTimestamp($_name) {
$formats = array("*-*-*.*.*-Y-m-d-H?i.*.*" => "backup-fensoft-3.1.5-2017-10-04-11h52.tar.gz");
foreach ($formats as $format => $example) {
$date = \DateTime::createFromFormat($format, $_name);
if ($date)
return $date->format('U');
}
return null;
}

static function withEqLogic($_eqLogic) {
\log::add('datatransfert', 'error', "withEqLogic unimplemented");
}
Expand All @@ -45,23 +55,30 @@ function remove($_cible) {
\log::add('datatransfert', 'error', "remove unimplemented");
}

function mkdir($_cible) {
\log::add('datatransfert', 'error', "mkdir unimplemented");
}

function removeOlder($_cible, $numberToKeep) {
\log::add('datatransfert', 'info', "removing old files except " . $numberToKeep . " in " . $_cible);
$ls = $this->ls($_cible);
$ls2 = array();
$lsskipped = array();
$ls2 = array();
$lsskipped = array();
foreach ($ls as $val) {
$guessed = $this->guessTimestamp($val["name"]);
if ($guessed != null)
$val["time"] = $guessed;
if ($val["time"] == null)
array_push($lsskipped, $val["name"]);
else
array_push($ls2, $val);
array_push($ls2, $val);
}
if (count($lsskipped) != 0)
\log::add('datatransfert', 'info', "unknown time, clean skipped for " . implode(',', $lsskipped));
if (count($lsskipped) != 0)
\log::add('datatransfert', 'info', "unknown time, clean skipped for " . implode(',', $lsskipped));
usort($ls2, "\\DataTransfert\\timesort");
$todel = array_slice($ls2, 0, -$numberToKeep);
foreach ($todel as $val) {
\log::add('datatransfert', 'info', "removing " . $_cible);
\log::add('datatransfert', 'info', "removing " . $_cible);
$this->remove($_cible . "/" . $val["name"]);
}
}
Expand All @@ -83,9 +100,11 @@ function dirname($_file) {
function put($_source, $_cible) {
if (isset($this->removeDupes) && $this->removeDupes == true) {
foreach ($this->ls(dirname($_cible)) as $val) {
if ($val["alias"] == basename($_cible)) {
$this->remove(dirname($_cible) . "/" . $val["name"]);
}
\log::add('datatransfert', 'debug', "dupes " . $val["alias"] . "==" . basename($_cible));

if ($val["alias"] == basename($_cible)) {
$this->remove(dirname($_cible) . "/" . $val["name"]);
}
}
}
$filesystem = $this->getFly($this->dirname($_cible));
Expand All @@ -102,16 +121,21 @@ function ls($_source) {
foreach ($filesystem->listContents($this->basename($_source), false) as $val) {
if ($val["type"] == "file") {
\log::add('datatransfert', 'debug', "list " . json_encode($val));
array_push($res, array("name" => $val["basename"], "alias" => $val["filename"], "time" => $this->timestamp($val)));
array_push($res, array("name" => $val["basename"], "alias" => $val["filename"] . ($val["extension"]==""?"":".".$val["extension"]), "time" => $this->timestamp($val)));
}
}
\log::add('datatransfert', 'debug', "list " . json_encode($res));
\log::add('datatransfert', 'debug', "list " . json_encode($res));
return $res;
}

function remove($_cible) {
$filesystem = $this->getFly($this->dirname($_cible));
$filesystem->delete($this->basename($_cible));
}

function mkdir($_cible) {
$filesystem = $this->getFly("");
$filesystem->createDir($_cible);
}
}
?>
6 changes: 0 additions & 6 deletions core/php/ftp.protocol.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,6 @@ function getFly($_base) {
}

function timestamp($_val) {
$formats = array("*-*-*.*.*-Y-m-d-H?i.*.*" => "backup-fensoft-3.1.5-2017-10-04-11h52.tar.gz");
foreach ($formats as $format => $example) {
$date = \DateTime::createFromFormat($format, $_val["basename"]);
if ($date)
return $date->format('U');
}
return null;
}
}
7 changes: 7 additions & 0 deletions core/php/gdrive.protocol.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,11 @@ function getFly($_base) {
$adapter = new \Hypweb\Flysystem\GoogleDrive\GoogleDriveAdapter($service, $_base);
return new \League\Flysystem\Filesystem($adapter);
}

function put($_source, $_cible) {
parent::put($_source, explode("/", $_cible)[0] . "/" . implode("_", array_slice(explode("/", $_cible), 1)));
}

function mkdir($_cible) {
}
}
4 changes: 4 additions & 0 deletions core/php/local.protocol.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,8 @@ function ls($_cible) {
function remove($_cible) {
unlink($_cible);
}

function mkdir($_cible) {
@mkdir($_cible, 0755, true);
}
}
4 changes: 4 additions & 0 deletions core/php/rclone.protocol.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,8 @@ function ls($_source) {
function remove($_source) {
$this->rclone->delete($_source);
}

function mkdir($_source) {
$this->rclone->mkdir($_source);
}
}
19 changes: 13 additions & 6 deletions desktop/js/datatransfert.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,22 +54,29 @@ function addCmdToTable(_cmd) {
tr += '<input class="cmdAttr form-control input-sm" data-l1key="name">';
tr += '</td>';
tr += '<td>';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="source" placeholder="{{Source}}">';
if (init(_cmd.eqType) != 'datatransfertInfo') {
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="source" placeholder="{{Source}}" value="' + backup_path + '">';
}
tr += '</td>';
tr += '<td>';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="cible" placeholder="{{Cible}}">';
if (init(_cmd.eqType) != 'datatransfertInfo') {
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="cible" placeholder="{{Cible}}">';
}
tr += '</td>';
tr += '<td>';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="filter_file" placeholder="{{Filtre sur fichier}}">';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="filter_recentfile" placeholder="{{n\'envoyer que X fichiers + recent}}" style="margin-top : 5px;">';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="remove_old" placeholder="{{ne conserver que X fichiers + recent}}" style="margin-top : 5px;">';
if (init(_cmd.eqType) != 'datatransfertInfo') {
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="filter_file" placeholder="{{Filtre sur fichier}}">';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="filter_recentfile" placeholder="{{n\'envoyer que X fichiers + recent}}" style="margin-top : 5px;">';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="configuration" data-l2key="remove_old" placeholder="{{ne conserver que X fichiers + recent}}" style="margin-top : 5px;">';
}
tr += '</td>';
tr += '<td>';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="type" value="action" style="display : none;">';
tr += '<input class="cmdAttr form-control input-sm" data-l1key="subType" value="other" style="display : none;">';
if (is_numeric(_cmd.id)) {
tr += '<a class="btn btn-default btn-xs cmdAction expertModeVisible" data-action="configure"><i class="fa fa-cogs"></i></a> ';
tr += '<a class="btn btn-default btn-xs cmdAction" data-action="test"><i class="fa fa-rss"></i> {{Tester}}</a>';
if (init(_cmd.eqType) != 'datatransfertInfo')
tr += '<a class="btn btn-default btn-xs cmdAction" data-action="test"><i class="fa fa-rss"></i> {{Tester}}</a>';
}
tr += '<i class="fa fa-minus-circle pull-right cmdAction cursor" data-action="remove"></i></td>';
tr += '</tr>';
Expand Down
4 changes: 4 additions & 0 deletions desktop/php/datatransfert.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
$eqLogics = eqLogic::byType('datatransfert');
?>

<script type="text/javascript">
var backup_path = "<?php echo realpath(calculPath(config::byKey('backup::path'))); ?>";
</script>

<div class="row row-overflow">
<div class="col-lg-2 col-md-3 col-sm-4">
<div class="bs-sidebar">
Expand Down

0 comments on commit d7e4f3f

Please sign in to comment.