Skip to content
This repository has been archived by the owner on Nov 25, 2020. It is now read-only.

Commit

Permalink
Update "Observe Storage" mechanism to make it more reliable and less …
Browse files Browse the repository at this point in the history 
…frequent.
  • Loading branch information
@cdujeu
cdujeu committed Feb 27, 2015
1 parent dba284a commit 56cda77
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 9 deletions.
87 changes: 78 additions & 9 deletions core/src/plugins/meta.syncable/class.ChangesTracker.php
View file
Expand Up @@ -36,6 +36,20 @@ public function init($options)
parent::init($options);
}

protected function excludeFromSync($path){
$excludedExtensions = array("dlpart");
$ext = pathinfo($path, PATHINFO_EXTENSION);
if(!empty($ext) && in_array($ext, $excludedExtensions)){
return true;
}
try{
$this->accessDriver->filterUserSelectionToHidden(array($path));
}catch(Exception $e){
return true;
}
return false;
}

protected function indexIsSync(){
// Grab all folders mtime and compare them
$repoIdentifier = $this->computeIdentifier($this->accessDriver->repository);
Expand All @@ -51,7 +65,12 @@ protected function indexIsSync(){
$children = dibi::query("SELECT [node_path],[mtime] FROM [ajxp_index] WHERE [repository_identifier] = %s AND [node_path] LIKE %s AND [node_path] NOT LIKE %s",
$repoIdentifier, "/%", "/%/%");
foreach($children as $cRow){
$mod["children"][substr($cRow->node_path, 1)] = $cRow->mtime;
$cp = substr($cRow->node_path, 1);
if(empty($cp)) continue;
if(PHP_OS == 'Darwin'){
$cp = Normalizer::normalize($cp, Normalizer::FORM_D);
}
$mod["children"][$cp] = $cRow->mtime;
}
$modified[] = $mod;

Expand All @@ -66,7 +85,7 @@ protected function indexIsSync(){
// Deleted folder!
$this->logDebug(__FUNCTION__, "Folder deleted directly on storage: ".$url);
$node = new AJXP_Node($url);
AJXP_Controller::applyHook("node.change", array(&$node, null, false));
AJXP_Controller::applyHook("node.change", array(&$node, null, false), true);
continue;
}
if($currentTime > $mtime){
Expand All @@ -76,10 +95,15 @@ protected function indexIsSync(){
"children" => array(),
"current_time" => $currentTime
);
$children = dibi::query("SELECT [node_path],[mtime],[md5] FROM [ajxp_index] WHERE [md5] != %s AND [repository_identifier] = %s AND [node_path] LIKE %s AND [node_path] NOT LIKE %s",
'directory', $repoIdentifier, "$path/%", "$path/%/%");
$children = dibi::query("SELECT [node_path],[mtime],[md5] FROM [ajxp_index] WHERE [repository_identifier] = %s AND [node_path] LIKE %s AND [node_path] NOT LIKE %s",
$repoIdentifier, "$path/%", "$path/%/%");
foreach($children as $cRow){
$mod["children"][substr($cRow->node_path, strlen($path)+1)] = $cRow->mtime;
$cp = substr($cRow->node_path, strlen($path)+1);
if(empty($cp)) continue;
if(PHP_OS == 'Darwin'){
$cp = Normalizer::normalize($cp, Normalizer::FORM_D);
}
$mod["children"][$cp] = $cRow->mtime;
}
$modified[] = $mod;
}
Expand All @@ -88,35 +112,50 @@ protected function indexIsSync(){
// NOW COMPUTE DIFFS
foreach($modified as $mod_data){
$url = $mod_data["url"];
$this->logDebug("Current folder is ".$url);
$current_time = $mod_data["current_time"];
$currentChildren = $mod_data["children"];
$files = scandir($url);
foreach($files as $f){
if($f[0] == ".") continue;
$nodeUrl = $url."/".$f;
$this->logDebug(__FUNCTION__, "Scanning ".$nodeUrl);
$node = new AJXP_Node($nodeUrl);
// Ignore dirs modified time
// if(is_dir($nodeUrl) && $mod_data["path"] != "/") continue;
if(!isSet($currentChildren[$f])){
if($this->excludeFromSync($nodeUrl)){
$this->logDebug(__FUNCTION__, "Excluding item detected on storage: ".$nodeUrl);
continue;
}
// New items detected
$this->logDebug(__FUNCTION__, "New item detected on storage: ".$nodeUrl);
AJXP_Controller::applyHook("node.change", array(null, &$node, false, true));
AJXP_Controller::applyHook("node.change", array(null, &$node, false, true), true);
continue;
}else {
if(is_dir($nodeUrl)) continue; // Make sure to not trigger a recursive indexation here.
if(filemtime($nodeUrl) > $currentChildren[$f]){
if($this->excludeFromSync($nodeUrl)){
$this->logDebug(__FUNCTION__, "Excluding item changed on storage: ".$nodeUrl);
continue;
}
// Changed!
$this->logDebug(__FUNCTION__, "Item modified directly on storage: ".$nodeUrl);
AJXP_Controller::applyHook("node.change", array(&$node, &$node, false));
AJXP_Controller::applyHook("node.change", array(&$node, &$node, false), true);
}
}
}
foreach($currentChildren as $cPath => $mtime){
$this->logDebug(__FUNCTION__, "Existing children ".$cPath);
if(!in_array($cPath, $files)){
if($this->excludeFromSync($url."/".$cPath)){
$this->logDebug(__FUNCTION__, "Excluding item deleted on storage: ".$url."/".$cPath);
continue;
}
// Deleted
$this->logDebug(__FUNCTION__, "File deleted directly on storage: ".$url."/".$cPath);
$node = new AJXP_Node($url."/".$cPath);
AJXP_Controller::applyHook("node.change", array(&$node, null, false));
AJXP_Controller::applyHook("node.change", array(&$node, null, false), true);
}
}
// Now "touch" parent directory
Expand All @@ -126,6 +165,26 @@ protected function indexIsSync(){
}
}

protected function getResyncTimestampFile($check = false){
$repo = ConfService::getRepository();
$sScope = $repo->securityScope();
$suffix = "-".$repo->getId();
if(!empty($sScope)) $suffix = "-".AuthService::getLoggedUser()->getId();
$file = $this->getPluginCacheDir(true, $check)."/storage_changes_time".$suffix;
return $file;
}

public function resyncAction($actionName, $httpVars, $fileVars)
{
if (ConfService::backgroundActionsSupported() && !ConfService::currentContextIsCommandLine()) {
AJXP_Controller::applyActionInBackground(ConfService::getRepository()->getId(), "resync_storage", $httpVars);
}else{
$file = $this->getResyncTimestampFile(true);
file_put_contents($file, time());
$this->indexIsSync();
}
}

public function switchActions($actionName, $httpVars, $fileVars)
{
if($actionName != "changes" || !isSet($httpVars["seq_id"])) return false;
Expand All @@ -138,7 +197,17 @@ public function switchActions($actionName, $httpVars, $fileVars)
$recycle = (!empty($recycle)?$recycle:false);

if($this->options["OBSERVE_STORAGE_CHANGES"] === true){
$this->indexIsSync();
// Do it every XX minutes
$minutes = 5;
if(isSet($this->options["OBSERVE_STORAGE_EVERY"])){
$minutes = intval($this->options["OBSERVE_STORAGE_EVERY"]);
}
$file = $this->getResyncTimestampFile();
$last = 0;
if(is_file($file)) $last = intval(file_get_contents($file));
if(time() - $last > $minutes * 60){
$this->resyncAction("resync_storage", array(), array());
}
}

HTMLWriter::charsetHeader('application/json', 'UTF-8');
Expand Down
6 changes: 6 additions & 0 deletions core/src/plugins/meta.syncable/manifest.xml
View file
Expand Up @@ -6,6 +6,7 @@
<global_param type="button" name="INSTALL_SQL" choices="run_plugin_action:meta.syncable:installSQLTables" label="CONF_MESSAGE[SQL Tables]" description="CONF_MESSAGE[Install SQL Tables]" mandatory="false"/>
<param name="REPO_SYNCABLE" type="boolean" label="Syncable Workspace" description="Workspace is syncable" default="true" scope="repository" expose="true"/>
<param name="OBSERVE_STORAGE_CHANGES" type="boolean" label="Observe storage changes [Experimental]" description="Continuously monitor underlying storage changes. This is experimental. Can be used if the storage content is modified OUTSIDE of Pydio." default="false" scope="repository" expose="false"/>
<param name="OBSERVE_STORAGE_EVERY" type="integer" label="Observe storage changes every..." description="If previous option is set to Yes, this will trigger a storage indexation every X minutes. This can be heavy in memory, so it can be a good practice to use something between 5 and 60 minutes." default="5" scope="repository" expose="false"/>
</server_settings>
<registry_contributions>
<actions>
Expand All @@ -19,6 +20,11 @@
</serverCallback>
</processing>
</action>
<action name="resync_storage">
<processing>
<serverCallback methodName="resyncAction" restParams="/" developerComment="Use the current index to compare with underlying storage and detect events"/>
</processing>
</action>
</actions>
<hooks>
<serverCallback hookName="node.change" methodName="updateNodesIndex" defer="true"/>
Expand Down

0 comments on commit 56cda77

Please sign in to comment.