Skip to content
This repository has been archived by the owner on Nov 25, 2020. It is now read-only.

Commit

Permalink
Compute a recursive size before copy/moving data and decide how to en…
Browse files Browse the repository at this point in the history 
…queue task.
  • Loading branch information
@cdujeu
cdujeu committed Jun 2, 2016
1 parent ef82c37 commit 103ca50
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 28 deletions.
20 changes: 16 additions & 4 deletions core/src/plugins/access.fs/class.fsAccessDriver.php
View file
Expand Up @@ -50,6 +50,7 @@
use Pydio\Core\Controller\HTMLWriter;
use Pydio\Core\PluginFramework\PluginsService;
use Pydio\Core\Utils\TextEncoder;
use Pydio\Tasks\Schedule;
use Pydio\Tasks\Task;
use Pydio\Tasks\TaskService;
use Zend\Diactoros\Response;
Expand Down Expand Up @@ -134,12 +135,12 @@ public function getResourceUrl($path)

/**
* @param String $directoryPath
* @param Repository $repositoryResolvedOptions
* @param array $repositoryResolvedOptions
* @return int
*/
public function directoryUsage($directoryPath, $repositoryResolvedOptions){
public function directoryUsage($directoryPath, $repositoryResolvedOptions = []){

$dir = $repositoryResolvedOptions["PATH"].$directoryPath;
$dir = (isSet($repositoryResolvedOptions["PATH"]) ? $repositoryResolvedOptions["PATH"] : $this->repository->getOption("PATH")).$directoryPath;
$size = -1;
if ( ( PHP_OS == "WIN32" || PHP_OS == "WINNT" || PHP_OS == "Windows") && class_exists("COM") ) {
$obj = new COM ( 'scripting.filesystemobject' );
Expand Down Expand Up @@ -825,9 +826,20 @@ public function switchAction(ServerRequestInterface &$request, ResponseInterface
throw new PydioException("", 113);
}
$taskId = $request->getAttribute("pydio-task-id");
if($taskId === null && (!$selection->isUnique() || !is_file($selection->getUniqueNode()->getUrl()))){
// Compute copy size
$size = 0;
$nodes = $selection->buildNodes();
$bgSizeThreshold = 10*1024*1024;
$bgWorkerThreshold = 80*1024*1024;
foreach($nodes as $node){
$size += $node->getSizeRecursive();
}
if($taskId === null && ($size > $bgSizeThreshold)){
$task = TaskService::actionAsTask($action, $httpVars);
$task->setFlags(Task::FLAG_STOPPABLE);
if($size > $bgWorkerThreshold){
$task->setSchedule(new Schedule(Schedule::TYPE_ONCE_DEFER));
}
$response = TaskService::getInstance()->enqueueTask($task, $request, $response);
break;
}
Expand Down
4 changes: 2 additions & 2 deletions core/src/plugins/access.s3/class.s3AccessDriver.php
View file
Expand Up @@ -148,10 +148,10 @@ public function getS3Service(){

/**
* @param String $directoryPath
* @param \Pydio\Access\Core\Model\Repository $repositoryResolvedOptions
* @param array $repositoryResolvedOptions
* @return int
*/
public function directoryUsage($directoryPath, $repositoryResolvedOptions){
public function directoryUsage($directoryPath, $repositoryResolvedOptions = []){
$client = $this->getS3Service();
$bucket = (isSet($repositoryResolvedOptions["CONTAINER"])?$repositoryResolvedOptions["CONTAINER"]:$this->repository->getOption("CONTAINER"));
$path = (isSet($repositoryResolvedOptions["PATH"])?$repositoryResolvedOptions["PATH"]:"");
Expand Down
41 changes: 20 additions & 21 deletions core/src/plugins/core.access/src/AbstractAccessDriver.php
View file
Expand Up @@ -27,10 +27,10 @@
use Pydio\Core\Services\AuthService;
use Pydio\Core\Controller\Controller;
use Pydio\Core\Utils\Utils;
use Pydio\Core\Controller\XMLWriter;
use Pydio\Core\PluginFramework\Plugin;
use Pydio\Core\Services\ConfService;
use Pydio\Core\Utils\TextEncoder;
use Pydio\Core\Utils\VarsFilter;
use Pydio\Log\Core\AJXP_Logger;
use Pydio\Tasks\Task;
use Pydio\Tasks\TaskService;
Expand Down Expand Up @@ -74,7 +74,7 @@ public function accessPreprocess(ServerRequestInterface &$request)
return;
}
$selection = new UserSelection($this->repository, $httpVars);
\Pydio\Core\Controller\Controller::applyHook("node.".$httpVars["hook_name"], array($selection->getUniqueNode(), $httpVars["hook_arg"]));
Controller::applyHook("node.".$httpVars["hook_name"], array($selection->getUniqueNode(), $httpVars["hook_arg"]));
}
if ($actionName == "ls") {
// UPWARD COMPATIBILTY
Expand Down Expand Up @@ -119,7 +119,7 @@ public function makeSharedRepositoryOptions($httpVars, $repository){}
* @return int
* @throws \Exception
*/
public function directoryUsage($directoryPath, $repositoryResolvedOptions){
public function directoryUsage($directoryPath, $repositoryResolvedOptions = []){
throw new \Exception("Current driver does not support recursive directory usage!");
}

Expand Down Expand Up @@ -189,7 +189,7 @@ public function crossRepositoryCopy(ServerRequestInterface &$requestInterface, R
}

/**
* @param String $destDir url of destination dir
* @param String $destFile url of destination file
* @param String $srcFile url of source file
* @param array $error accumulator for error messages
* @param array $success accumulator for success messages
Expand Down Expand Up @@ -228,7 +228,7 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
}
if (!$move) {
$size = filesize($realSrcFile);
\Pydio\Core\Controller\Controller::applyHook("node.before_create", array(new AJXP_Node($destFile), $size));
Controller::applyHook("node.before_create", array(new AJXP_Node($destFile), $size));
}
if (dirname($realSrcFile) == dirname($destFile) && basename($realSrcFile) == basename($destFile)) {
if ($move) {
Expand Down Expand Up @@ -285,7 +285,7 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
}
} else {
if ($move) {
\Pydio\Core\Controller\Controller::applyHook("node.before_path_change", array($srcNode));
Controller::applyHook("node.before_path_change", array($srcNode));
if(file_exists($destFile)) unlink($destFile);
if(AJXP_MetaStreamWrapper::nodesUseSameWrappers($realSrcFile, $destFile)){
rename($realSrcFile, $destFile);
Expand All @@ -297,7 +297,7 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
try {
$this->filecopy($realSrcFile, $destFile);
$this->changeMode($destFile, $destRepoData);
\Pydio\Core\Controller\Controller::applyHook("node.change", array($srcNode, $destNode, true));
Controller::applyHook("node.change", array($srcNode, $destNode, true));
} catch (\Exception $e) {
$error[] = $e->getMessage();
if(!empty($taskId)) TaskService::getInstance()->updateTaskStatus($taskId, Task::STATUS_FAILED, $e->getMessage());
Expand All @@ -306,7 +306,6 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
}
}

$successMessage = "";
if ($move) {
// Now delete original
// $this->deldir($realSrcFile); // both file and dir
Expand All @@ -316,9 +315,9 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
$messagePart = $mess[123]." ".$mess[122];
}
if (is_dir($destFile)) {
$successMessage = $mess[117]." ". \Pydio\Core\Utils\TextEncoder::toUTF8(basename($srcFile))." ".$messagePart;
$successMessage = $mess[117]." ". TextEncoder::toUTF8(basename($srcFile))." ".$messagePart;
} else {
$successMessage = $mess[34]." ". \Pydio\Core\Utils\TextEncoder::toUTF8(basename($srcFile))." ".$messagePart;
$successMessage = $mess[34]." ". TextEncoder::toUTF8(basename($srcFile))." ".$messagePart;
}
} else {
if (RecycleBinManager::recycleEnabled() && $destDir == "/".$srcRecycle) {
Expand All @@ -327,7 +326,7 @@ protected function copyOrMoveFile($destFile, $srcFile, &$error, &$success, $move
if (isSet($dirRes)) {
$successMessage = $mess[117]." ".TextEncoder::toUTF8(basename($destFile))." ".$mess[73]." ".TextEncoder::toUTF8($destDir)." (".TextEncoder::toUTF8($dirRes)." ".$mess[116].")";
} else {
$successMessage = $mess[34]." ". \Pydio\Core\Utils\TextEncoder::toUTF8(basename($destFile))." ".$mess[73]." ". \Pydio\Core\Utils\TextEncoder::toUTF8($destDir);
$successMessage = $mess[34]." ". TextEncoder::toUTF8(basename($destFile))." ".$mess[73]." ". TextEncoder::toUTF8($destDir);
}
}
$success[] = $successMessage;
Expand Down Expand Up @@ -427,7 +426,7 @@ protected function dircopy($srcdir, $dstdir, &$errors, &$success, $verbose = fal
}
closedir($curdir);
foreach ($recurse as $rec) {
if($verbose) echo "Dircopy $srcfile";
if($verbose && isSet($srcfile)) echo "Dircopy $srcfile";
$num += $this->dircopy($rec["src"], $rec["dest"], $errors, $success, $verbose, $convertSrcFile, $srcRepoData, $destRepoData, $taskId);
}
}
Expand Down Expand Up @@ -455,27 +454,27 @@ protected function changeMode($filePath, $repoData)
protected function deldir($location, $repoData)
{
if (is_dir($location)) {
\Pydio\Core\Controller\Controller::applyHook("node.before_path_change", array(new AJXP_Node($location)));
Controller::applyHook("node.before_path_change", array(new AJXP_Node($location)));
$all=opendir($location);
while (($file=readdir($all)) !== FALSE) {
if (is_dir("$location/$file") && $file !=".." && $file!=".") {
$this->deldir("$location/$file", $repoData);
if (file_exists("$location/$file")) {
rmdir("$location/$file");
@rmdir("$location/$file");
}
unset($file);
} elseif (!is_dir("$location/$file")) {
if (file_exists("$location/$file")) {
unlink("$location/$file");
@unlink("$location/$file");
}
unset($file);
}
}
closedir($all);
rmdir($location);
@rmdir($location);
} else {
if (file_exists("$location")) {
\Pydio\Core\Controller\Controller::applyHook("node.before_path_change", array(new AJXP_Node($location)));
Controller::applyHook("node.before_path_change", array(new AJXP_Node($location)));
$test = @unlink("$location");
if(!$test) throw new \Exception("Cannot delete file ".$location);
}
Expand All @@ -493,7 +492,7 @@ protected function deldir($location, $repoData)
* @param array $stat
* @param Repository $repoObject
* @param callable $remoteDetectionCallback
* @var oct $mode
* @var integer $mode
*/
public static function fixPermissions(&$stat, $repoObject, $remoteDetectionCallback = null)
{
Expand All @@ -513,7 +512,7 @@ public static function fixPermissions(&$stat, $repoObject, $remoteDetectionCallb
}

} else if (substr($fixPermPolicy, 0, strlen("file:")) == "file:") {
$filePath = \Pydio\Core\Utils\VarsFilter::filter(substr($fixPermPolicy, strlen("file:")));
$filePath = VarsFilter::filter(substr($fixPermPolicy, strlen("file:")));
if (file_exists($filePath)) {
// GET A GID/UID FROM FILE
$lines = file($filePath);
Expand Down Expand Up @@ -601,9 +600,9 @@ public function filterNodeName($nodePath, $nodeName, &$isLeaf, $lsOptions)
{
$showHiddenFiles = $this->getFilteredOption("SHOW_HIDDEN_FILES", $this->repository);
if($isLeaf === ""){
$isLeaf = (is_file($nodePath."/".$nodeName) || \Pydio\Core\Utils\Utils::isBrowsableArchive($nodeName));
$isLeaf = (is_file($nodePath."/".$nodeName) || Utils::isBrowsableArchive($nodeName));
}
if (\Pydio\Core\Utils\Utils::isHidden($nodeName) && !$showHiddenFiles) {
if (Utils::isHidden($nodeName) && !$showHiddenFiles) {
return false;
}
$nodeType = "d";
Expand Down
28 changes: 27 additions & 1 deletion core/src/plugins/core.access/src/Model/AJXP_Node.php
View file
Expand Up @@ -72,7 +72,7 @@ class AJXP_Node implements \JsonSerializable
*/
private $_repository;
/**
* @var .\AbstractAccessDriver
* @var AbstractAccessDriver
*/
private $_accessDriver;
/**
Expand Down Expand Up @@ -559,6 +559,32 @@ public function setInfoLoaded($level){
$this->nodeInfoLevel = $level;
}

/**
* Try the size of collection recursively.
* Will trigger the node.size.recursive hook, allowing certain plugins
* to perform the operation if they have the information (e.g. meta.syncable).
* Otherwise will use the directoryUsage() method of the accessDriver.
* @return int|mixed
*/
public function getSizeRecursive(){
$this->loadNodeInfo();
if($this->isLeaf()){
return $this->_metadata["bytesize"];
}else{
$result = -1;
Controller::applyHook("node.size.recursive", array(&$this, &$result));
if($result == -1){
try{
return $this->_accessDriver->directoryUsage($this->getPath(), []);
}catch(\Exception $e){
return -1;
}
}else{
return $result;
}
}
}

/**
* Magic getter for metadata
* @param $varName
Expand Down

0 comments on commit 103ca50

Please sign in to comment.