Reduce Batch size for delete - Fix usage of PATH parameter in S3 driv…

…er - Close #1365 at last !
pydio · Dec 8, 2017 · 2fa4fa6 · 2fa4fa6
1 parent 3258946
commit 2fa4fa6
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 15 deletions.
diff --git a/core/src/plugins/access.s3/S3AccessDriver.php b/core/src/plugins/access.s3/S3AccessDriver.php
@@ -21,12 +21,12 @@
  */
 namespace Pydio\Access\Driver\StreamProvider\S3;
 
-use DOMNode;
 use Pydio\Access\Core\Model\AJXP_Node;
 use Pydio\Access\Core\RecycleBinManager;
 
 use Pydio\Access\Driver\StreamProvider\FS\FsAccessDriver;
 use Pydio\Core\Model\ContextInterface;
+use Pydio\Core\Model\RepositoryInterface;
 use Pydio\Core\Services\ConfService;
 use Pydio\Core\Exception\PydioException;
 
@@ -40,7 +40,7 @@
 class S3AccessDriver extends FsAccessDriver
 {
     /**
-    * @var \Pydio\Access\Core\Model\Repository
+    * @var RepositoryInterface
     */
     public $repository;
     public $driverConf;
@@ -92,7 +92,7 @@ protected function initRepository(ContextInterface $contextInterface)
     }
 
     /**
-     * @return Aws\Common\Client\AbstractClient
+     * @return S3Client
      */
     public function getS3Service(){
         return $this->s3Client;
@@ -165,8 +165,7 @@ protected function appendUploadedData($folder, $source, $target){
 
 
     /**
-     * @param AJXP_Node $dir
-     * @param string $type
+     * @param AJXP_Node $node
      * @return bool
      */
     public function isWriteable(AJXP_Node $node)
@@ -183,7 +182,7 @@ public static function isRemote()
     }
 
     /**
-     * @param \Pydio\Access\Core\Model\AJXP_Node $ajxpNode
+     * @param AJXP_Node $node
      * @param bool $parentNode
      * @param bool $details
      * @return void
@@ -205,8 +204,7 @@ public function loadNodeInfo(&$node, $parentNode = false, $details = false)
     public function makeSharedRepositoryOptions(ContextInterface $ctx, $httpVars)
     {
         $newOptions                 = parent::makeSharedRepositoryOptions($ctx, $httpVars);
-        $newOptions["CONTAINER"]    = "AJXP_PARENT_OPTION:CONTAINER"; //$ctx->getRepository()->getContextOption($ctx, "CONTAINER");
-
+        $newOptions["CONTAINER"]    = "AJXP_PARENT_OPTION:CONTAINER";
         return $newOptions;
     }
 

diff --git a/core/src/plugins/access.s3/S3AccessWrapper.php b/core/src/plugins/access.s3/S3AccessWrapper.php
@@ -57,7 +57,7 @@ public static function getResolvedOptionsForNode($node)
         $options = [
             "TYPE" => "s3"
         ];
-        $optKeys = ["API_KEY", "SECRET_KEY", "CONTAINER", "SIGNATURE_VERSION", "STORAGE_URL", "REGION", "PROXY", "API_VERSION", "VHOST_NOT_SUPPORTED"];
+        $optKeys = ["API_KEY", "SECRET_KEY", "CONTAINER", "PATH", "SIGNATURE_VERSION", "STORAGE_URL", "REGION", "PROXY", "API_VERSION", "VHOST_NOT_SUPPORTED"];
         foreach($optKeys as $key){
             $options[$key] = $repository->getContextOption($context, $key);
         }
@@ -145,7 +145,7 @@ protected static function initPath($path, $streamType, $storeOpenContext = false
         $basePath       = $repoObject->getContextOption($node->getContext(), "PATH");
         $baseContainer  = $repoObject->getContextOption($node->getContext(), "CONTAINER");
         if (!empty($basePath)) {
-            $baseContainer .= rtrim($basePath, "/");
+            $baseContainer .= "/".trim($basePath, "/");
         }
         $p = $protocol . $baseContainer . str_replace("//", "/", $url["path"]);
         return $p;
@@ -159,7 +159,7 @@ protected static function initPath($path, $streamType, $storeOpenContext = false
      * @param String $mode
      * @param string $options
      * @param resource $context
-     * @return resource
+     * @return resource|bool
      * @internal param string $opened_path
      */
     public function stream_open($path, $mode, $options, &$context)
@@ -200,9 +200,10 @@ public function url_stat($path, $flags)
             $node       = new AJXP_Node($path);
             $ctx = $node->getContext();
             $repoObject = $node->getRepository();
+            $basePath = $repoObject->getContextOption($ctx, "PATH");
             $result = $this->getClientForContext($ctx)->listObjects([
                 'Bucket'  => $repoObject->getContextOption($ctx, "CONTAINER"),
-                'Prefix'  => ltrim($node->getPath(), "/"). '/',
+                'Prefix'  =>  ltrim( trim($basePath, "/") ."/". ltrim($node->getPath(), "/"), '/') . '/',
                 'MaxKeys' => 1
             ]);
             if (isSet($result['Contents']) && isSet($result['Contents'][0]['LastModified'])) {
@@ -244,7 +245,7 @@ protected function statForDir($lastModified){
      *
      * @param string $path
      * @param string $options
-     * @return resource
+     * @return resource|bool
      */
     public function dir_opendir($path, $options)
     {
@@ -260,8 +261,14 @@ public function dir_opendir($path, $options)
         return $this->dH !== false;
     }
 
+    /**
+     * @param string $path
+     * @param int $mode
+     * @param int $options
+     * @return bool
+     * @throws \Exception
+     */
     public function mkdir($path, $mode, $options){
-        $url        = UrlUtils::mbParseUrl($path);
         $node       = new AJXP_Node($path);
         $repoId     = $node->getRepositoryId();
         $repoObject = $node->getRepository();
@@ -414,7 +421,7 @@ public function rename($from, $to)
             $clear = \Aws\S3\BatchDelete::fromIterator($s3Client, $bucket, $s3Client->getIterator('ListObjects', array(
                 'Bucket' => $bucket,
                 'Prefix' => $fromKeyname
-            )));
+            )), ['batch_size' => 5]);
             $clear->delete();
 
             if (count($failed)) {

diff --git a/core/src/plugins/access.s3/manifest.xml b/core/src/plugins/access.s3/manifest.xml
@@ -13,6 +13,7 @@
 		<param name="SECRET_KEY" group="CONF_MESSAGE[Authentication]" type="string" label="CONF_MESSAGE[Secret Key]" description="CONF_MESSAGE[S3 secret key]" mandatory="true"/>
 		<param name="REGION" group="CONF_MESSAGE[Location]" type="select" choices="us-east-1|US Standard (Virginia),us-west-1|US West 1 (Northern California),us-west-2|US West 2 (Oregon),eu-west-1|EU (Ireland),eu-central-1|EU (Frankfurt - signature v4 only), ap-southeast-2|South-East (Sydney), ap-southeast-1|South-East (Singapore),ap-northeast-1|Asia Pacific (Japan),ap-northeast-2|Asia Pacific (Seoul - signature v4 only),sa-east-1|South America (Sao Paulo),us-gov-west-1|EU Governement Cloud" label="CONF_MESSAGE[Region]" description="CONF_MESSAGE[S3 storage region]" mandatory="false"/>
 		<param name="CONTAINER" group="CONF_MESSAGE[Location]" type="string" label="CONF_MESSAGE[Container]" description="CONF_MESSAGE[Root container in the S3 storage]" mandatory="false"/>
+		<param name="PATH" group="CONF_MESSAGE[Location]" type="string" label="CONF_MESSAGE[Inner Path]" description="CONF_MESSAGE[Path of the data inside the container. Empty value points to the root of the container. You can use dynamic keywords here.]" mandatory="false"/>
 		<param name="SIGNATURE_VERSION" group="CONF_MESSAGE[API]" type="select" choices="v2|Version 2,v4|Version 4" label="CONF_MESSAGE[Signature Version]" description="CONF_MESSAGE[API Signature Version. Leave empty if you are not sure, EU Frankfurt and AP Seoul should use v4 only.]" mandatory="false"/>
 		<param name="API_VERSION" group="CONF_MESSAGE[API]" type="string" label="CONF_MESSAGE[API Version]" description="CONF_MESSAGE[The API Version of AS3 is a date (like 2006-03-01). If not sure, put latest ]" mandatory="true" default="latest"/>
         <param name="PROXY" group="CONF_MESSAGE[Custom Storage]" type="string" label="CONF_MESSAGE[Proxy]" description="CONF_MESSAGE[If you are using a proxy, for example:  127.0.0.1:80 or username:password@127.0.0.1:8080]" mandatory="false"/>