From b009d2ca15342ef27a469715031bf42e74eb6349 Mon Sep 17 00:00:00 2001 From: dennisxrow Date: Mon, 5 Mar 2018 17:30:34 +0100 Subject: [PATCH 1/5] async file transfer --- modules/xrowvideo/upload.php | 54 +++++++++++++++---------------- settings/xrowvideo.ini.append.php | 1 + 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/modules/xrowvideo/upload.php b/modules/xrowvideo/upload.php index eff5bfd..37a7f2b 100644 --- a/modules/xrowvideo/upload.php +++ b/modules/xrowvideo/upload.php @@ -47,6 +47,7 @@ $mime2 = explode( '/', $mime['name'] ); $fileIni = eZINI::instance( 'file.ini' ); +$xrowvideoIni = eZINI::instance( 'xrowvideo.ini' ); $fileHandler = $fileIni->variable( 'ClusteringSettings', 'FileHandler' ); if($fileHandler == "eZDFSFileHandler") { @@ -165,11 +166,17 @@ } $targetchunk = $chunks -1; -$db = eZDB::instance(); -$db->begin(); -if( isset( $_REQUEST['chunk'] ) and $chunk == $targetchunk ) -{ +$closure = function () use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { + $db = eZDB::instance(); + $db->begin(); + $rootDir = getcwd(); + if (is_dir("ezpublish_legacy")) { + chdir("ezpublish_legacy"); + } + + echo "xrowvideo: Moving $storeNameNFS to $storeName\n"; rename($storeNameNFS, $storeName); + $contentObjectAttributeID = $attribute->attribute( 'id' ); $version = $attribute->attribute( 'version' ); @@ -180,39 +187,32 @@ $binary->store(); $fileHandler = eZClusterFileHandler::instance(); + echo "xrowvideo: $storeName starting fileStore()\n"; $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); - $mObj = new xrowMedia( $attribute ); - $mObj->updateMediaInfo(); - $mObj->addPendingAction(); - $attribute->setAttribute( 'data_text', $mObj->xml->saveXML() ); - $attribute->store(); $fileHandler->deleteLocal(); - eZLog::write( gmdate( 'D, d M Y H:i:s', time() ) . " ObjectID #" . $obj->ID . " completed", "xrowvideo.log"); -} -elseif( !isset( $_REQUEST['chunk'] ) ) -{ - $contentObjectAttributeID = $attribute->attribute( 'id' ); - $version = $attribute->attribute( 'version' ); - - $binary = eZBinaryFile::create( $contentObjectAttributeID, $version ); - $binary->setAttribute( 'filename', basename( $storeName ) ); - $binary->setAttribute( 'original_filename', $fileName ); - $binary->setAttribute( 'mime_type', $mime['name'] ); - $binary->store(); - - $fileHandler = eZClusterFileHandler::instance(); - $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); + chdir($rootDir); $mObj = new xrowMedia( $attribute ); $mObj->updateMediaInfo(); $mObj->addPendingAction(); $attribute->setAttribute( 'data_text', $mObj->xml->saveXML() ); $attribute->store(); - $fileHandler->deleteLocal(); + $db->commit(); + echo "xrowvideo: Done\n"; +}; + +if( (isset( $_REQUEST['chunk'] ) && $chunk == $targetchunk) || !isset( $_REQUEST['chunk'])) { + if ( $xrowvideoIni->hasVariable( 'xrowVideoSettings', 'AsyncFileTransfer' ) && $xrowvideoIni->variable('xrowVideoSettings', 'AsyncFileTransfer') === 'enabled' ) { + $container = ezpKernel::instance()->getServiceContainer(); + $mq = $container->get("xrow_mq"); + $mq->async($closure); + } else { + $closure(); + } eZLog::write( gmdate( 'D, d M Y H:i:s', time() ) . " ObjectID #" . $obj->ID . " completed", "xrowvideo.log"); } -$db->commit(); + // Return JSON-RPC response echo '{"jsonrpc" : "2.0", "result" : null, "id" : "'.basename( $storeName ).'"}'; eZExecution::cleanExit(); @@ -237,4 +237,4 @@ function storeName( $Filename = false, $suffix = false, $MimeCategory, $seed, $n $dest_name = $dir . '/' . md5( basename( $Filename ) . $seed ) . $suffixString; return $dest_name; -} \ No newline at end of file +} diff --git a/settings/xrowvideo.ini.append.php b/settings/xrowvideo.ini.append.php index 997fd48..8d9be93 100644 --- a/settings/xrowvideo.ini.append.php +++ b/settings/xrowvideo.ini.append.php @@ -4,6 +4,7 @@ Runtimes=html5 EnableTrackingwithGA=disabled TrackingGAID=1234567-8 +AsyncFileTransfer=disabled # enable and set this time, if your DB timeouts are smaller than 10000 #WaitingTimeOutTime=100000 From e9f6d883c3bd5a58da9162deb77df57fc9d4697c Mon Sep 17 00:00:00 2001 From: dennisxrow Date: Tue, 6 Mar 2018 10:42:36 +0100 Subject: [PATCH 2/5] Removed echo to prevent meddling with the response content moar comments --- modules/xrowvideo/upload.php | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/modules/xrowvideo/upload.php b/modules/xrowvideo/upload.php index 37a7f2b..0c9a454 100644 --- a/modules/xrowvideo/upload.php +++ b/modules/xrowvideo/upload.php @@ -10,8 +10,8 @@ $obj = $attribute->attribute( 'object' ); if ( !$obj OR - $obj->attribute( 'status' ) == eZContentObject::STATUS_ARCHIVED OR - !$obj->canEdit( false, false, false, $Params['Language'] ) ) + $obj->attribute( 'status' ) == eZContentObject::STATUS_ARCHIVED OR + !$obj->canEdit( false, false, false, $Params['Language'] ) ) { return $Module->handleError( eZError::KERNEL_NOT_AVAILABLE, 'kernel' ); } @@ -85,6 +85,7 @@ } if ( strpos( $contentType, 'multipart' ) !== false ) { + // Upload via chunking if ( isset( $_FILES['file']['tmp_name'] ) && is_uploaded_file( $_FILES['file']['tmp_name'] ) ) { // Open temp file @@ -126,6 +127,7 @@ } else { + // Upload via streaming set_time_limit(0); $mem = $attribute->attribute( 'contentclass_attribute' )->DataInt1 * 2;// twice the max upload size ini_set('memory_limit', $mem.'M'); @@ -166,15 +168,24 @@ } $targetchunk = $chunks -1; -$closure = function () use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { +/* + * This closure stores the file on the filesystem and database. + * The call to fileStore() may take several minutes to complete with files larger than 1GB, + * probably due to the fact that it transfers the file in 1MB chunks instead of simply moving it over. + * Thus exceeding most HTTP and database timeouts, e.g. wait_timeout. + * + * Therefore a asynchronous transfer was implemented further down using this $closure. + */ +$closure = function ($log = true) use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { $db = eZDB::instance(); $db->begin(); + + // If running eZ 5.4 we might have to move to the legacy root $rootDir = getcwd(); if (is_dir("ezpublish_legacy")) { chdir("ezpublish_legacy"); } - echo "xrowvideo: Moving $storeNameNFS to $storeName\n"; rename($storeNameNFS, $storeName); $contentObjectAttributeID = $attribute->attribute( 'id' ); @@ -187,9 +198,7 @@ $binary->store(); $fileHandler = eZClusterFileHandler::instance(); - echo "xrowvideo: $storeName starting fileStore()\n"; $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); - $fileHandler->deleteLocal(); chdir($rootDir); @@ -199,11 +208,13 @@ $attribute->setAttribute( 'data_text', $mObj->xml->saveXML() ); $attribute->store(); $db->commit(); - echo "xrowvideo: Done\n"; }; +// Store the received file if its the last chunk or if the data was send via streaming if( (isset( $_REQUEST['chunk'] ) && $chunk == $targetchunk) || !isset( $_REQUEST['chunk'])) { + // If AsyncFileTransfer is enabled perform the file processing asynchronous to prevent timeouts, otherwise just execute it if ( $xrowvideoIni->hasVariable( 'xrowVideoSettings', 'AsyncFileTransfer' ) && $xrowvideoIni->variable('xrowVideoSettings', 'AsyncFileTransfer') === 'enabled' ) { + // AsyncFileTransfer requires xrow mq-bundle installed and at least eZ 5.4 $container = ezpKernel::instance()->getServiceContainer(); $mq = $container->get("xrow_mq"); $mq->async($closure); From 823b829b833c814c50a5763e2ba482f03952b77f Mon Sep 17 00:00:00 2001 From: dennis Date: Wed, 7 Mar 2018 11:00:17 +0100 Subject: [PATCH 3/5] updated closure to use the legacy context --- modules/xrowvideo/upload.php | 68 +++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/modules/xrowvideo/upload.php b/modules/xrowvideo/upload.php index 0c9a454..fc1c8b5 100644 --- a/modules/xrowvideo/upload.php +++ b/modules/xrowvideo/upload.php @@ -176,38 +176,42 @@ * * Therefore a asynchronous transfer was implemented further down using this $closure. */ -$closure = function ($log = true) use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { - $db = eZDB::instance(); - $db->begin(); - - // If running eZ 5.4 we might have to move to the legacy root - $rootDir = getcwd(); - if (is_dir("ezpublish_legacy")) { - chdir("ezpublish_legacy"); - } - - rename($storeNameNFS, $storeName); - - $contentObjectAttributeID = $attribute->attribute( 'id' ); - $version = $attribute->attribute( 'version' ); - - $binary = eZBinaryFile::create( $contentObjectAttributeID, $version ); - $binary->setAttribute( 'filename', basename( $storeName ) ); - $binary->setAttribute( 'original_filename', $fileName ); - $binary->setAttribute( 'mime_type', $mime['name'] ); - $binary->store(); - - $fileHandler = eZClusterFileHandler::instance(); - $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); - $fileHandler->deleteLocal(); - chdir($rootDir); - - $mObj = new xrowMedia( $attribute ); - $mObj->updateMediaInfo(); - $mObj->addPendingAction(); - $attribute->setAttribute( 'data_text', $mObj->xml->saveXML() ); - $attribute->store(); - $db->commit(); +$closure = function () use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { + // Declare the $kernel as global to get the symfony kernel from the global scope + global $kernel; + $container = $kernel->getContainer(); + $legacyKernelClosure = $container->get('ezpublish_legacy.kernel'); + // Execute the legacy kernel closure to get the actual legacy kernel object + $legacyKernel = $legacyKernelClosure(); + + // Run callback in legacy context, important to get the correct working directory e.g. ezpublish_legacy + $legacyKernel->runCallback(function() use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { + $db = eZDB::instance(); + $db->begin(); + + rename($storeNameNFS, $storeName); + + $contentObjectAttributeID = $attribute->attribute( 'id' ); + $version = $attribute->attribute( 'version' ); + + $binary = eZBinaryFile::create( $contentObjectAttributeID, $version ); + $binary->setAttribute( 'filename', basename( $storeName ) ); + $binary->setAttribute( 'original_filename', $fileName ); + $binary->setAttribute( 'mime_type', $mime['name'] ); + $binary->store(); + + $fileHandler = eZClusterFileHandler::instance(); + // fileStore() is slow with large files + $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); + $fileHandler->deleteLocal(); + + $mObj = new xrowMedia( $attribute ); + $mObj->updateMediaInfo(); + $mObj->addPendingAction(); + $attribute->setAttribute( 'data_text', $mObj->xml->saveXML() ); + $attribute->store(); + $db->commit(); + }); }; // Store the received file if its the last chunk or if the data was send via streaming From 562e4c453cf4a78f15602c4e8f33b34c66de5ca6 Mon Sep 17 00:00:00 2001 From: dennis Date: Wed, 7 Mar 2018 16:05:30 +0100 Subject: [PATCH 4/5] create storage dir when necessary --- modules/xrowvideo/upload.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/xrowvideo/upload.php b/modules/xrowvideo/upload.php index fc1c8b5..97a18ed 100644 --- a/modules/xrowvideo/upload.php +++ b/modules/xrowvideo/upload.php @@ -189,6 +189,17 @@ $db = eZDB::instance(); $db->begin(); + /* + * Create the var/storage/original/video directory. + * The directories will get deleted by xrowMedia->updateMediaInfo() + * \_ eZDFSFileHandler->deleteLocal() + * \_ eZClusterFileHandler::cleanupEmptyDirectories() + * if the local file was the only file inside that directory tree. + */ + if (!file_exists(dirname($storeName))) { + mkdir(dirname($storeName), 0777, true); + } + rename($storeNameNFS, $storeName); $contentObjectAttributeID = $attribute->attribute( 'id' ); @@ -203,7 +214,6 @@ $fileHandler = eZClusterFileHandler::instance(); // fileStore() is slow with large files $fileHandler->fileStore( $storeName, 'binaryfile', false, $mime['name'] ); - $fileHandler->deleteLocal(); $mObj = new xrowMedia( $attribute ); $mObj->updateMediaInfo(); From 8abf0633b5e467c7214b5a290615cb59458f3429 Mon Sep 17 00:00:00 2001 From: dennis Date: Wed, 7 Mar 2018 16:48:25 +0100 Subject: [PATCH 5/5] added minimal logging for easier debugging --- modules/xrowvideo/upload.php | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/modules/xrowvideo/upload.php b/modules/xrowvideo/upload.php index 97a18ed..5b3a3ca 100644 --- a/modules/xrowvideo/upload.php +++ b/modules/xrowvideo/upload.php @@ -49,6 +49,11 @@ $fileIni = eZINI::instance( 'file.ini' ); $xrowvideoIni = eZINI::instance( 'xrowvideo.ini' ); $fileHandler = $fileIni->variable( 'ClusteringSettings', 'FileHandler' ); +$async = false; +if ( $xrowvideoIni->hasVariable( 'xrowVideoSettings', 'AsyncFileTransfer' ) && $xrowvideoIni->variable('xrowVideoSettings', 'AsyncFileTransfer') === 'enabled' ) { + $async = true; +} + if($fileHandler == "eZDFSFileHandler") { $nfs = true; @@ -168,15 +173,21 @@ } $targetchunk = $chunks -1; -/* - * This closure stores the file on the filesystem and database. +/** @var \closure $closure + * + * Stores the file on the filesystem and database. * The call to fileStore() may take several minutes to complete with files larger than 1GB, * probably due to the fact that it transfers the file in 1MB chunks instead of simply moving it over. * Thus exceeding most HTTP and database timeouts, e.g. wait_timeout. * * Therefore a asynchronous transfer was implemented further down using this $closure. */ -$closure = function () use ($storeName, $storeNameNFS, $fileName, $attribute, $mime) { +$closure = function () use ($storeName, $storeNameNFS, $fileName, $attribute, $mime, $async) { + // Only "log" when it's run asynchronous to avoid output in HTTP responses + if ($async) { + echo "xrowvideo: Saving $fileName (ContentObjectID = " . $attribute->ContentObjectID . ")\n"; + } + // Declare the $kernel as global to get the symfony kernel from the global scope global $kernel; $container = $kernel->getContainer(); @@ -222,12 +233,15 @@ $attribute->store(); $db->commit(); }); + if ($async) { + echo "xrowvideo: Saved $fileName (ContentObjectID = " . $attribute->ContentObjectID . ")\n"; + } }; // Store the received file if its the last chunk or if the data was send via streaming if( (isset( $_REQUEST['chunk'] ) && $chunk == $targetchunk) || !isset( $_REQUEST['chunk'])) { // If AsyncFileTransfer is enabled perform the file processing asynchronous to prevent timeouts, otherwise just execute it - if ( $xrowvideoIni->hasVariable( 'xrowVideoSettings', 'AsyncFileTransfer' ) && $xrowvideoIni->variable('xrowVideoSettings', 'AsyncFileTransfer') === 'enabled' ) { + if ($async) { // AsyncFileTransfer requires xrow mq-bundle installed and at least eZ 5.4 $container = ezpKernel::instance()->getServiceContainer(); $mq = $container->get("xrow_mq");