MDL-33550 Correctly process situation when file reference source is m…

…issing

- do not die with fatal error if source file in moodle internal repository is missing;
- moved code duplication for moodle repositories into class repository (functions send_file, get_reference_details, get_file_by_reference, get_file_reference);
- update file status after repository::sync_external_file so we know that it is missing (or not missing anymore). Do not run this function more than once for file within one request;
- display readable name for Private Files and Server files with the new format;
- display broken icon in filemanager if we know that source is missing, display information (for admin) where it was located before: see repository::get_reference_details() and extending classes;
- removed unnecessary queries in stored_file::sync_external_file();
- syncronize files before displaying it's size in mod_resource, do not query  directly

lang/en/repository.php

@@ -138,6 +138,7 @@
 $string['loading'] = 'Loading...';
 $string['login'] = 'Login';
 $string['logout'] = 'Logout';
+$string['lostsource'] = 'Error. Source is missing. {$a}';
 $string['makefileinternal'] = 'Make a copy of the file';
 $string['makefilelink'] = 'Link to the file directly';
 $string['makefilereference'] = 'Create an alias/shortcut to the file';
@@ -168,6 +169,7 @@
 $string['popup'] = 'Click "Login" button to login';
 $string['popupblockeddownload'] = 'The downloading window is blocked, please allow the popup window, and try again.';
 $string['preview'] = 'Preview';
+$string['privatefilesof'] = '{$a} Private files';
 $string['readonlyinstance'] = 'You cannot edit/delete a read-only instance';
 $string['referencesexist'] = 'There are {$a} alias/shortcut files that use this file as their source';
 $string['referenceslist'] = 'Aliases/Shortcuts';
@@ -202,6 +204,7 @@
 $string['upload'] = 'Upload this file';
 $string['uploading'] = 'Uploading...';
 $string['uploadsucc'] = 'The file has been uploaded successfully';
+$string['undisclosedsource'] = '(Undisclosed)';
 $string['undisclosedreference'] = '(Undisclosed)';
 $string['uselatestfile'] = 'Use latest file';
 $string['usercontextrepositorydisabled'] = 'You cannot edit this repository in user context';

lib/filebrowser/file_info.php

@@ -227,17 +227,13 @@ public function get_sortorder() {
     }
 
     /**
-     * Returns the localised human-readable name of the file together with
-     * virtual path
+     * Returns the localised human-readable name of the file together with virtual path
      *
+     * @see file_info_stored::get_readable_fullname()
      * @return string
      */
     public function get_readable_fullname() {
-        $fpath = array();
-        for ($parent = $this; $parent; $parent = $parent->get_parent()) {
-            array_unshift($fpath, $parent->get_visible_name());
-        }
-        return join('/', $fpath);
+        return null;
     }
 
     /**

lib/filebrowser/file_info_stored.php

@@ -113,6 +113,46 @@ public function get_visible_name() {
         }
     }
 
+    /**
+     * Returns the localised human-readable name of the file together with virtual path
+     *
+     * @return string
+     */
+    public function get_readable_fullname() {
+        global $CFG;
+        // retrieve the readable path with all parents (excluding the top most 'System')
+        $fpath = array();
+        for ($parent = $this; $parent && $parent->get_parent(); $parent = $parent->get_parent()) {
+            array_unshift($fpath, $parent->get_visible_name());
+        }
+
+        if ($this->lf->get_component() == 'user' && $this->lf->get_filearea() == 'private') {
+            // use the special syntax for user private files - 'USERNAME Private files: PATH'
+            $username = array_shift($fpath);
+            array_shift($fpath); // get rid of "Private Files/" in the beginning of the path
+            return get_string('privatefilesof', 'repository', $username). ': '. join('/', $fpath);
+        } else {
+            // for all other files (except user private files) return 'Server files: PATH'
+
+            // first, get and cache the name of the repository_local (will be used as prefix for file names):
+            static $replocalname = null;
+            if ($replocalname === null) {
+                require_once($CFG->dirroot . "/repository/lib.php");
+                $instances = repository::get_instances(array('type' => 'local'));
+                if (count($instances)) {
+                    $firstinstance = reset($instances);
+                    $replocalname = $firstinstance->get_name();
+                } else if (get_string_manager()->string_exists('pluginname', 'repository_local')) {
+                    $replocalname = get_string('pluginname', 'repository_local');
+                } else {
+                    $replocalname = get_string('arearoot', 'repository');
+                }
+            }
+
+            return $replocalname. ': '. join('/', $fpath);
+        }
+    }
+
     /**
      * Returns file download url
      *

lib/filelib.php

@@ -596,6 +596,9 @@ function file_get_drafarea_files($draftitemid, $filepath = '/') {
             $item->datemodified = $file->get_timemodified();
             $item->datecreated = $file->get_timecreated();
             $item->isref = $file->is_external_file();
+            if ($item->isref && $file->get_status() == 666) {
+                $item->originalmissing = true;
+            }
             // find the file this draft file was created from and count all references in local
             // system pointing to that file
             $source = unserialize($file->get_source());
@@ -2310,7 +2313,7 @@ function send_stored_file($stored_file, $lifetime=86400 , $filter=0, $forcedownl
     }
 
     // handle external resource
-    if ($stored_file->is_external_file()) {
+    if ($stored_file && $stored_file->is_external_file()) {
         $stored_file->send_file($lifetime, $filter, $forcedownload, $options);
         die;
     }

lib/filestorage/file_storage.php

@@ -1652,10 +1652,22 @@ public static function pack_reference($params) {
      * Unpack reference field
      *
      * @param string $str
+     * @param bool $cleanparams if set to true, array elements will be passed through {@link clean_param()}
      * @return array
      */
-    public static function unpack_reference($str) {
-        return unserialize(base64_decode($str));
+    public static function unpack_reference($str, $cleanparams = false) {
+        $params = unserialize(base64_decode($str));
+        if (is_array($params) && $cleanparams) {
+            $params = array(
+                'component' => is_null($params['component']) ? ''   : clean_param($params['component'], PARAM_COMPONENT),
+                'filearea'  => is_null($params['filearea'])  ? ''   : clean_param($params['filearea'], PARAM_AREA),
+                'itemid'    => is_null($params['itemid'])    ? 0    : clean_param($params['itemid'], PARAM_INT),
+                'filename'  => is_null($params['filename'])  ? null : clean_param($params['filename'], PARAM_FILE),
+                'filepath'  => is_null($params['filepath'])  ? null : clean_param($params['filepath'], PARAM_PATH),
+                'contextid' => is_null($params['contextid']) ? null : clean_param($params['contextid'], PARAM_INT)
+            );
+        }
+        return $params;
     }
 
     /**

lib/filestorage/stored_file.php

@@ -522,29 +522,16 @@ public function get_parent_directory() {
     }
 
     /**
-     * Sync external files
+     * Synchronize file if it is a reference and needs synchronizing
      *
-     * @return bool true if file content changed, false if not
+     * Updates contenthash and filesize
      */
     public function sync_external_file() {
-        global $CFG, $DB;
-        if (empty($this->file_record->referencefileid)) {
-            return false;
-        }
-        if (empty($this->file_record->referencelastsync) or ($this->file_record->referencelastsync + $this->file_record->referencelifetime < time())) {
+        global $CFG;
+        if (!empty($this->file_record->referencefileid)) {
             require_once($CFG->dirroot.'/repository/lib.php');
-            if (repository::sync_external_file($this)) {
-                $prevcontent = $this->file_record->contenthash;
-                $sql = "SELECT f.*, r.repositoryid, r.reference
-                          FROM {files} f
-                     LEFT JOIN {files_reference} r
-                               ON f.referencefileid = r.id
-                         WHERE f.id = ?";
-                $this->file_record = $DB->get_record_sql($sql, array($this->file_record->id), MUST_EXIST);
-                return ($prevcontent !== $this->file_record->contenthash);
-            }
+            repository::sync_external_file($this);
         }
-        return false;
     }
 
     /**
@@ -858,7 +845,45 @@ public function get_reference() {
      * @return string
      */
     public function get_reference_details() {
-        return $this->repository->get_reference_details($this->get_reference());
+        return $this->repository->get_reference_details($this->get_reference(), $this->get_status());
+    }
+
+    /**
+     * Called after reference-file has been synchronized with the repository
+     *
+     * We update contenthash, filesize and status in files table if changed
+     * and we always update lastsync in files_reference table
+     *
+     * @param type $contenthash
+     * @param type $filesize
+     */
+    public function set_synchronized($contenthash, $filesize, $status = 0) {
+        global $DB;
+        if (!$this->is_external_file()) {
+            return;
+        }
+        $now = time();
+        $filerecord = new stdClass();
+        if ($this->get_contenthash() !== $contenthash) {
+            $filerecord->contenthash = $contenthash;
+        }
+        if ($this->get_filesize() != $filesize) {
+            $filerecord->filesize = $filesize;
+        }
+        if ($this->get_status() != $status) {
+            $filerecord->status = $status;
+        }
+        $filerecord->referencelastsync = $now; // TODO MDL-33416 remove this
+        if (!empty($filerecord)) {
+            $this->update($filerecord);
+        }
+
+        $DB->set_field('files_reference', 'lastsync', $now, array('id'=>$this->get_referencefileid()));
+        // $this->file_record->lastsync = $now; // TODO MDL-33416 uncomment or remove
+    }
+
+    public function set_missingsource() {
+        $this->set_synchronized($this->get_contenthash(), 0, 666);
     }
 
     /**

lib/form/filemanager.js

@@ -528,6 +528,9 @@ M.form_filemanager.init = function(Y, options) {
                     if (node.refcount) {
                         classname = classname + ' fp-hasreferences';
                     }
+                    if (node.originalmissing) {
+                        classname = classname + ' fp-originalmissing';
+                    }
                     if (node.sortorder == 1) { classname = classname + ' fp-mainfile';}
                     return Y.Lang.trim(classname);
                 }

mod/resource/locallib.php

@@ -295,23 +295,27 @@ function resource_get_optional_details($resource, $cm) {
         $context = context_module::instance($cm->id);
         $size = '';
         $type = '';
-        if (!empty($options['showsize'])) {
-            $size = display_size($DB->get_field_sql(
-                    'SELECT SUM(filesize) FROM {files} WHERE contextid=?', array($context->id)));
+        $fs = get_file_storage();
+        $files = $fs->get_area_files($context->id, 'mod_resource', 'content', 0, 'sortorder DESC, id ASC', false);
+        if (!empty($options['showsize']) && count($files)) {
+            $sizebytes = 0;
+            foreach ($files as $file) {
+                // this will also synchronize the file size for external files if needed
+                $sizebytes += $file->get_filesize();
+            }
+            if ($sizebytes) {
+                $size = display_size($sizebytes);
+            }
         }
-        if (!empty($options['showtype'])) {
+        if (!empty($options['showtype']) && count($files)) {
             // For a typical file resource, the sortorder is 1 for the main file
             // and 0 for all other files. This sort approach is used just in case
             // there are situations where the file has a different sort order
-            $record = $DB->get_record_sql(
-                    'SELECT filename, mimetype FROM {files} WHERE contextid=? ORDER BY sortorder DESC',
-                    array($context->id), IGNORE_MULTIPLE);
+            $mainfile = reset($files);
+            $type = get_mimetype_description($mainfile);
             // Only show type if it is not unknown
-            if ($record) {
-                $type = get_mimetype_description($record);
-                if ($type === get_mimetype_description('document/unknown')) {
-                    $type = '';
-                }
+            if ($type === get_mimetype_description('document/unknown')) {
+                $type = '';
             }
         }
 

repository/boxnet/lib.php

@@ -267,44 +267,60 @@ public function get_file_reference($source) {
     }
 
     /**
-     * Get file from external repository by reference
+     * Returns information about file in this repository by reference
      * {@link repository::get_file_reference()}
      * {@link repository::get_file()}
      *
+     * Returns null if file not found or is not readable
+     *
      * @param stdClass $reference file reference db record
-     * @return stdClass|null|false
+     * @return null|stdClass with attribute 'filepath'
      */
     public function get_file_by_reference($reference) {
-        $fileinfo = new stdClass;
         $boxnetfile = $this->get_file($reference->reference);
-        $fileinfo->filepath = $boxnetfile['path'];
-        return $fileinfo;
+        // Please note that here we will ALWAYS receive a file
+        // If source file has been removed from external server, box.com still returns
+        // a plain/text file with content 'no such file' (filesize will be 12 bytes)
+        if (!empty($boxnetfile['path'])) {
+            return (object)array('filepath' => $boxnetfile['path']);
+        }
+        return null;
     }
 
     /**
      * Return human readable reference information
      * {@link stored_file::get_reference()}
      *
      * @param string $reference
-     * @return string|null
+     * @param int $filestatus status of the file, 0 - ok, 666 - source missing
+     * @return string
      */
-    public function get_reference_details($reference) {
+    public function get_reference_details($reference, $filestatus = 0) {
         // Indicate it's from box.net repository + secure URL
-        return $this->get_name() . ': ' . $reference;
+        $details = $this->get_name() . ': ' . $reference;
+        if (!$filestatus) {
+            return $details;
+        } else {
+            // at the moment for box.net files we never can be sure that source is missing
+            // because box.com never returns 404 error.
+            // So we never change the status and actually this part is unreachable
+            return get_string('lostsource', 'repository', $details);
+        }
     }
 
     /**
-     * Repository method to serve file
+     * Repository method to serve the referenced file
      *
-     * @param stored_file $storedfile
+     * @param stored_file $storedfile the file that contains the reference
      * @param int $lifetime Number of seconds before the file should expire from caches (default 24 hours)
      * @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
      * @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
      * @param array $options additional options affecting the file serving
      */
     public function send_file($storedfile, $lifetime=86400 , $filter=0, $forcedownload=false, array $options = null) {
         $ref = $storedfile->get_reference();
-        // Let box.net serve the file.
+        // Let box.net serve the file. It will return 'no such file' content if file not found
+        // also if file has the different name than alias, it will be returned with the box.net filename
         header('Location: ' . $ref);
     }
 }