Big refactoring to decouple meta.syncable from index.lucene: core.ind…

…ex is now responsible for triggering a recursive indexation and sending corresponding events.

core/src/plugins/core.index/class.CoreIndexer.php

@@ -0,0 +1,248 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+defined('AJXP_EXEC') or die( 'Access not allowed');
+
+class CoreIndexer extends AJXP_Plugin {
+
+    private $verboseIndexation = false;
+
+
+    public function logDebug($message = ""){
+        parent::logDebug("core.indexer", $message);
+        if($this->verboseIndexation && ConfService::currentContextIsCommandLine()){
+            print($message."\n");
+        }
+    }
+
+    public function applyAction($actionName, $httpVars, $fileVars)
+    {
+        $messages = ConfService::getMessages();
+
+        if ($actionName == "index") {
+
+            $repository = ConfService::getRepository();
+            $repositoryId = $repository->getId();
+            $userSelection = new UserSelection($repository, $httpVars);
+            if($userSelection->isEmpty()){
+                $userSelection->addFile("/");
+            }
+            $nodes = $userSelection->buildNodes($repository->driverInstance);
+
+            if (isSet($httpVars["verbose"]) && $httpVars["verbose"] == "true") {
+                $this->verboseIndexation = true;
+            }
+
+            if (ConfService::backgroundActionsSupported() && !ConfService::currentContextIsCommandLine()) {
+                AJXP_Controller::applyActionInBackground($repositoryId, "index", $httpVars);
+                AJXP_XMLWriter::header();
+                AJXP_XMLWriter::triggerBgAction("check_index_status", array("repository_id" => $repositoryId), sprintf($messages["core.index.8"], $nodes[0]->getPath()), true, 2);
+                if(!isSet($httpVars["inner_apply"])){
+                    AJXP_XMLWriter::close();
+                }
+                return null;
+            }
+
+            // GIVE BACK THE HAND TO USER
+            session_write_close();
+
+            foreach($nodes as $node){
+
+                // SIMPLE FILE
+                if(!is_dir($node->getUrl())){
+                    try{
+                        $this->logDebug("Indexing - node.index ".$node->getUrl());
+                        AJXP_Controller::applyHook("node.index", array($node));
+                    }catch (Exception $e){
+                        $this->logDebug("Error Indexing Node ".$node->getUrl()." (".$e->getMessage().")");
+                    }
+                }else{
+                    try{
+                        $this->recursiveIndexation($node);
+                    }catch (Exception $e){
+                        $this->logDebug("Indexation of ".$node->getUrl()." interrupted by error: (".$e->getMessage().")");
+                    }
+                }
+
+            }
+
+        } else if ($actionName == "check_index_status") {
+            $repoId = $httpVars["repository_id"];
+            list($status, $message) = $this->getIndexStatus(ConfService::getRepositoryById($repoId), AuthService::getLoggedUser());
+            if (!empty($status)) {
+                AJXP_XMLWriter::header();
+                AJXP_XMLWriter::triggerBgAction("check_index_status", array("repository_id" => $repoId), $message, true, 3);
+                AJXP_XMLWriter::close();
+            } else {
+                AJXP_XMLWriter::header();
+                AJXP_XMLWriter::triggerBgAction("info_message", array(), $messages["core.index.5"], true, 5);
+                AJXP_XMLWriter::close();
+            }
+        }
+        return null;
+    }
+
+    /**
+     *
+     * @param AJXP_Node $node
+     * @param int $depth
+     * @throws Exception
+     */
+    public function recursiveIndexation($node, $depth = 0)
+    {
+        $repository = $node->getRepository();
+        $user = $node->getUser();
+        $messages = ConfService::getMessages();
+        if($user == null && AuthService::usersEnabled()) $user = AuthService::getLoggedUser();
+        if($depth == 0){
+            $this->logDebug("Starting indexation - node.index.recursive.start  - ". memory_get_usage(true) ."  - ". $node->getUrl());
+            $this->setIndexStatus("RUNNING", str_replace("%s", $node->getPath(), $messages["core.index.8"]), $repository, $user);
+            AJXP_Controller::applyHook("node.index.recursive.start", array($node));
+        }else{
+            if($this->isInterruptRequired($repository, $user)){
+                $this->logDebug("Interrupting indexation! - node.index.recursive.end - ". $node->getUrl());
+                AJXP_Controller::applyHook("node.index.recursive.end", array($node));
+                $this->releaseStatus($repository, $user);
+                throw new Exception("User interrupted");
+            }
+        }
+
+        if(!ConfService::currentContextIsCommandLine()) @set_time_limit(120);
+        $url = $node->getUrl();
+        $this->logDebug("Indexing Node parent node ".$url);
+        $this->setIndexStatus("RUNNING", str_replace("%s", $node->getPath(), $messages["core.index.8"]), $repository, $user);
+        try {
+            AJXP_Controller::applyHook("node.index", array($node));
+        } catch (Exception $e) {
+            $this->logDebug("Error Indexing Node ".$url." (".$e->getMessage().")");
+        }
+
+        $handle = opendir($url);
+        if ($handle !== false) {
+            while ( ($child = readdir($handle)) != false) {
+                if($child[0] == ".") continue;
+                $childNode = new AJXP_Node(rtrim($url, "/")."/".$child);
+                $childUrl = $childNode->getUrl();
+                if(is_dir($childUrl)){
+                    $this->logDebug("Entering recursive indexation for ".$childUrl);
+                    $this->recursiveIndexation($childNode, $depth + 1);
+                }else{
+                    try {
+                        $this->logDebug("Indexing Node ".$childUrl);
+                        AJXP_Controller::applyHook("node.index", array($childNode));
+                    } catch (Exception $e) {
+                        $this->logDebug("Error Indexing Node ".$childUrl." (".$e->getMessage().")");
+                    }
+                }
+            }
+            closedir($handle);
+        } else {
+            $this->logDebug("Cannot open $url!!");
+        }
+        if($depth == 0){
+            $this->logDebug("End indexation - node.index.recursive.end - ". memory_get_usage(true) ."  -  ". $node->getUrl());
+            $this->setIndexStatus("RUNNING", "Indexation finished, cleaning...", $repository, $user);
+            AJXP_Controller::applyHook("node.index.recursive.end", array($node));
+            $this->releaseStatus($repository, $user);
+            $this->logDebug("End indexation - After node.index.recursive.end - ". memory_get_usage(true) ."  -  ". $node->getUrl());
+        }
+    }
+
+
+    /**
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     * @return Array
+     */
+    protected function buildIndexLockKey($repository, $user){
+        $scope = $repository->securityScope();
+        $key = $repository->getId();
+        if($scope == "USER"){
+            $key .= "-".$user->getId();
+        }else if($scope == "GROUP"){
+            $key .= "-".ltrim(str_replace("/", "__", $user->getGroupPath()), "__");
+        }
+        return $key;
+    }
+
+
+    /**
+     * @param String $status
+     * @param String $message
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     */
+    protected function setIndexStatus($status, $message, $repository, $user)
+    {
+        $iPath = (defined('AJXP_SHARED_CACHE_DIR')?AJXP_SHARED_CACHE_DIR:AJXP_CACHE_DIR)."/indexes";
+        if(!is_dir($iPath)) mkdir($iPath,0755, true);
+        $f = $iPath."/.indexation_status-".$this->buildIndexLockKey($repository, $user);
+        $this->logDebug("Updating file ".$f." with status $status - $message");
+        file_put_contents($f, strtoupper($status).":".$message);
+    }
+
+    /**
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     * @return Array Array(STATUS, Message)
+     */
+    protected function getIndexStatus($repository, $user)
+    {
+        $f = (defined('AJXP_SHARED_CACHE_DIR')?AJXP_SHARED_CACHE_DIR:AJXP_CACHE_DIR)."/indexes/.indexation_status-".$this->buildIndexLockKey($repository, $user);
+        if (file_exists($f)){
+            return explode(":", file_get_contents($f));
+        }else{
+            return array("", "");
+        }
+    }
+
+    /**
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     */
+    protected function releaseStatus($repository, $user)
+    {
+        $f = (defined('AJXP_SHARED_CACHE_DIR')?AJXP_SHARED_CACHE_DIR:AJXP_CACHE_DIR)."/indexes/.indexation_status-".$this->buildIndexLockKey($repository, $user);
+        $this->logDebug("Removing file ".$f);
+        @unlink($f);
+    }
+
+
+    /**
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     */
+    protected function requireInterrupt($repository, $user)
+    {
+        $this->setIndexStatus("INTERRUPT", "Interrupt required by user", $repository, $user);
+    }
+
+    /**
+     * @param Repository $repository
+     * @param AbstractAjxpUser $user
+     * @return boolean
+     */
+    protected function isInterruptRequired($repository, $user)
+    {
+        list($status, $message) = $this->getIndexStatus($repository, $user);
+        return ($status == "INTERRUPT");
+    }
+
+} 

core/src/plugins/core.index/i18n/de.php

@@ -0,0 +1,33 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+$mess = array(
+    "1" => "Index Inhalt",
+    "2" => "Index Verzeichnisinhalt",
+    "3" => "Indexieren im Hintergrund ausführen",
+    "4" => "Der Index scheint gesperrt zu sein, ein Indexierungsprozess könnte bereits laufen. Falls nicht, bitte die .ajxp_lock-Datei im Verzeichnis cache/indexes löschen.",
+    "5" => "Indexierung abgeschlossen",
+    "6" => "Warning, the repository is currently being indexed, please retry later.",
+    "7" => "First indexing of this repository, please wait and retry your search",
+    "8" => "Indexing %s in background",
+    "9" => "Cannot find index for current repository! You should trigger the indexation of the data first!",
+    "10" => "Indexing subfolders in background",
+);

core/src/plugins/core.index/i18n/en.php

@@ -0,0 +1,33 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+$mess = array(
+    "1" => "Index Content",
+    "2" => "Index folder content",
+    "3" => "Running indexation in background",
+    "4" => "The index seems to be locked, an indexation processs may be running. If not, please remove the .ajxp_lock file located in cache/indexes.",
+    "5" => "Indexation finished",
+    "6" => "The workspace is currently being indexed, please retry later. To force re-indexation, manually remove the .ajxp_lock file located in data/cache/indexes.",
+    "7" => "First indexing of this workspace, please wait and retry your search",
+    "8" => "Indexing %s in background",
+    "9" => "Cannot find index for current workspace! You should trigger the indexation of the data first!",
+    "10" => "Indexing subfolders in background",
+);

core/src/plugins/core.index/i18n/es.php

@@ -0,0 +1,33 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+$mess = array(
+    "1" => "Indexar contenido",
+    "2" => "Indexar el contenido de la carpeta",
+    "3" => "Ejecutando indexación en segundo plano",
+    "4" => "El índice parece estar bloqueado, puede que un proceso de indexación se esté realizando en este momento. En caso contrario borre el fichero  .ajxp_lock localizado en la carpeta cache/indexes.",
+    "5" => "Indexación finalizada",
+    "6" => "Atención, la carpeta actual está indexándose, inténtelo más tarde.",
+    "7" => "Primera indexación de esta carpeta, por favor espere y reintente más tarde su búsqueda",
+    "8" => "Indexando %s en segundo plano",
+    "9" => "No puedo encontrar el índice de la carpeta actual! Debería ejecutar la indexación de los datos primero!",
+    "10" => "Indexing subfolders in background",
+);

core/src/plugins/core.index/i18n/fr.php

@@ -0,0 +1,33 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+$mess = array(
+    "1" => "Indexer",
+    "2" => "Indexer le contenu du répertoire",
+    "3" => "Indexation en cours en arrière-plan",
+    "4" => "L'index est verrouillé, une indexation est peut-être en cours. Si ce n'est pas le cas, effacez manuellement le fichier \".ajxp_lock\" situé dans \"cache/indexes\"",
+    "5" => "Indexation terminée",
+    "6" => "Attention, le dépôt est en cours d'indexation, veuillez réessayer ultérieurement. Pour forcer une ré-indexation, effacez manuellement le fichier \".ajxp_lock\" situé dans \"data/cache/indexes\".",
+    "7" => "Lancement de la première indexation, veuillez attendre puis relancer votre recherche.",
+    "8" => "Indexation (%s) en arrière-plan.",
+    "9" => "Impossible de trouver un index pour ce dépôt, vous devrez déclencher l'indexation d'abord !",
+    "10" => "Indexation des sous-répertoires en arrière-plan",
+);