From 99449b648ee11dd31e8325aac3e16edeaee42ed4 Mon Sep 17 00:00:00 2001 From: cdujeu Date: Tue, 3 Dec 2013 11:30:09 +0100 Subject: [PATCH] New meta.syncable plugin keep tracks of all changes with a global revision number (inspired by couchdb mechanism) --- .../meta.syncable/class.ChangesTracker.php | 177 ++++++++++++++++++ core/src/plugins/meta.syncable/create.mysql | 45 +++++ core/src/plugins/meta.syncable/create.sqlite | 37 ++++ core/src/plugins/meta.syncable/manifest.xml | 20 ++ .../src/plugins/meta.syncable/plugin_doc.html | 1 + 5 files changed, 280 insertions(+) create mode 100644 core/src/plugins/meta.syncable/class.ChangesTracker.php create mode 100644 core/src/plugins/meta.syncable/create.mysql create mode 100644 core/src/plugins/meta.syncable/create.sqlite create mode 100644 core/src/plugins/meta.syncable/manifest.xml create mode 100644 core/src/plugins/meta.syncable/plugin_doc.html diff --git a/core/src/plugins/meta.syncable/class.ChangesTracker.php b/core/src/plugins/meta.syncable/class.ChangesTracker.php new file mode 100644 index 0000000000..49551f5c50 --- /dev/null +++ b/core/src/plugins/meta.syncable/class.ChangesTracker.php @@ -0,0 +1,177 @@ + + * This file is part of Pydio. + * + * Pydio is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Pydio is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Pydio. If not, see . + * + * The latest code can be found at . + */ + +defined('AJXP_EXEC') or die('Access not allowed'); + +/** + * Generates and caches and md5 hash of each file + * @package AjaXplorer_Plugins + * @subpackage Meta + */ +class ChangesTracker extends AJXP_Plugin +{ + protected $accessDriver; + private $sqlDriver; + + public function init($options) + { + $this->sqlDriver = AJXP_Utils::cleanDibiDriverParameters(array("group_switch_value" => "core")); + parent::init($options); + } + + public function initMeta($accessDriver) + { + $this->accessDriver = $accessDriver; + } + + public function switchActions($actionName, $httpVars, $fileVars) + { + if($actionName != "changes" || !isSet($httpVars["seq_id"])) return false; + + require_once(AJXP_BIN_FOLDER."/dibi.compact.php"); + dibi::connect($this->sqlDriver); + + HTMLWriter::charsetHeader('application/json', 'UTF-8'); + + $res = dibi::query("SELECT * FROM [ajxp_changes] + LEFT JOIN [ajxp_index] + ON [ajxp_changes].[node_id] = [ajxp_index].[node_id] + WHERE [ajxp_changes].[repository_identifier] = %s AND [seq] > %i + ORDER BY [ajxp_changes].[node_id], [seq] ASC", + $this->computeIdentifier(ConfService::getRepository()), AJXP_Utils::sanitize($httpVars["seq_id"], AJXP_SANITIZE_ALPHANUM)); + + echo '{"changes":['; + $previousNodeId = -1; + $previousRow = null; + $order = array("path"=>0, "content"=>1, "create"=>2, "delete"=>3); + $relocateAttrs = array("bytesize", "md5", "mtime", "node_path", "repository_identifier"); + foreach ($res as $row) { + $row->node = array(); + foreach ($relocateAttrs as $att) { + $row->node[$att] = $row->$att; + unset($row->$att); + } + if ($row->node_id == $previousNodeId) { + $previousRow->target = $row->target; + $previousRow->seq = $row->seq; + if ($order[$row->type] > $order[$previousRow->type]) { + $previousRow->type = $row->type; + } + } else { + if (isSet($previousRow) && ($previousRow->source != $previousRow->target || $previousRow->type == "content")) { + echo json_encode($previousRow) . ","; + } + $previousRow = $row; + $previousNodeId = $row->node_id; + } + $lastSeq = $row->seq; + flush(); + } + if (isSet($previousRow) && ($previousRow->source != $previousRow->target || $previousRow->type == "content")) { + echo json_encode($previousRow); + } + if (isSet($lastSeq)) { + echo '], "last_seq":'.$lastSeq.'}'; + } else { + $lastSeq = dibi::query("SELECT MAX([seq]) FROM [ajxp_changes]")->fetchSingle(); + if(empty($lastSeq)) $lastSeq = 1; + echo '], "last_seq":'.$lastSeq.'}'; + } + + } + + /** + * @param Repository $repository + * @return String + */ + protected function computeIdentifier($repository){ + $parts = array($repository->getId()); + if($repository->securityScope() == 'USER'){ + $parts[] = AuthService::getLoggedUser()->getId(); + }else if($repository->securityScope() == 'GROUP'){ + $parts[] = AuthService::getLoggedUser()->getGroupPath(); + } + return implode("-", $parts); + } + + /** + * @param AJXP_Node $oldNode + * @param AJXP_Node $newNode + * @param bool $copy + */ + public function updateNodesIndex($oldNode = null, $newNode = null, $copy = false) + { + + require_once(AJXP_BIN_FOLDER."/dibi.compact.php"); + try { + if ($newNode == null) { + $repoId = $this->computeIdentifier($oldNode->getRepository()); + // DELETE + dibi::query("DELETE FROM [ajxp_index] WHERE [node_path] LIKE %like~ AND [repository_identifier] = %s", $oldNode->getPath(), $repoId); + } else if ($oldNode == null) { + // CREATE + $stat = stat($newNode->getUrl()); + $res = dibi::query("INSERT INTO [ajxp_index]", array( + "node_path" => $newNode->getPath(), + "bytesize" => $stat["size"], + "mtime" => $stat["mtime"], + "md5" => $newNode->isLeaf()? md5_file($newNode->getUrl()):"directory", + "repository_identifier" => $repoId = $this->computeIdentifier($newNode->getRepository()) + )); + } else { + $repoId = $this->computeIdentifier($oldNode->getRepository()); + if ($oldNode->getPath() == $newNode->getPath()) { + // CONTENT CHANGE + $stat = stat($newNode->getUrl()); + dibi::query("UPDATE [ajxp_index] SET ", array( + "bytesize" => $stat["size"], + "mtime" => $stat["mtime"], + "md5" => md5_file($newNode->getUrl()) + ), "WHERE [node_path] = %s AND [repository_identifier] = %s", $oldNode->getPath(), $repoId); + } else { + // PATH CHANGE ONLY + $newNode->loadNodeInfo(); + if ($newNode->isLeaf()) { + dibi::query("UPDATE [ajxp_index] SET ", array( + "node_path" => $newNode->getPath(), + ), "WHERE [node_path] = %s AND [repository_identifier] = %s", $oldNode->getPath(), $repoId); + } else { + dibi::query("UPDATE [ajxp_index] SET [node_path]=REPLACE( REPLACE(CONCAT('$$$',[node_path]), CONCAT('$$$', %s), CONCAT('$$$', %s)) , '$$$', '') ", + $oldNode->getPath(), + $newNode->getPath() + , "WHERE [node_path] LIKE %like~ AND [repository_identifier] = %s", $oldNode->getPath(), $repoId); + } + + } + } + } catch (Exception $e) { + AJXP_Logger::error("[meta.syncable]", "Indexation", $e->getMessage()); + } + + } + + public function installSQLTables($param) + { + $p = AJXP_Utils::cleanDibiDriverParameters($param["SQL_DRIVER"]); + return AJXP_Utils::runCreateTablesQuery($p, $this->getBaseDir()."/create.sql"); + } + +} diff --git a/core/src/plugins/meta.syncable/create.mysql b/core/src/plugins/meta.syncable/create.mysql new file mode 100644 index 0000000000..d0523414b7 --- /dev/null +++ b/core/src/plugins/meta.syncable/create.mysql @@ -0,0 +1,45 @@ +CREATE TABLE IF NOT EXISTS `ajxp_changes` ( + `seq` int(20) NOT NULL AUTO_INCREMENT, + `repository_identifier` TEXT NOT NULL, + `node_id` bigint(20) NOT NULL, + `type` enum('create','delete','path','content') NOT NULL, + `source` text NOT NULL, + `target` text NOT NULL, + PRIMARY KEY (`seq`), + KEY `node_id` (`node_id`,`type`) +); + +CREATE TABLE IF NOT EXISTS `ajxp_index` ( + `node_id` int(20) NOT NULL AUTO_INCREMENT, + `node_path` text NOT NULL, + `bytesize` bigint(20) NOT NULL, + `md5` varchar(32) NOT NULL, + `mtime` int(11) NOT NULL, + `repository_identifier` text NOT NULL, + PRIMARY KEY (`node_id`) +); + +-- +-- Déclencheurs `ajxp_index` +-- +DROP TRIGGER IF EXISTS `LOG_DELETE`; +DELIMITER // +CREATE TRIGGER `LOG_DELETE` AFTER DELETE ON `ajxp_index` +FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type) + VALUES (old.repository_identifier, old.node_id, old.node_path, 'NULL', 'delete') +// +DELIMITER ; +DROP TRIGGER IF EXISTS `LOG_INSERT`; +DELIMITER // +CREATE TRIGGER `LOG_INSERT` AFTER INSERT ON `ajxp_index` +FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type) + VALUES (new.repository_identifier, new.node_id, 'NULL', new.node_path, 'create') +// +DELIMITER ; +DROP TRIGGER IF EXISTS `LOG_UPDATE`; +DELIMITER // +CREATE TRIGGER `LOG_UPDATE` AFTER UPDATE ON `ajxp_index` +FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type) + VALUES (new.repository_identifier, new.node_id, old.node_path, new.node_path, CASE old.node_path = new.node_path WHEN true THEN 'content' ELSE 'path' END) +// +DELIMITER ; diff --git a/core/src/plugins/meta.syncable/create.sqlite b/core/src/plugins/meta.syncable/create.sqlite new file mode 100644 index 0000000000..42e33ff9db --- /dev/null +++ b/core/src/plugins/meta.syncable/create.sqlite @@ -0,0 +1,37 @@ +CREATE TABLE ajxp_changes ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + repository_identifier TEXT, + node_id NUMERIC, + type TEXT, + source TEXT, + target TEXT +) + +CREATE TABLE ajxp_index ( + node_id INTEGER PRIMARY KEY AUTOINCREMENT, + repository_identifier TEXT, + node_path TEXT, + bytesize NUMERIC, + md5 TEXT, + mtime NUMERIC +) + +CREATE TRIGGER LOG_DELETE AFTER DELETE ON ajxp_index +BEGIN + INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (old.repository_identifer, old.node_id, old.node_path, "NULL", "delete"); +END + +CREATE TRIGGER LOG_INSERT AFTER INSERT ON ajxp_index +BEGIN +INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, "NULL", new.node_path, "create"); +END + +CREATE TRIGGER "LOG_UPDATE_CONTENT" AFTER UPDATE ON "ajxp_index" FOR EACH ROW WHEN old.node_path=new.node_path +BEGIN +INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, old.node_path, new.node_path, "content"); +END + +CREATE TRIGGER "LOG_UPDATE_PATH" AFTER UPDATE ON "ajxp_index" FOR EACH ROW WHEN old.node_path!=new.node_path +BEGIN +INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, old.node_path, new.node_path, "path"); +END \ No newline at end of file diff --git a/core/src/plugins/meta.syncable/manifest.xml b/core/src/plugins/meta.syncable/manifest.xml new file mode 100644 index 0000000000..01f42718e5 --- /dev/null +++ b/core/src/plugins/meta.syncable/manifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + diff --git a/core/src/plugins/meta.syncable/plugin_doc.html b/core/src/plugins/meta.syncable/plugin_doc.html new file mode 100644 index 0000000000..09f264e20b --- /dev/null +++ b/core/src/plugins/meta.syncable/plugin_doc.html @@ -0,0 +1 @@ +

This plugin generates an index of the workspaces nodes inside the DB. A changes feed is generated from this index using SQL Triggers, thus with high performance.

\ No newline at end of file