New meta.syncable plugin keep tracks of all changes with a global rev…

…ision number (inspired by couchdb mechanism)
pydio · Dec 3, 2013 · 99449b6 · 99449b6
1 parent c76f181
commit 99449b6
Show file tree

Hide file tree

Showing 5 changed files with 280 additions and 0 deletions.
diff --git a/core/src/plugins/meta.syncable/class.ChangesTracker.php b/core/src/plugins/meta.syncable/class.ChangesTracker.php
@@ -0,0 +1,177 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+defined('AJXP_EXEC') or die('Access not allowed');
+
+/**
+ * Generates and caches and md5 hash of each file
+ * @package AjaXplorer_Plugins
+ * @subpackage Meta
+ */
+class ChangesTracker extends AJXP_Plugin
+{
+    protected $accessDriver;
+    private $sqlDriver;
+
+    public function init($options)
+    {
+        $this->sqlDriver = AJXP_Utils::cleanDibiDriverParameters(array("group_switch_value" => "core"));
+        parent::init($options);
+    }
+
+    public function initMeta($accessDriver)
+    {
+        $this->accessDriver = $accessDriver;
+    }
+
+    public function switchActions($actionName, $httpVars, $fileVars)
+    {
+        if($actionName != "changes" || !isSet($httpVars["seq_id"])) return false;
+
+        require_once(AJXP_BIN_FOLDER."/dibi.compact.php");
+        dibi::connect($this->sqlDriver);
+
+        HTMLWriter::charsetHeader('application/json', 'UTF-8');
+
+        $res = dibi::query("SELECT * FROM [ajxp_changes]
+                LEFT JOIN [ajxp_index]
+                    ON [ajxp_changes].[node_id] = [ajxp_index].[node_id]
+                WHERE [ajxp_changes].[repository_identifier] = %s AND [seq] > %i
+                ORDER BY [ajxp_changes].[node_id], [seq] ASC",
+            $this->computeIdentifier(ConfService::getRepository()), AJXP_Utils::sanitize($httpVars["seq_id"], AJXP_SANITIZE_ALPHANUM));
+
+        echo '{"changes":[';
+        $previousNodeId = -1;
+        $previousRow = null;
+        $order = array("path"=>0, "content"=>1, "create"=>2, "delete"=>3);
+        $relocateAttrs = array("bytesize", "md5", "mtime", "node_path", "repository_identifier");
+        foreach ($res as $row) {
+            $row->node = array();
+            foreach ($relocateAttrs as $att) {
+                $row->node[$att] = $row->$att;
+                unset($row->$att);
+            }
+            if ($row->node_id == $previousNodeId) {
+                $previousRow->target = $row->target;
+                $previousRow->seq = $row->seq;
+                if ($order[$row->type] > $order[$previousRow->type]) {
+                    $previousRow->type = $row->type;
+                }
+            } else {
+                if (isSet($previousRow) && ($previousRow->source != $previousRow->target || $previousRow->type == "content")) {
+                    echo json_encode($previousRow) . ",";
+                }
+                $previousRow = $row;
+                $previousNodeId = $row->node_id;
+            }
+            $lastSeq = $row->seq;
+            flush();
+        }
+        if (isSet($previousRow) && ($previousRow->source != $previousRow->target || $previousRow->type == "content")) {
+            echo json_encode($previousRow);
+        }
+        if (isSet($lastSeq)) {
+            echo '], "last_seq":'.$lastSeq.'}';
+        } else {
+            $lastSeq = dibi::query("SELECT MAX([seq]) FROM [ajxp_changes]")->fetchSingle();
+            if(empty($lastSeq)) $lastSeq = 1;
+            echo '], "last_seq":'.$lastSeq.'}';
+        }
+
+    }
+
+    /**
+     * @param Repository $repository
+     * @return String
+     */
+    protected function computeIdentifier($repository){
+        $parts = array($repository->getId());
+        if($repository->securityScope() == 'USER'){
+            $parts[] = AuthService::getLoggedUser()->getId();
+        }else if($repository->securityScope() == 'GROUP'){
+            $parts[] = AuthService::getLoggedUser()->getGroupPath();
+        }
+        return implode("-", $parts);
+    }
+
+    /**
+     * @param AJXP_Node $oldNode
+     * @param AJXP_Node $newNode
+     * @param bool $copy
+     */
+    public function updateNodesIndex($oldNode = null, $newNode = null, $copy = false)
+    {
+
+        require_once(AJXP_BIN_FOLDER."/dibi.compact.php");
+        try {
+            if ($newNode == null) {
+                $repoId = $this->computeIdentifier($oldNode->getRepository());
+                // DELETE
+                dibi::query("DELETE FROM [ajxp_index] WHERE [node_path] LIKE %like~ AND [repository_identifier] = %s", $oldNode->getPath(), $repoId);
+            } else if ($oldNode == null) {
+                // CREATE
+                $stat = stat($newNode->getUrl());
+                $res = dibi::query("INSERT INTO [ajxp_index]", array(
+                    "node_path" => $newNode->getPath(),
+                    "bytesize"  => $stat["size"],
+                    "mtime"     => $stat["mtime"],
+                    "md5"       => $newNode->isLeaf()? md5_file($newNode->getUrl()):"directory",
+                    "repository_identifier" => $repoId = $this->computeIdentifier($newNode->getRepository())
+                ));
+            } else {
+                $repoId = $this->computeIdentifier($oldNode->getRepository());
+                if ($oldNode->getPath() == $newNode->getPath()) {
+                    // CONTENT CHANGE
+                    $stat = stat($newNode->getUrl());
+                    dibi::query("UPDATE [ajxp_index] SET ", array(
+                        "bytesize"  => $stat["size"],
+                        "mtime"     => $stat["mtime"],
+                        "md5"       => md5_file($newNode->getUrl())
+                    ), "WHERE [node_path] = %s AND [repository_identifier] = %s", $oldNode->getPath(), $repoId);
+                } else {
+                    // PATH CHANGE ONLY
+                    $newNode->loadNodeInfo();
+                    if ($newNode->isLeaf()) {
+                        dibi::query("UPDATE [ajxp_index] SET ", array(
+                            "node_path"  => $newNode->getPath(),
+                        ), "WHERE [node_path] = %s AND [repository_identifier] = %s", $oldNode->getPath(), $repoId);
+                    } else {
+                        dibi::query("UPDATE [ajxp_index] SET [node_path]=REPLACE( REPLACE(CONCAT('$$$',[node_path]), CONCAT('$$$', %s), CONCAT('$$$', %s)) , '$$$', '') ",
+                            $oldNode->getPath(),
+                            $newNode->getPath()
+                            , "WHERE [node_path] LIKE %like~ AND [repository_identifier] = %s", $oldNode->getPath(), $repoId);
+                    }
+
+                }
+            }
+        } catch (Exception $e) {
+            AJXP_Logger::error("[meta.syncable]", "Indexation", $e->getMessage());
+        }
+
+    }
+
+    public function installSQLTables($param)
+    {
+        $p = AJXP_Utils::cleanDibiDriverParameters($param["SQL_DRIVER"]);
+        return AJXP_Utils::runCreateTablesQuery($p, $this->getBaseDir()."/create.sql");
+    }
+
+}
diff --git a/core/src/plugins/meta.syncable/create.mysql b/core/src/plugins/meta.syncable/create.mysql
@@ -0,0 +1,45 @@
+CREATE TABLE IF NOT EXISTS `ajxp_changes` (
+  `seq` int(20) NOT NULL AUTO_INCREMENT,
+  `repository_identifier` TEXT NOT NULL,
+  `node_id` bigint(20) NOT NULL,
+  `type` enum('create','delete','path','content') NOT NULL,
+  `source` text NOT NULL,
+  `target` text NOT NULL,
+  PRIMARY KEY (`seq`),
+  KEY `node_id` (`node_id`,`type`)
+);
+
+CREATE TABLE IF NOT EXISTS `ajxp_index` (
+  `node_id` int(20) NOT NULL AUTO_INCREMENT,
+  `node_path` text NOT NULL,
+  `bytesize` bigint(20) NOT NULL,
+  `md5` varchar(32) NOT NULL,
+  `mtime` int(11) NOT NULL,
+  `repository_identifier` text NOT NULL,
+  PRIMARY KEY (`node_id`)
+);
+
+--
+-- Déclencheurs `ajxp_index`
+--
+DROP TRIGGER IF EXISTS `LOG_DELETE`;
+DELIMITER //
+CREATE TRIGGER `LOG_DELETE` AFTER DELETE ON `ajxp_index`
+FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type)
+  VALUES (old.repository_identifier, old.node_id, old.node_path, 'NULL', 'delete')
+//
+DELIMITER ;
+DROP TRIGGER IF EXISTS `LOG_INSERT`;
+DELIMITER //
+CREATE TRIGGER `LOG_INSERT` AFTER INSERT ON `ajxp_index`
+FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type)
+  VALUES (new.repository_identifier, new.node_id, 'NULL', new.node_path, 'create')
+//
+DELIMITER ;
+DROP TRIGGER IF EXISTS `LOG_UPDATE`;
+DELIMITER //
+CREATE TRIGGER `LOG_UPDATE` AFTER UPDATE ON `ajxp_index`
+FOR EACH ROW INSERT INTO ajxp_changes (repository_identifier, node_id,source,target,type)
+  VALUES (new.repository_identifier, new.node_id, old.node_path, new.node_path, CASE old.node_path = new.node_path WHEN true THEN 'content' ELSE 'path' END)
+//
+DELIMITER ;
diff --git a/core/src/plugins/meta.syncable/create.sqlite b/core/src/plugins/meta.syncable/create.sqlite
@@ -0,0 +1,37 @@
+CREATE TABLE ajxp_changes (
+  seq INTEGER PRIMARY KEY AUTOINCREMENT,
+  repository_identifier TEXT,
+  node_id NUMERIC,
+  type TEXT,
+  source TEXT,
+  target TEXT
+)
+
+CREATE TABLE ajxp_index (
+  node_id INTEGER PRIMARY KEY AUTOINCREMENT,
+  repository_identifier TEXT,
+  node_path TEXT,
+  bytesize NUMERIC,
+  md5 TEXT,
+  mtime NUMERIC
+)
+
+CREATE TRIGGER LOG_DELETE AFTER DELETE ON ajxp_index
+BEGIN
+ INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (old.repository_identifer, old.node_id, old.node_path, "NULL", "delete");
+END
+
+CREATE TRIGGER LOG_INSERT AFTER INSERT ON ajxp_index
+BEGIN
+INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, "NULL", new.node_path, "create");
+END
+
+CREATE TRIGGER "LOG_UPDATE_CONTENT" AFTER UPDATE ON "ajxp_index" FOR EACH ROW  WHEN old.node_path=new.node_path
+BEGIN
+INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, old.node_path, new.node_path, "content");
+END
+
+CREATE TRIGGER "LOG_UPDATE_PATH" AFTER UPDATE ON "ajxp_index" FOR EACH ROW  WHEN old.node_path!=new.node_path
+BEGIN
+INSERT INTO ajxp_changes (repository_identifer, node_id,source,target,type) VALUES (new.repository_identifer, new.node_id, old.node_path, new.node_path, "path");
+END
diff --git a/core/src/plugins/meta.syncable/manifest.xml b/core/src/plugins/meta.syncable/manifest.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<meta id="meta.syncable" label="CONF_MESSAGE[Syncable Workspace]" description="CONF_MESSAGE[Track changes on this workspace to enable the synchronization with an external client]" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:noNamespaceSchemaLocation="file:../core.ajaxplorer/ajxp_registry.xsd">
+    <class_definition filename="plugins/meta.syncable/class.ChangesTracker.php" classname="ChangesTracker"/>
+    <server_settings>
+        <global_param type="button" name="INSTALL_SQL" choices="run_plugin_action:meta.syncable:installSQLTables" label="CONF_MESSAGE[SQL Tables]" description="CONF_MESSAGE[Install SQL Tables]" mandatory="false"/>
+    </server_settings>
+    <registry_contributions>
+           <actions>
+               <action name="changes">
+                   <processing>
+                       <serverCallback methodName="switchActions" restParams="/seq_id"/>
+                   </processing>
+               </action>
+           </actions>
+   		<hooks>
+   			<serverCallback hookName="node.change" methodName="updateNodesIndex" defer="true"/>
+   		</hooks>
+    </registry_contributions>
+</meta>
diff --git a/core/src/plugins/meta.syncable/plugin_doc.html b/core/src/plugins/meta.syncable/plugin_doc.html
@@ -0,0 +1 @@
+<p>This plugin generates an index of the workspaces nodes inside the DB. A changes feed is generated from this index using SQL Triggers, thus with high performance.</p>