Base work for direct monitoring of the filesystem using watchdog

core/src/plugins/meta.monitor_fs/class.FSMonitoringManager.php

@@ -0,0 +1,57 @@
+<?php
+/*
+ * Copyright 2007-2013 Charles du Jeu - Abstrium SAS <team (at) pyd.io>
+ * This file is part of Pydio.
+ *
+ * Pydio is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Pydio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Pydio.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The latest code can be found at <http://pyd.io/>.
+ */
+
+defined('AJXP_EXEC') or die('Access not allowed');
+
+/**
+ * Monitor filesystem using Python
+ * @package AjaXplorer_Plugins
+ * @subpackage Meta
+ */
+class FSMonitoringManager extends AJXP_Plugin
+{
+    private $repoBase;
+
+    public function initMeta($accessDriver)
+    {
+        $repo = ConfService::getRepository();
+        $this->repoBase = $repo->getOption("PATH");
+    }
+
+    public function beforePathChange(AJXP_Node $node){
+        $this->informWatcher("path_change", $node->getPath());
+    }
+
+    public function beforeChange(AJXP_Node $node){
+        $this->informWatcher("content_change", $node->getPath());
+    }
+
+    public function beforeCreate(AJXP_Node $node){
+        $this->informWatcher("create", $node->getPath());
+    }
+
+    protected function informWatcher($action, $path)
+    {
+        $cmd = "python ".$this->getBaseDir()."/framework_watch.py --action=$action --path=". escapeshellarg($this->repoBase.SystemTextEncoding::fromUTF8($path));
+        AJXP_Controller::runCommandInBackground($cmd, $this->getBaseDir()."/cmd.out");
+    }
+
+}

core/src/plugins/meta.monitor_fs/filesystem_watch.py

@@ -0,0 +1,141 @@
+__author__ = 'charles'
+import time
+import os
+import sys
+import argparse
+import threading
+from watchdog.observers import Observer
+import logging
+import fnmatch
+
+from watchdog.events import LoggingEventHandler, FileSystemEventHandler
+
+
+class PydioEventHandler(FileSystemEventHandler):
+
+    def __init__(self, basepath, includes, excludes):
+        super(PydioEventHandler, self).__init__()
+        self.base = basepath
+        self.includes = includes
+        self.excludes = excludes
+
+    def remove_prefix(self, text):
+        return text[len(self.base):] if text.startswith(self.base) else text
+
+    def included(self, event, base=None):
+        if not base:
+            base = os.path.basename(event.src_path)
+        for i in self.includes:
+            if not fnmatch.fnmatch(base, i):
+                return False
+        for e in self.excludes:
+            if fnmatch.fnmatch(base, e):
+                return False
+        return True
+
+    def on_moved(self, event):
+        if not self.included(event):
+            return
+        logging.debug("Event: move noticed: " + event.event_type + " on file " + self.remove_prefix(event.src_path)
+                      + " at " + time.asctime())
+        t = (
+            self.remove_prefix(event.dest_path.decode('utf-8')),
+            self.remove_prefix(event.src_path.decode('utf-8')),
+        )
+
+    def on_created(self, event):
+        if not self.included(event):
+            return
+        logging.debug("Event: creation noticed: " + event.event_type +
+                         " on file " + self.remove_prefix(event.src_path) + " at " + time.asctime())
+
+        search_key = self.remove_prefix(event.src_path)
+        if event.is_directory:
+            hash_key = 'directory'
+        else:
+            #hash_key = hashfile(open(event.src_path, 'rb'), hashlib.md5())
+            pass
+
+    def on_deleted(self, event):
+        if not self.included(event):
+            return
+        logging.debug("Event: deletion noticed: " + event.event_type +
+                         " on file " + self.remove_prefix(event.src_path) + " at " + time.asctime())
+
+    def on_modified(self, event):
+        super(PydioEventHandler, self).on_modified(event)
+        if not self.included(event):
+            return
+
+        if event.is_directory:
+            files_in_dir = [event.src_path+"/"+f for f in os.listdir(event.src_path)]
+            if len(files_in_dir) > 0:
+                modified_filename = max(files_in_dir, key=os.path.getmtime)
+            else:
+                return
+            if os.path.isfile(modified_filename) and self.included(event=None, base=modified_filename):
+                logging.debug("Event: modified file : %s" % self.remove_prefix(modified_filename))
+        else:
+            modified_filename = event.src_path
+            logging.debug("Event: modified file : %s" % self.remove_prefix(modified_filename))
+
+
+class LocalWatcher(threading.Thread):
+
+    def __init__(self, local_path, includes, excludes):
+        threading.Thread.__init__(self)
+        self.basepath = local_path
+        self.observer = None
+        self.includes = includes
+        self.excludes = excludes
+
+    def stop(self):
+        self.observer.stop()
+
+    def run(self):
+        event_handler = PydioEventHandler(self.basepath, self.includes, self.excludes)
+
+        logging.info('Scanning for changes since last application launch')
+
+        # previous_snapshot = SqlSnapshot(self.basepath)
+        # snapshot = DirectorySnapshot(self.basepath, recursive=True)
+        # diff = DirectorySnapshotDiff(previous_snapshot, snapshot)
+        # for path in diff.dirs_created:
+        #     event_handler.on_created(DirCreatedEvent(path))
+        # for path in diff.files_created:
+        #     event_handler.on_created(FileCreatedEvent(path))
+        # for path in diff.dirs_moved:
+        #     event_handler.on_moved(DirMovedEvent(path[0], path[1]))
+        # for path in diff.files_moved:
+        #     event_handler.on_moved(FileMovedEvent(path[0], path[1]))
+        # for path in diff.files_deleted:
+        #     event_handler.on_deleted(FileDeletedEvent(path))
+        # for path in diff.dirs_deleted:
+        #     event_handler.on_deleted(DirDeletedEvent(path))
+        logging.info('Starting permanent monitor')
+        self.observer = Observer()
+        self.observer.schedule(event_handler, self.basepath, recursive=True)
+        self.observer.start()
+        while True:
+            time.sleep(1)
+        self.observer.join()
+
+
+if __name__ == "__main__":
+
+    logging.basicConfig(filename=os.path.dirname(os.path.realpath(__file__)) + '/filesystem.log',
+                        level=logging.DEBUG,
+                        format='%(asctime)s - %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    parser = argparse.ArgumentParser('Pydio Framework watcher')
+    parser.add_argument('-p', '--path', help='Node path', default=None)
+    args, _ = parser.parse_known_args()
+
+    watcher = LocalWatcher(args.path, includes=['*'], excludes=['.*'])
+
+    try:
+        watcher.start()
+    except (KeyboardInterrupt, SystemExit):
+        watcher.stop()
+        sys.exit()

core/src/plugins/meta.monitor_fs/framework_watch.py

@@ -0,0 +1,19 @@
+__author__ = 'charles'
+import logging
+import argparse
+import os
+
+if __name__ == "__main__":
+
+    logging.basicConfig(filename= os.path.dirname(os.path.realpath(__file__)) + '/framework.log',
+                        level=logging.DEBUG,
+                        format='%(asctime)s - %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    parser = argparse.ArgumentParser('Pydio Framework watcher')
+    parser.add_argument('-a', '--action', help='Event type', type=unicode, default=None)
+    parser.add_argument('-p', '--path', help='Node path', default=None)
+    args, _ = parser.parse_known_args()
+
+    path = args.path
+    logging.info("Event " + args.action +  " on " + args.path)

core/src/plugins/meta.monitor_fs/manifest.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ajxp_plugin id="meta.monitor_fs" label="CONF_MESSAGE[FileSystem Monitoring]" description="CONF_MESSAGE[If the workspace is meant to be modified directly through the file system, use this plugin to monitor the changes]" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+             xsi:noNamespaceSchemaLocation="file:../core.ajaxplorer/ajxp_registry.xsd">
+    <plugin_info>
+        <core_relation packaged="true" tested_version="5.2.0"/>
+        <plugin_author>Charles du Jeu</plugin_author>
+    </plugin_info>
+
+    <class_definition classname="FSMonitoringManager"
+                      filename="plugins/meta.monitor_fs/class.FSMonitoringManager.php"/>
+
+    <registry_contributions>
+        <hooks>
+            <serverCallback methodName="beforePathChange" hookName="node.before_path_change"/>
+            <serverCallback methodName="beforeCreate" hookName="node.before_create"/>
+            <serverCallback methodName="beforeChange" hookName="node.before_change"/>
+        </hooks>
+    </registry_contributions>
+</ajxp_plugin>