Update DirPluginScanner to rescan directory when necessary, add resca…

…n interval to set minimum time between rescans.

core/src/java/com/dtolabs/rundeck/core/plugins/DirPluginScanner.java

@@ -23,13 +23,15 @@
 */
 package com.dtolabs.rundeck.core.plugins;
 
+import com.dtolabs.rundeck.core.utils.StringArrayUtil;
 import com.dtolabs.rundeck.core.utils.cache.FileCache;
 import org.apache.log4j.Logger;
 
 import java.io.File;
 import java.io.FileFilter;
-import java.io.FilenameFilter;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
 
 /**
  * DirPluginScanner will scan all files in a directory matching a filter for valid plugins.
@@ -41,10 +43,14 @@ abstract class DirPluginScanner implements PluginScanner {
     final File extdir;
 
     final FileCache<ProviderLoader> filecache;
+    long lastScanAllCheckTime = -1;
+    HashSet<String> scanned = new HashSet<String>();
+    long scaninterval ;
 
-    protected DirPluginScanner(final File extdir, final FileCache<ProviderLoader> filecache) {
+    protected DirPluginScanner(final File extdir, final FileCache<ProviderLoader> filecache, final int rescanInterval) {
         this.extdir = extdir;
         this.filecache = filecache;
+        this.scaninterval = rescanInterval;
     }
 
     /**
@@ -57,25 +63,122 @@ protected DirPluginScanner(final File extdir, final FileCache<ProviderLoader> fi
      */
     public abstract FileFilter getFileFilter();
 
-    public final File scanForFile(final ProviderIdent ident) {
+    /**
+     * scan for matching file for the provider def
+     */
+    public final File scanForFile(final ProviderIdent ident) throws PluginScannerException {
         debug("scanForFile: " + ident);
         if (!extdir.exists() || !extdir.isDirectory()) {
             return null;
         }
         final File[] files = extdir.listFiles(getFileFilter());
+        if (shouldScanAll(files)) {
+            return scanAll(ident, files);
+        } else {
+            return scanFor(ident, files);
+        }
+    }
+
+    /**
+     * Return true if the entry has expired
+     */
+    public boolean isExpired(final ProviderIdent ident, final File file) {
+        return !file.exists() || !scanned.contains(memoFile(file));
+    }
+
+    /**
+     * Return true if any file has been added/removed/modified, and the last full scan has not happened within a certain
+     * interval
+     */
+    public boolean shouldRescan() {
+        return shouldScanAll(extdir.listFiles(getFileFilter()));
+    }
+
+    /**
+     * Return true if any file has been added/removed/modified, and the last full scan has not happened within a certain
+     * interval
+     */
+    private boolean shouldScanAll(final File[] files) {
+        if (lastScanAllCheckTime > 0 && lastScanAllCheckTime + scaninterval > System.currentTimeMillis()) {
+            //wait until scaninterval has passed to scanall again
+            log.debug("shouldScanAll: false, interval");
+            return false;
+        }
+        if (scanned.size() != files.length) {
+            log.debug("shouldScanAll: yes, count: " + scanned.size() + " vs " + files.length);
+            scanned.clear();
+            return true;
+        }else{
+            log.debug("(shouldScanAll: ...: " + scanned.size() + " vs " + files.length);
+        }
+        for (final File file : files) {
+            final String s = memoFile(file);
+            if (!scanned.contains(s)) {
+                log.debug("shouldScanAll: yes, file: " + s);
+                scanned.clear();
+                return true;
+            }
+        }
+        log.debug("shouldScanAll: false, no change");
+        lastScanAllCheckTime = System.currentTimeMillis();
+        return false;
+    }
+
+    private String memoFile(final File file) {
+        return file.getName() + ":" + file.lastModified() + ":" + file.length();
+    }
+
+    /**
+     * Return the first valid file found
+     */
+    private File scanFor(final ProviderIdent ident, final File[] files) throws PluginScannerException {
         for (final File file : files) {
             if (isValidPluginFile(file)) {
-                final ProviderLoader fileProviderLoader = filecache.get(file, this);
-                final boolean loaderFor = fileProviderLoader.isLoaderFor(ident);
-                debug("filecache result: " + fileProviderLoader + ", loaderForIdent: " + loaderFor);
-                if (null != fileProviderLoader && loaderFor) {
+                if (test(ident, file)) {
                     return file;
                 }
             }
         }
         return null;
     }
 
+    /**
+     * Test if a loader for this file matches the provider
+     */
+    private boolean test(final ProviderIdent ident, final File file) {
+        final ProviderLoader fileProviderLoader = filecache.get(file, this);
+        final boolean loaderFor = fileProviderLoader.isLoaderFor(ident);
+        debug("filecache result: " + fileProviderLoader + ", loaderForIdent: " + loaderFor);
+        return null != fileProviderLoader && loaderFor;
+    }
+
+    /**
+     * Rescan all files in the directory
+     */
+    private File scanAll(final ProviderIdent ident, final File[] files) throws PluginScannerException {
+        final List<File> candidates = new ArrayList<File>();
+        scanned.clear();
+        for (final File file : files) {
+            if (isValidPluginFile(file)) {
+                scanned.add(memoFile(file));
+                if (test(ident, file)) {
+                    candidates.add(file);
+                }
+            }
+        }
+        if (candidates.size() > 1) {
+            scanned.clear();
+            throw new PluginScannerException(
+                "More than one plugin file matched: " + StringArrayUtil.asString(candidates.toArray(), ","),
+                ident.getService(), ident.getProviderName());
+        }
+        lastScanAllCheckTime = System.currentTimeMillis();
+        if (candidates.size() > 0) {
+            return candidates.get(0);
+        }
+        return null;
+    }
+
     private void debug(final String s) {
         if (log.isDebugEnabled()) {
             log.debug(s);

core/src/java/com/dtolabs/rundeck/core/plugins/FilePluginCache.java

@@ -23,6 +23,7 @@
 */
 package com.dtolabs.rundeck.core.plugins;
 
+import com.dtolabs.rundeck.core.execution.service.ProviderLoaderException;
 import com.dtolabs.rundeck.core.utils.PairImpl;
 import com.dtolabs.rundeck.core.utils.cache.FileCache;
 import org.apache.log4j.Logger;
@@ -31,8 +32,8 @@
 import java.util.*;
 
 /**
- * FilePluginCache uses a filecache and a set of {@link PluginScanner}s to cache and create {@link ProviderLoader} instances
- * associated with files.
+ * FilePluginCache uses a filecache and a set of {@link PluginScanner}s to cache and create {@link ProviderLoader}
+ * instances associated with files.
  * <p/>
  * The instances are returned for {@link ProviderIdent} instances.
  *
@@ -96,23 +97,33 @@ private void remove(final ProviderIdent ident) {
      *
      * @return loader for the provider
      */
-    public synchronized ProviderLoader getLoaderForIdent(final ProviderIdent ident) {
+    public synchronized ProviderLoader getLoaderForIdent(final ProviderIdent ident) throws ProviderLoaderException {
         final cacheItem cacheItem = cache.get(ident);
         if (null == cacheItem) {
-            log.debug("getLoaderForIdent!: " + ident);
+            log.debug("getLoaderForIdent! " + ident);
             return rescanForItem(ident);
         }
-        log.debug("getLoaderForIdent: " + ident);
+
         final File file = cacheItem.getFirst();
-        if (!file.exists() || null == expire.get(ident) || file.lastModified() > expire.get(ident)) {
+        if (cacheItem.getSecond().isExpired(ident, file) || shouldRescan()) {
             remove(ident);
-            log.debug("getLoaderForIdent(reload): " + ident + ": " + file);
+            log.debug("getLoaderForIdent(expired): " + ident);
             return rescanForItem(ident);
         } else {
+            log.debug("getLoaderForIdent: " + ident);
             return loadFileProvider(cacheItem);
         }
     }
 
+    private boolean shouldRescan() {
+        for (final PluginScanner scanner : scanners) {
+            if(scanner.shouldRescan()){
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * return the loader stored in filecache for the file and scanner
      */
@@ -126,18 +137,33 @@ private ProviderLoader loadFileProvider(final cacheItem cached) {
     /**
      * Rescan for the ident and cache and return the loader
      */
-    private synchronized ProviderLoader rescanForItem(final ProviderIdent ident) {
+    private synchronized ProviderLoader rescanForItem(final ProviderIdent ident) throws ProviderLoaderException {
         log.debug("rescanForItem: " + ident);
+        File candidate = null;
+        PluginScanner cscanner = null;
         for (final PluginScanner scanner : scanners) {
             final File file = scanner.scanForFile(ident);
             if (null != file) {
-                log.debug("file scanned:" + file);
-                final cacheItem cacheItem = new cacheItem(file, scanner);
-                cache.put(ident, cacheItem);
-                expire.put(ident, file.lastModified());
-                return loadFileProvider(cacheItem);
+                log.debug("saw file: " + file);
+                if (null != candidate) {
+                    throw new ProviderLoaderException(
+                        "More than one plugin file matched: " + file + ", and " + candidate,
+                        ident.getService(), ident.getProviderName()
+                    );
+                }
+                candidate = file;
+                cscanner = scanner;
+            } else {
+                log.debug("scanner no result: " + scanner);
             }
         }
+        if (null != candidate) {
+            log.debug("file scanned:" + candidate);
+            final cacheItem cacheItem = new cacheItem(candidate, cscanner);
+            cache.put(ident, cacheItem);
+            expire.put(ident, candidate.lastModified());
+            return loadFileProvider(cacheItem);
+        }
         return null;
     }
 }

core/src/java/com/dtolabs/rundeck/core/plugins/JarPluginScanner.java

@@ -27,9 +27,6 @@
 import org.apache.log4j.Logger;
 
 import java.io.*;
-import java.util.jar.Attributes;
-import java.util.jar.JarInputStream;
-import java.util.jar.Manifest;
 
 /**
  * JarPluginScanner scans for java Jar plugins in the extensions dir.
@@ -44,8 +41,8 @@ public boolean accept(final File file) {
         }
     };
 
-    JarPluginScanner(final File extdir, final FileCache<ProviderLoader> filecache) {
-        super(extdir, filecache);
+    JarPluginScanner(final File extdir, final FileCache<ProviderLoader> filecache, final int rescanInterval) {
+        super(extdir, filecache, rescanInterval);
     }
 
     public boolean isValidPluginFile(final File file) {

core/src/java/com/dtolabs/rundeck/core/plugins/PluginCache.java

@@ -23,6 +23,8 @@
 */
 package com.dtolabs.rundeck.core.plugins;
 
+import com.dtolabs.rundeck.core.execution.service.ProviderLoaderException;
+
 /**
  * PluginCache can use PluginScanners and find ProviderLoaders for ProviderIdents.
  *
@@ -41,5 +43,5 @@ public interface PluginCache {
      *
      * @return loader for the provider
      */
-    ProviderLoader getLoaderForIdent(ProviderIdent ident);
+    ProviderLoader getLoaderForIdent(ProviderIdent ident) throws ProviderLoaderException;
 }

core/src/java/com/dtolabs/rundeck/core/plugins/PluginManagerService.java

@@ -53,8 +53,9 @@ private PluginManagerService(final File extdir, final File cachedir) {
         this.cachedir = cachedir;
         final FileCache<ProviderLoader> filecache = new FileCache<ProviderLoader>();
         cache = new FilePluginCache(filecache);
-        cache.addScanner(new JarPluginScanner(extdir, filecache));
-        cache.addScanner(new ScriptPluginScanner(extdir, cachedir, filecache));
+        final int rescanInterval = 5000;//TODO: use framework property to set interval
+        cache.addScanner(new JarPluginScanner(extdir, filecache, rescanInterval));
+        cache.addScanner(new ScriptPluginScanner(extdir, cachedir, filecache, rescanInterval));
         log.debug("Create PluginManagerService");
     }
 
@@ -84,11 +85,11 @@ private synchronized static PluginManagerService getInstanceForExtDir(final File
     }
 
 
-    public <T> T loadProvider(final PluggableService<T> service, final String providerName) throws ProviderLoaderException {
+    public synchronized <T> T loadProvider(final PluggableService<T> service, final String providerName) throws ProviderLoaderException {
         final ProviderIdent ident = new ProviderIdent(service.getName(), providerName);
         final ProviderLoader loaderForIdent = cache.getLoaderForIdent(ident);
         if (null == loaderForIdent) {
-            throw new MissingProviderException("Provider was not found", service.getName(), providerName);
+            throw new MissingProviderException("No matching plugin found", service.getName(), providerName);
         }
         final T load = loaderForIdent.load(service, providerName);
         if (null != load) {

core/src/java/com/dtolabs/rundeck/core/plugins/PluginScanner.java

@@ -42,5 +42,15 @@ interface PluginScanner extends FileCache.ItemCreator<ProviderLoader> {
     /**
      * Return a file plugin that can supply the given provider ident
      */
-    public File scanForFile(ProviderIdent ident);
+    public File scanForFile(ProviderIdent ident) throws PluginScannerException;
+
+    /**
+     * Return true if the ident and file pair is no longer valid
+     */
+    public boolean isExpired(final ProviderIdent ident, final File file) ;
+
+    /**
+     * Return true if the scanner determines need to rescan
+     */
+    public boolean shouldRescan();
 }

core/src/java/com/dtolabs/rundeck/core/plugins/PluginScannerException.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2011 DTO Solutions, Inc. (http://dtosolutions.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+* PluginScannerException.java
+* 
+* User: Greg Schueler <a href="mailto:greg@dtosolutions.com">greg@dtosolutions.com</a>
+* Created: 4/14/11 3:26 PM
+* 
+*/
+package com.dtolabs.rundeck.core.plugins;
+
+import com.dtolabs.rundeck.core.execution.service.ProviderLoaderException;
+
+import java.util.*;
+
+/**
+ * PluginScannerException is ...
+ *
+ * @author Greg Schueler <a href="mailto:greg@dtosolutions.com">greg@dtosolutions.com</a>
+ */
+class PluginScannerException extends ProviderLoaderException {
+    public PluginScannerException(String serviceName, String providerName) {
+        super(serviceName, providerName);
+    }
+
+    public PluginScannerException(String msg, String serviceName, String providerName) {
+        super(msg, serviceName, providerName);
+    }
+
+    public PluginScannerException(Exception cause, String serviceName, String providerName) {
+        super(cause, serviceName, providerName);
+    }
+
+    public PluginScannerException(String msg, Exception cause, String serviceName, String providerName) {
+        super(msg, cause, serviceName, providerName);
+    }
+}