diff --git a/pom.xml b/pom.xml index 731657f79..7ff542380 100644 --- a/pom.xml +++ b/pom.xml @@ -396,6 +396,12 @@ + + io.methvin + directory-watcher + ${directory-watcher.version} + + org.springframework diff --git a/src/main/java/org/craftercms/engine/service/context/FolderScanningSiteListResolver.java b/src/main/java/org/craftercms/engine/service/context/FolderScanningSiteListResolver.java index 215a09da4..0b0375532 100644 --- a/src/main/java/org/craftercms/engine/service/context/FolderScanningSiteListResolver.java +++ b/src/main/java/org/craftercms/engine/service/context/FolderScanningSiteListResolver.java @@ -144,5 +144,4 @@ protected File getSitesFolder() { return null; } } - } diff --git a/src/main/java/org/craftercms/engine/service/context/SiteContextFactory.java b/src/main/java/org/craftercms/engine/service/context/SiteContextFactory.java index ee2b363a0..6c3433f71 100644 --- a/src/main/java/org/craftercms/engine/service/context/SiteContextFactory.java +++ b/src/main/java/org/craftercms/engine/service/context/SiteContextFactory.java @@ -67,9 +67,12 @@ import org.tuckey.web.filters.urlrewrite.UrlRewriter; import javax.servlet.ServletContext; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URLClassLoader; import java.util.*; import java.util.concurrent.Executor; @@ -451,6 +454,25 @@ public SiteContext createContext(String siteName) { } } + /** + * Resolve root folder path + * @param siteName site name + * @return the root folder absolute path + */ + public String resolveRootFolderPath(String siteName) throws URISyntaxException { + Map macroValues = new HashMap<>(); + macroValues.put(siteNameMacroName, siteName); + + if (publishingTargetResolver instanceof SiteAwarePublishingTargetResolver) { + String target = ((SiteAwarePublishingTargetResolver) publishingTargetResolver).getPublishingTarget(siteName); + macroValues.put(publishingTargetMacroName, target); + } + + String resolvedRootFolderPath = macroResolver.resolveMacros(rootFolderPath, macroValues); + + return new File(new URI(resolvedRootFolderPath)).getAbsolutePath(); + } + protected HierarchicalConfiguration getConfig(SiteContext siteContext, String[] configPaths, ResourceLoader resourceLoader) { String siteName = siteContext.getSiteName(); diff --git a/src/main/java/org/craftercms/engine/service/context/SiteContextManager.java b/src/main/java/org/craftercms/engine/service/context/SiteContextManager.java index cb7dd7cae..c3d8ee1d8 100644 --- a/src/main/java/org/craftercms/engine/service/context/SiteContextManager.java +++ b/src/main/java/org/craftercms/engine/service/context/SiteContextManager.java @@ -16,8 +16,8 @@ package org.craftercms.engine.service.context; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.craftercms.commons.concurrent.locks.KeyBasedLockFactory; import org.craftercms.commons.concurrent.locks.WeakKeyBasedReentrantLockFactory; import org.craftercms.commons.entitlements.exception.EntitlementException; @@ -28,14 +28,17 @@ import org.springframework.beans.factory.DisposableBean; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; +import io.methvin.watcher.DirectoryWatcher; -import java.util.Collection; -import java.util.Iterator; -import java.util.Map; +import java.io.IOException; +import java.nio.file.*; +import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import java.util.stream.Collectors; import static java.lang.String.format; @@ -46,7 +49,7 @@ */ public class SiteContextManager implements ApplicationContextAware, DisposableBean { - private static final Log logger = LogFactory.getLog(SiteContextManager.class); + private static final Logger logger = LoggerFactory.getLogger(SiteContextManager.class); protected ApplicationContext applicationContext; protected KeyBasedLockFactory siteLockFactory; @@ -59,6 +62,41 @@ public class SiteContextManager implements ApplicationContextAware, DisposableBe protected Executor jobThreadPoolExecutor; protected String defaultSiteName; + /** + * Directory watcher registry for each site + */ + protected Map directoryWatcherRegistry; + + /** + * Directory watcher processed event hash + */ + protected Map directoryWatcherLastProcessedHash; + + /** + * Directory watcher counter to count modified events + */ + protected Map directoryWatcherCounter; + + /** + * Directory watcher site rebuild tasks + */ + protected Map directoryWatcherExecutor; + + /** + * Directory watcher watch paths + */ + protected String[] watcherPaths = {}; + + /** + * Directory watcher counter limit to run rebuild + */ + protected int watcherCounterLimit; + + /** + * Directory watcher interval period to run rebuild task check + */ + protected int watcherIntervalPeriod; + /** * Context build retry max count */ @@ -82,6 +120,10 @@ public class SiteContextManager implements ApplicationContextAware, DisposableBe public SiteContextManager() { siteLockFactory = new WeakKeyBasedReentrantLockFactory(); contextRegistry = new ConcurrentHashMap<>(); + directoryWatcherRegistry = new ConcurrentHashMap<>(); + directoryWatcherLastProcessedHash = new HashMap<>(); + directoryWatcherCounter = new ConcurrentHashMap<>(); + directoryWatcherExecutor = new ConcurrentHashMap<>(); } @Override @@ -144,6 +186,21 @@ public void setModePreview(final boolean modePreview) { this.modePreview = modePreview; } + @Required + public void setWatcherPaths(final String[] watcherPaths) { + this.watcherPaths = watcherPaths; + } + + @Required + public void setWatcherCounterLimit(final int watcherCounterLimit) { + this.watcherCounterLimit = watcherCounterLimit; + } + + @Required + public void setWatcherIntervalPeriod(final int watcherIntervalPeriod) { + this.watcherIntervalPeriod = watcherIntervalPeriod; + } + public void destroy() { destroyAllContexts(); } @@ -197,6 +254,139 @@ public void createContexts(boolean concurrent) { logger.info("=================================================="); } + /** + * Register files watcher for preview mode + * Any files from watcherPaths will be watched for CREATE, MODIFY, DELETE actions + * @param siteName site name + */ + protected void registerPreviewWatcher(String siteName) { + try { + String siteRootPath = contextFactory.resolveRootFolderPath(siteName); + List paths = Arrays.stream(watcherPaths) + .map(resource -> Paths.get(siteRootPath + resource)) + .collect(Collectors.toList()); + DirectoryWatcher watcher = DirectoryWatcher.builder() + .paths(paths) + .fileHasher(FileHasher.LAST_MODIFIED_TIME) + .listener(event -> { + switch (event.eventType()) { + case CREATE: + case DELETE: + case MODIFY: { + String hashValue = event.hash() != null ? event.hash().asString() : "none"; + logger.info("File watcher event type: '{}'. File affected: '{}'. Hash value: '{}'", event.eventType(), event.path(), hashValue); + // Only process if the hash is different from the last processed hash value, + // or the event hash is null in the case of DELETE + // This means the modified time has been updated from last processed event or the file is deleted + // This prevents multiple events for batch files change with same modified date such as from a git pull + String lastProcessedHash = directoryWatcherLastProcessedHash.get(siteName); + if (lastProcessedHash == null || event.hash() == null || !lastProcessedHash.equals(hashValue)) { + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); + try { + if (event.hash() != null) { + directoryWatcherLastProcessedHash.put(siteName, hashValue); + } + AtomicInteger counter = directoryWatcherCounter.get(siteName); + if (counter == null) { + counter = new AtomicInteger(0); + } + counter.getAndIncrement(); + directoryWatcherCounter.put(siteName, counter); + } finally { + siteLock.unlock(); + } + } else { + logger.debug("File watch for hash '{}' has already processed. No action required.", hashValue); + } + break; + } + default: + logger.debug("File watcher unhandled event type: '{}'. File affected: '{}'.", event.eventType(), event.path()); + } + }) + .build(); + + // Remove old watcher before register a new one + if (directoryWatcherRegistry.get(siteName) != null) { + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); + try { + DirectoryWatcher oldWatcher = directoryWatcherRegistry.remove(siteName); + oldWatcher.close(); + } finally { + siteLock.unlock(); + } + } + directoryWatcherRegistry.put(siteName, watcher); + watcher.watchAsync(); + } catch (Exception e) { + logger.error("Error while creating watcher for site: '{}'", siteName, e); + } + } + + /** + * Register preview rebuild task. + * When the watcher triggers, set a counter to 1 and sleep for 200 (configurable) milliseconds. + * Wake up and check: + * Is the counter 5 (one second has passed)? Then, trigger a rebuild. + * Has anything else changed (more changes since we slept)? If so, increment the counter and sleep for 200 milliseconds. + * If nothing has changed, then trigger a rebuild. + * @param siteName site name + * @param isFallback is fallback + */ + public void registerPreviewRebuildTask(String siteName, boolean isFallback) { + // Remove old executor then register a new one + if (directoryWatcherExecutor.get(siteName) != null) { + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); + try { + ScheduledExecutorService oldExecutor = directoryWatcherExecutor.remove(siteName); + oldExecutor.shutdown(); + } finally { + siteLock.unlock(); + } + } + + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + AtomicInteger rebuildCounter = new AtomicInteger(0); + AtomicInteger changeCounterFromLastTask = new AtomicInteger(0); + Runnable task = () -> { + try { + logger.debug("Running rebuild task for site '{}'.", siteName); + + // Is the counter reach limit (watcherCounterLimit * watcherIntervalPeriod second has passed)? + if (rebuildCounter.get() >= watcherCounterLimit) { + logger.debug("Counter reached '{}', rebuilding site '{}'", watcherCounterLimit, siteName); + rebuildContext(siteName, isFallback); + rebuildCounter.set(0); + directoryWatcherCounter.put(siteName, new AtomicInteger(0)); + changeCounterFromLastTask.set(0); + } + + // Check the directory watcher for new change + AtomicInteger nextChangeCounter = directoryWatcherCounter.get(siteName); + // Has anything else changed (more changes since we slept)? + logger.debug("Previous changed count for site '{}' is '{}'. Current is '{}'", siteName, changeCounterFromLastTask, nextChangeCounter); + if (nextChangeCounter != null && nextChangeCounter.get() > changeCounterFromLastTask.get()) { + logger.debug("Site '{}' has changed since last check, updating the change counter to '{}'.", siteName, nextChangeCounter); + changeCounterFromLastTask.set(nextChangeCounter.get()); + rebuildCounter.getAndIncrement(); + } else if (nextChangeCounter != null && nextChangeCounter.get() > 0) { // there were some modified but nothing new from last check + logger.debug("There were some change but nothing new from last check, rebuilding site '{}'.", siteName); + rebuildContext(siteName, isFallback); + rebuildCounter.set(0); + changeCounterFromLastTask.set(0); + directoryWatcherCounter.put(siteName, new AtomicInteger(0)); + } + } catch (Exception e) { + logger.error("Exception while perform rebuild check for site '{}'", siteName, e); + } + }; + directoryWatcherExecutor.put(siteName, executor); + executor.scheduleAtFixedRate(task, 0, watcherIntervalPeriod, TimeUnit.MILLISECONDS); + } + public void syncContexts() { logger.debug("Syncing the site contexts ..."); @@ -208,7 +398,7 @@ public void syncContexts() { try { destroyContext(siteName); } catch (Exception e) { - logger.error("Error destroying site context for site '" + siteName + "'", e); + logger.error("Error destroying site context for site '{}'", siteName, e); } } }); @@ -218,7 +408,7 @@ public void syncContexts() { try { getContext(siteName, false); } catch (Exception e) { - logger.error("Error creating site context for site '" + siteName + "'", e); + logger.error("Error creating site context for site '{}'", siteName, e); } }); } @@ -237,21 +427,29 @@ public void destroyAllContexts() { String siteName = siteContext.getSiteName(); logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); - Lock lock = siteLockFactory.getLock(siteName); - lock.lock(); + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); try { + if (directoryWatcherRegistry.get(siteName) != null) { + DirectoryWatcher watcher = directoryWatcherRegistry.remove(siteName); + watcher.close(); + } + if (directoryWatcherExecutor.get(siteName) != null) { + ScheduledExecutorService executor = directoryWatcherExecutor.remove(siteName); + executor.shutdown(); + } destroyContext(siteContext); } catch (Exception e) { - logger.error("Error destroying site context for site '" + siteName + "'", e); + logger.error("Error destroying site context for site '{}'", siteName, e); } finally { - lock.unlock(); + siteLock.unlock(); } logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); iter.remove(); @@ -278,27 +476,33 @@ public SiteContext getContext(String siteName, boolean fallback) { return null; } - Lock lock = siteLockFactory.getLock(siteName); - lock.lock(); + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); try { // Double check locking, in case the context has been created already by another thread siteContext = contextRegistry.get(siteName); if (siteContext == null) { logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); siteContext = createContext(siteName, fallback); + if (modePreview) { + // files watch register + registerPreviewWatcher(siteName); + registerPreviewRebuildTask(siteName, fallback); + } + logger.info("=================================================="); logger.info(""); logger.info("=================================================="); } } finally { - lock.unlock(); + siteLock.unlock(); } } else if (!siteContext.isValid()) { - logger.error("Site context " + siteContext + " is not valid anymore"); + logger.error("Site context '{}' is not valid anymore", siteContext); destroyContext(siteName); @@ -393,17 +597,31 @@ public void startDestroyContext(String siteName) { */ protected void destroyContext(String siteName) { SiteContext siteContext; - Lock lock = siteLockFactory.getLock(siteName); - lock.lock(); + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); try { + if (directoryWatcherRegistry.get(siteName) != null) { + try { + DirectoryWatcher watcher = directoryWatcherRegistry.remove(siteName); + watcher.close(); + } catch (IOException e) { + logger.warn("Error while removing directory watcher register for site '{}'", siteName, e); + } + } + + if (directoryWatcherExecutor.get(siteName) != null) { + ScheduledExecutorService executor = directoryWatcherExecutor.remove(siteName); + executor.shutdown(); + } + siteContext = contextRegistry.remove(siteName); } finally { - lock.unlock(); + siteLock.unlock(); } if (siteContext != null) { logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); try { @@ -413,7 +631,7 @@ protected void destroyContext(String siteName) { } logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); } } @@ -428,7 +646,7 @@ protected void destroyContexts(Collection siteNames) { try { destroyContext(siteName); } catch (Exception e) { - logger.error("Error destroying site context for site '" + siteName + "'", e); + logger.error("Error destroying site context for site '{}'", siteName, e); } } } @@ -452,17 +670,17 @@ protected SiteContext createContext(String siteName, boolean fallback) { contextRegistry.put(siteName, siteContext); - logger.info("Site context created: " + siteContext); + logger.info("Site context created: '{}'", siteContext); return siteContext; } protected SiteContext rebuildContext(String siteName, boolean fallback) { - Lock lock = siteLockFactory.getLock(siteName); - lock.lock(); + Lock siteLock = siteLockFactory.getLock(siteName); + siteLock.lock(); try { logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); SiteContext oldSiteContext = contextRegistry.get(siteName); @@ -471,19 +689,19 @@ protected SiteContext rebuildContext(String siteName, boolean fallback) { oldSiteContext.destroy(); logger.info("=================================================="); - logger.info(""); + logger.info("", siteName); logger.info("=================================================="); return newContext; } finally { - lock.unlock(); + siteLock.unlock(); } } protected void destroyContext(SiteContext siteContext) { siteContext.destroy(); - logger.info("Site context destroyed: " + siteContext); + logger.info("Site context destroyed: '{}'", siteContext); } protected boolean validateSiteCreationEntitlement() { diff --git a/src/main/resources/crafter/engine/mode/multi-tenant/mapped/services-context.xml b/src/main/resources/crafter/engine/mode/multi-tenant/mapped/services-context.xml index 0e992d961..ed10912e1 100644 --- a/src/main/resources/crafter/engine/mode/multi-tenant/mapped/services-context.xml +++ b/src/main/resources/crafter/engine/mode/multi-tenant/mapped/services-context.xml @@ -71,6 +71,9 @@ + + + diff --git a/src/main/resources/crafter/engine/mode/multi-tenant/simple/services-context.xml b/src/main/resources/crafter/engine/mode/multi-tenant/simple/services-context.xml index aa480fc39..41bb848e7 100644 --- a/src/main/resources/crafter/engine/mode/multi-tenant/simple/services-context.xml +++ b/src/main/resources/crafter/engine/mode/multi-tenant/simple/services-context.xml @@ -70,6 +70,9 @@ + + + diff --git a/src/main/resources/crafter/engine/mode/preview/services-context.xml b/src/main/resources/crafter/engine/mode/preview/services-context.xml index a4e0d24ae..904997e85 100644 --- a/src/main/resources/crafter/engine/mode/preview/services-context.xml +++ b/src/main/resources/crafter/engine/mode/preview/services-context.xml @@ -60,6 +60,9 @@ + + + diff --git a/src/main/resources/crafter/engine/mode/serverless/s3/services-context.xml b/src/main/resources/crafter/engine/mode/serverless/s3/services-context.xml index 1619c233e..03580dac8 100644 --- a/src/main/resources/crafter/engine/mode/serverless/s3/services-context.xml +++ b/src/main/resources/crafter/engine/mode/serverless/s3/services-context.xml @@ -59,6 +59,9 @@ + + + diff --git a/src/main/resources/crafter/engine/server-config.properties b/src/main/resources/crafter/engine/server-config.properties index b5cf26d6a..c71630876 100644 --- a/src/main/resources/crafter/engine/server-config.properties +++ b/src/main/resources/crafter/engine/server-config.properties @@ -461,3 +461,12 @@ crafter.engine.freemarker.statics.enable=false # The path pattern used to load plugin configuration crafter.engine.plugins.config.pattern=/config/plugins/${pluginId}/config.xml + +# Engine file change watcher paths +crafter.engine.watcher.paths=\ + /scripts,\ + /config/engine +# Engine file change watcher counter limit to rebuild +crafter.engine.watcher.counter.limit=5 +# Engine file change watcher thread interval in milliseconds +crafter.engine.watcher.interval.period=200 diff --git a/src/main/resources/crafter/engine/services/main-services-context.xml b/src/main/resources/crafter/engine/services/main-services-context.xml index 598f9bcf9..84c813664 100644 --- a/src/main/resources/crafter/engine/services/main-services-context.xml +++ b/src/main/resources/crafter/engine/services/main-services-context.xml @@ -544,6 +544,9 @@ + + +