From a22625caeee3ee93b8093a3182c2b69ec369004a Mon Sep 17 00:00:00 2001 From: Tomas Bjerre Date: Sat, 19 Aug 2023 19:27:40 +0200 Subject: [PATCH] feat: add caching feature in global config (refs #272) --- README.md | 22 ++-- debug.sh | 10 +- run.sh | 8 +- .../plugins/gwt/global/CacheConfig.java | 57 ++++++++ .../plugins/gwt/jobfinder/JobFinder.java | 2 +- .../gwt/jobfinder/JobFinderImpersonater.java | 124 +++++++++++++++++- .../gwt/global/CacheConfig/config.jelly | 27 ++++ .../gwt/global/CacheConfig/help-enabled.html | 5 + .../plugins/gwt/global/Whitelist/config.jelly | 2 +- .../gwt/global/Whitelist/help-enabled.html | 2 +- .../plugins/gwt/jobfinder/JobFinderTest.java | 3 + 11 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/jenkinsci/plugins/gwt/global/CacheConfig.java create mode 100644 src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/config.jelly create mode 100644 src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/help-enabled.html diff --git a/README.md b/README.md index 9b2ac29..f2a8e1f 100644 --- a/README.md +++ b/README.md @@ -65,14 +65,20 @@ The token can be supplied as a: - Request parameter: - `curl -vs http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=abc123 2>&1` + `curl -vs "http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=abc123" 2>&1` - Token header: - `curl -vs -H "token: abc123" http://localhost:8080/jenkins/generic-webhook-trigger/invoke 2>&1` + `curl -vs -H "token: abc123" "http://localhost:8080/jenkins/generic-webhook-trigger/invoke" 2>&1` - It will also detect `X-Gitlab-Token`. - _Authorization_ header of type _Bearer_ : - `curl -vs -H "Authorization: Bearer abc123" http://localhost:8080/jenkins/generic-webhook-trigger/invoke 2>&1` + `curl -vs -H "Authorization: Bearer abc123" "http://localhost:8080/jenkins/generic-webhook-trigger/invoke" 2>&1` + +## Cache jobs + +When plugin is used in large installations it may need some time to retrieve all configured jobs. This can be cached by enabling it in the global configuration. When enabled, the plugin will cache configured jobs for a configured time. The plugin will automatically refresh the cache so that any calls will use the cached value. This means the effect of any changes to any configured job will be delayed. + +The cache will only be used in invocations where a `token` is supplied. ## Trigger exactly one build @@ -106,26 +112,26 @@ If you are fiddling with expressions, you may want to checkout: It's probably easiest to do with `curl`. Given that you have configured a Jenkins job to trigger on Generic Webhook, here are some examples of how to start the jobs. ```bash -curl -vs http://localhost:8080/jenkins/generic-webhook-trigger/invoke 2>&1 +curl -vs "http://localhost:8080/jenkins/generic-webhook-trigger/invoke" 2>&1 ``` This should start your job, if the job has no `token` configured and no security enabled. If you have security enabled you may need to authenticate: ```bash -curl -vs http://theusername:thepasssword@localhost:8080/jenkins/generic-webhook-trigger/invoke 2>&1 +curl -vs "http://theusername:thepasssword@localhost:8080/jenkins/generic-webhook-trigger/invoke" 2>&1 ``` If your job has a `token` you don't need to supply other credentials. You can specify the `token` like this: ```bash -curl -vs http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=TOKEN_HERE 2>&1 +curl -vs "http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=TOKEN_HERE" 2>&1 ``` Please keep in mind, using a token always runs the triggered jobs with SYSTEM privileges. If you want to trigger with `token` and some post content, `curl` can dot that like this. ```bash -curl -v -H "Content-Type: application/json" -X POST -d '{ "app":{ "name":"some value" }}' http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=TOKEN_HERE +curl -v -H "Content-Type: application/json" -X POST -d '{ "app":{ "name":"some value" }}' "http://localhost:8080/jenkins/generic-webhook-trigger/invoke?token=TOKEN_HERE" ``` ## Screenshots @@ -372,7 +378,7 @@ pipeline { It can be triggered with something like: ```bash -curl -X POST -H "Content-Type: application/json" -H "headerWithNumber: nbr123" -H "headerWithString: a b c" -d '{ "before": "1848f12", "after": "5cab1", "ref": "refs/heads/develop" }' -vs http://admin:admin@localhost:8080/jenkins/generic-webhook-trigger/invoke?requestWithNumber=nbr%20123\&requestWithString=a%20string +curl -X POST -H "Content-Type: application/json" -H "headerWithNumber: nbr123" -H "headerWithString: a b c" -d '{ "before": "1848f12", "after": "5cab1", "ref": "refs/heads/develop" }' -vs "http://admin:admin@localhost:8080/jenkins/generic-webhook-trigger/invoke?requestWithNumber=nbr%20123&requestWithString=a%20string" ``` And the job will have this in the log: diff --git a/debug.sh b/debug.sh index 55d465f..057e85b 100755 --- a/debug.sh +++ b/debug.sh @@ -1,4 +1,8 @@ #!/bin/sh -mvn versions:update-properties -mvnDebug -q hpi:run -Djava.util.logging.config.file=logging.properties -Djenkins.version=2.204.1 -Denforcer.skip=true - +./mvnw versions:update-properties +MAVEN_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n" \ + ./mvnw hpi:run \ + -Djava.util.logging.config.file=logging.properties \ + -Djenkins.version=2.361.4 \ + -Denforcer.skip=true \ + -Dhudson.model.ParametersAction.keepUndefinedParameters=true diff --git a/run.sh b/run.sh index b79e106..d1bb256 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,7 @@ #!/bin/sh -mvn versions:update-properties -mvn hpi:run -Djava.util.logging.config.file=logging.properties -Djenkins.version=2.346.3 -Denforcer.skip=true +./mvnw versions:update-properties +./mvnw hpi:run \ + -Djava.util.logging.config.file=logging.properties \ + -Djenkins.version=2.361.4 \ + -Denforcer.skip=true \ + -Dhudson.model.ParametersAction.keepUndefinedParameters=true diff --git a/src/main/java/org/jenkinsci/plugins/gwt/global/CacheConfig.java b/src/main/java/org/jenkinsci/plugins/gwt/global/CacheConfig.java new file mode 100644 index 0000000..12e5ab1 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/gwt/global/CacheConfig.java @@ -0,0 +1,57 @@ +package org.jenkinsci.plugins.gwt.global; + +import hudson.Extension; +import java.io.Serializable; +import java.util.Optional; +import jenkins.model.GlobalConfiguration; +import net.sf.json.JSONObject; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.StaplerRequest; + +@Extension +public class CacheConfig extends GlobalConfiguration implements Serializable { + + private static final long serialVersionUID = -3077539230674127483L; + private static final int DEFAULT_GET_JOBS_CACHE_MINUTES = 15; + + public static CacheConfig get() { + return GlobalConfiguration.all().get(CacheConfig.class); + } + + private boolean cacheGetJobs; + private int cacheGetJobsMinutes; + + public CacheConfig(final boolean cacheGetJobs, final Integer cacheGetJobsMinutes) { + this.cacheGetJobs = cacheGetJobs; + this.cacheGetJobsMinutes = cacheGetJobsMinutes; + } + + public CacheConfig() { + this.load(); + } + + @Override + public boolean configure(final StaplerRequest req, final JSONObject json) throws FormException { + req.bindJSON(this, json); + this.save(); + return true; + } + + @DataBoundSetter + public void setCacheGetJobs(final boolean cacheGetJobs) { + this.cacheGetJobs = cacheGetJobs; + } + + public boolean isCacheGetJobs() { + return this.cacheGetJobs; + } + + @DataBoundSetter + public void setCacheGetJobsMinutes(final int cacheGetJobsMinutes) { + this.cacheGetJobsMinutes = cacheGetJobsMinutes; + } + + public int getCacheGetJobsMinutes() { + return Optional.ofNullable(this.cacheGetJobsMinutes).orElse(DEFAULT_GET_JOBS_CACHE_MINUTES); + } +} diff --git a/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinder.java b/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinder.java index c1a22e1..0c51eac 100644 --- a/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinder.java +++ b/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinder.java @@ -22,7 +22,7 @@ public final class JobFinder { - private static Logger LOG = Logger.getLogger(JobFinder.class.getSimpleName()); + private static Logger LOG = Logger.getLogger(JobFinder.class.getName()); private JobFinder() {} diff --git a/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderImpersonater.java b/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderImpersonater.java index 7b005ad..05a7fbf 100644 --- a/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderImpersonater.java +++ b/src/main/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderImpersonater.java @@ -1,14 +1,136 @@ package org.jenkinsci.plugins.gwt.jobfinder; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import hudson.security.ACL; +import java.time.Duration; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import jenkins.model.Jenkins; import jenkins.model.ParameterizedJobMixIn.ParameterizedJob; import org.acegisecurity.context.SecurityContext; import org.acegisecurity.context.SecurityContextHolder; +import org.jenkinsci.plugins.gwt.global.CacheConfig; public class JobFinderImpersonater { - public List getAllParameterizedJobs(boolean impersonate) { + private static final boolean DO_IMPERSONATE = true; + private static Logger LOGGER = Logger.getLogger(JobFinderImpersonater.class.getName()); + private static final long CACHE_REFRESH_INITIAL_DELAY = 0; + private ScheduledExecutorService scheduledExecutorService = null; + private LoadingCache> loadingCache = null; + private boolean cacheGetJobs = false; + private int cacheGetJobsMinutes = 0; + + public JobFinderImpersonater() {} + + public List getAllParameterizedJobs(final boolean impersonate) { + this.reconfigureCachingIfNecessary(); + final boolean useCache = CacheConfig.get().isCacheGetJobs(); + if (useCache && impersonate) { + try { + LOGGER.log(Level.FINE, "Using the cache"); + return this.getCachedJobs(); + } catch (final ExecutionException e) { + LOGGER.log(Level.SEVERE, "Was unable to getAllParameterizedJobs from cache.", e); + return doGetAllParameterizedJobs(impersonate); + } + } else if (useCache && !impersonate) { + LOGGER.log( + Level.INFO, + "Not using the cache because jobs are not retreieved with impersonation SYSTEM. " + + "SYSTEM is only impersonated when using a token." + + " If SYSTEM is not impersonated, only jobs available for the currently authenticated user is found."); + } + LOGGER.log(Level.FINE, "Not using the cache"); + return doGetAllParameterizedJobs(impersonate); + } + + private List getCachedJobs() throws ExecutionException { + return this.loadingCache.get(DO_IMPERSONATE); + } + + synchronized void reconfigureCachingIfNecessary() { + final boolean configCacheGetJobs = CacheConfig.get().isCacheGetJobs(); + final int configCacheGetJobsMinutes = CacheConfig.get().getCacheGetJobsMinutes(); + final boolean shouldReconfigure = + this.scheduledExecutorService == null + || this.loadingCache == null + || this.cacheGetJobs != configCacheGetJobs + || this.cacheGetJobsMinutes != configCacheGetJobsMinutes; + if (shouldReconfigure) { + LOGGER.log( + Level.INFO, + "Reconfiguring cache, was (enabled: " + + this.cacheGetJobs + + ", minutes: " + + this.cacheGetJobsMinutes + + ") changing to (enabled: " + + configCacheGetJobs + + ", minutes: " + + configCacheGetJobsMinutes + + ")"); + } else { + return; + } + + if (configCacheGetJobs) { + this.stopCaching(); + this.startCaching(); + try { + // Make a call to add the entry to cache + this.getCachedJobs(); + } catch (final ExecutionException e) { + LOGGER.log(Level.SEVERE, "Was unable to trigger cache", e); + } + } else { + this.stopCaching(); + } + this.cacheGetJobs = configCacheGetJobs; + this.cacheGetJobsMinutes = configCacheGetJobsMinutes; + } + + private void startCaching() { + final int cacheMinutes = CacheConfig.get().getCacheGetJobsMinutes(); + final int cacheRefreshDuration = cacheMinutes > 1 ? cacheMinutes - 1 : cacheMinutes; + + this.loadingCache = + CacheBuilder.newBuilder() // + .refreshAfterWrite(Duration.ofMinutes(cacheMinutes)) // + .build( + new CacheLoader>() { + @Override + public List load(final Boolean impersonate) throws Exception { + LOGGER.log(Level.FINE, "Loading the cache with impersonate " + impersonate); + return doGetAllParameterizedJobs(impersonate); + } + }); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + this.scheduledExecutorService.scheduleWithFixedDelay( + () -> { + LOGGER.log(Level.FINE, "Triggering cache refresh"); + this.loadingCache.asMap().keySet().forEach((key) -> this.loadingCache.refresh(key)); + }, + CACHE_REFRESH_INITIAL_DELAY, + cacheRefreshDuration, + TimeUnit.MINUTES); + } + + private void stopCaching() { + if (this.scheduledExecutorService != null) { + this.scheduledExecutorService.shutdown(); + } + if (this.loadingCache != null) { + this.loadingCache.invalidateAll(); + } + } + + private static List doGetAllParameterizedJobs(final boolean impersonate) { SecurityContext orig = null; try { if (impersonate) { diff --git a/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/config.jelly b/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/config.jelly new file mode 100644 index 0000000..dff29e0 --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/config.jelly @@ -0,0 +1,27 @@ + + + + + + + + + If checked, the plugin will cache available configured jobs. So that the plugin does not need to retrieve that list when invoked. It will only cache when the token-parameter is supplied. + + + + + + + Time, in minutes, to keep the jobs in cache before they are refreshed. + + + + + diff --git a/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/help-enabled.html b/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/help-enabled.html new file mode 100644 index 0000000..50192de --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/gwt/global/CacheConfig/help-enabled.html @@ -0,0 +1,5 @@ +
+

+ This feature is also documented here. +

+
diff --git a/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/config.jelly b/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/config.jelly index 11ce92b..bc170df 100644 --- a/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/config.jelly +++ b/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/config.jelly @@ -8,7 +8,7 @@ xmlns:t="/lib/hudson" xmlns:c="/lib/credentials"> - + diff --git a/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/help-enabled.html b/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/help-enabled.html index 88d93ca..50192de 100644 --- a/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/help-enabled.html +++ b/src/main/resources/org/jenkinsci/plugins/gwt/global/Whitelist/help-enabled.html @@ -1,5 +1,5 @@

- See Generic Webhook Trigger Plugin for details on how to configure and use this plugin. + This feature is also documented here.

diff --git a/src/test/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderTest.java b/src/test/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderTest.java index 80590c5..820fc90 100644 --- a/src/test/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderTest.java +++ b/src/test/java/org/jenkinsci/plugins/gwt/jobfinder/JobFinderTest.java @@ -36,6 +36,9 @@ public void before() { this.allParameterizedJobsByImpersonation = new ArrayList<>(); final JobFinderImpersonater jobFinderImpersonater = new JobFinderImpersonater() { + @Override + synchronized void reconfigureCachingIfNecessary() {} + @Override public List getAllParameterizedJobs(final boolean impersonate) { JobFinderTest.this.didImpersonate = impersonate;