Skip to content

Commit

Permalink
improve rudimentary cache for labels and globs; fix log config
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebullient
ebullient committed Feb 6, 2024
1 parent 395db39 commit 8eee4bf
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 137 deletions.
5 changes: 4 additions & 1 deletion src/main/java/org/commonhaus/automation/Main.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.commonhaus.automation;

import java.nio.file.Path;

import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;

Expand All @@ -14,6 +16,7 @@
public class Main {

public static void main(String... args) {
System.out.println("cwd=" + Path.of("").toAbsolutePath());
Quarkus.run(ApplicationRoot.class, args);
}

Expand All @@ -24,7 +27,7 @@ public static class ApplicationRoot implements QuarkusApplication {

@Override
public int run(String... args) throws Exception {
System.out.println("Do startup logic here. dryRun=" + botConfig.isDryRun());
System.out.println("dryRun=" + botConfig.isDryRun());

// Reminder: stop can happen elsewhere with Quarkus.asyncExit()
Quarkus.waitForExit();
Expand Down
19 changes: 2 additions & 17 deletions src/main/java/org/commonhaus/automation/github/LabelChanges.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import jakarta.inject.Inject;

import org.commonhaus.automation.github.QueryHelper.QueryContext;
import org.commonhaus.automation.github.model.ActionType;
import org.commonhaus.automation.github.model.DataDiscussion;
import org.commonhaus.automation.github.model.DataLabel;
import org.commonhaus.automation.github.model.EventPayload;
Expand Down Expand Up @@ -53,13 +52,7 @@ void onDiscussionLabelChangeEvent(GitHubEvent event, GitHub github,
}

Log.debugf("LabelChanges (%s): discussion %s changed label %s", event.getEventAction(), discussion.id, label);

// Don't fetch labels first: only add/remove if it's an item/id we know about
if (initialData.getActionType() == ActionType.labeled) {
queryContext.addCachedLabel(discussion.id, label);
} else {
queryContext.removeCachedLabel(discussion.id, label);
}
queryContext.modifyLabels(discussion.id, label, initialData.getActionType());
}

/**
Expand All @@ -84,14 +77,6 @@ void onRepositoryLabelChange(GitHubEvent event, GitHub github,
String cacheId = labelPayload.getRepository().getNodeId();

Log.debugf("LabelChanges (%s): repository %s changed label %s", event.getEventAction(), cacheId, label);

// Don't fetch labels first: only add/remove if it's an item/id we know about
if (initialData.getActionType() == ActionType.created) {
queryContext.addCachedLabel(cacheId, label);
} else if (initialData.getActionType() == ActionType.edited) {
queryContext.updateCachedLabel(cacheId, label);
} else {
queryContext.removeCachedLabel(cacheId, label);
}
queryContext.modifyLabels(cacheId, label, initialData.getActionType());
}
}
160 changes: 80 additions & 80 deletions src/main/java/org/commonhaus/automation/github/QueryHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;

import org.commonhaus.automation.AppConfig;
import org.commonhaus.automation.github.model.ActionType;
import org.commonhaus.automation.github.model.DataLabel;
import org.kohsuke.github.GHIOException;
import org.kohsuke.github.GitHub;
Expand All @@ -29,10 +31,61 @@

@ApplicationScoped
public class QueryHelper {
static final String LABEL = "labels";
static Cache<String, Object> cache = Caffeine.newBuilder()
.expireAfterWrite(6, TimeUnit.HOURS)
.build();
public enum QueryCache {
LABELS(Caffeine.newBuilder()
.expireAfterWrite(6, TimeUnit.HOURS)
.build()),
GLOB(Caffeine.newBuilder()
.maximumSize(200)
.build());

private final Cache<String, Object> cache;

private QueryCache(Cache<String, Object> cache) {
this.cache = cache;
}

@SuppressWarnings("unchecked")
public <T> T getCachedValue(String key, Class<T> type) {
T result = (T) cache.getIfPresent(key);
if (result != null) {
Log.debugf(":: HIT :: %s/%s ::: ", this.name(), key);
} else {
Log.debugf(":: MISS :: %s/%s ::: ", this.name(), key);
}
return result;
}

/**
* Put a value into the cache
*
* @param key
* @param value to be cached
* @return new value
*/
public <T> T putCachedValue(String key, T value) {
Log.debugf(":: PUT :: %s/%s ::: ", this.name(), key);
cache.put(key, value);
return value;
}

@SuppressWarnings("unchecked")
public <T> T computeIfPresent(String key, BiFunction<String, Object, T> mappingFunction) {
Log.debugf(":: UPDATE :: %s/%s ::: ", this.name(), key);
Object value = cache.asMap().computeIfPresent(key, mappingFunction);
return (T) value;
}

public void invalidate(String key) {
Log.debugf(":: INVALIDATE :: %s/%s ::: ", this.name(), key);
cache.invalidate(key);
}

public void invalidateAll() {
Log.debugf(":: INVALIDATE ALL :: %s ::: ", this.name());
cache.invalidateAll();
}
}

@Inject
AppConfig botConfig;
Expand All @@ -48,23 +101,6 @@ public QueryContext newQueryContext(EventData eventData, GitHub github) {
return newQueryContext(eventData).addExisting(github);
}

public static <T> T getCache(String itemId, String key, Class<T> type) {
String idx = itemId + ":" + key;
T result = (T) cache.getIfPresent(idx);
if (result != null) {
System.out.println(":: HIT :: " + idx + " ::: ");
} else {
System.out.println(":: MISS :: " + idx + " ::: ");
}
return result;
}

public static void putCache(String itemId, String key, Object value) {
String idx = itemId + ":" + key;
System.out.println(":: UPDATE :: " + idx + " ::: ");
cache.put(idx, value);
}

/**
* QueryContext is a helper class to encapsulate the GitHub API and GraphQL
* client.
Expand Down Expand Up @@ -143,7 +179,7 @@ protected GitHub getGitHub() {

@FunctionalInterface
public interface GitHubParameterApiCall<R> {
R apply(GitHub gh) throws GHIOException, IOException;
R apply(GitHub gh, boolean isDryRun) throws GHIOException, IOException;
}

/**
Expand All @@ -159,7 +195,7 @@ public <R> R execGitHubSync(GitHubParameterApiCall<R> ghApiCall) {
return null;
}
try {
return ghApiCall.apply(getGitHub());
return ghApiCall.apply(getGitHub(), botConfig.isDryRun());
} catch (GHIOException e) {
// TODO: Config to handle GHIOException (retry? quit? ensure notification?)
Log.errorf(e, "[%s] Error making GH Request: %s", evt.installationId(), e.toString());
Expand Down Expand Up @@ -235,29 +271,23 @@ public Response execRepoQuerySync(String query, Map<String, Object> variables) {
return execQuerySync(query, variables);
}

public <T> T getCache(String itemId, String key, Class<T> type) {
return QueryHelper.getCache(itemId, key, type);
}

public void putCache(String itemId, String key, Object value) {
QueryHelper.putCache(itemId, key, value);
}

/**
* Get labels for the labelable item
*
* @param cacheId Labelable item node id
* @return collection of labels for the item
*/
public Collection<DataLabel> getCachedLabels(String itemId) {
Collection<DataLabel> labels = getCache(itemId, "labels", Collection.class);
@SuppressWarnings("unchecked")
Set<DataLabel> labels = QueryHelper.QueryCache.LABELS.getCachedValue(itemId, Set.class);

if (labels != null) {
return labels;
}
// fresh fetch graphQL fetch
labels = DataLabel.queryLabels(this, itemId);
if (labels != null) {
putCache(itemId, LABEL, labels);
QueryHelper.QueryCache.LABELS.putCachedValue(itemId, labels);
}
return labels;
}
Expand All @@ -269,63 +299,33 @@ public Collection<DataLabel> getCachedLabels(String itemId) {
* @param label Label to add
* @return updated collection of labels for the item, or null if not cached
*/
public Collection<DataLabel> addCachedLabel(String cacheId, DataLabel label) {
Collection<DataLabel> labels = getCachedLabels(cacheId); // ensure labels are cached (if not already)
if (labels == null) {
return null;
}
labels.add(label);
return labels;
}

/**
* Remove a label from the list of known labels if the associated item exists
*
* @param cacheId Labelable item node id
* @param label Label to remove
* @return updated collection of labels for the item, or null if not cached
*/
public Collection<DataLabel> removeCachedLabel(String cacheId, DataLabel label) {
Collection<DataLabel> labels = getCachedLabels(cacheId); // ensure labels are cached (if not already)
if (labels == null) {
return null;
}
labels.remove(label);
return labels;
}

/**
* Update a label in the cache if the associated item exists
*
* @param cacheId Labelable item node id
* @param label Label to update/replace
* @return updated collection of labels for the item, or null if not cached
*/
public Collection<DataLabel> updateCachedLabel(String cacheId, DataLabel label) {
Collection<DataLabel> labels = getCachedLabels(cacheId); // ensure labels are cached (if not already)
if (labels == null) {
return null;
}
// only the id is equals/hashcode -- so remove and add
labels.remove(label);
labels.add(label);
return labels;
public Collection<DataLabel> modifyLabels(String cacheId, DataLabel label, ActionType action) {
return QueryHelper.QueryCache.LABELS.computeIfPresent(cacheId, (k, v) -> {
@SuppressWarnings("unchecked")
Set<DataLabel> labels = (Set<DataLabel>) v;
if (action == ActionType.deleted || action == ActionType.unlabeled || action == ActionType.edited) {
labels.remove(label);
}
if (action == ActionType.created || action == ActionType.labeled || action == ActionType.edited) {
labels.add(label);
}
return labels;
});
}

/**
* Add a label from the list of known labels if the associated item has been seen/cached
* Replace cached labels with result of modification (post-mutation)
*
* @param cacheId Labelable item node id
* @param labels updated Set of labels (e.g. post modify command)
* @return updated collection of labels for the item, or null if not cached
* @param labels updated Set of labels (e.g. post-mutation)
* @return updated collection of labels
*/
public Collection<DataLabel> modifyLabels(String cacheId, List<DataLabel> labels) {
public Collection<DataLabel> updateLabels(String cacheId, List<DataLabel> labels) {
Set<DataLabel> newLabels = DataLabel.modifyLabels(this, cacheId, labels);
if (newLabels == null) {
return null;
}
QueryHelper.putCache(cacheId, LABEL, newLabels);
return newLabels;
return QueryHelper.QueryCache.LABELS.putCachedValue(cacheId, newLabels);
}
}
}
2 changes: 1 addition & 1 deletion src/main/java/org/commonhaus/automation/github/actions/LabelAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void apply(QueryContext queryContext) {

if (!newLabels.isEmpty()) {
// Modify labels and update cache
queryContext.modifyLabels(nodeId, newLabels);
queryContext.updateLabels(nodeId, newLabels);
}
}
}
5 changes: 2 additions & 3 deletions src/main/java/org/commonhaus/automation/github/rules/MatchFilePath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,9 @@ public boolean matches(QueryContext queryContext) {
}

MatchingEngine compileGlob(String filenamePattern) {
MatchingEngine me = QueryHelper.getCache("GLOB", filenamePattern, MatchingEngine.class);
MatchingEngine me = QueryHelper.QueryCache.GLOB.getCachedValue(filenamePattern, MatchingEngine.class);
if (me == null) {
me = GlobPattern.compile(filenamePattern);
QueryHelper.putCache("GLOB", filenamePattern, me);
me = QueryHelper.QueryCache.GLOB.putCachedValue(filenamePattern, GlobPattern.compile(filenamePattern));
}
return me;
}
Expand Down
10 changes: 5 additions & 5 deletions src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
quarkus.log.category."org.commonhaus".level=DEBUG
quarkus.log.category."io.quarkiverse".level=DEBUG
quarkus.log.category."io.quarkus.cache".level=DEBUG

quarkus.package.vineflower.enabled=true

quarkus.log.file.enable=true
quarkus.log.file.async=true
quarkus.log.file.path=logs/debug.log
quarkus.log.file.level=TRACE
%prod.quarkus.log.file.enable=true
%prod.quarkus.log.file.async=true
%prod.quarkus.log.file.path=logs/bot.log
%prod.quarkus.log.file.level=INFO

quarkus.cache.caffeine."glob-cache".maximum-size=200
Loading

0 comments on commit 8eee4bf

Please sign in to comment.