From d2026020d77e62c2af3218f6f0813936122524c3 Mon Sep 17 00:00:00 2001 From: Max De Marzi Date: Tue, 18 Apr 2017 00:53:04 -0500 Subject: [PATCH] cache results --- .../java/com/maxdemarzi/search/Search.java | 87 +++++++++++-------- src/main/java/com/maxdemarzi/tags/Tags.java | 63 +++++++++----- src/main/java/com/maxdemarzi/users/Users.java | 1 + 3 files changed, 94 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/maxdemarzi/search/Search.java b/src/main/java/com/maxdemarzi/search/Search.java index 5e0be20..81e172a 100644 --- a/src/main/java/com/maxdemarzi/search/Search.java +++ b/src/main/java/com/maxdemarzi/search/Search.java @@ -1,5 +1,7 @@ package com.maxdemarzi.search; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.maxdemarzi.Labels; import com.maxdemarzi.RelationshipTypes; import com.maxdemarzi.users.Users; @@ -26,9 +28,8 @@ import java.io.IOException; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Map; +import java.util.*; +import java.util.concurrent.TimeUnit; import static com.maxdemarzi.Properties.*; import static com.maxdemarzi.Time.utc; @@ -45,6 +46,29 @@ public class Search { private static int postLabelId; private static int statusPropertyId; + // Cache + private static LoadingCache> searches = Caffeine.newBuilder() + .maximumSize(50_000) + .expireAfterWrite(1, TimeUnit.DAYS) + .refreshAfterWrite(1, TimeUnit.MINUTES) + .build(Search::performSearch); + + private static ArrayList performSearch(String term) throws SchemaRuleNotFoundException, IndexNotFoundKernelException { + ThreadToStatementContextBridge ctx = dbapi.getDependencyResolver().resolveDependency(ThreadToStatementContextBridge.class); + KernelStatement st = (KernelStatement)ctx.get(); + ReadOperations ops = st.readOperations(); + IndexDescriptor descriptor = ops.indexGetForLabelAndPropertyKey(postLabelId, statusPropertyId); + IndexReader reader = st.getStoreStatement().getIndexReader(descriptor); + + PrimitiveLongIterator hits = reader.containsString(term); + ArrayList results = new ArrayList<>(); + while(hits.hasNext()) { + results.add(hits.next()); + } + return results; + } + + public Search(@Context GraphDatabaseService db) throws NoSuchMethodException { this.dbapi = (GraphDatabaseAPI) db; try (Transaction tx = db.beginTx()) { @@ -54,9 +78,6 @@ public Search(@Context GraphDatabaseService db) throws NoSuchMethodException { statusPropertyId = ops.propertyKeyGetForName(STATUS); tx.success(); } - -// Method method = IndexReader.class.getDeclaredMethod("getIndexSearcher"); -// method.setAccessible(true); } @GET @@ -75,43 +96,39 @@ public Response getSearch(@QueryParam("q") final String q, Long latest = dateTime.toEpochSecond(ZoneOffset.UTC); try (Transaction tx = db.beginTx()) { - Node user = null; - if (username != null) { - user = Users.findUser(username, db); - } + final Node user = Users.findUser(username, db); + ArrayList postIds = searches.get(q); + Queue posts = new PriorityQueue<>(Comparator.comparing(m -> (Long) m.getProperty(TIME), reverseOrder())); - ThreadToStatementContextBridge ctx = dbapi.getDependencyResolver().resolveDependency(ThreadToStatementContextBridge.class); - KernelStatement st = (KernelStatement)ctx.get(); - ReadOperations ops = st.readOperations(); - IndexDescriptor descriptor = ops.indexGetForLabelAndPropertyKey(postLabelId, statusPropertyId); - IndexReader reader = st.getStoreStatement().getIndexReader(descriptor); - PrimitiveLongIterator hits = reader.containsString(q); - while(hits.hasNext()) { - Node post = db.getNodeById(hits.next()); + postIds.forEach(postId -> { + Node post = db.getNodeById(postId); Long time = (Long)post.getProperty("time"); if(time < latest) { - Map properties = post.getAllProperties(); - Node author = getAuthor(post, (Long) properties.get(TIME)); - properties.put(USERNAME, author.getProperty(USERNAME)); - properties.put(NAME, author.getProperty(NAME)); - properties.put(HASH, author.getProperty(HASH)); - properties.put(LIKES, post.getDegree(RelationshipTypes.LIKES)); - properties.put(REPOSTS, post.getDegree() - 1 - post.getDegree(RelationshipTypes.LIKES)); - if (user != null) { - properties.put(LIKED, userLikesPost(user, post)); - properties.put(REPOSTED, userRepostedPost(user, post)); - } - results.add(properties); + posts.add(post); } + }); - } + int count = 0; + while (count < limit && !posts.isEmpty()) { + count++; + Node post = posts.poll(); + Map properties = post.getAllProperties(); + Node author = getAuthor(post, (Long) properties.get(TIME)); + properties.put(USERNAME, author.getProperty(USERNAME)); + properties.put(NAME, author.getProperty(NAME)); + properties.put(HASH, author.getProperty(HASH)); + properties.put(LIKES, post.getDegree(RelationshipTypes.LIKES)); + properties.put(REPOSTS, post.getDegree() - 1 - post.getDegree(RelationshipTypes.LIKES)); + if (user != null) { + properties.put(LIKED, userLikesPost(user, post)); + properties.put(REPOSTED, userRepostedPost(user, post)); + } + results.add(properties); + } } - results.sort(Comparator.comparing(m -> (Long) m.get(TIME), reverseOrder())); - return Response.ok().entity(objectMapper.writeValueAsString( - results.subList(0, Math.min(results.size(), limit)))) - .build(); + return Response.ok().entity(objectMapper.writeValueAsString(results)).build(); } } diff --git a/src/main/java/com/maxdemarzi/tags/Tags.java b/src/main/java/com/maxdemarzi/tags/Tags.java index de13a48..e241fac 100644 --- a/src/main/java/com/maxdemarzi/tags/Tags.java +++ b/src/main/java/com/maxdemarzi/tags/Tags.java @@ -1,5 +1,7 @@ package com.maxdemarzi.tags; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.maxdemarzi.Labels; import com.maxdemarzi.RelationshipTypes; import org.codehaus.jackson.map.ObjectMapper; @@ -14,6 +16,7 @@ import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,6 +32,41 @@ public class Tags { private static final Pattern hashtagPattern = Pattern.compile("#(\\S+)"); private static final ObjectMapper objectMapper = new ObjectMapper(); + private static GraphDatabaseService db; + + public Tags(@Context GraphDatabaseService graphDatabaseService) { + db = graphDatabaseService; + } + + // Cache + private static LoadingCache>> trends = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.DAYS) + .refreshAfterWrite(5, TimeUnit.MINUTES) + .build(Tags::getTrends); + + private static List> getTrends(String key) { + ArrayList> results = new ArrayList<>(); + LocalDateTime dateTime = LocalDateTime.now(utc); + RelationshipType tagged = RelationshipType.withName("TAGGED_ON_" + + dateTime.format(dateFormatter)); + try (Transaction tx = db.beginTx()) { + ResourceIterator tags = db.findNodes(Labels.Tag); + while (tags.hasNext()) { + Node tag = tags.next(); + int taggings = tag.getDegree(tagged, Direction.INCOMING); + if ( taggings > 0) { + HashMap result = new HashMap<>(); + result.put(NAME, tag.getProperty(NAME)); + result.put(COUNT, taggings); + results.add(result); + } + } + tx.success(); + } + + results.sort(Comparator.comparing(m -> (Integer) m.get(COUNT), reverseOrder())); + return results.subList(0, Math.min(results.size(), 10)); + } @GET @Path("/{hashtag}") @@ -112,31 +150,12 @@ public static void createTags(Node post, HashMap input, LocalDa @GET public Response getTrends(@Context GraphDatabaseService db) throws IOException { - ArrayList> results = new ArrayList<>(); - LocalDateTime dateTime = LocalDateTime.now(utc); - RelationshipType tagged = RelationshipType.withName("TAGGED_ON_" + - dateTime.format(dateFormatter)); + List> results; try (Transaction tx = db.beginTx()) { - ResourceIterator tags = db.findNodes(Labels.Tag); - while (tags.hasNext()) { - Node tag = tags.next(); - int taggings = tag.getDegree(tagged, Direction.INCOMING); - if ( taggings > 0) { - HashMap result = new HashMap(); - result.put(NAME, tag.getProperty(NAME)); - result.put(COUNT, taggings); - results.add(result); - } - - } - tx.success(); + results = trends.get("trends"); } - results.sort(Comparator.comparing(m -> (Integer) m.get(COUNT), reverseOrder())); - - return Response.ok().entity(objectMapper.writeValueAsString( - results.subList(0, Math.min(results.size(), 10)))) - .build(); + return Response.ok().entity(objectMapper.writeValueAsString(results)).build(); } } diff --git a/src/main/java/com/maxdemarzi/users/Users.java b/src/main/java/com/maxdemarzi/users/Users.java index 9579397..e5a4331 100644 --- a/src/main/java/com/maxdemarzi/users/Users.java +++ b/src/main/java/com/maxdemarzi/users/Users.java @@ -255,6 +255,7 @@ public Response removeFollows(@PathParam("username") final String username, } public static Node findUser(String username, @Context GraphDatabaseService db) { + if (username == null) { return null; } Node user = db.findNode(Labels.User, USERNAME, username); if (user == null) { throw UserExceptions.userNotFound; } return user;