Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

initial commit

  • Loading branch information...
commit 0444f9db275d2e81872d6a3f4f83ff47142b6852 0 parents
@jexp authored
8 .gitignore
@@ -0,0 +1,8 @@
+*.db
+target
+*.iml
+*.ipr
+*.iws
+.idea
+.DS_Store
+
43 pom.xml
@@ -0,0 +1,43 @@
+<project>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.neo4j</groupId>
+ <artifactId>twitter-graph</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <name>Twitter-Graph</name>
+ <description>
+ </description>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.neo4j</groupId>
+ <artifactId>neo4j</artifactId>
+ <version>1.8.M06</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.data</groupId>
+ <artifactId>spring-data-neo4j</artifactId>
+ <version>2.1.0.RC2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <version>3.1.0.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.social</groupId>
+ <artifactId>spring-social-twitter</artifactId>
+ <version>1.0.2.RELEASE</version>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>spring</id>
+ <url>http://repo.springsource.org/libs-milestones</url>
+ </repository>
+ </repositories>
+</project>
57 src/main/java/org/neo4j/twitter_graph/TwitterGraph.java
@@ -0,0 +1,57 @@
+package org.neo4j.twitter_graph;
+
+import org.neo4j.twitter_graph.domain.Tweet;
+import org.neo4j.twitter_graph.services.TwitterService;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author mh
+ * @since 25.07.12
+ */
+public class TwitterGraph {
+ public static void main(String[] args) throws InterruptedException {
+ if (args.length < 2) {
+ System.out.println("Usage: TwitterGraph embedded|server #tag");
+ return;
+ }
+ final ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:TwitterGraph-"+args[0]+".xml");
+ addShutdownHook(ctx);
+ final TwitterService service = ctx.getBean(TwitterService.class);
+ Long lastTweetId=args.length==3 ? Long.parseLong(args[2]) : null;
+ while (true) {
+ final String search = args[1];
+ final List<Tweet> tweets = service.importTweets(search,lastTweetId);
+ if (!tweets.isEmpty()) {
+ lastTweetId = maxTweetId(tweets);
+ service.connectFollowers();
+ }
+ Thread.sleep(TimeUnit.MINUTES.toMillis(5));
+ }
+ }
+
+ private static Long maxTweetId(List<Tweet> tweets) {
+ final Tweet maxTweet = Collections.max(tweets, new TweetComparator());
+ return maxTweet!=null ? maxTweet.getTweetId() : null;
+ }
+
+ private static void addShutdownHook(final ClassPathXmlApplicationContext ctx) {
+ Runtime.getRuntime().addShutdownHook(new Thread(){
+ @Override
+ public void run() {
+ ctx.close();
+ }
+ });
+ }
+
+ private static class TweetComparator implements Comparator<Tweet> {
+ public int compare(Tweet o1, Tweet o2) {
+ return o1.getTweetId().compareTo(o2.getTweetId());
+ }
+ }
+}
17 src/main/java/org/neo4j/twitter_graph/domain/Follows.java
@@ -0,0 +1,17 @@
+package org.neo4j.twitter_graph.domain;
+
+import org.springframework.data.neo4j.annotation.EndNode;
+import org.springframework.data.neo4j.annotation.GraphId;
+import org.springframework.data.neo4j.annotation.RelationshipEntity;
+import org.springframework.data.neo4j.annotation.StartNode;
+
+/**
+ * @author mh
+ * @since 26.07.12
+ */
+@RelationshipEntity
+public class Follows {
+ @GraphId Long id;
+ @StartNode User follower;
+ @EndNode User user;
+}
33 src/main/java/org/neo4j/twitter_graph/domain/Tag.java
@@ -0,0 +1,33 @@
+package org.neo4j.twitter_graph.domain;
+
+import org.neo4j.graphdb.Direction;
+import org.springframework.data.neo4j.annotation.*;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+@NodeEntity
+public class Tag {
+ @GraphId Long id;
+ @Indexed(unique=true) String tag;
+
+ public Tag() {
+ }
+
+ public Tag(String tag) {
+ this.tag = tag;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ @Override
+ public String toString() {
+ return "#"+ tag;
+ }
+}
76 src/main/java/org/neo4j/twitter_graph/domain/Tweet.java
@@ -0,0 +1,76 @@
+package org.neo4j.twitter_graph.domain;
+
+import org.springframework.data.neo4j.annotation.*;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+@NodeEntity
+public class Tweet {
+ @GraphId Long id;
+
+ @Indexed(unique=true) Long tweetId;
+
+ String text;
+
+ @Fetch User sender;
+ @Fetch @RelatedTo(type="TAG") Collection<Tag> tags=new HashSet<Tag>();
+ @Fetch @RelatedTo(type="MENTION") private Set<User> mentions=new HashSet<User>();
+ @Fetch @RelatedTo(type="SOURCE") private Tweet source;
+
+ public Tweet() {
+ }
+
+ public Tweet(long tweetId, User sender, String text) {
+ this.tweetId = tweetId;
+ this.sender = sender;
+ this.text = text;
+ }
+
+ public void addMention(User mention) {
+ this.mentions.add(mention);
+ }
+ public Long getId() {
+ return id;
+ }
+
+ public Long getTweetId() {
+ return tweetId;
+ }
+
+ public User getSender() {
+ return sender;
+ }
+
+ @Override
+ public String toString() {
+ return "Tweet " + tweetId +
+ ": " + text +
+ " by " + sender;
+ }
+
+ public Set<User> getMentions() {
+ return mentions;
+ }
+
+ public Collection<Tag> getTags() {
+ return tags;
+ }
+
+ public void addTag(Tag tag) {
+ tags.add(tag);
+ }
+
+ public void setSource(Tweet source) {
+ this.source = source;
+ }
+
+ public Tweet getSource() {
+ return source;
+ }
+}
37 src/main/java/org/neo4j/twitter_graph/domain/User.java
@@ -0,0 +1,37 @@
+package org.neo4j.twitter_graph.domain;
+
+import org.springframework.data.neo4j.annotation.*;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+@NodeEntity
+public class User {
+ @GraphId Long id;
+
+ @Indexed(unique=true) String user;
+
+ public User() {
+ }
+
+ public User(String user) {
+ this.user = user;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ @Override
+ public String toString() {
+ return "@"+ user;
+ }
+}
11 src/main/java/org/neo4j/twitter_graph/repositories/TagRepository.java
@@ -0,0 +1,11 @@
+package org.neo4j.twitter_graph.repositories;
+
+import org.neo4j.twitter_graph.domain.Tag;
+import org.springframework.data.neo4j.repository.GraphRepository;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+public interface TagRepository extends GraphRepository<Tag> {
+}
15 src/main/java/org/neo4j/twitter_graph/repositories/TweetRepository.java
@@ -0,0 +1,15 @@
+package org.neo4j.twitter_graph.repositories;
+
+import org.neo4j.twitter_graph.domain.Tweet;
+import org.springframework.data.neo4j.repository.GraphRepository;
+
+import java.util.Collection;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+public interface TweetRepository extends GraphRepository<Tweet> {
+ Tweet findByTweetId(Long id);
+ Collection<Tweet> findByTagsTag(String tag);
+}
21 src/main/java/org/neo4j/twitter_graph/repositories/UserRepository.java
@@ -0,0 +1,21 @@
+package org.neo4j.twitter_graph.repositories;
+
+import org.neo4j.twitter_graph.domain.User;
+import org.springframework.data.neo4j.annotation.Query;
+import org.springframework.data.neo4j.repository.GraphRepository;
+import org.springframework.data.neo4j.repository.RelationshipOperationsRepository;
+
+import java.util.List;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+public interface UserRepository extends GraphRepository<User>, RelationshipOperationsRepository<User> {
+
+ @Query("START me=node:User(userName={0})" +
+ "MATCH me-[:POSTED]->tweet-[:MENTIONS]->user" +
+ "WHERE user.country = ”SE” and not me-[:FOLLOWS]->user" +
+ "RETURN user")
+ List<User> suggestFriends(User user);
+}
129 src/main/java/org/neo4j/twitter_graph/services/TwitterService.java
@@ -0,0 +1,129 @@
+package org.neo4j.twitter_graph.services;
+
+import org.neo4j.twitter_graph.domain.Follows;
+import org.neo4j.twitter_graph.domain.Tweet;
+import org.neo4j.twitter_graph.domain.Tag;
+import org.neo4j.twitter_graph.domain.User;
+import org.neo4j.twitter_graph.repositories.TagRepository;
+import org.neo4j.twitter_graph.repositories.TweetRepository;
+import org.neo4j.twitter_graph.repositories.UserRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.social.twitter.api.*;
+import org.springframework.social.twitter.api.impl.TwitterTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+@Service
+public class TwitterService {
+ private static final Pattern MENTION = Pattern.compile("@(\\p{Alnum}{3,})");
+ private static final Pattern TAG = Pattern.compile("#(\\p{Alnum}{3,})");
+ @Autowired
+ UserRepository userRepository;
+ @Autowired
+ TagRepository tagRepository;
+ @Autowired
+ TweetRepository tweetRepository;
+
+ @Transactional
+ public List<Tweet> importTweets(String search) {
+ return importTweets(search,null);
+ }
+
+ @Transactional
+ public List<Tweet> importTweets(String search, Long lastTweetId) {
+ System.out.println("Importing for " +search+ ", max tweet id: "+lastTweetId);
+
+ final SearchOperations searchOperations = new TwitterTemplate().searchOperations();
+
+ final SearchResults results = lastTweetId==null ? searchOperations.search(search,1,200) : searchOperations.search(search,1,200,lastTweetId,Long.MAX_VALUE);
+
+ final List<Tweet> result = new ArrayList<Tweet>();
+ for (org.springframework.social.twitter.api.Tweet tweet : results.getTweets()) {
+ result.add(importTweet(tweet));
+ }
+ return result;
+ }
+
+ @Transactional
+ protected Tweet importTweet(org.springframework.social.twitter.api.Tweet source) {
+ final String userName = source.getFromUser();
+ User user = userRepository.save(new User(userName));
+ final String text = source.getText();
+ final Tweet tweet = new Tweet(source.getId(), user, text);
+ System.out.println("Imported " + tweet);
+ addMentions(tweet, text);
+ addTags(tweet, text);
+ addOriginalTweet(tweet, source.getInReplyToStatusId());
+ return tweetRepository.save(tweet);
+ }
+
+ @Transactional
+ public void connectFollowers() {
+ final FriendOperations friendOperations = new TwitterTemplate().friendOperations();
+ Map<String,User> users=new HashMap<String, User>();
+ for (User user : userRepository.findAll()) {
+ users.put(user.getUser(),user);
+ }
+ for (Map.Entry<String, User> entry : users.entrySet()) {
+ addFriends(friendOperations, users, entry);
+ }
+ }
+
+ private void addFriends(FriendOperations friendOperations, Map<String, User> users, Map.Entry<String, User> entry) throws InterruptedException {
+ try {
+ final String name = entry.getKey();
+ final User user = entry.getValue();
+ for (TwitterProfile profile : friendOperations.getFriends(name)) {
+ final User friend = users.get(profile.getScreenName());
+ if (friend == null) continue;
+ System.out.println(name + " FOLLOWS " + friend.getUser());
+ userRepository.createRelationshipBetween(user, friend, Follows.class, "FOLLOWS");
+ }
+ Thread.sleep(TimeUnit.SECONDS.toMillis(2));
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ Thread.sleep(TimeUnit.SECONDS.toMillis(10));
+ }
+ }
+
+ private void addMentions(Tweet tweet, String text) {
+ for (String mention : extractMentions(text)) {
+ tweet.addMention(userRepository.save(new User(mention)));
+ }
+ }
+
+ private void addTags(Tweet tweet, String text) {
+ for (String tag : extractTokens(text, TAG)) {
+ tweet.addTag(tagRepository.save(new Tag(tag)));
+ }
+ }
+
+ private void addOriginalTweet(Tweet tweet, final Long replyId) {
+ if (replyId == null) return;
+ final Tweet source = tweetRepository.findByTweetId(replyId);
+ if (source == null) return;
+ tweet.setSource(source);
+ }
+
+ public Set<String> extractMentions(String text) {
+ return extractTokens(text, MENTION);
+ }
+
+ public Set<String> extractTokens(String text, Pattern p) {
+ final Matcher matcher = p.matcher(text);
+ Set<String> result=new LinkedHashSet<String>();
+ while (matcher.find()) {
+ result.add(matcher.group(1));
+ }
+ return result;
+ }
+}
23 src/main/resources/TwitterGraph-embedded.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:neo4j="http://www.springframework.org/schema/data/neo4j"
+ xmlns:tx="http://www.springframework.org/schema/tx"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context.xsd
+ http://www.springframework.org/schema/data/neo4j
+ http://www.springframework.org/schema/data/neo4j/spring-neo4j-2.0.xsd
+ http://www.springframework.org/schema/tx
+ http://www.springframework.org/schema/tx/spring-tx-2.5.xsd">
+
+ <context:component-scan base-package="org.neo4j.twitter_graph.services"/>
+ <context:annotation-config/>
+
+ <neo4j:repositories base-package="org.neo4j.twitter_graph.repositories"/>
+ <tx:annotation-driven mode="proxy"/>
+
+ <neo4j:config storeDirectory="twitter.db"/>
+</beans>
29 src/main/resources/TwitterGraph-server.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:neo4j="http://www.springframework.org/schema/data/neo4j"
+ xmlns:tx="http://www.springframework.org/schema/tx"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context.xsd
+ http://www.springframework.org/schema/data/neo4j
+ http://www.springframework.org/schema/data/neo4j/spring-neo4j-2.0.xsd
+ http://www.springframework.org/schema/tx
+ http://www.springframework.org/schema/tx/spring-tx-2.5.xsd">
+
+ <context:component-scan base-package="org.neo4j.twitter_graph.services"/>
+ <context:annotation-config/>
+
+ <neo4j:repositories base-package="org.neo4j.twitter_graph.repositories"/>
+ <tx:annotation-driven mode="proxy"/>
+
+ <bean id="graphDatabaseService"
+ class="org.springframework.data.neo4j.rest.SpringRestGraphDatabase">
+ <constructor-arg value="http://localhost:7473/db/data" />
+ </bean>
+
+ <neo4j:config graphDatabaseService="graphDatabaseService"/>
+
+</beans>
81 src/test/java/org/neo4j/twitter_graph/services/TwitterServiceTest.java
@@ -0,0 +1,81 @@
+package org.neo4j.twitter_graph.services;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.neo4j.helpers.collection.IteratorUtil;
+import org.neo4j.twitter_graph.domain.Tweet;
+import org.neo4j.twitter_graph.domain.User;
+import org.neo4j.twitter_graph.repositories.TweetRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.Collection;
+
+import static org.junit.Assert.*;
+import static org.junit.internal.matchers.IsCollectionContaining.hasItems;
+import static org.neo4j.helpers.collection.IteratorUtil.first;
+
+/**
+ * @author mh
+ * @since 24.07.12
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration
+public class TwitterServiceTest {
+ @Autowired
+ TwitterService twitterService;
+ @Autowired
+ TweetRepository tweetRepository;
+ @Test
+ @Transactional
+ public void testImportSimpleTweet() throws Exception {
+ final org.springframework.social.twitter.api.Tweet source = new org.springframework.social.twitter.api.Tweet(123L, "Text", null, "springsource", null, null, 234L, null, null);
+ final Tweet tweet = twitterService.importTweet(source);
+ assertNotNull(tweet.getId());
+ assertEquals((Long)123L,tweet.getTweetId());
+ final User sender = tweet.getSender();
+ assertNotNull(sender.getId());
+ assertEquals("springsource", sender.getUser());
+ }
+ @Test
+ @Transactional
+ public void testImportTweetWithMentions() throws Exception {
+ final org.springframework.social.twitter.api.Tweet source = new org.springframework.social.twitter.api.Tweet(123L, "Text @mesirii", null, "springsource", null, null, 234L, null, null);
+ final Tweet tweet = twitterService.importTweet(source);
+ assertEquals("mesirii", first(tweet.getMentions()).getUser());
+ }
+ @Test
+ @Transactional
+ public void testImportTweetWithTags() throws Exception {
+ final org.springframework.social.twitter.api.Tweet source = new org.springframework.social.twitter.api.Tweet(123L, "Text #neo4j", null, "springsource", null, null, 234L, null, null);
+ final Tweet tweet = twitterService.importTweet(source);
+ assertEquals("neo4j", first(tweet.getTags()).getTag());
+ }
+ @Test
+ @Transactional
+ public void testFindTweetsByTag() throws Exception {
+ final org.springframework.social.twitter.api.Tweet source = new org.springframework.social.twitter.api.Tweet(123L, "Text #neo4j", null, "springsource", null, null, 234L, null, null);
+ twitterService.importTweet(source);
+ final Collection<Tweet> tweets = tweetRepository.findByTagsTag("neo4j");
+ final Tweet tweet = first(tweets);
+ assertEquals("neo4j", first(tweet.getTags()).getTag());
+ assertEquals((Long)123L, tweet.getTweetId());
+ }
+
+ @Test
+ @Transactional
+ public void testImportTweets() throws Exception {
+ Collection<Tweet> tweets =twitterService.importTweets("#neo4j");
+ assertEquals("neo4j", first(first(tweets).getTags()).getTag());
+ }
+
+ @Test
+ public void testExtractMentions() throws Exception {
+ assertThat(twitterService.extractMentions("test @mesir11 test"), hasItems("mesir11"));
+ assertThat(twitterService.extractMentions("test @mesir11"), hasItems("mesir11"));
+ assertThat(twitterService.extractMentions("@mesir11 test"), hasItems("mesir11"));
+ assertThat(twitterService.extractMentions("@mesir11 test @SpringSource"), hasItems("mesir11","SpringSource"));
+ }
+}
23 src/test/resources/org/neo4j/twitter_graph/services/TwitterServiceTest-context.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:neo4j="http://www.springframework.org/schema/data/neo4j"
+ xmlns:tx="http://www.springframework.org/schema/tx"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context.xsd
+ http://www.springframework.org/schema/data/neo4j
+ http://www.springframework.org/schema/data/neo4j/spring-neo4j-2.0.xsd
+ http://www.springframework.org/schema/tx
+ http://www.springframework.org/schema/tx/spring-tx-2.5.xsd">
+
+ <context:component-scan base-package="org.neo4j.twitter_graph.services"/>
+ <context:annotation-config/>
+
+ <neo4j:repositories base-package="org.neo4j.twitter_graph.repositories"/>
+ <tx:annotation-driven mode="proxy"/>
+
+ <neo4j:config storeDirectory="target/twitter.db"/>
+</beans>
Please sign in to comment.
Something went wrong with that request. Please try again.