From a301664f700be5ec02828d2948075210ef612666 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Mon, 24 Jul 2017 02:13:21 +0200 Subject: [PATCH 01/11] fixed #138 --- src/main/resources/application.yml | 8 ++++---- src/main/resources/logback.xml | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 47516017..39a11429 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -33,14 +33,14 @@ spring: datasource: driverClassName: org.postgresql.Driver #initialSize: 5 - #max-total: 70 - #maxActive: 60 + max-total: 70 + maxActive: 60 #maxIdle: 5 #minIdle: 2 platform: POSTGRESQL #removeAbandoned: true - #tomcat: - #max-active: 200 + tomcat: + max-active: 60 #max-wait: 100000 #test-on-borrow: true url: ${DATABASE_URL} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index ad747094..a23b0f24 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,7 +1,6 @@ - - + From f366b3a87b0bc94c1a01b584f7edc9c06ef1adf6 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Mon, 24 Jul 2017 09:38:12 +0200 Subject: [PATCH 02/11] working on #141 #142 #143 --- .../scheduled/mq/endoint/UpdateTweets.java | 12 +++ .../mq/endoint/UpdateUserProfiles.java | 12 +++ .../UpdateUserProfilesFromMentions.java | 12 +++ .../mq/endoint/impl/StartTaskImpl.java | 6 +- .../mq/endoint/impl/UpdateTweetsImpl.java | 69 ++++++++++++++ .../UpdateUserProfilesFromMentionsImpl.java | 85 +++++++++++++++++ .../endoint/impl/UpdateUserProfilesImpl.java | 91 +++++++++++++++++++ src/main/resources/integration.xml | 39 ++++++-- 8 files changed, 317 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java new file mode 100644 index 00000000..e96a1769 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; + +import java.util.List; + +public interface UpdateTweets { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java new file mode 100644 index 00000000..9b254cd9 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; + +import java.util.List; + +public interface UpdateUserProfiles { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java new file mode 100644 index 00000000..cd2fc87b --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; + +import java.util.List; + +public interface UpdateUserProfilesFromMentions { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index 1603d389..b5b99946 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -92,21 +92,21 @@ public void fetchTweetsFromTwitterSearch() { @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; - messageSender(taskType); + sendAndReceive(taskType); } @Override @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; - messageSender(taskType); + sendAndReceive(taskType); } @Override @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) public void updateUserProfilesFromMentions() { TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; - messageSender(taskType); + sendAndReceive(taskType); } private void sendAndReceive(TaskType taskType){ diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java new file mode 100644 index 00000000..adf3f0b2 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -0,0 +1,69 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.twitter.api.Tweet; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.oodm.service.TweetService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateTweets; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateTweets") +public class UpdateTweetsImpl implements UpdateTweets { + + private final TwitterProperties twitterProperties; + + private final TweetService tweetService; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + public UpdateTweetsImpl(TwitterProperties twitterProperties, TweetService tweetService, TwitterApiService twitterApiService, TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.twitterProperties = twitterProperties; + this.tweetService = tweetService; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + CountedEntities countedEntities = countedEntitiesService.countAll(); + List tweets = new ArrayList<>(); + TaskMessage msgIn = message.getPayload(); + long taskId = msgIn.getTaskId(); + Task task = taskService.findById(taskId); + task = taskService.start(task,countedEntities); + List worklistTwitterIds = new ArrayList<>(); + boolean hasNext=true; + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + while(hasNext) { + Page tweetTwitterIds = tweetService.findAllTwitterIds(pageRequest); + hasNext = tweetTwitterIds.hasNext(); + worklistTwitterIds.addAll(tweetTwitterIds.getContent()); + } + for(Long tweetTwitterId : worklistTwitterIds){ + Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); + TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); + tweets.add(result); + } + return tweets; + } +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java new file mode 100644 index 00000000..3f9ad923 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java @@ -0,0 +1,85 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.twitter.api.TwitterProfile; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; +import org.woehlke.twitterwall.oodm.entities.Mention; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.MentionService; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateUserProfilesFromMentions; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateUserProfilesFromMentions") +public class UpdateUserProfilesFromMentionsImpl implements UpdateUserProfilesFromMentions { + + private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; + + private final TwitterProperties twitterProperties; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final MentionService mentionService; + + private final CountedEntitiesService countedEntitiesService; + + public UpdateUserProfilesFromMentionsImpl(TwitterwallSchedulerProperties twitterwallSchedulerProperties, TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, MentionService mentionService, CountedEntitiesService countedEntitiesService) { + this.twitterwallSchedulerProperties = twitterwallSchedulerProperties; + this.twitterProperties = twitterProperties; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.mentionService = mentionService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + CountedEntities countedEntities = countedEntitiesService.countAll(); + List userProfileList = new ArrayList<>(); + TaskMessage msgIn = message.getPayload(); + long id = msgIn.getTaskId(); + Task task = taskService.findById(id); + task = taskService.start(task,countedEntities); + List screenNames = new ArrayList<>(); + int allLoop = 0; + int loopId = 0; + boolean hasNext=true; + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + while (hasNext) { + Page allPersMentions = mentionService.getAll(pageRequest); + hasNext = allPersMentions.hasNext(); + long number = allPersMentions.getTotalElements(); + for (Mention onePersMentions : allPersMentions) { + if (!onePersMentions.hasPersistentUser()) { + String screenName = onePersMentions.getScreenName(); + screenNames.add(screenName); + } + } + pageRequest = pageRequest.next(); + } + for(String screenName:screenNames){ + TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName); + if(userProfile!=null){ + TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); + userProfileList.add(userMsg); + } + } + return userProfileList; + } +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java new file mode 100644 index 00000000..454699a4 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -0,0 +1,91 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.RateLimitExceededException; +import org.springframework.social.twitter.api.TwitterProfile; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.oodm.service.UserService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateUserProfiles; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateUserProfiles") +public class UpdateUserProfilesImpl implements UpdateUserProfiles { + + private static final Logger log = LoggerFactory.getLogger(UpdateUserProfilesImpl.class); + + private final TwitterProperties twitterProperties; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final UserService userService; + + private final CountedEntitiesService countedEntitiesService; + + @Autowired + public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, UserService userService, CountedEntitiesService countedEntitiesService) { + this.twitterProperties = twitterProperties; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.userService = userService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + String msg = "mqUpdateUserProfiles.splitMessage: "; + CountedEntities countedEntities = countedEntitiesService.countAll(); + List userProfileList = new ArrayList<>(); + TaskMessage msgIn = message.getPayload(); + long id = msgIn.getTaskId(); + Task task = taskService.findById(id); + task = taskService.start(task,countedEntities); + int allLoop = 0; + int loopId = 0; + boolean hasNext=true; + List worklistProfileTwitterIds = new ArrayList<>(); + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + while (hasNext) { + Page userProfileTwitterIds = userService.getAllTwitterIds(pageRequest); + hasNext = userProfileTwitterIds.hasNext(); + worklistProfileTwitterIds.addAll(userProfileTwitterIds.getContent()); + } + long number = worklistProfileTwitterIds.size(); + for(Long userProfileTwitterId:worklistProfileTwitterIds){ + allLoop++; + loopId++; + String counter = " ( " + loopId + " from " + number + " ) [" + allLoop + "] "; + log.debug(msg + counter); + TwitterProfile userProfile = null; + try { + userProfile = twitterApiService.getUserProfileForTwitterId(userProfileTwitterId); + } catch (RateLimitExceededException e) { + log.error(msg + counter+ "twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") ",e); + } + if(userProfile != null){ + TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); + userProfileList.add(userMsg); + } + } + return userProfileList; + } +} diff --git a/src/main/resources/integration.xml b/src/main/resources/integration.xml index 70ad9b2e..2690dd33 100644 --- a/src/main/resources/integration.xml +++ b/src/main/resources/integration.xml @@ -92,20 +92,47 @@ - + + + + + - + + + + + - + + + + + From d6af8875669006b1b33f7f0b2c4d34e7278c6d01 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Mon, 24 Jul 2017 10:00:20 +0200 Subject: [PATCH 03/11] working on #141 #142 #143 --- .../frontend/controller/TaskController.java | 32 ++++++----------- .../mq/endoint/impl/StartTaskImpl.java | 35 +++++++++++++++---- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java index 8e51b286..8b10ff78 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java @@ -11,7 +11,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -19,9 +18,7 @@ import org.woehlke.twitterwall.oodm.entities.TaskHistory; import org.woehlke.twitterwall.oodm.service.TaskHistoryService; import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.service.facade.FetchTweetsFromTwitterSearch; -import org.woehlke.twitterwall.scheduled.service.facade.FetchUsersFromDefinedUserList; -import org.woehlke.twitterwall.scheduled.service.facade.UpdateTweets; +import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; /** * Created by tw on 11.07.17. @@ -125,54 +122,45 @@ public String updateUserProfilesRequest(Model model) { @Async protected void fetchTweetsFromTwitterSearch() { - fetchTweetsFromTwitterSearch.fetchTweetsFromTwitterSearch();; + startTask.fetchTweetsFromTwitterSearch(); } @Async protected void updateTweets() { - updateTweets.updateTweets(); + startTask.updateTweets(); } @Async protected void fetchUsersFromDefinedUserList(){ - fetchUsersFromDefinedUserList.fetchUsersFromDefinedUserList(); + startTask.fetchUsersFromDefinedUserList(); } @Async protected void updateUserProfilesFromMentions(){ - + startTask.updateUserProfilesFromMentions(); } @Async protected void updateUserProfiles() { - + startTask.updateUserProfiles(); } private final TaskService taskService; private final TaskHistoryService taskHistoryService; - private final FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch; - - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; - - private final UpdateTweets updateTweets; - private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; + private final StartTask startTask; + @Autowired - public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, UpdateTweets updateTweets, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper, StartTask startTask) { this.taskService = taskService; this.taskHistoryService = taskHistoryService; - this.fetchTweetsFromTwitterSearch = fetchTweetsFromTwitterSearch; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.updateTweets = updateTweets; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; + this.startTask = startTask; } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index b5b99946..14474cec 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -85,31 +85,54 @@ private void messageSender(final TaskType taskType) { @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) public void fetchTweetsFromTwitterSearch() { TaskType taskType = TaskType.FETCH_TWEETS_FROM_TWITTER_SEARCH; - sendAndReceive(taskType); + sendAndReceiveTweet(taskType); } @Override @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; - sendAndReceive(taskType); + sendAndReceiveTweet(taskType); } @Override @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; - sendAndReceive(taskType); + sendAndReceiveUser(taskType); } @Override @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) public void updateUserProfilesFromMentions() { TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; - sendAndReceive(taskType); + sendAndReceiveUser(taskType); } - private void sendAndReceive(TaskType taskType){ + private void sendAndReceiveTweet(TaskType taskType){ + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create("Start via MQ", taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); + Object o = returnedMessage.getPayload(); + countedEntities = countedEntitiesService.countAll(); + if( o instanceof TweetResultList){ + TweetResultList msg = (TweetResultList) o; + long taskId = msg.getTaskId(); + task = taskService.findById(taskId); + taskService.done(task,countedEntities); + } else { + taskService.error(task,"Wrong type of returnedMessage",countedEntities); + } + } + + private void sendAndReceiveUser(TaskType taskType){ CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); @@ -136,7 +159,7 @@ private void sendAndReceive(TaskType taskType){ @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) public void fetchUsersFromDefinedUserList() { TaskType taskType = TaskType.FETCH_USERS_FROM_DEFINED_USER_LIST; - sendAndReceive(taskType); + sendAndReceiveUser(taskType); } @Override From 14f41a3409185b3a64a87dd3f710fc855b2cb246 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Mon, 24 Jul 2017 10:11:28 +0200 Subject: [PATCH 04/11] working on #141 #142 #143 --- .../woehlke/twitterwall/conf/TwitterProperties.java | 11 +++++++++++ .../scheduled/mq/endoint/impl/UpdateTweetsImpl.java | 5 +++++ .../impl/UpdateUserProfilesFromMentionsImpl.java | 5 +++++ .../mq/endoint/impl/UpdateUserProfilesImpl.java | 5 +++++ src/main/resources/application.yml | 1 + 5 files changed, 27 insertions(+) diff --git a/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java b/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java index 06eb24db..d2227f27 100644 --- a/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java +++ b/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java @@ -27,6 +27,9 @@ public class TwitterProperties { @NotNull private Integer pageSize; + @NotNull + private Integer millisToWaitBetweenTwoApiCalls; + @NotNull private String searchQuery; @@ -77,4 +80,12 @@ public String getSearchQuery() { public void setSearchQuery(String searchQuery) { this.searchQuery = searchQuery; } + + public Integer getMillisToWaitBetweenTwoApiCalls() { + return millisToWaitBetweenTwoApiCalls; + } + + public void setMillisToWaitBetweenTwoApiCalls(Integer millisToWaitBetweenTwoApiCalls) { + this.millisToWaitBetweenTwoApiCalls = millisToWaitBetweenTwoApiCalls; + } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java index adf3f0b2..2a9bcdd3 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -59,10 +59,15 @@ public List splitMessage(Message message) { hasNext = tweetTwitterIds.hasNext(); worklistTwitterIds.addAll(tweetTwitterIds.getContent()); } + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(Long tweetTwitterId : worklistTwitterIds){ Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); tweets.add(result); + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } } return tweets; } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java index 3f9ad923..8f961c59 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java @@ -73,12 +73,17 @@ public List splitMessage(Message message) { } pageRequest = pageRequest.next(); } + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(String screenName:screenNames){ TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName); if(userProfile!=null){ TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); userProfileList.add(userMsg); } + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } } return userProfileList; } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index 454699a4..4ac2b988 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -70,6 +70,7 @@ public List splitMessage(Message message) { worklistProfileTwitterIds.addAll(userProfileTwitterIds.getContent()); } long number = worklistProfileTwitterIds.size(); + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(Long userProfileTwitterId:worklistProfileTwitterIds){ allLoop++; loopId++; @@ -85,6 +86,10 @@ public List splitMessage(Message message) { TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); userProfileList.add(userMsg); } + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } } return userProfileList; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 39a11429..416e3024 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -66,6 +66,7 @@ twitter: # consumerSecret: ${TWITTER_CONSUMER_SECRET} pageSize: ${TWITTER_PAGE_SIZE} searchQuery: ${TWITTER_SEARCH_TERM} + millisToWaitBetweenTwoApiCalls: 5000 twitterwall: backend: twitter: From c4963b28a57c5728c32e483e30a5d3e7f76638e9 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Mon, 24 Jul 2017 22:50:27 +0200 Subject: [PATCH 05/11] working on #141 #142 #143 --- .../woehlke/twitterwall/ScheduledTasks.java | 246 +++--------------- .../controller/ApplicationController.java | 12 +- .../controller/CountedEntitiesController.java | 1 - .../controller/HashTagController.java | 57 +--- .../frontend/controller/MediaController.java | 7 +- .../controller/MentionController.java | 7 +- .../frontend/controller/PagesController.java | 9 - .../frontend/controller/TaskController.java | 9 - .../controller/TaskHistoryController.java | 7 +- .../frontend/controller/TestController.java | 11 +- .../controller/TickerSymbolController.java | 7 +- .../frontend/controller/TweetController.java | 1 - .../controller/UrlCacheController.java | 4 - .../frontend/controller/UrlController.java | 7 +- .../frontend/controller/UserController.java | 6 +- .../controller/common/ControllerHelper.java | 2 - .../common/HashTagsOverviewHelper.java | 8 + .../common/impl/ControllerHelperImpl.java | 13 - .../impl/HashTagsOverviewHelperImpl.java | 90 +++++++ .../mq/endoint/impl/StartTaskImpl.java | 199 ++++++-------- .../mq/endoint/impl/UpdateTweetsImpl.java | 3 +- .../endoint/impl/UpdateUserProfilesImpl.java | 3 +- 22 files changed, 237 insertions(+), 472 deletions(-) create mode 100644 src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java create mode 100644 src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java diff --git a/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java b/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java index 911762b4..bb8fb3b5 100644 --- a/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java +++ b/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java @@ -8,10 +8,7 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; -import org.woehlke.twitterwall.scheduled.service.facade.*; - -import java.text.SimpleDateFormat; -import java.util.Date; +import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; /** * Created by tw on 10.06.17. @@ -20,45 +17,50 @@ @Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = false) public class ScheduledTasks { - private static final Logger log = LoggerFactory.getLogger(ScheduledTasks.class); - - private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - - private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; - - private void logEnv(String msg){ - log.info("===================================================================="); - log.info(msg); - log.info("===================================================================="); - log.info("twitterwall.scheduler.allowUpdateTweets = "+ twitterwallSchedulerProperties.getAllowUpdateTweets()); - log.info("twitterwall.scheduler.allowUpdateUserProfiles = "+ twitterwallSchedulerProperties.getAllowUpdateUserProfiles()); - log.info("twitterwall.scheduler.allowUpdateUserProfilesFromMention = "+ twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention()); - log.info("twitterwall.scheduler.allowFetchTweetsFromTwitterSearch = "+ twitterwallSchedulerProperties.getAllowFetchTweetsFromTwitterSearch()); - log.info("twitterwall.scheduler.skipFortesting = "+ twitterwallSchedulerProperties.getSkipFortesting()); - log.info("twitterwall.scheduler.herokuDbRowsLimit = "+ twitterwallSchedulerProperties.getHerokuDbRowsLimit()); - log.info("twitterwall.scheduler.fetchUserList.name = "+ twitterwallSchedulerProperties.getFetchUserList().getName()); - log.info("twitterwall.scheduler.fetchUserList.allow = "+ twitterwallSchedulerProperties.getFetchUserList().getAllow()); - log.info("===================================================================="); + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) + public void fetchTweetsFromTwitterSearch() { + String msg = "fetch Tweets From TwitterSearch "; + if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.fetchTweetsFromTwitterSearch(); + } } - private final FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch; - - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) + public void updateTweets() { + String msg = "update Tweets "; + if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ + startTask.updateTweets(); + } + } - private final UpdateTweets updateTweets; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) + public void updateUserProfiles() { + String msg = "update User Profiles "; + if(twitterwallSchedulerProperties.getAllowUpdateUserProfiles() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.updateUserProfiles(); + } + } - private final UpdateUserProfiles updateUserProfiles; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) + public void updateUserProfilesFromMentions(){ + String msg = "update User Profiles From Mentions"; + if(twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.updateUserProfilesFromMentions(); + } + } - private final UpdateUserProfilesFromMentions updateUserProfilesFromMentions; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) + public void fetchUsersFromDefinedUserList(){ + String msg = "fetch Users from Defined User List "; + if(twitterwallSchedulerProperties.getFetchUserList().getAllow() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.fetchUsersFromDefinedUserList(); + } + } @Autowired - public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerProperties, FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, UpdateTweets updateTweets, UpdateUserProfiles updateUserProfiles, UpdateUserProfilesFromMentions updateUserProfilesFromMentions) { + public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerProperties, StartTask startTask) { this.twitterwallSchedulerProperties = twitterwallSchedulerProperties; - this.fetchTweetsFromTwitterSearch = fetchTweetsFromTwitterSearch; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.updateTweets = updateTweets; - this.updateUserProfiles = updateUserProfiles; - this.updateUserProfilesFromMentions = updateUserProfilesFromMentions; + this.startTask = startTask; } private final static long EINE_MINUTE = 60 * 1000; @@ -79,177 +81,9 @@ public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerPropert private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST = ZWOELF_STUNDEN; - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) - public void fetchTweetsFromTwitterSearch() { - String msg = "fetch Tweets From TwitterSearch "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ - log.info("START "+msg+": The time is now {}", dateFormat.format(new Date())); - try { - this.fetchTweetsFromTwitterSearch.fetchTweetsFromTwitterSearch(); - log.info("DONE "+msg+" (OK)"+": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMesg = e.getMessage(); - e.printStackTrace(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg +" : " + eMesg); - log.error("NOT DONE "+msg+" (NOK)");; - } catch (Exception e) { - msg += " (Exception) "; - String eMesg = e.getMessage(); - e.printStackTrace(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg +" : " + eMesg); - log.error("NOT DONE "+msg+" (NOK)");; - } - } - logEnv(msg); - } - - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) - public void updateTweets() { - String msg = "update Tweets "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ - log.info("START "+msg + ": The time is now {}", dateFormat.format(new Date())); - try { - this.updateTweets.updateTweets(); - log.info("DONE "+msg+" (OK)"+": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - log.warn(msg + e.getMessage()); - Throwable t = e.getCause(); - e.printStackTrace(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn("NOT DONE "+msg+" (NOK) {}", dateFormat.format(new Date()));; - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn("NOT DONE "+msg+" (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } - - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) - public void updateUserProfiles() { - String msg = "update User Profiles "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateUserProfiles() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - //this.updateUserProfiles.updateUserProfiles(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private static final Logger log = LoggerFactory.getLogger(ScheduledTasks.class); - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) - public void updateUserProfilesFromMentions(){ - String msg = "update User Profiles From Mentions"; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - //this.updateUserProfilesFromMentions.updateUserProfilesFromMentions(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) - public void fetchUsersFromDefinedUserList(){ - String msg = "fetch Users from Defined User List "; - logEnv(msg); - if(twitterwallSchedulerProperties.getFetchUserList().getAllow() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - this.fetchUsersFromDefinedUserList.fetchUsersFromDefinedUserList(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private final StartTask startTask; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java index 537fbe6f..91b024fe 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java @@ -20,7 +20,6 @@ public class ApplicationController { @RequestMapping(path="/domain/count") public String domainCount(Model model) { String msg = "/application/domain/count: "; - controllerHelper.logEnv(); String title = "Counted Entities"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.DATABASE.toString(); @@ -33,20 +32,13 @@ public String domainCount(Model model) { @RequestMapping(path="/management") public String dmanagementPage(Model model) { String msg = "/application/domain/count: "; - controllerHelper.logEnv(); String title = "Application Management"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.DATABASE.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - //CountedEntities countedEntities =this.countedEntitiesService.countAll(); - //model.addAttribute("countedEntities", countedEntities); - return "application/management"; } - - - private final CountedEntitiesService countedEntitiesService; - + @Autowired public ApplicationController(CountedEntitiesService countedEntitiesService, ControllerHelper controllerHelper, TwitterProperties twitterProperties) { this.countedEntitiesService = countedEntitiesService; @@ -54,6 +46,8 @@ public ApplicationController(CountedEntitiesService countedEntitiesService, Cont this.twitterProperties = twitterProperties; } + private final CountedEntitiesService countedEntitiesService; + private final ControllerHelper controllerHelper; private final TwitterProperties twitterProperties; diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java index 1dcb9a8e..af9140a1 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java @@ -88,7 +88,6 @@ public String domainCountUserprofile2Url(Model model) { } private void setUpThisPage(String title,Model model){ - controllerHelper.logEnv(); String subtitle = "Counted Entities"; String symbol = Symbols.DATABASE.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java index 4e79827d..8b591d3c 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java @@ -14,9 +14,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; +import org.woehlke.twitterwall.frontend.controller.common.HashTagsOverviewHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; -import org.woehlke.twitterwall.frontend.model.HashTagCounted; +import org.woehlke.twitterwall.frontend.model.HashTagOverview; import org.woehlke.twitterwall.oodm.entities.Tweet; import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.entities.HashTag; @@ -24,8 +25,6 @@ import org.woehlke.twitterwall.oodm.service.UserService; import org.woehlke.twitterwall.oodm.service.HashTagService; -import java.util.ArrayList; -import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,7 +40,6 @@ public class HashTagController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "HashTag"; String symbol = Symbols.HASHTAG.toString(); @@ -60,7 +58,6 @@ public String hashTagFromTweetsAndUsers( Model model) { String msg = "/hashtag/" + text + " "; - controllerHelper.logEnv(); Pattern p = Pattern.compile(HASHTAG_TEXT_PATTERN); Matcher m = p.matcher(text); if (m.matches()) { @@ -97,52 +94,13 @@ public String hashTagFromTweetsAndUsers( @RequestMapping(path="/overview") public String hashTagsOverview(Model model) { String msg = "/hashtag/overview "; - controllerHelper.logEnv(); String title = "HashTags"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.HASHTAG.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - List hashTagsTweets = new ArrayList<>(); - List hashTagsUsers = new ArrayList<>(); - Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterwallFrontendProperties.getPageSize()); - boolean hasNext = true; - while(hasNext) { - Page myPage = hashTagService.getAll(pageRequest); - for (HashTag hashTag : myPage.getContent()) { - Pageable pageRequestTeets = new PageRequest(0, 1); - Page tweets = tweetService.findTweetsForHashTag(hashTag, pageRequestTeets); - String myMSg = msg + " tweetService.findTweetsForHashTag= " + hashTag.getText(); - if (tweets == null) { - log.debug(myMSg + " result: null"); - } else { - long numberTweets = tweets.getTotalElements(); - log.debug(myMSg + " result: numberTweets=" + numberTweets); - if (numberTweets > 0) { - HashTagCounted c = new HashTagCounted(numberTweets, hashTag.getText()); - hashTagsTweets.add(c); - } - } - Pageable pageRequestUsers = new PageRequest(0, 1); - Page users = userService.getUsersForHashTag(hashTag, pageRequestUsers); - myMSg = msg + " userService.getUsersForHashTag= " + hashTag.getText(); - if (users == null) { - log.debug(myMSg + " result: null"); - } else { - long numberUsers = users.getTotalElements(); //userService.countUsersForHashTag(hashTag.getText()); - log.debug(myMSg + " result: numberUsers=" + numberUsers); - if (numberUsers > 0) { - HashTagCounted c = new HashTagCounted(numberUsers, hashTag.getText()); - hashTagsUsers.add(c); - } - } - } - hasNext = myPage.hasNext(); - if(hasNext){ - pageRequest=myPage.nextPageable(); - } - } - model.addAttribute("hashTagsTweets", hashTagsTweets); - model.addAttribute("hashTagsUsers", hashTagsUsers); + HashTagOverview overview = hashTagsOverviewHelper.getHashTagOverview(); + model.addAttribute("hashTagsTweets", overview.getHashTagsTweets()); + model.addAttribute("hashTagsUsers", overview.getHashTagsUsers()); return "hashtag/overview"; } @@ -160,13 +118,16 @@ public String hashTagsOverview(Model model) { private final ControllerHelper controllerHelper; + private final HashTagsOverviewHelper hashTagsOverviewHelper; + @Autowired - public HashTagController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, HashTagService hashTagService, TweetService tweetService, UserService userService, ControllerHelper controllerHelper) { + public HashTagController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, HashTagService hashTagService, TweetService tweetService, UserService userService, ControllerHelper controllerHelper, HashTagsOverviewHelper hashTagsOverviewHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.twitterProperties = twitterProperties; this.hashTagService = hashTagService; this.tweetService = tweetService; this.userService = userService; this.controllerHelper = controllerHelper; + this.hashTagsOverviewHelper = hashTagsOverviewHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java index 1769bf54..e809d08b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -26,7 +25,6 @@ public class MediaController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Media"; String symbol = Symbols.DATABASE.toString(); @@ -41,16 +39,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; @Autowired - public MediaController(MediaService mediaService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public MediaController(MediaService mediaService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.mediaService = mediaService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java index f3edb8ed..67d73038 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -25,7 +24,6 @@ public class MentionController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Mention"; String symbol = Symbols.DATABASE.toString(); @@ -38,16 +36,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final MentionService mentionService; private final ControllerHelper controllerHelper; @Autowired - public MentionController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, MentionService mentionService, ControllerHelper controllerHelper) { + public MentionController(TwitterwallFrontendProperties twitterwallFrontendProperties, MentionService mentionService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.mentionService = mentionService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java index b1f991c4..0cc2e8ff 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java @@ -29,14 +29,11 @@ public ModelAndView index(Model model) { @RequestMapping("/imprint") public String imprint(Model model) { log.info("-----------------------------------------"); - controllerHelper.logEnv(); String symbol = Symbols.IMPRINT.toString(); String title = "Impressum"; String subtitle = twitterwallFrontendProperties.getImprintSubtitle(); model = controllerHelper.setupPage(model, title, subtitle, symbol); User user = startTask.createImprintUser(); - //String screenName = twitterwallFrontendProperties.getImprintScreenName(); - //User user = createTestData.addUserForScreenName(screenName); model.addAttribute("user", user); log.info("-----------------------------------------"); return "imprint"; @@ -46,17 +43,11 @@ public String imprint(Model model) { private final TwitterwallFrontendProperties twitterwallFrontendProperties; - //private final TwitterProperties twitterProperties; - - //private final CreateTestData createTestData; - private final StartTask startTask; @Autowired public PagesController(TwitterwallFrontendProperties twitterwallFrontendProperties, StartTask startTask, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - //this.twitterProperties = twitterProperties; - //this.createTestData = createTestData; this.startTask = startTask; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java index 8b10ff78..0e685b97 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java @@ -30,7 +30,6 @@ public class TaskController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { String msg = "/task/all: "; - controllerHelper.logEnv(); String title = "Tasks"; String subtitle = "List aller Tasks"; String symbol = Symbols.TASK.toString(); @@ -44,15 +43,12 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp @RequestMapping(path="/{id}") public String getTaskById( @RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, - //@PathVariable long id, Model model) { @PathVariable("id") Task task, Model model) { String msg = "/task/ "+task.getId(); - controllerHelper.logEnv(); String title = "Tasks"; String subtitle = "List aller TasksHistory für Task"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - //Task oneTask = taskService.findById(id); Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize()); Page taskHistoryList = taskHistoryService.findByTask(task,pageRequest); model.addAttribute("task",task); @@ -63,7 +59,6 @@ public String getTaskById( @RequestMapping(path="/scheduled/tweets/fetch") public String fetchTweetsFromTwitterSearchRequest(Model model) { String msg = "/scheduled/tweets/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); @@ -75,7 +70,6 @@ public String fetchTweetsFromTwitterSearchRequest(Model model) { @RequestMapping(path="/scheduled/tweets/update") public String updateTweetsRequest(Model model) { String msg = "/scheduled/tweets/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); @@ -87,7 +81,6 @@ public String updateTweetsRequest(Model model) { @RequestMapping(path="/scheduled/users/list/fetch") public String fetchUsersFromDefinedUserListRequest(Model model){ String msg = "/scheduled/users/list/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/list/fetch"; String symbol = Symbols.TASK.toString(); @@ -99,7 +92,6 @@ public String fetchUsersFromDefinedUserListRequest(Model model){ @RequestMapping(path="/scheduled/users/mentions/update") public String updateUserProfilesFromMentionsRequest(Model model){ String msg = "/scheduled/users/mentions/update"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/mentions/update"; String symbol = Symbols.TASK.toString(); @@ -111,7 +103,6 @@ public String updateUserProfilesFromMentionsRequest(Model model){ @RequestMapping(path="/scheduled/users/update") public String updateUserProfilesRequest(Model model) { String msg = "/scheduled/users/update"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/update"; String symbol = Symbols.TASK.toString(); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java index 67eb257e..6091de54 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -27,7 +26,6 @@ public class TaskHistoryController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "TaskHistory"; String symbol = Symbols.DATABASE.toString(); @@ -42,15 +40,12 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; @Autowired - public TaskHistoryController(TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public TaskHistoryController(TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.taskHistoryService = taskHistoryService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java index 4c63670d..15e37a84 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java @@ -18,8 +18,6 @@ import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.service.UserService; import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; -import org.woehlke.twitterwall.scheduled.service.facade.CreateTestData; -import org.woehlke.twitterwall.scheduled.service.facade.FetchUsersFromDefinedUserList; /** @@ -32,7 +30,6 @@ public class TestController { @RequestMapping("/getTestData") public String getTestData(Model model) { - controllerHelper.logEnv(); model = controllerHelper.setupPage(model,"Test Data Tweets",twitterProperties.getSearchQuery(),Symbols.GET_TEST_DATA.toString()); String msg = "/getTestData : "; if(twitterwallFrontendProperties.getContextTest()){ @@ -64,10 +61,6 @@ public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ Contro private final UserService userService; - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; - - private final CreateTestData createTestData; - private final ControllerHelper controllerHelper; private final TwitterwallFrontendProperties twitterwallFrontendProperties; @@ -77,10 +70,8 @@ public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ Contro private final StartTask startTask; @Autowired - public TestController(UserService userService, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, CreateTestData createTestData, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, StartTask startTask) { + public TestController(UserService userService, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, StartTask startTask) { this.userService = userService; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.createTestData = createTestData; this.controllerHelper = controllerHelper; this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.twitterProperties = twitterProperties; diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java index 1d4547f4..a86ebdbe 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -27,7 +26,6 @@ public class TickerSymbolController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "TickerSymbol"; String symbol = Symbols.DATABASE.toString(); @@ -40,16 +38,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final TickerSymbolService tickerSymbolService; private final ControllerHelper controllerHelper; @Autowired - public TickerSymbolController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, TickerSymbolService tickerSymbolService, ControllerHelper controllerHelper) { + public TickerSymbolController(TwitterwallFrontendProperties twitterwallFrontendProperties, TickerSymbolService tickerSymbolService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.tickerSymbolService = tickerSymbolService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java index 8f820b0f..73858afc 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java @@ -28,7 +28,6 @@ public class TweetController { @RequestMapping("/all") public String getLatestTweets(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { - controllerHelper.logEnv(); model = controllerHelper.setupPage(model,"Tweets",twitterProperties.getSearchQuery(),Symbols.HOME.toString()); Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize(), Sort.Direction.DESC,"createdAt"); Page latest = tweetService.getAll(pageRequest); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java index dc8abe5c..2348f34e 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java @@ -27,7 +27,6 @@ public class UrlCacheController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "UrlCache"; String symbol = Symbols.DATABASE.toString(); @@ -42,8 +41,6 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelpe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final UrlCacheService urlCacheService; @Autowired @@ -51,6 +48,5 @@ public UrlCacheController(UrlCacheService urlCacheService, ControllerHelper cont this.urlCacheService = urlCacheService; this.controllerHelper = controllerHelper; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java index 0b975c72..bdcda11c 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; @@ -27,7 +26,6 @@ public class UrlController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Url"; String symbol = Symbols.DATABASE.toString(); @@ -40,16 +38,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelpe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final UrlService urlService; private final ControllerHelper controllerHelper; @Autowired - public UrlController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, UrlService urlService, ControllerHelper controllerHelper) { + public UrlController(TwitterwallFrontendProperties twitterwallFrontendProperties, UrlService urlService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.urlService = urlService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java index 76d4bce3..12276957 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java @@ -12,7 +12,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; @@ -113,18 +112,15 @@ public String getOnList(@RequestParam(name= "page" ,defaultValue=""+ControllerHe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; private static String subtitle = "Users"; @Autowired - public UserController(UserService userService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public UserController(UserService userService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.userService = userService; this.tweetService = tweetService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java index 15dd4976..54df3a47 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java @@ -10,8 +10,6 @@ public interface ControllerHelper { int FIRST_PAGE_NUMBER = 0; - void logEnv(); - ModelAndView setupPage(ModelAndView mav, String title, String subtitle, String symbol); Model setupPage(Model model, String title, String subtitle, String symbol); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java new file mode 100644 index 00000000..eb0e303c --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java @@ -0,0 +1,8 @@ +package org.woehlke.twitterwall.frontend.controller.common; + +import org.woehlke.twitterwall.frontend.model.HashTagOverview; + +public interface HashTagsOverviewHelper { + + HashTagOverview getHashTagOverview(); +} diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java index 618cd28f..31c9605b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java @@ -20,18 +20,6 @@ @Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = false) public class ControllerHelperImpl implements ControllerHelper { - public void logEnv(){ - log.info("--------------------------------------------------------------------"); - log.info("twitter.searchQuery = "+ twitterProperties.getSearchQuery()); - log.info("twitterwall.frontend.menuAppName = "+ twitterwallFrontendProperties.getMenuAppName()); - log.info("twitterwall.frontend.infoWebpage = "+ twitterwallFrontendProperties.getInfoWebpage()); - log.info("twitterwall.frontend.theme = "+ twitterwallFrontendProperties.getTheme()); - log.info("twitterwall.frontend.contextTest = "+ twitterwallFrontendProperties.getContextTest()); - log.info("twitterwall.frontend.imprintScreenName = "+ twitterwallFrontendProperties.getImprintScreenName()); - log.info("twitterwall.frontend.idGoogleAnalytics = "+ twitterwallFrontendProperties.getIdGoogleAnalytics()); - log.info("--------------------------------------------------------------------"); - } - private Page setupPage(Page page, String title, String subtitle, String symbol) { page.setTitle(title); page.setSubtitle(subtitle); @@ -49,7 +37,6 @@ private Page setupPage(Page page, String title, String subtitle, String symbol) } else { page.setGoogleAnalyticScriptHtml(""); } - logEnv(); log.info("--------------------------------------------------------------------"); log.info("setupPage = "+page.toString()); log.info("--------------------------------------------------------------------"); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java new file mode 100644 index 00000000..e6c18646 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java @@ -0,0 +1,90 @@ +package org.woehlke.twitterwall.frontend.controller.common.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; +import org.woehlke.twitterwall.frontend.controller.common.HashTagsOverviewHelper; +import org.woehlke.twitterwall.frontend.model.HashTagCounted; +import org.woehlke.twitterwall.frontend.model.HashTagOverview; +import org.woehlke.twitterwall.oodm.entities.HashTag; +import org.woehlke.twitterwall.oodm.entities.Tweet; +import org.woehlke.twitterwall.oodm.entities.User; +import org.woehlke.twitterwall.oodm.service.HashTagService; +import org.woehlke.twitterwall.oodm.service.TweetService; +import org.woehlke.twitterwall.oodm.service.UserService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component +public class HashTagsOverviewHelperImpl implements HashTagsOverviewHelper { + + + public HashTagsOverviewHelperImpl(UserService userService, HashTagService hashTagService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties) { + this.userService = userService; + this.hashTagService = hashTagService; + this.tweetService = tweetService; + this.twitterwallFrontendProperties = twitterwallFrontendProperties; + } + + @Override + public HashTagOverview getHashTagOverview() { + String msg = "getHashTagOverview"; + List hashTagsTweets = new ArrayList<>(); + List hashTagsUsers = new ArrayList<>(); + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterwallFrontendProperties.getPageSize()); + boolean hasNext = true; + while(hasNext) { + Page myPage = hashTagService.getAll(pageRequest); + for (HashTag hashTag : myPage.getContent()) { + Pageable pageRequestTeets = new PageRequest(0, 1); + Page tweets = tweetService.findTweetsForHashTag(hashTag, pageRequestTeets); + String myMSg = msg + " tweetService.findTweetsForHashTag= " + hashTag.getText(); + if (tweets == null) { + log.debug(myMSg + " result: null"); + } else { + long numberTweets = tweets.getTotalElements(); + log.debug(myMSg + " result: numberTweets=" + numberTweets); + if (numberTweets > 0) { + HashTagCounted c = new HashTagCounted(numberTweets, hashTag.getText()); + hashTagsTweets.add(c); + } + } + Pageable pageRequestUsers = new PageRequest(0, 1); + Page users = userService.getUsersForHashTag(hashTag, pageRequestUsers); + myMSg = msg + " userService.getUsersForHashTag= " + hashTag.getText(); + if (users == null) { + log.debug(myMSg + " result: null"); + } else { + long numberUsers = users.getTotalElements(); //userService.countUsersForHashTag(hashTag.getText()); + log.debug(myMSg + " result: numberUsers=" + numberUsers); + if (numberUsers > 0) { + HashTagCounted c = new HashTagCounted(numberUsers, hashTag.getText()); + hashTagsUsers.add(c); + } + } + } + hasNext = myPage.hasNext(); + if(hasNext){ + pageRequest=myPage.nextPageable(); + } + } + return null; + } + + private static final Logger log = LoggerFactory.getLogger(HashTagsOverviewHelperImpl.class); + + private final UserService userService; + + private final HashTagService hashTagService; + + private final TweetService tweetService; + + private final TwitterwallFrontendProperties twitterwallFrontendProperties; +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index 14474cec..511e5058 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -5,8 +5,8 @@ import org.springframework.integration.support.MessageBuilder; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Tweet; import org.woehlke.twitterwall.oodm.entities.User; @@ -26,137 +26,31 @@ @Component public class StartTaskImpl implements StartTask { - private final MessageChannel startTaskChannel; - - private final TaskService taskService; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public StartTaskImpl(MessageChannel startTaskChannel, TaskService taskService, CountedEntitiesService countedEntitiesService) { - this.startTaskChannel = startTaskChannel; - this.taskService = taskService; - this.countedEntitiesService = countedEntitiesService; - } - - private void messageSender(final TaskType taskType) { - CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); - TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); - Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); - MessagingTemplate mqTemplate = new MessagingTemplate(); - mqTemplate.send(startTaskChannel, mqMessage); - } - - - private final static long EINE_MINUTE = 60 * 1000; - - private final static long FUENF_MINUTEN = 5 * EINE_MINUTE; - - private final static long EINE_STUNDE = 60 * EINE_MINUTE; - - private final static long ZWOELF_STUNDEN = 12 * EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS = EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER = ZWOELF_STUNDEN; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS = ZWOELF_STUNDEN; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION = EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST = ZWOELF_STUNDEN; - - - /* - twitterwall.mq.starttask.time.fetchTweetsFromTwitterSearch - twitterwall.mq.starttask.time.updateTweets - twitterwall.mq.starttask.time.updateUserProfiles - twitterwall.mq.starttask.time.updateUserProfilesFromMentions - twitterwall.mq.starttask.time.fetchUsersFromDefinedUserList - */ - - @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) public void fetchTweetsFromTwitterSearch() { TaskType taskType = TaskType.FETCH_TWEETS_FROM_TWITTER_SEARCH; sendAndReceiveTweet(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; sendAndReceiveTweet(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; sendAndReceiveUser(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) public void updateUserProfilesFromMentions() { TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; sendAndReceiveUser(taskType); } - private void sendAndReceiveTweet(TaskType taskType){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); - TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); - Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); - MessagingTemplate mqTemplate = new MessagingTemplate(); - Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); - Object o = returnedMessage.getPayload(); - countedEntities = countedEntitiesService.countAll(); - if( o instanceof TweetResultList){ - TweetResultList msg = (TweetResultList) o; - long taskId = msg.getTaskId(); - task = taskService.findById(taskId); - taskService.done(task,countedEntities); - } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); - } - } - - private void sendAndReceiveUser(TaskType taskType){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); - TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); - Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); - MessagingTemplate mqTemplate = new MessagingTemplate(); - Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); - Object o = returnedMessage.getPayload(); - countedEntities = countedEntitiesService.countAll(); - if( o instanceof UserMessage){ - UserMessage msg = (UserMessage) o; - long taskId = msg.getTaskId(); - task = taskService.findById(taskId); - taskService.done(task,countedEntities); - } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); - } - } - @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) public void fetchUsersFromDefinedUserList() { TaskType taskType = TaskType.FETCH_USERS_FROM_DEFINED_USER_LIST; sendAndReceiveUser(taskType); @@ -193,25 +87,25 @@ public User createImprintUser() { public List createTestDataForUser() { CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_USER; - Task task = taskService.create("Start via MQ", taskType,countedEntities); + Task task = taskService.create("Start via MQ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); - if( o instanceof UserResultList){ + if (o instanceof UserResultList) { UserResultList result = (UserResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + taskService.done(task, countedEntities); return result.getUserList(); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + taskService.error(task, "Wrong type of returnedMessage", countedEntities); return new ArrayList<>(); } } @@ -220,28 +114,87 @@ public List createTestDataForUser() { public List createTestDataForTweets() { CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_TWEETS; + Task task = taskService.create("Start via MQ", taskType, countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); + Object o = returnedMessage.getPayload(); + countedEntities = countedEntitiesService.countAll(); + if (o instanceof TweetResultList) { + TweetResultList result = (TweetResultList) o; + long taskId = result.getTaskId(); + task = taskService.findById(taskId); + taskService.done(task, countedEntities); + return result.getTweetList(); + } else { + taskService.error(task, "Wrong type of returnedMessage", countedEntities); + return new ArrayList<>(); + } + } + + private void sendAndReceiveTweet(TaskType taskType){ + CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); if( o instanceof TweetResultList){ - TweetResultList result = (TweetResultList) o; - long taskId =result.getTaskId(); + TweetResultList msg = (TweetResultList) o; + long taskId = msg.getTaskId(); + task = taskService.findById(taskId); + taskService.done(task,countedEntities); + } else { + taskService.error(task,"Wrong type of returnedMessage",countedEntities); + } + } + + private void sendAndReceiveUser(TaskType taskType){ + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create("Start via MQ", taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); + Object o = returnedMessage.getPayload(); + countedEntities = countedEntitiesService.countAll(); + if( o instanceof UserMessage){ + UserMessage msg = (UserMessage) o; + long taskId = msg.getTaskId(); task = taskService.findById(taskId); taskService.done(task,countedEntities); - return result.getTweetList(); } else { taskService.error(task,"Wrong type of returnedMessage",countedEntities); - return new ArrayList<>(); } } + + private final MessageChannel startTaskChannel; + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + @Autowired + public StartTaskImpl(MessageChannel startTaskChannel, TaskService taskService, CountedEntitiesService countedEntitiesService, TwitterwallSchedulerProperties twitterwallSchedulerProperties) { + this.startTaskChannel = startTaskChannel; + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java index 2a9bcdd3..4b97b9e6 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -56,8 +56,9 @@ public List splitMessage(Message message) { Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); while(hasNext) { Page tweetTwitterIds = tweetService.findAllTwitterIds(pageRequest); - hasNext = tweetTwitterIds.hasNext(); worklistTwitterIds.addAll(tweetTwitterIds.getContent()); + hasNext = tweetTwitterIds.hasNext(); + pageRequest = pageRequest.next(); } int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(Long tweetTwitterId : worklistTwitterIds){ diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index 4ac2b988..7f51d22c 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -66,8 +66,9 @@ public List splitMessage(Message message) { Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); while (hasNext) { Page userProfileTwitterIds = userService.getAllTwitterIds(pageRequest); - hasNext = userProfileTwitterIds.hasNext(); worklistProfileTwitterIds.addAll(userProfileTwitterIds.getContent()); + hasNext = userProfileTwitterIds.hasNext(); + pageRequest = pageRequest.next(); } long number = worklistProfileTwitterIds.size(); int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); From 2f4874b2748b83197e6df6c27bcc425dc990c192 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 02:33:37 +0200 Subject: [PATCH 06/11] working on #141 #142 #143 --- .../common/impl/HashTagsOverviewHelperImpl.java | 2 +- .../twitterwall/oodm/service/TweetService.java | 4 ---- .../scheduled/mq/endoint/impl/StartTaskImpl.java | 3 +++ .../mq/endoint/impl/UpdateTweetsImpl.java | 15 ++++++++++++--- .../mq/endoint/impl/UpdateUserProfilesImpl.java | 10 +++++++--- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java index e6c18646..f302fe8e 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java @@ -75,7 +75,7 @@ public HashTagOverview getHashTagOverview() { pageRequest=myPage.nextPageable(); } } - return null; + return new HashTagOverview(hashTagsTweets,hashTagsUsers); } private static final Logger log = LoggerFactory.getLogger(HashTagsOverviewHelperImpl.class); diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java index 21e7e591..b6645732 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java @@ -16,12 +16,8 @@ */ public interface TweetService extends DomainServiceWithIdTwitter,DomainServiceWithTask { - //Page findLatestTweets(Pageable pageRequest); - Page findTweetsForHashTag(HashTag hashtag, Pageable pageRequest); - //long countTweetsForHashTag(String hashtagText); - Page findTweetsForUser(User user, Pageable pageRequest); Page findAllTwitterIds(Pageable pageRequest); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index 511e5058..9faa3a1c 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -5,6 +5,7 @@ import org.springframework.integration.support.MessageBuilder; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; import org.woehlke.twitterwall.oodm.entities.Task; @@ -32,12 +33,14 @@ public void fetchTweetsFromTwitterSearch() { sendAndReceiveTweet(taskType); } + @Async @Override public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; sendAndReceiveTweet(taskType); } + @Async @Override public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java index 4b97b9e6..be4ac88e 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -46,7 +48,6 @@ public UpdateTweetsImpl(TwitterProperties twitterProperties, TweetService tweetS @Override public List splitMessage(Message message) { CountedEntities countedEntities = countedEntitiesService.countAll(); - List tweets = new ArrayList<>(); TaskMessage msgIn = message.getPayload(); long taskId = msgIn.getTaskId(); Task task = taskService.findById(taskId); @@ -55,13 +56,18 @@ public List splitMessage(Message message) { boolean hasNext=true; Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); while(hasNext) { - Page tweetTwitterIds = tweetService.findAllTwitterIds(pageRequest); - worklistTwitterIds.addAll(tweetTwitterIds.getContent()); + Page tweetTwitterIds = tweetService.getAll(pageRequest); + for(org.woehlke.twitterwall.oodm.entities.Tweet tweetTwitterId:tweetTwitterIds.getContent()){ + log.debug("### tweetService.findAllTwitterIds (1): "+tweetTwitterId.getIdTwitter()); + worklistTwitterIds.add(tweetTwitterId.getIdTwitter()); + } hasNext = tweetTwitterIds.hasNext(); pageRequest = pageRequest.next(); } int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); + List tweets = new ArrayList<>(); for(Long tweetTwitterId : worklistTwitterIds){ + log.debug("### tweetService.findAllTwitterIds (2): "+tweetTwitterId); Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); tweets.add(result); @@ -72,4 +78,7 @@ public List splitMessage(Message message) { } return tweets; } + + private static final Logger log = LoggerFactory.getLogger(UpdateTweetsImpl.class); + } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index 7f51d22c..70012d8c 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Component; import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; import org.woehlke.twitterwall.oodm.service.TaskService; import org.woehlke.twitterwall.oodm.service.UserService; @@ -54,7 +55,6 @@ public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiSer public List splitMessage(Message message) { String msg = "mqUpdateUserProfiles.splitMessage: "; CountedEntities countedEntities = countedEntitiesService.countAll(); - List userProfileList = new ArrayList<>(); TaskMessage msgIn = message.getPayload(); long id = msgIn.getTaskId(); Task task = taskService.findById(id); @@ -65,13 +65,17 @@ public List splitMessage(Message message) { List worklistProfileTwitterIds = new ArrayList<>(); Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); while (hasNext) { - Page userProfileTwitterIds = userService.getAllTwitterIds(pageRequest); - worklistProfileTwitterIds.addAll(userProfileTwitterIds.getContent()); + Page userProfileTwitterIds = userService.getAll(pageRequest); + for(User user:userProfileTwitterIds.getContent()){ + log.debug(msg+ " userService.getAllTwitterIds: "+user.getIdTwitter()); + worklistProfileTwitterIds.add(user.getIdTwitter()); + } hasNext = userProfileTwitterIds.hasNext(); pageRequest = pageRequest.next(); } long number = worklistProfileTwitterIds.size(); int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); + List userProfileList = new ArrayList<>(); for(Long userProfileTwitterId:worklistProfileTwitterIds){ allLoop++; loopId++; From 5b35c844879f5b35e52a49d2b1913bfe0ebfc948 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 10:33:00 +0200 Subject: [PATCH 07/11] working on #141 #142 #143 --- .../frontend/controller/TaskController.java | 44 ++---- .../frontend/controller/TestController.java | 26 ++-- .../twitterwall/oodm/service/TaskService.java | 5 + .../oodm/service/impl/TaskServiceImpl.java | 30 +++- .../scheduled/mq/channel/SenderType.java | 6 + .../scheduled/mq/endoint/AsyncStartTask.java | 15 ++ .../mq/endoint/FetchDataFromDatabase.java | 13 -- .../FetchDataFromRemoteTwitterApi.java | 13 -- .../scheduled/mq/endoint/StartTask.java | 12 +- .../scheduled/mq/endoint/TweetFinisher.java | 3 + .../scheduled/mq/endoint/UserFinisher.java | 2 + .../mq/endoint/impl/AsyncStartTaskImpl.java | 82 +++++++++++ .../impl/FetchDataFromDatabaseImpl.java | 62 --------- .../FetchDataFromRemoteTwitterApiImpl.java | 70 ---------- .../mq/endoint/impl/StartTaskImpl.java | 69 +++++++--- .../mq/endoint/impl/TweetFinisherImpl.java | 44 ++++++ .../mq/endoint/impl/UpdateTweetsImpl.java | 1 + .../UpdateUserProfilesFromMentionsImpl.java | 15 +- .../endoint/impl/UpdateUserProfilesImpl.java | 2 + .../mq/endoint/impl/UserFinisherImpl.java | 46 ++++++- .../mq/endoint/impl/UserPersistorImpl.java | 6 +- src/main/resources/integration.xml | 130 +++++++++++++++++- src/main/resources/logback.xml | 1 + 23 files changed, 447 insertions(+), 250 deletions(-) create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java delete mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java delete mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java create mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java delete mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java delete mode 100644 src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java index 0e685b97..babc717d 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java @@ -5,7 +5,6 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.PathVariable; @@ -18,7 +17,7 @@ import org.woehlke.twitterwall.oodm.entities.TaskHistory; import org.woehlke.twitterwall.oodm.service.TaskHistoryService; import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; /** * Created by tw on 11.07.17. @@ -63,7 +62,7 @@ public String fetchTweetsFromTwitterSearchRequest(Model model) { String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.fetchTweetsFromTwitterSearch(); + mqAsyncStartTask.fetchUsersFromDefinedUserList(); return "scheduled/taskStarted"; } @@ -74,7 +73,7 @@ public String updateTweetsRequest(Model model) { String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateTweets(); + mqAsyncStartTask.updateTweets(); return "scheduled/taskStarted"; } @@ -85,7 +84,7 @@ public String fetchUsersFromDefinedUserListRequest(Model model){ String subtitle = "/scheduled/users/list/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.fetchUsersFromDefinedUserList(); + mqAsyncStartTask.fetchUsersFromDefinedUserList(); return "scheduled/taskStarted"; } @@ -96,7 +95,7 @@ public String updateUserProfilesFromMentionsRequest(Model model){ String subtitle = "/scheduled/users/mentions/update"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateUserProfilesFromMentions(); + mqAsyncStartTask.updateUserProfilesFromMentions(); return "scheduled/taskStarted"; } @@ -107,35 +106,10 @@ public String updateUserProfilesRequest(Model model) { String subtitle = "/scheduled/users/update"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateUserProfiles(); + mqAsyncStartTask.updateUserProfiles(); return "scheduled/taskStarted"; } - @Async - protected void fetchTweetsFromTwitterSearch() { - startTask.fetchTweetsFromTwitterSearch(); - } - - @Async - protected void updateTweets() { - startTask.updateTweets(); - } - - @Async - protected void fetchUsersFromDefinedUserList(){ - startTask.fetchUsersFromDefinedUserList(); - } - - @Async - protected void updateUserProfilesFromMentions(){ - startTask.updateUserProfilesFromMentions(); - } - - @Async - protected void updateUserProfiles() { - startTask.updateUserProfiles(); - } - private final TaskService taskService; private final TaskHistoryService taskHistoryService; @@ -144,14 +118,14 @@ protected void updateUserProfiles() { private final ControllerHelper controllerHelper; - private final StartTask startTask; + private final AsyncStartTask mqAsyncStartTask; @Autowired - public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper, StartTask startTask) { + public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper, AsyncStartTask mqAsyncStartTask) { this.taskService = taskService; this.taskHistoryService = taskHistoryService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.controllerHelper = controllerHelper; - this.startTask = startTask; + this.mqAsyncStartTask = mqAsyncStartTask; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java index 15e37a84..66269e30 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java @@ -6,7 +6,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; @@ -17,6 +16,7 @@ import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.service.UserService; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; @@ -33,8 +33,8 @@ public String getTestData(Model model) { model = controllerHelper.setupPage(model,"Test Data Tweets",twitterProperties.getSearchQuery(),Symbols.GET_TEST_DATA.toString()); String msg = "/getTestData : "; if(twitterwallFrontendProperties.getContextTest()){ - model.addAttribute("latestTweets", startTask.createTestDataForTweets()); - model.addAttribute("users", startTask.createTestDataForUser()); + model.addAttribute("latestTweets", mqStartTask.createTestDataForTweets()); + model.addAttribute("users", mqStartTask.createTestDataForUser()); } else { model.addAttribute("latestTweets",null); model.addAttribute("users",null); @@ -46,7 +46,9 @@ public String getTestData(Model model) { public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize()); String msg = "getOnListRenew: "; - this.startOnListRenew(); + log.info(msg+"START startTask.fetchUsersFromDefinedUserList: "); + mqAsyncStartTask.fetchUsersFromDefinedUserList(); + log.info(msg+"DONE startTask.fetchUsersFromDefinedUserList: "); log.info(msg+"START userService.findOnList(): "); Page usersOnList = userService.getOnList(pageRequest); log.info(msg+"DONE userService.findOnList(): "); @@ -67,22 +69,18 @@ public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ Contro private final TwitterProperties twitterProperties; - private final StartTask startTask; + private final AsyncStartTask mqAsyncStartTask; + + private final StartTask mqStartTask; @Autowired - public TestController(UserService userService, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, StartTask startTask) { + public TestController(UserService userService, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, AsyncStartTask mqAsyncStartTask, StartTask mqStartTask) { this.userService = userService; this.controllerHelper = controllerHelper; this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.twitterProperties = twitterProperties; - this.startTask = startTask; + this.mqAsyncStartTask = mqAsyncStartTask; + this.mqStartTask = mqStartTask; } - @Async - protected void startOnListRenew(){ - String msg = "startOnListRenew: "; - log.info(msg+"START startTask.fetchUsersFromDefinedUserList: "); - startTask.fetchUsersFromDefinedUserList(); - log.info(msg+"DONE startTask.fetchUsersFromDefinedUserList: "); - } } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java index fbf6d526..ea8be218 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java @@ -31,4 +31,9 @@ public interface TaskService extends DomainObjectMinimalService { Task error(Task task, String msg,CountedEntities countedEntities); Task start(Task task,CountedEntities countedEntities); + + Task finalError(Task task, String s, CountedEntities countedEntities); + + Task done(String logMsg, Task task, CountedEntities countedEntities); + } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java index 0a8684f3..aa1dcc35 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java @@ -72,7 +72,7 @@ public Task done(Task task,CountedEntities countedEntities) { task.setTaskStatus(TaskStatus.FINISHED); task.setTimeLastUpdate(new Date()); task = taskRepository.save(task); - TaskHistory event = new TaskHistory("done",task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); + TaskHistory event = new TaskHistory("DONE ",task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); event.setIdTask(task.getId()); event.setTask(task); event = taskHistoryRepository.save(event); @@ -175,7 +175,33 @@ public Task start(Task task,CountedEntities countedEntities) { task.setTaskStatus(TaskStatus.RUNNING); task.setTimeLastUpdate(new Date()); task = taskRepository.save(task); - TaskHistory event = new TaskHistory("start",task.getTaskStatus(),TaskStatus.RUNNING,countedEntities); + TaskHistory event = new TaskHistory("START",task.getTaskStatus(),TaskStatus.RUNNING,countedEntities); + event.setIdTask(task.getId()); + event.setTask(task); + event = taskHistoryRepository.save(event); + log.debug(task.toString()); + return task; + } + + @Override + public Task finalError(Task task, String msg, CountedEntities countedEntities) { + task.setTaskStatus(TaskStatus.ERROR); + task.setTimeLastUpdate(new Date()); + task = taskRepository.save(task); + TaskHistory event = new TaskHistory("FINAL ERROR: "+msg,task.getTaskStatus(),TaskStatus.ERROR,countedEntities); + event.setIdTask(task.getId()); + event.setTask(task); + event = taskHistoryRepository.save(event); + log.debug(task.toString()); + return task; + } + + @Override + public Task done(String logMsg, Task task, CountedEntities countedEntities) { + task.setTaskStatus(TaskStatus.FINISHED); + task.setTimeLastUpdate(new Date()); + task = taskRepository.save(task); + TaskHistory event = new TaskHistory("DONE "+logMsg,task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); event.setIdTask(task.getId()); event.setTask(task); event = taskHistoryRepository.save(event); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java new file mode 100644 index 00000000..f2d707f4 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java @@ -0,0 +1,6 @@ +package org.woehlke.twitterwall.scheduled.mq.channel; + +public enum SenderType { + SEND_AND_WAIT_FOR_RESULT_SENDER, + FIRE_AND_FORGET_SENDER +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java new file mode 100644 index 00000000..0ccb44c2 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java @@ -0,0 +1,15 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +public interface AsyncStartTask { + + void updateTweets(); + + void updateUserProfiles(); + + void updateUserProfilesFromMentions(); + + void fetchTweetsFromTwitterSearch(); + + void fetchUsersFromDefinedUserList(); + +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java deleted file mode 100644 index 610f6566..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint; - -import org.springframework.messaging.Message; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; - -public interface FetchDataFromDatabase { - - void updateTweets(Message mqMessage); - - void updateUserProfiles(Message mqMessage); - - void updateUserProfilesFromMentions(Message mqMessage); -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java deleted file mode 100644 index 4a0484b8..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint; - -import org.springframework.messaging.Message; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; - -public interface FetchDataFromRemoteTwitterApi { - - void fetchTweetsFromTwitterSearch(Message mqMessage); - - void fetchUsersFromDefinedUserList(Message mqMessage); - - void createTestDataForUser(Message mqMessage); -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java index 7a344693..495ccf74 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java @@ -5,17 +5,7 @@ import java.util.List; -public interface StartTask { - - void fetchTweetsFromTwitterSearch(); - - void updateTweets(); - - void updateUserProfiles(); - - void updateUserProfilesFromMentions(); - - void fetchUsersFromDefinedUserList(); +public interface StartTask extends AsyncStartTask { User createImprintUser(); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java index 07a5145e..af391679 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java @@ -7,5 +7,8 @@ import java.util.List; public interface TweetFinisher { + TweetResultList finish(Message> incomingMessageList); + + void finishAsnyc(Message> incomingMessageList); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java index 19b3c5cf..e1daaf0a 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java @@ -9,4 +9,6 @@ public interface UserFinisher { UserResultList finish(Message> incomingMessageList); + + void finishAsnyc(Message> incomingMessageList); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java new file mode 100644 index 00000000..856c8b7f --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java @@ -0,0 +1,82 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.channel.ExecutorChannel; +import org.springframework.integration.core.MessagingTemplate; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.entities.parts.TaskType; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +@Component("mqAsyncStartTask") +public class AsyncStartTaskImpl implements AsyncStartTask { + + @Override + public void fetchTweetsFromTwitterSearch() { + TaskType taskType = TaskType.FETCH_TWEETS_FROM_TWITTER_SEARCH; + send(taskType); + } + + @Override + public void updateTweets() { + TaskType taskType = TaskType.UPDATE_TWEETS; + send(taskType); + } + + @Override + public void updateUserProfiles() { + TaskType taskType = TaskType.UPDATE_USER_PROFILES; + send(taskType); + } + + @Override + public void updateUserProfilesFromMentions() { + TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; + send(taskType); + } + + @Override + public void fetchUsersFromDefinedUserList() { + TaskType taskType = TaskType.FETCH_USERS_FROM_DEFINED_USER_LIST; + send(taskType); + } + + private void send(TaskType taskType){ + String msg = "Start task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create(msg, taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(msg); + mqTemplate.send(executorChannelForAsyncStart, mqMessage); + } + + @Autowired + public AsyncStartTaskImpl(TaskService taskService, CountedEntitiesService countedEntitiesService, ExecutorChannel executorChannelForAsyncStart) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + this.executorChannelForAsyncStart = executorChannelForAsyncStart; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private final ExecutorChannel executorChannelForAsyncStart; + + private static final Logger log = LoggerFactory.getLogger(AsyncStartTaskImpl.class); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java deleted file mode 100644 index 2191993e..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint.impl; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.Message; -import org.springframework.stereotype.Component; -import org.woehlke.twitterwall.oodm.entities.Task; -import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; -import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.mq.endoint.FetchDataFromDatabase; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; -import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; - -@Component("mqFetchDataFromDatabase") -public class FetchDataFromDatabaseImpl implements FetchDataFromDatabase { - - private static final Logger log = LoggerFactory.getLogger(FetchDataFromDatabaseImpl.class); - - private final TaskService taskService; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public FetchDataFromDatabaseImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { - this.taskService = taskService; - this.countedEntitiesService = countedEntitiesService; - } - - - private void react(String logMsg,Message mqMessage){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - TaskMessage receivedMessage = mqMessage.getPayload(); - Task task = taskService.findById(receivedMessage.getTaskId()); - taskService.start(task,countedEntities); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"received: "+mqMessage.getPayload().toString()); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - } - - @Override - public void updateTweets(Message mqMessage) { - String logMsg = "updateTweets: "; - react(logMsg,mqMessage); - } - - @Override - public void updateUserProfiles(Message mqMessage) { - String logMsg = "updateUserProfiles: "; - react(logMsg,mqMessage); - } - - @Override - public void updateUserProfilesFromMentions(Message mqMessage) { - String logMsg = "updateUserProfilesFromMentions: "; - react(logMsg,mqMessage); - } -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java deleted file mode 100644 index 0ce240ac..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint.impl; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.Message; -import org.springframework.stereotype.Component; -import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; -import org.woehlke.twitterwall.oodm.entities.Task; -import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; -import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; -import org.woehlke.twitterwall.scheduled.mq.endoint.FetchDataFromRemoteTwitterApi; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; -import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; - -@Component("mqFetchDataFromRemoteTwitterApi") -public class FetchDataFromRemoteTwitterApiImpl implements FetchDataFromRemoteTwitterApi { - - private static final Logger log = LoggerFactory.getLogger(FetchDataFromRemoteTwitterApiImpl.class); - - private final TaskService taskService; - - private final TwitterApiService twitterApiService; - - private final TwitterwallFrontendProperties twitterwallFrontendProperties; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public FetchDataFromRemoteTwitterApiImpl(TaskService taskService, TwitterApiService twitterApiService, TwitterwallFrontendProperties twitterwallFrontendProperties, CountedEntitiesService countedEntitiesService) { - this.taskService = taskService; - this.twitterApiService = twitterApiService; - this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.countedEntitiesService = countedEntitiesService; - } - - private void react(String logMsg,Message mqMessage){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - TaskMessage receivedMessage = mqMessage.getPayload(); - Task task = taskService.findById(receivedMessage.getTaskId()); - taskService.start(task,countedEntities); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"received: "+mqMessage.getPayload().toString()); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - } - - @Override - public void fetchTweetsFromTwitterSearch(Message mqMessage) { - String logMsg = "fetchTweetsFromTwitterSearch: "; - this.react(logMsg, mqMessage); - } - - @Override - public void fetchUsersFromDefinedUserList(Message mqMessage) { - String logMsg = "fetchUsersFromDefinedUserList: "; - this.react(logMsg, mqMessage); - } - - @Override - public void createTestDataForUser(Message mqMessage) { - String logMsg = "createTestDataForUser: "; - this.react(logMsg, mqMessage); - } - -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index 9faa3a1c..a2ad1d61 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -1,19 +1,20 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.core.MessagingTemplate; import org.springframework.integration.support.MessageBuilder; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Tweet; import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; import org.woehlke.twitterwall.oodm.entities.parts.TaskType; import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; import org.woehlke.twitterwall.scheduled.mq.msg.TweetResultList; @@ -24,7 +25,7 @@ import java.util.ArrayList; import java.util.List; -@Component +@Component("mqStartTask") public class StartTaskImpl implements StartTask { @Override @@ -33,14 +34,12 @@ public void fetchTweetsFromTwitterSearch() { sendAndReceiveTweet(taskType); } - @Async @Override public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; sendAndReceiveTweet(taskType); } - @Async @Override public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; @@ -63,6 +62,7 @@ public void fetchUsersFromDefinedUserList() { public User createImprintUser() { CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_CREATE_IMPRINT_USER; + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; Task task = taskService.create("Start via MQ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -71,6 +71,7 @@ public User createImprintUser() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -78,10 +79,14 @@ public User createImprintUser() { UserMessage msg = (UserMessage) o; long taskId = msg.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return msg.getUser(); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return null; } } @@ -90,6 +95,7 @@ public User createImprintUser() { public List createTestDataForUser() { CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_USER; + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; Task task = taskService.create("Start via MQ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -98,6 +104,7 @@ public List createTestDataForUser() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -105,10 +112,14 @@ public List createTestDataForUser() { UserResultList result = (UserResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - taskService.done(task, countedEntities); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return result.getUserList(); } else { - taskService.error(task, "Wrong type of returnedMessage", countedEntities); + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return new ArrayList<>(); } } @@ -117,7 +128,8 @@ public List createTestDataForUser() { public List createTestDataForTweets() { CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_TWEETS; - Task task = taskService.create("Start via MQ", taskType, countedEntities); + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + Task task = taskService.create("Start via MQ by Scheduler ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) .setHeader("task_id", task.getId()) @@ -125,6 +137,7 @@ public List createTestDataForTweets() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -132,17 +145,22 @@ public List createTestDataForTweets() { TweetResultList result = (TweetResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - taskService.done(task, countedEntities); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return result.getTweetList(); } else { - taskService.error(task, "Wrong type of returnedMessage", countedEntities); + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return new ArrayList<>(); } } private void sendAndReceiveTweet(TaskType taskType){ + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); + Task task = taskService.create("Start via MQ by Scheduler ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) .setHeader("task_id", task.getId()) @@ -150,6 +168,7 @@ private void sendAndReceiveTweet(TaskType taskType){ .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -157,15 +176,19 @@ private void sendAndReceiveTweet(TaskType taskType){ TweetResultList msg = (TweetResultList) o; long taskId = msg.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg,task,countedEntities); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); } } private void sendAndReceiveUser(TaskType taskType){ + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); + Task task = taskService.create(logMsg, taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) .setHeader("task_id", task.getId()) @@ -173,6 +196,7 @@ private void sendAndReceiveUser(TaskType taskType){ .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); + log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -180,9 +204,13 @@ private void sendAndReceiveUser(TaskType taskType){ UserMessage msg = (UserMessage) o; long taskId = msg.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg,task,countedEntities); + log.info(logMsg); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); } } @@ -192,12 +220,15 @@ private void sendAndReceiveUser(TaskType taskType){ private final CountedEntitiesService countedEntitiesService; + private static final Logger log = LoggerFactory.getLogger(StartTaskImpl.class); + @Autowired - public StartTaskImpl(MessageChannel startTaskChannel, TaskService taskService, CountedEntitiesService countedEntitiesService, TwitterwallSchedulerProperties twitterwallSchedulerProperties) { + public StartTaskImpl(MessageChannel startTaskChannel,TaskService taskService, CountedEntitiesService countedEntitiesService) { this.startTaskChannel = startTaskChannel; this.taskService = taskService; this.countedEntitiesService = countedEntitiesService; } + } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java index f330ede8..e2f3690f 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java @@ -1,11 +1,19 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.Message; import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Tweet; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.TweetFinisher; import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; import org.woehlke.twitterwall.scheduled.mq.msg.TweetResultList; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; import java.util.ArrayList; import java.util.List; @@ -13,6 +21,7 @@ @Component("mqTweetFinisher") public class TweetFinisherImpl implements TweetFinisher { + @Override public TweetResultList finish(Message> incomingMessageList){ List resultList = new ArrayList<>(); long taskId = 0L; @@ -23,4 +32,39 @@ public TweetResultList finish(Message> incomingMessageLis TweetResultList result = new TweetResultList(taskId,resultList); return result; } + + @Override + public void finishAsnyc(Message> incomingMessageList) { + List userMessageList = incomingMessageList.getPayload(); + CountedEntities countedEntities = countedEntitiesService.countAll(); + if(incomingMessageList.getHeaders().containsKey("task_id")){ + long taskId = (Long) incomingMessageList.getHeaders().get( "task_id"); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(msgDone,task,countedEntities); + log.info(msgDone); + } else { + if(userMessageList.size()>0) { + long taskId = userMessageList.get(0).getTaskId(); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(task,countedEntities); + log.info(msgDone); + } else { + log.warn("finishAsnyc: No Tweets performed. via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER); + } + } + } + + @Autowired + public TweetFinisherImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private static final Logger log = LoggerFactory.getLogger(UserFinisherImpl.class); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java index be4ac88e..c26e3510 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -71,6 +71,7 @@ public List splitMessage(Message message) { Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); tweets.add(result); + log.debug("### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); try { Thread.sleep(millisToWaitBetweenTwoApiCalls); } catch (InterruptedException e) { diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java index 8f961c59..4054fce9 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -7,7 +9,6 @@ import org.springframework.social.twitter.api.TwitterProfile; import org.springframework.stereotype.Component; import org.woehlke.twitterwall.conf.TwitterProperties; -import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; import org.woehlke.twitterwall.oodm.entities.Mention; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; @@ -27,7 +28,7 @@ @Component("mqUpdateUserProfilesFromMentions") public class UpdateUserProfilesFromMentionsImpl implements UpdateUserProfilesFromMentions { - private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; + private static final Logger log = LoggerFactory.getLogger(UpdateUserProfilesFromMentionsImpl.class); private final TwitterProperties twitterProperties; @@ -39,8 +40,7 @@ public class UpdateUserProfilesFromMentionsImpl implements UpdateUserProfilesFro private final CountedEntitiesService countedEntitiesService; - public UpdateUserProfilesFromMentionsImpl(TwitterwallSchedulerProperties twitterwallSchedulerProperties, TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, MentionService mentionService, CountedEntitiesService countedEntitiesService) { - this.twitterwallSchedulerProperties = twitterwallSchedulerProperties; + public UpdateUserProfilesFromMentionsImpl(TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, MentionService mentionService, CountedEntitiesService countedEntitiesService) { this.twitterProperties = twitterProperties; this.twitterApiService = twitterApiService; this.taskService = taskService; @@ -50,6 +50,8 @@ public UpdateUserProfilesFromMentionsImpl(TwitterwallSchedulerProperties twitter @Override public List splitMessage(Message message) { + String msg ="splitMessage: "; + log.debug(msg+ " START"); CountedEntities countedEntities = countedEntitiesService.countAll(); List userProfileList = new ArrayList<>(); TaskMessage msgIn = message.getPayload(); @@ -76,10 +78,11 @@ public List splitMessage(Message message) { int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(String screenName:screenNames){ TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName); - if(userProfile!=null){ - TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); + if(userProfile!=null) { + TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn, userProfile); userProfileList.add(userMsg); } + log.debug("### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); try { Thread.sleep(millisToWaitBetweenTwoApiCalls); } catch (InterruptedException e) { diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index 70012d8c..4d7eddfd 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -54,6 +54,7 @@ public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiSer @Override public List splitMessage(Message message) { String msg = "mqUpdateUserProfiles.splitMessage: "; + log.debug(msg+ " START"); CountedEntities countedEntities = countedEntitiesService.countAll(); TaskMessage msgIn = message.getPayload(); long id = msgIn.getTaskId(); @@ -96,6 +97,7 @@ public List splitMessage(Message message) { } catch (InterruptedException e) { } } + log.debug(msg+ " DONE"); return userProfileList; } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java index 1b12e187..4c7bce3c 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java @@ -1,11 +1,20 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.Message; import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.User; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.UserFinisher; import org.woehlke.twitterwall.scheduled.mq.msg.UserMessage; import org.woehlke.twitterwall.scheduled.mq.msg.UserResultList; +import org.woehlke.twitterwall.scheduled.service.backend.impl.TwitterApiServiceImpl; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; import java.util.ArrayList; import java.util.List; @@ -13,7 +22,6 @@ @Component("mqUserFinisher") public class UserFinisherImpl implements UserFinisher { - @Override public UserResultList finish(Message> incomingMessageList) { long taskId = 0L; @@ -28,4 +36,40 @@ public UserResultList finish(Message> incomingMessageList) { UserResultList userResultList = new UserResultList(taskId,users); return userResultList; } + + @Override + public void finishAsnyc(Message> incomingMessageList) { + List userMessageList = incomingMessageList.getPayload(); + CountedEntities countedEntities = countedEntitiesService.countAll(); + if(incomingMessageList.getHeaders().containsKey("task_id")){ + long taskId = (Long) incomingMessageList.getHeaders().get( "task_id"); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(msgDone,task,countedEntities); + log.info(msgDone); + } else { + if(userMessageList.size()>0) { + long taskId = userMessageList.get(0).getTaskId(); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(task,countedEntities); + log.info(msgDone); + } else { + log.warn("finishAsnyc: No Users performed. via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER); + } + } + } + + + @Autowired + public UserFinisherImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private static final Logger log = LoggerFactory.getLogger(UserFinisherImpl.class); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java index 296f4d17..568be8b5 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java @@ -26,10 +26,10 @@ public UserPersistorImpl(TaskService taskService, StoreUserProcess storeUserProc @Override public UserMessage persistUser(Message incomingUserMessage) { UserMessage receivedMessage = incomingUserMessage.getPayload(); - long id = receivedMessage.getTwitterProfileMessage().getTaskMessage().getTaskId(); - Task task = taskService.findById(id); + long taskId = receivedMessage.getTwitterProfileMessage().getTaskMessage().getTaskId(); + Task task = taskService.findById(taskId); User user = storeUserProcess.storeUserProcess(receivedMessage.getUser(),task); - UserMessage userMessage = new UserMessage(user,id); + UserMessage userMessage = new UserMessage(user,taskId); return userMessage; } } diff --git a/src/main/resources/integration.xml b/src/main/resources/integration.xml index 2690dd33..3b67ae9a 100644 --- a/src/main/resources/integration.xml +++ b/src/main/resources/integration.xml @@ -80,7 +80,7 @@ - + @@ -203,4 +203,132 @@ method="persistUser" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index a23b0f24..44db7f4e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -14,6 +14,7 @@ + From 7216c06e43025e8f862f839e5c6b92076c12d93b Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 10:44:15 +0200 Subject: [PATCH 08/11] working on #141 #142 #143 --- .../scheduled/mq/endoint/impl/StartTaskImpl.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index a2ad1d61..beb518f7 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -60,9 +60,10 @@ public void fetchUsersFromDefinedUserList() { @Override public User createImprintUser() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_CREATE_IMPRINT_USER; String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -71,7 +72,6 @@ public User createImprintUser() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -93,9 +93,10 @@ public User createImprintUser() { @Override public List createTestDataForUser() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_USER; String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -104,7 +105,6 @@ public List createTestDataForUser() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -126,9 +126,10 @@ public List createTestDataForUser() { @Override public List createTestDataForTweets() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_TWEETS; String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ by Scheduler ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -137,7 +138,6 @@ public List createTestDataForTweets() { .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -159,6 +159,7 @@ public List createTestDataForTweets() { private void sendAndReceiveTweet(TaskType taskType){ String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ by Scheduler ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); @@ -168,7 +169,6 @@ private void sendAndReceiveTweet(TaskType taskType){ .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); @@ -187,6 +187,7 @@ private void sendAndReceiveTweet(TaskType taskType){ private void sendAndReceiveUser(TaskType taskType){ String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create(logMsg, taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); @@ -196,7 +197,6 @@ private void sendAndReceiveUser(TaskType taskType){ .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(logMsg); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); From d2c161e85cd8f67575062ecc333703c38b69396e Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 15:10:50 +0200 Subject: [PATCH 09/11] working on #141 #142 #143 --- .../frontend/controller/TaskController.java | 2 +- .../oodm/repositories/MentionRepository.java | 3 + .../oodm/service/MentionService.java | 4 ++ .../oodm/service/impl/MentionServiceImpl.java | 5 ++ .../mq/endoint/impl/AsyncStartTaskImpl.java | 4 +- .../mq/endoint/impl/UpdateTweetsImpl.java | 10 +++- .../UpdateUserProfilesFromMentionsImpl.java | 14 +++-- .../endoint/impl/UpdateUserProfilesImpl.java | 14 ++--- .../backend/impl/TwitterApiServiceImpl.java | 57 ++++++++++++++++--- 9 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java index babc717d..6aacdb1b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java @@ -62,7 +62,7 @@ public String fetchTweetsFromTwitterSearchRequest(Model model) { String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - mqAsyncStartTask.fetchUsersFromDefinedUserList(); + mqAsyncStartTask.fetchTweetsFromTwitterSearch(); return "scheduled/taskStarted"; } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java b/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java index 8f4dd8e6..bf59d71e 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.oodm.repositories; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Repository; import org.woehlke.twitterwall.oodm.entities.Mention; import org.woehlke.twitterwall.oodm.repositories.common.DomainRepository; @@ -17,4 +19,5 @@ public interface MentionRepository extends DomainRepository,MentionRepo Mention findByScreenName(String screenName); + Page findAllByUserNull(Pageable pageRequest); } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java index 32495a08..cfd056fe 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.oodm.service; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Mention; import org.woehlke.twitterwall.oodm.service.common.DomainServiceWithIdTwitter; @@ -13,4 +15,6 @@ public interface MentionService extends DomainServiceWithScreenName,DomainServiceWithIdTwitter,DomainServiceWithTask { Mention createProxyMention(Mention mention, Task task); + + Page getAllWithoutPersistentUser(Pageable pageRequest); } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java index 50420a2b..63998d92 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java @@ -65,4 +65,9 @@ public Mention createProxyMention(Mention mention, Task task) { mention = mentionRepository.save(mention); return mention; } + + @Override + public Page getAllWithoutPersistentUser(Pageable pageRequest) { + return mentionRepository.findAllByUserNull(pageRequest); + } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java index 856c8b7f..a28321f6 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java @@ -51,7 +51,8 @@ public void fetchUsersFromDefinedUserList() { } private void send(TaskType taskType){ - String msg = "Start task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; + String msg = "START Task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; + log.info(msg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create(msg, taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); @@ -61,7 +62,6 @@ private void send(TaskType taskType){ .setHeader("task_type", task.getTaskType()) .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); - log.info(msg); mqTemplate.send(executorChannelForAsyncStart, mqMessage); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java index c26e3510..f5a82362 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -55,10 +55,14 @@ public List splitMessage(Message message) { List worklistTwitterIds = new ArrayList<>(); boolean hasNext=true; Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + int lfdNr = 0; + int all = 0; while(hasNext) { Page tweetTwitterIds = tweetService.getAll(pageRequest); for(org.woehlke.twitterwall.oodm.entities.Tweet tweetTwitterId:tweetTwitterIds.getContent()){ - log.debug("### tweetService.findAllTwitterIds (1): "+tweetTwitterId.getIdTwitter()); + lfdNr++; + all++; + log.debug("### tweetService.findAllTwitterIds from DB ("+lfdNr+"): "+tweetTwitterId.getIdTwitter()); worklistTwitterIds.add(tweetTwitterId.getIdTwitter()); } hasNext = tweetTwitterIds.hasNext(); @@ -66,8 +70,10 @@ public List splitMessage(Message message) { } int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); List tweets = new ArrayList<>(); + lfdNr = 0; for(Long tweetTwitterId : worklistTwitterIds){ - log.debug("### tweetService.findAllTwitterIds (2): "+tweetTwitterId); + lfdNr++; + log.debug("### witterApiService.findOneTweetById from Twiiter API ("+lfdNr+" of "+all+"): "+tweetTwitterId); Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); tweets.add(result); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java index 4054fce9..39fa3cd3 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java @@ -59,17 +59,19 @@ public List splitMessage(Message message) { Task task = taskService.findById(id); task = taskService.start(task,countedEntities); List screenNames = new ArrayList<>(); - int allLoop = 0; - int loopId = 0; + int lfdNr = 0; + int all = 0; boolean hasNext=true; Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); while (hasNext) { - Page allPersMentions = mentionService.getAll(pageRequest); + Page allPersMentions = mentionService.getAllWithoutPersistentUser(pageRequest); hasNext = allPersMentions.hasNext(); - long number = allPersMentions.getTotalElements(); for (Mention onePersMentions : allPersMentions) { if (!onePersMentions.hasPersistentUser()) { + lfdNr++; + all++; String screenName = onePersMentions.getScreenName(); + log.debug("### mentionService.getAll from DB ("+lfdNr+"): "+screenName); screenNames.add(screenName); } } @@ -77,12 +79,14 @@ public List splitMessage(Message message) { } int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); for(String screenName:screenNames){ + lfdNr++; + log.debug("### twitterApiService.getUserProfileForScreenName("+screenName+") from Twiiter API ("+lfdNr+" of "+all+")"); TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName); if(userProfile!=null) { TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn, userProfile); userProfileList.add(userMsg); } - log.debug("### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); + log.debug(msg+"### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); try { Thread.sleep(millisToWaitBetweenTwoApiCalls); } catch (InterruptedException e) { diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index 4d7eddfd..d4378598 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -53,14 +53,13 @@ public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiSer @Override public List splitMessage(Message message) { - String msg = "mqUpdateUserProfiles.splitMessage: "; + String msg = "### mqUpdateUserProfiles.splitMessage: "; log.debug(msg+ " START"); CountedEntities countedEntities = countedEntitiesService.countAll(); TaskMessage msgIn = message.getPayload(); long id = msgIn.getTaskId(); Task task = taskService.findById(id); task = taskService.start(task,countedEntities); - int allLoop = 0; int loopId = 0; boolean hasNext=true; List worklistProfileTwitterIds = new ArrayList<>(); @@ -68,7 +67,8 @@ public List splitMessage(Message message) { while (hasNext) { Page userProfileTwitterIds = userService.getAll(pageRequest); for(User user:userProfileTwitterIds.getContent()){ - log.debug(msg+ " userService.getAllTwitterIds: "+user.getIdTwitter()); + loopId++; + log.debug(msg+ "### userService.getAllTwitterIds: ("+loopId+") "+user.getIdTwitter()); worklistProfileTwitterIds.add(user.getIdTwitter()); } hasNext = userProfileTwitterIds.hasNext(); @@ -78,20 +78,20 @@ public List splitMessage(Message message) { int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); List userProfileList = new ArrayList<>(); for(Long userProfileTwitterId:worklistProfileTwitterIds){ - allLoop++; - loopId++; - String counter = " ( " + loopId + " from " + number + " ) [" + allLoop + "] "; + String counter = " ( " + loopId + " from " + number + " ) "; log.debug(msg + counter); TwitterProfile userProfile = null; try { + log.debug(msg+"### twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter); userProfile = twitterApiService.getUserProfileForTwitterId(userProfileTwitterId); } catch (RateLimitExceededException e) { - log.error(msg + counter+ "twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") ",e); + log.error(msg + "### ERROR: twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter,e); } if(userProfile != null){ TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); userProfileList.add(userMsg); } + log.debug(msg + "### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); try { Thread.sleep(millisToWaitBetweenTwoApiCalls); } catch (InterruptedException e) { diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java index c886dba8..8e937de6 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java @@ -30,9 +30,16 @@ public class TwitterApiServiceImpl implements TwitterApiService { public List findTweetsForSearchQuery() { String msg = MSG+"findTweetsForSearchQuery: "; log.debug(msg); - List fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets(); + List fetchedTweets; + try { + fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets(); + } catch (Exception e) { + fetchedTweets = new ArrayList<>(); + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; - if(fetchedTweets == null){ + if(fetchedTweets.size()==0){ log.debug(msg+" result.size: 0"); return new ArrayList<>(); } else { @@ -45,7 +52,14 @@ public List findTweetsForSearchQuery() { public Tweet findOneTweetById(long id) { String msg = MSG+"findOneTweetById: "+id; log.debug(msg); - Tweet result = getTwitterProxy().timelineOperations().getStatus(id); + Tweet result; + try { + result = getTwitterProxy().timelineOperations().getStatus(id); + } catch (Exception e){ + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" Id: "+result.getId()); return result; @@ -55,7 +69,14 @@ public Tweet findOneTweetById(long id) { public List getUserProfilesForTwitterIds(long... userProfileTwitterIds) { String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterIds; log.debug(msg); - List result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds); + List result; + try { + result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds); + } catch (Exception e){ + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" size: "+result.size()); return result; @@ -65,7 +86,14 @@ public List getUserProfilesForTwitterIds(long... userProfileTwit public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) { String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterId; log.debug(msg); - TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId); + TwitterProfile result; + try { + result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId); + } catch (Exception e) { + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" Id: "+result.getId()); log.debug(msg+" ScreenName: "+result.getScreenName()); @@ -77,8 +105,14 @@ public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) { public TwitterProfile getUserProfileForScreenName(String screenName) { String msg = MSG+"getUserProfileForScreenName: "+screenName; log.debug(msg); - TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(screenName); - log.debug(msg+result.getId()); + TwitterProfile result; + try { + result= getTwitterProxy().userOperations().getUserProfile(screenName); + } catch (Exception e) { + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" ScreenName: "+result.getScreenName()); log.debug(msg+" Name: "+result.getName()); @@ -89,7 +123,14 @@ public TwitterProfile getUserProfileForScreenName(String screenName) { public List findUsersFromDefinedList(String screenName,String fetchUserListName) { String msg = MSG+"findUsersFromDefinedList: "+fetchUserListName+" "; log.debug(msg); - List result = getTwitterProxy().listOperations().getListMembers(screenName,fetchUserListName); + List result; + try { + result = getTwitterProxy().listOperations().getListMembers(screenName, fetchUserListName); + } catch (Exception e) { + result = new ArrayList<>(); + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } log.debug(msg+" result.size: "+result.size()); return result; } From 8417566ae9974a24dcb36d9503ae8b2ce2af099b Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 15:17:43 +0200 Subject: [PATCH 10/11] working on #141 #142 #143 --- .../mq/endoint/impl/AsyncStartTaskImpl.java | 2 +- .../scheduled/mq/endoint/impl/StartTaskImpl.java | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java index a28321f6..2d986afa 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java @@ -51,7 +51,7 @@ public void fetchUsersFromDefinedUserList() { } private void send(TaskType taskType){ - String msg = "START Task "+taskType+"via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; + String msg = "START Task "+taskType+" via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; log.info(msg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create(msg, taskType,countedEntities); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index beb518f7..bd72224f 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -79,12 +79,12 @@ public User createImprintUser() { UserMessage msg = (UserMessage) o; long taskId = msg.getTaskId(); task = taskService.findById(taskId); - logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; taskService.done(logMsg, task, countedEntities); log.info(logMsg); return msg.getUser(); } else { - logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; taskService.finalError(task,logMsg,countedEntities); log.error(logMsg); return null; @@ -94,7 +94,7 @@ public User createImprintUser() { @Override public List createTestDataForUser() { TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_USER; - String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + String logMsg = "Start task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; log.info(logMsg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType, countedEntities); @@ -112,12 +112,12 @@ public List createTestDataForUser() { UserResultList result = (UserResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; taskService.done(logMsg, task, countedEntities); log.info(logMsg); return result.getUserList(); } else { - logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; taskService.finalError(task,logMsg,countedEntities); log.error(logMsg); return new ArrayList<>(); @@ -127,7 +127,7 @@ public List createTestDataForUser() { @Override public List createTestDataForTweets() { TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_TWEETS; - String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + String logMsg = "Start task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; log.info(logMsg); CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ by Scheduler ", taskType, countedEntities); @@ -145,12 +145,12 @@ public List createTestDataForTweets() { TweetResultList result = (TweetResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; taskService.done(logMsg, task, countedEntities); log.info(logMsg); return result.getTweetList(); } else { - logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; taskService.finalError(task,logMsg,countedEntities); log.error(logMsg); return new ArrayList<>(); From eab0403a6ec32383fd8dbc9d7d7c10a9713e6508 Mon Sep 17 00:00:00 2001 From: phasenraum2010 Date: Tue, 25 Jul 2017 15:53:41 +0200 Subject: [PATCH 11/11] fixed #141, fixed #142, fixed #143 --- .../scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java | 2 ++ src/main/resources/logback.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java index d4378598..70808cb0 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -75,12 +75,14 @@ public List splitMessage(Message message) { pageRequest = pageRequest.next(); } long number = worklistProfileTwitterIds.size(); + loopId = 0; int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); List userProfileList = new ArrayList<>(); for(Long userProfileTwitterId:worklistProfileTwitterIds){ String counter = " ( " + loopId + " from " + number + " ) "; log.debug(msg + counter); TwitterProfile userProfile = null; + loopId++; try { log.debug(msg+"### twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter); userProfile = twitterApiService.getUserProfileForTwitterId(userProfileTwitterId); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 44db7f4e..97327e3e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -14,7 +14,7 @@ - +