diff --git a/pom.xml b/pom.xml index c0f09ba3..3771e8ec 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.woehlke.twitterwall twitterwall2 - 1.0.15-SNAPSHOT + 1.0.16-SNAPSHOT twitterwall2 Twitterwall with spring:boot for heroku diff --git a/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java b/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java index 911762b4..bb8fb3b5 100644 --- a/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java +++ b/src/main/java/org/woehlke/twitterwall/ScheduledTasks.java @@ -8,10 +8,7 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.woehlke.twitterwall.conf.TwitterwallSchedulerProperties; -import org.woehlke.twitterwall.scheduled.service.facade.*; - -import java.text.SimpleDateFormat; -import java.util.Date; +import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; /** * Created by tw on 10.06.17. @@ -20,45 +17,50 @@ @Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = false) public class ScheduledTasks { - private static final Logger log = LoggerFactory.getLogger(ScheduledTasks.class); - - private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); - - private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; - - private void logEnv(String msg){ - log.info("===================================================================="); - log.info(msg); - log.info("===================================================================="); - log.info("twitterwall.scheduler.allowUpdateTweets = "+ twitterwallSchedulerProperties.getAllowUpdateTweets()); - log.info("twitterwall.scheduler.allowUpdateUserProfiles = "+ twitterwallSchedulerProperties.getAllowUpdateUserProfiles()); - log.info("twitterwall.scheduler.allowUpdateUserProfilesFromMention = "+ twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention()); - log.info("twitterwall.scheduler.allowFetchTweetsFromTwitterSearch = "+ twitterwallSchedulerProperties.getAllowFetchTweetsFromTwitterSearch()); - log.info("twitterwall.scheduler.skipFortesting = "+ twitterwallSchedulerProperties.getSkipFortesting()); - log.info("twitterwall.scheduler.herokuDbRowsLimit = "+ twitterwallSchedulerProperties.getHerokuDbRowsLimit()); - log.info("twitterwall.scheduler.fetchUserList.name = "+ twitterwallSchedulerProperties.getFetchUserList().getName()); - log.info("twitterwall.scheduler.fetchUserList.allow = "+ twitterwallSchedulerProperties.getFetchUserList().getAllow()); - log.info("===================================================================="); + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) + public void fetchTweetsFromTwitterSearch() { + String msg = "fetch Tweets From TwitterSearch "; + if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.fetchTweetsFromTwitterSearch(); + } } - private final FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch; - - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) + public void updateTweets() { + String msg = "update Tweets "; + if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ + startTask.updateTweets(); + } + } - private final UpdateTweets updateTweets; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) + public void updateUserProfiles() { + String msg = "update User Profiles "; + if(twitterwallSchedulerProperties.getAllowUpdateUserProfiles() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.updateUserProfiles(); + } + } - private final UpdateUserProfiles updateUserProfiles; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) + public void updateUserProfilesFromMentions(){ + String msg = "update User Profiles From Mentions"; + if(twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.updateUserProfilesFromMentions(); + } + } - private final UpdateUserProfilesFromMentions updateUserProfilesFromMentions; + @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) + public void fetchUsersFromDefinedUserList(){ + String msg = "fetch Users from Defined User List "; + if(twitterwallSchedulerProperties.getFetchUserList().getAllow() && !twitterwallSchedulerProperties.getSkipFortesting()) { + startTask.fetchUsersFromDefinedUserList(); + } + } @Autowired - public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerProperties, FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, UpdateTweets updateTweets, UpdateUserProfiles updateUserProfiles, UpdateUserProfilesFromMentions updateUserProfilesFromMentions) { + public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerProperties, StartTask startTask) { this.twitterwallSchedulerProperties = twitterwallSchedulerProperties; - this.fetchTweetsFromTwitterSearch = fetchTweetsFromTwitterSearch; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.updateTweets = updateTweets; - this.updateUserProfiles = updateUserProfiles; - this.updateUserProfilesFromMentions = updateUserProfilesFromMentions; + this.startTask = startTask; } private final static long EINE_MINUTE = 60 * 1000; @@ -79,177 +81,9 @@ public ScheduledTasks(TwitterwallSchedulerProperties twitterwallSchedulerPropert private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST = ZWOELF_STUNDEN; - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) - public void fetchTweetsFromTwitterSearch() { - String msg = "fetch Tweets From TwitterSearch "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ - log.info("START "+msg+": The time is now {}", dateFormat.format(new Date())); - try { - this.fetchTweetsFromTwitterSearch.fetchTweetsFromTwitterSearch(); - log.info("DONE "+msg+" (OK)"+": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMesg = e.getMessage(); - e.printStackTrace(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg +" : " + eMesg); - log.error("NOT DONE "+msg+" (NOK)");; - } catch (Exception e) { - msg += " (Exception) "; - String eMesg = e.getMessage(); - e.printStackTrace(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg +" : " + eMesg); - log.error("NOT DONE "+msg+" (NOK)");; - } - } - logEnv(msg); - } - - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) - public void updateTweets() { - String msg = "update Tweets "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateTweets() && !twitterwallSchedulerProperties.getSkipFortesting()){ - log.info("START "+msg + ": The time is now {}", dateFormat.format(new Date())); - try { - this.updateTweets.updateTweets(); - log.info("DONE "+msg+" (OK)"+": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - log.warn(msg + e.getMessage()); - Throwable t = e.getCause(); - e.printStackTrace(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn("NOT DONE "+msg+" (NOK) {}", dateFormat.format(new Date()));; - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn("NOT DONE "+msg+" (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } - - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) - public void updateUserProfiles() { - String msg = "update User Profiles "; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateUserProfiles() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - //this.updateUserProfiles.updateUserProfiles(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private static final Logger log = LoggerFactory.getLogger(ScheduledTasks.class); - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) - public void updateUserProfilesFromMentions(){ - String msg = "update User Profiles From Mentions"; - logEnv(msg); - if(twitterwallSchedulerProperties.getAllowUpdateUserProfilesFromMention() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - //this.updateUserProfilesFromMentions.updateUserProfilesFromMentions(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private final TwitterwallSchedulerProperties twitterwallSchedulerProperties; - //@Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) - public void fetchUsersFromDefinedUserList(){ - String msg = "fetch Users from Defined User List "; - logEnv(msg); - if(twitterwallSchedulerProperties.getFetchUserList().getAllow() && !twitterwallSchedulerProperties.getSkipFortesting()) { - log.info("START " + msg + ": The time is now {}", dateFormat.format(new Date())); - try { - this.fetchUsersFromDefinedUserList.fetchUsersFromDefinedUserList(); - log.info("DONE " + msg + " (OK)" + ": The time is now {}", dateFormat.format(new Date())); - } catch (RuntimeException e) { - msg += " (RuntimeException) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.warn(msg + eMsg); - log.warn(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } catch (Exception e) { - msg += " (Exception) "; - String eMsg = e.getMessage(); - Throwable t = e.getCause(); - while(t != null){ - log.warn(msg + t.getMessage()); - t = t.getCause(); - } - log.error(msg + eMsg); - log.error(msg + " NOT DONE " + msg + " (NOK) {}", dateFormat.format(new Date())); - } - } - logEnv(msg); - } + private final StartTask startTask; } diff --git a/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java b/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java index 06eb24db..d2227f27 100644 --- a/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java +++ b/src/main/java/org/woehlke/twitterwall/conf/TwitterProperties.java @@ -27,6 +27,9 @@ public class TwitterProperties { @NotNull private Integer pageSize; + @NotNull + private Integer millisToWaitBetweenTwoApiCalls; + @NotNull private String searchQuery; @@ -77,4 +80,12 @@ public String getSearchQuery() { public void setSearchQuery(String searchQuery) { this.searchQuery = searchQuery; } + + public Integer getMillisToWaitBetweenTwoApiCalls() { + return millisToWaitBetweenTwoApiCalls; + } + + public void setMillisToWaitBetweenTwoApiCalls(Integer millisToWaitBetweenTwoApiCalls) { + this.millisToWaitBetweenTwoApiCalls = millisToWaitBetweenTwoApiCalls; + } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java index 537fbe6f..91b024fe 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/ApplicationController.java @@ -20,7 +20,6 @@ public class ApplicationController { @RequestMapping(path="/domain/count") public String domainCount(Model model) { String msg = "/application/domain/count: "; - controllerHelper.logEnv(); String title = "Counted Entities"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.DATABASE.toString(); @@ -33,20 +32,13 @@ public String domainCount(Model model) { @RequestMapping(path="/management") public String dmanagementPage(Model model) { String msg = "/application/domain/count: "; - controllerHelper.logEnv(); String title = "Application Management"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.DATABASE.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - //CountedEntities countedEntities =this.countedEntitiesService.countAll(); - //model.addAttribute("countedEntities", countedEntities); - return "application/management"; } - - - private final CountedEntitiesService countedEntitiesService; - + @Autowired public ApplicationController(CountedEntitiesService countedEntitiesService, ControllerHelper controllerHelper, TwitterProperties twitterProperties) { this.countedEntitiesService = countedEntitiesService; @@ -54,6 +46,8 @@ public ApplicationController(CountedEntitiesService countedEntitiesService, Cont this.twitterProperties = twitterProperties; } + private final CountedEntitiesService countedEntitiesService; + private final ControllerHelper controllerHelper; private final TwitterProperties twitterProperties; diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java index 1dcb9a8e..af9140a1 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/CountedEntitiesController.java @@ -88,7 +88,6 @@ public String domainCountUserprofile2Url(Model model) { } private void setUpThisPage(String title,Model model){ - controllerHelper.logEnv(); String subtitle = "Counted Entities"; String symbol = Symbols.DATABASE.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java index 4e79827d..8b591d3c 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/HashTagController.java @@ -14,9 +14,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; +import org.woehlke.twitterwall.frontend.controller.common.HashTagsOverviewHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; -import org.woehlke.twitterwall.frontend.model.HashTagCounted; +import org.woehlke.twitterwall.frontend.model.HashTagOverview; import org.woehlke.twitterwall.oodm.entities.Tweet; import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.entities.HashTag; @@ -24,8 +25,6 @@ import org.woehlke.twitterwall.oodm.service.UserService; import org.woehlke.twitterwall.oodm.service.HashTagService; -import java.util.ArrayList; -import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,7 +40,6 @@ public class HashTagController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "HashTag"; String symbol = Symbols.HASHTAG.toString(); @@ -60,7 +58,6 @@ public String hashTagFromTweetsAndUsers( Model model) { String msg = "/hashtag/" + text + " "; - controllerHelper.logEnv(); Pattern p = Pattern.compile(HASHTAG_TEXT_PATTERN); Matcher m = p.matcher(text); if (m.matches()) { @@ -97,52 +94,13 @@ public String hashTagFromTweetsAndUsers( @RequestMapping(path="/overview") public String hashTagsOverview(Model model) { String msg = "/hashtag/overview "; - controllerHelper.logEnv(); String title = "HashTags"; String subtitle = twitterProperties.getSearchQuery(); String symbol = Symbols.HASHTAG.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - List hashTagsTweets = new ArrayList<>(); - List hashTagsUsers = new ArrayList<>(); - Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterwallFrontendProperties.getPageSize()); - boolean hasNext = true; - while(hasNext) { - Page myPage = hashTagService.getAll(pageRequest); - for (HashTag hashTag : myPage.getContent()) { - Pageable pageRequestTeets = new PageRequest(0, 1); - Page tweets = tweetService.findTweetsForHashTag(hashTag, pageRequestTeets); - String myMSg = msg + " tweetService.findTweetsForHashTag= " + hashTag.getText(); - if (tweets == null) { - log.debug(myMSg + " result: null"); - } else { - long numberTweets = tweets.getTotalElements(); - log.debug(myMSg + " result: numberTweets=" + numberTweets); - if (numberTweets > 0) { - HashTagCounted c = new HashTagCounted(numberTweets, hashTag.getText()); - hashTagsTweets.add(c); - } - } - Pageable pageRequestUsers = new PageRequest(0, 1); - Page users = userService.getUsersForHashTag(hashTag, pageRequestUsers); - myMSg = msg + " userService.getUsersForHashTag= " + hashTag.getText(); - if (users == null) { - log.debug(myMSg + " result: null"); - } else { - long numberUsers = users.getTotalElements(); //userService.countUsersForHashTag(hashTag.getText()); - log.debug(myMSg + " result: numberUsers=" + numberUsers); - if (numberUsers > 0) { - HashTagCounted c = new HashTagCounted(numberUsers, hashTag.getText()); - hashTagsUsers.add(c); - } - } - } - hasNext = myPage.hasNext(); - if(hasNext){ - pageRequest=myPage.nextPageable(); - } - } - model.addAttribute("hashTagsTweets", hashTagsTweets); - model.addAttribute("hashTagsUsers", hashTagsUsers); + HashTagOverview overview = hashTagsOverviewHelper.getHashTagOverview(); + model.addAttribute("hashTagsTweets", overview.getHashTagsTweets()); + model.addAttribute("hashTagsUsers", overview.getHashTagsUsers()); return "hashtag/overview"; } @@ -160,13 +118,16 @@ public String hashTagsOverview(Model model) { private final ControllerHelper controllerHelper; + private final HashTagsOverviewHelper hashTagsOverviewHelper; + @Autowired - public HashTagController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, HashTagService hashTagService, TweetService tweetService, UserService userService, ControllerHelper controllerHelper) { + public HashTagController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, HashTagService hashTagService, TweetService tweetService, UserService userService, ControllerHelper controllerHelper, HashTagsOverviewHelper hashTagsOverviewHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.twitterProperties = twitterProperties; this.hashTagService = hashTagService; this.tweetService = tweetService; this.userService = userService; this.controllerHelper = controllerHelper; + this.hashTagsOverviewHelper = hashTagsOverviewHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java index 1769bf54..e809d08b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/MediaController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -26,7 +25,6 @@ public class MediaController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Media"; String symbol = Symbols.DATABASE.toString(); @@ -41,16 +39,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; @Autowired - public MediaController(MediaService mediaService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public MediaController(MediaService mediaService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.mediaService = mediaService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java index f3edb8ed..67d73038 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/MentionController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -25,7 +24,6 @@ public class MentionController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Mention"; String symbol = Symbols.DATABASE.toString(); @@ -38,16 +36,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final MentionService mentionService; private final ControllerHelper controllerHelper; @Autowired - public MentionController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, MentionService mentionService, ControllerHelper controllerHelper) { + public MentionController(TwitterwallFrontendProperties twitterwallFrontendProperties, MentionService mentionService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.mentionService = mentionService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java index b1f991c4..0cc2e8ff 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/PagesController.java @@ -29,14 +29,11 @@ public ModelAndView index(Model model) { @RequestMapping("/imprint") public String imprint(Model model) { log.info("-----------------------------------------"); - controllerHelper.logEnv(); String symbol = Symbols.IMPRINT.toString(); String title = "Impressum"; String subtitle = twitterwallFrontendProperties.getImprintSubtitle(); model = controllerHelper.setupPage(model, title, subtitle, symbol); User user = startTask.createImprintUser(); - //String screenName = twitterwallFrontendProperties.getImprintScreenName(); - //User user = createTestData.addUserForScreenName(screenName); model.addAttribute("user", user); log.info("-----------------------------------------"); return "imprint"; @@ -46,17 +43,11 @@ public String imprint(Model model) { private final TwitterwallFrontendProperties twitterwallFrontendProperties; - //private final TwitterProperties twitterProperties; - - //private final CreateTestData createTestData; - private final StartTask startTask; @Autowired public PagesController(TwitterwallFrontendProperties twitterwallFrontendProperties, StartTask startTask, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - //this.twitterProperties = twitterProperties; - //this.createTestData = createTestData; this.startTask = startTask; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java index 8e51b286..6aacdb1b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskController.java @@ -5,13 +5,11 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -19,9 +17,7 @@ import org.woehlke.twitterwall.oodm.entities.TaskHistory; import org.woehlke.twitterwall.oodm.service.TaskHistoryService; import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.service.facade.FetchTweetsFromTwitterSearch; -import org.woehlke.twitterwall.scheduled.service.facade.FetchUsersFromDefinedUserList; -import org.woehlke.twitterwall.scheduled.service.facade.UpdateTweets; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; /** * Created by tw on 11.07.17. @@ -33,7 +29,6 @@ public class TaskController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { String msg = "/task/all: "; - controllerHelper.logEnv(); String title = "Tasks"; String subtitle = "List aller Tasks"; String symbol = Symbols.TASK.toString(); @@ -47,15 +42,12 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp @RequestMapping(path="/{id}") public String getTaskById( @RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, - //@PathVariable long id, Model model) { @PathVariable("id") Task task, Model model) { String msg = "/task/ "+task.getId(); - controllerHelper.logEnv(); String title = "Tasks"; String subtitle = "List aller TasksHistory für Task"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - //Task oneTask = taskService.findById(id); Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize()); Page taskHistoryList = taskHistoryService.findByTask(task,pageRequest); model.addAttribute("task",task); @@ -66,113 +58,74 @@ public String getTaskById( @RequestMapping(path="/scheduled/tweets/fetch") public String fetchTweetsFromTwitterSearchRequest(Model model) { String msg = "/scheduled/tweets/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.fetchTweetsFromTwitterSearch(); + mqAsyncStartTask.fetchTweetsFromTwitterSearch(); return "scheduled/taskStarted"; } @RequestMapping(path="/scheduled/tweets/update") public String updateTweetsRequest(Model model) { String msg = "/scheduled/tweets/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/tweets/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateTweets(); + mqAsyncStartTask.updateTweets(); return "scheduled/taskStarted"; } @RequestMapping(path="/scheduled/users/list/fetch") public String fetchUsersFromDefinedUserListRequest(Model model){ String msg = "/scheduled/users/list/fetch"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/list/fetch"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.fetchUsersFromDefinedUserList(); + mqAsyncStartTask.fetchUsersFromDefinedUserList(); return "scheduled/taskStarted"; } @RequestMapping(path="/scheduled/users/mentions/update") public String updateUserProfilesFromMentionsRequest(Model model){ String msg = "/scheduled/users/mentions/update"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/mentions/update"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateUserProfilesFromMentions(); + mqAsyncStartTask.updateUserProfilesFromMentions(); return "scheduled/taskStarted"; } @RequestMapping(path="/scheduled/users/update") public String updateUserProfilesRequest(Model model) { String msg = "/scheduled/users/update"; - controllerHelper.logEnv(); String title = "Scheduled Task started"; String subtitle = "/scheduled/users/update"; String symbol = Symbols.TASK.toString(); model = controllerHelper.setupPage(model,title,subtitle,symbol); - this.updateUserProfiles(); + mqAsyncStartTask.updateUserProfiles(); return "scheduled/taskStarted"; } - @Async - protected void fetchTweetsFromTwitterSearch() { - fetchTweetsFromTwitterSearch.fetchTweetsFromTwitterSearch();; - } - - @Async - protected void updateTweets() { - updateTweets.updateTweets(); - } - - @Async - protected void fetchUsersFromDefinedUserList(){ - fetchUsersFromDefinedUserList.fetchUsersFromDefinedUserList(); - } - - @Async - protected void updateUserProfilesFromMentions(){ - - } - - @Async - protected void updateUserProfiles() { - - } - private final TaskService taskService; private final TaskHistoryService taskHistoryService; - private final FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch; - - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; - - private final UpdateTweets updateTweets; - private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; + private final AsyncStartTask mqAsyncStartTask; + @Autowired - public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, FetchTweetsFromTwitterSearch fetchTweetsFromTwitterSearch, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, UpdateTweets updateTweets, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public TaskController(TaskService taskService, TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper, AsyncStartTask mqAsyncStartTask) { this.taskService = taskService; this.taskHistoryService = taskHistoryService; - this.fetchTweetsFromTwitterSearch = fetchTweetsFromTwitterSearch; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.updateTweets = updateTweets; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; + this.mqAsyncStartTask = mqAsyncStartTask; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java index 67eb257e..6091de54 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TaskHistoryController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -27,7 +26,6 @@ public class TaskHistoryController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "TaskHistory"; String symbol = Symbols.DATABASE.toString(); @@ -42,15 +40,12 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; @Autowired - public TaskHistoryController(TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public TaskHistoryController(TaskHistoryService taskHistoryService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.taskHistoryService = taskHistoryService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java index 4c63670d..66269e30 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TestController.java @@ -6,7 +6,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; @@ -17,9 +16,8 @@ import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.oodm.entities.User; import org.woehlke.twitterwall.oodm.service.UserService; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; -import org.woehlke.twitterwall.scheduled.service.facade.CreateTestData; -import org.woehlke.twitterwall.scheduled.service.facade.FetchUsersFromDefinedUserList; /** @@ -32,12 +30,11 @@ public class TestController { @RequestMapping("/getTestData") public String getTestData(Model model) { - controllerHelper.logEnv(); model = controllerHelper.setupPage(model,"Test Data Tweets",twitterProperties.getSearchQuery(),Symbols.GET_TEST_DATA.toString()); String msg = "/getTestData : "; if(twitterwallFrontendProperties.getContextTest()){ - model.addAttribute("latestTweets", startTask.createTestDataForTweets()); - model.addAttribute("users", startTask.createTestDataForUser()); + model.addAttribute("latestTweets", mqStartTask.createTestDataForTweets()); + model.addAttribute("users", mqStartTask.createTestDataForUser()); } else { model.addAttribute("latestTweets",null); model.addAttribute("users",null); @@ -49,7 +46,9 @@ public String getTestData(Model model) { public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize()); String msg = "getOnListRenew: "; - this.startOnListRenew(); + log.info(msg+"START startTask.fetchUsersFromDefinedUserList: "); + mqAsyncStartTask.fetchUsersFromDefinedUserList(); + log.info(msg+"DONE startTask.fetchUsersFromDefinedUserList: "); log.info(msg+"START userService.findOnList(): "); Page usersOnList = userService.getOnList(pageRequest); log.info(msg+"DONE userService.findOnList(): "); @@ -64,34 +63,24 @@ public String getOnListRenew(@RequestParam(name= "page" ,defaultValue=""+ Contro private final UserService userService; - private final FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList; - - private final CreateTestData createTestData; - private final ControllerHelper controllerHelper; private final TwitterwallFrontendProperties twitterwallFrontendProperties; private final TwitterProperties twitterProperties; - private final StartTask startTask; + private final AsyncStartTask mqAsyncStartTask; + + private final StartTask mqStartTask; @Autowired - public TestController(UserService userService, FetchUsersFromDefinedUserList fetchUsersFromDefinedUserList, CreateTestData createTestData, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, StartTask startTask) { + public TestController(UserService userService, ControllerHelper controllerHelper, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, AsyncStartTask mqAsyncStartTask, StartTask mqStartTask) { this.userService = userService; - this.fetchUsersFromDefinedUserList = fetchUsersFromDefinedUserList; - this.createTestData = createTestData; this.controllerHelper = controllerHelper; this.twitterwallFrontendProperties = twitterwallFrontendProperties; this.twitterProperties = twitterProperties; - this.startTask = startTask; + this.mqAsyncStartTask = mqAsyncStartTask; + this.mqStartTask = mqStartTask; } - @Async - protected void startOnListRenew(){ - String msg = "startOnListRenew: "; - log.info(msg+"START startTask.fetchUsersFromDefinedUserList: "); - startTask.fetchUsersFromDefinedUserList(); - log.info(msg+"DONE startTask.fetchUsersFromDefinedUserList: "); - } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java index 1d4547f4..a86ebdbe 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TickerSymbolController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.Symbols; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; @@ -27,7 +26,6 @@ public class TickerSymbolController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "TickerSymbol"; String symbol = Symbols.DATABASE.toString(); @@ -40,16 +38,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelp private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final TickerSymbolService tickerSymbolService; private final ControllerHelper controllerHelper; @Autowired - public TickerSymbolController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, TickerSymbolService tickerSymbolService, ControllerHelper controllerHelper) { + public TickerSymbolController(TwitterwallFrontendProperties twitterwallFrontendProperties, TickerSymbolService tickerSymbolService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.tickerSymbolService = tickerSymbolService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java index 8f820b0f..73858afc 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/TweetController.java @@ -28,7 +28,6 @@ public class TweetController { @RequestMapping("/all") public String getLatestTweets(@RequestParam(name= "page" ,defaultValue=""+ ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model) { - controllerHelper.logEnv(); model = controllerHelper.setupPage(model,"Tweets",twitterProperties.getSearchQuery(),Symbols.HOME.toString()); Pageable pageRequest = new PageRequest(page, twitterwallFrontendProperties.getPageSize(), Sort.Direction.DESC,"createdAt"); Page latest = tweetService.getAll(pageRequest); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java index dc8abe5c..2348f34e 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlCacheController.java @@ -27,7 +27,6 @@ public class UrlCacheController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "UrlCache"; String symbol = Symbols.DATABASE.toString(); @@ -42,8 +41,6 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelpe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final UrlCacheService urlCacheService; @Autowired @@ -51,6 +48,5 @@ public UrlCacheController(UrlCacheService urlCacheService, ControllerHelper cont this.urlCacheService = urlCacheService; this.controllerHelper = controllerHelper; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java index 0b975c72..bdcda11c 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UrlController.java @@ -9,7 +9,6 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; @@ -27,7 +26,6 @@ public class UrlController { @RequestMapping(path="/all") public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelper.FIRST_PAGE_NUMBER) int page, Model model){ - controllerHelper.logEnv(); String subtitle = "all"; String title = "Url"; String symbol = Symbols.DATABASE.toString(); @@ -40,16 +38,13 @@ public String getAll(@RequestParam(name= "page" ,defaultValue=""+ControllerHelpe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final UrlService urlService; private final ControllerHelper controllerHelper; @Autowired - public UrlController(TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, UrlService urlService, ControllerHelper controllerHelper) { + public UrlController(TwitterwallFrontendProperties twitterwallFrontendProperties, UrlService urlService, ControllerHelper controllerHelper) { this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.urlService = urlService; this.controllerHelper = controllerHelper; } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java index 76d4bce3..12276957 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/UserController.java @@ -12,7 +12,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.woehlke.twitterwall.conf.TwitterProperties; import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; import org.woehlke.twitterwall.frontend.controller.common.ControllerHelper; import org.woehlke.twitterwall.frontend.controller.common.Symbols; @@ -113,18 +112,15 @@ public String getOnList(@RequestParam(name= "page" ,defaultValue=""+ControllerHe private final TwitterwallFrontendProperties twitterwallFrontendProperties; - private final TwitterProperties twitterProperties; - private final ControllerHelper controllerHelper; private static String subtitle = "Users"; @Autowired - public UserController(UserService userService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties, TwitterProperties twitterProperties, ControllerHelper controllerHelper) { + public UserController(UserService userService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties, ControllerHelper controllerHelper) { this.userService = userService; this.tweetService = tweetService; this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.twitterProperties = twitterProperties; this.controllerHelper = controllerHelper; } } diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java index 15dd4976..54df3a47 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/ControllerHelper.java @@ -10,8 +10,6 @@ public interface ControllerHelper { int FIRST_PAGE_NUMBER = 0; - void logEnv(); - ModelAndView setupPage(ModelAndView mav, String title, String subtitle, String symbol); Model setupPage(Model model, String title, String subtitle, String symbol); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java new file mode 100644 index 00000000..eb0e303c --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/HashTagsOverviewHelper.java @@ -0,0 +1,8 @@ +package org.woehlke.twitterwall.frontend.controller.common; + +import org.woehlke.twitterwall.frontend.model.HashTagOverview; + +public interface HashTagsOverviewHelper { + + HashTagOverview getHashTagOverview(); +} diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java index 618cd28f..31c9605b 100644 --- a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/ControllerHelperImpl.java @@ -20,18 +20,6 @@ @Transactional(propagation = Propagation.REQUIRES_NEW, readOnly = false) public class ControllerHelperImpl implements ControllerHelper { - public void logEnv(){ - log.info("--------------------------------------------------------------------"); - log.info("twitter.searchQuery = "+ twitterProperties.getSearchQuery()); - log.info("twitterwall.frontend.menuAppName = "+ twitterwallFrontendProperties.getMenuAppName()); - log.info("twitterwall.frontend.infoWebpage = "+ twitterwallFrontendProperties.getInfoWebpage()); - log.info("twitterwall.frontend.theme = "+ twitterwallFrontendProperties.getTheme()); - log.info("twitterwall.frontend.contextTest = "+ twitterwallFrontendProperties.getContextTest()); - log.info("twitterwall.frontend.imprintScreenName = "+ twitterwallFrontendProperties.getImprintScreenName()); - log.info("twitterwall.frontend.idGoogleAnalytics = "+ twitterwallFrontendProperties.getIdGoogleAnalytics()); - log.info("--------------------------------------------------------------------"); - } - private Page setupPage(Page page, String title, String subtitle, String symbol) { page.setTitle(title); page.setSubtitle(subtitle); @@ -49,7 +37,6 @@ private Page setupPage(Page page, String title, String subtitle, String symbol) } else { page.setGoogleAnalyticScriptHtml(""); } - logEnv(); log.info("--------------------------------------------------------------------"); log.info("setupPage = "+page.toString()); log.info("--------------------------------------------------------------------"); diff --git a/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java new file mode 100644 index 00000000..f302fe8e --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/frontend/controller/common/impl/HashTagsOverviewHelperImpl.java @@ -0,0 +1,90 @@ +package org.woehlke.twitterwall.frontend.controller.common.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; +import org.woehlke.twitterwall.frontend.controller.common.HashTagsOverviewHelper; +import org.woehlke.twitterwall.frontend.model.HashTagCounted; +import org.woehlke.twitterwall.frontend.model.HashTagOverview; +import org.woehlke.twitterwall.oodm.entities.HashTag; +import org.woehlke.twitterwall.oodm.entities.Tweet; +import org.woehlke.twitterwall.oodm.entities.User; +import org.woehlke.twitterwall.oodm.service.HashTagService; +import org.woehlke.twitterwall.oodm.service.TweetService; +import org.woehlke.twitterwall.oodm.service.UserService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component +public class HashTagsOverviewHelperImpl implements HashTagsOverviewHelper { + + + public HashTagsOverviewHelperImpl(UserService userService, HashTagService hashTagService, TweetService tweetService, TwitterwallFrontendProperties twitterwallFrontendProperties) { + this.userService = userService; + this.hashTagService = hashTagService; + this.tweetService = tweetService; + this.twitterwallFrontendProperties = twitterwallFrontendProperties; + } + + @Override + public HashTagOverview getHashTagOverview() { + String msg = "getHashTagOverview"; + List hashTagsTweets = new ArrayList<>(); + List hashTagsUsers = new ArrayList<>(); + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterwallFrontendProperties.getPageSize()); + boolean hasNext = true; + while(hasNext) { + Page myPage = hashTagService.getAll(pageRequest); + for (HashTag hashTag : myPage.getContent()) { + Pageable pageRequestTeets = new PageRequest(0, 1); + Page tweets = tweetService.findTweetsForHashTag(hashTag, pageRequestTeets); + String myMSg = msg + " tweetService.findTweetsForHashTag= " + hashTag.getText(); + if (tweets == null) { + log.debug(myMSg + " result: null"); + } else { + long numberTweets = tweets.getTotalElements(); + log.debug(myMSg + " result: numberTweets=" + numberTweets); + if (numberTweets > 0) { + HashTagCounted c = new HashTagCounted(numberTweets, hashTag.getText()); + hashTagsTweets.add(c); + } + } + Pageable pageRequestUsers = new PageRequest(0, 1); + Page users = userService.getUsersForHashTag(hashTag, pageRequestUsers); + myMSg = msg + " userService.getUsersForHashTag= " + hashTag.getText(); + if (users == null) { + log.debug(myMSg + " result: null"); + } else { + long numberUsers = users.getTotalElements(); //userService.countUsersForHashTag(hashTag.getText()); + log.debug(myMSg + " result: numberUsers=" + numberUsers); + if (numberUsers > 0) { + HashTagCounted c = new HashTagCounted(numberUsers, hashTag.getText()); + hashTagsUsers.add(c); + } + } + } + hasNext = myPage.hasNext(); + if(hasNext){ + pageRequest=myPage.nextPageable(); + } + } + return new HashTagOverview(hashTagsTweets,hashTagsUsers); + } + + private static final Logger log = LoggerFactory.getLogger(HashTagsOverviewHelperImpl.class); + + private final UserService userService; + + private final HashTagService hashTagService; + + private final TweetService tweetService; + + private final TwitterwallFrontendProperties twitterwallFrontendProperties; +} diff --git a/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java b/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java index 8f4dd8e6..bf59d71e 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/repositories/MentionRepository.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.oodm.repositories; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Repository; import org.woehlke.twitterwall.oodm.entities.Mention; import org.woehlke.twitterwall.oodm.repositories.common.DomainRepository; @@ -17,4 +19,5 @@ public interface MentionRepository extends DomainRepository,MentionRepo Mention findByScreenName(String screenName); + Page findAllByUserNull(Pageable pageRequest); } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java index 32495a08..cfd056fe 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/MentionService.java @@ -1,5 +1,7 @@ package org.woehlke.twitterwall.oodm.service; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Mention; import org.woehlke.twitterwall.oodm.service.common.DomainServiceWithIdTwitter; @@ -13,4 +15,6 @@ public interface MentionService extends DomainServiceWithScreenName,DomainServiceWithIdTwitter,DomainServiceWithTask { Mention createProxyMention(Mention mention, Task task); + + Page getAllWithoutPersistentUser(Pageable pageRequest); } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java index fbf6d526..ea8be218 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/TaskService.java @@ -31,4 +31,9 @@ public interface TaskService extends DomainObjectMinimalService { Task error(Task task, String msg,CountedEntities countedEntities); Task start(Task task,CountedEntities countedEntities); + + Task finalError(Task task, String s, CountedEntities countedEntities); + + Task done(String logMsg, Task task, CountedEntities countedEntities); + } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java b/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java index 21e7e591..b6645732 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/TweetService.java @@ -16,12 +16,8 @@ */ public interface TweetService extends DomainServiceWithIdTwitter,DomainServiceWithTask { - //Page findLatestTweets(Pageable pageRequest); - Page findTweetsForHashTag(HashTag hashtag, Pageable pageRequest); - //long countTweetsForHashTag(String hashtagText); - Page findTweetsForUser(User user, Pageable pageRequest); Page findAllTwitterIds(Pageable pageRequest); diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java index 50420a2b..63998d92 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/MentionServiceImpl.java @@ -65,4 +65,9 @@ public Mention createProxyMention(Mention mention, Task task) { mention = mentionRepository.save(mention); return mention; } + + @Override + public Page getAllWithoutPersistentUser(Pageable pageRequest) { + return mentionRepository.findAllByUserNull(pageRequest); + } } diff --git a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java index 0a8684f3..aa1dcc35 100644 --- a/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/oodm/service/impl/TaskServiceImpl.java @@ -72,7 +72,7 @@ public Task done(Task task,CountedEntities countedEntities) { task.setTaskStatus(TaskStatus.FINISHED); task.setTimeLastUpdate(new Date()); task = taskRepository.save(task); - TaskHistory event = new TaskHistory("done",task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); + TaskHistory event = new TaskHistory("DONE ",task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); event.setIdTask(task.getId()); event.setTask(task); event = taskHistoryRepository.save(event); @@ -175,7 +175,33 @@ public Task start(Task task,CountedEntities countedEntities) { task.setTaskStatus(TaskStatus.RUNNING); task.setTimeLastUpdate(new Date()); task = taskRepository.save(task); - TaskHistory event = new TaskHistory("start",task.getTaskStatus(),TaskStatus.RUNNING,countedEntities); + TaskHistory event = new TaskHistory("START",task.getTaskStatus(),TaskStatus.RUNNING,countedEntities); + event.setIdTask(task.getId()); + event.setTask(task); + event = taskHistoryRepository.save(event); + log.debug(task.toString()); + return task; + } + + @Override + public Task finalError(Task task, String msg, CountedEntities countedEntities) { + task.setTaskStatus(TaskStatus.ERROR); + task.setTimeLastUpdate(new Date()); + task = taskRepository.save(task); + TaskHistory event = new TaskHistory("FINAL ERROR: "+msg,task.getTaskStatus(),TaskStatus.ERROR,countedEntities); + event.setIdTask(task.getId()); + event.setTask(task); + event = taskHistoryRepository.save(event); + log.debug(task.toString()); + return task; + } + + @Override + public Task done(String logMsg, Task task, CountedEntities countedEntities) { + task.setTaskStatus(TaskStatus.FINISHED); + task.setTimeLastUpdate(new Date()); + task = taskRepository.save(task); + TaskHistory event = new TaskHistory("DONE "+logMsg,task.getTaskStatus(),TaskStatus.FINISHED,countedEntities); event.setIdTask(task.getId()); event.setTask(task); event = taskHistoryRepository.save(event); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java new file mode 100644 index 00000000..f2d707f4 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/channel/SenderType.java @@ -0,0 +1,6 @@ +package org.woehlke.twitterwall.scheduled.mq.channel; + +public enum SenderType { + SEND_AND_WAIT_FOR_RESULT_SENDER, + FIRE_AND_FORGET_SENDER +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java new file mode 100644 index 00000000..0ccb44c2 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/AsyncStartTask.java @@ -0,0 +1,15 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +public interface AsyncStartTask { + + void updateTweets(); + + void updateUserProfiles(); + + void updateUserProfilesFromMentions(); + + void fetchTweetsFromTwitterSearch(); + + void fetchUsersFromDefinedUserList(); + +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java deleted file mode 100644 index 610f6566..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromDatabase.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint; - -import org.springframework.messaging.Message; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; - -public interface FetchDataFromDatabase { - - void updateTweets(Message mqMessage); - - void updateUserProfiles(Message mqMessage); - - void updateUserProfilesFromMentions(Message mqMessage); -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java deleted file mode 100644 index 4a0484b8..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/FetchDataFromRemoteTwitterApi.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint; - -import org.springframework.messaging.Message; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; - -public interface FetchDataFromRemoteTwitterApi { - - void fetchTweetsFromTwitterSearch(Message mqMessage); - - void fetchUsersFromDefinedUserList(Message mqMessage); - - void createTestDataForUser(Message mqMessage); -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java index 7a344693..495ccf74 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/StartTask.java @@ -5,17 +5,7 @@ import java.util.List; -public interface StartTask { - - void fetchTweetsFromTwitterSearch(); - - void updateTweets(); - - void updateUserProfiles(); - - void updateUserProfilesFromMentions(); - - void fetchUsersFromDefinedUserList(); +public interface StartTask extends AsyncStartTask { User createImprintUser(); diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java index 07a5145e..af391679 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/TweetFinisher.java @@ -7,5 +7,8 @@ import java.util.List; public interface TweetFinisher { + TweetResultList finish(Message> incomingMessageList); + + void finishAsnyc(Message> incomingMessageList); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java new file mode 100644 index 00000000..e96a1769 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateTweets.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; + +import java.util.List; + +public interface UpdateTweets { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java new file mode 100644 index 00000000..9b254cd9 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfiles.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; + +import java.util.List; + +public interface UpdateUserProfiles { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java new file mode 100644 index 00000000..cd2fc87b --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UpdateUserProfilesFromMentions.java @@ -0,0 +1,12 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint; + +import org.springframework.messaging.Message; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; + +import java.util.List; + +public interface UpdateUserProfilesFromMentions { + + List splitMessage(Message message); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java index 19b3c5cf..e1daaf0a 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/UserFinisher.java @@ -9,4 +9,6 @@ public interface UserFinisher { UserResultList finish(Message> incomingMessageList); + + void finishAsnyc(Message> incomingMessageList); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java new file mode 100644 index 00000000..2d986afa --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/AsyncStartTaskImpl.java @@ -0,0 +1,82 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.channel.ExecutorChannel; +import org.springframework.integration.core.MessagingTemplate; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.entities.parts.TaskType; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; +import org.woehlke.twitterwall.scheduled.mq.endoint.AsyncStartTask; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +@Component("mqAsyncStartTask") +public class AsyncStartTaskImpl implements AsyncStartTask { + + @Override + public void fetchTweetsFromTwitterSearch() { + TaskType taskType = TaskType.FETCH_TWEETS_FROM_TWITTER_SEARCH; + send(taskType); + } + + @Override + public void updateTweets() { + TaskType taskType = TaskType.UPDATE_TWEETS; + send(taskType); + } + + @Override + public void updateUserProfiles() { + TaskType taskType = TaskType.UPDATE_USER_PROFILES; + send(taskType); + } + + @Override + public void updateUserProfilesFromMentions() { + TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; + send(taskType); + } + + @Override + public void fetchUsersFromDefinedUserList() { + TaskType taskType = TaskType.FETCH_USERS_FROM_DEFINED_USER_LIST; + send(taskType); + } + + private void send(TaskType taskType){ + String msg = "START Task "+taskType+" via MQ by "+SenderType.FIRE_AND_FORGET_SENDER; + log.info(msg); + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create(msg, taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + mqTemplate.send(executorChannelForAsyncStart, mqMessage); + } + + @Autowired + public AsyncStartTaskImpl(TaskService taskService, CountedEntitiesService countedEntitiesService, ExecutorChannel executorChannelForAsyncStart) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + this.executorChannelForAsyncStart = executorChannelForAsyncStart; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private final ExecutorChannel executorChannelForAsyncStart; + + private static final Logger log = LoggerFactory.getLogger(AsyncStartTaskImpl.class); +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java deleted file mode 100644 index 2191993e..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromDatabaseImpl.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint.impl; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.Message; -import org.springframework.stereotype.Component; -import org.woehlke.twitterwall.oodm.entities.Task; -import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; -import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.mq.endoint.FetchDataFromDatabase; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; -import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; - -@Component("mqFetchDataFromDatabase") -public class FetchDataFromDatabaseImpl implements FetchDataFromDatabase { - - private static final Logger log = LoggerFactory.getLogger(FetchDataFromDatabaseImpl.class); - - private final TaskService taskService; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public FetchDataFromDatabaseImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { - this.taskService = taskService; - this.countedEntitiesService = countedEntitiesService; - } - - - private void react(String logMsg,Message mqMessage){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - TaskMessage receivedMessage = mqMessage.getPayload(); - Task task = taskService.findById(receivedMessage.getTaskId()); - taskService.start(task,countedEntities); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"received: "+mqMessage.getPayload().toString()); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - } - - @Override - public void updateTweets(Message mqMessage) { - String logMsg = "updateTweets: "; - react(logMsg,mqMessage); - } - - @Override - public void updateUserProfiles(Message mqMessage) { - String logMsg = "updateUserProfiles: "; - react(logMsg,mqMessage); - } - - @Override - public void updateUserProfilesFromMentions(Message mqMessage) { - String logMsg = "updateUserProfilesFromMentions: "; - react(logMsg,mqMessage); - } -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java deleted file mode 100644 index 0ce240ac..00000000 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/FetchDataFromRemoteTwitterApiImpl.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.woehlke.twitterwall.scheduled.mq.endoint.impl; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.messaging.Message; -import org.springframework.stereotype.Component; -import org.woehlke.twitterwall.conf.TwitterwallFrontendProperties; -import org.woehlke.twitterwall.oodm.entities.Task; -import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; -import org.woehlke.twitterwall.oodm.service.TaskService; -import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; -import org.woehlke.twitterwall.scheduled.mq.endoint.FetchDataFromRemoteTwitterApi; -import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; -import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; - -@Component("mqFetchDataFromRemoteTwitterApi") -public class FetchDataFromRemoteTwitterApiImpl implements FetchDataFromRemoteTwitterApi { - - private static final Logger log = LoggerFactory.getLogger(FetchDataFromRemoteTwitterApiImpl.class); - - private final TaskService taskService; - - private final TwitterApiService twitterApiService; - - private final TwitterwallFrontendProperties twitterwallFrontendProperties; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public FetchDataFromRemoteTwitterApiImpl(TaskService taskService, TwitterApiService twitterApiService, TwitterwallFrontendProperties twitterwallFrontendProperties, CountedEntitiesService countedEntitiesService) { - this.taskService = taskService; - this.twitterApiService = twitterApiService; - this.twitterwallFrontendProperties = twitterwallFrontendProperties; - this.countedEntitiesService = countedEntitiesService; - } - - private void react(String logMsg,Message mqMessage){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - TaskMessage receivedMessage = mqMessage.getPayload(); - Task task = taskService.findById(receivedMessage.getTaskId()); - taskService.start(task,countedEntities); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"received: "+mqMessage.getPayload().toString()); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - log.info(logMsg+"##############################################"); - } - - @Override - public void fetchTweetsFromTwitterSearch(Message mqMessage) { - String logMsg = "fetchTweetsFromTwitterSearch: "; - this.react(logMsg, mqMessage); - } - - @Override - public void fetchUsersFromDefinedUserList(Message mqMessage) { - String logMsg = "fetchUsersFromDefinedUserList: "; - this.react(logMsg, mqMessage); - } - - @Override - public void createTestDataForUser(Message mqMessage) { - String logMsg = "createTestDataForUser: "; - this.react(logMsg, mqMessage); - } - -} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java index 1603d389..bd72224f 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/StartTaskImpl.java @@ -1,11 +1,12 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.core.MessagingTemplate; import org.springframework.integration.support.MessageBuilder; import org.springframework.messaging.Message; import org.springframework.messaging.MessageChannel; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Tweet; @@ -13,6 +14,7 @@ import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; import org.woehlke.twitterwall.oodm.entities.parts.TaskType; import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.StartTask; import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; import org.woehlke.twitterwall.scheduled.mq.msg.TweetResultList; @@ -23,126 +25,45 @@ import java.util.ArrayList; import java.util.List; -@Component +@Component("mqStartTask") public class StartTaskImpl implements StartTask { - private final MessageChannel startTaskChannel; - - private final TaskService taskService; - - private final CountedEntitiesService countedEntitiesService; - - @Autowired - public StartTaskImpl(MessageChannel startTaskChannel, TaskService taskService, CountedEntitiesService countedEntitiesService) { - this.startTaskChannel = startTaskChannel; - this.taskService = taskService; - this.countedEntitiesService = countedEntitiesService; - } - - private void messageSender(final TaskType taskType) { - CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); - TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); - Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); - MessagingTemplate mqTemplate = new MessagingTemplate(); - mqTemplate.send(startTaskChannel, mqMessage); - } - - - private final static long EINE_MINUTE = 60 * 1000; - - private final static long FUENF_MINUTEN = 5 * EINE_MINUTE; - - private final static long EINE_STUNDE = 60 * EINE_MINUTE; - - private final static long ZWOELF_STUNDEN = 12 * EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS = EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER = ZWOELF_STUNDEN; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS = ZWOELF_STUNDEN; - - private final static long FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION = EINE_STUNDE; - - private final static long FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST = ZWOELF_STUNDEN; - - - /* - twitterwall.mq.starttask.time.fetchTweetsFromTwitterSearch - twitterwall.mq.starttask.time.updateTweets - twitterwall.mq.starttask.time.updateUserProfiles - twitterwall.mq.starttask.time.updateUserProfilesFromMentions - twitterwall.mq.starttask.time.fetchUsersFromDefinedUserList - */ - - @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_TWEETS) public void fetchTweetsFromTwitterSearch() { TaskType taskType = TaskType.FETCH_TWEETS_FROM_TWITTER_SEARCH; - sendAndReceive(taskType); + sendAndReceiveTweet(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_TWEETS) public void updateTweets() { TaskType taskType = TaskType.UPDATE_TWEETS; - messageSender(taskType); + sendAndReceiveTweet(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER) public void updateUserProfiles() { TaskType taskType = TaskType.UPDATE_USER_PROFILES; - messageSender(taskType); + sendAndReceiveUser(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_UPDATE_USER_BY_MENTION) public void updateUserProfilesFromMentions() { TaskType taskType = TaskType.UPDATE_USER_PROFILES_FROM_MENTIONS; - messageSender(taskType); - } - - private void sendAndReceive(TaskType taskType){ - CountedEntities countedEntities = countedEntitiesService.countAll(); - Task task = taskService.create("Start via MQ", taskType,countedEntities); - TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); - Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); - MessagingTemplate mqTemplate = new MessagingTemplate(); - Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); - Object o = returnedMessage.getPayload(); - countedEntities = countedEntitiesService.countAll(); - if( o instanceof UserMessage){ - UserMessage msg = (UserMessage) o; - long taskId = msg.getTaskId(); - task = taskService.findById(taskId); - taskService.done(task,countedEntities); - } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); - } + sendAndReceiveUser(taskType); } @Override - @Scheduled(fixedRate = FIXED_RATE_FOR_SCHEDULAR_FETCH_USER_LIST) public void fetchUsersFromDefinedUserList() { TaskType taskType = TaskType.FETCH_USERS_FROM_DEFINED_USER_LIST; - sendAndReceive(taskType); + sendAndReceiveUser(taskType); } @Override public User createImprintUser() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_CREATE_IMPRINT_USER; + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); Task task = taskService.create("Start via MQ", taskType,countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) @@ -158,67 +79,156 @@ public User createImprintUser() { UserMessage msg = (UserMessage) o; long taskId = msg.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return msg.getUser(); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return null; } } @Override public List createTestDataForUser() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_USER; - Task task = taskService.create("Start via MQ", taskType,countedEntities); + String logMsg = "Start task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create("Start via MQ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); - if( o instanceof UserResultList){ + if (o instanceof UserResultList) { UserResultList result = (UserResultList) o; long taskId = result.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return result.getUserList(); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return new ArrayList<>(); } } @Override public List createTestDataForTweets() { - CountedEntities countedEntities = countedEntitiesService.countAll(); TaskType taskType = TaskType.CONTROLLER_GET_TESTDATA_TWEETS; - Task task = taskService.create("Start via MQ", taskType,countedEntities); + String logMsg = "Start task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create("Start via MQ by Scheduler ", taskType, countedEntities); TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); Message mqMessage = MessageBuilder.withPayload(taskMessage) - .setHeader("task_id", task.getId()) - .setHeader("task_uid", task.getUniqueId()) - .setHeader("task_type", task.getTaskType()) - .build(); + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); MessagingTemplate mqTemplate = new MessagingTemplate(); Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); Object o = returnedMessage.getPayload(); countedEntities = countedEntitiesService.countAll(); - if( o instanceof TweetResultList){ + if (o instanceof TweetResultList) { TweetResultList result = (TweetResultList) o; - long taskId =result.getTaskId(); + long taskId = result.getTaskId(); task = taskService.findById(taskId); - taskService.done(task,countedEntities); + logMsg = "Sucessfully finished task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg, task, countedEntities); + log.info(logMsg); return result.getTweetList(); } else { - taskService.error(task,"Wrong type of returnedMessage",countedEntities); + logMsg = "Finished with Error: task "+taskType+" via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); return new ArrayList<>(); } } + + private void sendAndReceiveTweet(TaskType taskType){ + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create("Start via MQ by Scheduler ", taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); + Object o = returnedMessage.getPayload(); + countedEntities = countedEntitiesService.countAll(); + if( o instanceof TweetResultList){ + TweetResultList msg = (TweetResultList) o; + long taskId = msg.getTaskId(); + task = taskService.findById(taskId); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg,task,countedEntities); + } else { + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); + } + } + + private void sendAndReceiveUser(TaskType taskType){ + String logMsg = "Start task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + log.info(logMsg); + CountedEntities countedEntities = countedEntitiesService.countAll(); + Task task = taskService.create(logMsg, taskType,countedEntities); + TaskMessage taskMessage = new TaskMessage(task.getId(), taskType, task.getTimeStarted()); + Message mqMessage = MessageBuilder.withPayload(taskMessage) + .setHeader("task_id", task.getId()) + .setHeader("task_uid", task.getUniqueId()) + .setHeader("task_type", task.getTaskType()) + .build(); + MessagingTemplate mqTemplate = new MessagingTemplate(); + Message returnedMessage = mqTemplate.sendAndReceive(startTaskChannel, mqMessage); + Object o = returnedMessage.getPayload(); + countedEntities = countedEntitiesService.countAll(); + if( o instanceof UserMessage){ + UserMessage msg = (UserMessage) o; + long taskId = msg.getTaskId(); + task = taskService.findById(taskId); + logMsg = "Sucessfully finished task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER; + taskService.done(logMsg,task,countedEntities); + log.info(logMsg); + } else { + logMsg = "Finished with Error: task "+taskType+"via MQ by "+ SenderType.SEND_AND_WAIT_FOR_RESULT_SENDER+": Wrong type of returnedMessage"; + taskService.finalError(task,logMsg,countedEntities); + log.error(logMsg); + } + } + + private final MessageChannel startTaskChannel; + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private static final Logger log = LoggerFactory.getLogger(StartTaskImpl.class); + + @Autowired + public StartTaskImpl(MessageChannel startTaskChannel,TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.startTaskChannel = startTaskChannel; + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java index f330ede8..e2f3690f 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/TweetFinisherImpl.java @@ -1,11 +1,19 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.Message; import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.Tweet; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.TweetFinisher; import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; import org.woehlke.twitterwall.scheduled.mq.msg.TweetResultList; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; import java.util.ArrayList; import java.util.List; @@ -13,6 +21,7 @@ @Component("mqTweetFinisher") public class TweetFinisherImpl implements TweetFinisher { + @Override public TweetResultList finish(Message> incomingMessageList){ List resultList = new ArrayList<>(); long taskId = 0L; @@ -23,4 +32,39 @@ public TweetResultList finish(Message> incomingMessageLis TweetResultList result = new TweetResultList(taskId,resultList); return result; } + + @Override + public void finishAsnyc(Message> incomingMessageList) { + List userMessageList = incomingMessageList.getPayload(); + CountedEntities countedEntities = countedEntitiesService.countAll(); + if(incomingMessageList.getHeaders().containsKey("task_id")){ + long taskId = (Long) incomingMessageList.getHeaders().get( "task_id"); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(msgDone,task,countedEntities); + log.info(msgDone); + } else { + if(userMessageList.size()>0) { + long taskId = userMessageList.get(0).getTaskId(); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(task,countedEntities); + log.info(msgDone); + } else { + log.warn("finishAsnyc: No Tweets performed. via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER); + } + } + } + + @Autowired + public TweetFinisherImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private static final Logger log = LoggerFactory.getLogger(UserFinisherImpl.class); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java new file mode 100644 index 00000000..f5a82362 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateTweetsImpl.java @@ -0,0 +1,91 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.twitter.api.Tweet; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.oodm.service.TweetService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateTweets; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TweetFromTwitter; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateTweets") +public class UpdateTweetsImpl implements UpdateTweets { + + private final TwitterProperties twitterProperties; + + private final TweetService tweetService; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + public UpdateTweetsImpl(TwitterProperties twitterProperties, TweetService tweetService, TwitterApiService twitterApiService, TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.twitterProperties = twitterProperties; + this.tweetService = tweetService; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + CountedEntities countedEntities = countedEntitiesService.countAll(); + TaskMessage msgIn = message.getPayload(); + long taskId = msgIn.getTaskId(); + Task task = taskService.findById(taskId); + task = taskService.start(task,countedEntities); + List worklistTwitterIds = new ArrayList<>(); + boolean hasNext=true; + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + int lfdNr = 0; + int all = 0; + while(hasNext) { + Page tweetTwitterIds = tweetService.getAll(pageRequest); + for(org.woehlke.twitterwall.oodm.entities.Tweet tweetTwitterId:tweetTwitterIds.getContent()){ + lfdNr++; + all++; + log.debug("### tweetService.findAllTwitterIds from DB ("+lfdNr+"): "+tweetTwitterId.getIdTwitter()); + worklistTwitterIds.add(tweetTwitterId.getIdTwitter()); + } + hasNext = tweetTwitterIds.hasNext(); + pageRequest = pageRequest.next(); + } + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); + List tweets = new ArrayList<>(); + lfdNr = 0; + for(Long tweetTwitterId : worklistTwitterIds){ + lfdNr++; + log.debug("### witterApiService.findOneTweetById from Twiiter API ("+lfdNr+" of "+all+"): "+tweetTwitterId); + Tweet foundTweetFromTwitter = twitterApiService.findOneTweetById(tweetTwitterId); + TweetFromTwitter result = new TweetFromTwitter(task.getId(),foundTweetFromTwitter); + tweets.add(result); + log.debug("### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } + } + return tweets; + } + + private static final Logger log = LoggerFactory.getLogger(UpdateTweetsImpl.class); + +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java new file mode 100644 index 00000000..39fa3cd3 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesFromMentionsImpl.java @@ -0,0 +1,97 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.twitter.api.TwitterProfile; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.oodm.entities.Mention; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.MentionService; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateUserProfilesFromMentions; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateUserProfilesFromMentions") +public class UpdateUserProfilesFromMentionsImpl implements UpdateUserProfilesFromMentions { + + private static final Logger log = LoggerFactory.getLogger(UpdateUserProfilesFromMentionsImpl.class); + + private final TwitterProperties twitterProperties; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final MentionService mentionService; + + private final CountedEntitiesService countedEntitiesService; + + public UpdateUserProfilesFromMentionsImpl(TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, MentionService mentionService, CountedEntitiesService countedEntitiesService) { + this.twitterProperties = twitterProperties; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.mentionService = mentionService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + String msg ="splitMessage: "; + log.debug(msg+ " START"); + CountedEntities countedEntities = countedEntitiesService.countAll(); + List userProfileList = new ArrayList<>(); + TaskMessage msgIn = message.getPayload(); + long id = msgIn.getTaskId(); + Task task = taskService.findById(id); + task = taskService.start(task,countedEntities); + List screenNames = new ArrayList<>(); + int lfdNr = 0; + int all = 0; + boolean hasNext=true; + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + while (hasNext) { + Page allPersMentions = mentionService.getAllWithoutPersistentUser(pageRequest); + hasNext = allPersMentions.hasNext(); + for (Mention onePersMentions : allPersMentions) { + if (!onePersMentions.hasPersistentUser()) { + lfdNr++; + all++; + String screenName = onePersMentions.getScreenName(); + log.debug("### mentionService.getAll from DB ("+lfdNr+"): "+screenName); + screenNames.add(screenName); + } + } + pageRequest = pageRequest.next(); + } + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); + for(String screenName:screenNames){ + lfdNr++; + log.debug("### twitterApiService.getUserProfileForScreenName("+screenName+") from Twiiter API ("+lfdNr+" of "+all+")"); + TwitterProfile userProfile = twitterApiService.getUserProfileForScreenName(screenName); + if(userProfile!=null) { + TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn, userProfile); + userProfileList.add(userMsg); + } + log.debug(msg+"### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } + } + return userProfileList; + } +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java new file mode 100644 index 00000000..70808cb0 --- /dev/null +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UpdateUserProfilesImpl.java @@ -0,0 +1,105 @@ +package org.woehlke.twitterwall.scheduled.mq.endoint.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.messaging.Message; +import org.springframework.social.RateLimitExceededException; +import org.springframework.social.twitter.api.TwitterProfile; +import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.conf.TwitterProperties; +import org.woehlke.twitterwall.oodm.entities.Task; +import org.woehlke.twitterwall.oodm.entities.User; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.oodm.service.UserService; +import org.woehlke.twitterwall.scheduled.mq.endoint.UpdateUserProfiles; +import org.woehlke.twitterwall.scheduled.mq.msg.TaskMessage; +import org.woehlke.twitterwall.scheduled.mq.msg.TwitterProfileMessage; +import org.woehlke.twitterwall.scheduled.service.backend.TwitterApiService; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; + +import java.util.ArrayList; +import java.util.List; + +import static org.woehlke.twitterwall.frontend.controller.common.ControllerHelper.FIRST_PAGE_NUMBER; + +@Component("mqUpdateUserProfiles") +public class UpdateUserProfilesImpl implements UpdateUserProfiles { + + private static final Logger log = LoggerFactory.getLogger(UpdateUserProfilesImpl.class); + + private final TwitterProperties twitterProperties; + + private final TwitterApiService twitterApiService; + + private final TaskService taskService; + + private final UserService userService; + + private final CountedEntitiesService countedEntitiesService; + + @Autowired + public UpdateUserProfilesImpl(TwitterProperties twitterProperties, TwitterApiService twitterApiService, TaskService taskService, UserService userService, CountedEntitiesService countedEntitiesService) { + this.twitterProperties = twitterProperties; + this.twitterApiService = twitterApiService; + this.taskService = taskService; + this.userService = userService; + this.countedEntitiesService = countedEntitiesService; + } + + @Override + public List splitMessage(Message message) { + String msg = "### mqUpdateUserProfiles.splitMessage: "; + log.debug(msg+ " START"); + CountedEntities countedEntities = countedEntitiesService.countAll(); + TaskMessage msgIn = message.getPayload(); + long id = msgIn.getTaskId(); + Task task = taskService.findById(id); + task = taskService.start(task,countedEntities); + int loopId = 0; + boolean hasNext=true; + List worklistProfileTwitterIds = new ArrayList<>(); + Pageable pageRequest = new PageRequest(FIRST_PAGE_NUMBER, twitterProperties.getPageSize()); + while (hasNext) { + Page userProfileTwitterIds = userService.getAll(pageRequest); + for(User user:userProfileTwitterIds.getContent()){ + loopId++; + log.debug(msg+ "### userService.getAllTwitterIds: ("+loopId+") "+user.getIdTwitter()); + worklistProfileTwitterIds.add(user.getIdTwitter()); + } + hasNext = userProfileTwitterIds.hasNext(); + pageRequest = pageRequest.next(); + } + long number = worklistProfileTwitterIds.size(); + loopId = 0; + int millisToWaitBetweenTwoApiCalls = twitterProperties.getMillisToWaitBetweenTwoApiCalls(); + List userProfileList = new ArrayList<>(); + for(Long userProfileTwitterId:worklistProfileTwitterIds){ + String counter = " ( " + loopId + " from " + number + " ) "; + log.debug(msg + counter); + TwitterProfile userProfile = null; + loopId++; + try { + log.debug(msg+"### twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter); + userProfile = twitterApiService.getUserProfileForTwitterId(userProfileTwitterId); + } catch (RateLimitExceededException e) { + log.error(msg + "### ERROR: twitterApiService.getUserProfileForTwitterId("+userProfileTwitterId+") "+counter,e); + } + if(userProfile != null){ + TwitterProfileMessage userMsg = new TwitterProfileMessage(msgIn,userProfile); + userProfileList.add(userMsg); + } + log.debug(msg + "### waiting now for (ms): "+millisToWaitBetweenTwoApiCalls); + try { + Thread.sleep(millisToWaitBetweenTwoApiCalls); + } catch (InterruptedException e) { + } + } + log.debug(msg+ " DONE"); + return userProfileList; + } +} diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java index 1b12e187..4c7bce3c 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserFinisherImpl.java @@ -1,11 +1,20 @@ package org.woehlke.twitterwall.scheduled.mq.endoint.impl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.Message; import org.springframework.stereotype.Component; +import org.woehlke.twitterwall.oodm.entities.Task; import org.woehlke.twitterwall.oodm.entities.User; +import org.woehlke.twitterwall.oodm.entities.parts.CountedEntities; +import org.woehlke.twitterwall.oodm.service.TaskService; +import org.woehlke.twitterwall.scheduled.mq.channel.SenderType; import org.woehlke.twitterwall.scheduled.mq.endoint.UserFinisher; import org.woehlke.twitterwall.scheduled.mq.msg.UserMessage; import org.woehlke.twitterwall.scheduled.mq.msg.UserResultList; +import org.woehlke.twitterwall.scheduled.service.backend.impl.TwitterApiServiceImpl; +import org.woehlke.twitterwall.scheduled.service.persist.CountedEntitiesService; import java.util.ArrayList; import java.util.List; @@ -13,7 +22,6 @@ @Component("mqUserFinisher") public class UserFinisherImpl implements UserFinisher { - @Override public UserResultList finish(Message> incomingMessageList) { long taskId = 0L; @@ -28,4 +36,40 @@ public UserResultList finish(Message> incomingMessageList) { UserResultList userResultList = new UserResultList(taskId,users); return userResultList; } + + @Override + public void finishAsnyc(Message> incomingMessageList) { + List userMessageList = incomingMessageList.getPayload(); + CountedEntities countedEntities = countedEntitiesService.countAll(); + if(incomingMessageList.getHeaders().containsKey("task_id")){ + long taskId = (Long) incomingMessageList.getHeaders().get( "task_id"); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(msgDone,task,countedEntities); + log.info(msgDone); + } else { + if(userMessageList.size()>0) { + long taskId = userMessageList.get(0).getTaskId(); + Task task = taskService.findById(taskId); + String msgDone = "Sucessfully finished task "+task.getTaskType()+"via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER; + taskService.done(task,countedEntities); + log.info(msgDone); + } else { + log.warn("finishAsnyc: No Users performed. via MQ by "+ SenderType.FIRE_AND_FORGET_SENDER); + } + } + } + + + @Autowired + public UserFinisherImpl(TaskService taskService, CountedEntitiesService countedEntitiesService) { + this.taskService = taskService; + this.countedEntitiesService = countedEntitiesService; + } + + private final TaskService taskService; + + private final CountedEntitiesService countedEntitiesService; + + private static final Logger log = LoggerFactory.getLogger(UserFinisherImpl.class); } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java index 296f4d17..568be8b5 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/mq/endoint/impl/UserPersistorImpl.java @@ -26,10 +26,10 @@ public UserPersistorImpl(TaskService taskService, StoreUserProcess storeUserProc @Override public UserMessage persistUser(Message incomingUserMessage) { UserMessage receivedMessage = incomingUserMessage.getPayload(); - long id = receivedMessage.getTwitterProfileMessage().getTaskMessage().getTaskId(); - Task task = taskService.findById(id); + long taskId = receivedMessage.getTwitterProfileMessage().getTaskMessage().getTaskId(); + Task task = taskService.findById(taskId); User user = storeUserProcess.storeUserProcess(receivedMessage.getUser(),task); - UserMessage userMessage = new UserMessage(user,id); + UserMessage userMessage = new UserMessage(user,taskId); return userMessage; } } diff --git a/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java b/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java index c886dba8..8e937de6 100644 --- a/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java +++ b/src/main/java/org/woehlke/twitterwall/scheduled/service/backend/impl/TwitterApiServiceImpl.java @@ -30,9 +30,16 @@ public class TwitterApiServiceImpl implements TwitterApiService { public List findTweetsForSearchQuery() { String msg = MSG+"findTweetsForSearchQuery: "; log.debug(msg); - List fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets(); + List fetchedTweets; + try { + fetchedTweets = getTwitterProxy().searchOperations().search(twitterProperties.getSearchQuery(), twitterProperties.getPageSize()).getTweets(); + } catch (Exception e) { + fetchedTweets = new ArrayList<>(); + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; - if(fetchedTweets == null){ + if(fetchedTweets.size()==0){ log.debug(msg+" result.size: 0"); return new ArrayList<>(); } else { @@ -45,7 +52,14 @@ public List findTweetsForSearchQuery() { public Tweet findOneTweetById(long id) { String msg = MSG+"findOneTweetById: "+id; log.debug(msg); - Tweet result = getTwitterProxy().timelineOperations().getStatus(id); + Tweet result; + try { + result = getTwitterProxy().timelineOperations().getStatus(id); + } catch (Exception e){ + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" Id: "+result.getId()); return result; @@ -55,7 +69,14 @@ public Tweet findOneTweetById(long id) { public List getUserProfilesForTwitterIds(long... userProfileTwitterIds) { String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterIds; log.debug(msg); - List result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds); + List result; + try { + result = getTwitterProxy().userOperations().getUsers(userProfileTwitterIds); + } catch (Exception e){ + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" size: "+result.size()); return result; @@ -65,7 +86,14 @@ public List getUserProfilesForTwitterIds(long... userProfileTwit public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) { String msg = MSG+"getUserProfileForTwitterId: "+userProfileTwitterId; log.debug(msg); - TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId); + TwitterProfile result; + try { + result = getTwitterProxy().userOperations().getUserProfile(userProfileTwitterId); + } catch (Exception e) { + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" Id: "+result.getId()); log.debug(msg+" ScreenName: "+result.getScreenName()); @@ -77,8 +105,14 @@ public TwitterProfile getUserProfileForTwitterId(long userProfileTwitterId) { public TwitterProfile getUserProfileForScreenName(String screenName) { String msg = MSG+"getUserProfileForScreenName: "+screenName; log.debug(msg); - TwitterProfile result = getTwitterProxy().userOperations().getUserProfile(screenName); - log.debug(msg+result.getId()); + TwitterProfile result; + try { + result= getTwitterProxy().userOperations().getUserProfile(screenName); + } catch (Exception e) { + result = null; + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } msg += " result: "; log.debug(msg+" ScreenName: "+result.getScreenName()); log.debug(msg+" Name: "+result.getName()); @@ -89,7 +123,14 @@ public TwitterProfile getUserProfileForScreenName(String screenName) { public List findUsersFromDefinedList(String screenName,String fetchUserListName) { String msg = MSG+"findUsersFromDefinedList: "+fetchUserListName+" "; log.debug(msg); - List result = getTwitterProxy().listOperations().getListMembers(screenName,fetchUserListName); + List result; + try { + result = getTwitterProxy().listOperations().getListMembers(screenName, fetchUserListName); + } catch (Exception e) { + result = new ArrayList<>(); + log.debug(msg + e.getMessage()); + e.printStackTrace(); + } log.debug(msg+" result.size: "+result.size()); return result; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a7f6e65c..a337b3c7 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -66,6 +66,7 @@ twitter: # consumerSecret: ${TWITTER_CONSUMER_SECRET} pageSize: ${TWITTER_PAGE_SIZE} searchQuery: ${TWITTER_SEARCH_TERM} + millisToWaitBetweenTwoApiCalls: 5000 twitterwall: backend: twitter: diff --git a/src/main/resources/integration.xml b/src/main/resources/integration.xml index 70ad9b2e..3b67ae9a 100644 --- a/src/main/resources/integration.xml +++ b/src/main/resources/integration.xml @@ -80,7 +80,7 @@ - + @@ -92,20 +92,47 @@ - + + + + + - + + + + + - + + + + + @@ -176,4 +203,132 @@ method="persistUser" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index f040319c..97327e3e 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,7 +1,6 @@ - +