From e8695179e14c9866267b5e9f089bb037b3b29c72 Mon Sep 17 00:00:00 2001 From: Austin Keener Date: Sat, 7 May 2016 01:21:02 +0900 Subject: [PATCH] Changed how GoogleSearch accesses Google's search API due to Google finally removing the AJAX endpoint. Now uses the Google Custom Search API. Created SearchResult as an abstraction of results returned from a Search Engine. Will be useful for Bing support when Google Search usage cap is met. Rewrote the GoogleSearch system to be Static and also to use the new SearchResult api. --- .../java/net/dv8tion/discord/Settings.java | 11 +++ .../net/dv8tion/discord/SettingsManager.java | 6 ++ src/main/java/net/dv8tion/discord/Yui.java | 13 ++- .../commands/AnimeNewsNetworkCommand.java | 20 ++-- .../discord/commands/MyAnimeListCommand.java | 14 +-- .../discord/commands/SearchCommand.java | 8 +- .../dv8tion/discord/util/GoogleSearch.java | 97 +++++++++---------- .../dv8tion/discord/util/SearchResult.java | 75 ++++++++++++++ 8 files changed, 169 insertions(+), 75 deletions(-) create mode 100644 src/main/java/net/dv8tion/discord/util/SearchResult.java diff --git a/src/main/java/net/dv8tion/discord/Settings.java b/src/main/java/net/dv8tion/discord/Settings.java index 54122d5..e95e62d 100644 --- a/src/main/java/net/dv8tion/discord/Settings.java +++ b/src/main/java/net/dv8tion/discord/Settings.java @@ -26,6 +26,7 @@ */ public class Settings { private String botToken; + private String googleApiKey; private String proxyHost; private String proxyPort; private Boolean useBetaBuilds; @@ -42,6 +43,16 @@ public void setBotToken(String botToken) this.botToken = botToken; } + public String getGoogleApiKey() + { + return googleApiKey; + } + + public void setGoogleApiKey(String googleApiKey) + { + this.googleApiKey = googleApiKey; + } + public String getProxyHost() { return proxyHost; diff --git a/src/main/java/net/dv8tion/discord/SettingsManager.java b/src/main/java/net/dv8tion/discord/SettingsManager.java index 9ca88c3..c9bd7a6 100644 --- a/src/main/java/net/dv8tion/discord/SettingsManager.java +++ b/src/main/java/net/dv8tion/discord/SettingsManager.java @@ -92,6 +92,7 @@ public void saveSettings() { private Settings getDefaultSettings() { Settings newSettings = new Settings(); newSettings.setBotToken(""); + newSettings.setGoogleApiKey(""); newSettings.setProxyHost(""); newSettings.setProxyPort("8080"); newSettings.setUseBetaBuilds(new Boolean(false)); @@ -120,6 +121,11 @@ private void checkOldSettingsFile() settings.setBotToken(defaults.getBotToken()); modified = true; } + if (settings.getGoogleApiKey() == null) + { + settings.setGoogleApiKey(defaults.getGoogleApiKey()); + modified = true; + } if (settings.getUseBetaBuilds() == null) { settings.setUseBetaBuilds(defaults.getUseBetaBuilds()); diff --git a/src/main/java/net/dv8tion/discord/Yui.java b/src/main/java/net/dv8tion/discord/Yui.java index 3b2a1d7..a8446b0 100644 --- a/src/main/java/net/dv8tion/discord/Yui.java +++ b/src/main/java/net/dv8tion/discord/Yui.java @@ -29,6 +29,7 @@ import net.dv8tion.discord.commands.*; import net.dv8tion.discord.util.Database; +import net.dv8tion.discord.util.GoogleSearch; import net.dv8tion.jda.JDA; import net.dv8tion.jda.JDABuilder; import net.dv8tion.jda.entities.Guild; @@ -111,10 +112,14 @@ private static void setupBot() HelpCommand help = new HelpCommand(); jdaBuilder.addListener(help.registerCommand(help)); jdaBuilder.addListener(help.registerCommand(new TestCommand())); - jdaBuilder.addListener(help.registerCommand(new SearchCommand())); - jdaBuilder.addListener(help.registerCommand(new NyaaCommand())); - jdaBuilder.addListener(help.registerCommand(new MyAnimeListCommand())); - jdaBuilder.addListener(help.registerCommand(new AnimeNewsNetworkCommand())); + if (settings.getGoogleApiKey() != null && !settings.getGoogleApiKey().isEmpty()) + { + GoogleSearch.setup(settings.getGoogleApiKey()); + jdaBuilder.addListener(help.registerCommand(new SearchCommand())); + jdaBuilder.addListener(help.registerCommand(new NyaaCommand())); + jdaBuilder.addListener(help.registerCommand(new MyAnimeListCommand())); + jdaBuilder.addListener(help.registerCommand(new AnimeNewsNetworkCommand())); + } jdaBuilder.addListener(help.registerCommand(new ReloadCommand())); jdaBuilder.addListener(help.registerCommand(new UpdateCommand())); jdaBuilder.addListener(help.registerCommand(new PermissionsCommand())); diff --git a/src/main/java/net/dv8tion/discord/commands/AnimeNewsNetworkCommand.java b/src/main/java/net/dv8tion/discord/commands/AnimeNewsNetworkCommand.java index 57a062c..4d63681 100644 --- a/src/main/java/net/dv8tion/discord/commands/AnimeNewsNetworkCommand.java +++ b/src/main/java/net/dv8tion/discord/commands/AnimeNewsNetworkCommand.java @@ -24,6 +24,7 @@ import net.dv8tion.discord.util.Downloader; import net.dv8tion.discord.util.GoogleSearch; +import net.dv8tion.discord.util.SearchResult; import net.dv8tion.jda.events.message.MessageReceivedEvent; import net.dv8tion.jda.events.message.guild.GuildMessageReceivedEvent; import net.dv8tion.jda.events.message.priv.PrivateMessageReceivedEvent; @@ -43,12 +44,11 @@ public class AnimeNewsNetworkCommand extends Command @Override public void onCommand(MessageReceivedEvent e, String[] args) { - GoogleSearch search = new GoogleSearch( - String.format("%s+%s", - StringUtils.join(args, "+", 1, args.length), - "site:animenewsnetwork.com")); + List results = GoogleSearch.performSearch( + "018291224751151548851:g6kjw0k_cp8", + StringUtils.join(args, "+", 1, args.length)); - sendMessage(e, handleSearch(search)); + sendMessage(e, handleSearch(results.get(0))); } @Override @@ -80,9 +80,9 @@ public List getUsageInstructions() + " - This will return the manga page for Boku no Hero Academia (hopefully)"); } - private String handleSearch(GoogleSearch search) + private String handleSearch(SearchResult result) { - String url = search.getUrl(0); + String url = result.getUrl(); if (url.contains(ANIME_URL) || url.contains(MANGA_URL)) { String title = null; @@ -98,7 +98,7 @@ private String handleSearch(GoogleSearch search) if (m.find()) title = m.group(); else - title = search.getTitle(0); + title = result.getTitle(); Pattern p2 = Pattern.compile(ALT_TITLE_REGEX); Matcher m2 = p2.matcher(xmlReturn); @@ -110,7 +110,7 @@ private String handleSearch(GoogleSearch search) if (m3.find()) summary = m3.group(); else - summary = search.getContent(0); + summary = result.getContent(); Pattern p4 = Pattern.compile("type=\"Picture\".*?>.*?"); Matcher m4 = p4.matcher(xmlReturn); @@ -137,7 +137,7 @@ private String handleSearch(GoogleSearch search) } else { - return search.getSuggestedReturn(); + return result.getSuggestedReturn(); } } } diff --git a/src/main/java/net/dv8tion/discord/commands/MyAnimeListCommand.java b/src/main/java/net/dv8tion/discord/commands/MyAnimeListCommand.java index 0af1ded..a567e10 100644 --- a/src/main/java/net/dv8tion/discord/commands/MyAnimeListCommand.java +++ b/src/main/java/net/dv8tion/discord/commands/MyAnimeListCommand.java @@ -22,6 +22,7 @@ import net.dv8tion.discord.util.Downloader; import net.dv8tion.discord.util.GoogleSearch; +import net.dv8tion.discord.util.SearchResult; import net.dv8tion.jda.events.message.MessageReceivedEvent; import org.apache.commons.lang3.StringUtils; @@ -34,12 +35,11 @@ public class MyAnimeListCommand extends Command @Override public void onCommand(MessageReceivedEvent e, String[] args) { - GoogleSearch search = new GoogleSearch( - String.format("%s+%s", - StringUtils.join(args, "+", 1, args.length), - "site:myanimelist.net")); + List results = GoogleSearch.performSearch( + "018291224751151548851:pwowlyhmpyc", + StringUtils.join(args, "+", 1, args.length)); - sendMessage(e, search.getSuggestedReturn()); + sendMessage(e, results.get(0).getSuggestedReturn()); } @Override @@ -72,9 +72,9 @@ public List getUsageInstructions() } @SuppressWarnings("unused") - private String handleSearch(GoogleSearch search) + private String handleSearch(SearchResult result) { - String url = search.getUrl(0); + String url = result.getUrl(); if (url.contains(ANIME_URL)) { System.out.println("this is anime"); diff --git a/src/main/java/net/dv8tion/discord/commands/SearchCommand.java b/src/main/java/net/dv8tion/discord/commands/SearchCommand.java index 936c31e..45a02d4 100644 --- a/src/main/java/net/dv8tion/discord/commands/SearchCommand.java +++ b/src/main/java/net/dv8tion/discord/commands/SearchCommand.java @@ -21,6 +21,7 @@ import net.dv8tion.discord.util.GoogleSearch; +import net.dv8tion.discord.util.SearchResult; import net.dv8tion.jda.events.message.MessageReceivedEvent; import org.apache.commons.lang3.StringUtils; @@ -44,11 +45,12 @@ public void onCommand(MessageReceivedEvent e, String[] args) return; } - GoogleSearch search = new GoogleSearch( + List results = GoogleSearch.performSearch( + "018291224751151548851%3Ajzifriqvl1o", StringUtils.join(args, "+", 1, args.length) + ((filter != null) ? ("+" + filter) : "")); - sendMessage(e, search.getSuggestedReturn()); + sendMessage(e, results.get(0).getSuggestedReturn()); } @Override @@ -73,6 +75,6 @@ public String getName() public List getUsageInstructions() { return Collections.singletonList( - ".google * **OR** .wiki ** **OR** .urban **\n"); + ".google ** **OR** .wiki ** **OR** .urban **\n"); } } \ No newline at end of file diff --git a/src/main/java/net/dv8tion/discord/util/GoogleSearch.java b/src/main/java/net/dv8tion/discord/util/GoogleSearch.java index 5bec4d1..983a799 100644 --- a/src/main/java/net/dv8tion/discord/util/GoogleSearch.java +++ b/src/main/java/net/dv8tion/discord/util/GoogleSearch.java @@ -22,68 +22,66 @@ import java.net.URL; import java.net.URLConnection; import java.net.URLDecoder; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.Random; +import net.dv8tion.discord.Permissions; import org.apache.commons.lang3.StringEscapeUtils; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonParser; +import org.json.JSONArray; +import org.json.JSONObject; public class GoogleSearch { - private JsonArray results; + public static final String GOOGLE_URL = "https://www.googleapis.com/customsearch/v1?cx=%s&key=%s&num=%d&q=%s"; + private static String GOOGLE_API_KEY = null; + private static LocalDateTime dayStartTime = null; + private static int currentGoogleUsage = 0; - public GoogleSearch(String terms) + public static void setup(String googleApiKey) { - performSearch(terms.replaceAll(" ", "+")); + GOOGLE_API_KEY = googleApiKey; + dayStartTime = LocalDateTime.now(); } - public String getTitle(int resultIndex) + public static List performSearch(String engineId, String terms) { - String title = results.get(resultIndex).getAsJsonObject().get("title").toString(); - return cleanString(title); + return performSearch(engineId, terms, 1); } - public String getContent(int resultIndex) + public static List performSearch(String engineId, String terms, int requiredResultsCount) { - String content = results.get(resultIndex).getAsJsonObject().get("content").toString(); - return cleanString(content); - } - - public String getUrl(int resultIndex) - { - String url = results.get(resultIndex).getAsJsonObject().get("url").toString(); - url = cleanString(url); try { - return URLDecoder.decode(url, "UTF-8"); - } - catch (UnsupportedEncodingException e) - { - e.printStackTrace(); - } - return url; - } - - public String getSuggestedReturn() - { - return getUrl(0) + " - *" + getTitle(0) + "*: \"" + getContent(0) + "\""; - } + if (GOOGLE_API_KEY == null) + throw new IllegalStateException("Google API Key is null, Cannot preform google search without a key! Set one in the settings!"); + if (engineId == null || engineId.isEmpty()) + throw new IllegalArgumentException("Google Custom Search Engine id cannot be null or empty!"); - public int getResultCount() - { - return results.size(); - } + LocalDateTime currentTime = LocalDateTime.now(); + if (currentTime.isAfter(dayStartTime.plusDays(1))) + { + dayStartTime = currentTime; + currentGoogleUsage = 1; + } + else if (currentGoogleUsage >= 80) + { + throw new IllegalStateException("Google usage has reached the premature security cap of 80"); + } - private void performSearch(String terms) { - try { - StringBuilder searchURLString = new StringBuilder(); - searchURLString.append("https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q="); - searchURLString.append(terms); + terms = terms.replace(" ", "%20"); + String searchUrl = String.format(GOOGLE_URL, engineId, GOOGLE_API_KEY, requiredResultsCount, terms); - URL searchURL = new URL(searchURLString.toString()); + URL searchURL = new URL(searchUrl); URLConnection conn = searchURL.openConnection(); + currentGoogleUsage++; conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 " + randomName(10)); BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); StringBuilder json = new StringBuilder(); @@ -92,26 +90,23 @@ private void performSearch(String terms) { json.append(line).append("\n"); } in.close(); - JsonElement element = new JsonParser().parse(json.toString()); - results = element.getAsJsonObject().getAsJsonObject("responseData").getAsJsonArray("results"); + + JSONArray jsonResults = new JSONObject(json.toString()).getJSONArray("items"); + List results = new LinkedList<>(); + for (int i = 0; i < jsonResults.length(); i++) + { + results.add(SearchResult.fromGoogle(jsonResults.getJSONObject(i))); + } + return results; } catch (IOException e) { e.printStackTrace(); + return null; } } - private String cleanString(String uncleanString) - { - return StringEscapeUtils.unescapeJava( - StringEscapeUtils.unescapeHtml4( - uncleanString - .replaceAll("\\s+", " ") - .replaceAll("\\<.*?>", "") - .replaceAll("\"", ""))); - } - - private String randomName(int randomLength) + private static String randomName(int randomLength) { char[] characters = new char[] {'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z', diff --git a/src/main/java/net/dv8tion/discord/util/SearchResult.java b/src/main/java/net/dv8tion/discord/util/SearchResult.java new file mode 100644 index 0000000..75e54c6 --- /dev/null +++ b/src/main/java/net/dv8tion/discord/util/SearchResult.java @@ -0,0 +1,75 @@ +/** + * Copyright 2015-2016 Austin Keener + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.dv8tion.discord.util; + +import org.apache.commons.lang3.StringEscapeUtils; +import org.json.JSONObject; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + +public class SearchResult +{ + private String title; + private String content; + private String url; + + public static SearchResult fromGoogle(JSONObject googleResult) + { + SearchResult result = new SearchResult(); + result.title = cleanString(googleResult.getString("title")); + result.content = cleanString(googleResult.getString("snippet")); + try + { + result.url = URLDecoder.decode(cleanString(googleResult.getString("link")), "UTF-8"); + } + catch (UnsupportedEncodingException e) + { + e.printStackTrace(); + } + return result; + } + + public String getTitle() + { + return content; + } + + public String getContent() + { + return content; + } + + public String getUrl() + { + return url; + } + + public String getSuggestedReturn() + { + return url + " - *" + title + "*: \"" + content + "\""; + } + + private static String cleanString(String uncleanString) + { + return StringEscapeUtils.unescapeJava( + StringEscapeUtils.unescapeHtml4( + uncleanString + .replaceAll("\\s+", " ") + .replaceAll("\\<.*?>", "") + .replaceAll("\"", ""))); + } +}