Changed how GoogleSearch accesses Google's search API due to Google f…

…inally removing the AJAX endpoint. Now uses the Google Custom Search API.

Created SearchResult as an abstraction of results returned from a Search Engine. Will be useful for Bing support when Google Search usage cap is met.
Rewrote the GoogleSearch system to be Static and also to use the new SearchResult api.

src/main/java/net/dv8tion/discord/Settings.java

@@ -26,6 +26,7 @@
  */
 public class Settings {
     private String botToken;
+    private String googleApiKey;
     private String proxyHost;
     private String proxyPort;
     private Boolean useBetaBuilds;
@@ -42,6 +43,16 @@ public void setBotToken(String botToken)
         this.botToken = botToken;
     }
 
+    public String getGoogleApiKey()
+    {
+        return googleApiKey;
+    }
+
+    public void setGoogleApiKey(String googleApiKey)
+    {
+        this.googleApiKey = googleApiKey;
+    }
+
     public String getProxyHost()
     {
         return proxyHost;

src/main/java/net/dv8tion/discord/SettingsManager.java

@@ -92,6 +92,7 @@ public void saveSettings() {
     private Settings getDefaultSettings() {
         Settings newSettings = new Settings();
         newSettings.setBotToken("");
+        newSettings.setGoogleApiKey("");
         newSettings.setProxyHost("");
         newSettings.setProxyPort("8080");
         newSettings.setUseBetaBuilds(new Boolean(false));
@@ -120,6 +121,11 @@ private void checkOldSettingsFile()
             settings.setBotToken(defaults.getBotToken());
             modified = true;
         }
+        if (settings.getGoogleApiKey() == null)
+        {
+            settings.setGoogleApiKey(defaults.getGoogleApiKey());
+            modified = true;
+        }
         if (settings.getUseBetaBuilds() == null)
         {
             settings.setUseBetaBuilds(defaults.getUseBetaBuilds());

src/main/java/net/dv8tion/discord/Yui.java

@@ -29,6 +29,7 @@
 import net.dv8tion.discord.commands.*;
 import net.dv8tion.discord.util.Database;
 
+import net.dv8tion.discord.util.GoogleSearch;
 import net.dv8tion.jda.JDA;
 import net.dv8tion.jda.JDABuilder;
 import net.dv8tion.jda.entities.Guild;
@@ -111,10 +112,14 @@ private static void setupBot()
             HelpCommand help = new HelpCommand();
             jdaBuilder.addListener(help.registerCommand(help));
             jdaBuilder.addListener(help.registerCommand(new TestCommand()));
-            jdaBuilder.addListener(help.registerCommand(new SearchCommand()));
-            jdaBuilder.addListener(help.registerCommand(new NyaaCommand()));
-            jdaBuilder.addListener(help.registerCommand(new MyAnimeListCommand()));
-            jdaBuilder.addListener(help.registerCommand(new AnimeNewsNetworkCommand()));
+            if (settings.getGoogleApiKey() != null && !settings.getGoogleApiKey().isEmpty())
+            {
+                GoogleSearch.setup(settings.getGoogleApiKey());
+                jdaBuilder.addListener(help.registerCommand(new SearchCommand()));
+                jdaBuilder.addListener(help.registerCommand(new NyaaCommand()));
+                jdaBuilder.addListener(help.registerCommand(new MyAnimeListCommand()));
+                jdaBuilder.addListener(help.registerCommand(new AnimeNewsNetworkCommand()));
+            }
             jdaBuilder.addListener(help.registerCommand(new ReloadCommand()));
             jdaBuilder.addListener(help.registerCommand(new UpdateCommand()));
             jdaBuilder.addListener(help.registerCommand(new PermissionsCommand()));

src/main/java/net/dv8tion/discord/commands/AnimeNewsNetworkCommand.java

@@ -24,6 +24,7 @@
 import net.dv8tion.discord.util.Downloader;
 import net.dv8tion.discord.util.GoogleSearch;
 
+import net.dv8tion.discord.util.SearchResult;
 import net.dv8tion.jda.events.message.MessageReceivedEvent;
 import net.dv8tion.jda.events.message.guild.GuildMessageReceivedEvent;
 import net.dv8tion.jda.events.message.priv.PrivateMessageReceivedEvent;
@@ -43,12 +44,11 @@ public class AnimeNewsNetworkCommand extends Command
     @Override
     public void onCommand(MessageReceivedEvent e, String[] args)
     {
-        GoogleSearch search = new GoogleSearch(
-                String.format("%s+%s",
-                        StringUtils.join(args, "+", 1, args.length),
-                        "site:animenewsnetwork.com"));
+        List<SearchResult> results = GoogleSearch.performSearch(
+                "018291224751151548851:g6kjw0k_cp8",
+                StringUtils.join(args, "+", 1, args.length));
 
-        sendMessage(e, handleSearch(search));
+        sendMessage(e, handleSearch(results.get(0)));
     }
 
     @Override
@@ -80,9 +80,9 @@ public List<String> getUsageInstructions()
                 + " - This will return the manga page for Boku no Hero Academia (hopefully)");
     }
 
-    private String handleSearch(GoogleSearch search)
+    private String handleSearch(SearchResult result)
     {
-        String url = search.getUrl(0);
+        String url = result.getUrl();
         if (url.contains(ANIME_URL) || url.contains(MANGA_URL))
         {
             String title = null;
@@ -98,7 +98,7 @@ private String handleSearch(GoogleSearch search)
             if (m.find())
                 title = m.group();
             else
-                title = search.getTitle(0);
+                title = result.getTitle();
 
             Pattern p2 = Pattern.compile(ALT_TITLE_REGEX);
             Matcher m2 = p2.matcher(xmlReturn);
@@ -110,7 +110,7 @@ private String handleSearch(GoogleSearch search)
             if (m3.find())
                 summary = m3.group();
             else
-                summary = search.getContent(0);
+                summary = result.getContent();
 
             Pattern p4 = Pattern.compile("type=\"Picture\".*?>.*?</info>");
             Matcher m4 = p4.matcher(xmlReturn);
@@ -137,7 +137,7 @@ private String handleSearch(GoogleSearch search)
         }
         else
         {
-            return search.getSuggestedReturn();
+            return result.getSuggestedReturn();
         }
     }
 }

src/main/java/net/dv8tion/discord/commands/MyAnimeListCommand.java

@@ -22,6 +22,7 @@
 import net.dv8tion.discord.util.Downloader;
 import net.dv8tion.discord.util.GoogleSearch;
 
+import net.dv8tion.discord.util.SearchResult;
 import net.dv8tion.jda.events.message.MessageReceivedEvent;
 import org.apache.commons.lang3.StringUtils;
 
@@ -34,12 +35,11 @@ public class MyAnimeListCommand extends Command
     @Override
     public void onCommand(MessageReceivedEvent e, String[] args)
     {
-        GoogleSearch search = new GoogleSearch(
-                String.format("%s+%s",
-                        StringUtils.join(args, "+", 1, args.length),
-                        "site:myanimelist.net"));
+        List<SearchResult> results = GoogleSearch.performSearch(
+                "018291224751151548851:pwowlyhmpyc",
+                StringUtils.join(args, "+", 1, args.length));
 
-        sendMessage(e, search.getSuggestedReturn());
+        sendMessage(e, results.get(0).getSuggestedReturn());
     }
 
     @Override
@@ -72,9 +72,9 @@ public List<String> getUsageInstructions()
     }
 
     @SuppressWarnings("unused")
-    private String handleSearch(GoogleSearch search)
+    private String handleSearch(SearchResult result)
     {
-        String url = search.getUrl(0);
+        String url = result.getUrl();
         if (url.contains(ANIME_URL))
         {
             System.out.println("this is anime");

src/main/java/net/dv8tion/discord/commands/SearchCommand.java

@@ -21,6 +21,7 @@
 
 import net.dv8tion.discord.util.GoogleSearch;
 
+import net.dv8tion.discord.util.SearchResult;
 import net.dv8tion.jda.events.message.MessageReceivedEvent;
 import org.apache.commons.lang3.StringUtils;
 
@@ -44,11 +45,12 @@ public void onCommand(MessageReceivedEvent e, String[] args)
 				return;
 		}
 
-		GoogleSearch search = new GoogleSearch(
+		List<SearchResult> results = GoogleSearch.performSearch(
+				"018291224751151548851%3Ajzifriqvl1o",
 				StringUtils.join(args, "+", 1, args.length)
 						+ ((filter != null) ? ("+" + filter) : ""));
 
-		sendMessage(e, search.getSuggestedReturn());
+		sendMessage(e, results.get(0).getSuggestedReturn());
 	}
 
 	@Override
@@ -73,6 +75,6 @@ public String getName()
     public List<String> getUsageInstructions()
     {
         return Collections.singletonList(
-				".google *<search terms>  **OR** .wiki *<search terms>*  **OR**  .urban *<search terms>*\n");
+				".google *<search terms>*  **OR** .wiki *<search terms>* **OR** .urban *<search terms>*\n");
     }
 }

src/main/java/net/dv8tion/discord/util/GoogleSearch.java

@@ -22,68 +22,66 @@
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLDecoder;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Random;
 
+import net.dv8tion.discord.Permissions;
 import org.apache.commons.lang3.StringEscapeUtils;
 
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonParser;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 public class GoogleSearch
 {
-    private JsonArray results;
+    public static final String GOOGLE_URL = "https://www.googleapis.com/customsearch/v1?cx=%s&key=%s&num=%d&q=%s";
+    private static String GOOGLE_API_KEY = null;
+    private static LocalDateTime dayStartTime = null;
+    private static int currentGoogleUsage = 0;
 
-    public GoogleSearch(String terms)
+    public static void setup(String googleApiKey)
     {
-        performSearch(terms.replaceAll(" ", "+"));
+        GOOGLE_API_KEY = googleApiKey;
+        dayStartTime = LocalDateTime.now();
     }
 
-    public String getTitle(int resultIndex)
+    public static List<SearchResult> performSearch(String engineId, String terms)
     {
-        String title = results.get(resultIndex).getAsJsonObject().get("title").toString();
-        return cleanString(title);
+        return performSearch(engineId, terms, 1);
     }
 
-    public String getContent(int resultIndex)
+    public static List<SearchResult> performSearch(String engineId, String terms, int requiredResultsCount)
     {
-        String content =  results.get(resultIndex).getAsJsonObject().get("content").toString();
-        return cleanString(content);
-    }
-
-    public String getUrl(int resultIndex)
-    {
-        String url = results.get(resultIndex).getAsJsonObject().get("url").toString();
-        url = cleanString(url);
         try
         {
-            return URLDecoder.decode(url, "UTF-8");
-        }
-        catch (UnsupportedEncodingException e)
-        {
-            e.printStackTrace();
-        }
-        return url;
-    }
-
-    public String getSuggestedReturn()
-    {
-        return getUrl(0) + " - *" + getTitle(0) + "*: \"" + getContent(0) + "\"";
-    }
+            if (GOOGLE_API_KEY == null)
+                throw new IllegalStateException("Google API Key is null, Cannot preform google search without a key! Set one in the settings!");
+            if (engineId == null || engineId.isEmpty())
+                throw new IllegalArgumentException("Google Custom Search Engine id cannot be null or empty!");
 
-    public int getResultCount()
-    {
-        return results.size();
-    }
+            LocalDateTime currentTime = LocalDateTime.now();
+            if (currentTime.isAfter(dayStartTime.plusDays(1)))
+            {
+                dayStartTime = currentTime;
+                currentGoogleUsage = 1;
+            }
+            else if (currentGoogleUsage >= 80)
+            {
+                throw new IllegalStateException("Google usage has reached the premature security cap of 80");
+            }
 
-    private void performSearch(String terms) {
-        try {
-            StringBuilder searchURLString = new StringBuilder();
-            searchURLString.append("https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=");
-            searchURLString.append(terms);
+            terms = terms.replace(" ", "%20");
+            String searchUrl = String.format(GOOGLE_URL, engineId, GOOGLE_API_KEY, requiredResultsCount, terms);
 
-            URL searchURL = new URL(searchURLString.toString());
+            URL searchURL = new URL(searchUrl);
             URLConnection conn = searchURL.openConnection();
+            currentGoogleUsage++;
             conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 " + randomName(10));
             BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
             StringBuilder json = new StringBuilder();
@@ -92,26 +90,23 @@ private void performSearch(String terms) {
                 json.append(line).append("\n");
             }
             in.close();
-            JsonElement element = new JsonParser().parse(json.toString());
-            results = element.getAsJsonObject().getAsJsonObject("responseData").getAsJsonArray("results");
+
+            JSONArray jsonResults = new JSONObject(json.toString()).getJSONArray("items");
+            List<SearchResult> results = new LinkedList<>();
+            for (int i = 0; i < jsonResults.length(); i++)
+            {
+                results.add(SearchResult.fromGoogle(jsonResults.getJSONObject(i)));
+            }
+            return results;
         }
         catch (IOException e)
         {
             e.printStackTrace();
+            return null;
         }
     }
 
-    private String cleanString(String uncleanString)
-    {
-        return StringEscapeUtils.unescapeJava(
-                StringEscapeUtils.unescapeHtml4(
-                        uncleanString
-                            .replaceAll("\\s+", " ")
-                            .replaceAll("\\<.*?>", "")
-                            .replaceAll("\"", "")));
-    }
-
-    private String randomName(int randomLength)
+    private static String randomName(int randomLength)
     {
         char[] characters = new char[]
                 {'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',