Choco31415 · Choco31415 · Nov 18, 2017 · Nov 18, 2017
diff --git a/src/main/java/WikiBot/Core/GenericBot.java b/src/main/java/WikiBot/Core/GenericBot.java
@@ -1658,13 +1658,13 @@ private String APIcommandHTTP(APIcommand command) {
 			}
 		}
 
-		//Follor the url!
+		//Follow the url!
 		try {
-			String[] output = getURL(url, command.shouldUnescapeHTML());
+			String output = removeBOM(EntityUtils.toString(getURL(url)));
 			if (output == null) {
 				throw new NetworkError("Cannot connect to server at: " + baseURL);
 			} else {
-				return ArrayUtils.compactArray(output, "\n");
+				return output;
 			}
 		} catch (IOException e) {
 			throw new Error(e);

diff --git a/src/main/java/WikiBot/Core/NetworkingBase.java b/src/main/java/WikiBot/Core/NetworkingBase.java
@@ -1,10 +1,6 @@
 package WikiBot.Core;
 
-import org.apache.commons.lang3.StringEscapeUtils;
-
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
@@ -14,21 +10,31 @@
 import java.net.URLEncoder;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Level;
 
+import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.X509TrustManager;
 
-import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicNameValuePair;
@@ -65,9 +71,53 @@ public class NetworkingBase extends javax.swing.JPanel {
 	//Special characters.
 	private static final String UTF8_BOM = "\uFEFF";
 
+		// One-off SSL trust manager class.
+		public class HttpsTrustManager implements X509TrustManager {
+			@Override
+			public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+				// TODO Auto-generated method stub
+
+			}
+
+			@Override
+			public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+				// TODO Auto-generated method stub
+
+			}
+
+			@Override
+			public X509Certificate[] getAcceptedIssuers() {
+				// TODO Auto-generated method stub
+				return new X509Certificate[]{};
+			}
+
+		}
+
+	private void setupSSLclient() {
+		// Handle SSL
+		SSLContext sslcontext = null;
+		SSLConnectionSocketFactory factory = null;
+
+
+			try {
+				sslcontext = SSLContexts.custom().useProtocol("SSL").build();
+				sslcontext.init(null, new X509TrustManager[]{new HttpsTrustManager()}, new SecureRandom());
+		        factory = new SSLConnectionSocketFactory(sslcontext, new NoopHostnameVerifier());
+			} catch (KeyManagementException | NoSuchAlgorithmException e) {
+				// TODO Auto-generated catch block
+				logError("Could not create SSL context. Entering fail state.");
+				throw new Error(e.getMessage());
+			}
+
+
+
+		// Create client and context for web stuff.
+		httpclient = HttpClientBuilder.create().setSSLSocketFactory(factory).build();
+	}
+
 	//Instantiation.
 	public NetworkingBase() {
-		httpclient = HttpClientBuilder.create().build();
+		setupSSLclient();
 		context =  HttpClientContext.create();
 	}
 
@@ -187,38 +237,22 @@ public void setLogPropagation(boolean set) {
 	 * @param ur The url you want to get.
 	 * @param unescapeHTML4 Unescapes HTML4 text. Ex: & #039;
 	 */
-	protected String[] getURL(String ur, boolean unescapeHTML4) throws IOException {
+	protected HttpEntity getURL(String ur) throws IOException {
 		logFiner("Loading: " + ur);
+
+  		//This method actual fetches a web page, and turns it into a more easily use-able format.		  		//This method actual fetches a web page, and turns it into a more easily use-able format.
+		HttpResponse response = null;
+	 	HttpGet httpost = new HttpGet(ur);
 
-		//This method actual fetches a web page, and turns it into a more easily use-able format.
-        URL oracle = null;
+		// Fetch the url.
 		try {
-			oracle = new URL(ur);
-		} catch (MalformedURLException e) {
-			System.err.println(e.getMessage());
+			response = httpclient.execute(httpost, context);
+		} catch (SocketException|NoHttpResponseException e) {
+			throw new NetworkError("Cannot connect to server at: " + ur);
+		} catch (IOException e) {
+			e.printStackTrace();
 		}
-
-		BufferedReader in = null;
-        try {
-            in = new BufferedReader(new InputStreamReader(oracle.openStream(), StandardCharsets.UTF_8));
-        } catch (IOException e) {
-        	logError("Connection cannot be opened.");
-        	return null;
-        }
-
-        ArrayList<String> page = new ArrayList<String>();
-        String inputLine;
-        while ((inputLine = in.readLine()) != null) {
-        	if (unescapeHTML4) {
-        		inputLine = StringEscapeUtils.unescapeHtml4(StringEscapeUtils.unescapeHtml4(inputLine));
-        	}
-        	inputLine = removeBOM(inputLine);
-
-            page.add(inputLine);
-        }
-        in.close();
-
-        return page.toArray(new String[page.size()]);
+		return response.getEntity();
 	}
 
 	/*

diff --git a/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java b/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java
@@ -10,12 +10,12 @@
 import java.util.logging.Level;
 
 import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.http.util.EntityUtils;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import WikiBot.Core.NetworkingBase;
-import WikiBot.Utils.ArrayUtils;
 import WikiBot.Utils.FileUtils;
 
 public class FamilyGenerator extends NetworkingBase {
@@ -192,7 +192,7 @@ private void addInterwikiFamily() throws IOException, URISyntaxException  {
 			url = URLapi + "/api.php?action=query&meta=siteinfo&siprop=interwikimap&sifilteriw=local&format=json";
 		}
 
-		String serverOutput = ArrayUtils.compactArray(getURL(url, true));
+		String serverOutput = removeBOM(EntityUtils.toString(getURL(url)));
 		serverOutput = StringEscapeUtils.unescapeHtml4(StringEscapeUtils.unescapeHtml4(serverOutput));
 
 		// Read in the Json!!!
@@ -418,7 +418,7 @@ private String getMediawikiURL(String url) throws URISyntaxException {
 	 * @throws IOException
 	 */
 	private String getMWversion(String apiURL) throws IOException {
-		String serverOutput = ArrayUtils.compactArray(getURL(apiURL + "/api.php?action=query&meta=siteinfo&format=json", true));
+		String serverOutput = removeBOM(EntityUtils.toString(getURL(apiURL + "/api.php?action=query&meta=siteinfo&format=json")));
 
 		// Read in the JSON!!!
 		ObjectMapper mapper = new ObjectMapper();

diff --git a/src/main/resources/Families/Scratch.txt b/src/main/resources/Families/Scratch.txt
@@ -1,9 +1,9 @@
-[{"prefix": "de", "version": "1.26.3", "url": "http://scratch-dach.info/w"},
+[{"prefix": "de", "version": "1.26.3", "url": "https://scratch-dach.info/w"},
 {"prefix": "en", "version": "1.22.7", "url": "https://wiki.scratch.mit.edu/w"},
-{"prefix": "fr", "version": "1.26.3", "url": "http://fr.scratch-wiki.info/w"},
-{"prefix": "hu", "version": "1.26.3", "url": "http://hu.scratch-wiki.info/w"},
-{"prefix": "id", "version": "1.26.3", "url": "http://scratch-indo.info/w"},
-{"prefix": "ja", "version": "1.26.3", "url": "http://jp.scratch-wiki.info/w"},
-{"prefix": "nl", "version": "1.26.3", "url": "http://nl.scratch-wiki.info/w"},
-{"prefix": "ru", "version": "1.26.3", "url": "http://scratch-ru.info/w"},
-{"prefix": "test", "version": "1.25.3", "url": "http://test.scratch-wiki.info/w"}]
+{"prefix": "fr", "version": "1.26.3", "url": "https://fr.scratch-wiki.info/w"},
+{"prefix": "hu", "version": "1.26.3", "url": "https://hu.scratch-wiki.info/w"},
+{"prefix": "id", "version": "1.26.3", "url": "https://scratch-indo.info/w"},
+{"prefix": "ja", "version": "1.26.3", "url": "https://jp.scratch-wiki.info/w"},
+{"prefix": "nl", "version": "1.26.3", "url": "https://nl.scratch-wiki.info/w"},
+{"prefix": "ru", "version": "1.26.3", "url": "https://scratch-ru.info/w"},
+{"prefix": "test", "version": "1.25.3", "url": "https://test.scratch-wiki.info/w"}]