diff --git a/src/main/java/WikiBot/Core/GenericBot.java b/src/main/java/WikiBot/Core/GenericBot.java index 6aa97ad..ad6e1fe 100644 --- a/src/main/java/WikiBot/Core/GenericBot.java +++ b/src/main/java/WikiBot/Core/GenericBot.java @@ -1658,13 +1658,13 @@ private String APIcommandHTTP(APIcommand command) { } } - //Follor the url! + //Follow the url! try { - String[] output = getURL(url, command.shouldUnescapeHTML()); + String output = removeBOM(EntityUtils.toString(getURL(url))); if (output == null) { throw new NetworkError("Cannot connect to server at: " + baseURL); } else { - return ArrayUtils.compactArray(output, "\n"); + return output; } } catch (IOException e) { throw new Error(e); diff --git a/src/main/java/WikiBot/Core/NetworkingBase.java b/src/main/java/WikiBot/Core/NetworkingBase.java index feeb4b7..2bbc0f6 100644 --- a/src/main/java/WikiBot/Core/NetworkingBase.java +++ b/src/main/java/WikiBot/Core/NetworkingBase.java @@ -1,10 +1,6 @@ package WikiBot.Core; -import org.apache.commons.lang3.StringEscapeUtils; - -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -14,21 +10,31 @@ import java.net.URLEncoder; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.X509TrustManager; -import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContexts; import org.apache.http.cookie.Cookie; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicNameValuePair; @@ -65,9 +71,53 @@ public class NetworkingBase extends javax.swing.JPanel { //Special characters. private static final String UTF8_BOM = "\uFEFF"; + // One-off SSL trust manager class. + public class HttpsTrustManager implements X509TrustManager { + @Override + public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { + // TODO Auto-generated method stub + + } + + @Override + public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { + // TODO Auto-generated method stub + + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + // TODO Auto-generated method stub + return new X509Certificate[]{}; + } + + } + + private void setupSSLclient() { + // Handle SSL + SSLContext sslcontext = null; + SSLConnectionSocketFactory factory = null; + + + try { + sslcontext = SSLContexts.custom().useProtocol("SSL").build(); + sslcontext.init(null, new X509TrustManager[]{new HttpsTrustManager()}, new SecureRandom()); + factory = new SSLConnectionSocketFactory(sslcontext, new NoopHostnameVerifier()); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + // TODO Auto-generated catch block + logError("Could not create SSL context. Entering fail state."); + throw new Error(e.getMessage()); + } + + + + // Create client and context for web stuff. + httpclient = HttpClientBuilder.create().setSSLSocketFactory(factory).build(); + } + //Instantiation. public NetworkingBase() { - httpclient = HttpClientBuilder.create().build(); + setupSSLclient(); context = HttpClientContext.create(); } @@ -187,38 +237,22 @@ public void setLogPropagation(boolean set) { * @param ur The url you want to get. * @param unescapeHTML4 Unescapes HTML4 text. Ex: & #039; */ - protected String[] getURL(String ur, boolean unescapeHTML4) throws IOException { + protected HttpEntity getURL(String ur) throws IOException { logFiner("Loading: " + ur); + + //This method actual fetches a web page, and turns it into a more easily use-able format. //This method actual fetches a web page, and turns it into a more easily use-able format. + HttpResponse response = null; + HttpGet httpost = new HttpGet(ur); - //This method actual fetches a web page, and turns it into a more easily use-able format. - URL oracle = null; + // Fetch the url. try { - oracle = new URL(ur); - } catch (MalformedURLException e) { - System.err.println(e.getMessage()); + response = httpclient.execute(httpost, context); + } catch (SocketException|NoHttpResponseException e) { + throw new NetworkError("Cannot connect to server at: " + ur); + } catch (IOException e) { + e.printStackTrace(); } - - BufferedReader in = null; - try { - in = new BufferedReader(new InputStreamReader(oracle.openStream(), StandardCharsets.UTF_8)); - } catch (IOException e) { - logError("Connection cannot be opened."); - return null; - } - - ArrayList page = new ArrayList(); - String inputLine; - while ((inputLine = in.readLine()) != null) { - if (unescapeHTML4) { - inputLine = StringEscapeUtils.unescapeHtml4(StringEscapeUtils.unescapeHtml4(inputLine)); - } - inputLine = removeBOM(inputLine); - - page.add(inputLine); - } - in.close(); - - return page.toArray(new String[page.size()]); + return response.getEntity(); } /* diff --git a/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java b/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java index 7716fa1..2b8c6fa 100644 --- a/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java +++ b/src/main/java/WikiBot/MediawikiData/FamilyGenerator.java @@ -10,12 +10,12 @@ import java.util.logging.Level; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.http.util.EntityUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import WikiBot.Core.NetworkingBase; -import WikiBot.Utils.ArrayUtils; import WikiBot.Utils.FileUtils; public class FamilyGenerator extends NetworkingBase { @@ -192,7 +192,7 @@ private void addInterwikiFamily() throws IOException, URISyntaxException { url = URLapi + "/api.php?action=query&meta=siteinfo&siprop=interwikimap&sifilteriw=local&format=json"; } - String serverOutput = ArrayUtils.compactArray(getURL(url, true)); + String serverOutput = removeBOM(EntityUtils.toString(getURL(url))); serverOutput = StringEscapeUtils.unescapeHtml4(StringEscapeUtils.unescapeHtml4(serverOutput)); // Read in the Json!!! @@ -418,7 +418,7 @@ private String getMediawikiURL(String url) throws URISyntaxException { * @throws IOException */ private String getMWversion(String apiURL) throws IOException { - String serverOutput = ArrayUtils.compactArray(getURL(apiURL + "/api.php?action=query&meta=siteinfo&format=json", true)); + String serverOutput = removeBOM(EntityUtils.toString(getURL(apiURL + "/api.php?action=query&meta=siteinfo&format=json"))); // Read in the JSON!!! ObjectMapper mapper = new ObjectMapper(); diff --git a/src/main/resources/Families/Scratch.txt b/src/main/resources/Families/Scratch.txt index 9d47640..68ac327 100644 --- a/src/main/resources/Families/Scratch.txt +++ b/src/main/resources/Families/Scratch.txt @@ -1,9 +1,9 @@ -[{"prefix": "de", "version": "1.26.3", "url": "http://scratch-dach.info/w"}, +[{"prefix": "de", "version": "1.26.3", "url": "https://scratch-dach.info/w"}, {"prefix": "en", "version": "1.22.7", "url": "https://wiki.scratch.mit.edu/w"}, -{"prefix": "fr", "version": "1.26.3", "url": "http://fr.scratch-wiki.info/w"}, -{"prefix": "hu", "version": "1.26.3", "url": "http://hu.scratch-wiki.info/w"}, -{"prefix": "id", "version": "1.26.3", "url": "http://scratch-indo.info/w"}, -{"prefix": "ja", "version": "1.26.3", "url": "http://jp.scratch-wiki.info/w"}, -{"prefix": "nl", "version": "1.26.3", "url": "http://nl.scratch-wiki.info/w"}, -{"prefix": "ru", "version": "1.26.3", "url": "http://scratch-ru.info/w"}, -{"prefix": "test", "version": "1.25.3", "url": "http://test.scratch-wiki.info/w"}] \ No newline at end of file +{"prefix": "fr", "version": "1.26.3", "url": "https://fr.scratch-wiki.info/w"}, +{"prefix": "hu", "version": "1.26.3", "url": "https://hu.scratch-wiki.info/w"}, +{"prefix": "id", "version": "1.26.3", "url": "https://scratch-indo.info/w"}, +{"prefix": "ja", "version": "1.26.3", "url": "https://jp.scratch-wiki.info/w"}, +{"prefix": "nl", "version": "1.26.3", "url": "https://nl.scratch-wiki.info/w"}, +{"prefix": "ru", "version": "1.26.3", "url": "https://scratch-ru.info/w"}, +{"prefix": "test", "version": "1.25.3", "url": "https://test.scratch-wiki.info/w"}] \ No newline at end of file