From babe4febbf758c45054fd62cfbb0c58c7203df73 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Fri, 5 Jun 2020 22:13:47 +0800 Subject: [PATCH 01/10] fix retry bug --- .../java/org/wikidata/wdtk/wikibaseapi/WbEditingAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/WbEditingAction.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/WbEditingAction.java index 5d131603a..1b0160be6 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/WbEditingAction.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/WbEditingAction.java @@ -763,7 +763,7 @@ private JsonNode performAPIAction( retry--; } - if (lastException != null) { + if (retry == 0 && lastException != null) { logger.error("Gave up after several retries. Last error was: " + lastException.toString()); throw lastException; From 43f98ef8b6cba23752d268e51e643b2c29de8753 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Fri, 5 Jun 2020 22:21:20 +0800 Subject: [PATCH 02/10] migrate to OkHTTP --- .../wdtk/wikibaseapi/ApiConnection.java | 259 ++++++------- .../wdtk/wikibaseapi/BasicApiConnection.java | 137 +++---- .../wdtk/wikibaseapi/OAuthApiConnection.java | 82 +--- .../wikibaseapi/BasicApiConnectionTest.java | 364 +++++++----------- .../wikibaseapi/OAuthApiConnectionTest.java | 154 +++----- .../src/test/resources/loginError.json | 2 +- .../src/test/resources/loginError2.json | 1 - 7 files changed, 368 insertions(+), 631 deletions(-) delete mode 100644 wdtk-wikibaseapi/src/test/resources/loginError2.json diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index 18a9b732b..568bb1fa4 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -20,43 +20,31 @@ * #L% */ -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import okhttp3.*; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.wikidata.wdtk.util.WebResourceFetcherImpl; import org.wikidata.wdtk.wikibaseapi.apierrors.AssertUserFailedException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorHandler; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; -import java.net.URL; import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; +import java.util.*; +import java.util.concurrent.TimeUnit; /** - * Class to build up and hold a connection to a Wikibase API, managing cookies - * and login. - * - * This should no longer be instantiated directly: please use one of the subclasses - * {@link BasicApiConnection} and {@link OAuthApiConnection} instead. This - * class will become an interface in a future release. + * Class to build up and hold a connection to a Wikibase API. * * @author Michael Guenther * @author Antonin Delpeuch + * @author Lu Liu */ @JsonIgnoreProperties(ignoreUnknown = true) public abstract class ApiConnection { @@ -71,7 +59,7 @@ public abstract class ApiConnection { * URL of the API of test.wikidata.org. */ public final static String URL_TEST_WIKIDATA_API = "https://test.wikidata.org/w/api.php"; - + /** * URL of the API of commons.wikimedia.org. */ @@ -86,48 +74,56 @@ public abstract class ApiConnection { * Name of the HTTP parameter to submit the requested result format to the * API. */ - public final static String PARAM_FORMAT = "format"; + protected final static String PARAM_FORMAT = "format"; /** - * MediaWiki assert= parameter to ensure we are editting while logged in. + * MediaWiki assert parameter to ensure we are editing while logged in. */ - private static final String ASSERT_PARAMETER = "assert"; + protected static final String ASSERT_PARAMETER = "assert"; + + protected static final MediaType URLENCODED_MEDIA_TYPE = MediaType.parse("application/x-www-form-urlencoded"); /** * URL to access the Wikibase API. */ - final String apiBaseUrl; + protected final String apiBaseUrl; /** * True after successful login. */ - boolean loggedIn = false; + protected boolean loggedIn = false; /** * User name used to log in. */ - String username = ""; - + protected String username = ""; + /** * Map of requested tokens. */ - final Map tokens; - + @NotNull + protected final Map tokens; + /** * Maximum time to wait for when establishing a connection, in milliseconds. * For negative values, no timeout is set. */ - int connectTimeout = -1; - + protected int connectTimeout = -1; + /** * Maximum time to wait for a server response once the connection was established. * For negative values, no timeout is set. */ - int readTimeout = -1; + protected int readTimeout = -1; + + /** + * Http client used for making requests. + */ + private OkHttpClient client; /** * Mapper object used for deserializing JSON data. */ - final ObjectMapper mapper = new ObjectMapper(); + private final ObjectMapper mapper = new ObjectMapper(); /** * Creates an object to manage a connection to the Web API of a Wikibase @@ -156,6 +152,19 @@ public ApiConnection(String apiBaseUrl, Map tokens) { this.tokens = tokens != null ? tokens : new HashMap<>(); } + /** + * Subclasses can customized their own {@link OkHttpClient.Builder} instances. + * + * An example: + *
+	 * 	    return new OkHttpClient.Builder()
+	 * 		        .connectTimeout(5, TimeUnit.MILLISECONDS)
+	 * 		        .readTimeout(5, TimeUnit.MILLISECONDS)
+	 * 		        .cookieJar(...);
+	 * 
+ */ + protected abstract OkHttpClient.Builder getBuilder(); + /** * Getter for the apiBaseUrl. */ @@ -167,25 +176,25 @@ public String getApiBaseUrl() { * Returns true if a user is logged in. This does not perform * any request to the server: it just returns our own internal state. * To check if our authentication credentials are still considered - * valid by the remote server, use checkCredentials(). + * valid by the remote server, use {@link ApiConnection#checkCredentials()}. * * @return true if the connection is in a logged in state */ public boolean isLoggedIn() { - return this.loggedIn; + return loggedIn; } - + /** * Checks that the credentials are still valid for the * user currently logged in. This can fail if (for instance) * the cookies expired, or were invalidated by a logout from * a different client. - * + * * This method queries the API and throws {@link AssertUserFailedException} * if the check failed. This does not update the state of the connection * object. - * @throws MediaWikiApiErrorException - * @throws IOException + * @throws MediaWikiApiErrorException + * @throws IOException */ public void checkCredentials() throws IOException, MediaWikiApiErrorException { Map parameters = new HashMap<>(); @@ -200,14 +209,66 @@ public void checkCredentials() throws IOException, MediaWikiApiErrorException { * @return name of the logged in user */ public String getCurrentUser() { - return this.username; + return username; + } + + /** + * Sets the maximum time to wait for a server response once the connection was established, in milliseconds. + * For negative values, no timeout is set. + * + * @see HttpURLConnection#setReadTimeout + */ + public void setReadTimeout(int timeout) { + readTimeout = timeout; + if (timeout >= 0) { + OkHttpClient.Builder builder = getBuilder(); + builder.readTimeout(timeout, TimeUnit.MILLISECONDS); + client = builder.build(); + } } - + + /** + * Sets the maximum time to wait for when establishing a connection, in milliseconds. + * For negative values, no timeout is set. + * + * @see HttpURLConnection#setConnectTimeout + */ + public void setConnectTimeout(int timeout) { + connectTimeout = timeout; + if (timeout >= 0) { + OkHttpClient.Builder builder = getBuilder(); + builder.connectTimeout(timeout, TimeUnit.MILLISECONDS); + client = builder.build(); + } + } + + /** + * Maximum time to wait for when establishing a connection, in milliseconds. + * For negative values, no timeout is set, which is the default behaviour (for + * backwards compatibility). + * + * @see HttpURLConnection#getConnectTimeout + */ + public int getConnectTimeout() { + return connectTimeout; + } + + /** + * Maximum time to wait for a server response once the connection was established. + * For negative values, no timeout is set, which is the default behaviour (for backwards + * compatibility). + * + * @see HttpURLConnection#getReadTimeout + */ + public int getReadTimeout() { + return readTimeout; + } + /** * Logs the current user out. * * @throws IOException - * @throws MediaWikiApiErrorException + * @throws MediaWikiApiErrorException */ public abstract void logout() throws IOException, MediaWikiApiErrorException; @@ -215,7 +276,7 @@ public String getCurrentUser() { * Return a token of given type. * @param tokenType The kind of token to retrieve like "csrf" or "login" * @return a token - * @throws MediaWikiApiErrorException + * @throws MediaWikiApiErrorException * if MediaWiki returned an error * @throws IOException * if a network error occurred @@ -246,10 +307,10 @@ void clearToken(String tokenType) { * * @param tokenType The kind of token to retrieve like "csrf" or "login" * @return newly retrieved token - * @throws IOException + * @throws IOException * if a network error occurred * @throws MediaWikiApiErrorException - * if MediaWiki returned an error when fetching the token + * if MediaWiki returned an error when fetching the token */ private String fetchToken(String tokenType) throws IOException, MediaWikiApiErrorException { Map params = new HashMap<>(); @@ -309,67 +370,21 @@ public JsonNode sendJsonRequest(String requestMethod, Map paramet */ public InputStream sendRequest(String requestMethod, Map parameters) throws IOException { + Request request; String queryString = getQueryString(parameters); - URL url = new URL(this.apiBaseUrl); - HttpURLConnection connection = (HttpURLConnection) WebResourceFetcherImpl - .getUrlConnection(url); - - setupConnection(requestMethod, queryString, connection); - OutputStreamWriter writer = new OutputStreamWriter( - connection.getOutputStream()); - writer.write(queryString); - writer.flush(); - writer.close(); - - int rc = connection.getResponseCode(); - if (rc != 200) { - logger.warn("Error: API request returned response code " + rc); + if ("GET".equalsIgnoreCase(requestMethod)) { + request = new Request.Builder().url(apiBaseUrl + "?" + queryString).build(); + } else if ("POST".equalsIgnoreCase(requestMethod)) { + request = new Request.Builder().url(apiBaseUrl).post(RequestBody.create(URLENCODED_MEDIA_TYPE, queryString)).build(); + } else { + throw new IllegalArgumentException("Expected the requestMethod to be either GET or POST, but got " + requestMethod); } - InputStream iStream = connection.getInputStream(); - processResponseHeaders(connection.getHeaderFields()); - return iStream; - } - - /** - * Configures a given {@link HttpURLConnection} object to send requests. - * Takes the request method (either "POST" or "GET") and query string. - * - * @param requestMethod - * either "POST" or "GET" - * @param queryString - * the query string to submit - * @param connection - * the connection to configure - * @throws IOException - * if the given protocol is not valid - */ - protected void setupConnection(String requestMethod, String queryString, - HttpURLConnection connection) throws IOException { - connection.setRequestMethod(requestMethod); - connection.setDoInput(true); - connection.setDoOutput(true); - connection.setUseCaches(false); - if(connectTimeout >= 0) { - connection.setConnectTimeout(connectTimeout); + if (client == null) { + client = getBuilder().build(); } - if(readTimeout >= 0) { - connection.setReadTimeout(readTimeout); - } - connection.setRequestProperty("Content-Type", - "application/x-www-form-urlencoded"); - } - - /** - * Method called after each request with the response headers received - * from the server. Can be used to store cookies returned by the server for instance. - * By default, does nothing. - * - * @param headerFields - * the headers returned by the server - */ - public void processResponseHeaders(Map> headerFields) { - + Response response = client.newCall(request).execute(); + return Objects.requireNonNull(response.body()).byteStream(); } /** @@ -501,47 +516,5 @@ public static String implodeObjects(Iterable objects) { } return builder.toString(); } - - /** - * Maximum time to wait for when establishing a connection, in milliseconds. - * For negative values, no timeout is set, which is the default behaviour (for - * backwards compatibility). - * - * @see HttpURLConnection#getConnectTimeout - */ - public int getConnectTimeout() { - return connectTimeout; - } - - /** - * Sets the maximum time to wait for when establishing a connection, in milliseconds. - * For negative values, no timeout is set. - * - * @see HttpURLConnection#setConnectTimeout - */ - public void setConnectTimeout(int timeout) { - connectTimeout = timeout; - } - - /** - * Maximum time to wait for a server response once the connection was established. - * For negative values, no timeout is set, which is the default behaviour (for backwards - * compatibility). - * - * @see HttpURLConnection#getReadTimeout - */ - public int getReadTimeout() { - return readTimeout; - } - - /** - * Sets the maximum time to wait for a server response once the connection was established. - * For negative values, no timeout is set. - * - * @see HttpURLConnection#setReadTimeout - */ - public void setReadTimeout(int timeout) { - readTimeout = timeout; - } } diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index be3c83624..9fcc1b096 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -21,14 +21,17 @@ */ import java.io.IOException; -import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import okhttp3.Cookie; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import org.jetbrains.annotations.NotNull; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import com.fasterxml.jackson.annotation.JsonCreator; @@ -119,17 +122,6 @@ public class BasicApiConnection extends ApiConnection { */ protected final static String LOGIN_WRONG_TOKEN = "WrongToken"; - /** - * Name of the HTTP parameter to submit cookies to the API. - */ - public final static String PARAM_COOKIE = "Cookie"; - - /** - * Name of the HTTP response header field that provides us with cookies we - * should set. - */ - final static String HEADER_FIELD_SET_COOKIE = "Set-Cookie"; - /** * Password used to log in. */ @@ -178,6 +170,12 @@ protected BasicApiConnection( this.loggedIn = loggedIn; } + @Override + protected OkHttpClient.Builder getBuilder() { + return new OkHttpClient.Builder() + .cookieJar(new ApiConnectionCookieJar()); + } + /** * Creates an API connection to test.wikidata.org. * @@ -309,63 +307,6 @@ public void clearCookies() throws IOException, MediaWikiApiErrorException { this.cookies.clear(); } - @Override - public void processResponseHeaders(Map> headerFields) { - fillCookies(headerFields); - } - - /** - * Reads out the Set-Cookie Header Fields and fills the cookie map of the - * API connection with it. - */ - void fillCookies(Map> headerFields) { - List headerCookies = new ArrayList<>(); - for (Map.Entry> headers : headerFields.entrySet()) { - if (HEADER_FIELD_SET_COOKIE.equalsIgnoreCase(headers.getKey())) { - headerCookies.addAll(headers.getValue()); - } - } - if (!headerCookies.isEmpty()) { - for (String cookie : headerCookies) { - String[] cookieResponse = cookie.split(";\\p{Space}??"); - for (String cookieLine : cookieResponse) { - String[] entry = cookieLine.split("="); - if (entry.length == 2) { - this.cookies.put(entry[0], entry[1]); - } - if (entry.length == 1) { - this.cookies.put(entry[0], ""); - } - } - } - } - } - - /** - * Returns the string representation of the currently stored cookies. This - * data is added to the connection before making requests. - * - * @return cookie string - */ - protected String getCookieString() { - StringBuilder result = new StringBuilder(); - boolean first = true; - for (Entry entry : this.cookies.entrySet()) { - if (first) { - first = false; - } else { - result.append("; "); - } - result.append(entry.getKey()); - if (!"".equals(entry.getValue())) { - result.append("=").append(entry.getValue()); - } - - } - return result.toString(); - } - - /** * Returns a user-readable message for a given API response. * @@ -406,27 +347,6 @@ protected String getLoginErrorMessage(String loginResult) { } } - /** - * Configures a given {@link HttpURLConnection} object to send requests. - * Takes the request method (either "POST" or "GET") and query string. - * - * @param requestMethod - * either "POST" or "GET" - * @param queryString - * the query string to submit - * @param connection - * the connection to configure - * @throws IOException - * if the given protocol is not valid - */ - @Override - protected void setupConnection(String requestMethod, String queryString, - HttpURLConnection connection) throws IOException { - super.setupConnection(requestMethod, queryString, connection); - connection.setRequestProperty(PARAM_COOKIE, - getCookieString()); - } - /** * Logs the current user out. * @@ -449,4 +369,39 @@ public void logout() throws IOException, MediaWikiApiErrorException { this.password = ""; } } + + private class ApiConnectionCookieJar implements CookieJar { + + private String domain; + + public ApiConnectionCookieJar() { + int firstSlash = apiBaseUrl.indexOf("/", apiBaseUrl.indexOf("//") + 2); + domain = apiBaseUrl.substring(apiBaseUrl.indexOf("//") + 2, firstSlash); + if (domain.contains(":")) { // Cookies do not provide isolation by port. + domain = domain.substring(0, domain.indexOf(":")); + } + } + + @Override + public void saveFromResponse(@NotNull HttpUrl url, @NotNull List cookieList) { + for (Cookie cookie : cookieList) { + cookies.put(cookie.name(), cookie.value()); + } + } + + @NotNull + @Override + public List loadForRequest(@NotNull HttpUrl url) { + List cookieList = new ArrayList<>(); + for (Map.Entry entry : cookies.entrySet()) { + Cookie cookie = new Cookie.Builder() + .domain(domain) + .name(entry.getKey()) + .value(entry.getValue()) + .build(); + cookieList.add(cookie); + } + return cookieList; + } + } } diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java index 3ea09d3e6..23d0cdeed 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java @@ -20,7 +20,6 @@ * #L% */ - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; @@ -31,11 +30,8 @@ import se.akerfeldt.okhttp.signpost.SigningInterceptor; import java.io.IOException; -import java.io.InputStream; import java.util.HashMap; import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; /** * A connection to the MediaWiki/Wikibase API which uses OAuth @@ -52,10 +48,6 @@ public class OAuthApiConnection extends ApiConnection { private String accessToken; private String accessSecret; - private OkHttpClient client; - - private static final MediaType URLENCODED_MEDIA_TYPE = MediaType.parse("application/x-www-form-urlencoded"); - /** * Constructs an OAuth connection to the given MediaWiki API endpoint. *

@@ -88,42 +80,15 @@ public OAuthApiConnection( this.consumerSecret = consumerSecret; this.accessToken = accessToken; this.accessSecret = accessSecret; - OkHttpOAuthConsumer consumer = new OkHttpOAuthConsumer(consumerKey, consumerSecret); - consumer.setTokenWithSecret(accessToken, accessSecret); - client = new OkHttpClient.Builder() - .addInterceptor(new SigningInterceptor(consumer)) - .build(); loggedIn = true; } - /** - * Sends a request to the API with the given parameters and the given - * request method and returns the result string. - *

- * WARNING: You probably want to use {@link ApiConnection#sendJsonRequest} - * that execute the request using JSON content format, - * throws the errors and logs the warnings. - * - * @param requestMethod either POST or GET - * @param parameters Maps parameter keys to values. Out of this map the function - * will create a query string for the request. - * @return API result - * @throws IOException - */ @Override - public InputStream sendRequest(String requestMethod, Map parameters) throws IOException { - Request request; - String queryString = getQueryString(parameters); - if ("GET".equalsIgnoreCase(requestMethod)) { - request = new Request.Builder().url(apiBaseUrl + "?" + queryString).build(); - } else if ("POST".equalsIgnoreCase(requestMethod)) { - request = new Request.Builder().url(apiBaseUrl).post(RequestBody.create(URLENCODED_MEDIA_TYPE, queryString)).build(); - } else { - throw new IllegalArgumentException("Expected the requestMethod to be either GET or POST, but got " + requestMethod); - } - - Response response = client.newCall(request).execute(); - return Objects.requireNonNull(response.body()).byteStream(); + protected OkHttpClient.Builder getBuilder() { + OkHttpOAuthConsumer consumer = new OkHttpOAuthConsumer(consumerKey, consumerSecret); + consumer.setTokenWithSecret(accessToken, accessSecret); + return new OkHttpClient.Builder() + .addInterceptor(new SigningInterceptor(consumer)); } /** @@ -183,7 +148,7 @@ public String getCurrentUser() { Map params = new HashMap<>(); params.put(PARAM_ACTION, "query"); params.put("meta", "userinfo"); - JsonNode root = sendJsonRequest("GET", params); + JsonNode root = sendJsonRequest("POST", params); JsonNode nameNode = root.path("query").path("userinfo").path("name"); if (nameNode.isMissingNode()) { throw new AssertUserFailedException("The path \"query/userinfo/name\" doesn't exist in the json response"); @@ -216,39 +181,4 @@ public String getAccessSecret() { return accessSecret; } - @Override - public void setConnectTimeout(int timeout) { - super.setConnectTimeout(timeout); - updateTimeoutSetting(); - } - - @Override - public void setReadTimeout(int timeout) { - super.setReadTimeout(timeout); - updateTimeoutSetting(); - } - - private void updateTimeoutSetting() { - if (!loggedIn) throw new IllegalStateException("Cannot update connection settings after logging out"); - - // avoid instantiating new objects if possible - if (connectTimeout < 0 && readTimeout < 0) return; - - OkHttpClient.Builder builder = new OkHttpClient.Builder(); - - OkHttpOAuthConsumer consumer = new OkHttpOAuthConsumer(consumerKey, consumerSecret); - consumer.setTokenWithSecret(accessToken, accessSecret); - builder.addInterceptor(new SigningInterceptor(consumer)); - - if (connectTimeout >= 0) { - builder.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); - } - - if (readTimeout >= 0) { - builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS); - } - - // rebuild the client - client = builder.build(); - } } diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index 98463e6f5..693967936 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,27 +23,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyInt; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeSet; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.wikidata.wdtk.util.CompressionType; +import java.util.*; + +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.jetbrains.annotations.NotNull; +import org.junit.*; +import org.wikidata.wdtk.testing.MockStringContentFactory; import org.wikidata.wdtk.wikibaseapi.apierrors.AssertUserFailedException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; @@ -54,12 +48,18 @@ public class BasicApiConnectionTest { - final ObjectMapper mapper = new ObjectMapper(); + private final ObjectMapper mapper = new ObjectMapper(); - MockBasicApiConnection con; + private static MockWebServer server; + private BasicApiConnection connection; + + private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"" + server.url("/w/api.php").toString() + + "\",\"cookies\":{\"GeoIP\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"testwikidatawikiSession\":\"c18ef92637227283bcda73bcf95cfaf5\",\"WMF-Last-Access\":\"18-Aug-2015\"}," + + "\"username\":\"username\"," + + "\"loggedIn\":true," + + "\"connectTimeout\":-1," + + "\"readTimeout\":-1}\n"; - private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"connectTimeout\":-1,\"readTimeout\":-1,\"baseUrl\":\"https://mocked.api.connection/w/api.php\",\"loggedIn\":true,\"cookies\":{\"Path\":\"/\",\"GeoIP\":\"DE:13:Dresden:51.0500:13.7500:v4\",\" path\":\"/\",\" Domain\":\".wikidata.org\",\"testwikidatawikiSession\":\"c18ef92637227283bcda73bcf95cfaf5\",\" secure\":\"\",\"WMF-Last-Access\":\"18-Aug-2015\",\"Expires\":\"Sat, 19 Sep 2015 12:00:00 GMT\",\"HttpOnly\":\"\",\" Path\":\"/\",\" httponly\":\"\"},\"username\":\"username\"}"; - Set split(String str, char ch) { Set set = new TreeSet<>(); StringTokenizer stok = new StringTokenizer(str, "" + ch); @@ -69,155 +69,128 @@ Set split(String str, char ch) { return set; } - @Before - public void setUp() throws Exception { - this.con = new MockBasicApiConnection(); - Map params = new HashMap<>(); - params.put("action", "query"); - params.put("meta", "tokens"); - params.put("type", "csrf"); - params.put("format", "json"); - params.put("assert", "user"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/query-csrf-token-loggedin-response.json", CompressionType.NONE); - params.clear(); - params.put("action", "query"); - params.put("meta", "tokens"); - params.put("type", "login"); - params.put("format", "json"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/query-login-token.json", CompressionType.NONE); - params.clear(); - params.put("action", "login"); - params.put("lgname", "username"); - params.put("lgpassword", "password"); - params.put("lgtoken", "b5780b6e2f27e20b450921d9461010b4"); - params.put("format", "json"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/loginSuccess.json", CompressionType.NONE); - - params.clear(); - params.put("action", "login"); - params.put("lgname", "username2"); - params.put("lgpassword", "password2"); - params.put("lgtoken", "anothertoken"); - params.put("format", "json"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/loginError.json", CompressionType.NONE); - - params.clear(); - params.put("action", "login"); - params.put("lgname", "username3"); - params.put("lgpassword", "password3"); - params.put("lgtoken", "b5780b6e2f27e20b450921d9461010b4"); - params.put("format", "json"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/loginError2.json", CompressionType.NONE); + private static MockResponse makeJsonResponseFrom(String path) throws IOException { + String body = MockStringContentFactory.getStringFromUrl(BasicApiConnectionTest.class.getResource(path)); + return new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .addHeader("Set-Cookie", "WMF-Last-Access=18-Aug-2015;Path=/;HttpOnly;Expires=Sat, 19 Sep 2015 12:00:00 GMT") + .addHeader("Set-Cookie", "GeoIP=DE:13:Dresden:51.0500:13.7500:v4; Path=/; Domain=" + server.getHostName()) + .addHeader("Set-Cookie", "testwikidatawikiSession=c18ef92637227283bcda73bcf95cfaf5; path=/; secure; httponly") + .setBody(body); + } - params.clear(); - params.put("action", "logout"); - params.put("assert", "user"); - params.put("format", "json"); - params.put("token", "42307b93c79b0cb558d2dfb4c3c92e0955e06041+\\"); - this.con.setWebResource(params, "{}"); + @BeforeClass + public static void init() throws Exception { + Dispatcher dispatcher = new Dispatcher() { + @NotNull + @Override + public MockResponse dispatch(@NotNull RecordedRequest request) throws InterruptedException { + if ("/w/api.php?languages=fr&format=json&action=wbgetentities&ids=Q8&sitefilter=enwiki&props=info".equals(request.getPath())) { + return new MockResponse() + .setHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"entities\":{\"Q8\":{\"pageid\":134,\"ns\":0,\"title\":\"Q8\",\"lastrevid\":1174289176,\"modified\":\"2020-05-05T12:39:07Z\",\"type\":\"item\",\"id\":\"Q8\"}},\"success\":1}\n"); + } + try { + switch (request.getBody().readUtf8()) { + case "meta=tokens&format=json&action=query&type=login": + return makeJsonResponseFrom("/query-login-token.json"); + case "lgtoken=b5780b6e2f27e20b450921d9461010b4&lgpassword=password&format=json&action=login&lgname=username": + return makeJsonResponseFrom("/loginSuccess.json"); + case "lgtoken=b5780b6e2f27e20b450921d9461010b4&lgpassword=password1&format=json&action=login&lgname=username1": + return makeJsonResponseFrom("/loginError.json"); + case "meta=tokens&assert=user&format=json&action=query&type=csrf": + return makeJsonResponseFrom("/query-csrf-token-loggedin-response.json"); + case "assert=user&format=json&action=logout&token=42307b93c79b0cb558d2dfb4c3c92e0955e06041%2B%5C": + return new MockResponse().setHeader("Content-Type", "application/json; charset=utf-8").setBody("{}"); + case "assert=user&format=json&action=query": + return makeJsonResponseFrom("/assert-user-failed.json"); + } + }catch (Exception e) { + return new MockResponse().setResponseCode(404); + } + return new MockResponse().setResponseCode(404); + } + }; + + server = new MockWebServer(); + server.setDispatcher(dispatcher); + server.start(); + } - params.clear(); - params.put("action", "query"); - params.put("assert", "user"); - params.put("format", "json"); - this.con.setWebResourceFromPath(params, this.getClass(), - "/assert-user-failed.json", CompressionType.NONE); - params.clear(); + @AfterClass + public static void finish() throws IOException { + server.shutdown(); } - @Test - public void testGetToken() throws LoginFailedException, IOException, MediaWikiApiErrorException { - this.con.login("username", "password"); - assertNotNull(this.con.getOrFetchToken("csrf")); + @Before + public void setUp() { + connection = new BasicApiConnection(server.url("/w/api.php").toString()); } - @Test(expected = IOException.class) - public void testGetTokenWithoutLogin() throws IOException, MediaWikiApiErrorException { - this.con.getOrFetchToken("csrf"); + @Test + public void testGetToken() throws LoginFailedException, IOException, MediaWikiApiErrorException, InterruptedException { + connection.login("username", "password"); + assertNotNull(connection.getOrFetchToken("csrf")); } @Test - public void testGetLoginToken() throws IOException, MediaWikiApiErrorException { - assertNotNull(this.con.getOrFetchToken("login")); + public void testGetLoginToken() throws IOException, MediaWikiApiErrorException, InterruptedException, LoginFailedException { + assertNotNull(connection.getOrFetchToken("login")); } @Test public void testConfirmLogin() throws LoginFailedException, IOException, MediaWikiApiErrorException { - String token = this.con.getOrFetchToken("login"); - this.con.confirmLogin(token, "username", "password"); + String token = connection.getOrFetchToken("login"); + connection.confirmLogin(token, "username", "password"); } @Test public void testLogin() throws LoginFailedException { - assertFalse(this.con.loggedIn); - this.con.login("username", "password"); - assertEquals("username", this.con.getCurrentUser()); - assertEquals("password", this.con.password); - assertTrue(this.con.isLoggedIn()); + assertFalse(connection.loggedIn); + connection.login("username", "password"); + assertEquals("username", connection.getCurrentUser()); + assertEquals("password", connection.password); + assertTrue(connection.isLoggedIn()); } - + @Test public void testSerialize() throws LoginFailedException, IOException { - - Map> headerFields = new HashMap<>(); - List cookieList = testCookieList(); - headerFields.put("Set-Cookie", cookieList); - - con.login("username", "password"); - con.fillCookies(headerFields); - - - assertTrue(this.con.isLoggedIn()); - String jsonSerialization = mapper.writeValueAsString(this.con); + connection.login("username", "password"); + assertTrue(connection.isLoggedIn()); + String jsonSerialization = mapper.writeValueAsString(connection); JsonNode tree1 = mapper.readTree(LOGGED_IN_SERIALIZED_CONNECTION); JsonNode tree2 = mapper.readTree(jsonSerialization); Assert.assertEquals(tree1, tree2); } - + @Test public void testDeserialize() throws IOException { - - Map> headerFields = new HashMap>(); - List cookieList = testCookieList(); - headerFields.put("Set-Cookie", cookieList); - - ApiConnection newConn = mapper.readValue(LOGGED_IN_SERIALIZED_CONNECTION, BasicApiConnection.class); - assertTrue(newConn.isLoggedIn()); - assertEquals("username", newConn.getCurrentUser()); - assertEquals("https://mocked.api.connection/w/api.php", newConn.getApiBaseUrl()); + BasicApiConnection newConnection = mapper.readValue(LOGGED_IN_SERIALIZED_CONNECTION, BasicApiConnection.class); + assertTrue(newConnection.isLoggedIn()); + assertEquals("username", newConnection.getCurrentUser()); + assertEquals(-1, newConnection.getConnectTimeout()); + assertEquals(-1, newConnection.getReadTimeout()); + assertEquals(server.url("/w/api.php").toString(), newConnection.getApiBaseUrl()); + Map cookies = newConnection.getCookies(); + assertEquals("18-Aug-2015", cookies.get("WMF-Last-Access")); + assertEquals("DE:13:Dresden:51.0500:13.7500:v4", cookies.get("GeoIP")); + assertEquals("c18ef92637227283bcda73bcf95cfaf5", cookies.get("testwikidatawikiSession")); } @Test public void testLogout() throws IOException, LoginFailedException, MediaWikiApiErrorException { - this.con.login("username", "password"); - this.con.logout(); - assertEquals("", this.con.username); - assertEquals("", this.con.password); - assertFalse(this.con.loggedIn); - } - - @Test(expected = LoginFailedException.class) - public void loginErrors() throws LoginFailedException { - // This will fail because the token returned is not correct - this.con.login("username2", "password2"); + connection.login("username", "password"); + connection.logout(); + assertEquals("", connection.username); + assertEquals("", connection.password); + assertFalse(connection.loggedIn); } @Test(expected = LoginFailedException.class) public void loginUserErrors() throws LoginFailedException { // This will fail because the user is not known - this.con.login("username3", "password3"); - } - - @Test(expected = LoginFailedException.class) - public void loginIoErrors() throws LoginFailedException { - // This will fail because the request will throw an IO exception - this.con.login("notmocked", "notmocked"); + connection.login("username1", "password1"); } @Test @@ -231,7 +204,7 @@ public void testGetQueryString() { assertEquals( split("lgtoken=b5780b6e2f27e20b450921d9461010b4&lgpassword=password" + "&action=login&lgname=username&format=json", '&'), - split(con.getQueryString(params), '&')); + split(connection.getQueryString(params), '&')); } @Test @@ -239,7 +212,7 @@ public void testWarnings() throws IOException { JsonNode root; URL path = this.getClass().getResource("/warnings.json"); root = mapper.readTree(path.openStream()); - List warnings = con.getWarnings(root); + List warnings = connection.getWarnings(root); List expectedWarnings = Arrays .asList("[main]: Unrecognized parameter: 'rmparam'", "[query]: Unrecognized value for parameter 'list': raremodule", @@ -254,58 +227,14 @@ public void testErrors() throws IOException, JsonNode root; URL path = this.getClass().getResource("/error.json"); root = mapper.readTree(path.openStream()); - con.checkErrors(root); - } - - @Test - public void testFillCookies() { - Map> headerFields = new HashMap<>(); - List cookieList = testCookieList(); - headerFields.put("Set-Cookie", cookieList); - con.fillCookies(headerFields); - assertEquals(con.cookies.get(" Domain"), ".wikidata.org"); - } - - @Test - public void testFillCookiesCaseInsensitive() { - // for https://github.com/Wikidata/Wikidata-Toolkit/issues/491 - Map> headerFields = new HashMap<>(); - List cookieList = testCookieList(); - headerFields.put("set-cookie", cookieList); - con.fillCookies(headerFields); - assertEquals(con.cookies.get(" Domain"), ".wikidata.org"); - } - - @Test - public void testGetCookieString() { - Map> headerFields = new HashMap<>(); - List cookieList = testCookieList(); - headerFields.put("Set-Cookie", cookieList); - con.fillCookies(headerFields); - assertEquals( - split("HttpOnly; httponly; Path=/; GeoIP=DE:13:Dresden:51.0500:13.7500:v4; " - + "Domain=.wikidata.org; Expires=Sat, 19 Sep 2015 12:00:00 GMT; secure; path=/; " - + "testwikidatawikiSession=c18ef92637227283bcda73bcf95cfaf5; WMF-Last-Access=18-Aug-2015; Path=/", ';'), - split(con.getCookieString(), ';')); - } - - @Test - public void testSetupConnection() throws IOException { - URL url = new URL("http://example.org/"); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - con.setupConnection("POST", "", connection); - assertEquals("POST", - connection.getRequestMethod()); - assertEquals("application/x-www-form-urlencoded", - connection.getRequestProperty("Content-Type")); - + connection.checkErrors(root); } @Test public void testClearCookies() throws IOException, MediaWikiApiErrorException { - con.cookies.put("Content", "some content"); - con.clearCookies(); - assertTrue(con.cookies.keySet().isEmpty()); + connection.cookies.put("Content", "some content"); + connection.clearCookies(); + assertTrue(connection.cookies.keySet().isEmpty()); } @Test @@ -348,16 +277,16 @@ public void testErrorMessages() { i++; } } - + @Test(expected = AssertUserFailedException.class) public void testCheckCredentials() throws IOException, MediaWikiApiErrorException, LoginFailedException { // we first login successfully - this.con.login("username", "password"); - assertTrue(this.con.isLoggedIn()); + connection.login("username", "password"); + assertTrue(connection.isLoggedIn()); // after a while, the credentials expire - this.con.checkCredentials(); + connection.checkCredentials(); } - + /** * For backwards compatibility: by defaults, no timeouts * are set by us, we use HttpURLConnection's defaults. @@ -365,47 +294,36 @@ public void testCheckCredentials() throws IOException, MediaWikiApiErrorExceptio */ @Test public void testNoTimeouts() throws IOException { - HttpURLConnection urlConn = mock(HttpURLConnection.class); - - this.con.setupConnection("GET", "foo=bar", urlConn); - - verify(urlConn, times(0)).setConnectTimeout(anyInt()); - verify(urlConn, times(0)).setReadTimeout(anyInt()); + assertEquals(-1, connection.getConnectTimeout()); + assertEquals(-1, connection.getReadTimeout()); } - + @Test public void testConnectTimeout() throws IOException { - HttpURLConnection urlConn = mock(HttpURLConnection.class); - - this.con.setConnectTimeout(1000); - this.con.setupConnection("GET", "foo=bar", urlConn); - - assertEquals(con.getConnectTimeout(), 1000); - verify(urlConn, times(1)).setConnectTimeout(1000); - verify(urlConn, times(0)).setReadTimeout(anyInt()); + connection.setConnectTimeout(5000); + assertEquals(5000, connection.getConnectTimeout()); } - + @Test public void testReadTimeout() throws IOException { - HttpURLConnection urlConn = mock(HttpURLConnection.class); - - this.con.setReadTimeout(2000); - this.con.setupConnection("GET", "foo=bar", urlConn); - - assertEquals(con.getReadTimeout(), 2000); - verify(urlConn, times(0)).setConnectTimeout(anyInt()); - verify(urlConn, times(1)).setReadTimeout(2000); + connection.setReadTimeout(5000); + assertEquals(5000, connection.getReadTimeout()); } - private List testCookieList() { - List cookieList = new ArrayList<>(); - cookieList - .add("WMF-Last-Access=18-Aug-2015;Path=/;HttpOnly;Expires=Sat, 19 Sep 2015 12:00:00 GMT"); - cookieList - .add("GeoIP=DE:13:Dresden:51.0500:13.7500:v4; Path=/; Domain=.wikidata.org"); - cookieList - .add("testwikidatawikiSession=c18ef92637227283bcda73bcf95cfaf5; path=/; secure; httponly"); - return cookieList; + @Test + public void testGetMethod() throws IOException, MediaWikiApiErrorException { + Map parameters = new HashMap<>(); + parameters.put("action", "wbgetentities"); + parameters.put("languages", "fr"); + parameters.put("ids", "Q8"); + parameters.put("sitefilter", "enwiki"); + parameters.put("props", "info"); + JsonNode root = connection.sendJsonRequest("GET", parameters); + assertEquals("{\"entities\":{\"Q8\":{\"pageid\":134,\"ns\":0,\"title\":\"Q8\",\"lastrevid\":1174289176,\"modified\":\"2020-05-05T12:39:07Z\",\"type\":\"item\",\"id\":\"Q8\"}},\"success\":1}", mapper.writeValueAsString(root)); } + @Test(expected = IllegalArgumentException.class) + public void testUnsupportedMethod() throws IOException, MediaWikiApiErrorException { + connection.sendJsonRequest("PUT", new HashMap<>()); + } } diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java index 133a57910..e7d45ca77 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.HttpUrl; +import okhttp3.mockwebserver.Dispatcher; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; +import org.jetbrains.annotations.NotNull; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.EntityDocument; @@ -45,24 +49,53 @@ public class OAuthApiConnectionTest { private static final String ACCESS_TOKEN = "access_token"; private static final String ACCESS_SECRET = "access_secret"; + private static MockWebServer server; + private OAuthApiConnection connection; + private final ObjectMapper mapper = new ObjectMapper(); - private static final String NOT_LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"http://kubernetes.docker.internal:55690/w/api.php\",\"consumerKey\":null,\"consumerSecret\":null,\"accessToken\":null,\"accessSecret\":null,\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":false,\"username\":\"\"}"; - private static final String LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"http://kubernetes.docker.internal:55178/w/api.php\",\"consumerKey\":\"consumer_key\",\"consumerSecret\":\"consumer_secret\",\"accessToken\":\"access_token\",\"accessSecret\":\"access_secret\",\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":true,\"username\":\"foo\"}"; + private final String NOT_LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\",\"consumerKey\":null,\"consumerSecret\":null,\"accessToken\":null,\"accessSecret\":null,\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":false,\"username\":\"\"}"; + private final String LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\",\"consumerKey\":\"consumer_key\",\"consumerSecret\":\"consumer_secret\",\"accessToken\":\"access_token\",\"accessSecret\":\"access_secret\",\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":true,\"username\":\"foo\"}"; + + @BeforeClass + public static void init() throws IOException { + Dispatcher dispatcher = new Dispatcher() { + @NotNull + @Override + public MockResponse dispatch(@NotNull RecordedRequest request) throws InterruptedException { + switch (request.getBody().readUtf8()) { + case "languages=fr&assert=user&format=json&action=wbgetentities&ids=Q8&sitefilter=enwiki&props=info%7Cdatatype%7Clabels%7Caliases%7Cdescriptions%7Csitelinks": + return new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"entities\":{\"Q8\":{\"pageid\":134,\"ns\":0,\"title\":\"Q8\",\"lastrevid\":1174289176,\"modified\":\"2020-05-05T12:39:07Z\",\"type\":\"item\",\"id\":\"Q8\",\"labels\":{\"fr\":{\"language\":\"fr\",\"value\":\"bonheur\"}},\"descriptions\":{\"fr\":{\"language\":\"fr\",\"value\":\"état émotionnel\"}},\"aliases\":{\"fr\":[{\"language\":\"fr\",\"value\":\":)\"},{\"language\":\"fr\",\"value\":\"\uD83D\uDE04\"},{\"language\":\"fr\",\"value\":\"\uD83D\uDE03\"}]},\"sitelinks\":{\"enwiki\":{\"site\":\"enwiki\",\"title\":\"Happiness\",\"badges\":[]}}}},\"success\":1}"); + case "meta=userinfo&assert=user&format=json&action=query": + return new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody("{\"batchcomplete\":\"\",\"query\":{\"userinfo\":{\"id\":2333,\"name\":\"foo\"}}}"); + default: + return new MockResponse().setResponseCode(404); + } + } + }; + + server = new MockWebServer(); + server.setDispatcher(dispatcher); + server.start(); + } + + @AfterClass + public static void finish() throws IOException { + server.shutdown(); + } + + @Before + public void setUp() { + connection = new OAuthApiConnection(server.url("/w/api.php").toString(), + CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET); + } @Test public void testFetchOnlineData() throws IOException, MediaWikiApiErrorException, InterruptedException { - MockWebServer server = new MockWebServer(); - server.enqueue(new MockResponse().addHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"entities\":{\"Q8\":{\"pageid\":134,\"ns\":0,\"title\":\"Q8\",\"lastrevid\":1174289176,\"modified\":\"2020-05-05T12:39:07Z\",\"type\":\"item\",\"id\":\"Q8\",\"labels\":{\"fr\":{\"language\":\"fr\",\"value\":\"bonheur\"}},\"descriptions\":{\"fr\":{\"language\":\"fr\",\"value\":\"état émotionnel\"}},\"aliases\":{\"fr\":[{\"language\":\"fr\",\"value\":\":)\"},{\"language\":\"fr\",\"value\":\"\uD83D\uDE04\"},{\"language\":\"fr\",\"value\":\"\uD83D\uDE03\"}]},\"sitelinks\":{\"enwiki\":{\"site\":\"enwiki\",\"title\":\"Happiness\",\"badges\":[]}}}},\"success\":1}")); - server.start(); - HttpUrl apiBaseUrl = server.url("/w/api.php"); - ApiConnection connection = new OAuthApiConnection(apiBaseUrl.toString(), - CONSUMER_KEY, CONSUMER_SECRET, - ACCESS_TOKEN, ACCESS_SECRET); - assertTrue(connection.isLoggedIn()); - - // We can still fetch unprotected data without logging in. WikibaseDataFetcher wbdf = new WikibaseDataFetcher(connection, Datamodel.SITE_WIKIDATA); wbdf.getFilter().setSiteLinkFilter(Collections.singleton("enwiki")); wbdf.getFilter().setLanguageFilter(Collections.singleton("fr")); @@ -79,25 +112,10 @@ public void testFetchOnlineData() throws IOException, MediaWikiApiErrorException } assertEquals("The French label for entity Q8 is bonheur\n" + "and its English Wikipedia page has the title Happiness.", result); - - RecordedRequest request = server.takeRequest(); - assertNotNull(request.getHeader("Authorization")); - - server.shutdown(); } @Test - public void testLogout() throws IOException, LoginFailedException, InterruptedException { - MockWebServer server = new MockWebServer(); - // user info - server.enqueue(new MockResponse() - .addHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"batchcomplete\":\"\",\"query\":{\"userinfo\":{\"id\":2333,\"name\":\"foo\"}}}")); - server.start(); - HttpUrl apiBaseUrl = server.url("w/api.php"); - - OAuthApiConnection connection = new OAuthApiConnection(apiBaseUrl.toString(), - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET); + public void testLogout() throws IOException, InterruptedException { assertTrue(connection.isLoggedIn()); assertEquals("foo", connection.getCurrentUser()); @@ -108,58 +126,24 @@ public void testLogout() throws IOException, LoginFailedException, InterruptedEx RecordedRequest request = server.takeRequest(); assertNotNull(request.getHeader("Authorization")); - - assertEquals("/w/api.php?meta=userinfo&assert=user&format=json&action=query", request.getPath()); - - server.shutdown(); } @Test public void testSerialize() throws IOException, LoginFailedException { - MockWebServer server = new MockWebServer(); - // user info - server.enqueue(new MockResponse() - .addHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"batchcomplete\":\"\",\"query\":{\"userinfo\":{\"id\":2333,\"name\":\"foo\"}}}")); - server.start(); - HttpUrl apiBaseUrl = server.url("w/api.php"); - - OAuthApiConnection connection = new OAuthApiConnection(apiBaseUrl.toString(), - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET); String jsonSerialization = mapper.writeValueAsString(connection); - // The baseUrl field may be different because the server port is random in every run. - // But other fields are required to be the same. - assertEquals(LOGGED_IN_SERIALIZED.substring(LOGGED_IN_SERIALIZED.indexOf("/w/api.php")), - jsonSerialization.substring(jsonSerialization.indexOf("/w/api.php"))); - server.shutdown(); + assertEquals(LOGGED_IN_SERIALIZED, jsonSerialization); } @Test public void testDeserialize() throws IOException { - OAuthApiConnection connection = mapper.readValue(LOGGED_IN_SERIALIZED, OAuthApiConnection.class); - assertTrue(connection.isLoggedIn()); - assertEquals(CONSUMER_KEY, connection.getConsumerKey()); - assertEquals(CONSUMER_SECRET, connection.getConsumerSecret()); - assertEquals(ACCESS_TOKEN, connection.getAccessToken()); - assertEquals(ACCESS_SECRET, connection.getAccessSecret()); - assertEquals("http://kubernetes.docker.internal:55178/w/api.php", connection.getApiBaseUrl()); - - // To get the username, we have to start a server at http://kubernetes.docker.internal:55178/w/api.php - // and return the corresponding json response. - // But we cannot control the port of the mocked server, so we just change the url here. - MockWebServer server = new MockWebServer(); - // user info - server.enqueue(new MockResponse() - .addHeader("Content-Type", "application/json; charset=utf-8") - .setBody("{\"batchcomplete\":\"\",\"query\":{\"userinfo\":{\"id\":2333,\"name\":\"foo\"}}}")); - server.start(); - HttpUrl apiBaseUrl = server.url("w/api.php"); - OAuthApiConnection connection1 = new OAuthApiConnection(apiBaseUrl.toString(), - connection.getConsumerKey(), connection.getConsumerSecret(), - connection.getAccessToken(), connection.getAccessSecret()); - - assertEquals("foo", connection1.getCurrentUser()); - server.shutdown(); + OAuthApiConnection newConnection = mapper.readValue(LOGGED_IN_SERIALIZED, OAuthApiConnection.class); + assertTrue(newConnection.isLoggedIn()); + assertEquals(CONSUMER_KEY, newConnection.getConsumerKey()); + assertEquals(CONSUMER_SECRET, newConnection.getConsumerSecret()); + assertEquals(ACCESS_TOKEN, newConnection.getAccessToken()); + assertEquals(ACCESS_SECRET, newConnection.getAccessSecret()); + assertEquals(server.url("/w/api.php").toString(), newConnection.getApiBaseUrl()); + assertEquals("foo", newConnection.getCurrentUser()); } @Test @@ -170,28 +154,6 @@ public void testDeserializeNotLogin() throws IOException { assertNull(CONSUMER_SECRET, connection.getConsumerSecret()); assertNull(ACCESS_TOKEN, connection.getAccessToken()); assertNull(ACCESS_SECRET, connection.getAccessSecret()); - assertEquals("http://kubernetes.docker.internal:55690/w/api.php", connection.getApiBaseUrl()); - } - - - @Test - public void testConnectionTimeout() { - ApiConnection connection = new OAuthApiConnection(ApiConnection.URL_WIKIDATA_API, - CONSUMER_KEY, CONSUMER_SECRET, - ACCESS_TOKEN, ACCESS_SECRET); - connection.setConnectTimeout(2000); - assertEquals(2000, connection.getConnectTimeout()); - assertEquals(-1, connection.getReadTimeout()); + assertEquals(server.url("/w/api.php").toString(), connection.getApiBaseUrl()); } - - @Test - public void testReadTimeout() { - ApiConnection connection = new OAuthApiConnection(ApiConnection.URL_WIKIDATA_API, - CONSUMER_KEY, CONSUMER_SECRET, - ACCESS_TOKEN, ACCESS_SECRET); - connection.setReadTimeout(2000); - assertEquals(-1, connection.getConnectTimeout()); - assertEquals(2000, connection.getReadTimeout()); - } - } diff --git a/wdtk-wikibaseapi/src/test/resources/loginError.json b/wdtk-wikibaseapi/src/test/resources/loginError.json index 256e095af..dc9f307c3 100644 --- a/wdtk-wikibaseapi/src/test/resources/loginError.json +++ b/wdtk-wikibaseapi/src/test/resources/loginError.json @@ -1 +1 @@ -{"login":{"result":"WrongToken"}} +{"login":{"result":"NotExists"}} diff --git a/wdtk-wikibaseapi/src/test/resources/loginError2.json b/wdtk-wikibaseapi/src/test/resources/loginError2.json deleted file mode 100644 index dc9f307c3..000000000 --- a/wdtk-wikibaseapi/src/test/resources/loginError2.json +++ /dev/null @@ -1 +0,0 @@ -{"login":{"result":"NotExists"}} From 33cc4b3b5889973073f976311cbf8d3ce59aa778 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Sat, 6 Jun 2020 09:13:34 +0800 Subject: [PATCH 03/10] clean up @NotNull annotations --- .../java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java | 2 -- .../org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java | 6 ++---- .../wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java | 8 ++------ .../wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java | 5 ++--- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index 568bb1fa4..fc803c942 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.wikidata.wdtk.wikibaseapi.apierrors.AssertUserFailedException; @@ -100,7 +99,6 @@ public abstract class ApiConnection { /** * Map of requested tokens. */ - @NotNull protected final Map tokens; /** diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index 9fcc1b096..2a6e15339 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -31,7 +31,6 @@ import okhttp3.CookieJar; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; -import org.jetbrains.annotations.NotNull; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import com.fasterxml.jackson.annotation.JsonCreator; @@ -383,15 +382,14 @@ public ApiConnectionCookieJar() { } @Override - public void saveFromResponse(@NotNull HttpUrl url, @NotNull List cookieList) { + public void saveFromResponse(HttpUrl url, List cookieList) { for (Cookie cookie : cookieList) { cookies.put(cookie.name(), cookie.value()); } } - @NotNull @Override - public List loadForRequest(@NotNull HttpUrl url) { + public List loadForRequest(HttpUrl url) { List cookieList = new ArrayList<>(); for (Map.Entry entry : cookies.entrySet()) { Cookie cookie = new Cookie.Builder() diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index 693967936..aca7295f2 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -23,19 +23,15 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import java.io.IOException; import java.net.URL; import java.util.*; -import okhttp3.HttpUrl; import okhttp3.mockwebserver.Dispatcher; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -import org.jetbrains.annotations.NotNull; import org.junit.*; import org.wikidata.wdtk.testing.MockStringContentFactory; import org.wikidata.wdtk.wikibaseapi.apierrors.AssertUserFailedException; @@ -82,9 +78,9 @@ private static MockResponse makeJsonResponseFrom(String path) throws IOException @BeforeClass public static void init() throws Exception { Dispatcher dispatcher = new Dispatcher() { - @NotNull + @Override - public MockResponse dispatch(@NotNull RecordedRequest request) throws InterruptedException { + public MockResponse dispatch(RecordedRequest request) throws InterruptedException { if ("/w/api.php?languages=fr&format=json&action=wbgetentities&ids=Q8&sitefilter=enwiki&props=info".equals(request.getPath())) { return new MockResponse() .setHeader("Content-Type", "application/json; charset=utf-8") diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java index e7d45ca77..54d6d86f0 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java @@ -26,7 +26,6 @@ import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -import org.jetbrains.annotations.NotNull; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -60,9 +59,9 @@ public class OAuthApiConnectionTest { @BeforeClass public static void init() throws IOException { Dispatcher dispatcher = new Dispatcher() { - @NotNull + @Override - public MockResponse dispatch(@NotNull RecordedRequest request) throws InterruptedException { + public MockResponse dispatch(RecordedRequest request) throws InterruptedException { switch (request.getBody().readUtf8()) { case "languages=fr&assert=user&format=json&action=wbgetentities&ids=Q8&sitefilter=enwiki&props=info%7Cdatatype%7Clabels%7Caliases%7Cdescriptions%7Csitelinks": return new MockResponse() From 97442ecc4589e51edfd0cc506a29484bb7e9abd7 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Sat, 6 Jun 2020 09:16:07 +0800 Subject: [PATCH 04/10] rename getBuilder to getClientBuilder --- .../java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java | 8 ++++---- .../org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java | 2 +- .../org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index fc803c942..f4020d616 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -161,7 +161,7 @@ public ApiConnection(String apiBaseUrl, Map tokens) { * .cookieJar(...); * */ - protected abstract OkHttpClient.Builder getBuilder(); + protected abstract OkHttpClient.Builder getClientBuilder(); /** * Getter for the apiBaseUrl. @@ -219,7 +219,7 @@ public String getCurrentUser() { public void setReadTimeout(int timeout) { readTimeout = timeout; if (timeout >= 0) { - OkHttpClient.Builder builder = getBuilder(); + OkHttpClient.Builder builder = getClientBuilder(); builder.readTimeout(timeout, TimeUnit.MILLISECONDS); client = builder.build(); } @@ -234,7 +234,7 @@ public void setReadTimeout(int timeout) { public void setConnectTimeout(int timeout) { connectTimeout = timeout; if (timeout >= 0) { - OkHttpClient.Builder builder = getBuilder(); + OkHttpClient.Builder builder = getClientBuilder(); builder.connectTimeout(timeout, TimeUnit.MILLISECONDS); client = builder.build(); } @@ -379,7 +379,7 @@ public InputStream sendRequest(String requestMethod, } if (client == null) { - client = getBuilder().build(); + client = getClientBuilder().build(); } Response response = client.newCall(request).execute(); return Objects.requireNonNull(response.body()).byteStream(); diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index 2a6e15339..a441c4699 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -170,7 +170,7 @@ protected BasicApiConnection( } @Override - protected OkHttpClient.Builder getBuilder() { + protected OkHttpClient.Builder getClientBuilder() { return new OkHttpClient.Builder() .cookieJar(new ApiConnectionCookieJar()); } diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java index 23d0cdeed..2ff1a711d 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java @@ -84,7 +84,7 @@ public OAuthApiConnection( } @Override - protected OkHttpClient.Builder getBuilder() { + protected OkHttpClient.Builder getClientBuilder() { OkHttpOAuthConsumer consumer = new OkHttpOAuthConsumer(consumerKey, consumerSecret); consumer.setTokenWithSecret(accessToken, accessSecret); return new OkHttpClient.Builder() From 7649482b47a9a28cfef5fe9cdd9e8f702c80fc4c Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Sat, 6 Jun 2020 10:21:45 +0800 Subject: [PATCH 05/10] fix timeout settings bug --- .../wdtk/wikibaseapi/ApiConnection.java | 35 +++++++++++-------- .../wikibaseapi/BasicApiConnectionTest.java | 8 +++++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index f4020d616..e369c5b61 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -211,33 +211,40 @@ public String getCurrentUser() { } /** - * Sets the maximum time to wait for a server response once the connection was established, in milliseconds. + * Sets the maximum time to wait for when establishing a connection, in milliseconds. * For negative values, no timeout is set. * - * @see HttpURLConnection#setReadTimeout + * @see HttpURLConnection#setConnectTimeout */ - public void setReadTimeout(int timeout) { - readTimeout = timeout; + public void setConnectTimeout(int timeout) { + connectTimeout = timeout; if (timeout >= 0) { - OkHttpClient.Builder builder = getClientBuilder(); - builder.readTimeout(timeout, TimeUnit.MILLISECONDS); - client = builder.build(); + updateTimeoutSettings(); } } /** - * Sets the maximum time to wait for when establishing a connection, in milliseconds. + * Sets the maximum time to wait for a server response once the connection was established, in milliseconds. * For negative values, no timeout is set. * - * @see HttpURLConnection#setConnectTimeout + * @see HttpURLConnection#setReadTimeout */ - public void setConnectTimeout(int timeout) { - connectTimeout = timeout; + public void setReadTimeout(int timeout) { + readTimeout = timeout; if (timeout >= 0) { - OkHttpClient.Builder builder = getClientBuilder(); - builder.connectTimeout(timeout, TimeUnit.MILLISECONDS); - client = builder.build(); + updateTimeoutSettings(); + } + } + + private void updateTimeoutSettings() { + OkHttpClient.Builder builder = getClientBuilder(); + if (connectTimeout >= 0) { + builder.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); + } + if (readTimeout >= 0) { + builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS); } + client = builder.build(); } /** diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index aca7295f2..b5ccf41d0 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -306,6 +306,14 @@ public void testReadTimeout() throws IOException { assertEquals(5000, connection.getReadTimeout()); } + @Test + public void testTimeouts() { + connection.setConnectTimeout(5000); + connection.setReadTimeout(5000); + assertEquals(5000, connection.getConnectTimeout()); + assertEquals(5000, connection.getReadTimeout()); + } + @Test public void testGetMethod() throws IOException, MediaWikiApiErrorException { Map parameters = new HashMap<>(); From 09bcee032468e100c5301694d7b5cf1bfb86e7ad Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Sat, 6 Jun 2020 14:51:09 +0800 Subject: [PATCH 06/10] serialize/deserialize connect timeout, read timeout and tokens --- .../wdtk/wikibaseapi/ApiConnection.java | 18 +++++-- .../wdtk/wikibaseapi/BasicApiConnection.java | 39 +++++---------- .../wdtk/wikibaseapi/OAuthApiConnection.java | 49 ++++++++++++++++--- .../wikibaseapi/BasicApiConnectionTest.java | 18 ++++--- .../wikibaseapi/OAuthApiConnectionTest.java | 28 ++++++++++- 5 files changed, 106 insertions(+), 46 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index e369c5b61..231e29b1a 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -21,6 +21,7 @@ */ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.*; @@ -166,6 +167,7 @@ public ApiConnection(String apiBaseUrl, Map tokens) { /** * Getter for the apiBaseUrl. */ + @JsonProperty("baseUrl") public String getApiBaseUrl() { return apiBaseUrl; } @@ -206,10 +208,18 @@ public void checkCredentials() throws IOException, MediaWikiApiErrorException { * * @return name of the logged in user */ + @JsonProperty("username") public String getCurrentUser() { return username; } + /** + * Returns the map of tokens (such as csrf token and login token) currently used in this connection. + */ + public Map getTokens() { + return Collections.unmodifiableMap(tokens); + } + /** * Sets the maximum time to wait for when establishing a connection, in milliseconds. * For negative values, no timeout is set. @@ -219,7 +229,7 @@ public String getCurrentUser() { public void setConnectTimeout(int timeout) { connectTimeout = timeout; if (timeout >= 0) { - updateTimeoutSettings(); + buildClient(); } } @@ -232,11 +242,11 @@ public void setConnectTimeout(int timeout) { public void setReadTimeout(int timeout) { readTimeout = timeout; if (timeout >= 0) { - updateTimeoutSettings(); + buildClient(); } } - private void updateTimeoutSettings() { + private void buildClient() { OkHttpClient.Builder builder = getClientBuilder(); if (connectTimeout >= 0) { builder.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); @@ -386,7 +396,7 @@ public InputStream sendRequest(String requestMethod, } if (client == null) { - client = getClientBuilder().build(); + buildClient(); } Response response = client.newCall(request).execute(); return Objects.requireNonNull(response.body()).byteStream(); diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index a441c4699..08d0d58ac 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -147,14 +147,13 @@ public BasicApiConnection(String apiBaseUrl) { /** * Deserializes an existing BasicApiConnection from JSON. * - * @param apiBaseUrl - * base URL of the API to use, e.g. "https://www.wikidata.org/w/api.php/" - * @param cookies - * map of cookies used for this session - * @param loggedIn - * true if login succeeded. - * @param tokens - * map of tokens used for this session + * @param apiBaseUrl base URL of the API to use, e.g. "https://www.wikidata.org/w/api.php/" + * @param cookies map of cookies used for this session + * @param username name of the current user + * @param loggedIn true if login succeeded. + * @param tokens map of tokens used for this session + * @param connectTimeout the maximum time to wait for when establishing a connection, in milliseconds + * @param readTimeout the maximum time to wait for a server response once the connection was established, in milliseconds */ @JsonCreator protected BasicApiConnection( @@ -162,11 +161,15 @@ protected BasicApiConnection( @JsonProperty("cookies") Map cookies, @JsonProperty("username") String username, @JsonProperty("loggedIn") boolean loggedIn, - @JsonProperty("tokens") Map tokens) { + @JsonProperty("tokens") Map tokens, + @JsonProperty("connectTimeout") int connectTimeout, + @JsonProperty("readTimeout") int readTimeout) { super(apiBaseUrl, tokens); this.username = username; this.cookies = cookies; this.loggedIn = loggedIn; + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; } @Override @@ -270,24 +273,6 @@ protected void confirmLogin(String token, String username, String password) } } - @Override - @JsonProperty("baseUrl") - public String getApiBaseUrl() { - return super.getApiBaseUrl(); - } - - @Override - @JsonProperty("loggedIn") - public boolean isLoggedIn() { - return super.isLoggedIn(); - } - - @Override - @JsonProperty("username") - public String getCurrentUser() { - return super.getCurrentUser(); - } - /** * Returns the map of cookies currently used in this connection. */ diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java index 2ff1a711d..9768409c9 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; -import okhttp3.*; +import okhttp3.OkHttpClient; import org.wikidata.wdtk.wikibaseapi.apierrors.AssertUserFailedException; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import se.akerfeldt.okhttp.signpost.OkHttpOAuthConsumer; @@ -68,19 +68,54 @@ public class OAuthApiConnection extends ApiConnection { * @param accessToken the access token obtained via the OAuth process * @param accessSecret the secret key obtained via the OAuth process */ + public OAuthApiConnection(String apiBaseUrl, + String consumerKey, + String consumerSecret, + String accessToken, + String accessSecret) { + super(apiBaseUrl, null); + this.consumerKey = consumerKey; + this.consumerSecret = consumerSecret; + this.accessToken = accessToken; + this.accessSecret = accessSecret; + loggedIn = true; + } + + /** + * Deserializes an existing OAuthApiConnection from JSON. + * + * @param apiBaseUrl the MediaWiki API endpoint, such as "https://www.wikidata.org/w/api.php" + * @param consumerKey the OAuth 1.0a consumer key + * @param consumerSecret the OAuth 1.0a consumer secret + * @param accessToken the access token obtained via the OAuth process + * @param accessSecret the secret key obtained via the OAuth process + * @param username name of the current user + * @param loggedIn true if login succeeded. + * @param tokens map of tokens used for this session + * @param connectTimeout the maximum time to wait for when establishing a connection, in milliseconds + * @param readTimeout the maximum time to wait for a server response once the connection was established, in milliseconds + */ @JsonCreator - public OAuthApiConnection( + protected OAuthApiConnection( @JsonProperty("baseUrl") String apiBaseUrl, @JsonProperty("consumerKey") String consumerKey, @JsonProperty("consumerSecret") String consumerSecret, @JsonProperty("accessToken") String accessToken, - @JsonProperty("accessSecret") String accessSecret) { - super(apiBaseUrl, null); + @JsonProperty("accessSecret") String accessSecret, + @JsonProperty("username") String username, + @JsonProperty("loggedIn") boolean loggedIn, + @JsonProperty("tokens") Map tokens, + @JsonProperty("connectTimeout") int connectTimeout, + @JsonProperty("readTimeout") int readTimeout) { + super(apiBaseUrl, tokens); this.consumerKey = consumerKey; this.consumerSecret = consumerSecret; this.accessToken = accessToken; this.accessSecret = accessSecret; - loggedIn = true; + this.username = username; + this.loggedIn = loggedIn; + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; } @Override diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index b5ccf41d0..848784a54 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -49,12 +49,13 @@ public class BasicApiConnectionTest { private static MockWebServer server; private BasicApiConnection connection; - private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"" + server.url("/w/api.php").toString() + - "\",\"cookies\":{\"GeoIP\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"testwikidatawikiSession\":\"c18ef92637227283bcda73bcf95cfaf5\",\"WMF-Last-Access\":\"18-Aug-2015\"}," + + private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"" + server.url("/w/api.php").toString() + "\"," + + "\"cookies\":{\"GeoIP\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"testwikidatawikiSession\":\"c18ef92637227283bcda73bcf95cfaf5\",\"WMF-Last-Access\":\"18-Aug-2015\"}," + "\"username\":\"username\"," + "\"loggedIn\":true," + - "\"connectTimeout\":-1," + - "\"readTimeout\":-1}\n"; + "\"tokens\":{\"login\":\"b5780b6e2f27e20b450921d9461010b4\"}," + + "\"connectTimeout\":5000," + + "\"readTimeout\":6000}\n"; Set split(String str, char ch) { Set set = new TreeSet<>(); @@ -153,6 +154,8 @@ public void testLogin() throws LoginFailedException { @Test public void testSerialize() throws LoginFailedException, IOException { connection.login("username", "password"); + connection.setConnectTimeout(5000); + connection.setReadTimeout(6000); assertTrue(connection.isLoggedIn()); String jsonSerialization = mapper.writeValueAsString(connection); JsonNode tree1 = mapper.readTree(LOGGED_IN_SERIALIZED_CONNECTION); @@ -165,13 +168,16 @@ public void testDeserialize() throws IOException { BasicApiConnection newConnection = mapper.readValue(LOGGED_IN_SERIALIZED_CONNECTION, BasicApiConnection.class); assertTrue(newConnection.isLoggedIn()); assertEquals("username", newConnection.getCurrentUser()); - assertEquals(-1, newConnection.getConnectTimeout()); - assertEquals(-1, newConnection.getReadTimeout()); + assertEquals(5000, newConnection.getConnectTimeout()); + assertEquals(6000, newConnection.getReadTimeout()); assertEquals(server.url("/w/api.php").toString(), newConnection.getApiBaseUrl()); Map cookies = newConnection.getCookies(); assertEquals("18-Aug-2015", cookies.get("WMF-Last-Access")); assertEquals("DE:13:Dresden:51.0500:13.7500:v4", cookies.get("GeoIP")); assertEquals("c18ef92637227283bcda73bcf95cfaf5", cookies.get("testwikidatawikiSession")); + Map tokens = newConnection.getTokens(); + assertEquals("b5780b6e2f27e20b450921d9461010b4", tokens.get("login")); + assertNull(tokens.get("csrf")); } @Test diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java index 54d6d86f0..ee5e67a5e 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnectionTest.java @@ -53,8 +53,26 @@ public class OAuthApiConnectionTest { private final ObjectMapper mapper = new ObjectMapper(); - private final String NOT_LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\",\"consumerKey\":null,\"consumerSecret\":null,\"accessToken\":null,\"accessSecret\":null,\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":false,\"username\":\"\"}"; - private final String LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\",\"consumerKey\":\"consumer_key\",\"consumerSecret\":\"consumer_secret\",\"accessToken\":\"access_token\",\"accessSecret\":\"access_secret\",\"connectTimeout\":-1,\"readTimeout\":-1,\"loggedIn\":true,\"username\":\"foo\"}"; + private final String NOT_LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\"," + + "\"consumerKey\":null," + + "\"consumerSecret\":null," + + "\"accessToken\":null," + + "\"accessSecret\":null," + + "\"username\":\"\"," + + "\"loggedIn\":false," + + "\"tokens\":{}," + + "\"connectTimeout\":-1," + + "\"readTimeout\":-1}"; + private final String LOGGED_IN_SERIALIZED = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\"," + + "\"consumerKey\":\"consumer_key\"," + + "\"consumerSecret\":\"consumer_secret\"," + + "\"accessToken\":\"access_token\"," + + "\"accessSecret\":\"access_secret\"," + + "\"username\":\"foo\"," + + "\"loggedIn\":true," + + "\"tokens\":{}," + + "\"connectTimeout\":-1," + + "\"readTimeout\":-1}"; @BeforeClass public static void init() throws IOException { @@ -143,6 +161,9 @@ public void testDeserialize() throws IOException { assertEquals(ACCESS_SECRET, newConnection.getAccessSecret()); assertEquals(server.url("/w/api.php").toString(), newConnection.getApiBaseUrl()); assertEquals("foo", newConnection.getCurrentUser()); + assertEquals(-1, connection.getConnectTimeout()); + assertEquals(-1, connection.getReadTimeout()); + assertTrue(connection.getTokens().isEmpty()); } @Test @@ -154,5 +175,8 @@ public void testDeserializeNotLogin() throws IOException { assertNull(ACCESS_TOKEN, connection.getAccessToken()); assertNull(ACCESS_SECRET, connection.getAccessSecret()); assertEquals(server.url("/w/api.php").toString(), connection.getApiBaseUrl()); + assertEquals(-1, connection.getConnectTimeout()); + assertEquals(-1, connection.getReadTimeout()); + assertTrue(connection.getTokens().isEmpty()); } } From 320cca7d7fc63ae8c546c78f6d67bc78325d04e2 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Sat, 6 Jun 2020 15:29:22 +0800 Subject: [PATCH 07/10] fix timeout settings bug again :) --- .../wdtk/wikibaseapi/ApiConnection.java | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index 231e29b1a..aba5eb02c 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -92,6 +92,7 @@ public abstract class ApiConnection { * True after successful login. */ protected boolean loggedIn = false; + /** * User name used to log in. */ @@ -228,9 +229,7 @@ public Map getTokens() { */ public void setConnectTimeout(int timeout) { connectTimeout = timeout; - if (timeout >= 0) { - buildClient(); - } + client = null; } /** @@ -241,20 +240,7 @@ public void setConnectTimeout(int timeout) { */ public void setReadTimeout(int timeout) { readTimeout = timeout; - if (timeout >= 0) { - buildClient(); - } - } - - private void buildClient() { - OkHttpClient.Builder builder = getClientBuilder(); - if (connectTimeout >= 0) { - builder.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); - } - if (readTimeout >= 0) { - builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS); - } - client = builder.build(); + client = null; } /** @@ -402,6 +388,17 @@ public InputStream sendRequest(String requestMethod, return Objects.requireNonNull(response.body()).byteStream(); } + private void buildClient() { + OkHttpClient.Builder builder = getClientBuilder(); + if (connectTimeout >= 0) { + builder.connectTimeout(connectTimeout, TimeUnit.MILLISECONDS); + } + if (readTimeout >= 0) { + builder.readTimeout(readTimeout, TimeUnit.MILLISECONDS); + } + client = builder.build(); + } + /** * Checks if an API response contains an error and throws a suitable * exception in this case. From 1b311eca535877b281d9d1b24daa2ca8306a0a33 Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Mon, 8 Jun 2020 10:05:29 +0800 Subject: [PATCH 08/10] use JavaNetCookieJar to manager cookies --- pom.xml | 1 + wdtk-wikibaseapi/pom.xml | 7 +- .../wdtk/wikibaseapi/BasicApiConnection.java | 98 +++++++++++-------- .../wikibaseapi/BasicApiConnectionTest.java | 29 +++--- 4 files changed, 77 insertions(+), 58 deletions(-) diff --git a/pom.xml b/pom.xml index 0f9c2f672..b4c2ca692 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,7 @@ 1.7.30 1.5.0 1.1.0 + 4.2.2 diff --git a/wdtk-wikibaseapi/pom.xml b/wdtk-wikibaseapi/pom.xml index 958c9212f..19be8ef05 100644 --- a/wdtk-wikibaseapi/pom.xml +++ b/wdtk-wikibaseapi/pom.xml @@ -32,10 +32,15 @@ ${project.version} test + + com.squareup.okhttp3 + okhttp-urlconnection + ${okhttpVersion} + com.squareup.okhttp3 mockwebserver - 4.2.2 + ${okhttpVersion} test diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index 08d0d58ac..9cdfd8fd9 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -21,16 +21,12 @@ */ import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; +import java.net.*; import java.util.HashMap; import java.util.List; import java.util.Map; -import okhttp3.Cookie; -import okhttp3.CookieJar; -import okhttp3.HttpUrl; -import okhttp3.OkHttpClient; +import okhttp3.*; import org.wikidata.wdtk.wikibaseapi.apierrors.MediaWikiApiErrorException; import com.fasterxml.jackson.annotation.JsonCreator; @@ -126,10 +122,11 @@ public class BasicApiConnection extends ApiConnection { */ @JsonIgnore String password = ""; + /** - * Map of cookies that are currently set. + * Used for managing and serializing/deserializing cookies. */ - final Map cookies; + private final CookieManager cookieManager; /** * Creates an object to manage a connection to the Web API of a Wikibase @@ -141,7 +138,8 @@ public class BasicApiConnection extends ApiConnection { */ public BasicApiConnection(String apiBaseUrl) { super(apiBaseUrl); - this.cookies = new HashMap<>(); + cookieManager = new CookieManager(); + cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL); } /** @@ -158,7 +156,7 @@ public BasicApiConnection(String apiBaseUrl) { @JsonCreator protected BasicApiConnection( @JsonProperty("baseUrl") String apiBaseUrl, - @JsonProperty("cookies") Map cookies, + @JsonProperty("cookies") List cookies, @JsonProperty("username") String username, @JsonProperty("loggedIn") boolean loggedIn, @JsonProperty("tokens") Map tokens, @@ -166,16 +164,23 @@ protected BasicApiConnection( @JsonProperty("readTimeout") int readTimeout) { super(apiBaseUrl, tokens); this.username = username; - this.cookies = cookies; this.loggedIn = loggedIn; this.connectTimeout = connectTimeout; this.readTimeout = readTimeout; + + cookieManager = new CookieManager(); + cookieManager.setCookiePolicy(CookiePolicy.ACCEPT_ALL); + CookieStore cookieStore = cookieManager.getCookieStore(); + // We only deal with apiBaseUrl here. + URI uri = URI.create(apiBaseUrl); + cookies.stream().map(HttpCookieWrapper::toHttpCookie) + .forEach(cookie -> cookieStore.add(uri, cookie)); } @Override protected OkHttpClient.Builder getClientBuilder() { return new OkHttpClient.Builder() - .cookieJar(new ApiConnectionCookieJar()); + .cookieJar(new JavaNetCookieJar(cookieManager)); } /** @@ -276,9 +281,8 @@ protected void confirmLogin(String token, String username, String password) /** * Returns the map of cookies currently used in this connection. */ - @JsonProperty("cookies") - public Map getCookies() { - return Collections.unmodifiableMap(this.cookies); + public List getCookies() { + return cookieManager.getCookieStore().getCookies(); } /** @@ -288,7 +292,7 @@ public Map getCookies() { */ public void clearCookies() throws IOException, MediaWikiApiErrorException { logout(); - this.cookies.clear(); + cookieManager.getCookieStore().removeAll(); } /** @@ -354,37 +358,45 @@ public void logout() throws IOException, MediaWikiApiErrorException { } } - private class ApiConnectionCookieJar implements CookieJar { - - private String domain; + /** + * Wrapper for {@link HttpCookie}. + * + * Used for json deserialization. + * + * Since {@link HttpCookie} is final, we can't extend it here. + */ + protected static class HttpCookieWrapper { - public ApiConnectionCookieJar() { - int firstSlash = apiBaseUrl.indexOf("/", apiBaseUrl.indexOf("//") + 2); - domain = apiBaseUrl.substring(apiBaseUrl.indexOf("//") + 2, firstSlash); - if (domain.contains(":")) { // Cookies do not provide isolation by port. - domain = domain.substring(0, domain.indexOf(":")); - } - } + private HttpCookie httpCookie; - @Override - public void saveFromResponse(HttpUrl url, List cookieList) { - for (Cookie cookie : cookieList) { - cookies.put(cookie.name(), cookie.value()); - } + @JsonCreator + public HttpCookieWrapper(@JsonProperty("name") String name, + @JsonProperty("value") String value, + @JsonProperty("comment") String comment, + @JsonProperty("commentURL") String commentURL, + @JsonProperty("domain") String domain, + @JsonProperty("maxAge") int maxAge, + @JsonProperty("path") String path, + @JsonProperty("portlist") String portlist, + @JsonProperty("secure") boolean secure, + @JsonProperty("httpOnly") boolean httpOnly, + @JsonProperty("version") int version, + @JsonProperty("discard") boolean discard) { + httpCookie = new HttpCookie(name, value); + httpCookie.setComment(comment); + httpCookie.setCommentURL(commentURL); + httpCookie.setDomain(domain); + httpCookie.setMaxAge(maxAge); + httpCookie.setPath(path); + httpCookie.setPortlist(portlist); + httpCookie.setSecure(secure); + httpCookie.setHttpOnly(httpOnly); + httpCookie.setVersion(version); + httpCookie.setDiscard(discard); } - @Override - public List loadForRequest(HttpUrl url) { - List cookieList = new ArrayList<>(); - for (Map.Entry entry : cookies.entrySet()) { - Cookie cookie = new Cookie.Builder() - .domain(domain) - .name(entry.getKey()) - .value(entry.getValue()) - .build(); - cookieList.add(cookie); - } - return cookieList; + public HttpCookie toHttpCookie() { + return httpCookie; } } } diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index 848784a54..ab20c385e 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.net.HttpCookie; import java.net.URL; import java.util.*; @@ -49,13 +50,7 @@ public class BasicApiConnectionTest { private static MockWebServer server; private BasicApiConnection connection; - private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"" + server.url("/w/api.php").toString() + "\"," + - "\"cookies\":{\"GeoIP\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"testwikidatawikiSession\":\"c18ef92637227283bcda73bcf95cfaf5\",\"WMF-Last-Access\":\"18-Aug-2015\"}," + - "\"username\":\"username\"," + - "\"loggedIn\":true," + - "\"tokens\":{\"login\":\"b5780b6e2f27e20b450921d9461010b4\"}," + - "\"connectTimeout\":5000," + - "\"readTimeout\":6000}\n"; + private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"http://kubernetes.docker.internal:" + server.getPort() + "/w/api.php\",\"cookies\":[{\"name\":\"GeoIP\",\"value\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"comment\":null,\"commentURL\":null,\"domain\":\".kubernetes.docker.internal\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":false,\"httpOnly\":false,\"version\":0,\"discard\":false},{\"name\":\"testwikidatawikiSession\",\"value\":\"c18ef92637227283bcda73bcf95cfaf5\",\"comment\":null,\"commentURL\":null,\"domain\":\"kubernetes.docker.internal\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":true,\"httpOnly\":true,\"version\":0,\"discard\":false}],\"username\":\"username\",\"loggedIn\":true,\"tokens\":{\"login\":\"b5780b6e2f27e20b450921d9461010b4\"},\"connectTimeout\":5000,\"readTimeout\":6000}\n"; Set split(String str, char ch) { Set set = new TreeSet<>(); @@ -171,10 +166,15 @@ public void testDeserialize() throws IOException { assertEquals(5000, newConnection.getConnectTimeout()); assertEquals(6000, newConnection.getReadTimeout()); assertEquals(server.url("/w/api.php").toString(), newConnection.getApiBaseUrl()); - Map cookies = newConnection.getCookies(); - assertEquals("18-Aug-2015", cookies.get("WMF-Last-Access")); - assertEquals("DE:13:Dresden:51.0500:13.7500:v4", cookies.get("GeoIP")); - assertEquals("c18ef92637227283bcda73bcf95cfaf5", cookies.get("testwikidatawikiSession")); + List cookies = newConnection.getCookies(); + for (HttpCookie cookie : cookies) { + if (cookie.getName().equals("GeoIP")) { + assertEquals("DE:13:Dresden:51.0500:13.7500:v4", cookie.getValue()); + } else { + assertEquals("testwikidatawikiSession", cookie.getName()); + assertEquals("c18ef92637227283bcda73bcf95cfaf5", cookie.getValue()); + } + } Map tokens = newConnection.getTokens(); assertEquals("b5780b6e2f27e20b450921d9461010b4", tokens.get("login")); assertNull(tokens.get("csrf")); @@ -233,10 +233,11 @@ public void testErrors() throws IOException, } @Test - public void testClearCookies() throws IOException, MediaWikiApiErrorException { - connection.cookies.put("Content", "some content"); + public void testClearCookies() throws LoginFailedException, IOException, MediaWikiApiErrorException { + connection.login("username", "password"); + assertFalse(connection.getCookies().isEmpty()); connection.clearCookies(); - assertTrue(connection.cookies.keySet().isEmpty()); + assertTrue(connection.getCookies().isEmpty()); } @Test From a26f3b274512135db5da0ff429c50f2fb6b2a34b Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Mon, 8 Jun 2020 11:44:47 +0800 Subject: [PATCH 09/10] skip cookie domains comparison --- .../wdtk/wikibaseapi/BasicApiConnectionTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java index ab20c385e..8be3eba73 100644 --- a/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java +++ b/wdtk-wikibaseapi/src/test/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnectionTest.java @@ -50,7 +50,7 @@ public class BasicApiConnectionTest { private static MockWebServer server; private BasicApiConnection connection; - private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"http://kubernetes.docker.internal:" + server.getPort() + "/w/api.php\",\"cookies\":[{\"name\":\"GeoIP\",\"value\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"comment\":null,\"commentURL\":null,\"domain\":\".kubernetes.docker.internal\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":false,\"httpOnly\":false,\"version\":0,\"discard\":false},{\"name\":\"testwikidatawikiSession\",\"value\":\"c18ef92637227283bcda73bcf95cfaf5\",\"comment\":null,\"commentURL\":null,\"domain\":\"kubernetes.docker.internal\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":true,\"httpOnly\":true,\"version\":0,\"discard\":false}],\"username\":\"username\",\"loggedIn\":true,\"tokens\":{\"login\":\"b5780b6e2f27e20b450921d9461010b4\"},\"connectTimeout\":5000,\"readTimeout\":6000}\n"; + private String LOGGED_IN_SERIALIZED_CONNECTION = "{\"baseUrl\":\"" + server.url("/w/api.php") + "\",\"cookies\":[{\"name\":\"GeoIP\",\"value\":\"DE:13:Dresden:51.0500:13.7500:v4\",\"comment\":null,\"commentURL\":null,\"domain\":\"domain comparison should be skipped\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":false,\"httpOnly\":false,\"version\":0,\"discard\":false},{\"name\":\"testwikidatawikiSession\",\"value\":\"c18ef92637227283bcda73bcf95cfaf5\",\"comment\":null,\"commentURL\":null,\"domain\":\"domain comparison should be skipped\",\"maxAge\":-1,\"path\":\"/\",\"portlist\":null,\"secure\":true,\"httpOnly\":true,\"version\":0,\"discard\":false}],\"username\":\"username\",\"loggedIn\":true,\"tokens\":{\"login\":\"b5780b6e2f27e20b450921d9461010b4\"},\"connectTimeout\":5000,\"readTimeout\":6000}"; Set split(String str, char ch) { Set set = new TreeSet<>(); @@ -153,9 +153,10 @@ public void testSerialize() throws LoginFailedException, IOException { connection.setReadTimeout(6000); assertTrue(connection.isLoggedIn()); String jsonSerialization = mapper.writeValueAsString(connection); - JsonNode tree1 = mapper.readTree(LOGGED_IN_SERIALIZED_CONNECTION); - JsonNode tree2 = mapper.readTree(jsonSerialization); - Assert.assertEquals(tree1, tree2); + // We skip comparing the cookie domains here, since they depend on + // the mocked web server's host, which is system dependent. + jsonSerialization = jsonSerialization.replaceAll("\"domain\":\"[^\"]*\"", "\"domain\":\"domain comparison should be skipped\""); + assertEquals(LOGGED_IN_SERIALIZED_CONNECTION, jsonSerialization); } @Test From 3c980365ec5527128fdc3ff3e1dd2cc08beeb8fb Mon Sep 17 00:00:00 2001 From: afkbrb <2w6f8c@gmail.com> Date: Mon, 8 Jun 2020 13:22:51 +0800 Subject: [PATCH 10/10] add @JsonProperty annotations to getters --- .../org/wikidata/wdtk/wikibaseapi/ApiConnection.java | 4 ++++ .../wdtk/wikibaseapi/BasicApiConnection.java | 1 + .../wdtk/wikibaseapi/OAuthApiConnection.java | 12 ------------ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java index aba5eb02c..8e2184249 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/ApiConnection.java @@ -181,6 +181,7 @@ public String getApiBaseUrl() { * * @return true if the connection is in a logged in state */ + @JsonProperty("loggedIn") public boolean isLoggedIn() { return loggedIn; } @@ -217,6 +218,7 @@ public String getCurrentUser() { /** * Returns the map of tokens (such as csrf token and login token) currently used in this connection. */ + @JsonProperty("tokens") public Map getTokens() { return Collections.unmodifiableMap(tokens); } @@ -250,6 +252,7 @@ public void setReadTimeout(int timeout) { * * @see HttpURLConnection#getConnectTimeout */ + @JsonProperty("connectTimeout") public int getConnectTimeout() { return connectTimeout; } @@ -261,6 +264,7 @@ public int getConnectTimeout() { * * @see HttpURLConnection#getReadTimeout */ + @JsonProperty("readTimeout") public int getReadTimeout() { return readTimeout; } diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java index 9cdfd8fd9..0d5fa7124 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/BasicApiConnection.java @@ -281,6 +281,7 @@ protected void confirmLogin(String token, String username, String password) /** * Returns the map of cookies currently used in this connection. */ + @JsonProperty("cookies") public List getCookies() { return cookieManager.getCookieStore().getCookies(); } diff --git a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java index 9768409c9..cd2216d35 100644 --- a/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java +++ b/wdtk-wikibaseapi/src/main/java/org/wikidata/wdtk/wikibaseapi/OAuthApiConnection.java @@ -161,18 +161,6 @@ public void checkCredentials() throws IOException, MediaWikiApiErrorException { super.checkCredentials(); } - @Override - @JsonProperty("baseUrl") - public String getApiBaseUrl() { - return super.getApiBaseUrl(); - } - - @Override - @JsonProperty("loggedIn") - public boolean isLoggedIn() { - return super.isLoggedIn(); - } - @Override @JsonProperty("username") public String getCurrentUser() {