Merge remote-tracking branch 'origin/master' into clt-with-metadata

Conflicts:
	wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java

RELEASE-NOTES.md

@@ -1,6 +1,14 @@
 Wikidata Toolkit Release Notes
 ==============================
 
+Version 0.4.0
+-------------
+
+Bug fixes:
+* Support RDF export of Monolingual Text Value data in statements.
+* Significant performance improvements in RDF export of taxonomy data.   
+
+
 Version 0.3.0
 -------------
 

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/SitesImpl.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -32,17 +32,17 @@
  * Implementation of the {@link Sites} interface that allows sites to be
  * registered. Objects of this type are not immutable, since they are not data
  * objects, but the {@link Sites} interface only supports read access.
- * 
+ *
  * @author Markus Kroetzsch
- * 
+ *
  */
 public class SitesImpl implements Sites {
 
 	/**
 	 * Simple record for holding information about a site.
-	 * 
+	 *
 	 * @author Markus Kroetzsch
-	 * 
+	 *
 	 */
 	class SiteInformation {
 		final String siteKey;
@@ -83,7 +83,7 @@ class SiteInformation {
 
 		/**
 		 * Returns the file URL.
-		 * 
+		 *
 		 * @see Sites#getFileUrl(String, String)
 		 * @param fileName
 		 *            the file name
@@ -98,7 +98,7 @@ String getFileUrl(String fileName) {
 		 * page titles on MediaWiki sites, since this is how MediaWiki page URLs
 		 * are constructed. For other sites, this might not be the case and
 		 * spaces will just be escaped in the standard way using "+".
-		 * 
+		 *
 		 * @see Sites#getPageUrl(String, String)
 		 * @param pageTitle
 		 *            the page title, not escaped
@@ -126,23 +126,7 @@ String getPageUrl(String pageTitle) {
 
 	final HashMap<String, SiteInformation> sites = new HashMap<String, SiteInformation>();
 
-	/**
-	 * Sets the stored information for the site of the given key to the given
-	 * values.
-	 * 
-	 * @param siteKey
-	 *            the global site key
-	 * @param group
-	 *            the site group
-	 * @param languageCode
-	 *            the site MediaWiki language code
-	 * @param siteType
-	 *            the site type
-	 * @param filePath
-	 *            the file path with $1 as a placeholder for the file name
-	 * @param pagePath
-	 *            the page path with $1 as a placeholder for the page title
-	 */
+	@Override
 	public void setSiteInformation(String siteKey, String group,
 			String languageCode, String siteType, String filePath,
 			String pagePath) {

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +21,7 @@
  */
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 
 /**
@@ -33,17 +34,23 @@
 public class EntityDocumentProcessorBroker implements EntityDocumentProcessor {
 
 	final List<EntityDocumentProcessor> entityDocumentProcessors = new ArrayList<EntityDocumentProcessor>();
+	final HashSet<EntityDocumentProcessor> entityDocumentProcessorRegistry = new HashSet<>();
 
 	/**
 	 * Registers a listener which will be called for all entity documents that
-	 * are processed.
+	 * are processed. The method avoids duplicates in the sense that the exact
+	 * same object cannot be registered twice.
 	 *
 	 * @param entityDocumentProcessor
 	 *            the listener to register
 	 */
 	public void registerEntityDocumentProcessor(
 			EntityDocumentProcessor entityDocumentProcessor) {
-		this.entityDocumentProcessors.add(entityDocumentProcessor);
+		if (!this.entityDocumentProcessorRegistry
+				.contains(entityDocumentProcessor)) {
+			this.entityDocumentProcessors.add(entityDocumentProcessor);
+			this.entityDocumentProcessorRegistry.add(entityDocumentProcessor);
+		}
 	}
 
 	@Override

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/Sites.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,13 +27,47 @@
  * configuration of a MediaWiki site and therefore not fixed.
  * <p>
  * This is not a Wikibase data object as such, but part of the general
- * configuration of a Wikibase site.
- * 
+ * configuration of a Wikibase site. The interface supports modification, e.g.,
+ * to insert additional associations. This can be useful to augment data
+ * manually (even when loading most of the data from a file dump). For example,
+ * some of Wikimedia's data exports are more frequent than their sites table
+ * exports, so it might be useful to add some very recent sites.
+ *
  * @author Markus Kroetzsch
- * 
+ *
  */
 public interface Sites {
 
+	/**
+	 * Sets the stored information for the site of the given key to the given
+	 * values.
+	 * <p>
+	 * Note that the path URLs given here should be absolute. In MediaWiki, it
+	 * is common to use protocol-relative paths (starting with "//" rather than
+	 * with "http://" or "https://"). The code in this class is not prepared to
+	 * handle this yet (URL-returning methods would need to allow for a
+	 * preferred protocol to be specified).
+	 *
+	 * @param siteKey
+	 *            the global site key, e.g., "enwiki" or "fawikivoyage"
+	 * @param group
+	 *            the site group, e.g., "wikipedia" or "wikivoyage"
+	 * @param languageCode
+	 *            the site MediaWiki language code, e.g., "en" or "fa"
+	 * @param siteType
+	 *            the site type, typically "mediawiki"
+	 * @param filePath
+	 *            the file path with $1 as a placeholder for the file name,
+	 *            e.g., "http://en.wikipedia.org/w/$1" or
+	 *            "http://fa.wikivoyage.org/w/$1"
+	 * @param pagePath
+	 *            the page path with $1 as a placeholder for the page title,
+	 *            e.g., "http://en.wikipedia.org/wiki/$1" or
+	 *            "http://fa.wikivoyage.org/wiki/$1"
+	 */
+	void setSiteInformation(String siteKey, String group, String languageCode,
+			String siteType, String filePath, String pagePath);
+
 	/**
 	 * Returns the MediaWiki language code for the given site, or null if there
 	 * is no such data for this site key.
@@ -44,7 +78,7 @@ public interface Sites {
 	 * are a number of <a
 	 * href="http://meta.wikimedia.org/wiki/Special_language_codes"
 	 * >exceptions</a>.
-	 * 
+	 *
 	 * @param siteKey
 	 *            the global site key
 	 * @return the corresponding MediaWiki language code, or null if not known
@@ -58,7 +92,7 @@ public interface Sites {
 	 * "wikipedia", "wikisource", "wikivoyage", and "wikiquote", used for most
 	 * sites of these projects, but also singleton groups like "commons" and
 	 * "wikimania2013".
-	 * 
+	 *
 	 * @param siteKey
 	 *            the global site key
 	 * @return the corresponding group, or null if not known
@@ -69,7 +103,7 @@ public interface Sites {
 	 * Returns the URL for the page of the given name, or null if the site is
 	 * not known. All characters in the page title will be escaped for use in
 	 * URLs.
-	 * 
+	 *
 	 * @param siteKey
 	 *            the global site key
 	 * @param pageTitle
@@ -82,7 +116,7 @@ public interface Sites {
 	/**
 	 * Returns the URL for the given site link, or null if its site key is not
 	 * known.
-	 * 
+	 *
 	 * @param siteLink
 	 *            the SiteLink object
 	 * @return the page URL for this site link, or null if not known
@@ -96,7 +130,7 @@ public interface Sites {
 	 * calling the API of the site. Also note that this method does not
 	 * construct URLs for files uploaded to a MediaWiki site using the given
 	 * file name; such files are usually placed in some subdirectory.
-	 * 
+	 *
 	 * @param siteKey
 	 *            the global site key
 	 * @param fileName
@@ -109,7 +143,7 @@ public interface Sites {
 	/**
 	 * Returns the type for the given site, or null if there is no such data for
 	 * this site key. For MediaWiki sites, this is "mediawiki".
-	 * 
+	 *
 	 * @param siteKey
 	 *            the global site key
 	 * @return the corresponding type, or null if not known

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/json/jackson/JacksonMonolingualTextValue.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -45,11 +45,11 @@ public class JacksonMonolingualTextValue implements MonolingualTextValue {
 	/**
 	 * The language code.
 	 */
-	private String languageCode;
+	private String languageCode = "";
 	/**
 	 * The text value.
 	 */
-	private String text;
+	private String text = "";
 
 	/**
 	 * Constructor. Creates an empty object that can be populated during JSON

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/json/jackson/JacksonObjectFactory.java

@@ -121,7 +121,9 @@ public TimeValue getTimeValue(long year, byte month, byte day, byte hour,
 	public GlobeCoordinatesValue getGlobeCoordinatesValue(long latitude,
 			long longitude, long precision, String globeIri) {
 		JacksonInnerGlobeCoordinates innerCoordinates = new JacksonInnerGlobeCoordinates(
-				latitude, longitude, precision, globeIri);
+				((double)latitude / GlobeCoordinatesValue.PREC_DEGREE), 
+				((double)longitude / GlobeCoordinatesValue.PREC_DEGREE), 
+				((double)precision / GlobeCoordinatesValue.PREC_DEGREE), globeIri);
 		JacksonValueGlobeCoordinates result = new JacksonValueGlobeCoordinates();
 		result.setValue(innerCoordinates);
 		return result;

wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/json/jackson/JacksonPropertyDocument.java

@@ -30,6 +30,7 @@
 import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
@@ -41,6 +42,7 @@
  * @author Fredo Erxleben
  *
  */
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class JacksonPropertyDocument extends JacksonTermedDocument implements
 		PropertyDocument {