T270657: Retrieve current timestamp to avoid conflicts with deletions

WPCleaner · Feb 13, 2021 · 4926f2b · 4926f2b
1 parent 2fbc02c
commit 4926f2b
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 2 deletions.
diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/impl/MediaWikiAPI.java b/WikipediaCleaner/src/org/wikipediacleaner/api/impl/MediaWikiAPI.java
@@ -20,6 +20,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.zip.GZIPInputStream;
 
 import org.apache.commons.httpclient.Header;
@@ -730,7 +731,7 @@ private boolean constructContents(Page page, Element root, String query)
         page.setExisting(Boolean.FALSE);
       }
       page.setPageId(node.getAttributeValue("pageid"));
-      page.setStartTimestamp(node.getAttributeValue("starttimestamp"));
+      Optional.ofNullable(node.getAttributeValue("starttimestamp")).ifPresent(timestamp -> page.setStartTimestamp(timestamp));
     }
     XPathExpression<Element> xpa = XPathFactory.instance().compile(
         query + "/revisions/rev", Filters.element());

diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/request/ApiRequest.java b/WikipediaCleaner/src/org/wikipediacleaner/api/request/ApiRequest.java
@@ -146,6 +146,12 @@ public abstract class ApiRequest {
    */
   public final static String PROPERTY_CONTINUE_DEFAULT = "";
 
+  /** Current timestamp */
+  public final static String PROPERTY_CURRENT_TIMESTAMP = "curtimestamp";
+
+  /** Value to include current timestamp in the response */
+  public final static String PROPERTY_CURRENT_TIMESTAMP_YES = "1";
+
   // ==========================================================================
   // Wiki management
   // ==========================================================================

diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/request/ApiXmlResult.java b/WikipediaCleaner/src/org/wikipediacleaner/api/request/ApiXmlResult.java
@@ -13,8 +13,11 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.zip.GZIPInputStream;
 
+import javax.annotation.Nonnull;
+
 import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpMethod;
@@ -280,6 +283,21 @@ protected boolean shouldContinue(
     return result;
   }
 
+  /**
+   * Retrieve current timestamp from the result.
+   * 
+   * @param root Root of the DOM tree.
+   * @return Optional current timestamp.
+   */
+  @Nonnull
+  protected Optional<String> getCurrentTimestamp(Element root) {
+    XPathExpression<Element> xpa = XPathFactory.instance().compile(
+        "/api", Filters.element());
+    return Optional
+        .ofNullable(xpa.evaluateFirst(root))
+        .map(node -> node.getAttributeValue("curtimestamp"));
+  }
+
   /**
    * Get a page corresponding to a page node.
    * 

diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiRevisionsRequest.java b/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiRevisionsRequest.java
@@ -208,6 +208,9 @@ public void loadContent(
     }
 
     Map<String, String> properties = getProperties(ACTION_QUERY, result.getFormat());
+    properties.put(
+        PROPERTY_CURRENT_TIMESTAMP,
+        PROPERTY_CURRENT_TIMESTAMP_YES);
     properties.put(
         PROPERTY_PROP,
         PROPERTY_PROP_REVISIONS + "|" + PROPERTY_PROP_INFO);

diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiXmlPropertiesResult.java b/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiXmlPropertiesResult.java
@@ -12,6 +12,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.apache.commons.httpclient.HttpClient;
 import org.jdom2.Attribute;
@@ -61,7 +62,7 @@ public void updatePageInformation(Element node, Page page) throws JDOMException
     if (attrTitle != null) {
       page.setTitle(attrTitle.getValue());
     }
-    page.setStartTimestamp(node.getAttributeValue("starttimestamp"));
+    Optional.ofNullable(node.getAttributeValue("starttimestamp")).ifPresent(timestamp -> page.setStartTimestamp(timestamp));
     Attribute attrRedirect = node.getAttribute("redirect");
     if (attrRedirect != null) {
       page.getRedirects().isRedirect(true);

diff --git a/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiXmlRevisionsResult.java b/WikipediaCleaner/src/org/wikipediacleaner/api/request/query/prop/ApiXmlRevisionsResult.java
@@ -12,6 +12,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.apache.commons.httpclient.HttpClient;
 import org.jdom2.Element;
@@ -62,6 +63,9 @@ public boolean executeLastRevision(
       Map<String, String> normalization = new HashMap<>();
       retrieveNormalization(root, normalization);
 
+      // Retrieve current timestamp
+      Optional<String> currentTimestamp = getCurrentTimestamp(root);
+
       // Retrieve pages
       XPathExpression<Element> xpa = XPathFactory.instance().compile(
           "/api/query/pages/page", Filters.element());
@@ -96,6 +100,7 @@ public boolean executeLastRevision(
             }
             if (samePage) {
               page.setNamespace(namespace);
+              currentTimestamp.ifPresent(timestamp -> page.setStartTimestamp(timestamp));
               updatePageInformation(pageNode, page);
 
               // Retrieve revisions