Add telemetry to the pull process and hook UI events

- Eagerly get all the batches to get the total submissions of the operation
- Make Cursor comparable to reduce the last cursor from the batch list

src/org/opendatakit/briefcase/model/FormStatus.java

@@ -54,6 +54,10 @@ public synchronized void clearStatusHistory() {
     isSuccessful = true;
   }
 
+  public synchronized void setStatusString(String statusString) {
+    setStatusString(statusString, false);
+  }
+
   public synchronized void setStatusString(String statusString, boolean isSuccessful) {
     this.statusString = statusString;
     if (statusHistory.length() > STATUS_HISTORY_MAX_BYTES) {

src/org/opendatakit/briefcase/pull/PullForm.java

@@ -37,32 +37,33 @@
 import org.opendatakit.briefcase.reused.OptionalProduct;
 import org.opendatakit.briefcase.reused.RemoteServer;
 import org.opendatakit.briefcase.reused.http.Http;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class PullForm {
-  private static final Logger log = LoggerFactory.getLogger(PullForm.class);
-
   public static String pull(FormStatus form, boolean includeIncomplete, RemoteServer server, Path briefcaseDir, Http http) {
+    PullTracker tracker = new PullTracker(form);
+
     // Download the blank form
     String formXml = downloadForm(form, server, http);
     writeForm(form, briefcaseDir, formXml);
+    tracker.trackBlankForm();
 
     // Download attachments of the blank form
-    downloadFormAttachments(form, briefcaseDir, http);
+    downloadFormAttachments(form, briefcaseDir, http, tracker);
 
     // Get all the submission batches
     List<InstanceIdBatch> batches = getInstanceIdBatches(server, http, form.getFormId(), includeIncomplete);
+    tracker.trackBatches(batches);
 
     // For all submissions in all batches...
     SubmissionKeyGenerator subKeyGen = SubmissionKeyGenerator.from(formXml);
     batches.stream().flatMap(batch -> batch.getInstanceIds().stream()).forEach(instanceId -> {
       // Download the submission
       DownloadedSubmission submission = downloadSubmission(form, server, briefcaseDir, http, subKeyGen, instanceId);
       writeSubmission(form, submission, briefcaseDir);
+      tracker.trackSubmission();
 
       // Download attachments of the submission
-      downloadSubmissionAttachments(form, submission, briefcaseDir, http);
+      downloadSubmissionAttachments(form, submission, briefcaseDir, http, tracker);
     });
 
     // Return the last cursor received
@@ -82,35 +83,33 @@ private static DownloadedSubmission downloadSubmission(FormStatus form, RemoteSe
     if (!exists(instanceDir))
       createDirectories(instanceDir);
     String submissionKey = subKeyGen.buildKey(instanceId);
-    log.info("Got xml for instance ID {} of form {}", instanceId, form.getFormId());
     return http.execute(server.getDownloadSubmissionRequest(submissionKey)).orElseThrow(BriefcaseException::new);
   }
 
-  private static void downloadSubmissionAttachments(FormStatus form, DownloadedSubmission submission, Path briefcaseDir, Http http) {
+  private static void downloadSubmissionAttachments(FormStatus form, DownloadedSubmission submission, Path briefcaseDir, Http http, PullTracker tracker) {
     Path mediaDir = form.getSubmissionDir(briefcaseDir, submission.getInstanceId());
     if (!exists(mediaDir))
       createDirectories(mediaDir);
-    downloadMediaFiles(submission.getAttachments(), mediaDir, http);
+    downloadMediaFiles(submission.getAttachments(), mediaDir, http, tracker);
   }
 
-  private static void downloadFormAttachments(FormStatus form, Path briefcaseDir, Http http) {
+  private static void downloadFormAttachments(FormStatus form, Path briefcaseDir, Http http, PullTracker tracker) {
     form.getManifestUrl().ifPresent(manifestUrl -> {
       Path mediaDir = form.getFormMediaDir(briefcaseDir);
       if (!exists(mediaDir))
         createDirectories(mediaDir);
       List<MediaFile> mediaFiles = http.execute(get(manifestUrl).asXmlElement().withMapper(PullForm::parseMediaFiles).build()).get();
-      downloadMediaFiles(mediaFiles, mediaDir, http);
+      downloadMediaFiles(mediaFiles, mediaDir, http, tracker);
     });
   }
 
-  private static void downloadMediaFiles(List<MediaFile> mediaFiles, Path mediaDir, Http http) {
-    mediaFiles.stream()
-        .filter(mediaFile -> mediaFile.needsUpdate(mediaDir))
-        .forEach(mediaFile -> {
-          Path target = mediaFile.getTargetPath(mediaDir);
-          http.execute(get(mediaFile.getDownloadUrl()).downloadTo(target).build());
-          log.info("Downloaded mediaFile at {}", target);
-        });
+  private static void downloadMediaFiles(List<MediaFile> mediaFiles, Path mediaDir, Http http, PullTracker tracker) {
+    List<MediaFile> mediaFilesToDownload = mediaFiles.stream().filter(mediaFile -> mediaFile.needsUpdate(mediaDir)).collect(toList());
+    mediaFilesToDownload.forEach(mediaFile -> {
+      Path target = mediaFile.getTargetPath(mediaDir);
+      http.execute(get(mediaFile.getDownloadUrl()).downloadTo(target).build());
+    });
+    tracker.trackMediaFiles(mediaFiles, mediaFilesToDownload);
   }
 
   private static void writeForm(FormStatus form, Path briefcaseDir, String formXml) {
@@ -119,13 +118,11 @@ private static void writeForm(FormStatus form, Path briefcaseDir, String formXml
       createDirectories(formDir);
     Path formFile = formDir.resolve(stripIllegalChars(form.getFormName()) + ".xml");
     write(formFile, formXml, CREATE, TRUNCATE_EXISTING);
-    log.info("Wrote blank form at {}", formFile);
   }
 
   private static void writeSubmission(FormStatus form, DownloadedSubmission submission, Path briefcaseDir) {
     Path submissionFile = form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).resolve("submission.xml");
     write(submissionFile, submission.getContent(), CREATE, TRUNCATE_EXISTING);
-    log.info("Wrote submission at {}", submissionFile);
   }
 
   private static List<MediaFile> parseMediaFiles(XmlElement root) {

src/org/opendatakit/briefcase/pull/PullTracker.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2018 Nafundi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.opendatakit.briefcase.pull;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.bushe.swing.event.EventBus;
+import org.opendatakit.briefcase.model.FormStatus;
+import org.opendatakit.briefcase.model.FormStatusEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class PullTracker {
+  private static final Logger log = LoggerFactory.getLogger(PullTracker.class);
+  private final FormStatus form;
+  private int totalSubmissions;
+  private AtomicInteger submissionCounter = new AtomicInteger(0);
+
+  PullTracker(FormStatus form) {
+    this.form = form;
+  }
+
+  void trackBlankForm() {
+    form.setStatusString("Downloaded blank form");
+    log.info("Downloaded blank form {}", form.getFormName());
+    fireUIEvent();
+  }
+
+  void trackBatches(List<InstanceIdBatch> batches) {
+    totalSubmissions = batches.stream().map(InstanceIdBatch::count).reduce(0, Integer::sum);
+    form.setStatusString("Downloading " + totalSubmissions + " submissions");
+    log.info("Downloaded {} submissions", totalSubmissions);
+    fireUIEvent();
+  }
+
+  void trackSubmission() {
+    form.setStatusString("Downloaded submission " + submissionCounter.incrementAndGet() + " of " + totalSubmissions);
+    log.info("Downloaded submission {} of {}", submissionCounter.get(), totalSubmissions);
+    fireUIEvent();
+  }
+
+  void trackMediaFiles(List<MediaFile> manifestMediaFiles, List<MediaFile> downloadedMediaFiles) {
+    if (!downloadedMediaFiles.isEmpty()) {
+      form.setStatusString("Downloaded " + downloadedMediaFiles.size() + " attachments");
+      log.info("Downloaded {} attachments", downloadedMediaFiles.size());
+      fireUIEvent();
+    }
+    if (manifestMediaFiles.size() > downloadedMediaFiles.size()) {
+      int number = manifestMediaFiles.size() - downloadedMediaFiles.size();
+      form.setStatusString("Ignoring " + number + " attachments (already present)");
+      log.info("Ignoring {} attachments (already present)", number);
+      fireUIEvent();
+    }
+  }
+
+  private void fireUIEvent() {
+    EventBus.publish(new FormStatusEvent(form));
+  }
+}