From 55fb9b804322893cde45c26a03fa5dce9acd46da Mon Sep 17 00:00:00 2001 From: Guille Date: Wed, 12 Dec 2018 11:10:36 +0100 Subject: [PATCH] Add telemetry to the pull process and hook UI events - Eagerly get all the batches to get the total submissions of the operation - Make Cursor comparable to reduce the last cursor from the batch list --- .../briefcase/model/FormStatus.java | 4 + .../opendatakit/briefcase/pull/PullForm.java | 39 +++++----- .../briefcase/pull/PullTracker.java | 73 +++++++++++++++++++ 3 files changed, 95 insertions(+), 21 deletions(-) create mode 100644 src/org/opendatakit/briefcase/pull/PullTracker.java diff --git a/src/org/opendatakit/briefcase/model/FormStatus.java b/src/org/opendatakit/briefcase/model/FormStatus.java index 17ae2a1bb..59a51e878 100644 --- a/src/org/opendatakit/briefcase/model/FormStatus.java +++ b/src/org/opendatakit/briefcase/model/FormStatus.java @@ -54,6 +54,10 @@ public synchronized void clearStatusHistory() { isSuccessful = true; } + public synchronized void setStatusString(String statusString) { + setStatusString(statusString, false); + } + public synchronized void setStatusString(String statusString, boolean isSuccessful) { this.statusString = statusString; if (statusHistory.length() > STATUS_HISTORY_MAX_BYTES) { diff --git a/src/org/opendatakit/briefcase/pull/PullForm.java b/src/org/opendatakit/briefcase/pull/PullForm.java index 4d80c3d38..7ca16ac8e 100644 --- a/src/org/opendatakit/briefcase/pull/PullForm.java +++ b/src/org/opendatakit/briefcase/pull/PullForm.java @@ -37,22 +37,22 @@ import org.opendatakit.briefcase.reused.OptionalProduct; import org.opendatakit.briefcase.reused.RemoteServer; import org.opendatakit.briefcase.reused.http.Http; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PullForm { - private static final Logger log = LoggerFactory.getLogger(PullForm.class); - public static String pull(FormStatus form, boolean includeIncomplete, RemoteServer server, Path briefcaseDir, Http http) { + PullTracker tracker = new PullTracker(form); + // Download the blank form String formXml = downloadForm(form, server, http); writeForm(form, briefcaseDir, formXml); + tracker.trackBlankForm(); // Download attachments of the blank form - downloadFormAttachments(form, briefcaseDir, http); + downloadFormAttachments(form, briefcaseDir, http, tracker); // Get all the submission batches List batches = getInstanceIdBatches(server, http, form.getFormId(), includeIncomplete); + tracker.trackBatches(batches); // For all submissions in all batches... SubmissionKeyGenerator subKeyGen = SubmissionKeyGenerator.from(formXml); @@ -60,9 +60,10 @@ public static String pull(FormStatus form, boolean includeIncomplete, RemoteServ // Download the submission DownloadedSubmission submission = downloadSubmission(form, server, briefcaseDir, http, subKeyGen, instanceId); writeSubmission(form, submission, briefcaseDir); + tracker.trackSubmission(); // Download attachments of the submission - downloadSubmissionAttachments(form, submission, briefcaseDir, http); + downloadSubmissionAttachments(form, submission, briefcaseDir, http, tracker); }); // Return the last cursor received @@ -82,35 +83,33 @@ private static DownloadedSubmission downloadSubmission(FormStatus form, RemoteSe if (!exists(instanceDir)) createDirectories(instanceDir); String submissionKey = subKeyGen.buildKey(instanceId); - log.info("Got xml for instance ID {} of form {}", instanceId, form.getFormId()); return http.execute(server.getDownloadSubmissionRequest(submissionKey)).orElseThrow(BriefcaseException::new); } - private static void downloadSubmissionAttachments(FormStatus form, DownloadedSubmission submission, Path briefcaseDir, Http http) { + private static void downloadSubmissionAttachments(FormStatus form, DownloadedSubmission submission, Path briefcaseDir, Http http, PullTracker tracker) { Path mediaDir = form.getSubmissionDir(briefcaseDir, submission.getInstanceId()); if (!exists(mediaDir)) createDirectories(mediaDir); - downloadMediaFiles(submission.getAttachments(), mediaDir, http); + downloadMediaFiles(submission.getAttachments(), mediaDir, http, tracker); } - private static void downloadFormAttachments(FormStatus form, Path briefcaseDir, Http http) { + private static void downloadFormAttachments(FormStatus form, Path briefcaseDir, Http http, PullTracker tracker) { form.getManifestUrl().ifPresent(manifestUrl -> { Path mediaDir = form.getFormMediaDir(briefcaseDir); if (!exists(mediaDir)) createDirectories(mediaDir); List mediaFiles = http.execute(get(manifestUrl).asXmlElement().withMapper(PullForm::parseMediaFiles).build()).get(); - downloadMediaFiles(mediaFiles, mediaDir, http); + downloadMediaFiles(mediaFiles, mediaDir, http, tracker); }); } - private static void downloadMediaFiles(List mediaFiles, Path mediaDir, Http http) { - mediaFiles.stream() - .filter(mediaFile -> mediaFile.needsUpdate(mediaDir)) - .forEach(mediaFile -> { - Path target = mediaFile.getTargetPath(mediaDir); - http.execute(get(mediaFile.getDownloadUrl()).downloadTo(target).build()); - log.info("Downloaded mediaFile at {}", target); - }); + private static void downloadMediaFiles(List mediaFiles, Path mediaDir, Http http, PullTracker tracker) { + List mediaFilesToDownload = mediaFiles.stream().filter(mediaFile -> mediaFile.needsUpdate(mediaDir)).collect(toList()); + mediaFilesToDownload.forEach(mediaFile -> { + Path target = mediaFile.getTargetPath(mediaDir); + http.execute(get(mediaFile.getDownloadUrl()).downloadTo(target).build()); + }); + tracker.trackMediaFiles(mediaFiles, mediaFilesToDownload); } private static void writeForm(FormStatus form, Path briefcaseDir, String formXml) { @@ -119,13 +118,11 @@ private static void writeForm(FormStatus form, Path briefcaseDir, String formXml createDirectories(formDir); Path formFile = formDir.resolve(stripIllegalChars(form.getFormName()) + ".xml"); write(formFile, formXml, CREATE, TRUNCATE_EXISTING); - log.info("Wrote blank form at {}", formFile); } private static void writeSubmission(FormStatus form, DownloadedSubmission submission, Path briefcaseDir) { Path submissionFile = form.getSubmissionDir(briefcaseDir, submission.getInstanceId()).resolve("submission.xml"); write(submissionFile, submission.getContent(), CREATE, TRUNCATE_EXISTING); - log.info("Wrote submission at {}", submissionFile); } private static List parseMediaFiles(XmlElement root) { diff --git a/src/org/opendatakit/briefcase/pull/PullTracker.java b/src/org/opendatakit/briefcase/pull/PullTracker.java new file mode 100644 index 000000000..46944775a --- /dev/null +++ b/src/org/opendatakit/briefcase/pull/PullTracker.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2018 Nafundi + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.opendatakit.briefcase.pull; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.bushe.swing.event.EventBus; +import org.opendatakit.briefcase.model.FormStatus; +import org.opendatakit.briefcase.model.FormStatusEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class PullTracker { + private static final Logger log = LoggerFactory.getLogger(PullTracker.class); + private final FormStatus form; + private int totalSubmissions; + private AtomicInteger submissionCounter = new AtomicInteger(0); + + PullTracker(FormStatus form) { + this.form = form; + } + + void trackBlankForm() { + form.setStatusString("Downloaded blank form"); + log.info("Downloaded blank form {}", form.getFormName()); + fireUIEvent(); + } + + void trackBatches(List batches) { + totalSubmissions = batches.stream().map(InstanceIdBatch::count).reduce(0, Integer::sum); + form.setStatusString("Downloading " + totalSubmissions + " submissions"); + log.info("Downloaded {} submissions", totalSubmissions); + fireUIEvent(); + } + + void trackSubmission() { + form.setStatusString("Downloaded submission " + submissionCounter.incrementAndGet() + " of " + totalSubmissions); + log.info("Downloaded submission {} of {}", submissionCounter.get(), totalSubmissions); + fireUIEvent(); + } + + void trackMediaFiles(List manifestMediaFiles, List downloadedMediaFiles) { + if (!downloadedMediaFiles.isEmpty()) { + form.setStatusString("Downloaded " + downloadedMediaFiles.size() + " attachments"); + log.info("Downloaded {} attachments", downloadedMediaFiles.size()); + fireUIEvent(); + } + if (manifestMediaFiles.size() > downloadedMediaFiles.size()) { + int number = manifestMediaFiles.size() - downloadedMediaFiles.size(); + form.setStatusString("Ignoring " + number + " attachments (already present)"); + log.info("Ignoring {} attachments (already present)", number); + fireUIEvent(); + } + } + + private void fireUIEvent() { + EventBus.publish(new FormStatusEvent(form)); + } +}