Remembering previously entered values (auto-fill) (#2882)

* Pass last-saved instance into JR from FormLoaderTask

* Refactor a few related FileUtils

* Update last-saved instance on save

* Pass last-saved src from InstanceGoogleSheetsUploader also

* Move LAST_SAVED_FILENAME to FileUtils

* Make form MEDIA_SUFFIX a constant

* Factor out getLastSavedSrcIfExists util

* Rename old occurrences of form fileName => basename

* Update one other occurrence of -media string

* Make MEDIA_SUFFIX private to FormUtils

* Make LAST_SAVED_PATH private to FormUtils

* Remove temporary variable

* Make sure dirs exist before trying to write

* Always create last-saved.xml if it doesn't exist

* Ensure last-saved.xml is deleted for encrypted forms

It will always be deleted on save if finalized, along with the rest of
the plaintext files.

Because of this, InstanceSyncTask doesn't need to delete it (nor does it
even know of its existence because it's in a separate directory) so it
just passes in `null`.

collect_app/src/main/java/org/odk/collect/android/logic/FormController.java

@@ -49,6 +49,7 @@
 import org.javarosa.xpath.expr.XPathExpression;
 import org.odk.collect.android.exception.JavaRosaException;
 import org.odk.collect.android.utilities.AuditEventLogger;
+import org.odk.collect.android.utilities.FileUtils;
 import org.odk.collect.android.utilities.RegexUtils;
 import org.odk.collect.android.views.ODKView;
 
@@ -168,6 +169,11 @@ public String getAbsoluteInstancePath() {
         return instanceFile != null ? instanceFile.getAbsolutePath() : null;
     }
 
+    @Nullable
+    public String getLastSavedPath() {
+        return mediaFolder != null ? FileUtils.getLastSavedPath(mediaFolder) : null;
+    }
+
     public void setIndexWaitingForData(FormIndex index) {
         indexWaitingForData = index;
     }

collect_app/src/main/java/org/odk/collect/android/provider/FormsProvider.java

@@ -212,10 +212,7 @@ public synchronized Uri insert(@NonNull Uri uri, ContentValues initialValues) {
                 values.put(FormsColumns.JRCACHE_FILE_PATH, cachePath);
             }
             if (!values.containsKey(FormsColumns.FORM_MEDIA_PATH)) {
-                String pathNoExtension = filePath.substring(0,
-                        filePath.lastIndexOf('.'));
-                String mediaPath = pathNoExtension + "-media";
-                values.put(FormsColumns.FORM_MEDIA_PATH, mediaPath);
+                values.put(FormsColumns.FORM_MEDIA_PATH, FileUtils.constructMediaPath(filePath));
             }
 
             SQLiteDatabase db = formsDatabaseHelper.getWritableDatabase();

collect_app/src/main/java/org/odk/collect/android/tasks/FormLoaderTask.java

@@ -122,10 +122,7 @@ protected FECWrapper doInBackground(String... path) {
 
         final String formPath = path[0];
         final File formXml = new File(formPath);
-
-        // set paths to /sdcard/odk/forms/formfilename-media/
-        final String formFileName = formXml.getName().substring(0, formXml.getName().lastIndexOf("."));
-        final File formMediaDir = new File(formXml.getParent(), formFileName + "-media");
+        final File formMediaDir = FileUtils.getFormMediaDir(formXml);
 
         final ReferenceManager referenceManager = ReferenceManager.instance();
 
@@ -138,7 +135,7 @@ protected FECWrapper doInBackground(String... path) {
             referenceManager.addReferenceFactory(new FileReferenceFactory(Collect.ODK_ROOT));
         }
 
-        addSessionRootTranslators(formFileName, referenceManager,
+        addSessionRootTranslators(formMediaDir.getName(), referenceManager,
                 "images", "image", "audio", "video", "file");
 
         FormDef formDef = null;
@@ -219,9 +216,9 @@ protected FECWrapper doInBackground(String... path) {
         return data;
     }
 
-    private void addSessionRootTranslators(String formFileName, ReferenceManager referenceManager, String... hostStrings) {
-        // Set jr://... to point to /sdcard/odk/forms/filename-media/
-        final String translatedPrefix = String.format("jr://file/forms/%s-media/", formFileName);
+    private void addSessionRootTranslators(String formMediaDir, ReferenceManager referenceManager, String... hostStrings) {
+        // Set jr://... to point to /sdcard/odk/forms/formBasename-media/
+        final String translatedPrefix = String.format("jr://file/forms/" + formMediaDir + "/");
         for (String t : hostStrings) {
             referenceManager.addSessionRootTranslator(new RootTranslator(String.format("jr://%s/", t), translatedPrefix));
         }
@@ -242,7 +239,8 @@ private FormDef createFormDefFromCacheOrXml(String formPath, File formXml) {
             Timber.i("Attempting to load from: %s", formXml.getAbsolutePath());
             final long start = System.currentTimeMillis();
             fis = new FileInputStream(formXml);
-            FormDef formDefFromXml = XFormUtils.getFormFromInputStream(fis);
+            String lastSavedSrc = FileUtils.getOrCreateLastSavedSrc(formXml);
+            FormDef formDefFromXml = XFormUtils.getFormFromInputStream(fis, lastSavedSrc);
             if (formDefFromXml == null) {
                 errorMsg = "Error reading XForm file";
             } else {

collect_app/src/main/java/org/odk/collect/android/tasks/InstanceSyncTask.java

@@ -265,7 +265,7 @@ private void encryptInstance(Cursor instanceCursor, String candidateInstance,
                 instancesDao.updateInstance(values, InstanceColumns.INSTANCE_FILE_PATH + "=?", new String[]{candidateInstance});
 
                 SaveToDiskTask.manageFilesAfterSavingEncryptedForm(instanceXml, submissionXml);
-                if (!EncryptionUtils.deletePlaintextFiles(instanceXml)) {
+                if (!EncryptionUtils.deletePlaintextFiles(instanceXml, null)) {
                     Timber.e("Error deleting plaintext files for %s", instanceXml.getAbsolutePath());
                 }
             }

collect_app/src/main/java/org/odk/collect/android/tasks/SaveToDiskTask.java

@@ -270,10 +270,14 @@ private void exportData(boolean markCompleted) throws IOException, EncryptionExc
 
         writeFile(payload, instancePath);
 
+        // Write SMS data
         final ByteArrayPayload payloadSms = formController.getFilledInFormSMS();
-        // Write SMS to card
         writeFile(payloadSms, getSmsInstancePath(instancePath));
 
+        // Write last-saved instance
+        String lastSavedPath = formController.getLastSavedPath();
+        writeFile(payload, lastSavedPath);
+
         // update the uri. We have exported the reloadable instance, so update status...
         // Since we saved a reloadable instance, it is flagged as re-openable so that if any error
         // occurs during the packaging of the data for the server fails (e.g., encryption),
@@ -346,7 +350,7 @@ private void exportData(boolean markCompleted) throws IOException, EncryptionExc
             // if encrypted, delete all plaintext files
             // (anything not named instanceXml or anything not ending in .enc)
             if (isEncrypted) {
-                if (!EncryptionUtils.deletePlaintextFiles(instanceXml)) {
+                if (!EncryptionUtils.deletePlaintextFiles(instanceXml, new File(lastSavedPath))) {
                     Timber.e("Error deleting plaintext files for %s", instanceXml.getAbsolutePath());
                 }
             }
@@ -395,14 +399,13 @@ static void writeFile(ByteArrayPayload payload, String path) throws IOException
 
         // read from data stream
         byte[] data = new byte[len];
-        // try {
         int read = is.read(data, 0, len);
         if (read > 0) {
+            // Make sure the directory path to this file exists.
+            file.getParentFile().mkdirs();
             // write xml file
             RandomAccessFile randomAccessFile = null;
             try {
-                // String filename = path + File.separator +
-                // path.substring(path.lastIndexOf(File.separator) + 1) + ".xml";
                 randomAccessFile = new RandomAccessFile(file, "rws");
                 randomAccessFile.write(data);
             } finally {

collect_app/src/main/java/org/odk/collect/android/upload/InstanceGoogleSheetsUploader.java

@@ -43,6 +43,7 @@
 import org.odk.collect.android.preferences.GeneralSharedPreferences;
 import org.odk.collect.android.preferences.GeneralKeys;
 import org.odk.collect.android.tasks.FormLoaderTask;
+import org.odk.collect.android.utilities.FileUtils;
 import org.odk.collect.android.utilities.TextUtils;
 import org.odk.collect.android.utilities.UrlUtils;
 import org.odk.collect.android.utilities.gdrive.DriveHelper;
@@ -301,8 +302,12 @@ private String uploadMediaFile(Instance instance, String fileName) throws Upload
 
     private TreeElement getInstanceElement(String formFilePath, File instanceFile) throws UploadException {
         FormDef formDef;
+
+        File formXml = new File(formFilePath);
+        String lastSavedSrc = FileUtils.getOrCreateLastSavedSrc(formXml);
+
         try {
-            formDef = XFormUtils.getFormFromInputStream(new FileInputStream(new File(formFilePath)));
+            formDef = XFormUtils.getFormFromInputStream(new FileInputStream(formXml), lastSavedSrc);
             FormLoaderTask.importData(instanceFile, new FormEntryController(new FormEntryModel(formDef)));
         } catch (IOException | RuntimeException e) {
             throw new UploadException(e);

collect_app/src/main/java/org/odk/collect/android/utilities/EncryptionUtils.java

@@ -17,6 +17,7 @@
 import android.content.ContentResolver;
 import android.database.Cursor;
 import android.net.Uri;
+import android.support.annotation.Nullable;
 import android.util.Base64;
 
 import org.apache.commons.io.IOUtils;
@@ -427,16 +428,17 @@ private static void encryptFile(File file, EncryptedFormInformation formInfo)
         }
     }
 
-    public static boolean deletePlaintextFiles(File instanceXml) {
+    public static boolean deletePlaintextFiles(File instanceXml, @Nullable File lastSaved) {
         // NOTE: assume the directory containing the instanceXml contains ONLY
         // files related to this one instance.
         File instanceDir = instanceXml.getParentFile();
 
         boolean allSuccessful = true;
-        // encrypt files that do not end with ".enc", and do not start with ".";
+
+        // Delete files that do not end with ".enc", and do not start with ".";
         // ignore directories
-        File[] allFiles = instanceDir.listFiles();
-        for (File f : allFiles) {
+        File[] instanceFiles = instanceDir.listFiles();
+        for (File f : instanceFiles) {
             if (f.equals(instanceXml)) {
                 continue; // don't touch instance file
             }
@@ -449,6 +451,12 @@ public static boolean deletePlaintextFiles(File instanceXml) {
                 // short-circuit
             }
         }
+
+        // Delete the last-saved instance, if one exists.
+        if (lastSaved != null && lastSaved.exists()) {
+            allSuccessful &= lastSaved.delete();
+        }
+
         return allSuccessful;
     }
 

collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java

@@ -43,6 +43,7 @@
 import java.net.FileNameMap;
 import java.net.URLConnection;
 import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
@@ -67,6 +68,16 @@ public class FileUtils {
     public static final String BASE64_RSA_PUBLIC_KEY = "base64RsaPublicKey";
     public static final String AUTO_DELETE = "autoDelete";
     public static final String AUTO_SEND = "autoSend";
+
+    /** Suffix for the form media directory. */
+    private static final String MEDIA_SUFFIX = "-media";
+
+    /** Filename of the last-saved instance data. */
+    public static final String LAST_SAVED_FILENAME = "last-saved.xml";
+
+    /** Valid XML stub that can be parsed without error. */
+    private static final String STUB_XML = "<?xml version='1.0' ?><stub />";
+
     static int bufSize = 16 * 1024; // May be set by unit test
 
     private FileUtils() {
@@ -357,9 +368,54 @@ public static void deleteAndReport(File file) {
         }
     }
 
+    public static String getFormBasename(File formXml) {
+        return getFormBasename(formXml.getName());
+    }
+
+    public static String getFormBasename(String formFilePath) {
+        return formFilePath.substring(0, formFilePath.lastIndexOf('.'));
+    }
+
     public static String constructMediaPath(String formFilePath) {
-        String pathNoExtension = formFilePath.substring(0, formFilePath.lastIndexOf('.'));
-        return pathNoExtension + "-media";
+        return getFormBasename(formFilePath) + MEDIA_SUFFIX;
+    }
+
+    public static File getFormMediaDir(File formXml) {
+        final String formFileName = getFormBasename(formXml);
+        return new File(formXml.getParent(), formFileName + MEDIA_SUFFIX);
+    }
+
+    public static String getFormBasenameFromMediaFolder(File mediaFolder) {
+        /*
+         * TODO (from commit 37e3467): Apparently the form name is neither
+         * in the formController nor the formDef. In fact, it doesn't seem to
+         * be saved into any object in JavaRosa. However, the mediaFolder
+         * has the substring of the file name in it, so we extract the file name
+         * from here. Awkward...
+         */
+        return mediaFolder.getName().split(MEDIA_SUFFIX)[0];
+    }
+
+    public static File getLastSavedFile(File formXml) {
+        return new File(getFormMediaDir(formXml), LAST_SAVED_FILENAME);
+    }
+
+    public static String getLastSavedPath(File mediaFolder) {
+        return mediaFolder.getAbsolutePath() + File.separator + LAST_SAVED_FILENAME;
+    }
+
+    /**
+     * Returns the path to the last-saved file for this form,
+     * creating a valid stub if it doesn't yet exist.
+     */
+    public static String getOrCreateLastSavedSrc(File formXml) {
+        File lastSavedFile = getLastSavedFile(formXml);
+
+        if (!lastSavedFile.exists()) {
+            write(lastSavedFile, STUB_XML.getBytes(Charset.forName("UTF-8")));
+        }
+
+        return "jr://file/" + LAST_SAVED_FILENAME;
     }
 
     /**
@@ -455,6 +511,9 @@ public static byte[] read(File file) {
     }
 
     public static void write(File file, byte[] data) {
+        // Make sure the directory path to this file exists.
+        file.getParentFile().mkdirs();
+
         try (FileOutputStream fos = new FileOutputStream(file)) {
             fos.write(data);
             fos.close();

collect_app/src/main/java/org/odk/collect/android/widgets/OSMWidget.java

@@ -25,6 +25,7 @@
 import org.odk.collect.android.application.Collect;
 import org.odk.collect.android.http.CollectServerClient;
 import org.odk.collect.android.logic.FormController;
+import org.odk.collect.android.utilities.FileUtils;
 import org.odk.collect.android.utilities.ViewIds;
 import org.odk.collect.android.widgets.interfaces.BinaryWidget;
 
@@ -63,14 +64,7 @@ public OSMWidget(Context context, FormEntryPrompt prompt) {
 
         FormController formController = Collect.getInstance().getFormController();
 
-        /*
-         * NH: I'm trying to find the form xml file name, but this is neither
-         * in the formController nor the formDef. In fact, it doesn't seem to
-         * be saved into any object in JavaRosa. However, the mediaFolder
-         * has the substring of the file name in it, so I extract the file name
-         * from here. Awkward...
-         */
-        formFileName = formController.getMediaFolder().getName().split("-media")[0];
+        formFileName = FileUtils.getFormBasenameFromMediaFolder(formController.getMediaFolder());
 
         instanceDirectory = formController.getInstanceFile().getParent();
         instanceId = formController.getSubmissionMetadata().instanceId;