Merge branch 'master' into TWO_PRODUCTS

rust-toolchain

@@ -1 +1 @@
-nightly-2019-11-01
+nightly-2020-02-01

src/config.rs

@@ -253,6 +253,7 @@ pub struct MassStorageConfig {
     #[serde(flatten)]
     pub location: MountableDeviceLocation,
     pub extensions: Vec<String>,
+    pub cleanup_extensions: Option<Vec<String>>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
@@ -299,6 +300,8 @@ pub enum ConfigError {
     InvalidApiBase(url::ParseError),
     #[fail(display = "The token file does not exist. Did you login?")]
     NoTokenFile,
+    #[fail(display = "mass_storage entries must have an extensions = [...] key")]
+    MassStorageMissingExtensions,
 }
 
 impl FromStr for Config {
@@ -339,6 +342,7 @@ impl Config {
         }
 
         Config::check_staging(&config.staging)?;
+        Config::check_mass_storages(config.mass_storages())?;
 
         if let Some(base) = &config.stokepile.api_base {
             if let Err(err) = url::Url::parse(&base) {
@@ -363,6 +367,16 @@ impl Config {
         Ok(())
     }
 
+    #[must_use]
+    fn check_mass_storages(storages: &[MassStorageConfig]) -> Result<(), ConfigError> {
+        for ms in storages {
+            if ms.extensions.len() == 0 {
+                Err(ConfigError::MassStorageMissingExtensions)?
+            }
+        }
+        Ok(())
+    }
+
     /// Get the api base of this config, or return the default
     pub fn api_base(&self) -> &str {
         match &self.stokepile.api_base {
@@ -625,6 +639,7 @@ mod tests {
                 name: "video".into(),
                 location: MountableDeviceLocation::from_mountpoint("/mnt/stokepile/mass_storage".into()),
                 extensions: vec!["mp4".into()],
+                cleanup_extensions: Some(vec!["lrv".into(), "thm".into()]),
             }])
         );
 
@@ -856,11 +871,13 @@ token="DROPBOX_TOKEN_GOES_HERE"
                     name: "front".into(),
                     location: MountableDeviceLocation::Mountpoint("/mnt/stokepile/front".into()),
                     extensions: vec!["mp4".into()],
+                    cleanup_extensions: None,
                 },
                 MassStorageConfig {
                     name: "back".into(),
                     location: MountableDeviceLocation::Label("back_mass_storage".into()),
                     extensions: vec!["mov".into()],
+                    cleanup_extensions: None,
                 }
             ]
         )
@@ -924,6 +941,26 @@ extensions = ["mov"]
         assert_no_flysights(&config);
     }
 
+    #[test]
+    fn rejects_mass_storage_without_extensions() {
+        let err = Config::from_str(
+            r#"
+[stokepile]
+[staging]
+mountpoint = "/test"
+[dropbox]
+token="DROPBOX_TOKEN_GOES_HERE"
+
+[[mass_storage]]
+name = "front"
+mountpoint="/mnt/stokepile/front"
+extensions = []
+"#,
+        )
+        .unwrap_err();
+        assert_eq!(err, ConfigError::MassStorageMissingExtensions);
+    }
+
     #[test]
     fn test_gopros() {
         let config = Config::from_str(

src/mass_storage.rs

@@ -53,31 +53,74 @@ impl StageFromDevice for MountedMassStorage {
     type FileType = MassStorageFile;
 
     fn files(&self) -> Result<Vec<MassStorageFile>, Error> {
+        // Screw it
         let mut out = vec![];
-        for entry in WalkDir::new(&self.mount.path()) {
-            let entry = entry?;
-            if entry.file_type().is_dir() {
-                continue;
-            }
-
-            let path = entry.path();
-            if let Some(ext) = path.extension() {
-                let extension = ext.to_str().unwrap().to_lowercase();
-                if !self.mass_storage.extensions.contains(&extension) {
-                    continue;
-                }
 
-                out.push(MassStorageFile {
-                    capturedatetime: path.metadata()?.modified()?.into(),
-                    file: File::open(path)
-                        .context("Opening content file for MountedMassStorage")?,
-                    source_path: path.to_path_buf(),
-                    extension,
-                });
-            }
+        for ref path in self.files_matching_extensions() {
+            // Could definitely lift this into some domain object
+            let extension = path.extension().unwrap().to_str().unwrap().to_lowercase();
+            let file = MassStorageFile {
+                capturedatetime: path.metadata()?.modified()?.into(),
+                file: File::open(path)
+                    .context("Opening content file for MountedMassStorage")?,
+                source_path: path.to_path_buf(),
+                extension,
+            };
+            out.push(file);
         }
         Ok(out)
     }
+
+    fn cleanup(&self) -> Result<(), Error> {
+        for path in self.files_matching_cleanup_extensions() {
+            fs::remove_file(path)?;
+        }
+        Ok(())
+    }
+}
+
+impl MountedMassStorage {
+    /// Returns `PathBuf`s for all the files matching the `extensions` field on this MassStorage.
+    pub fn files_matching_extensions(&self) -> Vec<PathBuf> {
+        self.map_files_with_extensions(&self.mass_storage.extensions[..]).collect()
+    }
+
+    /// Returns `PathBuf`s for all the files matching the `cleanup_extensions` field on this MassStorage.
+    pub fn files_matching_cleanup_extensions(&self) -> Vec<PathBuf> {
+        match &self.mass_storage.cleanup_extensions {
+            Some(extns) => {
+                self.map_files_with_extensions(extns).collect()
+            },
+            None => {
+                vec![]
+            },
+        }
+    }
+
+
+    fn map_files_with_extensions<'a>(&self, extensions: &'a [String]) -> impl Iterator<Item=PathBuf> + 'a {
+        WalkDir::new(&self.mount.path())
+            .into_iter()
+            .filter_map(move |entry| {
+                // TODO(richo) Do we think this actually can fail?
+                let entry = entry.unwrap();
+                if entry.file_type().is_dir() {
+                    return None;
+                }
+
+                let path = entry.path();
+                if let Some(ext) = path.extension() {
+                    let extension = ext.to_str().unwrap().to_lowercase();
+                    if !extensions.contains(&extension) {
+                        return None;
+                    }
+
+                    return Some(path.to_path_buf());
+                } else {
+                    return None;
+                }
+            })
+    }
 }
 
 impl MountableFilesystem for MassStorageConfig {
@@ -138,6 +181,7 @@ mod tests {
             name: "data".into(),
             location: MountableDeviceLocation::from_mountpoint("test-data/mass_storage".into()),
             extensions: extensions(),
+            cleanup_extensions: None,
         };
         let mounted = mass_storage.mount_for_test();
 
@@ -158,16 +202,23 @@ mod tests {
             name: "data".into(),
             location: MountableDeviceLocation::from_mountpoint(source.path().to_path_buf()),
             extensions: extensions(),
+            cleanup_extensions: Some(vec!["lrv".into()]),
         };
 
         let mounted = mass_storage.mount_for_test();
 
-        mounted.stage_files("data", &dest).unwrap();
+        // Confirm that we see the lrv files before hand
+        assert_eq!(mounted.files_matching_cleanup_extensions().len(), 2);
+
+        let mounted = mounted.stage_files_for_test("data", &dest).unwrap();
         // TODO(richo) test harder
         let iter = fs::read_dir(&dest.staging_location()).unwrap();
         let files: Vec<_> = iter.collect();
 
         // Two files for the two mp4 files, two files for the manifests
         assert_eq!(files.len(), 4);
+
+        // Assert that the original lrv's are gone.
+        assert_eq!(mounted.files_matching_cleanup_extensions().len(), 0);
     }
 }

src/staging.rs

@@ -335,6 +335,17 @@ pub trait StageFromDevice: Sized {
         Ok(i)
     }
 
+    /// As per `stage_files` but returns the underlying object so you can inspect it in a test
+    /// setting.
+    #[cfg(test)]
+    fn stage_files_for_test<T: StagingLocation>(self, name: &str, stager: &Stager<T>) -> Result<Self, Error> {
+        for file in self.files()? {
+            stager.stage(file, name)?;
+        }
+        self.cleanup()?;
+        Ok(self)
+    }
+
     fn cleanup(&self) -> Result<(), Error> {
         Ok(())
     }

src/web/models/device.rs

@@ -32,6 +32,7 @@ impl From<Device> for config::DeviceConfig {
                     // TODO(richo) add a metadata field and store this there
                     extensions: vec!["mp4".into()],
                     location: MountableDeviceLocation::from_label(device.identifier),
+                    cleanup_extensions: None,
                 })
             }
             "flysight" => config::DeviceConfig::Flysight(FlysightConfig {

stokepile.toml.example

@@ -30,6 +30,7 @@ mountpoint="/mnt/stokepile/mass_storage"
 # The extensions of files that we should be archiving
 # Only files with this extension will be uploaded and removed, leaving the directories intact
 extensions = ["mp4"]
+cleanup_extensions = ["lrv", "thm"]
 
 #  [gswoop]
 #  binary = "/Applications/gSwoop.app/Contents/MacOS/gswoop"