Update #208: archive xml if missing item, add optional low ttl

hdsdi3g · Mar 29, 2017 · 6fc5b1a · 6fc5b1a
1 parent d582d9d
commit 6fc5b1a
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 13 deletions.
diff --git a/UPGRADE.txt b/UPGRADE.txt
@@ -1,7 +1,7 @@
 === MyDMAM Upgrade document ===
 
 #239 FTP server admin (in Play side) can create FTP account user directories. Just add some ftpservergroups/group name/base_working_dir keys in configuration
-#208 Add watchfolderopts.max_founded_items configuration entry for limit big watchfolders job explosion
+#208 Add watchfolderopts.max_founded_items and dontkeepdone configuration entry for limit big watchfolders job explosion
 
 v0.19
 #234  Change configuration keys: move auth.* to play.*

diff --git a/app/hd3gtv/mydmam/transcode/kit/PKitOpAtomTo1A_XMLBased.java b/app/hd3gtv/mydmam/transcode/kit/PKitOpAtomTo1A_XMLBased.java
@@ -144,7 +144,9 @@ public List<File> process(File physical_source, Container source_indexing_result
 			if (file_atom_0.exists() && file_atom_0.isFile()) {
 				mxf_files.add(new Atom(file_atom_0));
 			} else {
-				throw new FileNotFoundException("Can't found main MXF file " + file_atom_0);
+				Loggers.Transcode.warn("Can't found main MXF file " + file_atom_0 + ", archive xml file to xml-old");
+				FileUtils.copyFile(physical_source, new File(FilenameUtils.removeExtension(physical_source.getAbsolutePath()) + ".xml-old"));
+				return null;
 			}
 			if (file_atom_1.exists() && file_atom_1.isFile()) {
 				mxf_files.add(new Atom(file_atom_1));
@@ -383,7 +385,6 @@ public boolean accept(File dir, String name) {
 			mxf_files.forEach(atom -> {
 				atom.path.delete();
 			});
-			// physical_source.delete();
 
 			return null;
 		}

diff --git a/app/hd3gtv/mydmam/transcode/watchfolder/AbstractFoundedFile.java b/app/hd3gtv/mydmam/transcode/watchfolder/AbstractFoundedFile.java
@@ -66,17 +66,22 @@ public enum Status {
 		map_job_target = MyDMAM.gson_kit.getGsonSimple().fromJson(cols.getStringValue("map_job_target", "{}"), GsonKit.type_HashMap_String_String);
 	}
 
-	void saveToCassandra(MutationBatch mutator) {
+	void saveToCassandra(MutationBatch mutator, boolean terminate) {
 		if (Loggers.Transcode_WatchFolder.isTraceEnabled()) {
 			Loggers.Transcode_WatchFolder.trace("Prepare saveToCassandra for:\t" + this);
 		}
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("path", path, WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("storage_name", storage_name, WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("filedate", date, WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("filesize", size, WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("status", status.name(), WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("last_checked", last_checked, WatchFolderTranscoder.TTL_CASSANDRA);
-		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("map_job_target", MyDMAM.gson_kit.getGsonSimple().toJson(map_job_target), WatchFolderTranscoder.TTL_CASSANDRA);
+		int ttl = WatchFolderTranscoder.TTL_CASSANDRA;
+		if (terminate && WatchFolderTranscoder.DONT_KEEP_DONE) {
+			ttl = WatchFolderTranscoder.TTL_CASSANDRA_SHORT;
+		}
+
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("path", path, ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("storage_name", storage_name, ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("filedate", date, ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("filesize", size, ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("status", status.name(), ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("last_checked", last_checked, ttl);
+		mutator.withRow(WatchFolderDB.CF_WATCHFOLDERS, getPathIndexKey()).putColumn("map_job_target", MyDMAM.gson_kit.getGsonSimple().toJson(map_job_target), ttl);
 	}
 
 	AbstractFoundedFile(AbstractFile found_file, String storage_name) {

diff --git a/app/hd3gtv/mydmam/transcode/watchfolder/WatchFolderDB.java b/app/hd3gtv/mydmam/transcode/watchfolder/WatchFolderDB.java
@@ -127,7 +127,7 @@ static void push(List<AbstractFoundedFile> files) throws ConnectionException {
 			if (Loggers.Transcode_WatchFolder.isTraceEnabled()) {
 				Loggers.Transcode_WatchFolder.trace("Save FoundedFile in DB\t" + files.get(pos));
 			}
-			files.get(pos).saveToCassandra(mutator);
+			files.get(pos).saveToCassandra(mutator, false);
 		}
 		mutator.execute();
 	}
@@ -138,12 +138,13 @@ static void switchStatus(String path_index_key, AbstractFoundedFile.Status new_s
 		a_file.status = new_status;
 
 		MutationBatch mutator = CassandraDb.prepareMutationBatch();
-		a_file.saveToCassandra(mutator);
+		a_file.saveToCassandra(mutator, AbstractFoundedFile.Status.PROCESSED.equals(new_status));
 
 		if (Loggers.Transcode_WatchFolder.isDebugEnabled()) {
 			Loggers.Transcode_WatchFolder.debug("Switch FoundedFile status: " + path_index_key + " is now " + new_status + " for this file: " + a_file.storage_name + ":" + a_file.path);
 		}
 		mutator.execute();
+		a_file.close();
 	}
 
 	static ColumnPrefixDistributedRowLock<String> prepareLock(String pathindexkey) {

diff --git a/app/hd3gtv/mydmam/transcode/watchfolder/WatchFolderTranscoder.java b/app/hd3gtv/mydmam/transcode/watchfolder/WatchFolderTranscoder.java
@@ -31,6 +31,9 @@
 public class WatchFolderTranscoder {
 
 	static final int TTL_CASSANDRA = (int) TimeUnit.HOURS.toSeconds(24);
+	static final int TTL_CASSANDRA_SHORT = (int) TimeUnit.MINUTES.toSeconds(10);
+	static final boolean DONT_KEEP_DONE = Configuration.global.getValueBoolean("watchfolderopts", "dontkeepdone");
+
 	static final long TTL_ES = TimeUnit.HOURS.toMillis(24);
 
 	private transient ThreadGroup wf_group;

diff --git a/conf/app.d-examples/transcoding.yml.example b/conf/app.d-examples/transcoding.yml.example
@@ -127,3 +127,4 @@ watchfoldertranscoder:
 #watchfolderopts
 #    max_founded_items: 100 # Limits the quantity of founded files stored in database (also and the limit of the created transcoding Jobs quantity)
 #                           # After the operation, the founded file entries and all transcoding related Jobs will be removed
+#    dontkeepdone: true     # Don't keep too long time founded files DB references after processing. If max_founded_items > 0, this param has not behaviors.