Save metadata in operation output dir. Closes #1917 (#1919)

src/plugins/GenerateJob/templates/main.ejs

@@ -2,6 +2,7 @@ import json
 import deepforge
 import signal
 import sys
+import os
 
 def signal_handler(signum, frame):
     sig = signal.Signals(signum)
@@ -34,13 +35,13 @@ print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation #
 })%>)
 print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
 
-result_types = {}
 <% outputs.forEach(name => { %>
-with open('outputs/<%= name %>', 'wb') as outfile:
+os.makedirs('outputs/<%= name %>/', exist_ok=True)
+with open('outputs/<%= name %>/metadata.json', 'w') as outfile:
+    metadata = {}
+    metadata['type'] = deepforge.serialization.get_full_class_name(<%= name %>)
+    json.dump(metadata, outfile)
+
+with open('outputs/<%= name %>/data', 'wb') as outfile:
     deepforge.serialization.dump(<%= name %>, outfile)
-    result_types['<%= name %>'] = deepforge.serialization.get_full_class_name(<%= name %>)
 <% }); %>
-
-# Save the result types
-with open('result-types.json', 'w') as f:
-    json.dump(result_types, f)

src/plugins/GenerateJob/templates/start.js

@@ -5,6 +5,7 @@
 const childProcess = require('child_process');
 const {spawn} = childProcess;
 const fs = require('fs');
+const fsp = fs.promises;
 const {promisify} = require('util');
 const mkdir = promisify(fs.mkdir);
 const pipeline = promisify(require('stream').pipeline);
@@ -162,19 +163,19 @@ requirejs([
 
     async function uploadOutputData(exitCode) {
         if (exitCode === 0) {
-            const results = require('./result-types.json');
+            const outputNames = await fsp.readdir('./outputs');
+            const results = {};
             const storageId = Config.storage.id;
             const config = Config.storage.config;
             const client = await Storage.getClient(storageId, logger, config);
-            const outputNames = Object.keys(results);
             const storageDir = Config.storage.dir;
 
             for (let i = outputNames.length; i--;) {
                 const filename = outputNames[i];
                 const storagePath = `${storageDir}/${filename}`;
-                const contentsStream = fs.createReadStream(`outputs/${filename}`);
+                const contentsStream = fs.createReadStream(`outputs/${filename}/data`);
                 const dataInfo = await client.putFileStream(storagePath, contentsStream);
-                const type = results[filename];
+                const type = require(`./outputs/${filename}/metadata.json`).type;
                 results[filename] = {type, dataInfo};
             }