Write out registry.xml file in case jobs are missing this file

lib/galaxy/jobs/handler.py

@@ -94,6 +94,16 @@ def job_pair_for_id(self, id):
         job = self.sa_session.query(model.Job).get(id)
         return job, self.job_wrapper(job, use_persisted_destination=True)
 
+    def __write_registry_file_if_absent(self, job):
+        job_wrapper =  self.job_wrapper(job)
+        cwd = job_wrapper.working_directory
+        datatypes_config = os.path.join(cwd, "registry.xml")
+        if not os.path.exists(datatypes_config):
+            try:
+                self.app.datatypes_registry.to_xml_file(path=datatypes_config)
+            except OSError:
+                pass
+
     def __check_jobs_at_startup(self):
         """
         Checks all jobs that are in the 'new', 'queued' or 'running' state in
@@ -121,6 +131,7 @@ def __check_jobs_at_startup(self):
                         (model.Job.handler == self.app.config.server_name)).all()
 
         for job in jobs_at_startup:
+            self.__write_registry_file_if_absent(job)
             if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True):
                 log.warning("(%s) Tool '%s' removed from tool config, unable to recover job" % (job.id, job.tool_id))
                 self.job_wrapper(job).fail('This tool was disabled before the job completed.  Please contact your Galaxy administrator.')

lib/galaxy_ext/metadata/set_metadata.py

@@ -80,6 +80,16 @@ def set_metadata():
 
     # Set up datatypes registry
     datatypes_config = sys.argv.pop(1)
+    if not os.path.exists(datatypes_config):
+        # This path should exist, except for jobs that started running on release 17.05, where a global
+        # datatypes_config (instead of a datatypes_config per job) was used. For a while release 17.05
+        # would remove the global datatypes config on shutdown and toolbox reload, which would lead to
+        # failed metadata jobs. To remedy this we scan jobs at startup for missing registry.xml files,
+        # and if we detect such a job we write out the current registry.xml file.
+        datatypes_config = os.path.join(tool_job_working_directory, "registry.xml")
+        if not os.path.exists(datatypes_config):
+            print("Metadata setting failed because registry.xml could not be found. You may retry setting metadata.")
+            sys.exit(1)
     datatypes_registry = galaxy.datatypes.registry.Registry()
     datatypes_registry.load_datatypes(root_dir=galaxy_root, config=datatypes_config)
     galaxy.model.set_datatypes_registry(datatypes_registry)