Add soft time limit for Celery tasks (#3019)

refinery/config/settings/base.py

@@ -336,6 +336,7 @@ def get_setting(name, settings=local_settings, default=None):
 CELERYD_MAX_TASKS_PER_CHILD = get_setting("CELERYD_MAX_TASKS_PER_CHILD")
 CELERY_ROUTES = {"file_store.tasks.FileImportTask": {"queue": "file_import"}}
 CELERY_ACCEPT_CONTENT = ['pickle']
+CELERYD_TASK_SOFT_TIME_LIMIT = 60  # seconds
 CELERYBEAT_SCHEDULE = {
     'collect_site_statistics': {
         'task': 'core.tasks.collect_site_statistics',

refinery/file_store/tasks.py

@@ -22,6 +22,8 @@
 
 class FileImportTask(celery.Task):
 
+    soft_time_limit = 3600  # 1 hour
+
     def run(self, item_uuid):
         """Download or copy data file for FileStoreItem specified by UUID
         Fail the task in case of errors (http://stackoverflow.com/a/33143545)
@@ -68,7 +70,7 @@ def run(self, item_uuid):
                     file_store_name = self.import_s3_to_path(item.source)
                 else:
                     file_store_name = self.import_url_to_path(item.source)
-        except RuntimeError as exc:
+        except (RuntimeError, celery.exceptions.SoftTimeLimitExceeded) as exc:
             logger.error("File import failed: %s", exc)
             self.update_state(state=celery.states.FAILURE,
                               meta='Failed to import file')