Merge pull request #1126 from rfw/robust_fs_new_crashes

mozilla-services · Mar 8, 2013 · c7d46a7 · c7d46a7
2 parents 4a90f10 + 086596a
commit c7d46a7
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 14 deletions.
diff --git a/socorro/app/fetch_transform_save_app.py b/socorro/app/fetch_transform_save_app.py
@@ -104,9 +104,32 @@ def transform(self, crash_id):
         source to the destination without changing the data.  While this may
         be good enough for the raw crashmover, the processor would override
         this method to create and save processed crashes"""
-        raw_crash = self.source.get_raw_crash(crash_id)
-        dumps = self.source.get_raw_dumps(crash_id)
-        self.destination.save_raw_crash(raw_crash, dumps, crash_id)
+        try:
+            raw_crash = self.source.get_raw_crash(crash_id)
+        except Exception as x:
+            self.config.logger.error(
+                "reading raw_crash: %s",
+                str(x),
+                exc_info=True
+            )
+            raw_crash = {}
+        try:
+            dumps = self.source.get_raw_dumps(crash_id)
+        except Exception as x:
+            self.config.logger.error(
+                "reading dump: %s",
+                str(x),
+                exc_info=True
+            )
+            dumps = {}
+        try:
+            self.destination.save_raw_crash(raw_crash, dumps, crash_id)
+        except Exception as x:
+            self.config.logger.error(
+                "writing raw: %s",
+                str(x),
+                exc_info=True
+            )
 
     #--------------------------------------------------------------------------
     def quit_check(self):

diff --git a/socorro/external/fs/crashstorage.py b/socorro/external/fs/crashstorage.py
@@ -415,7 +415,15 @@ def _visit_minute_slot(self, minute_slot_base):
                         self._get_date_root_name(crash_id)
                     ])
                     yield crash_id
-                    os.unlink(date_root_path)
+
+                    try:
+                        os.unlink(date_root_path)
+                    except OSError as e:
+                        self.logger.error("could not find a date root in "
+                                          "%s; is crash corrupt?",
+                                          namedir,
+                                          exc_info=True)
+
                     os.unlink(namedir)
 
     def new_crashes(self):
@@ -473,9 +481,9 @@ def new_crashes(self):
                         os.rmdir(minute_slot_base)
                     except OSError as e:
                         self.logger.error("could not fully remove directory: "
-                                          "%s; are there more crashes in it? "
-                                          "%s",
-                                          minute_slot_base, e)
+                                          "%s; are there more crashes in it?",
+                                          minute_slot_base,
+                                          exc_info=True)
 
                 if not skip_dir and hour_slot < current_slot[0]:
                     try:
@@ -485,9 +493,9 @@ def new_crashes(self):
                         os.rmdir(hour_slot_base)
                     except OSError as e:
                        self.logger.error("could not fully remove directory: "
-                                          "%s; are there more crashes in it? "
-                                          "%s",
-                                          hour_slot_base, e)
+                                          "%s; are there more crashes in it?",
+                                          hour_slot_base,
+                                          exc_info=True)
 
 
 class FSLegacyDatedRadixTreeStorage(FSLegacyRadixTreeStorage,
@@ -527,9 +535,19 @@ def _visit_minute_slot(self, minute_slot_base):
                         namedir,
                         self._get_date_root_name(crash_id)
                     ])
+
                     yield crash_id
-                    os.unlink(date_root_path)
+
+                    try:
+                        os.unlink(date_root_path)
+                    except OSError as e:
+                        self.logger.error("could not find a date root in "
+                                          "%s; is crash corrupt?",
+                                          webhead_slot_base,
+                                          exc_info=True)
+
                     os.unlink(namedir)
+
             elif stat.S_ISDIR(st_result.st_mode):
                 webhead_slot = crash_id_or_webhead
                 webhead_slot_base = os.sep.join([minute_slot_base,
@@ -544,6 +562,8 @@ def _visit_minute_slot(self, minute_slot_base):
                     os.rmdir(webhead_slot_base)
                 except OSError as e:
                     self.logger.error("could not fully remove directory: "
-                                      "%s; are there more crashes in it? "
-                                      "%s",
-                                      webhead_slot_base, e)
+                                      "%s; are there more crashes in it?",
+                                      webhead_slot_base,
+                                      exc_info=True)
+            else:
+                self.logger.critical("unknown file %s found", namedir)