Skip to content
Permalink
Browse files
fix: report error and possible cause if metadata cleanup fails (#1554)
  • Loading branch information
johanneskoester committed Apr 1, 2022
1 parent 77d5a08 commit 68661341efa0a3de4e03de3fb1b8f3117de66efe
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 3 deletions.
@@ -186,7 +186,7 @@ def cleanup_locks(self):
shutil.rmtree(self._lockdir)

def cleanup_metadata(self, path):
self._delete_record(self._metadata_path, path)
return self._delete_record(self._metadata_path, path)

def cleanup_shadow(self):
if os.path.exists(self.shadow_path):
@@ -403,9 +403,14 @@ def _delete_record(self, subject, id):
recdirs = os.path.relpath(os.path.dirname(recpath), start=subject)
if recdirs != ".":
os.removedirs(recdirs)
return True
except OSError as e:
if e.errno != 2: # not missing
if e.errno != 2:
# not missing
raise e
else:
# file is missing, report failure
return False

@lru_cache()
def _read_record_cached(self, subject, id):
@@ -764,8 +764,18 @@ def files(items):
self.persistence.deactivate_cache()

if cleanup_metadata:
failed = []
for f in cleanup_metadata:
self.persistence.cleanup_metadata(f)
success = self.persistence.cleanup_metadata(f)
if not success:
failed.append(f)
if failed:
logger.warning(
"Failed to clean up metadata for the following files because the metadata was not present.\n"
"If this is expected, there is nothing to do.\nOtherwise, the reason might be file system latency "
"or still running jobs.\nConsider running metadata cleanup again.\nFiles:\n"
+ "\n".join(failed)
)
return True

if unlock:
@@ -1589,3 +1589,7 @@ def test_github_issue1500():

def test_github_issue1542():
run(dpath("test_github_issue1542"), dryrun=True)


def test_cleanup_metadata_fail():
run(dpath("test09"), cleanup_metadata=["xyz"])

0 comments on commit 6866134

Please sign in to comment.