Skip to content

Commit

Permalink
fix(update): Don't deactive nodes with I/O issues
Browse files Browse the repository at this point in the history
Instead, they'll just be skipped by the running process,
which has the same effect as an inactive node, except that
alpenhornd will re-check the node each time through the main
loop.

This should mean alpenhorn can recover from intermittant I/O issues
on /project space at cedar.
  • Loading branch information
ketiltrout committed Oct 23, 2023
1 parent 72a50b7 commit 24c5299
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 10 deletions.
10 changes: 2 additions & 8 deletions alpenhorn/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,15 +274,9 @@ def update_active(self) -> bool:
if self.io.check_active():
return True
else:
# Mark the node as inactive in the database
self.db.active = False
self.db.save(only=[StorageNode.active])
log.info(
f'Correcting the database: node "{self.name}" is now set to '
"inactive."
)
log.warning(f'Ignoring node "{self.name}": not ready for I/O.')
else:
log.warning(f'Attempted to update inactive node "{self.name}"')
log.warning(f'Ignoring node "{self.name}": deactivated during update.')

return False

Expand Down
4 changes: 2 additions & 2 deletions tests/test_update_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,8 @@ def test_update_active(unode):
# Pretend node is actually not active
with patch.object(unode.io, "check_active", lambda: False):
assert not unode.update_active()
assert not unode.db.active
assert not StorageNode.select(StorageNode.active).limit(1).scalar()
assert unode.db.active
assert StorageNode.select(StorageNode.active).limit(1).scalar()


def test_update_free_space(unode):
Expand Down

0 comments on commit 24c5299

Please sign in to comment.