s3ql crashing under Debian 10 with google cloud

[johnnychicago]

Hello --

s3ql was always very stable until I upgraded Debian to 10. Now I see regular crashes (multiple times daily)

~# s3qladm --version S3QL 3.0 ~# cat /etc/issue Debian GNU/Linux 10 \n \l

Here's what it fails with, after a few GB of transfer:

`~# mount.s3ql --allow-other --fg --compress lzma-6 --cachesize 524288 --cachedir /[redacted] --authfile [redacted] [bucket] /s3/gs_cloud_data
Using 4 upload threads.
Autodetected 1048532 file descriptors available for cache entries
Requesting new access token
Ignoring locally cached metadata (outdated).
Downloading and decompressing metadata...
Reading metadata...
..objects..
..blocks..
..inodes..
..inode_blocks..
..symlink_targets..
..names..
..contents..
..ext_attributes..
Mounting gs://[redacted] at /s3/gs_cloud_data...
ERROR: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/lib/s3ql/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 445, in _upload_loop
self._do_upload(*tmp)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 472, in _do_upload
% obj_id).get_obj_size()
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 389, in exit
self.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 383, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 933, in close
self.metadata, size=self.obj_size)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 485, in write_fh
raise _map_request_error(exc, key) or exc
s3ql.backends.gs.RequestError: <RequestError, code=401, reason='Unauthorized', message='Invalid Credentials'>
WARNING: Unhandled top-level exception during shutdown (will not be re-raised)
ERROR: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/lib/s3ql/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 445, in _upload_loop
self._do_upload(*tmp)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 472, in _do_upload
% obj_id).get_obj_size()
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 389, in exit
self.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 383, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 933, in close
self.metadata, size=self.obj_size)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 485, in write_fh
raise _map_request_error(exc, key) or exc
s3ql.backends.gs.RequestError: <RequestError, code=401, reason='Unauthorized', message='Invalid Credentials'>
WARNING: Unhandled top-level exception during shutdown (will not be re-raised)
ERROR: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/lib/s3ql/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 445, in _upload_loop
self._do_upload(*tmp)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 472, in _do_upload
% obj_id).get_obj_size()
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 389, in exit
self.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 383, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 933, in close
self.metadata, size=self.obj_size)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 485, in write_fh
raise _map_request_error(exc, key) or exc
s3ql.backends.gs.RequestError: <RequestError, code=401, reason='Unauthorized', message='Invalid Credentials'>
WARNING: Unhandled top-level exception during shutdown (will not be re-raised)
ERROR: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/lib/s3ql/s3ql/mount.py", line 58, in run_with_except_hook
run_old(*args, **kw)
File "/usr/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 445, in _upload_loop
self._do_upload(*tmp)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 472, in _do_upload
% obj_id).get_obj_size()
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 279, in perform_write
return fn(fh)
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 389, in exit
self.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 383, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/comprenc.py", line 548, in close
self.fh.close()
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 933, in close
self.metadata, size=self.obj_size)
File "/usr/lib/s3ql/s3ql/backends/common.py", line 108, in wrapped
return method(*a, **kw)
File "/usr/lib/s3ql/s3ql/backends/gs.py", line 485, in write_fh
raise _map_request_error(exc, key) or exc
s3ql.backends.gs.RequestError: <RequestError, code=401, reason='Unauthorized', message='Invalid Credentials'>
ERROR: Unable to flush cache, no upload threads left alive
Unmounting file system...
ERROR: Uncaught top-level exception:
Traceback (most recent call last):
File "/usr/lib/s3ql/s3ql/mount.py", line 214, in main
raise RuntimeError('Received signal %d, terminating' % (ret,))
RuntimeError: Received signal 15, terminating

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/bin/mount.s3ql", line 11, in
load_entry_point('s3ql==3.0', 'console_scripts', 'mount.s3ql')()
File "/usr/lib/s3ql/s3ql/mount.py", line 229, in main
unmount_clean = True
File "/usr/lib/python3.7/contextlib.py", line 511, in exit
raise exc_details[1]
File "/usr/lib/python3.7/contextlib.py", line 496, in exit
if cb(*exc_details):
File "/usr/lib/python3.7/contextlib.py", line 383, in _exit_wrapper
callback(*args, **kwds)
File "/usr/lib/s3ql/s3ql/block_cache.py", line 431, in destroy
os.rmdir(self.path)
OSError: [Errno 39] Directory not empty: '/[redacted]'

`
I've mounted a filesystem on linode object storage, that seems to be holding. But the google one fails within an hour or so of use.

[johnnychicago]

My apologies - just realized this is about this issue:
#84
Upgrading using buster-backports seems to fix this.