Skip to content

Commit

Permalink
Sort the result of os.listdir.
Browse files Browse the repository at this point in the history
On Linux this should make no difference, but on Mac it may.
It at least solves some spurious test failures on my Mac.
  • Loading branch information
mauritsvanrees committed Apr 11, 2018
1 parent 9398099 commit 18b22f4
Show file tree
Hide file tree
Showing 5 changed files with 31 additions and 30 deletions.
1 change: 1 addition & 0 deletions news/45.bugfix
@@ -0,0 +1 @@
Sort os.listdir, for improved functioning on Mac. [maurits]
8 changes: 4 additions & 4 deletions src/collective/recipe/backup/copyblobs.py
Expand Up @@ -280,7 +280,7 @@ def get_valid_directories(container, name):
"""
valid_entries = []
for entry in os.listdir(container):
for entry in sorted(os.listdir(container)):
if not entry.startswith(name + '.'):
continue
entry_start, entry_num = entry.rsplit('.', 1)
Expand Down Expand Up @@ -343,7 +343,7 @@ def get_valid_archives(container, name):
>>> remove('dirtest')
"""
valid_entries = []
for entry in os.listdir(container):
for entry in sorted(os.listdir(container)):
matched = re.match('^{0}\.(\d+)\.tar(\.gz)?$'.format(name), entry)
if matched is None:
continue
Expand Down Expand Up @@ -463,7 +463,7 @@ def get_blob_backup_dirs(backup_location, only_timestamps=False):
If only_timestamps is True, we only return backups that have timestamps.
That is useful when restoring.
"""
filenames = os.listdir(backup_location)
filenames = sorted(os.listdir(backup_location))
logger.debug('Looked up filenames in the target dir: %s found. %r.',
len(filenames), filenames)
backup_dirs = []
Expand Down Expand Up @@ -527,7 +527,7 @@ def get_blob_backup_archives(
If only_timestamps is True, we only return backups that have timestamps.
That is useful when restoring.
"""
filenames = os.listdir(backup_location)
filenames = sorted(os.listdir(backup_location))
logger.debug('Looked up filenames in the target dir: %s found. %r.',
len(filenames), filenames)
backup_archives = []
Expand Down
2 changes: 1 addition & 1 deletion src/collective/recipe/backup/repozorunner.py
Expand Up @@ -328,7 +328,7 @@ def cleanup(backup_location, keep=0):
"Value of 'keep' is %r, we don't want to remove anything.", keep)
return
logger.debug('Trying to clean up old backups.')
filenames = os.listdir(backup_location)
filenames = sorted(os.listdir(backup_location))
logger.debug('Looked up filenames in the target dir: %s found. %r.',
len(filenames), filenames)
num_backups = int(keep)
Expand Down
12 changes: 6 additions & 6 deletions src/collective/recipe/backup/tests/backup_blobs_dir.rst
Expand Up @@ -75,7 +75,7 @@ Now cleanup and try with filestamps.
>>> backup_blobs('blobs', 'backups', timestamps=True)
>>> ls('backups')
d blobs.20...-...-...-...-...
>>> backup0 = os.listdir('backups')[0]
>>> backup0 = sorted(os.listdir('backups'))[0]
>>> timestamp0 = backup0[len('blobs.'):]
>>> ls('backups', backup0, 'blobs')
d dir
Expand All @@ -92,7 +92,7 @@ Wait a while, so we get a different timestamp, and then change some stuff.
>>> ls('backups')
d blobs.20...-...-...-...-...
d blobs.20...-...-...-...-...
>>> backup1 = os.listdir('backups')[-1]
>>> backup1 = sorted(os.listdir('backups'))[-1]
>>> timestamp1 = backup1[len('blobs.'):]
>>> timestamp0 < timestamp1
True
Expand All @@ -114,9 +114,9 @@ to any filestorage backup.
... fs_backup_location='fs')
>>> ls('backups')
d blobs.20...-...-...-...-...
>>> len(os.listdir('backups')) # The dots could shadow other backups.
>>> len(sorted(os.listdir('backups'))) # The dots could shadow other backups.
1
>>> backup1 == os.listdir('backups')[0]
>>> backup1 == sorted(os.listdir('backups'))[0]
True
>>> ls('backups', backup1, 'blobs')
d dir
Expand All @@ -131,7 +131,7 @@ Pretend there is a newer filestorage backup and a blob change.
>>> ls('backups')
d blobs.20...-...-...-...-...
d blobs.2100-01-01-00-00-00
>>> len(os.listdir('backups')) # The dots could shadow a third backup
>>> len(sorted(os.listdir('backups'))) # The dots could shadow a third backup
2
>>> ls('backups', 'blobs.2100-01-01-00-00-00', 'blobs')
d dir
Expand All @@ -145,7 +145,7 @@ Remove the oldest filestorage backup.
... fs_backup_location='fs')
>>> ls('backups')
d blobs.2100-01-01-00-00-00
>>> len(os.listdir('backups'))
>>> len(sorted(os.listdir('backups')))
1

Cleanup:
Expand Down
38 changes: 19 additions & 19 deletions src/collective/recipe/backup/tests/blob_timestamps.rst
Expand Up @@ -78,17 +78,17 @@ Test the snapshotbackup first, as that should be easiest.
--backup -f /sample-buildout/var/filestorage/Data.fs -r /sample-buildout/var/snapshotbackups -F --gzip
>>> ls('var', 'blobstoragesnapshots')
d blobstorage.20...-...-...-...-...-...
>>> timestamp0 = os.listdir('var/blobstoragesnapshots/')[0]
>>> timestamp0 = sorted(os.listdir('var/blobstoragesnapshots/'))[0]
>>> ls('var', 'blobstoragesnapshots', timestamp0)
d blobstorage
>>> ls('var', 'blobstoragesnapshots_foo')
d blobstorage-foo.20...-...-...-...-...-...
>>> foo_timestamp0 = os.listdir('var/blobstoragesnapshots_foo/')[0]
>>> foo_timestamp0 = sorted(os.listdir('var/blobstoragesnapshots_foo/'))[0]
>>> ls('var', 'blobstoragesnapshots_foo', foo_timestamp0)
d blobstorage-foo
>>> ls('var', 'blobstoragesnapshots_bar')
d blobstorage-bar.20...-...-...-...-...-...
>>> bar_timestamp0 = os.listdir('var/blobstoragesnapshots_bar/')[0]
>>> bar_timestamp0 = sorted(os.listdir('var/blobstoragesnapshots_bar/'))[0]
>>> ls('var', 'blobstoragesnapshots_bar', bar_timestamp0)
d blobstorage-bar

Expand Down Expand Up @@ -118,9 +118,9 @@ Note that due to the timestamps no renaming takes place from blobstorage.0 to bl
>>> ls('var/blobstoragesnapshots')
d blobstorage.20...-...-...-...-...-...
d blobstorage.20...-...-...-...-...-...
>>> timestamp0 == os.listdir('var/blobstoragesnapshots/')[0]
>>> timestamp0 == sorted(os.listdir('var/blobstoragesnapshots/'))[0]
True
>>> timestamp1 = os.listdir('var/blobstoragesnapshots/')[1]
>>> timestamp1 = sorted(os.listdir('var/blobstoragesnapshots/'))[1]
>>> ls('var', 'blobstoragesnapshots', timestamp1, 'blobstorage')
- blob1.txt
- blob2.txt
Expand All @@ -135,9 +135,9 @@ Note that due to the timestamps no renaming takes place from blobstorage.0 to bl
>>> ls('var', 'blobstoragesnapshots_foo')
d blobstorage-foo.20...-...-...-...-...-...
d blobstorage-foo.20...-...-...-...-...-...
>>> foo_timestamp0 == os.listdir('var/blobstoragesnapshots_foo/')[0]
>>> foo_timestamp0 == sorted(os.listdir('var/blobstoragesnapshots_foo/'))[0]
True
>>> foo_timestamp1 = os.listdir('var/blobstoragesnapshots_foo/')[1]
>>> foo_timestamp1 = sorted(os.listdir('var/blobstoragesnapshots_foo/'))[1]
>>> ls('var', 'blobstoragesnapshots_foo', foo_timestamp1, 'blobstorage-foo')
- blob-foo1.txt
- blob-foo2.txt
Expand Down Expand Up @@ -178,11 +178,11 @@ But let's test it for good measure::
d blobstorage.20...-...-...-...-...-...
d blobstorage.20...-...-...-...-...-...
d blobstorage.20...-...-...-...-...-...
>>> timestamp0 == os.listdir('var/blobstoragesnapshots/')[0]
>>> timestamp0 == sorted(os.listdir('var/blobstoragesnapshots/'))[0]
True
>>> timestamp1 == os.listdir('var/blobstoragesnapshots/')[1]
>>> timestamp1 == sorted(os.listdir('var/blobstoragesnapshots/'))[1]
True
>>> timestamp2 = os.listdir('var/blobstoragesnapshots/')[2]
>>> timestamp2 = sorted(os.listdir('var/blobstoragesnapshots/'))[2]
>>> ls('var', 'blobstoragesnapshots', timestamp2, 'blobstorage')
- blob1.txt
>>> ls('var', 'blobstoragesnapshots', timestamp1, 'blobstorage')
Expand All @@ -200,11 +200,11 @@ But let's test it for good measure::
d blobstorage-foo.20...-...-...-...-...-...
d blobstorage-foo.20...-...-...-...-...-...
d blobstorage-foo.20...-...-...-...-...-...
>>> foo_timestamp0 == os.listdir('var/blobstoragesnapshots_foo/')[0]
>>> foo_timestamp0 == sorted(os.listdir('var/blobstoragesnapshots_foo/'))[0]
True
>>> foo_timestamp1 == os.listdir('var/blobstoragesnapshots_foo/')[1]
>>> foo_timestamp1 == sorted(os.listdir('var/blobstoragesnapshots_foo/'))[1]
True
>>> foo_timestamp2 = os.listdir('var/blobstoragesnapshots_foo/')[2]
>>> foo_timestamp2 = sorted(os.listdir('var/blobstoragesnapshots_foo/'))[2]
>>> ls('var', 'blobstoragesnapshots_foo', foo_timestamp2, 'blobstorage-foo')
- blob-foo2.txt
>>> ls('var', 'blobstoragesnapshots_foo', foo_timestamp1, 'blobstorage-foo')
Expand Down Expand Up @@ -244,14 +244,14 @@ Let's see how a bin/backup goes:
--backup -f /sample-buildout/var/filestorage/foo.fs -r /sample-buildout/var/backups_foo --quick --gzip
--backup -f /sample-buildout/var/filestorage/bar.fs -r /sample-buildout/var/backups_bar --quick --gzip
--backup -f /sample-buildout/var/filestorage/Data.fs -r /sample-buildout/var/backups --quick --gzip
>>> backup_timestamp0 = os.listdir('var/blobstoragebackups/')[0]
>>> backup_timestamp0 = sorted(os.listdir('var/blobstoragebackups/'))[0]
>>> ls('var', 'blobstoragebackups')
d blobstorage.20...-...-...-...-...-...
>>> ls('var', 'blobstoragebackups', backup_timestamp0)
d blobstorage
>>> ls('var', 'blobstoragebackups', backup_timestamp0, 'blobstorage')
- blob1.txt
>>> foo_backup_timestamp0 = os.listdir('var/blobstoragebackups_foo/')[0]
>>> foo_backup_timestamp0 = sorted(os.listdir('var/blobstoragebackups_foo/'))[0]
>>> ls('var', 'blobstoragebackups_foo')
d blobstorage-foo.20...-...-...-...-...-...
>>> ls('var', 'blobstoragebackups_foo', foo_backup_timestamp0)
Expand Down Expand Up @@ -282,9 +282,9 @@ We try again with an extra 'blob' and a changed 'blob':
>>> ls('var', 'blobstoragebackups')
d blobstorage.20...-...-...-...-...-...
d blobstorage.20...-...-...-...-...-...
>>> backup_timestamp0 == os.listdir('var/blobstoragebackups/')[0]
>>> backup_timestamp0 == sorted(os.listdir('var/blobstoragebackups/'))[0]
True
>>> backup_timestamp1 = os.listdir('var/blobstoragebackups/')[1]
>>> backup_timestamp1 = sorted(os.listdir('var/blobstoragebackups/'))[1]
>>> ls('var', 'blobstoragebackups', backup_timestamp1, 'blobstorage')
- blob1.txt
- blob2.txt
Expand Down Expand Up @@ -583,7 +583,7 @@ Now we test it::
--backup -f /sample-buildout/var/filestorage/Data.fs -r /sample-buildout/var/zipbackups -F --gzip
>>> ls('var', 'blobstoragezips')
- blobstorage.20...-...-...-...-...-....tar
>>> zip_timestamp0 = os.listdir('var/blobstoragezips')[0]
>>> zip_timestamp0 = sorted(os.listdir('var/blobstoragezips'))[0]

Keep is ignored by zipbackup, always using 1 as value.
Pause a short time to avoid getting an error for overwriting the previous file::
Expand All @@ -599,7 +599,7 @@ Pause a short time to avoid getting an error for overwriting the previous file::
--backup -f /sample-buildout/var/filestorage/Data.fs -r /sample-buildout/var/zipbackups -F --gzip
>>> ls('var', 'blobstoragezips')
- blobstorage.20...-...-...-...-...-....tar
>>> zip_timestamp1 = os.listdir('var/blobstoragezips')[0]
>>> zip_timestamp1 = sorted(os.listdir('var/blobstoragezips'))[0]
>>> zip_timestamp0 == zip_timestamp1
False

Expand Down

0 comments on commit 18b22f4

Please sign in to comment.