diff --git a/.github/workflows/ci-python3-dependencies.yml b/.github/workflows/ci-python3-dependencies.yml index 1a0baca..5eaa7a4 100644 --- a/.github/workflows/ci-python3-dependencies.yml +++ b/.github/workflows/ci-python3-dependencies.yml @@ -33,7 +33,7 @@ jobs: const latestPythonMinorVersion = pythonVersions[0].split('.').slice(0, 2).join('.') // Check if new Python minor version exists - if (latestPythonMinorVersion === '3.12') { + if (latestPythonMinorVersion === '3.13') { return } diff --git a/.github/workflows/ci-python3-freebsd.yml b/.github/workflows/ci-python3-freebsd.yml index 21d9942..c48120e 100644 --- a/.github/workflows/ci-python3-freebsd.yml +++ b/.github/workflows/ci-python3-freebsd.yml @@ -64,13 +64,13 @@ jobs: pip list # Check syntax by compiling code - python -m compileall -f . + python -W error -bb -m compileall -f . # Run unit tests - su runner -c 'python -bb test/test.py --unit --exit-early' + su runner -c 'python -W error -bb test/test.py --unit --exit-early' # Run integration tests (internal) - su runner -c 'ulimit -n; ulimit -n 4096; python -bb test/test.py -i --exit-early' + su runner -c 'ulimit -n; ulimit -n 4096; python -W error -bb test/test.py -i --exit-early' # Run integration tests (external process) su runner -c 'ulimit -n; ulimit -n 4096; test/test.py -e --exit-early' diff --git a/.github/workflows/ci-python3.yml b/.github/workflows/ci-python3.yml index ae9eb34..a7fcabb 100644 --- a/.github/workflows/ci-python3.yml +++ b/.github/workflows/ci-python3.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: os: [ ubuntu-24.04, ubuntu-22.04, ubuntu-20.04 ] - python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] # Exclude unsupported OS/Python version combinations exclude: - os: ubuntu-22.04 @@ -58,16 +58,16 @@ jobs: pip list - name: Check syntax by compiling code - run: python -m compileall -f . + run: python -W error -bb -m compileall -f . - name: Run unit tests - run: python -bb test/test.py --unit --exit-early + run: python -W error -bb test/test.py --unit --exit-early - name: Run integration tests (internal) run: | ulimit -n ulimit -n 4096 - python -bb test/test.py -i --exit-early + python -W error -bb test/test.py -i --exit-early - name: Run integration tests (external process) run: | diff --git a/Changelog b/Changelog index 00d5dde..2cf2835 100644 --- a/Changelog +++ b/Changelog @@ -2,6 +2,9 @@ * WARNING: This is a development snapshot, not a stable release. * Drop support for Python 3.5. * Python 3.6 is now the minimum supported version. + * Add support for Python 3.12 (Thanks to Louis Sautier). + * Add support for Python 3.13. + * Correctly close files in various situations. 2022-10-30 - v3.0.0: * Tested platforms: Linux and FreeBSD. If there is interest in supporting more platforms (e.g. OSX and Windows), please get in contact with the project. diff --git a/lib/cfv/common.py b/lib/cfv/common.py index 3f7d86d..fe13d95 100644 --- a/lib/cfv/common.py +++ b/lib/cfv/common.py @@ -29,6 +29,7 @@ __version__ = '3.0.1.dev0' __homepage__ = 'https://github.com/cfv-project/cfv' +import contextlib import copy import errno import getopt @@ -466,8 +467,6 @@ def test_chksumfile(self, file, filename): cache.set_verified(filename) try: cf_stats = stats.make_sub_stats() - if not file: - file = fileutil.open_read(filename, config) self.do_test_chksumfile(file) cf_stats.sub_stats_end(stats) view.ev_test_cf_done(filename, cf_stats) @@ -1532,8 +1531,8 @@ def getimagedimensions(filename): return '0', '0' try: from PIL import Image - im1 = Image.open(filename) - return list(map(str, im1.size)) + with Image.open(filename) as im1: + return list(map(str, im1.size)) except (ImportError, IOError): return '0', '0' @@ -1694,26 +1693,33 @@ def visit_dir(name, st=None, noisy=1): def test(filename, typename, restrict_typename='auto'): - if typename != 'auto': - cf = cftypes.get_handler(typename)() - cf.test_chksumfile(None, filename) - return + class UnexpectedHandlerException(Exception): + pass - try: - file = fileutil.open_read(filename, config) + def get_cf_handler(file, typename, restrict_typename): + if typename != 'auto': + return cftypes.get_handler(typename)() cftype = cftypes.auto_chksumfile_match(file) if restrict_typename != 'auto' and cftypes.get_handler(restrict_typename) != cftype: - return + raise UnexpectedHandlerException() if cftype: - cf = cftype() + return cftype() + + try: + with contextlib.closing(fileutil.open_read(filename, config)) as file: + cf = get_cf_handler(file, typename, restrict_typename) + if not cf: + view.ev_test_cf_unrecognized(filename, file._decode_errs) + stats.cferror += 1 + return + cf.test_chksumfile(file, filename) - return + except UnexpectedHandlerException: + return except EnvironmentError as a: stats.cferror += 1 view.ev_cf_enverror(filename, a) return -1 - view.ev_test_cf_unrecognized(filename, file._decode_errs) - stats.cferror += 1 def make(cftype, ifilename, testfiles): diff --git a/lib/cfv/fileutil.py b/lib/cfv/fileutil.py index cc3dcce..6ca19b1 100644 --- a/lib/cfv/fileutil.py +++ b/lib/cfv/fileutil.py @@ -104,6 +104,7 @@ def _done_peeking(self, raw): self.readline = self._readline self.read = fileobj.read self.seek = fileobj.seek + self.close = fileobj.close def seek(self, *args): self._done_peeking(raw=1) @@ -117,6 +118,9 @@ def read(self, *args): self._done_peeking(raw=1) return self.read(*args) + def close(self): + self.fileobj.close() + def PeekFileNonseekable(fileobj, filename, encoding): return PeekFile(BytesIO(fileobj.read()), filename, encoding) diff --git a/lib/cfv/hash.py b/lib/cfv/hash.py index cf111d2..862a697 100644 --- a/lib/cfv/hash.py +++ b/lib/cfv/hash.py @@ -45,27 +45,31 @@ def finish(m, s): if callback: callback(s) - if f == sys.stdin.buffer or _nommap or callback: - return finish(hasher(), 0) - else: - s = os.path.getsize(filename) - try: - if s > _MAX_MMAP: - # Work around python 2.[56] problem with md5 of large mmap objects - raise OverflowError - m = hasher(dommap(f.fileno(), s)) - except OverflowError: - # mmap size is limited by C's int type, which even on 64 bit - # arches is often 32 bits, so we can't use sys.maxint - # either. If we get the error, just assume 32 bits. - mmapsize = min(s, _FALLBACK_MMAP) - m = hasher(dommap(f.fileno(), mmapsize)) - f.seek(mmapsize) - # unfortunatly, python's mmap module doesn't support the - # offset parameter, so we just have to do the rest of the - # file the old fashioned way. - return finish(m, mmapsize) - return m.digest(), s + try: + if f == sys.stdin.buffer or _nommap or callback: + return finish(hasher(), 0) + else: + s = os.path.getsize(filename) + try: + if s > _MAX_MMAP: + # Work around python 2.[56] problem with md5 of large mmap objects + raise OverflowError + m = hasher(dommap(f.fileno(), s)) + except OverflowError: + # mmap size is limited by C's int type, which even on 64 bit + # arches is often 32 bits, so we can't use sys.maxint + # either. If we get the error, just assume 32 bits. + mmapsize = min(s, _FALLBACK_MMAP) + m = hasher(dommap(f.fileno(), mmapsize)) + f.seek(mmapsize) + # unfortunatly, python's mmap module doesn't support the + # offset parameter, so we just have to do the rest of the + # file the old fashioned way. + return finish(m, mmapsize) + return m.digest(), s + finally: + if filename != '': + f.close() def getfilechecksumgeneric(algo): diff --git a/setup.py b/setup.py index ee17045..44a4709 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,8 @@ def _get_version(path): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], keywords='cfv checksum verify sfv csv crc bsdmd5 md5sum sha1sum sha224sum sha256sum sha384sum sha512sum torrent par par2', project_urls={ diff --git a/test/cfvtest.py b/test/cfvtest.py index 6427d46..f67dc5a 100644 --- a/test/cfvtest.py +++ b/test/cfvtest.py @@ -242,7 +242,7 @@ def all_unittests_suite(): assert modules_to_test alltests = unittest.TestSuite() for module in map(my_import, modules_to_test): - alltests.addTest(unittest.findTestCases(module)) + alltests.addTest(unittest.defaultTestLoader.loadTestsFromModule(module)) import cfv.common libdir = os.path.split(cfv.common.__file__)[0] diff --git a/test/test.py b/test/test.py index 4fd5f7c..c62b9ad 100755 --- a/test/test.py +++ b/test/test.py @@ -1354,7 +1354,7 @@ def test_encoding2(): raw_fnok += 1 flag_ok_raw = True try: - open(os.path.join(d, fn), 'rb') + open(os.path.join(d, fn), 'rb').close() except (EnvironmentError, UnicodeError): files_fnerrs += 1 else: @@ -1394,14 +1394,13 @@ def test_encoding2(): def largefile2GB_test(): # hope you have sparse file support ;) fn = os.path.join('bigfile2', 'bigfile') - f = open(fn, 'wb') try: - f.write(b'hi') - f.seek(2 ** 30) - f.write(b'foo') - f.seek(2 ** 31) - f.write(b'bar') - f.close() + with open(fn, 'wb') as f: + f.write(b'hi') + f.seek(2 ** 30) + f.write(b'foo') + f.seek(2 ** 31) + f.write(b'bar') test_generic(cfvcmd + ' -v -T -p %s' % 'bigfile2', rcurry(cfv_all_test, ok=6)) finally: os.unlink(fn) @@ -1410,16 +1409,15 @@ def largefile2GB_test(): def largefile4GB_test(): # hope you have sparse file support ;) fn = os.path.join('bigfile', 'bigfile') - f = open(fn, 'wb') try: - f.write(b'hi') - f.seek(2 ** 30) - f.write(b'foo') - f.seek(2 ** 31) - f.write(b'bar') - f.seek(2 ** 32) - f.write(b'baz') - f.close() + with open(fn, 'wb') as f: + f.write(b'hi') + f.seek(2 ** 30) + f.write(b'foo') + f.seek(2 ** 31) + f.write(b'bar') + f.seek(2 ** 32) + f.write(b'baz') test_generic(cfvcmd + ' -v -T -p %s' % 'bigfile', rcurry(cfv_all_test, ok=10)) finally: os.unlink(fn) @@ -1766,7 +1764,7 @@ def sfvverify(f): else: if t == 'par': try: - open('data1'.encode('utf-16le').decode('utf-16be'), 'rb') + open('data1'.encode('utf-16le').decode('utf-16be'), 'rb').close() except UnicodeError: nf = 0 err = 4 @@ -1778,7 +1776,7 @@ def sfvverify(f): test_generic(cfvcmd + ' --encoding=cp500 -i -T -f test.' + t, rcurry(cfv_all_test, cferror=4)) else: try: - open(b'data1'.decode('cp500'), 'rb') + open(b'data1'.decode('cp500'), 'rb').close() except UnicodeError: nf = 0 err = 4 @@ -1890,3 +1888,4 @@ def copytree(src, dst, ignore=None): sys.exit(failed) finally: shutil.rmtree(tmpdatapath) + logfile.close()