From 8516c1ad8fcbd1bd8166244fe259fd8f6bdbbcfb Mon Sep 17 00:00:00 2001 From: Fabian Vogt Date: Tue, 5 Dec 2023 11:56:29 +0100 Subject: [PATCH 1/2] factory-package-news.py: Compress data files with zstd It's less than 1/3rd of the uncompressed size. --- dist/package/openSUSE-release-tools.spec | 2 ++ factory-package-news/factory-package-news.py | 34 +++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 6ffef65a5..17db2c8ab 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -57,6 +57,8 @@ Requires: python3-lxml Requires: python3-pycurl Requires: python3-python-dateutil Requires: python3-pyxdg +# factory-package-news.py +Requires: python3-pyzstd Requires: python3-requests # typing extensions are needed on SLE & Leap %if 0%{?suse_version} <= 1500 diff --git a/factory-package-news/factory-package-news.py b/factory-package-news/factory-package-news.py index 3db553c82..a0acb8599 100755 --- a/factory-package-news/factory-package-news.py +++ b/factory-package-news/factory-package-news.py @@ -9,6 +9,7 @@ import pickle import cmdln import re +import pyzstd SRPM_RE = re.compile( r'(?P.+)-(?P[^-]+)-(?P[^-]+)\.(?P(?:no)?src\.rpm)$') @@ -27,6 +28,17 @@ def utf8str(content): return str(content, 'utf-8') if isinstance(content, bytes) else content +# Older versions wrote uncompressed files, support both for now. +def open_zstd_or_plain(path: str): + f = open(path, 'rb') + if f.read(4) == b'\x28\xb5\x2f\xfd': + f.close() + return pyzstd.open(path, 'rb') + + f.seek(0, os.SEEK_SET) + return f + + class ChangeLogger(cmdln.Cmdln): def __init__(self, *args, **kwargs): cmdln.Cmdln.__init__(self, args, kwargs) @@ -165,8 +177,8 @@ def do_save(self, subcmd, opts, *dirs): if not opts.snapshot: raise Exception("missing snapshot option") - f = open(os.path.join(opts.dir, opts.snapshot), 'wb') - pickle.dump([data_version, self.readChangeLogs(dirs)], f) + with pyzstd.open(os.path.join(opts.dir, opts.snapshot), 'wb') as f: + pickle.dump([data_version, self.readChangeLogs(dirs)], f) def do_dump(self, subcmd, opts, *dirs): """${cmd_name}: pprint the package changelog information @@ -182,9 +194,9 @@ def do_inspect(self, subcmd, opts, filename, package): ${cmd_usage} ${cmd_option_list} """ - f = open(filename, 'rb') - (v, (pkgs, changelogs)) = pickle.load( - f, encoding='utf-8', errors='backslashreplace') + with open_zstd_or_plain(filename) as f: + (v, (pkgs, changelogs)) = pickle.load( + f, encoding='utf-8', errors='backslashreplace') pprint(pkgs[package]) pprint(changelogs[pkgs[package]['sourcerpm']]) @@ -209,14 +221,14 @@ def do_diff(self, subcmd, opts, version1, version2): if not os.path.isdir(opts.dir): raise Exception("%s must be a directory" % opts.dir) - f = open(os.path.join(opts.dir, version1), 'rb') - (v, (v1pkgs, v1changelogs)) = pickle.load(f, - encoding='utf-8', errors='backslashreplace') + with open_zstd_or_plain(os.path.join(opts.dir, version1)) as f: + (v, (v1pkgs, v1changelogs)) = pickle.load(f, + encoding='utf-8', errors='backslashreplace') if v != data_version: raise Exception("not matching version %s in %s" % (v, version1)) - f = open(os.path.join(opts.dir, version2), 'rb') - (v, (v2pkgs, v2changelogs)) = pickle.load(f, - encoding='utf-8', errors='backslashreplace') + with open_zstd_or_plain(os.path.join(opts.dir, version2)) as f: + (v, (v2pkgs, v2changelogs)) = pickle.load(f, + encoding='utf-8', errors='backslashreplace') if v != data_version: raise Exception("not matching version %s in %s" % (v, version2)) From b82749ac01f0027c32c11b0dbe2ee2c8c3650645 Mon Sep 17 00:00:00 2001 From: Fabian Vogt Date: Mon, 11 Dec 2023 13:52:31 +0100 Subject: [PATCH 2/2] factory-package-news.py: Port from pyzstd to zstandard ariel has only zstandard 0.13.0, so not even the .open method exists. --- dist/package/openSUSE-release-tools.spec | 2 +- factory-package-news/factory-package-news.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 17db2c8ab..b56a737fd 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -58,7 +58,7 @@ Requires: python3-pycurl Requires: python3-python-dateutil Requires: python3-pyxdg # factory-package-news.py -Requires: python3-pyzstd +Requires: python3-zstandard Requires: python3-requests # typing extensions are needed on SLE & Leap %if 0%{?suse_version} <= 1500 diff --git a/factory-package-news/factory-package-news.py b/factory-package-news/factory-package-news.py index a0acb8599..442861d22 100755 --- a/factory-package-news/factory-package-news.py +++ b/factory-package-news/factory-package-news.py @@ -9,7 +9,8 @@ import pickle import cmdln import re -import pyzstd +# Can be replaced with pyzstd (for pyzstd.open) once available on ariel +import zstandard SRPM_RE = re.compile( r'(?P.+)-(?P[^-]+)-(?P[^-]+)\.(?P(?:no)?src\.rpm)$') @@ -31,11 +32,11 @@ def utf8str(content): # Older versions wrote uncompressed files, support both for now. def open_zstd_or_plain(path: str): f = open(path, 'rb') - if f.read(4) == b'\x28\xb5\x2f\xfd': - f.close() - return pyzstd.open(path, 'rb') - + magic = f.read(4) f.seek(0, os.SEEK_SET) + if magic == b'\x28\xb5\x2f\xfd': + return zstandard.ZstdDecompressor().stream_reader(f, closefd=True) + return f @@ -177,8 +178,9 @@ def do_save(self, subcmd, opts, *dirs): if not opts.snapshot: raise Exception("missing snapshot option") - with pyzstd.open(os.path.join(opts.dir, opts.snapshot), 'wb') as f: - pickle.dump([data_version, self.readChangeLogs(dirs)], f) + with open(os.path.join(opts.dir, opts.snapshot), 'wb') as fraw: + with zstandard.ZstdCompressor().stream_writer(fraw, closefd=False) as f: + pickle.dump([data_version, self.readChangeLogs(dirs)], f) def do_dump(self, subcmd, opts, *dirs): """${cmd_name}: pprint the package changelog information