Merge pull request #3039 from Vogtinator/zstdnews

factory-package-news.py: Compress data files with zstd

dist/package/openSUSE-release-tools.spec

@@ -57,6 +57,8 @@ Requires:       python3-lxml
 Requires:       python3-pycurl
 Requires:       python3-python-dateutil
 Requires:       python3-pyxdg
+# factory-package-news.py
+Requires:       python3-zstandard
 Requires:       python3-requests
 # typing extensions are needed on SLE & Leap
 %if 0%{?suse_version} <= 1500

factory-package-news/factory-package-news.py

@@ -9,6 +9,8 @@
 import pickle
 import cmdln
 import re
+# Can be replaced with pyzstd (for pyzstd.open) once available on ariel
+import zstandard
 
 SRPM_RE = re.compile(
     r'(?P<name>.+)-(?P<version>[^-]+)-(?P<release>[^-]+)\.(?P<suffix>(?:no)?src\.rpm)$')
@@ -27,6 +29,17 @@ def utf8str(content):
     return str(content, 'utf-8') if isinstance(content, bytes) else content
 
 
+# Older versions wrote uncompressed files, support both for now.
+def open_zstd_or_plain(path: str):
+    f = open(path, 'rb')
+    magic = f.read(4)
+    f.seek(0, os.SEEK_SET)
+    if magic == b'\x28\xb5\x2f\xfd':
+        return zstandard.ZstdDecompressor().stream_reader(f, closefd=True)
+
+    return f
+
+
 class ChangeLogger(cmdln.Cmdln):
     def __init__(self, *args, **kwargs):
         cmdln.Cmdln.__init__(self, args, kwargs)
@@ -165,8 +178,9 @@ def do_save(self, subcmd, opts, *dirs):
         if not opts.snapshot:
             raise Exception("missing snapshot option")
 
-        f = open(os.path.join(opts.dir, opts.snapshot), 'wb')
-        pickle.dump([data_version, self.readChangeLogs(dirs)], f)
+        with open(os.path.join(opts.dir, opts.snapshot), 'wb') as fraw:
+            with zstandard.ZstdCompressor().stream_writer(fraw, closefd=False) as f:
+                pickle.dump([data_version, self.readChangeLogs(dirs)], f)
 
     def do_dump(self, subcmd, opts, *dirs):
         """${cmd_name}: pprint the package changelog information
@@ -182,9 +196,9 @@ def do_inspect(self, subcmd, opts, filename, package):
         ${cmd_usage}
         ${cmd_option_list}
         """
-        f = open(filename, 'rb')
-        (v, (pkgs, changelogs)) = pickle.load(
-            f, encoding='utf-8', errors='backslashreplace')
+        with open_zstd_or_plain(filename) as f:
+            (v, (pkgs, changelogs)) = pickle.load(
+                f, encoding='utf-8', errors='backslashreplace')
         pprint(pkgs[package])
         pprint(changelogs[pkgs[package]['sourcerpm']])
 
@@ -209,14 +223,14 @@ def do_diff(self, subcmd, opts, version1, version2):
         if not os.path.isdir(opts.dir):
             raise Exception("%s must be a directory" % opts.dir)
 
-        f = open(os.path.join(opts.dir, version1), 'rb')
-        (v, (v1pkgs, v1changelogs)) = pickle.load(f,
-                                                  encoding='utf-8', errors='backslashreplace')
+        with open_zstd_or_plain(os.path.join(opts.dir, version1)) as f:
+            (v, (v1pkgs, v1changelogs)) = pickle.load(f,
+                                                      encoding='utf-8', errors='backslashreplace')
         if v != data_version:
             raise Exception("not matching version %s in %s" % (v, version1))
-        f = open(os.path.join(opts.dir, version2), 'rb')
-        (v, (v2pkgs, v2changelogs)) = pickle.load(f,
-                                                  encoding='utf-8', errors='backslashreplace')
+        with open_zstd_or_plain(os.path.join(opts.dir, version2)) as f:
+            (v, (v2pkgs, v2changelogs)) = pickle.load(f,
+                                                      encoding='utf-8', errors='backslashreplace')
         if v != data_version:
             raise Exception("not matching version %s in %s" % (v, version2))