Merge pull request openSUSE#3033 from Vogtinator/zst

Handle ZSTD compressed primary.xml.zst

CONTENTS.md

@@ -178,7 +178,7 @@ Script to generate necessary FileProvides lines needed by OBS from repo data.
 * Sources: [repo2fileprovides.py](repo2fileprovides.py)
 * Documentation: --
 * Package: --
-* Usage: repo2fileprovides.py primary.xml(.gz)
+* Usage: repo2fileprovides.py primary.xml(.gz|.zst)
 
 ### Bots
 

osclib/repochecks.py

@@ -174,17 +174,15 @@ def installcheck(directories, arch, whitelist, ignore_conflicts):
 
 
 def mirrorRepomd(cachedir, url):
-    # Use repomd.xml to get the location of primary.xml.gz
+    # Use repomd.xml to get the location of primary.xml.*
     repoindex = ET.fromstring(requests.get('{}/repodata/repomd.xml'.format(url)).content)
     primarypath = repoindex.xpath("string(./repo:data[@type='primary']/repo:location/@href)",
                                   namespaces={'repo': 'http://linux.duke.edu/metadata/repo'})
-    if not primarypath.endswith(".xml.gz"):
-        raise Exception('unsupported primary format')
 
     primarydest = os.path.join(cachedir, os.path.basename(primarypath))
     if not os.path.exists(primarydest):
         # Delete the old files first
-        for oldfile in glob.glob(glob.escape(cachedir) + "/*.xml.gz"):
+        for oldfile in glob.glob(glob.escape(cachedir) + "/*.xml.*"):
             os.unlink(oldfile)
 
         with tempfile.NamedTemporaryFile(dir=cachedir) as primarytemp:

pkglistgen/update_repo_handler.py

@@ -1,8 +1,6 @@
 
 import glob
-import gzip
 import hashlib
-import io
 import logging
 import os.path
 import re
@@ -99,12 +97,11 @@ def parse_repomd(repo, baseurl):
         if sha != sha_expected:
             raise Exception('checksums do not match {} != {}'.format(sha, sha_expected))
 
-        content = gzip.GzipFile(fileobj=io.BytesIO(primary.content))
         os.lseek(f.fileno(), 0, os.SEEK_SET)
-        f.write(content.read())
+        f.write(primary.content)
         f.flush()
         os.lseek(f.fileno(), 0, os.SEEK_SET)
-        repo.add_rpmmd(solv.xfopen_fd(None, f.fileno()), None, 0)
+        repo.add_rpmmd(solv.xfopen_fd(url, f.fileno()), None, 0)
         return True
 
     return False
@@ -132,13 +129,13 @@ def parse_susetags(repo, baseurl):
         if packages.status_code != requests.codes.ok:
             raise Exception(url + ' does not exist')
 
-        content = gzip.GzipFile(fileobj=io.BytesIO(packages.content))
         os.lseek(f.fileno(), 0, os.SEEK_SET)
-        f.write(content.read())
+        f.write(packages.content)
         f.flush()
         os.lseek(f.fileno(), 0, os.SEEK_SET)
         try:
-            repo.add_susetags(f, defvendorid, None, solv.Repo.REPO_NO_INTERNALIZE | solv.Repo.SUSETAGS_RECORD_SHARES)
+            repo.add_susetags(solv.xfopen_fd(url, f.fileno()), defvendorid, None,
+                              solv.Repo.REPO_NO_INTERNALIZE | solv.Repo.SUSETAGS_RECORD_SHARES)
         except TypeError:
             logger.error(f"Failed to add susetags for {url}")
             return False

repo2fileprovides.py

@@ -1,18 +1,22 @@
 #!/usr/bin/python3
 import gzip
+import pyzstd
 import sys
 from collections import defaultdict
 from lxml import etree
 
 if len(sys.argv) != 2:
     print("Script to generate necessary FileProvides lines needed by OBS from repo data.", file=sys.stderr)
-    print("Usage: repo2fileprovides.py primary.xml(.gz)", file=sys.stderr)
+    print("Usage: repo2fileprovides.py primary.xml(.gz|.zst)", file=sys.stderr)
     sys.exit(1)
 
 repofilename = sys.argv[1]
-xmlfile = open(repofilename, 'rb')
 if repofilename.endswith('.gz'):
-    xmlfile = gzip.GzipFile(fileobj=xmlfile)
+    xmlfile = gzip.GzipFile(repofilename)
+elif repofilename.endswith('.zst'):
+    xmlfile = pyzstd.ZstdFile(repofilename)
+else:
+    xmlfile = open(repofilename, 'rb')
 
 NS = {'md': 'http://linux.duke.edu/metadata/common',
       'rpm': 'http://linux.duke.edu/metadata/rpm'}