Merge 304e000 into bac6644

totest-manager.py

@@ -10,14 +10,19 @@
 
 import cmdln
 import datetime
+import glob
 import json
 import os
 import re
 import sys
 import urllib2
 import logging
+import shutil
 import signal
+import subprocess
+import tempfile
 import time
+import zlib
 
 from xml.etree import cElementTree as ET
 from openqa_client.client import OpenQA_Client
@@ -37,6 +42,9 @@
 QA_FAILED = 2
 QA_PASSED = 3
 
+REPOMD_NAMESPACES = {'md': "http://linux.duke.edu/metadata/common",
+                     'repo': "http://linux.duke.edu/metadata/repo",
+                     'rpm': "http://linux.duke.edu/metadata/rpm"}
 
 class NotFoundException(Exception):
     pass
@@ -222,6 +230,56 @@ def update_openqa_status_message(self):
                 self.openqa.openqa_request(
                     'POST', 'groups/%s/comments' % group_id, data=data)
 
+    def _get_repo_mirror(self, snapshot):
+        """Get the URL of the repo mirror from openQA"""
+        jobs = self.find_openqa_results(snapshot)
+        # Find the MIRROR_HTTP variable, most jobs have it
+        http_mirror = None
+        for job in jobs:
+            if 'settings' in job and 'MIRROR_HTTP' in job['settings']:
+                return job['settings']['SUSEMIRROR']
+
+    def _download_file(self, url):
+        try:
+            req = urllib2.Request(url=url)
+            f = urllib2.urlopen(req)
+            return f.read()
+        except urllib2.HTTPError:
+            return None
+
+    def _get_repo_primary(self, baseurl):
+        repoindex_xml = self._download_file(baseurl + "/repodata/repomd.xml")
+        repoindex = ET.fromstring(repoindex_xml)
+        primary_elem = repoindex.find("./repo:data[@type='primary']/repo:location",
+                                    REPOMD_NAMESPACES)
+        path_primary = primary_elem.attrib['href']
+        primary_xml = self._download_file(baseurl + "/" + path_primary)
+        return zlib.decompress(primary_xml, zlib.MAX_WBITS|32)
+
+    def _download_snapshot_rpm(self, name, baseurl, primary_tree):
+        """Download the RPM package for a given package name from openQA's mirror.
+        The local path is returned, it's the responsibility of the caller to clean it up."""
+
+        packages = primary_tree.findall("md:package[@type='rpm']",
+                                  REPOMD_NAMESPACES)
+
+        pkg_location = None
+        for pkg in packages:
+            pkg_name = pkg.find("md:name", REPOMD_NAMESPACES).text
+            if pkg_name != name:
+                continue
+
+            pkg_location = pkg.find("md:location", REPOMD_NAMESPACES).attrib['href']
+            break
+
+        if not pkg_location:
+            logger.fatal('could not find the location for the %s package' % (name))
+            return None
+
+        destination = tempfile.NamedTemporaryFile(delete=False)
+        destination.write(self._download_file(baseurl + '/' + pkg_location))
+        return destination.name
+
     def overall_result(self, snapshot):
         """Analyze the openQA jobs of a given snapshot Returns a QAResult"""
 
@@ -435,7 +493,28 @@ def _release_package(self, project, package, set_release=None):
         else:
             self.api.retried_POST(url)
 
-    def _release(self, set_release=None):
+    def _release_docker_image(self, pkg, snapshot, repo_url, primary_tree):
+        """The docker image is wrapped in an RPM in the ToTest repo"""
+
+        # Download the RPM to a temporary location
+        rpm_path = self._download_snapshot_rpm(pkg['pkg'], repo_url, primary_tree)
+        # Create a temporary dir for extraction
+        tmp_path = tempfile.mkdtemp()
+        # Extract the .tar.xz inside the RPM into the dir
+        subprocess.call("rpm2cpio '%s' | cpio -i --to-stdout \*.tar.xz | tar -xJf - -C '%s'" % (rpm_path, tmp_path), shell=True)
+        # Remove the RPM file again
+        os.unlink(rpm_path)
+        # Parse the manifest to get the name of the root tar.xz
+        manifest_path = glob.glob(tmp_path + '/manifest.json')[0]
+        manifest = json.loads(open(manifest_path).read())
+        root_path = tmp_path + '/' + manifest[0]['Layers'][0]
+        # TODO: Copy it into the repo and push.
+        print root_path
+        # And remove the temporary directory
+        shutil.rmtree(tmp_path)
+
+
+    def _release(self, snapshot, set_release=None):
         for product in self.ftp_products:
             self._release_package(self.project, product)
 
@@ -446,13 +525,19 @@ def _release(self, set_release=None):
         for cd in self.main_products:
             self._release_package(self.project, cd, set_release=set_release)
 
+        if self.docker_packages:
+            repo_url = self._get_repo_mirror(snapshot) + "/suse"
+            primary_tree = ET.fromstring(self._get_repo_primary(repo_url))
+            for docker_pkg in self.docker_packages:
+                self._release_docker_image(docker_pkg, snapshot, repo_url, primary_tree)
+
     def update_totest(self, snapshot=None):
         release = 'Snapshot%s' % snapshot if snapshot else None
         logger.info('Updating snapshot %s' % snapshot)
         if not self.dryrun:
             self.api.switch_flag_in_prj(self.test_project, flag='publish', state='disable')
-
-        self._release(set_release=release)
+        snapshot = "20180228"
+        self._release(snapshot, set_release=release)
 
     def publish_factory_totest(self):
         logger.info('Publish test project content')
@@ -630,6 +715,13 @@ class ToTestFactory(ToTestBase):
                        'livecd-tumbleweed-gnome',
                        'livecd-tumbleweed-x11']
 
+    docker_packages = [{'pkg': 'opensuse-image',
+                        'repo': 'docker-images-build',
+                        'branch': 'openSUSE-Tumbleweed',
+                        'filename': 'x86_64/openSUSE-Tumbleweed.base.x86_64.tar.xz'}]
+
+    docker_openqa_tests = ['ext4']
+
     def __init__(self, *args, **kwargs):
         ToTestBase.__init__(self, *args, **kwargs)