Permalink
Cannot retrieve contributors at this time
Fetching contributors…
Cannot retrieve contributors at this time
| # | |
| # Represents OS distribution specific install data | |
| # | |
| # Copyright 2006-2007, 2013 Red Hat, Inc. | |
| # Daniel P. Berrange <berrange@redhat.com> | |
| # | |
| # This program is free software; you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation; either version 2 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program; if not, write to the Free Software | |
| # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, | |
| # MA 02110-1301 USA. | |
| import ConfigParser | |
| import ftplib | |
| import io | |
| import logging | |
| import os | |
| import re | |
| import stat | |
| import subprocess | |
| import tempfile | |
| import urllib2 | |
| import urlparse | |
| import requests | |
| from .osdict import OSDB | |
| ######################################################################### | |
| # Backends for the various URL types we support (http, ftp, nfs, local) # | |
| ######################################################################### | |
| class _URLFetcher(object): | |
| """ | |
| This is a generic base class for fetching/extracting files from | |
| a media source, such as CD ISO, NFS server, or HTTP/FTP server | |
| """ | |
| _block_size = 16384 | |
| def __init__(self, location, scratchdir, meter): | |
| self.location = location | |
| self.scratchdir = scratchdir | |
| self.meter = meter | |
| self._srcdir = None | |
| logging.debug("Using scratchdir=%s", scratchdir) | |
| #################### | |
| # Internal helpers # | |
| #################### | |
| def _make_full_url(self, filename): | |
| """ | |
| Generate a full fetchable URL from the passed filename, which | |
| is relative to the self.location | |
| """ | |
| ret = self._srcdir or self.location | |
| if not filename: | |
| return ret | |
| if not ret.endswith("/"): | |
| ret += "/" | |
| return ret + filename | |
| def _grabURL(self, filename, fileobj): | |
| """ | |
| Download the filename from self.location, and write contents to | |
| fileobj | |
| """ | |
| url = self._make_full_url(filename) | |
| try: | |
| urlobj, size = self._grabber(url) | |
| except Exception as e: | |
| raise ValueError(_("Couldn't acquire file %s: %s") % | |
| (url, str(e))) | |
| logging.debug("Fetching URI: %s", url) | |
| self.meter.start( | |
| text=_("Retrieving file %s...") % os.path.basename(filename), | |
| size=size) | |
| total = self._write(urlobj, fileobj) | |
| self.meter.end(total) | |
| def _write(self, urlobj, fileobj): | |
| """ | |
| Write the contents of urlobj to python file like object fileobj | |
| """ | |
| total = 0 | |
| while 1: | |
| buff = urlobj.read(self._block_size) | |
| if not buff: | |
| break | |
| fileobj.write(buff) | |
| total += len(buff) | |
| self.meter.update(total) | |
| return total | |
| def _grabber(self, url): | |
| """ | |
| Returns the urlobj, size for the passed URL. urlobj is whatever | |
| data needs to be passed to self._write | |
| """ | |
| raise NotImplementedError("must be implemented in subclass") | |
| ############## | |
| # Public API # | |
| ############## | |
| def prepareLocation(self): | |
| """ | |
| Perform any necessary setup | |
| """ | |
| pass | |
| def cleanupLocation(self): | |
| """ | |
| Perform any necessary cleanup | |
| """ | |
| pass | |
| def _hasFile(self, url): | |
| raise NotImplementedError("Must be implemented in subclass") | |
| def hasFile(self, filename): | |
| """ | |
| Return True if self.location has the passed filename | |
| """ | |
| url = self._make_full_url(filename) | |
| ret = self._hasFile(url) | |
| logging.debug("hasFile(%s) returning %s", url, ret) | |
| return ret | |
| def acquireFile(self, filename): | |
| """ | |
| Grab the passed filename from self.location and save it to | |
| a temporary file, returning the temp filename | |
| """ | |
| prefix = "virtinst-" + os.path.basename(filename) + "." | |
| # pylint: disable=redefined-variable-type | |
| if "VIRTINST_TEST_SUITE" in os.environ: | |
| fn = os.path.join("/tmp", prefix) | |
| fileobj = open(fn, "wb") | |
| else: | |
| fileobj = tempfile.NamedTemporaryFile( | |
| dir=self.scratchdir, prefix=prefix, delete=False) | |
| fn = fileobj.name | |
| self._grabURL(filename, fileobj) | |
| logging.debug("Saved file to " + fn) | |
| return fn | |
| def acquireFileContent(self, filename): | |
| """ | |
| Grab the passed filename from self.location and return it as a string | |
| """ | |
| fileobj = io.BytesIO() | |
| self._grabURL(filename, fileobj) | |
| return fileobj.getvalue() | |
| class _HTTPURLFetcher(_URLFetcher): | |
| def _hasFile(self, url): | |
| """ | |
| We just do a HEAD request to see if the file exists | |
| """ | |
| try: | |
| response = requests.head(url, allow_redirects=True) | |
| response.raise_for_status() | |
| except Exception as e: | |
| logging.debug("HTTP hasFile request failed: %s", str(e)) | |
| return False | |
| return True | |
| def _grabber(self, url): | |
| """ | |
| Use requests for this | |
| """ | |
| response = requests.get(url, stream=True) | |
| response.raise_for_status() | |
| try: | |
| size = int(response.headers.get('content-length')) | |
| except Exception: | |
| size = None | |
| return response, size | |
| def _write(self, urlobj, fileobj): | |
| """ | |
| The requests object doesn't have a file-like read() option, so | |
| we need to implemente it ourselves | |
| """ | |
| total = 0 | |
| for data in urlobj.iter_content(chunk_size=self._block_size): | |
| fileobj.write(data) | |
| total += len(data) | |
| self.meter.update(total) | |
| return total | |
| class _FTPURLFetcher(_URLFetcher): | |
| _ftp = None | |
| def prepareLocation(self): | |
| if self._ftp: | |
| return | |
| try: | |
| parsed = urlparse.urlparse(self.location) | |
| self._ftp = ftplib.FTP() | |
| self._ftp.connect(parsed.hostname, parsed.port) | |
| self._ftp.login() | |
| # Force binary mode | |
| self._ftp.voidcmd("TYPE I") | |
| except Exception as e: | |
| raise ValueError(_("Opening URL %s failed: %s.") % | |
| (self.location, str(e))) | |
| def _grabber(self, url): | |
| """ | |
| Use urllib2 and ftplib to grab the file | |
| """ | |
| request = urllib2.Request(url) | |
| urlobj = urllib2.urlopen(request) | |
| size = self._ftp.size(urlparse.urlparse(url)[2]) | |
| return urlobj, size | |
| def cleanupLocation(self): | |
| if not self._ftp: | |
| return | |
| try: | |
| self._ftp.quit() | |
| except Exception: | |
| logging.debug("Error quitting ftp connection", exc_info=True) | |
| self._ftp = None | |
| def _hasFile(self, url): | |
| path = urlparse.urlparse(url)[2] | |
| try: | |
| try: | |
| # If it's a file | |
| self._ftp.size(path) | |
| except ftplib.all_errors: | |
| # If it's a dir | |
| self._ftp.cwd(path) | |
| except ftplib.all_errors as e: | |
| logging.debug("FTP hasFile: couldn't access %s: %s", | |
| url, str(e)) | |
| return False | |
| return True | |
| class _LocalURLFetcher(_URLFetcher): | |
| """ | |
| For grabbing files from a local directory | |
| """ | |
| def _hasFile(self, url): | |
| return os.path.exists(url) | |
| def _grabber(self, url): | |
| urlobj = open(url, "r") | |
| size = os.path.getsize(url) | |
| return urlobj, size | |
| class _MountedURLFetcher(_LocalURLFetcher): | |
| """ | |
| Fetcher capable of extracting files from a NFS server | |
| or loopback mounted file, or local CDROM device | |
| """ | |
| _in_test_suite = bool("VIRTINST_TEST_SUITE" in os.environ) | |
| _mounted = False | |
| def prepareLocation(self): | |
| if self._mounted: | |
| return | |
| if self._in_test_suite: | |
| self._srcdir = os.environ["VIRTINST_TEST_URL_DIR"] | |
| else: | |
| self._srcdir = tempfile.mkdtemp(prefix="virtinstmnt.", | |
| dir=self.scratchdir) | |
| mountcmd = "/bin/mount" | |
| logging.debug("Preparing mount at " + self._srcdir) | |
| if self.location.startswith("nfs:"): | |
| cmd = [mountcmd, "-o", "ro", self.location[4:], self._srcdir] | |
| else: | |
| if stat.S_ISBLK(os.stat(self.location)[stat.ST_MODE]): | |
| mountopt = "ro" | |
| else: | |
| mountopt = "ro,loop" | |
| cmd = [mountcmd, "-o", mountopt, self.location, self._srcdir] | |
| logging.debug("mount cmd: %s", cmd) | |
| if not self._in_test_suite: | |
| ret = subprocess.call(cmd) | |
| if ret != 0: | |
| self.cleanupLocation() | |
| raise ValueError(_("Mounting location '%s' failed") % | |
| (self.location)) | |
| self._mounted = True | |
| def cleanupLocation(self): | |
| if not self._mounted: | |
| return | |
| logging.debug("Cleaning up mount at " + self._srcdir) | |
| try: | |
| if not self._in_test_suite: | |
| cmd = ["/bin/umount", self._srcdir] | |
| subprocess.call(cmd) | |
| try: | |
| os.rmdir(self._srcdir) | |
| except Exception: | |
| pass | |
| finally: | |
| self._mounted = False | |
| def fetcherForURI(uri, *args, **kwargs): | |
| if uri.startswith("http://") or uri.startswith("https://"): | |
| fclass = _HTTPURLFetcher | |
| elif uri.startswith("ftp://"): | |
| fclass = _FTPURLFetcher | |
| elif uri.startswith("nfs:"): | |
| fclass = _MountedURLFetcher | |
| elif os.path.isdir(uri): | |
| # Pointing to a local tree | |
| fclass = _LocalURLFetcher | |
| else: | |
| # Pointing to a path, like an .iso to mount | |
| fclass = _MountedURLFetcher | |
| return fclass(uri, *args, **kwargs) | |
| ############################################### | |
| # Helpers for detecting distro from given URL # | |
| ############################################### | |
| def _grabTreeinfo(fetcher): | |
| """ | |
| See if the URL has treeinfo, and if so return it as a ConfigParser | |
| object. | |
| """ | |
| try: | |
| tmptreeinfo = fetcher.acquireFile(".treeinfo") | |
| except ValueError: | |
| return None | |
| try: | |
| treeinfo = ConfigParser.SafeConfigParser() | |
| treeinfo.read(tmptreeinfo) | |
| finally: | |
| os.unlink(tmptreeinfo) | |
| try: | |
| treeinfo.get("general", "family") | |
| except ConfigParser.NoSectionError: | |
| logging.debug("Did not find 'family' section in treeinfo") | |
| return None | |
| logging.debug("treeinfo family=%s", treeinfo.get("general", "family")) | |
| return treeinfo | |
| def _distroFromSUSEContent(fetcher, arch, vmtype=None): | |
| # Parse content file for the 'LABEL' field containing the distribution name | |
| # None if no content, GenericDistro if unknown label type. | |
| try: | |
| cbuf = fetcher.acquireFileContent("content") | |
| except ValueError: | |
| return None | |
| distribution = None | |
| distro_version = None | |
| distro_summary = None | |
| distro_distro = None | |
| distro_arch = None | |
| lines = cbuf.splitlines()[1:] | |
| for line in lines: | |
| if line.startswith("LABEL "): | |
| distribution = line.split(' ', 1) | |
| elif line.startswith("DISTRO "): | |
| distro_distro = line.rsplit(',', 1) | |
| elif line.startswith("VERSION "): | |
| distro_version = line.split(' ', 1) | |
| if len(distro_version) > 1: | |
| d_version = distro_version[1].split('-', 1) | |
| if len(d_version) > 1: | |
| distro_version[1] = d_version[0] | |
| elif line.startswith("SUMMARY "): | |
| distro_summary = line.split(' ', 1) | |
| elif line.startswith("BASEARCHS "): | |
| distro_arch = line.split(' ', 1) | |
| elif line.startswith("DEFAULTBASE "): | |
| distro_arch = line.split(' ', 1) | |
| elif line.startswith("REPOID "): | |
| distro_arch = line.rsplit('/', 1) | |
| if distribution and distro_version and distro_arch: | |
| break | |
| if not distribution: | |
| if distro_summary: | |
| distribution = distro_summary | |
| elif distro_distro: | |
| distribution = distro_distro | |
| if distro_arch: | |
| arch = distro_arch[1].strip() | |
| # Fix for 13.2 official oss repo | |
| if arch.find("i586-x86_64") != -1: | |
| arch = "x86_64" | |
| else: | |
| if cbuf.find("x86_64") != -1: | |
| arch = "x86_64" | |
| elif cbuf.find("i586") != -1: | |
| arch = "i586" | |
| elif cbuf.find("s390x") != -1: | |
| arch = "s390x" | |
| def _parse_sle_distribution(d): | |
| sle_version = d[1].strip().rsplit(' ')[4] | |
| if len(d[1].strip().rsplit(' ')) > 5: | |
| sle_version = sle_version + '.' + d[1].strip().rsplit(' ')[5][2] | |
| return ['VERSION', sle_version] | |
| dclass = GenericDistro | |
| if distribution: | |
| if re.match(".*SUSE Linux Enterprise Server*", distribution[1]) or \ | |
| re.match(".*SUSE SLES*", distribution[1]): | |
| dclass = SLESDistro | |
| if distro_version is None: | |
| distro_version = _parse_sle_distribution(distribution) | |
| elif re.match(".*SUSE Linux Enterprise Desktop*", distribution[1]): | |
| dclass = SLEDDistro | |
| if distro_version is None: | |
| distro_version = _parse_sle_distribution(distribution) | |
| elif re.match(".*openSUSE.*", distribution[1]): | |
| dclass = OpensuseDistro | |
| if distro_version is None: | |
| distro_version = ['VERSION', distribution[0].strip().rsplit(':')[4]] | |
| if distro_version is None: | |
| return None | |
| ob = dclass(fetcher, arch, vmtype) | |
| if dclass != GenericDistro: | |
| ob.version_from_content = distro_version | |
| # Explictly call this, so we populate os_type/variant info | |
| ob.isValidStore() | |
| return ob | |
| def getDistroStore(guest, fetcher): | |
| stores = [] | |
| logging.debug("Finding distro store for location=%s", fetcher.location) | |
| arch = guest.os.arch | |
| _type = guest.os.os_type | |
| urldistro = OSDB.lookup_os(guest.os_variant).urldistro | |
| treeinfo = _grabTreeinfo(fetcher) | |
| if not treeinfo: | |
| dist = _distroFromSUSEContent(fetcher, arch, _type) | |
| if dist: | |
| return dist | |
| stores = _allstores[:] | |
| # If user manually specified an os_distro, bump it's URL class | |
| # to the top of the list | |
| if urldistro: | |
| logging.debug("variant=%s has distro=%s, looking for matching " | |
| "distro store to prioritize", | |
| guest.os_variant, urldistro) | |
| found_store = None | |
| for store in stores: | |
| if store.urldistro == urldistro: | |
| found_store = store | |
| if found_store: | |
| logging.debug("Prioritizing distro store=%s", found_store) | |
| stores.remove(found_store) | |
| stores.insert(0, found_store) | |
| else: | |
| logging.debug("No matching store found, not prioritizing anything") | |
| if treeinfo: | |
| stores.sort(key=lambda x: not x.uses_treeinfo) | |
| for sclass in stores: | |
| store = sclass(fetcher, arch, _type) | |
| store.treeinfo = treeinfo | |
| if store.isValidStore(): | |
| logging.debug("Detected distro name=%s osvariant=%s", | |
| store.name, store.os_variant) | |
| return store | |
| # No distro was detected. See if the URL even resolves, and if not | |
| # give the user a hint that maybe they mistyped. This won't always | |
| # be true since some webservers don't allow directory listing. | |
| # http://www.redhat.com/archives/virt-tools-list/2014-December/msg00048.html | |
| extramsg = "" | |
| if not fetcher.hasFile(""): | |
| extramsg = (": " + | |
| _("The URL could not be accessed, maybe you mistyped?")) | |
| raise ValueError( | |
| _("Could not find an installable distribution at '%s'%s\n\n" | |
| "The location must be the root directory of an install tree.\n" | |
| "See virt-install man page for various distro examples." % | |
| (fetcher.location, extramsg))) | |
| ################## | |
| # Distro classes # | |
| ################## | |
| class Distro(object): | |
| """ | |
| An image store is a base class for retrieving either a bootable | |
| ISO image, or a kernel+initrd pair for a particular OS distribution | |
| """ | |
| name = None | |
| urldistro = None | |
| uses_treeinfo = False | |
| # osdict variant value | |
| os_variant = None | |
| _boot_iso_paths = [] | |
| _hvm_kernel_paths = [] | |
| _xen_kernel_paths = [] | |
| version_from_content = [] | |
| def __init__(self, fetcher, arch, vmtype): | |
| self.fetcher = fetcher | |
| self.type = vmtype | |
| self.arch = arch | |
| self.uri = fetcher.location | |
| # This is set externally | |
| self.treeinfo = None | |
| def isValidStore(self): | |
| """Determine if uri points to a tree of the store's distro""" | |
| raise NotImplementedError | |
| def acquireKernel(self, guest): | |
| kernelpath = None | |
| initrdpath = None | |
| if self.treeinfo: | |
| try: | |
| kernelpath = self._getTreeinfoMedia("kernel") | |
| initrdpath = self._getTreeinfoMedia("initrd") | |
| except ConfigParser.NoSectionError: | |
| pass | |
| if not kernelpath or not initrdpath: | |
| # fall back to old code | |
| if self.type is None or self.type == "hvm": | |
| paths = self._hvm_kernel_paths | |
| else: | |
| paths = self._xen_kernel_paths | |
| for kpath, ipath in paths: | |
| if self.fetcher.hasFile(kpath) and self.fetcher.hasFile(ipath): | |
| kernelpath = kpath | |
| initrdpath = ipath | |
| if not kernelpath or not initrdpath: | |
| raise RuntimeError(_("Couldn't find %(type)s kernel for " | |
| "%(distro)s tree.") % | |
| {"distro": self.name, "type": self.type}) | |
| return self._kernelFetchHelper(guest, kernelpath, initrdpath) | |
| def acquireBootDisk(self, guest): | |
| ignore = guest | |
| if self.treeinfo: | |
| return self.fetcher.acquireFile(self._getTreeinfoMedia("boot.iso")) | |
| for path in self._boot_iso_paths: | |
| if self.fetcher.hasFile(path): | |
| return self.fetcher.acquireFile(path) | |
| raise RuntimeError(_("Could not find boot.iso in %s tree." % | |
| self.name)) | |
| def _check_osvariant_valid(self, os_variant): | |
| return OSDB.lookup_os(os_variant) is not None | |
| def get_osdict_info(self): | |
| """ | |
| Return (distro, variant) tuple, checking to make sure they are valid | |
| osdict entries | |
| """ | |
| if not self.os_variant: | |
| return None | |
| if not self._check_osvariant_valid(self.os_variant): | |
| logging.debug("%s set os_variant to %s, which is not in osdict.", | |
| self, self.os_variant) | |
| return None | |
| return self.os_variant | |
| def _get_method_arg(self): | |
| return "method" | |
| def _getTreeinfoMedia(self, mediaName): | |
| if self.type == "xen": | |
| t = "xen" | |
| else: | |
| t = self.treeinfo.get("general", "arch") | |
| return self.treeinfo.get("images-%s" % t, mediaName) | |
| def _fetchAndMatchRegex(self, filename, regex): | |
| # Fetch 'filename' and return True/False if it matches the regex | |
| try: | |
| content = self.fetcher.acquireFileContent(filename) | |
| except ValueError: | |
| return False | |
| for line in content.splitlines(): | |
| if re.match(regex, line): | |
| return True | |
| return False | |
| def _kernelFetchHelper(self, guest, kernelpath, initrdpath): | |
| # Simple helper for fetching kernel + initrd and performing | |
| # cleanup if necessary | |
| ignore = guest | |
| kernel = self.fetcher.acquireFile(kernelpath) | |
| args = '' | |
| if not self.fetcher.location.startswith("/"): | |
| args += "%s=%s" % (self._get_method_arg(), self.fetcher.location) | |
| try: | |
| initrd = self.fetcher.acquireFile(initrdpath) | |
| return kernel, initrd, args | |
| except Exception: | |
| os.unlink(kernel) | |
| raise | |
| class GenericDistro(Distro): | |
| """ | |
| Generic distro store. Check well known paths for kernel locations | |
| as a last resort if we can't recognize any actual distro | |
| """ | |
| name = "Generic" | |
| uses_treeinfo = True | |
| _xen_paths = [("images/xen/vmlinuz", | |
| "images/xen/initrd.img"), # Fedora | |
| ] | |
| _hvm_paths = [("images/pxeboot/vmlinuz", | |
| "images/pxeboot/initrd.img"), # Fedora | |
| ("ppc/ppc64/vmlinuz", | |
| "ppc/ppc64/initrd.img"), # CenOS 7 ppc64le | |
| ] | |
| _iso_paths = ["images/boot.iso", # RH/Fedora | |
| "boot/boot.iso", # Suse | |
| "current/images/netboot/mini.iso", # Debian | |
| "install/images/boot.iso", # Mandriva | |
| ] | |
| # Holds values to use when actually pulling down media | |
| _valid_kernel_path = None | |
| _valid_iso_path = None | |
| def isValidStore(self): | |
| if self.treeinfo: | |
| # Use treeinfo to pull down media paths | |
| if self.type == "xen": | |
| typ = "xen" | |
| else: | |
| typ = self.treeinfo.get("general", "arch") | |
| kernelSection = "images-%s" % typ | |
| isoSection = "images-%s" % self.treeinfo.get("general", "arch") | |
| if self.treeinfo.has_section(kernelSection): | |
| try: | |
| self._valid_kernel_path = ( | |
| self._getTreeinfoMedia("kernel"), | |
| self._getTreeinfoMedia("initrd")) | |
| except (ConfigParser.NoSectionError, | |
| ConfigParser.NoOptionError) as e: | |
| logging.debug(e) | |
| if self.treeinfo.has_section(isoSection): | |
| try: | |
| self._valid_iso_path = self.treeinfo.get(isoSection, | |
| "boot.iso") | |
| except ConfigParser.NoOptionError as e: | |
| logging.debug(e) | |
| if self.type == "xen": | |
| kern_list = self._xen_paths | |
| else: | |
| kern_list = self._hvm_paths | |
| # If validated media paths weren't found (no treeinfo), check against | |
| # list of media location paths. | |
| for kern, init in kern_list: | |
| if (self._valid_kernel_path is None and | |
| self.fetcher.hasFile(kern) and | |
| self.fetcher.hasFile(init)): | |
| self._valid_kernel_path = (kern, init) | |
| break | |
| for iso in self._iso_paths: | |
| if (self._valid_iso_path is None and | |
| self.fetcher.hasFile(iso)): | |
| self._valid_iso_path = iso | |
| break | |
| if self._valid_kernel_path or self._valid_iso_path: | |
| return True | |
| return False | |
| def acquireKernel(self, guest): | |
| if self._valid_kernel_path is None: | |
| raise ValueError(_("Could not find a kernel path for virt type " | |
| "'%s'" % self.type)) | |
| return self._kernelFetchHelper(guest, | |
| self._valid_kernel_path[0], | |
| self._valid_kernel_path[1]) | |
| def acquireBootDisk(self, guest): | |
| if self._valid_iso_path is None: | |
| raise ValueError(_("Could not find a boot iso path for this tree.")) | |
| return self.fetcher.acquireFile(self._valid_iso_path) | |
| class RedHatDistro(Distro): | |
| """ | |
| Base image store for any Red Hat related distros which have | |
| a common layout | |
| """ | |
| uses_treeinfo = True | |
| _version_number = None | |
| _boot_iso_paths = ["images/boot.iso"] | |
| _hvm_kernel_paths = [("images/pxeboot/vmlinuz", | |
| "images/pxeboot/initrd.img")] | |
| _xen_kernel_paths = [("images/xen/vmlinuz", | |
| "images/xen/initrd.img")] | |
| def isValidStore(self): | |
| raise NotImplementedError() | |
| def _get_method_arg(self): | |
| if (self._version_number is not None and | |
| ((self.urldistro == "rhel" and self._version_number >= 7) or | |
| (self.urldistro == "fedora" and self._version_number >= 19))): | |
| return "inst.repo" | |
| return "method" | |
| # Fedora distro check | |
| class FedoraDistro(RedHatDistro): | |
| name = "Fedora" | |
| urldistro = "fedora" | |
| def isValidStore(self): | |
| if not self.treeinfo: | |
| return self.fetcher.hasFile("Fedora") | |
| if not re.match(".*Fedora.*", self.treeinfo.get("general", "family")): | |
| return False | |
| ver = self.treeinfo.get("general", "version") | |
| if not ver: | |
| logging.debug("No version found in .treeinfo") | |
| return False | |
| logging.debug("Found treeinfo version=%s", ver) | |
| latest_variant = OSDB.latest_fedora_version() | |
| if re.match("fedora[0-9]+", latest_variant): | |
| latest_vernum = int(latest_variant[6:]) | |
| else: | |
| logging.debug("Failed to parse version number from latest " | |
| "fedora variant=%s. Using safe default 22", latest_variant) | |
| latest_vernum = 22 | |
| # rawhide trees changed to use version=Rawhide in Apr 2016 | |
| if ver in ["development", "rawhide", "Rawhide"]: | |
| self._version_number = latest_vernum | |
| self.os_variant = latest_variant | |
| return True | |
| # Dev versions can be like '23_Alpha' | |
| if "_" in ver: | |
| ver = ver.split("_")[0] | |
| # Typical versions are like 'fedora-23' | |
| vernum = str(ver).split("-")[0] | |
| if vernum.isdigit(): | |
| vernum = int(vernum) | |
| else: | |
| logging.debug("Failed to parse version number from treeinfo " | |
| "version=%s, using vernum=latest=%s", ver, latest_vernum) | |
| vernum = latest_vernum | |
| if vernum > latest_vernum: | |
| self.os_variant = latest_variant | |
| else: | |
| self.os_variant = "fedora" + str(vernum) | |
| self._version_number = vernum | |
| return True | |
| # Red Hat Enterprise Linux distro check | |
| class RHELDistro(RedHatDistro): | |
| name = "Red Hat Enterprise Linux" | |
| urldistro = "rhel" | |
| def isValidStore(self): | |
| if self.treeinfo: | |
| # Matches: | |
| # Red Hat Enterprise Linux | |
| # RHEL Atomic Host | |
| m = re.match(".*(Red Hat Enterprise Linux|RHEL).*", | |
| self.treeinfo.get("general", "family")) | |
| ret = (m is not None) | |
| if ret: | |
| self._variantFromVersion() | |
| return ret | |
| if (self.fetcher.hasFile("Server") or | |
| self.fetcher.hasFile("Client")): | |
| self.os_variant = "rhel5" | |
| return True | |
| return self.fetcher.hasFile("RedHat") | |
| ################################ | |
| # osdict autodetection helpers # | |
| ################################ | |
| def _parseTreeinfoVersion(self, verstr): | |
| def _safeint(c): | |
| try: | |
| val = int(c) | |
| except Exception: | |
| val = 0 | |
| return val | |
| version = _safeint(verstr[0]) | |
| update = 0 | |
| # RHEL has version=5.4, scientific linux=54 | |
| updinfo = verstr.split(".") | |
| if len(updinfo) > 1: | |
| update = _safeint(updinfo[1]) | |
| elif len(verstr) > 1: | |
| update = _safeint(verstr[1]) | |
| return version, update | |
| def _variantFromVersion(self): | |
| ver = self.treeinfo.get("general", "version") | |
| name = None | |
| if self.treeinfo.has_option("general", "name"): | |
| name = self.treeinfo.get("general", "name") | |
| if not ver: | |
| return | |
| if name and name.startswith("Red Hat Enterprise Linux Server for ARM"): | |
| # Kind of a hack, but good enough for the time being | |
| version = 7 | |
| update = 0 | |
| else: | |
| version, update = self._parseTreeinfoVersion(ver) | |
| self._version_number = version | |
| self._setRHELVariant(version, update) | |
| def _setRHELVariant(self, version, update): | |
| base = "rhel" + str(version) | |
| if update < 0: | |
| update = 0 | |
| ret = None | |
| while update >= 0: | |
| tryvar = base + ".%s" % update | |
| if not self._check_osvariant_valid(tryvar): | |
| update -= 1 | |
| continue | |
| ret = tryvar | |
| break | |
| if not ret: | |
| # Try plain rhel5, rhel6, whatev | |
| if self._check_osvariant_valid(base): | |
| ret = base | |
| if ret: | |
| self.os_variant = ret | |
| # CentOS distro check | |
| class CentOSDistro(RHELDistro): | |
| name = "CentOS" | |
| urldistro = "centos" | |
| def isValidStore(self): | |
| if not self.treeinfo: | |
| return self.fetcher.hasFile("CentOS") | |
| m = re.match(".*CentOS.*", self.treeinfo.get("general", "family")) | |
| ret = (m is not None) | |
| if ret: | |
| self._variantFromVersion() | |
| if self.os_variant: | |
| new_variant = self.os_variant.replace("rhel", "centos") | |
| if self._check_osvariant_valid(new_variant): | |
| self.os_variant = new_variant | |
| return ret | |
| # Scientific Linux distro check | |
| class SLDistro(RHELDistro): | |
| name = "Scientific Linux" | |
| urldistro = None | |
| _boot_iso_paths = RHELDistro._boot_iso_paths + ["images/SL/boot.iso"] | |
| _hvm_kernel_paths = RHELDistro._hvm_kernel_paths + [ | |
| ("images/SL/pxeboot/vmlinuz", "images/SL/pxeboot/initrd.img")] | |
| def isValidStore(self): | |
| if self.treeinfo: | |
| m = re.match(".*Scientific.*", | |
| self.treeinfo.get("general", "family")) | |
| ret = (m is not None) | |
| if ret: | |
| self._variantFromVersion() | |
| return ret | |
| return self.fetcher.hasFile("SL") | |
| class SuseDistro(Distro): | |
| name = "SUSE" | |
| _boot_iso_paths = ["boot/boot.iso"] | |
| def __init__(self, *args, **kwargs): | |
| Distro.__init__(self, *args, **kwargs) | |
| if re.match(r'i[4-9]86', self.arch): | |
| self.arch = 'i386' | |
| oldkern = "linux" | |
| oldinit = "initrd" | |
| if self.arch == "x86_64": | |
| oldkern += "64" | |
| oldinit += "64" | |
| if self.arch == "s390x": | |
| self._hvm_kernel_paths = [("boot/%s/linux" % self.arch, | |
| "boot/%s/initrd" % self.arch)] | |
| # No Xen on s390x | |
| self._xen_kernel_paths = [] | |
| else: | |
| # Tested with Opensuse >= 10.2, 11, and sles 10 | |
| self._hvm_kernel_paths = [("boot/%s/loader/linux" % self.arch, | |
| "boot/%s/loader/initrd" % self.arch)] | |
| # Tested with Opensuse 10.0 | |
| self._hvm_kernel_paths.append(("boot/loader/%s" % oldkern, | |
| "boot/loader/%s" % oldinit)) | |
| # Tested with SLES 12 for ppc64le | |
| self._hvm_kernel_paths.append(("boot/%s/linux" % self.arch, | |
| "boot/%s/initrd" % self.arch)) | |
| # Matches Opensuse > 10.2 and sles 10 | |
| self._xen_kernel_paths = [("boot/%s/vmlinuz-xen" % self.arch, | |
| "boot/%s/initrd-xen" % self.arch)] | |
| def _variantFromVersion(self): | |
| distro_version = self.version_from_content[1].strip() | |
| version = distro_version.split('.', 1)[0].strip() | |
| self.os_variant = self.urldistro | |
| if int(version) >= 10: | |
| if self.os_variant.startswith(("sles", "sled")): | |
| sp_version = None | |
| if len(distro_version.split('.', 1)) == 2: | |
| sp_version = 'sp' + distro_version.split('.', 1)[1].strip() | |
| self.os_variant += version | |
| if sp_version: | |
| self.os_variant += sp_version | |
| else: | |
| # Tumbleweed 8 digit date | |
| if len(version) == 8: | |
| self.os_variant += "tumbleweed" | |
| else: | |
| self.os_variant += distro_version | |
| else: | |
| self.os_variant += "9" | |
| def isValidStore(self): | |
| # self.version_from_content is the VERSION line from the contents file | |
| if (not self.version_from_content or | |
| self.version_from_content[1] is None): | |
| return False | |
| self._variantFromVersion() | |
| self.os_variant = self._detect_osdict_from_url() | |
| # Reset kernel name for sle11 source on s390x | |
| if self.arch == "s390x": | |
| if self.os_variant == "sles11" or self.os_variant == "sled11": | |
| self._hvm_kernel_paths = [("boot/%s/vmrdr.ikr" % self.arch, | |
| "boot/%s/initrd" % self.arch)] | |
| return True | |
| def _get_method_arg(self): | |
| return "install" | |
| ################################ | |
| # osdict autodetection helpers # | |
| ################################ | |
| def _detect_osdict_from_url(self): | |
| root = "opensuse" | |
| oses = [n for n in OSDB.list_os() if n.name.startswith(root)] | |
| for osobj in oses: | |
| codename = osobj.name[len(root):] | |
| if re.search("/%s/" % codename, self.uri): | |
| return osobj.name | |
| return self.os_variant | |
| class SLESDistro(SuseDistro): | |
| urldistro = "sles" | |
| class SLEDDistro(SuseDistro): | |
| urldistro = "sled" | |
| # Suse image store is harder - we fetch the kernel RPM and a helper | |
| # RPM and then munge bits together to generate a initrd | |
| class OpensuseDistro(SuseDistro): | |
| urldistro = "opensuse" | |
| class DebianDistro(Distro): | |
| # ex. http://ftp.egr.msu.edu/debian/dists/sarge/main/installer-i386/ | |
| # daily builds: http://d-i.debian.org/daily-images/amd64/ | |
| name = "Debian" | |
| urldistro = "debian" | |
| def __init__(self, *args, **kwargs): | |
| Distro.__init__(self, *args, **kwargs) | |
| self._url_prefix = "" | |
| self._treeArch = self._find_treearch() | |
| self._installer_dirname = self.name.lower() + "-installer" | |
| def _find_treearch(self): | |
| for pattern in ["^.*/installer-(\w+)/?$", | |
| "^.*/daily-images/(\w+)/?$"]: | |
| arch = re.findall(pattern, self.uri) | |
| if not arch: | |
| continue | |
| logging.debug("Found pattern=%s treearch=%s in uri", | |
| pattern, arch[0]) | |
| return arch[0] | |
| # Check for standard 'i386' and 'amd64' which will be | |
| # in the URI name for --location $ISO mounts | |
| for arch in ["i386", "amd64", "x86_64", "arm64"]: | |
| if arch in self.uri: | |
| logging.debug("Found treearch=%s in uri", arch) | |
| if arch == "x86_64": | |
| arch = "amd64" | |
| return arch | |
| # Otherwise default to i386 | |
| arch = "i386" | |
| logging.debug("No treearch found in uri, defaulting to arch=%s", arch) | |
| return arch | |
| def _set_media_paths(self): | |
| self._boot_iso_paths = ["%s/netboot/mini.iso" % self._url_prefix] | |
| hvmroot = "%s/netboot/%s/%s/" % (self._url_prefix, | |
| self._installer_dirname, | |
| self._treeArch) | |
| initrd_basename = "initrd.gz" | |
| kernel_basename = "linux" | |
| if self._treeArch in ["ppc64el"]: | |
| kernel_basename = "vmlinux" | |
| if self._treeArch == "s390x": | |
| hvmroot = "%s/generic/" % self._url_prefix | |
| kernel_basename = "kernel.%s" % self.name.lower() | |
| initrd_basename = "initrd.%s" % self.name.lower() | |
| self._hvm_kernel_paths = [ | |
| (hvmroot + kernel_basename, hvmroot + initrd_basename)] | |
| xenroot = "%s/netboot/xen/" % self._url_prefix | |
| self._xen_kernel_paths = [(xenroot + "vmlinuz", xenroot + "initrd.gz")] | |
| def _check_manifest(self, filename): | |
| if not self.fetcher.hasFile(filename): | |
| return False | |
| if self.arch == "s390x": | |
| regex = ".*generic/kernel\.%s.*" % self.name.lower() | |
| else: | |
| regex = ".*%s.*" % self._installer_dirname | |
| if not self._fetchAndMatchRegex(filename, regex): | |
| logging.debug("Regex didn't match, not a %s distro", self.name) | |
| return False | |
| return True | |
| def _check_info(self, filename): | |
| if not self.fetcher.hasFile(filename): | |
| return False | |
| regex = "%s.*" % self.name | |
| if not self._fetchAndMatchRegex(filename, regex): | |
| logging.debug("Regex didn't match, not a %s distro", self.name) | |
| return False | |
| return True | |
| def _is_regular_tree(self): | |
| # For regular trees | |
| if not self._check_manifest("current/images/MANIFEST"): | |
| return False | |
| self._url_prefix = "current/images" | |
| self._set_media_paths() | |
| self.os_variant = self._detect_debian_osdict_from_url() | |
| return True | |
| def _is_daily_tree(self): | |
| # For daily trees | |
| if not self._check_manifest("daily/MANIFEST"): | |
| return False | |
| self._url_prefix = "daily" | |
| self._set_media_paths() | |
| self.os_variant = self._detect_debian_osdict_from_url() | |
| return True | |
| def _is_install_cd(self): | |
| # For install CDs | |
| if not self._check_info(".disk/info"): | |
| return False | |
| if self.arch == "x86_64": | |
| kernel_initrd_pair = ("install.amd/vmlinuz", "install.amd/initrd.gz") | |
| elif self.arch == "i686": | |
| kernel_initrd_pair = ("install.386/vmlinuz", "install.386/initrd.gz") | |
| elif self.arch == "s390x": | |
| kernel_initrd_pair = ("boot/linux_vm", "boot/root.bin") | |
| else: | |
| kernel_initrd_pair = ("install/vmlinuz", "install/initrd.gz") | |
| self._hvm_kernel_paths += [kernel_initrd_pair] | |
| self._xen_kernel_paths += [kernel_initrd_pair] | |
| return True | |
| def isValidStore(self): | |
| return any(check() for check in [ | |
| self._is_regular_tree, | |
| self._is_daily_tree, | |
| self._is_install_cd, | |
| ]) | |
| ################################ | |
| # osdict autodetection helpers # | |
| ################################ | |
| def _detect_debian_osdict_from_url(self): | |
| root = self.name.lower() | |
| oses = [n for n in OSDB.list_os() if n.name.startswith(root)] | |
| if self._url_prefix == "daily": | |
| logging.debug("Appears to be debian 'daily' URL, using latest " | |
| "debian OS") | |
| return oses[0].name | |
| for osobj in oses: | |
| if osobj.codename: | |
| # Ubuntu codenames look like 'Warty Warthog' | |
| codename = osobj.codename.split()[0].lower() | |
| else: | |
| if " " not in osobj.label: | |
| continue | |
| # Debian labels look like 'Debian Sarge' | |
| codename = osobj.label.split()[1].lower() | |
| if ("/%s/" % codename) in self.uri: | |
| logging.debug("Found codename=%s in the URL string", codename) | |
| return osobj.name | |
| logging.debug("Didn't find any known codename in the URL string") | |
| return self.os_variant | |
| class UbuntuDistro(DebianDistro): | |
| # http://archive.ubuntu.com/ubuntu/dists/natty/main/installer-amd64/ | |
| name = "Ubuntu" | |
| urldistro = "ubuntu" | |
| def _is_tree_iso(self): | |
| # For trees based on ISO's | |
| if not self._check_info("install/netboot/version.info"): | |
| return False | |
| self._url_prefix = "install" | |
| self._set_media_paths() | |
| self.os_variant = self._detect_debian_osdict_from_url() | |
| return True | |
| def _is_install_cd(self): | |
| # For install CDs | |
| if not self._check_info(".disk/info"): | |
| return False | |
| if not self.arch == "s390x": | |
| kernel_initrd_pair = ("install/vmlinuz", "install/initrd.gz") | |
| else: | |
| kernel_initrd_pair = ("boot/kernel.ubuntu", "boot/initrd.ubuntu") | |
| self._hvm_kernel_paths += [kernel_initrd_pair] | |
| self._xen_kernel_paths += [kernel_initrd_pair] | |
| return True | |
| class MandrivaDistro(Distro): | |
| # ftp://ftp.uwsg.indiana.edu/linux/mandrake/official/2007.1/x86_64/ | |
| name = "Mandriva/Mageia" | |
| urldistro = "mandriva" | |
| _boot_iso_paths = ["install/images/boot.iso"] | |
| _xen_kernel_paths = [] | |
| def __init__(self, *args, **kwargs): | |
| Distro.__init__(self, *args, **kwargs) | |
| self._hvm_kernel_paths = [] | |
| # At least Mageia 5 uses arch in the names | |
| self._hvm_kernel_paths += [ | |
| ("isolinux/%s/vmlinuz" % self.arch, | |
| "isolinux/%s/all.rdz" % self.arch)] | |
| # Kernels for HVM: valid for releases 2007.1, 2008.*, 2009.0 | |
| self._hvm_kernel_paths += [ | |
| ("isolinux/alt0/vmlinuz", "isolinux/alt0/all.rdz")] | |
| def isValidStore(self): | |
| # Don't support any paravirt installs | |
| if self.type is not None and self.type != "hvm": | |
| return False | |
| # Mandriva websites / media appear to have a VERSION | |
| # file in top level which we can use as our 'magic' | |
| # check for validity | |
| if not self.fetcher.hasFile("VERSION"): | |
| return False | |
| for name in ["Mandriva", "Mageia"]: | |
| if self._fetchAndMatchRegex("VERSION", ".*%s.*" % name): | |
| return True | |
| logging.debug("Regex didn't match, not a %s distro", self.name) | |
| return False | |
| class ALTLinuxDistro(Distro): | |
| # altlinux doesn't have installable URLs, so this is just for a | |
| # mounted ISO | |
| name = "ALT Linux" | |
| urldistro = "altlinux" | |
| _boot_iso_paths = [("altinst", "live")] | |
| _hvm_kernel_paths = [("syslinux/alt0/vmlinuz", "syslinux/alt0/full.cz")] | |
| _xen_kernel_paths = [] | |
| def isValidStore(self): | |
| # Don't support any paravirt installs | |
| if self.type is not None and self.type != "hvm": | |
| return False | |
| if not self.fetcher.hasFile(".disk/info"): | |
| return False | |
| if self._fetchAndMatchRegex(".disk/info", ".*%s.*" % self.name): | |
| return True | |
| logging.debug("Regex didn't match, not a %s distro", self.name) | |
| return False | |
| # Build list of all *Distro classes | |
| def _build_distro_list(): | |
| allstores = [] | |
| for obj in globals().values(): | |
| if isinstance(obj, type) and issubclass(obj, Distro) and obj.name: | |
| allstores.append(obj) | |
| seen_urldistro = [] | |
| for obj in allstores: | |
| if obj.urldistro and obj.urldistro in seen_urldistro: | |
| raise RuntimeError("programming error: duplicate urldistro=%s" % | |
| obj.urldistro) | |
| seen_urldistro.append(obj.urldistro) | |
| # Always stick GenericDistro at the end, since it's a catchall | |
| allstores.remove(GenericDistro) | |
| allstores.append(GenericDistro) | |
| return allstores | |
| _allstores = _build_distro_list() |