Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
reportclient: introduce DNF debug info downloader 
DebugInfoDownload is an abstract template class.

DNFDebugInfoDownload and YumDebugInfoDownload implements the abstract
methods.

I did this in hope to use this schema for adding support of libzypp on
OpenSUSE in the future.

Related to #291

Signed-off-by: Jakub Filak <jfilak@redhat.com>
  • Loading branch information
Jakub Filak committed Mar 5, 2015
1 parent c024366 commit 7a1f41e
Show file tree
Hide file tree
Showing 5 changed files with 405 additions and 181 deletions.
4 changes: 3 additions & 1 deletion po/POTFILES.in
View file
Expand Up @@ -3,8 +3,10 @@
# Please keep this file sorted alphabetically.
src/cli/cli.c
src/cli/cli-report.c
src/client-python/reportclient/debuginfo.py
src/client-python/reportclient/__init__.py
src/client-python/reportclient/debuginfo.py
src/client-python/reportclient/dnfdebuginfo.py
src/client-python/reportclient/yumdebuginfo.py
src/gtk-helpers/ask_dialogs.c
src/gtk-helpers/config_dialog.c
src/gtk-helpers/event_config_dialog.c
Expand Down
4 changes: 3 additions & 1 deletion src/client-python/reportclient/Makefile.am
View file
Expand Up @@ -2,7 +2,9 @@ clientexecdir = $(pyexecdir)/reportclient

clientexec_PYTHON = \
__init__.py \
debuginfo.py
debuginfo.py \
dnfdebuginfo.py \
yumdebuginfo.py

clientexec_LTLIBRARIES = _reportclient.la

Expand Down
247 changes: 68 additions & 179 deletions src/client-python/reportclient/debuginfo.py
View file
@@ -1,3 +1,22 @@
#!/usr/bin/python
# coding=UTF-8

## Copyright (C) 2015 ABRT team <abrt-devel-list@redhat.com>
## Copyright (C) 2015 Red Hat, Inc.

## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.

## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
"""
This module provides classes and functions used to download and manage
debuginfos.
Expand All @@ -10,16 +29,10 @@
import shutil
from subprocess import Popen

from yum import _, YumBase
from yum.callbacks import DownloadBaseCallback
from yum.Errors import YumBaseError

from reportclient import (_, log1, log2, RETURN_OK, RETURN_FAILURE,
RETURN_CANCEL_BY_USER, verbose, ask_yes_no,
error_msg)

TMPDIR=""

def ensure_abrt_uid(fn):
"""
Ensures that the function is called using abrt's uid and gid
Expand Down Expand Up @@ -134,21 +147,21 @@ def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm, exact_files=
print(_("Can't extract files from '{0}'").format(unpacked_cpio_path))
return RETURN_FAILURE

def clean_up():
def clean_up(tmp_dir):
"""
Removes the temporary directory.
"""

if TMPDIR:
if tmp_dir:
try:
shutil.rmtree(TMPDIR)
shutil.rmtree(tmp_dir)
except OSError as ex:
if ex.errno != errno.ENOENT:
error_msg(_("Can't remove '{0}': {1}").format(TMPDIR, ex))
error_msg(_("Can't remove '{0}': {1}").format(tmp_dir, str(ex)))

class MyDownloadCallback(DownloadBaseCallback):
class DownloadProgress(object):
"""
This class serves as a download progress handler for yum's progress bar.
This class serves as a download progress handler.
"""

def __init__(self, total_pkgs):
Expand All @@ -163,20 +176,16 @@ def __init__(self, total_pkgs):
self.downloaded_pkgs = 0
self.last_pct = 0
self.last_time = 0
DownloadBaseCallback.__init__(self)

def updateProgress(self, name, frac, fread, ftime):
def update(self, name, pct):
"""
A method used to update the progress
Arguments:
name - filename
frac - progress fracment (0 -> 1)
fread - formated string containing BytesRead
ftime - formated string containing remaining or elapsed time
pct - percent downloaded
"""

pct = int(frac * 100)
if pct == self.last_pct:
log2("percentage is the same, not updating progress")
return
Expand Down Expand Up @@ -212,18 +221,8 @@ def updateProgress(self, name, frac, fread, ftime):

sys.stdout.flush()

def downloadErrorCallback(callBackObj):
"""
A callback function for mirror errors.
"""

print(_("Problem '{0!s}' occured while downloading from mirror: '{1!s}'. Trying next one").format(
callBackObj.exception, callBackObj.mirror))
# explanation of the return value can be found here:
# /usr/lib/python2.7/site-packages/urlgrabber/mirror.py
return {'fail':0}

class DebugInfoDownload(YumBase):
class DebugInfoDownload(object):
"""
This class is used to manage download of debuginfos.
"""
Expand All @@ -233,28 +232,9 @@ def __init__(self, cache, tmp, repo_pattern="*debug*", keep_rpms=False,
self.old_stdout = -1
self.cachedir = cache
self.tmpdir = tmp
global TMPDIR
TMPDIR = tmp
self.keeprpms = keep_rpms
self.noninteractive = noninteractive
self.repo_pattern=repo_pattern
YumBase.__init__(self)
self.mute_stdout()
#self.conf.cache = os.geteuid() != 0
# Setup yum (Ts, RPM db, Repo & Sack)
# doConfigSetup() takes some time, let user know what we are doing
print(_("Initializing yum"))
try:
# Saw this exception here:
# cannot open Packages index using db3 - Permission denied (13)
# yum.Errors.YumBaseError: Error: rpmdb open failed
self.doConfigSetup()
except YumBaseError as ex:
self.unmute_stdout()
print(_("Error initializing yum (YumBase.doConfigSetup): '{0!s}'").format(ex))
#return 1 - can't do this in constructor
exit(1)
self.unmute_stdout()
self.repo_pattern = repo_pattern

def mute_stdout(self):
"""
Expand Down Expand Up @@ -293,140 +273,72 @@ def setup_tmp_dirs(self):

return RETURN_OK

def prepare(self):
pass

def initialize_progress(self, updater):
pass

def initialize_repositories(self):
pass

def triage(self, files):
pass

def download_package(self, pkg):
pass

# return value will be used as exitcode. So 0 = ok, !0 - error
def download(self, files, download_exact_files=False):
"""
Downloads rpms into a temporary directory
Downloads rpms shipping given files into a temporary directory
Arguments:
package_files_dict - a dict containing {pkg: file list} entries
total_pkgs - total number of packages to download
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""

installed_size = 0
total_pkgs = 0
todownload_size = 0
downloaded_pkgs = 0
# nothing to download?
if not files:
return RETURN_FAILURE

# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval

print(_("Initializing package manager"))
self.prepare()
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()

# make yumdownloader work as non root user
if not self.setCacheDir():
print(_("Error: can't make cachedir, exiting"))
return RETURN_FAILURE

# disable all not needed
for repo in self.repos.listEnabled():
try:
repo.close()
self.repos.disableRepo(repo.id)
except YumBaseError as ex:
print(_("Can't disable repository '{0!s}': {1!s}").format(repo.id, str(ex)))

# This takes some time, let user know what we are doing
print(_("Setting up yum repositories"))
# setting-up repos one-by-one, so we can skip the broken ones...
# this helps when users are using 3rd party repos like rpmfusion
# in rawhide it results in: Can't find valid base url...
for r in self.repos.findRepos(pattern=self.repo_pattern):
try:
rid = self.repos.enableRepo(r.id)
self.repos.doSetup(thisrepo=str(r.id))
log1("enabled repo %s", rid)
setattr(r, "skip_if_unavailable", True)
# yes, we want async download, otherwise our progressCallback
# is not called and the internal yum's one is used,
# which causes artifacts on output
try:
setattr(r, "_async", False)
except (NameError, AttributeError) as ex:
print(ex)
print(_("Can't disable async download, the output might contain artifacts!"))
except YumBaseError as ex:
print(_("Can't setup {0}: {1}, disabling").format(r.id, ex))
self.repos.disableRepo(r.id)

# This is somewhat "magic", it unpacks the metadata making it usable.
# Looks like this is the moment when yum talks to remote servers,
# which takes time (sometimes minutes), let user know why
# we have "paused":
print(_("Looking for needed packages in repositories"))
try:
self.repos.populateSack(mdtype='metadata', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving metadata: '{0!s}'").format(ex))
#we don't want to die here, some metadata might be already retrieved
# so there is a chance we already have what we need
#return 1

try:
# Saw this exception here:
# raise Errors.NoMoreMirrorsRepoError, errstr
# NoMoreMirrorsRepoError: failure:
# repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
# from fedora-debuginfo: [Errno 256] No more mirrors to try.
self.repos.populateSack(mdtype='filelists', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving filelists: '{0!s}'").format(ex))
# we don't want to die here, some repos might be already processed
# so there is a chance we already have what we need
#return 1
print(_("Setting up repositories"))
self.initialize_repositories()

#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()

not_found = []
package_files_dict = {}
for debuginfo_path in files:
log2("yum whatprovides %s", debuginfo_path)
pkg = self.pkgSack.searchFiles(debuginfo_path)
# sometimes one file is provided by more rpms, we can use either of
# them, so let's use the first match
if pkg:
if pkg[0] in package_files_dict:
package_files_dict[pkg[0]].append(debuginfo_path)
else:
package_files_dict[pkg[0]] = [debuginfo_path]
todownload_size += float(pkg[0].size)
installed_size += float(pkg[0].installedsize)
total_pkgs += 1

log2("found pkg for %s: %s", debuginfo_path, pkg[0])
else:
log2("not found pkg for %s", debuginfo_path)
not_found.append(debuginfo_path)

# connect our progress update callback
dnlcb = MyDownloadCallback(total_pkgs)
self.repos.setProgressBar(dnlcb)
self.repos.setMirrorFailureCallback(downloadErrorCallback)
print(_("Looking for needed packages in repositories"))
package_files_dict, not_found, todownload_size, installed_size = self.triage(files)

if verbose != 0 or len(not_found) != 0:
print(_("Can't find packages for {0} debuginfo files").format(len(not_found)))
if verbose != 0 or total_pkgs != 0:
print(_("Packages to download: {0}").format(total_pkgs))
if verbose != 0 or len(package_files_dict) != 0:
print(_("Packages to download: {0}").format(len(package_files_dict)))
question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
todownload_size / (1024*1024),
installed_size / (1024*1024)
)
if self.noninteractive == False and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
Expand All @@ -447,36 +359,13 @@ def download(self, files, download_exact_files=False):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER

progress_observer = DownloadProgress(len(package_files_dict))
self.initialize_progress(progress_observer)

for pkg, files in package_files_dict.items():
dnlcb.downloaded_pkgs = downloaded_pkgs
repo.cache = 0
remote = pkg.returnSimple('relativepath')
local = os.path.basename(remote)
retval = self.setup_tmp_dirs()
# continue only if the tmp dirs are ok
if retval != RETURN_OK:
return retval

remote_path = pkg.returnSimple('remote_url')
# check if the pkg is in a local repo and copy it if it is
err = None
if remote_path.startswith('file:///'):
pkg_path = remote_path[7:]
log2("copying from local repo: %s", remote)
try:
shutil.copy(pkg_path, self.tmpdir)
except OSError as ex:
print(_("Cannot copy file '{0}': {1}").format(pkg_path, ex))
continue
else:
# pkg is in a remote repo, we need to download it to tmpdir
local = os.path.join(self.tmpdir, local)
pkg.localpath = local # Hack: to set the localpath we want
err = self.downloadPkgs(pkglist=[pkg])
# normalize the name
# just str(pkg) doesn't work because it can have epoch
pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch
package_file_name = pkg_nvra + ".rpm"
# Download
package_file_name, err = self.download_package(pkg)

if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
Expand All @@ -492,10 +381,10 @@ def download(self, files, download_exact_files=False):
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
clean_up()
clean_up(self.tmpdir)
return RETURN_FAILURE

downloaded_pkgs += 1
progress_observer.downloaded_pkgs += 1

if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
Expand All @@ -505,7 +394,7 @@ def download(self, files, download_exact_files=False):
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove %s, probably contains an error log").format(self.tmpdir))
error_msg(_("Can't remove {0}, probably contains an error log").format(self.tmpdir))

return RETURN_OK

Expand Down

0 comments on commit 7a1f41e

Please sign in to comment.