diff --git a/examples/c/Makefile b/examples/c/Makefile index 49c509e07..6b47368d5 100644 --- a/examples/c/Makefile +++ b/examples/c/Makefile @@ -5,7 +5,11 @@ CC=gcc CFLAGS= -Wall -Wextra -g -std=c99 -O3 -I../../ `pkg-config --cflags glib-2.0` LINKFLAGS= -L../../build/librepo/ -lrepo `pkg-config --libs glib-2.0` -all: download_repo download_packages download_repo_with_callback +all: \ + download_repo \ + download_packages \ + download_repo_with_callback \ + fastestmirror download_repo: $(CC) $(CFLAGS) download_repo.c $(LINKFLAGS) -o download_repo @@ -16,8 +20,15 @@ download_packages: download_repo_with_callback: $(CC) $(CFLAGS) download_repo_with_callback.c $(LINKFLAGS) -o download_repo_with_callback +fastestmirror: + $(CC) $(CFLAGS) fastestmirror.c $(LINKFLAGS) -o fastestmirror + clean: - rm -f download_repo download_packages download_repo_with_callback + rm -f \ + download_repo \ + download_packages \ + download_repo_with_callback \ + fastestmirror run: LD_LIBRARY_PATH="../../build/librepo/" ./download_repo diff --git a/examples/c/fastestmirror.c b/examples/c/fastestmirror.c new file mode 100644 index 000000000..e3d90d559 --- /dev/null +++ b/examples/c/fastestmirror.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +static void +log_handler_cb(const gchar *log_domain G_GNUC_UNUSED, + GLogLevelFlags log_level G_GNUC_UNUSED, + const gchar *message, + gpointer user_data G_GNUC_UNUSED) +{ + g_print ("%s\n", message); +} + +int +main(int argc, char *argv[]) +{ + int rc = EXIT_SUCCESS; + GSList *list = NULL; + GError *tmp_err = NULL; + + g_log_set_handler("librepo", G_LOG_LEVEL_ERROR | + G_LOG_LEVEL_CRITICAL | + G_LOG_LEVEL_DEBUG | + G_LOG_LEVEL_WARNING, + log_handler_cb, NULL); + + if (argc < 2) { + g_printerr("Usage: %s ...\n", argv[0]); + return EXIT_FAILURE; + } + + for (int x = 1; x < argc; x++) + list = g_slist_prepend(list, argv[x]); + list = g_slist_reverse(list); + + gboolean ret = lr_fastestmirror(NULL, &list, &tmp_err); + if (!ret) { + g_printerr("Error encountered: %s\n", tmp_err->message); + g_error_free(tmp_err); + rc = EXIT_FAILURE; + } + + for (GSList *elem = list; elem; elem = g_slist_next(elem)) { + gchar *url = elem->data; + g_print("%s\n", url); + } + + g_slist_free(list); + + return rc; +} diff --git a/examples/python/yum_repo_simple_download.py b/examples/python/yum_repo_simple_download.py index 997266a21..884f64001 100755 --- a/examples/python/yum_repo_simple_download.py +++ b/examples/python/yum_repo_simple_download.py @@ -14,7 +14,7 @@ import librepo # Metalink URL -METALINK_URL = "https://mirrors.fedoraproject.org/metalink?repo=fedora-18&arch=x86_64" +METALINK_URL = "https://mirrors.fedoraproject.org/metalink?repo=fedora-19&arch=x86_64" # Destination directory (note: This directory must exists!) DESTDIR = "downloaded_metadata" @@ -28,6 +28,8 @@ h.mirrorlist = METALINK_URL # Destination directory for metadata h.destdir = DESTDIR + # Use the fastest mirror + h.fastestmirror = True try: h.perform(r) diff --git a/librepo/CMakeLists.txt b/librepo/CMakeLists.txt index 1319964e9..b79bb3c42 100644 --- a/librepo/CMakeLists.txt +++ b/librepo/CMakeLists.txt @@ -2,6 +2,7 @@ SET (librepo_SRCS checksum.c downloader.c downloadtarget.c + fastestmirror.c gpg.c handle.c lrmirrorlist.c @@ -19,6 +20,7 @@ SET (librepo_SRCS SET(librepo_HEADERS checksum.h + fastestmirror.h gpg.h handle.h librepo.h diff --git a/librepo/fastestmirror.c b/librepo/fastestmirror.c new file mode 100644 index 000000000..d677938a7 --- /dev/null +++ b/librepo/fastestmirror.c @@ -0,0 +1,366 @@ +/* librepo - A library providing (libcURL like) API to downloading repository + * Copyright (C) 2013 Tomas Mlcoch + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, + * USA. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "handle_internal.h" +#include "rcodes.h" +#include "fastestmirror.h" +#include "fastestmirror_internal.h" + +#define LENGT_OF_MEASUREMENT 2.0 // Number of seconds (float point!) +#define HALF_OF_SECOND_IN_MICROS 500000 + +typedef struct { + gchar *url; // Points to string passed by the user + CURL *curl; + double plain_connect_time; +} LrFastestMirror; + +static LrFastestMirror * +lr_lrfastestmirror_new() +{ + LrFastestMirror *mirror = g_new0(LrFastestMirror, 1); + mirror->plain_connect_time = 0.0; + return mirror; +} + +void +lr_lrfastestmirror_free(LrFastestMirror *mirror) +{ + if (!mirror) + return; + if (mirror->curl) + curl_easy_cleanup(mirror->curl); + g_free(mirror); +} + +/** Create list of LrFastestMirror based on input list of URLs. + */ +gboolean +lr_fastestmirror_prepare(LrHandle *handle, + GSList *in_list, + GSList **out_list, + GError **err) +{ + gboolean ret = TRUE; + GSList *list = NULL; + + assert(!err || *err == NULL); + + if (!in_list) { + *out_list = NULL; + return TRUE; + } + + for (GSList *elem = in_list; elem; elem = g_slist_next(elem)) { + gchar *url = elem->data; + CURLcode curlcode; + CURL *curlh; + + if (handle) + curlh = curl_easy_duphandle(handle->curl_handle); + else + curlh = lr_get_curl_handle(); + + if (!curlh) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURL, + "Cannot create curl handle"); + ret = FALSE; + break; + } + + LrFastestMirror *mirror = lr_lrfastestmirror_new(); + mirror->url = url; + mirror->curl = curlh; + + curlcode = curl_easy_setopt(curlh, CURLOPT_URL, url); + if (curlcode != CURLE_OK) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURL, + "curl_easy_setopt(_, CURLOPT_URL, %s) failed: %s", + url, curl_easy_strerror(curlcode)); + ret = FALSE; + break; + } + + curlcode = curl_easy_setopt(curlh, CURLOPT_CONNECT_ONLY, 1); + if (curlcode != CURLE_OK) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURL, + "curl_easy_setopt(_, CURLOPT_CONNECT_ONLY, 1) failed: %s", + curl_easy_strerror(curlcode)); + ret = FALSE; + break; + } + + list = g_slist_append(list, mirror); + } + + if (ret) { + *out_list = list; + } else { + assert(!err || *err); + g_slist_free_full(list, (GDestroyNotify)lr_lrfastestmirror_free); + *out_list = NULL; + } + + return ret; +} + +static gboolean +lr_fastestmirror_perform(GSList *list, GError **err) +{ + assert(!err || *err == NULL); + + if (!list) + return TRUE; + + CURLM *multihandle = curl_multi_init(); + if (!multihandle) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURL, + "curl_multi_init() error"); + return FALSE; + } + + // Add curl easy handles to multi handle + for (GSList *elem = list; elem; elem = g_slist_next(elem)) { + LrFastestMirror *mirror = elem->data; + curl_multi_add_handle(multihandle, mirror->curl); + } + + int still_running; + gdouble elapsed_time = 0.0; + GTimer *timer = g_timer_new(); + g_timer_start(timer); + + do { + struct timeval timeout; + int rc, cm_rc; + int maxfd = -1; + long curl_timeout = -1; + fd_set fdread, fdwrite, fdexcep; + + FD_ZERO(&fdread); + FD_ZERO(&fdwrite); + FD_ZERO(&fdexcep); + + // Set suitable timeout to play around with + timeout.tv_sec = 0; + timeout.tv_usec = HALF_OF_SECOND_IN_MICROS; + + cm_rc = curl_multi_timeout(multihandle, &curl_timeout); + if (cm_rc != CURLM_OK) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURLM, + "curl_multi_timeout() error: %s", + curl_multi_strerror(cm_rc)); + curl_multi_cleanup(multihandle); + return FALSE; + } + + // Set timeout to a reasonable value + if (curl_timeout >= 0) { + timeout.tv_sec = curl_timeout / 1000; + if (timeout.tv_sec >= 1) { + timeout.tv_sec = 0; + timeout.tv_usec = HALF_OF_SECOND_IN_MICROS; + } else { + timeout.tv_usec = (curl_timeout % 1000) * 1000; + if (timeout.tv_usec > HALF_OF_SECOND_IN_MICROS) + timeout.tv_usec = HALF_OF_SECOND_IN_MICROS; + } + } + + // Get file descriptors from the transfers + cm_rc = curl_multi_fdset(multihandle, &fdread, &fdwrite, + &fdexcep, &maxfd); + if (cm_rc != CURLM_OK) { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_CURLM, + "curl_multi_fdset() error: %s", + curl_multi_strerror(cm_rc)); + return FALSE; + } + + rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); + if (rc < 0) { + if (errno == EINTR) { + g_debug("%s: select() interrupted by signal", __func__); + } else { + g_set_error(err, LR_FASTESTMIRROR_ERROR, LRE_SELECT, + "select() error: %s", strerror(errno)); + return FALSE; + } + } + + curl_multi_perform(multihandle, &still_running); + + // Break loop after some reasonable amount of time + elapsed_time = g_timer_elapsed(timer, NULL); + + } while(still_running && elapsed_time < LENGT_OF_MEASUREMENT); + + g_timer_destroy(timer); + + // Remove curl easy handles from multi handle + // and calculate plain_connect_time + for (GSList *elem = list; elem; elem = g_slist_next(elem)) { + LrFastestMirror *mirror = elem->data; + CURL *curl = mirror->curl; + + // Remove handle + curl_multi_remove_handle(multihandle, curl); + + // Calculate plain_connect_time + char *effective_url; + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + + if (!effective_url) { + // No effective url is most likely an error + mirror->plain_connect_time = DBL_MAX; + } else if (g_str_has_prefix(effective_url, "file://")) { + // Local directories are considered to be the best mirrors + mirror->plain_connect_time = 0.0; + } else { + // Get connect time + double namelookup_time; + double connect_time; + curl_easy_getinfo(curl, CURLINFO_NAMELOOKUP_TIME, &namelookup_time); + curl_easy_getinfo(curl, CURLINFO_CONNECT_TIME, &connect_time); + + if (connect_time == 0.0) { + // Zero connect time is most likely an error + connect_time = DBL_MAX; + } + + mirror->plain_connect_time = connect_time; + //g_debug("%s: name_lookup: %3.6f connect_time: %3.6f (%3.6f) | %s", + // __func__, namelookup_time, connect_time, + // mirror->plain_connect_time, mirror->url); + } + } + + curl_multi_cleanup(multihandle); + return TRUE; +} + +gboolean +lr_fastestmirror(LrHandle *handle, GSList **list, GError **err) +{ + assert(!err || *err == NULL); + + g_debug("%s: Fastest mirror determination in progress...", __func__); + + if (!list || *list == NULL) + return TRUE; + + gboolean ret; + GSList *lrfastestmirrors; + ret = lr_fastestmirror_prepare(handle, *list, &lrfastestmirrors, err); + if (!ret) { + g_debug("%s: Error while lr_fastestmirror_prepare()", __func__); + return FALSE; + } + + ret = lr_fastestmirror_perform(lrfastestmirrors, err); + if (!ret) { + g_debug("%s: Error while lr_fastestmirror_perform()", __func__); + g_slist_free_full(lrfastestmirrors, + (GDestroyNotify)lr_lrfastestmirror_free); + return FALSE; + } + + // Sort the mirrors by the connection time + GSList *new_list = NULL; + while (lrfastestmirrors) { + LrFastestMirror *mirror = lrfastestmirrors->data; + double min_value = mirror->plain_connect_time; + + for (GSList *m=g_slist_next(lrfastestmirrors); m; m=g_slist_next(m)) { + LrFastestMirror *current_mirror = m->data; + if (current_mirror->plain_connect_time < min_value) { + min_value = current_mirror->plain_connect_time; + mirror = current_mirror; + } + } + + g_debug("%s: %3.6f : %s", __func__, min_value, mirror->url); + new_list = g_slist_append(new_list, mirror->url); + lrfastestmirrors = g_slist_remove(lrfastestmirrors, mirror); + lr_lrfastestmirror_free(mirror); + } + + assert(!lrfastestmirrors); + g_slist_free(*list); + *list = new_list; + + return TRUE; +} + +gboolean +lr_fastestmirror_sort_internalmirrorlist(LrHandle *handle, + GSList **list, + GError **err) +{ + assert(!err || *err == NULL); + + if (!list || *list == NULL) + return TRUE; + + // Prepare list of urls + GSList *list_of_urls = NULL; + for (GSList *elem = *list; elem; elem = g_slist_next(elem)) { + LrInternalMirror *imirror = elem->data; + list_of_urls = g_slist_prepend(list_of_urls, imirror->url); + } + list_of_urls = g_slist_reverse(list_of_urls); + + // Sort this list by the connection time + gboolean ret = lr_fastestmirror(handle, &list_of_urls, err); + if (!ret) { + g_debug("%s: lr_fastestmirror failed", __func__); + return FALSE; + } + + // Apply sorted order to the source list + GSList *new_list = NULL; + for (GSList *elem = list_of_urls; elem; elem = g_slist_next(elem)) { + gchar *url = elem->data; + for (GSList *ime = *list; ime; ime = g_slist_next(ime)) { + LrInternalMirror *im = ime->data; + if (!g_strcmp0(im->url, url)) { + new_list = g_slist_prepend(new_list, im); + url = NULL; // Just for the assert + break; + } + } + assert(url == NULL); + } + new_list = g_slist_reverse(new_list); + + g_slist_free(list_of_urls); + g_slist_free(*list); + *list = new_list; + + return TRUE; +} diff --git a/librepo/fastestmirror.h b/librepo/fastestmirror.h new file mode 100644 index 000000000..8507c284a --- /dev/null +++ b/librepo/fastestmirror.h @@ -0,0 +1,44 @@ +/* librepo - A library providing (libcURL like) API to downloading repository + * Copyright (C) 2013 Tomas Mlcoch + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, + * USA. + */ + +#ifndef LR_FASTESTMIRROR_H +#define LR_FASTESTMIRROR_H + +#include + +#include "url_substitution.h" +#include "mirrorlist.h" +#include "metalink.h" +#include "handle.h" + +G_BEGIN_DECLS + +/** Sorts list or mirror URLs by its the connections times. + * @param handle LrHandle or NULL + * @param list Pointer to the GSList of urls (char* or gchar*) + * that will be sorted. + * @param err GError ** + * @return TRUE if everything is ok, FALSE is err is set. + */ +gboolean +lr_fastestmirror(LrHandle *handle, GSList **list, GError **err); + +G_END_DECLS + +#endif diff --git a/librepo/fastestmirror_internal.h b/librepo/fastestmirror_internal.h new file mode 100644 index 000000000..3b2efb614 --- /dev/null +++ b/librepo/fastestmirror_internal.h @@ -0,0 +1,36 @@ +/* librepo - A library providing (libcURL like) API to downloading repository + * Copyright (C) 2013 Tomas Mlcoch + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, + * USA. + */ + +#ifndef LR_FASTESTMIRROR_INTERNAL_H +#define LR_FASTESTMIRROR_INTERNAL_H + +#include + +#include "handle.h" + +G_BEGIN_DECLS + +gboolean +lr_fastestmirror_sort_internalmirrorlist(LrHandle *handle, + GSList **list, + GError **err); + +G_END_DECLS + +#endif diff --git a/librepo/handle.c b/librepo/handle.c index 43ac734fa..4707cd7b7 100644 --- a/librepo/handle.c +++ b/librepo/handle.c @@ -44,6 +44,7 @@ #include "yum_internal.h" #include "url_substitution.h" #include "downloader.h" +#include "fastestmirror_internal.h" CURL * lr_get_curl_handle() @@ -451,6 +452,10 @@ lr_handle_setopt(LrHandle *handle, break; } + case LRO_FASTESTMIRROR: + handle->fastestmirror = va_arg(arg, long) ? 1 : 0; + break; + default: g_set_error(err, LR_HANDLE_ERROR, LRE_BADOPTARG, "Unknown option"); @@ -749,7 +754,7 @@ lr_handle_prepare_internal_mirrorlist(LrHandle *handle, GError **err) // Create internal mirrorlist - g_debug("Preparing internal mirrorlist"); + g_debug("%s: Preparing internal mirrorlist", __func__); // Get local path in case of local repository gchar *local_path = NULL; @@ -821,6 +826,17 @@ lr_handle_prepare_internal_mirrorlist(LrHandle *handle, GError **err) handle->internal_mirrorlist, handle->metalink_mirrors); + // If enabled, sort internal mirrorlist by the connection + // speed (the LRO_FASTESTMIRROR option) + if (handle->fastestmirror) { + g_debug("%s: Sorting internal mirrorlist by connection speed", + __func__); + gboolean ret = lr_fastestmirror_sort_internalmirrorlist( + handle, &(handle->internal_mirrorlist), err); + if (!ret) + return FALSE; + } + // Prepare mirrors (the list that is reported via LRI_MIRRORS) // This list contains mirrors from mirrorlist and/or metalink // even if they are not explicitly specified, but are available @@ -994,7 +1010,7 @@ lr_handle_getinfo(LrHandle *handle, case LRI_LOCAL: lnum = va_arg(arg, long *); - *lnum = handle->local; + *lnum = (long) handle->local; break; case LRI_PROGRESSCB: { @@ -1100,6 +1116,11 @@ lr_handle_getinfo(LrHandle *handle, break; } + case LRI_FASTESTMIRROR: + lnum = va_arg(arg, long *); + *lnum = (long) handle->fastestmirror; + break; + default: rc = FALSE; g_set_error(err, LR_HANDLE_ERROR, LRE_UNKNOWNOPT, diff --git a/librepo/handle.h b/librepo/handle.h index 1a5ce1b33..ed4bd4e5d 100644 --- a/librepo/handle.h +++ b/librepo/handle.h @@ -170,6 +170,11 @@ typedef enum { After set the list to the handle, it has not to be freed! Handle itself takes care about freeing the list. */ + LRO_FASTESTMIRROR, /*!< (long 1 or 0) + Sort the internal mirrorlist, after it is constructed, by the + determined connection speed. + Disabled by default. */ + /* Repo common options */ LRO_GPGCHECK, /*!< (long 1 or 0) @@ -234,6 +239,7 @@ typedef enum { NOTE: Returned list must be freed as well as all its items! You could use g_strfreev() function. */ LRI_METALINK, /*!< (LrMetalink *) */ + LRI_FASTESTMIRROR, /*!< (long *) */ LRI_SENTINEL, } LrHandleInfoOption; /*!< Handle info options */ diff --git a/librepo/handle_internal.h b/librepo/handle_internal.h index f840fa04d..221a4811e 100644 --- a/librepo/handle_internal.h +++ b/librepo/handle_internal.h @@ -49,10 +49,13 @@ struct _LrHandle { LrInternalMirrorlist *urls_mirrors; /*!< Mirrors from urls */ + int fastestmirror; /*!< + Should be internal mirrorlist sorted by connection time */ + // Mirrorlist related stuff char *mirrorlist; /*!< - XXX: TODO + XXX: Deprecated! List of or metalink */ char *mirrorlisturl; /*!< diff --git a/librepo/librepo.h b/librepo/librepo.h index 9896294d7..aa52afaa0 100644 --- a/librepo/librepo.h +++ b/librepo/librepo.h @@ -23,6 +23,7 @@ #include #include "checksum.h" +#include "fastestmirror.h" #include "gpg.h" #include "handle.h" #include "metalink.h" @@ -39,6 +40,7 @@ #include "yum.h" // Low level downloading interface +// (API could be changed significantly between two versions) #include "downloader.h" #include "downloadtarget.h" diff --git a/librepo/python/__init__.py b/librepo/python/__init__.py index 4d28b6cf9..ec7ff549a 100644 --- a/librepo/python/__init__.py +++ b/librepo/python/__init__.py @@ -188,6 +188,11 @@ for variables in ulrs (e.g.: "http://foo/$version/"). ``[("releasever", "f18"), ("basearch", "i386")]`` +.. data:: LRO_FASTESTMIRROR + + *Boolean*. If True, internal mirrorlist is sorted + by the determined connection speed, after it is constructed. + .. data:: LRO_GPGCHECK *Boolean*. Set True to enable gpg check (if available) of downloaded repo. @@ -242,6 +247,7 @@ .. data:: LRI_VARSUB .. data:: LRI_MIRRORS .. data:: LRI_METALINK +.. data:: LRI_FASTESTMIRROR .. _proxy-type-label: @@ -519,6 +525,7 @@ LRO_MAXPARALLELDOWNLOADS = _librepo.LRO_MAXPARALLELDOWNLOADS LRO_MAXDOWNLOADSPERMIRROR = _librepo.LRO_MAXDOWNLOADSPERMIRROR LRO_VARSUB = _librepo.LRO_VARSUB +LRO_FASTESTMIRROR = _librepo.LRO_FASTESTMIRROR LRO_GPGCHECK = _librepo.LRO_GPGCHECK LRO_CHECKSUM = _librepo.LRO_CHECKSUM LRO_YUMDLIST = _librepo.LRO_YUMDLIST @@ -553,6 +560,7 @@ "maxparalleldownloads": LRO_MAXPARALLELDOWNLOADS, "maxdownloadspermirror":LRO_MAXDOWNLOADSPERMIRROR, "varsub": LRO_VARSUB, + "fastestmirror": LRO_FASTESTMIRROR, "gpgcheck": LRO_GPGCHECK, "checksum": LRO_CHECKSUM, "yumdlist": LRO_YUMDLIST, @@ -576,6 +584,7 @@ LRI_VARSUB = _librepo.LRI_VARSUB LRI_MIRRORS = _librepo.LRI_MIRRORS LRI_METALINK = _librepo.LRI_METALINK +LRI_FASTESTMIRROR = _librepo.LRI_FASTESTMIRROR ATTR_TO_LRI = { "update": LRI_UPDATE, @@ -596,6 +605,7 @@ "varsub": LRI_VARSUB, "mirrors": LRI_MIRRORS, "metalink": LRI_METALINK, + "fastestmirror": LRI_FASTESTMIRROR, } LR_CHECK_GPG = _librepo.LR_CHECK_GPG @@ -845,6 +855,10 @@ class Handle(_librepo.Handle): See: :data:`.LRO_VARSUB` + .. attribute:: fastestmirror: + + See: :data:`.LRO_FASTESTMIRROR` + .. attribute:: gpgcheck: See: :data:`.LRO_GPGCHECK` diff --git a/librepo/python/handle-py.c b/librepo/python/handle-py.c index 42c9f20e3..41751610a 100644 --- a/librepo/python/handle-py.c +++ b/librepo/python/handle-py.c @@ -201,6 +201,7 @@ setopt(_HandleObject *self, PyObject *args) case LRO_CHECKSUM: case LRO_INTERRUPTIBLE: case LRO_FETCHMIRRORS: + case LRO_FASTESTMIRROR: { long d; @@ -562,6 +563,7 @@ getinfo(_HandleObject *self, PyObject *args) case LRI_REPOTYPE: case LRI_FETCHMIRRORS: case LRI_MAXMIRRORTRIES: + case LRI_FASTESTMIRROR: res = lr_handle_getinfo(self->handle, &tmp_err, (LrHandleInfoOption)option, diff --git a/librepo/python/librepomodule.c b/librepo/python/librepomodule.c index 76444945d..a2c2e55e6 100644 --- a/librepo/python/librepomodule.c +++ b/librepo/python/librepomodule.c @@ -248,6 +248,7 @@ init_librepo(void) PyModule_AddIntConstant(m, "LRO_MAXPARALLELDOWNLOADS", LRO_MAXPARALLELDOWNLOADS); PyModule_AddIntConstant(m, "LRO_MAXDOWNLOADSPERMIRROR", LRO_MAXDOWNLOADSPERMIRROR); PyModule_AddIntConstant(m, "LRO_VARSUB", LRO_VARSUB); + PyModule_AddIntConstant(m, "LRO_FASTESTMIRROR", LRO_FASTESTMIRROR); PyModule_AddIntConstant(m, "LRO_GPGCHECK", LRO_GPGCHECK); PyModule_AddIntConstant(m, "LRO_CHECKSUM", LRO_CHECKSUM); PyModule_AddIntConstant(m, "LRO_YUMDLIST", LRO_YUMDLIST); @@ -273,6 +274,7 @@ init_librepo(void) PyModule_AddIntConstant(m, "LRI_VARSUB", LRI_VARSUB); PyModule_AddIntConstant(m, "LRI_MIRRORS", LRI_MIRRORS); PyModule_AddIntConstant(m, "LRI_METALINK", LRI_METALINK); + PyModule_AddIntConstant(m, "LRI_FASTESTMIRROR", LRI_FASTESTMIRROR); // Check options PyModule_AddIntConstant(m, "LR_CHECK_GPG", LR_CHECK_GPG); diff --git a/librepo/rcodes.c b/librepo/rcodes.c index 982486ef4..cef32d2cd 100644 --- a/librepo/rcodes.c +++ b/librepo/rcodes.c @@ -112,6 +112,12 @@ lr_downloader_error_quark(void) return g_quark_from_static_string("lr_downloader_error"); } +GQuark +lr_fastestmirror_error_quark(void) +{ + return g_quark_from_static_string("lr_fastestmirror_error"); +} + GQuark lr_gpg_error_quark(void) { diff --git a/librepo/rcodes.h b/librepo/rcodes.h index f94dacbbc..1e171f3f9 100644 --- a/librepo/rcodes.h +++ b/librepo/rcodes.h @@ -121,6 +121,7 @@ const char *lr_strerror(int rc); /** Error domains for GError */ #define LR_CHECKSUM_ERROR lr_checksum_error_quark() #define LR_DOWNLOADER_ERROR lr_downloader_error_quark() +#define LR_FASTESTMIRROR_ERROR lr_fastestmirror_error_quark() #define LR_GPG_ERROR lr_gpg_error_quark() #define LR_HANDLE_ERROR lr_handle_error_quark() #define LR_METALINK_ERROR lr_metalink_error_quark() @@ -133,6 +134,7 @@ const char *lr_strerror(int rc); GQuark lr_checksum_error_quark(void); GQuark lr_downloader_error_quark(void); +GQuark lr_fastestmirror_error_quark(void); GQuark lr_gpg_error_quark(void); GQuark lr_handle_error_quark(void); GQuark lr_metalink_error_quark(void); diff --git a/tests/python/tests/servermock/yum_mock/config.py b/tests/python/tests/servermock/yum_mock/config.py index 06e4653c8..fca2ad26e 100644 --- a/tests/python/tests/servermock/yum_mock/config.py +++ b/tests/python/tests/servermock/yum_mock/config.py @@ -26,6 +26,7 @@ METALINK_BADCHECKSUM = METALINK_DIR+"badchecksum.xml" METALINK_NOURLS = METALINK_DIR+"nourls.xml" METALINK_BADFIRSTURL = METALINK_DIR+"badfirsturl.xml" +METALINK_BADFIRSTHOST = METALINK_DIR+"badfirsthost.xml" METALINK_FIRSTURLHASCORRUPTEDFILES = METALINK_DIR+"firsturlhascorruptedfiles.xml" METALINK_VARSUB = METALINK_DIR+"varsub.xml" METALINK_VARSUB_LIST = [("version", "01")] diff --git a/tests/python/tests/servermock/yum_mock/static/metalink/badfirsthost.xml b/tests/python/tests/servermock/yum_mock/static/metalink/badfirsthost.xml new file mode 100644 index 000000000..66f392d21 --- /dev/null +++ b/tests/python/tests/servermock/yum_mock/static/metalink/badfirsthost.xml @@ -0,0 +1,19 @@ + + + + + 1347459931 + 2621 + + f76409f67a84bcd516131d5cc98e57e1 + 75125e73304c21945257d9041a908d0d01d2ca16 + bef5d33dc68f47adc7b31df448851b1e9e6bae27840f28700fff144881482a6a + e40060c747895562e945a68967a04d1279e4bd8507413681f83c322479aa564027fdf3962c2d875089bfcb9317d3a623465f390dc1f4acef294711168b807af0 + + + http://256.0.0.1/foo/bar/bad_url + http://127.0.0.1:5000/yum/static/01/repodata/repomd.xml + + + + diff --git a/tests/python/tests/test_handle.py b/tests/python/tests/test_handle.py index 5c5f3f7f1..43fddb13c 100644 --- a/tests/python/tests/test_handle.py +++ b/tests/python/tests/test_handle.py @@ -101,6 +101,12 @@ def test_handle_setopt_getinfo(self): h.setopt(librepo.LRO_VARSUB, None) self.assertEqual(h.getinfo(librepo.LRI_VARSUB), None) + self.assertEqual(h.getinfo(librepo.LRI_FASTESTMIRROR), False) + h.setopt(librepo.LRO_FASTESTMIRROR, True) + self.assertEqual(h.getinfo(librepo.LRI_FASTESTMIRROR), True) + h.setopt(librepo.LRO_FASTESTMIRROR, False) + self.assertEqual(h.getinfo(librepo.LRI_FASTESTMIRROR), False) + def test_handle_setget_attr(self): """No exception should be raised.""" h = librepo.Handle() @@ -195,6 +201,12 @@ def test_handle_setget_attr(self): h.varsub = None self.assertEqual(h.varsub, None) + self.assertEqual(h.fastestmirror, False) + h.fastestmirror = True + self.assertEqual(h.fastestmirror, True) + h.fastestmirror = False + self.assertEqual(h.fastestmirror, False) + def test_handle_setopt_none_value(self): """Using None in setopt.""" h = librepo.Handle() @@ -233,6 +245,8 @@ def test_handle_setopt_none_value(self): h.maxmirrortries = None h.setopt(librepo.LRO_VARSUB, None) h.varsub = None + h.setopt(librepo.LRO_FASTESTMIRROR, None) + h.fastestmirror = None h.setopt(librepo.LRO_GPGCHECK, None) h.gpgcheck = None h.setopt(librepo.LRO_CHECKSUM, None) diff --git a/tests/python/tests/test_yum_repo_downloading.py b/tests/python/tests/test_yum_repo_downloading.py index 2c7b9314f..d398801c5 100644 --- a/tests/python/tests/test_yum_repo_downloading.py +++ b/tests/python/tests/test_yum_repo_downloading.py @@ -747,6 +747,51 @@ def test_download_repo_01_via_metalink_badfirsturl(self): if yum_repo[key] and (key not in ("url", "destdir")): self.assertTrue(os.path.isfile(yum_repo[key])) + def test_download_repo_01_via_metalink_badfirsturl_maxmirrortries(self): + h = librepo.Handle() + r = librepo.Result() + + url = "%s%s" % (MOCKURL, config.METALINK_BADFIRSTHOST) + h.setopt(librepo.LRO_MIRRORLIST, url) + h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) + h.setopt(librepo.LRO_DESTDIR, self.tmpdir) + h.setopt(librepo.LRO_MAXMIRRORTRIES, 1) + + # Because first host is bad and maxmirrortries == 1 + # Download should fail + self.assertRaises(librepo.LibrepoException, h.perform, (r)) + + def test_download_repo_01_via_metalink_badfirsthost_fastestmirror(self): + h = librepo.Handle() + r = librepo.Result() + + url = "%s%s" % (MOCKURL, config.METALINK_BADFIRSTHOST) + h.setopt(librepo.LRO_MIRRORLIST, url) + h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO) + h.setopt(librepo.LRO_DESTDIR, self.tmpdir) + h.setopt(librepo.LRO_FASTESTMIRROR, True) + h.setopt(librepo.LRO_MAXMIRRORTRIES, 1) + + # First host is bad, but fastestmirror is used and thus + # working mirror should be added to the first position + # and download should be successfull even if maxmirrortries + # is equal to 1. + h.perform(r) + + yum_repo = r.getinfo(librepo.LRR_YUM_REPO) + yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD) + + self.assertTrue(yum_repo) + self.assertTrue(yum_repomd) + self.assertEqual(yum_repo["url"], "http://127.0.0.1:5000/yum/static/01/") + + # Test if all mentioned files really exist + self.assertTrue(os.path.isdir(yum_repo["destdir"])) + for key in yum_repo.iterkeys(): + if yum_repo[key] and (key not in ("url", "destdir")): + self.assertTrue(os.path.isfile(yum_repo[key])) + + def test_download_repo_01_via_metalink_firsturlhascorruptedfiles(self): h = librepo.Handle() r = librepo.Result()