Permalink
Browse files

fastestmirror: Add support for cache - New options: LRO_FASTESTMIRROR…

…CACHE and LRO_FASTESTMIRRORMAXAGE
  • Loading branch information...
Tojaj committed Oct 15, 2013
1 parent 6058299 commit b8a063763ccd8a84b8ec21a643461eaace9b9c08

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -87,6 +87,7 @@ lr_handle_init()
handle = lr_malloc0(sizeof(LrHandle));
handle->curl_handle = curl;
handle->fastestmirrormaxage = LRO_FASTESTMIRRORMAXAGE_DEFAULT;
handle->mirrorlist_fd = -1;
handle->metalink_fd = -1;
handle->checks |= LR_CHECK_CHECKSUM;
@@ -108,6 +109,7 @@ lr_handle_free(LrHandle *handle)
if (handle->metalink_fd != -1)
close(handle->mirrorlist_fd);
lr_handle_free_list(&handle->urls);
lr_free(handle->fastestmirrorcache);
lr_free(handle->mirrorlist);
lr_free(handle->mirrorlisturl);
lr_free(handle->metalinkurl);
@@ -456,6 +458,26 @@ lr_handle_setopt(LrHandle *handle,
handle->fastestmirror = va_arg(arg, long) ? 1 : 0;
break;
case LRO_FASTESTMIRRORCACHE: {
char *fastestmirrorcache = va_arg(arg, char *);
if (handle->fastestmirrorcache) lr_free(handle->fastestmirrorcache);
handle->fastestmirrorcache = g_strdup(fastestmirrorcache);
break;
}
case LRO_FASTESTMIRRORMAXAGE:
val_long = va_arg(arg, long);
if (val_long < LRO_FASTESTMIRRORMAXAGE_MIN) {
g_set_error(err, LR_HANDLE_ERROR, LRE_BADOPTARG,
"Value of LRO_FASTESTMIRRORMAXAGE is too low.");
ret = FALSE;
} else {
handle->fastestmirrormaxage = val_long;
}
break;
default:
g_set_error(err, LR_HANDLE_ERROR, LRE_BADOPTARG,
"Unknown option");
@@ -1124,6 +1146,16 @@ lr_handle_getinfo(LrHandle *handle,
*lnum = (long) handle->fastestmirror;
break;
case LRI_FASTESTMIRRORCACHE:
str = va_arg(arg, char **);
*str = handle->fastestmirrorcache;
break;
case LRI_FASTESTMIRRORMAXAGE:
lnum = va_arg(arg, long *);
*lnum = (long) handle->fastestmirrormaxage;
break;
default:
rc = FALSE;
g_set_error(err, LR_HANDLE_ERROR, LRE_UNKNOWNOPT,
@@ -36,6 +36,12 @@ G_BEGIN_DECLS
*/
typedef struct _LrHandle LrHandle;
/** LRO_FASTESTMIRRORMAXAGE default value */
#define LRO_FASTESTMIRRORMAXAGE_DEFAULT 2592000 // 30 days
/** LRO_FASTESTMIRRORMAXAGE minimal allowed value */
#define LRO_FASTESTMIRRORMAXAGE_MIN 0
/** LRO_PROXYPORT default value */
#define LRO_PROXYPORT_DEFAULT 1080
@@ -175,6 +181,15 @@ typedef enum {
determined connection speed.
Disabled by default. */
LRO_FASTESTMIRRORCACHE, /*!< (char *)
Path to the fastestmirror's cache file.
Used when LRO_FASTESTMIRROR is enabled.
If it doesn't exists, it will be created. */
LRO_FASTESTMIRRORMAXAGE, /*< (long)
Maximum age of a record in cache (seconds).
Default: 2592000 (30 days). */
/* Repo common options */
LRO_GPGCHECK, /*!< (long 1 or 0)
@@ -240,6 +255,8 @@ typedef enum {
You could use g_strfreev() function. */
LRI_METALINK, /*!< (LrMetalink *) */
LRI_FASTESTMIRROR, /*!< (long *) */
LRI_FASTESTMIRRORCACHE, /*!< (char **) */
LRI_FASTESTMIRRORMAXAGE, /*!< (long *) */
LRI_SENTINEL,
} LrHandleInfoOption; /*!< Handle info options */
@@ -52,6 +52,12 @@ struct _LrHandle {
int fastestmirror; /*!<
Should be internal mirrorlist sorted by connection time */
char *fastestmirrorcache; /*!<
Path to the fastestmirror's cache file. */
long fastestmirrormaxage; /*!<
Maximum age of a record in cache (seconds). */
// Mirrorlist related stuff
char *mirrorlist; /*!<
@@ -292,6 +292,7 @@ lr_download_packages(GSList *targets,
// Do Fastest Mirror resolving for all handles in one shot
if (fmr_handles) {
fmr_handles = g_slist_reverse(fmr_handles);
ret = lr_fastestmirror_sort_internalmirrorlists(fmr_handles, err);
g_slist_free(fmr_handles);
@@ -193,6 +193,15 @@
*Boolean*. If True, internal mirrorlist is sorted
by the determined connection speed, after it is constructed.
.. data:: LRO_FASTESTMIRRORCACHE
*String or None*. Path to the cache file. If cache file it doesn't exists
it will be created.
.. data:: LRO_FASTESTMIRRORMAXAGE
*Integer or None*. Max age of cache record. Older records will not be used.
.. data:: LRO_GPGCHECK
*Boolean*. Set True to enable gpg check (if available) of downloaded repo.
@@ -248,6 +257,8 @@
.. data:: LRI_MIRRORS
.. data:: LRI_METALINK
.. data:: LRI_FASTESTMIRROR
.. data:: LRI_FASTESTMIRRORCACHE
.. data:: LRI_FASTESTMIRRORMAXAGE
.. _proxy-type-label:
@@ -533,6 +544,8 @@
LRO_MAXDOWNLOADSPERMIRROR = _librepo.LRO_MAXDOWNLOADSPERMIRROR
LRO_VARSUB = _librepo.LRO_VARSUB
LRO_FASTESTMIRROR = _librepo.LRO_FASTESTMIRROR
LRO_FASTESTMIRRORCACHE = _librepo.LRO_FASTESTMIRRORCACHE
LRO_FASTESTMIRRORMAXAGE = _librepo.LRO_FASTESTMIRRORMAXAGE
LRO_GPGCHECK = _librepo.LRO_GPGCHECK
LRO_CHECKSUM = _librepo.LRO_CHECKSUM
LRO_YUMDLIST = _librepo.LRO_YUMDLIST
@@ -568,6 +581,8 @@
"maxdownloadspermirror":LRO_MAXDOWNLOADSPERMIRROR,
"varsub": LRO_VARSUB,
"fastestmirror": LRO_FASTESTMIRROR,
"fastestmirrorcache": LRO_FASTESTMIRRORCACHE,
"fastestmirrormaxage": LRO_FASTESTMIRRORMAXAGE,
"gpgcheck": LRO_GPGCHECK,
"checksum": LRO_CHECKSUM,
"yumdlist": LRO_YUMDLIST,
@@ -592,6 +607,8 @@
LRI_MIRRORS = _librepo.LRI_MIRRORS
LRI_METALINK = _librepo.LRI_METALINK
LRI_FASTESTMIRROR = _librepo.LRI_FASTESTMIRROR
LRI_FASTESTMIRRORCACHE = _librepo.LRI_FASTESTMIRRORCACHE
LRI_FASTESTMIRRORMAXAGE = _librepo.LRI_FASTESTMIRRORMAXAGE
ATTR_TO_LRI = {
"update": LRI_UPDATE,
@@ -613,6 +630,8 @@
"mirrors": LRI_MIRRORS,
"metalink": LRI_METALINK,
"fastestmirror": LRI_FASTESTMIRROR,
"fastestmirrorcache": LRI_FASTESTMIRRORCACHE,
"fastestmirrormaxage": LRI_FASTESTMIRRORMAXAGE,
}
LR_CHECK_GPG = _librepo.LR_CHECK_GPG
@@ -870,6 +889,14 @@ class Handle(_librepo.Handle):
See: :data:`.LRO_FASTESTMIRROR`
.. attribute:: fastestmirrorcache:
See: :data:`.LRO_FASTESTMIRRORCACHE`
.. attribute:: fastestmirrormaxage:
See: :data:`.LRO_FASTESTMIRRORMAXAGE`
.. attribute:: gpgcheck:
See: :data:`.LRO_GPGCHECK`
@@ -161,7 +161,9 @@ setopt(_HandleObject *self, PyObject *args)
case LRO_PROXY:
case LRO_PROXYUSERPWD:
case LRO_DESTDIR:
case LRO_USERAGENT: {
case LRO_USERAGENT:
case LRO_FASTESTMIRRORCACHE:
{
char *str = NULL, *alloced = NULL;
if (PyUnicode_Check(obj)) {
@@ -222,6 +224,7 @@ setopt(_HandleObject *self, PyObject *args)
*/
case LRO_PROXYTYPE:
case LRO_REPOTYPE:
case LRO_FASTESTMIRRORMAXAGE:
{
int badarg = 0;
long d;
@@ -543,6 +546,7 @@ getinfo(_HandleObject *self, PyObject *args)
case LRI_METALINKURL:
case LRI_DESTDIR:
case LRI_USERAGENT:
case LRI_FASTESTMIRRORCACHE:
res = lr_handle_getinfo(self->handle,
&tmp_err,
(LrHandleInfoOption)option,
@@ -558,6 +562,7 @@ getinfo(_HandleObject *self, PyObject *args)
case LRI_FETCHMIRRORS:
case LRI_MAXMIRRORTRIES:
case LRI_FASTESTMIRROR:
case LRI_FASTESTMIRRORMAXAGE:
res = lr_handle_getinfo(self->handle,
&tmp_err,
(LrHandleInfoOption)option,
@@ -249,6 +249,8 @@ init_librepo(void)
PyModule_AddIntConstant(m, "LRO_MAXDOWNLOADSPERMIRROR", LRO_MAXDOWNLOADSPERMIRROR);
PyModule_AddIntConstant(m, "LRO_VARSUB", LRO_VARSUB);
PyModule_AddIntConstant(m, "LRO_FASTESTMIRROR", LRO_FASTESTMIRROR);
PyModule_AddIntConstant(m, "LRO_FASTESTMIRRORCACHE", LRO_FASTESTMIRRORCACHE);
PyModule_AddIntConstant(m, "LRO_FASTESTMIRRORMAXAGE", LRO_FASTESTMIRRORMAXAGE);
PyModule_AddIntConstant(m, "LRO_GPGCHECK", LRO_GPGCHECK);
PyModule_AddIntConstant(m, "LRO_CHECKSUM", LRO_CHECKSUM);
PyModule_AddIntConstant(m, "LRO_YUMDLIST", LRO_YUMDLIST);
@@ -275,6 +277,8 @@ init_librepo(void)
PyModule_AddIntConstant(m, "LRI_MIRRORS", LRI_MIRRORS);
PyModule_AddIntConstant(m, "LRI_METALINK", LRI_METALINK);
PyModule_AddIntConstant(m, "LRI_FASTESTMIRROR", LRI_FASTESTMIRROR);
PyModule_AddIntConstant(m, "LRI_FASTESTMIRRORCACHE", LRI_FASTESTMIRRORCACHE);
PyModule_AddIntConstant(m, "LRI_FASTESTMIRRORMAXAGE", LRI_FASTESTMIRRORMAXAGE);
// Check options
PyModule_AddIntConstant(m, "LR_CHECK_GPG", LR_CHECK_GPG);
@@ -840,6 +840,45 @@ def test_download_repo_01_via_metalink_badfirsthost_fastestmirror(self):
if yum_repo[key] and (key not in ("url", "destdir")):
self.assertTrue(os.path.isfile(yum_repo[key]))
def test_download_repo_01_via_metalink_badfirsthost_fastestmirror_with_cache(self):
h = librepo.Handle()
r = librepo.Result()
cache = os.path.join(self.tmpdir, "fastestmirror.cache")
self.assertFalse(os.path.exists(cache))
url = "%s%s" % (MOCKURL, config.METALINK_BADFIRSTHOST)
h.setopt(librepo.LRO_MIRRORLIST, url)
h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
h.setopt(librepo.LRO_DESTDIR, self.tmpdir)
h.setopt(librepo.LRO_FASTESTMIRROR, True)
h.setopt(librepo.LRO_FASTESTMIRRORCACHE, cache)
h.setopt(librepo.LRO_MAXMIRRORTRIES, 1)
# First host is bad, but fastestmirror is used and thus
# working mirror should be added to the first position
# and download should be successfull even if maxmirrortries
# is equal to 1.
h.perform(r)
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
self.assertTrue(yum_repo)
self.assertTrue(yum_repomd)
self.assertEqual(yum_repo["url"], "http://127.0.0.1:5000/yum/static/01/")
self.assertTrue(os.path.exists(cache))
shutil.rmtree(os.path.join(self.tmpdir, "repodata"))
# Try again, this time, fastestmirror cache should be used
h.perform()
yum_repo = r.getinfo(librepo.LRR_YUM_REPO)
yum_repomd = r.getinfo(librepo.LRR_YUM_REPOMD)
self.assertTrue(yum_repo)
self.assertTrue(yum_repomd)
self.assertEqual(yum_repo["url"], "http://127.0.0.1:5000/yum/static/01/")
self.assertTrue(os.path.exists(cache))
def test_download_repo_01_via_metalink_firsturlhascorruptedfiles(self):
h = librepo.Handle()
@@ -48,6 +48,8 @@ START_TEST(test_handle)
LrUrlVars *vars = NULL;
vars = lr_urlvars_set(vars, "foo", "bar");
fail_if(!lr_handle_setopt(h, NULL, LRO_VARSUB, vars));
fail_if(!lr_handle_setopt(h, NULL, LRO_FASTESTMIRRORCACHE,
"/var/cache/fastestmirror.librepo"));
lr_handle_free(h);
}
END_TEST
@@ -90,7 +92,7 @@ START_TEST(test_handle_getinfo)
str = NULL;
fail_if(!lr_handle_getinfo(h, NULL, LRI_USERAGENT, &str));
fail_if(num != 0);
fail_if(str != NULL);
strlist = NULL;
fail_if(!lr_handle_getinfo(h, NULL, LRI_YUMDLIST, &strlist));
@@ -108,6 +110,14 @@ START_TEST(test_handle_getinfo)
fail_if(!lr_handle_getinfo(h, NULL, LRI_VARSUB, &vars));
fail_if(strlist != NULL);
str = NULL;
fail_if(!lr_handle_getinfo(h, NULL, LRI_FASTESTMIRRORCACHE, &str));
fail_if(str != NULL);
num = -1;
fail_if(!lr_handle_getinfo(h, NULL, LRI_FASTESTMIRRORMAXAGE, &num));
fail_if(num != LRO_FASTESTMIRRORMAXAGE_DEFAULT);
lr_handle_free(h);
}
END_TEST

0 comments on commit b8a0637

Please sign in to comment.